Switch to ETags for AWS S3 targets

R/class_aws.R

@@ -224,7 +224,6 @@ store_upload_object_aws <- function(store) {
       bucket = bucket,
       region = store_aws_region(store$file$path),
       endpoint = store_aws_endpoint(store$file$path),
-      metadata = list("targets-hash" = store$file$hash),
       part_size = aws$part_size,
       args = aws$args,
       max_tries = aws$max_tries,
@@ -247,6 +246,7 @@ store_upload_object_aws <- function(store) {
     invert = TRUE
   )
   store$file$path <- c(path, paste0("version=", head$VersionId))
+  store$file$hash <- digest_chr64(head$ETag)
   invisible()
 }
 
@@ -271,7 +271,7 @@ store_aws_hash <- function(store) {
     close_connection = aws$close_connection,
     s3_force_path_style = aws$s3_force_path_style
   )
-  head$Metadata[["targets-hash"]]
+  digest_chr64(head$ETag)
 }
 # nocov end
 

R/utils_aws.R

@@ -102,7 +102,7 @@ aws_s3_list_etags <- function(
   out <- list()
   for (page in pages) {
     for (object in page$Contents) {
-      out[[object$Key]] <- eval(parse(text = object$ETag))
+      out[[object$Key]] <- object$ETag
     }
   }
   out

tests/aws/test-aws_endpoint.R

@@ -39,7 +39,7 @@ tar_test("aws qs format data gets stored", {
   })
   expr <- tar_tidy_eval(expr, environment(), TRUE)
   eval(as.call(list(`tar_script`, expr, ask = FALSE)))
-  tar_make(callr_function = NULL)
+  tar_make(callr_function = NULL, reporter = "silent")
   expect_true(
     aws_s3_exists(
       bucket = bucket_name,
@@ -97,10 +97,10 @@ tar_test("aws qs format invalidation", {
   })
   expr <- tar_tidy_eval(expr, environment(), TRUE)
   eval(as.call(list(`tar_script`, expr, ask = FALSE)))
-  tar_make(callr_function = NULL)
+  tar_make(callr_function = NULL, reporter = "silent")
   expect_equal(tar_progress(x)$progress, "built")
   expect_equal(tar_progress(y)$progress, "built")
-  tar_make(callr_function = NULL)
+  tar_make(callr_function = NULL, reporter = "silent")
   progress <- tar_progress()
   progress <- progress[progress$progress != "skipped", ]
   expect_equal(nrow(progress), 0L)
@@ -122,7 +122,7 @@ tar_test("aws qs format invalidation", {
   })
   expr <- tar_tidy_eval(expr, environment(), TRUE)
   eval(as.call(list(`tar_script`, expr, ask = FALSE)))
-  tar_make(callr_function = NULL)
+  tar_make(callr_function = NULL, reporter = "silent")
   expect_equal(tar_progress(x)$progress, "built")
   expect_equal(tar_progress(y)$progress, "built")
   expect_equal(tar_read(x), "x_value2")

tests/aws/test-aws_hpc.R

@@ -29,6 +29,6 @@ tar_test("AWS S3 + HPC", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make_future()
+  tar_make_future(reporter = "silent")
   expect_equal(tar_read(c), 2L)
 })

tests/aws/test-aws_long.R

@@ -39,7 +39,7 @@ tar_test("pipeline continuously uploads metadata", {
   expr <- tar_tidy_eval(expr, environment(), TRUE)
   eval(as.call(list(`tar_script`, expr, ask = FALSE)))
   R.utils::withTimeout(
-    expr = tar_make(seconds_meta_upload = 1),
+    expr = tar_make(seconds_meta_upload = 1, reporter = "silent"),
     timeout = 30,
     onTimeout = "silent"
   )

tests/aws/test-aws_meta.R

@@ -29,7 +29,7 @@ tar_test("AWS meta", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -52,7 +52,7 @@ tar_test("AWS meta", {
   }
   unlink(path_meta(path_store_default()))
   expect_equal(sort(tar_outdated()), sort(c("a", "b", "c")))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "skipped"))
   tar_destroy()
   for (file in c("meta", "process", "progress", "crew")) {
@@ -106,7 +106,7 @@ tar_test("AWS tar_meta_delete()", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -179,7 +179,7 @@ tar_test("AWS tar_meta_upload()", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -244,7 +244,7 @@ tar_test("AWS tar_meta_download()", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -297,7 +297,7 @@ tar_test("AWS tar_meta_sync() upload", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -362,7 +362,7 @@ tar_test("AWS tar_meta_sync() download", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {

tests/gcp/test-gcp_hpc.R

@@ -31,6 +31,6 @@ tar_test("gcp + HPC", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make_future()
+  tar_make_future(reporter = "silent")
   expect_equal(tar_read(c), 2L)
 })

tests/gcp/test-gcp_long.R

@@ -41,7 +41,7 @@ tar_test("pipeline continuously uploads metadata", {
   expr <- tar_tidy_eval(expr, environment(), TRUE)
   eval(as.call(list(`tar_script`, expr, ask = FALSE)))
   R.utils::withTimeout(
-    expr = tar_make(seconds_meta_upload = 1),
+    expr = tar_make(seconds_meta_upload = 1, reporter = "silent"),
     timeout = 30,
     onTimeout = "silent"
   )

tests/gcp/test-gcp_meta.R

@@ -30,7 +30,7 @@ tar_test("gcp meta", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -53,7 +53,7 @@ tar_test("gcp meta", {
   }
   unlink(path_meta(path_store_default()))
   expect_equal(sort(tar_outdated()), sort(c("a", "b", "c")))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "skipped"))
   tar_destroy()
   for (file in c("meta", "process", "progress", "crew")) {
@@ -108,7 +108,7 @@ tar_test("gcp tar_meta_delete()", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -182,7 +182,7 @@ tar_test("gcp tar_meta_upload()", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -248,7 +248,7 @@ tar_test("gcp tar_meta_download()", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -302,7 +302,7 @@ tar_test("gcp tar_meta_sync() upload", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -368,7 +368,7 @@ tar_test("gcp tar_meta_sync() download", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {