diff --git a/DESCRIPTION b/DESCRIPTION
index 927be4a1..bcb39ca5 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -35,8 +35,7 @@ Imports:
     progress,
     Rcpp,
     rlang (>= 0.4.9),
-    tibble,
-    withr
+    tibble
 Suggests: 
     blob,
     covr,
@@ -46,7 +45,8 @@ Suggests:
     readr,
     sodium,
     testthat (>= 3.1.5),
-    wk (>= 0.3.2)
+    wk (>= 0.3.2),
+    withr
 LinkingTo: 
     progress,
     rapidjsonr,
diff --git a/R/bq-download.R b/R/bq-download.R
index f67c01ec..f88f5234 100644
--- a/R/bq-download.R
+++ b/R/bq-download.R
@@ -85,7 +85,7 @@ bq_table_download <-
     start_index <- params$start_index
 
     schema_path <- bq_download_schema(x, tempfile())
-    withr::defer(file.remove(schema_path))
+    defer(file.remove(schema_path))
 
     if (n_max == 0) {
       table_data <- bq_parse_files(
@@ -134,7 +134,7 @@ bq_table_download <-
     )
     curl::multi_run(pool = pool)
     path_first_chunk <- chunk_plan$dat$path[1]
-    withr::defer(file.remove(path_first_chunk))
+    defer(file.remove(path_first_chunk))
 
     chunk_data <- bq_parse_file(schema_path, path_first_chunk)
     n_got <- nrow(chunk_data)
@@ -198,7 +198,7 @@ bq_table_download <-
       )
     }
     curl::multi_run(pool = pool)
-    withr::defer(file.remove(chunk_plan$dat$path))
+    defer(file.remove(chunk_plan$dat$path))
 
     table_data <- bq_parse_files(
       schema_path,
@@ -329,7 +329,7 @@ bq_download_callback <- function(path, progress = NULL) {
     )
 
     con <- file(path, open = "wb")
-    withr::defer(close(con))
+    defer(close(con))
     writeBin(result$content, con)
   }
 }
diff --git a/R/bq-perform.R b/R/bq-perform.R
index 89f5cb64..20cffcd4 100644
--- a/R/bq-perform.R
+++ b/R/bq-perform.R
@@ -175,7 +175,7 @@ export_json <- function(values) {
   })
 
   con <- rawConnection(raw(0), "r+")
-  on.exit(close(con))
+  defer(close(con))
   jsonlite::stream_out(values, con, verbose = FALSE, na = "null")
 
   rawToChar(rawConnectionValue(con))
diff --git a/R/utils.R b/R/utils.R
index fed9a7b6..42afecc7 100644
--- a/R/utils.R
+++ b/R/utils.R
@@ -78,3 +78,7 @@ print.bq_bytes <- function(x, ...) {
 }
 # nocov end
 
+defer <- function (expr, env = caller_env(), after = FALSE) {
+  thunk <- as.call(list(function() expr))
+  do.call(on.exit, list(thunk, TRUE, after), envir = env)
+}
diff --git a/tests/testthat/test-bq-dataset.R b/tests/testthat/test-bq-dataset.R
index f6206c7a..3d19ce0a 100644
--- a/tests/testthat/test-bq-dataset.R
+++ b/tests/testthat/test-bq-dataset.R
@@ -11,7 +11,7 @@ test_that("can create and delete datasets", {
 
 test_that("can update dataset metadata", {
   ds <- bq_dataset(bq_test_project(), random_name())
-  on.exit(bq_dataset_delete(ds))
+  defer(bq_dataset_delete(ds))
 
   bq_dataset_create(ds, description = "a", friendly_name = "b")
   bq_dataset_update(ds, description = "b")
diff --git a/tests/testthat/test-bq-table.R b/tests/testthat/test-bq-table.R
index 9de2abaa..2400ee6e 100644
--- a/tests/testthat/test-bq-table.R
+++ b/tests/testthat/test-bq-table.R
@@ -51,7 +51,7 @@ test_that("can roundtrip via save + load", {
   gs <- gs_test_object()
 
   bq_table_save(tb1, gs)
-  on.exit(gs_object_delete(gs))
+  defer(gs_object_delete(gs))
   bq_table_load(tb2, gs)
 
   df <- bq_table_download(tb2)
diff --git a/tests/testthat/test-dplyr.R b/tests/testthat/test-dplyr.R
index 0bc812fa..0416957b 100644
--- a/tests/testthat/test-dplyr.R
+++ b/tests/testthat/test-dplyr.R
@@ -44,7 +44,7 @@ test_that("can collect and compute (no dataset)", {
   # compute: persistent
   name <- paste0("basedata.", random_name())
   temp <- dplyr::compute(bq_mtcars, temporary = FALSE, name = name)
-  on.exit(DBI::dbRemoveTable(con, name))
+  defer(DBI::dbRemoveTable(con, name))
 
   expect_true(DBI::dbExistsTable(con, name))
 })
@@ -66,7 +66,7 @@ test_that("can collect and compute (with dataset)", {
   # compute: persistent
   name <- random_name()
   temp <- dplyr::compute(bq_mtcars, temporary = FALSE, name = name)
-  on.exit(DBI::dbRemoveTable(con, name))
+  defer(DBI::dbRemoveTable(con, name))
 
   expect_true(DBI::dbExistsTable(con, name))
 })