[187316637]: recover from flaky dataset creation

Crunch-io · Apr 8, 2024 · e343776 · e343776
1 parent 3656e2c
commit e343776
Show file tree

Hide file tree

Showing 56 changed files with 120 additions and 88 deletions.
diff --git a/tests/testthat/helper-contexts.R b/tests/testthat/helper-contexts.R
@@ -34,9 +34,9 @@ new.dataset.with.setup <- function(df = NULL, ...) {
         ## Just return it
         out <- df
     } else if (is.null(df)) {
-        out <- createDataset(name = unique.name, ...)
+        out <- flakyRecoverCreateDataset(name = unique.name, ...)
     } else {
-        out <- suppressMessages(newDataset(df, name = unique.name, ...))
+        out <- suppressMessages(flakyRecoverNewDataset(df, name = unique.name, ...))
     }
     objects_to_purge <<- c(objects_to_purge, self(out))
     return(out)

diff --git a/tests/testthat/helper.R b/tests/testthat/helper.R
@@ -19,7 +19,8 @@ uncached <- httpcache::uncached
 ## See other options in inst/crunch-test.R
 crunch:::set_crunch_opts(
     crunch.debug = FALSE,
-    crunch.timeout = 60, ## In case an import fails to start, don't wait forever
+    crunch.timeout = 20, ## In case an import fails to start, don't wait forever
+    crunch.max.flaky.new.datasets = 2,
     crunch.require.confirmation = TRUE,
     crunch.check.updates = FALSE,
     crunch.namekey.dataset = "alias",
@@ -40,6 +41,37 @@ crunch:::.onLoad()
 ## Test serialize and deserialize
 cereal <- function(x) fromJSON(toJSON(x), simplifyVector = FALSE)
 
+assign("dataset.retries", 0, envir = globalenv())
+flakyDatasetExpr <- function(expr) {
+    # Because of flaky builds, when uploading a new dataset fails
+    # Try once to upload it again, on up to MAX_RETRIES datasets
+    # during all of testing
+    MEX_RETRIES <- envOrOption("crunch.max.flaky.new.datasets", 2)
+
+    out <- try(expr)
+    if (!inherits(out, "try-error")) return(out)
+
+    num_failures <- get("dataset.retries", envir = globalenv()) + 1
+    assign("dataset.retries", num_failures, envir = globalenv())
+    err_msg <- attr(out, "condition")$message
+
+    if (num_failures > MEX_RETRIES) {
+        stop("Failed to create more than ", MAX_RETRIES, " datasets. ", err_msg)
+    }
+
+    warning(paste0(
+        "Failed to create new Dataset (#", num_failures, ") - ", err_msg, ". Trying again."
+    ))
+    expr
+}
+
+flakyRecoverNewDataset <- function(...) {
+    flakyDatasetExpr(newDataset(...))
+}
+flakyRecoverCreateDataset <- function(...) {
+    flakyDatasetExpr(createDataset(...))
+}
+
 
 datasetFixturePath <- function(filename) {
     # Try in mocks tempdir
@@ -67,10 +99,10 @@ newDatasetFromFixture <- function(filename) {
     m <- fromJSON(datasetFixturePath(paste0(filename, ".json")),
         simplifyVector = FALSE
     )
-    return(suppressMessages(createWithMetadataAndFile(
+    return(suppressMessages(flakyDatasetExpr(createWithMetadataAndFile(
         m,
         datasetFixturePath(paste0(filename, ".csv"))
-    )))
+    ))))
 }
 
 ## Data frames to make datasets with

diff --git a/tests/testthat/test-active-filter.R b/tests/testthat/test-active-filter.R
@@ -214,7 +214,7 @@ with_mock_crunch({
 })
 
 with_test_authentication({
-    ds <- newDataset(df)
+    ds <- flakyRecoverNewDataset(df)
     ds2 <- ds[ds$v4 == "C", ]
     ds2b <- ds[ds$v4 != "B", ]
     ds3 <- ds[ds$v3 > 11, ]

diff --git a/tests/testthat/test-add-variable-definition.R b/tests/testthat/test-add-variable-definition.R
@@ -66,7 +66,7 @@ test_that("Can make a VarDef with no values", {
 
 
 with_test_authentication({
-    ds <- newDataset(df)
+    ds <- flakyRecoverNewDataset(df)
     test_that("Wrapping VarDef has same result as just ds<-", {
         ds$newvar <- VarDef(df$v4)
         expect_true("newvar" %in% names(ds))

diff --git a/tests/testthat/test-add-variable.R b/tests/testthat/test-add-variable.R
@@ -348,7 +348,7 @@ with_mock_crunch({
 })
 
 with_test_authentication({
-    ds <- newDataset(df)
+    ds <- flakyRecoverNewDataset(df)
     test_that("addVariable creates a new remote numeric variable", {
         ds <- addVariables(
             ds,
@@ -420,7 +420,7 @@ with_test_authentication({
     })
 
     test_that("Adding text variables (debugging)", {
-        ds <- newDataset(data.frame(x = 1:1024))
+        ds <- flakyRecoverNewDataset(data.frame(x = 1:1024))
         ds$a_text_var <- "12345 Some text that is definitely >4 characters"
         ds$a_factor <- factor("Different text")
         ds$the_name <- name(ds)
@@ -463,7 +463,7 @@ with_test_authentication({
         )
     })
 
-    ds <- newDataset(df)
+    ds <- flakyRecoverNewDataset(df)
     test_that("Categorical to R and back", {
         v4 <- as.vector(ds$v4)
         expect_identical(levels(v4), c("B", "C"))

diff --git a/tests/testthat/test-append-arrays.R b/tests/testthat/test-append-arrays.R
@@ -2,10 +2,10 @@ context("Appending datasets with arrays")
 
 with_test_authentication({
     whereas("Appending arrays with mismatching names and aliases", {
-        part1 <- mrdf.setup(newDataset(mrdf), selections = "1.0")
+        part1 <- mrdf.setup(flakyRecoverNewDataset(mrdf), selections = "1.0")
         names(subvariables(part1$MR)) <- c("One", "Two", "Three")
         ## Aliases are c("mr_1", "mr_2", "mr_3")
-        part2 <- mrdf.setup(newDataset(mrdf), selections = "1.0")
+        part2 <- mrdf.setup(flakyRecoverNewDataset(mrdf), selections = "1.0")
         names(subvariables(part2$MR)) <- c("Loneliest", "Two", "Three")
         aliases(subvariables(part2$MR))[3] <- "alt"
         ## Aliases are c("mr_1", "mr_2", "alt")

diff --git a/tests/testthat/test-append-conflicts.R b/tests/testthat/test-append-conflicts.R
@@ -2,8 +2,8 @@ context("Handling append conflicts")
 
 with_test_authentication({
     whereas("When attempting to append an array and a numeric", {
-        part1 <- mrdf.setup(newDataset(mrdf))
-        part2 <- newDataset(mrdf[c("mr_3", "v4")])
+        part1 <- mrdf.setup(flakyRecoverNewDataset(mrdf))
+        part2 <- flakyRecoverNewDataset(mrdf[c("mr_3", "v4")])
         alias(part2$mr_3) <- "CA"
         name(part2$CA) <- "Bad var"
         test_that("setup for append array type mismatch", {
@@ -44,10 +44,10 @@ with_test_authentication({
     })
 
     whereas("When attempting to append text and numeric", {
-        part1 <- newDataset(df[, 2:5])
+        part1 <- flakyRecoverNewDataset(df[, 2:5])
         d2 <- df
         d2$v2 <- d2$v3 ## v2 was text, now is numeric
-        part2 <- newDataset(d2)
+        part2 <- flakyRecoverNewDataset(d2)
         test_that("compareDatasets catches that", {
             comp <- compareDatasets(part1, part2)
             expect_prints(summary(comp), "Type mismatch: 1")

diff --git a/tests/testthat/test-append-debug.R b/tests/testthat/test-append-debug.R
@@ -3,7 +3,7 @@ context("Debugging append")
 with_test_authentication({
     with(temp.option(crunch = list(crunch.timeout = 60)), {
         whereas("Appending with an exclusion on the incoming dataset", {
-            part0 <- createDataset(name = now())
+            part0 <- flakyRecoverCreateDataset(name = now())
             part1 <- newDatasetFromFixture("apidocs")
             exclusion(part1) <- part1$q1 == "Dog"
             part2 <- newDatasetFromFixture("apidocs")
@@ -22,8 +22,8 @@ with_test_authentication({
         })
 
         whereas("When appending different arrays containing the same subvars", {
-            part1 <- mrdf.setup(newDataset(mrdf), name = "CA1")
-            part2 <- mrdf.setup(newDataset(mrdf), name = "CA2")
+            part1 <- mrdf.setup(flakyRecoverNewDataset(mrdf), name = "CA1")
+            part2 <- mrdf.setup(flakyRecoverNewDataset(mrdf), name = "CA2")
 
             test_that("The arrays with different aliases have the same subvar aliases", {
                 expect_identical(
@@ -116,10 +116,10 @@ with_test_authentication({
 
         test_that("Derivations with comparison operators can be appended", {
             df1 <- data.frame(foo = rnorm(100), bar = c(1:100))
-            ds1 <- newDataset(df1)
+            ds1 <- flakyRecoverNewDataset(df1)
 
             df2 <- data.frame(bar = c(100:1))
-            ds2 <- newDataset(df2)
+            ds2 <- flakyRecoverNewDataset(df2)
 
             ds1$new1 <- makeCaseVariable(
                 `less than one` = ds1$foo < 1,

diff --git a/tests/testthat/test-append-sparse.R b/tests/testthat/test-append-sparse.R
@@ -1,11 +1,11 @@
 context("Appends with sparse data")
 
 with_test_authentication({
-    part1 <- newDataset(data.frame(
+    part1 <- flakyRecoverNewDataset(data.frame(
         v4 = factor(rep(c("a", "b"), 500)),
         v5 = factor(rep(c("a", "b"), 500))
     ))
-    part2 <- mrdf.setup(newDataset(data.frame(
+    part2 <- mrdf.setup(flakyRecoverNewDataset(data.frame(
         mr_1 = c(1, 0, 1, 1, 0, rep(NA, 995)),
         mr_2 = c(rep(NA, 995), 0, 1, 1, 1, 0),
         v4 = as.factor(LETTERS[2:3])

diff --git a/tests/testthat/test-append-subset.R b/tests/testthat/test-append-subset.R
@@ -52,8 +52,8 @@ with_mock_crunch({
 })
 
 with_test_authentication({
-    ds1 <- newDataset(df[1:3])
-    ds2 <- newDataset(df[2:5])
+    ds1 <- flakyRecoverNewDataset(df[1:3])
+    ds2 <- flakyRecoverNewDataset(df[2:5])
     test_that("We can select variables to append", {
         ds1 <- appendDataset(ds1, ds2[c("v2", "v5")])
         expect_equal(ncol(ds1), 4)

diff --git a/tests/testthat/test-append-subvariables.R b/tests/testthat/test-append-subvariables.R
@@ -4,7 +4,7 @@ with_test_authentication({
     # Revisit after https://www.pivotaltracker.com/n/projects/2172644/stories/186660623
     # (reusable subvar codes) ships
     # whereas("When appending a dataset with unbound subvariables", {
-    #     part1 <- mrdf.setup(newDataset(mrdf), selections = "1.0")
+    #     part1 <- mrdf.setup(flakyRecoverNewDataset(mrdf), selections = "1.0")
     #     mr_cats <- categories(part1$MR)
     #     subvar_cats <- categories(part1$MR$mr_1)
     #     dichotomized_cats <- Categories(
@@ -15,7 +15,7 @@ with_test_authentication({
     #     ## Dichotomize this way so that categories get aligned
     #     ## (via supertype)
     #
-    #     part2 <- mrdf.setup(newDataset(mrdf))
+    #     part2 <- mrdf.setup(flakyRecoverNewDataset(mrdf))
     #     unbind(part2$CA)
     #     part2 <- refresh(part2)
     #     undichotomized_cats <- Categories(
@@ -93,9 +93,9 @@ with_test_authentication({
     # })
 
     whereas("When appending arrays with different subsets of subvariables", {
-        part1 <- mrdf.setup(newDataset(mrdf[-3]), selections = "1.0")
+        part1 <- mrdf.setup(flakyRecoverNewDataset(mrdf[-3]), selections = "1.0")
         part1 <- saveVersion(part1, "Before appending")
-        part2 <- mrdf.setup(newDataset(mrdf[-1]), selections = "1.0")
+        part2 <- mrdf.setup(flakyRecoverNewDataset(mrdf[-1]), selections = "1.0")
         test_that("set up MR for appending", {
             expect_true(is.Multiple(part1$MR))
             expect_identical(

diff --git a/tests/testthat/test-appending.R b/tests/testthat/test-appending.R
@@ -44,8 +44,8 @@ with_mock_crunch({
 
 
 with_test_authentication({
-    part1 <- newDataset(df)
-    part2 <- newDataset(df)
+    part1 <- flakyRecoverNewDataset(df)
+    part2 <- flakyRecoverNewDataset(df)
     cats <- categories(part1$v4)
 
     ## Set a primary key to test that it gets unset

diff --git a/tests/testthat/test-as-data-frame.R b/tests/testthat/test-as-data-frame.R
@@ -185,7 +185,7 @@ with_mock_crunch({
 })
 
 with_test_authentication({
-    ds <- newDataset(df)
+    ds <- flakyRecoverNewDataset(df)
 
     # change the alias of v6 to be something that includes spaces/punctuation
     alias(ds$v6) <- "vee six !"
@@ -297,7 +297,7 @@ with_test_authentication({
 
     test_that("Multiple response variables in as.data.frame(force=TRUE)", {
         skip_on_local_backend("Vagrant host doesn't serve files correctly")
-        mrds <- mrdf.setup(newDataset(mrdf, name = "test-mrdfmr"), selections = "1.0")
+        mrds <- mrdf.setup(flakyRecoverNewDataset(mrdf, name = "test-mrdfmr"), selections = "1.0")
         mrds_df <- as.data.frame(mrds, force = TRUE)
         expect_equal(ncol(mrds_df), 4)
         expect_equal(names(mrds_df), c("mr_1", "mr_2", "mr_3", "v4"))

diff --git a/tests/testthat/test-case-when-variable.R b/tests/testthat/test-case-when-variable.R
@@ -195,7 +195,7 @@ with_mock_crunch({
 })
 
 with_test_authentication({
-    ds <- newDataset(
+    ds <- flakyRecoverNewDataset(
         data.frame(
             fav_brand1 = factor(
                 c("Coke", "Diet Coke", "Diet Pepsi", "Coke", "Pepsi", "Water"),

diff --git a/tests/testthat/test-categories.R b/tests/testthat/test-categories.R
@@ -410,7 +410,7 @@ with_mock_crunch({
 
 with_test_authentication({
     whereas("When editing categories", {
-        ds <- newDataset(df[, 4, drop = FALSE])
+        ds <- flakyRecoverNewDataset(df[, 4, drop = FALSE])
         test_that("categories setters persist to the server", {
             expect_equal(names(categories(ds$v4)), c("B", "C", "No Data"))
             expect_equivalent(

diff --git a/tests/testthat/test-change-category-id.R b/tests/testthat/test-change-category-id.R
@@ -51,7 +51,7 @@ with_mock_crunch({
 })
 
 with_test_authentication({
-    ds <- newDataset(data.frame(
+    ds <- flakyRecoverNewDataset(data.frame(
         samevalue = df$v4[1:4],
         diffvalue = df$v4[1:4],
         excluded = df$v4[1:4],

diff --git a/tests/testthat/test-crunchbox.R b/tests/testthat/test-crunchbox.R
@@ -338,7 +338,7 @@ with_test_authentication({
     testdf <- as.data.frame(sapply(letters[1:8], function(x) df$v4, simplify = FALSE))
     testdf$cat <- as.factor(letters[1:10])
     testdf$num <- 1
-    ds <- newDataset(testdf)
+    ds <- flakyRecoverNewDataset(testdf)
     ds$mr <- makeMR(ds[letters[1:8]],
         name = "Excessively long variable name to trigger the check for length",
         selections = "B"

diff --git a/tests/testthat/test-cube-dims.R b/tests/testthat/test-cube-dims.R
@@ -215,7 +215,7 @@ with_mock_crunch({
 })
 
 with_test_authentication({
-    ds <- newDataset(data.frame(x = c(LETTERS[1:10])), name = "test for notes")
+    ds <- flakyRecoverNewDataset(data.frame(x = c(LETTERS[1:10])), name = "test for notes")
 
     test_that("empty variable description are empty string", {
         expect_equal(description(ds$x), "")

diff --git a/tests/testthat/test-cube-errors.R b/tests/testthat/test-cube-errors.R
@@ -154,7 +154,7 @@ with_mock_crunch({
 })
 
 with_test_authentication({
-    ds <- newDataset(df[, 1:4])
+    ds <- flakyRecoverNewDataset(df[, 1:4])
     test_that("All variables must be present in data", {
         expect_error(
             crtabs(~ aaa + v3, data = ds),

diff --git a/tests/testthat/test-cube-with-array.R b/tests/testthat/test-cube-with-array.R
@@ -3,7 +3,7 @@ context("Cubes with categorical array and multiple response")
 with_test_authentication({
     cubemrdf <- mrdf
     cubemrdf$v5 <- as.factor(c("A", "A", "B", "B"))
-    mrds <- mrdf.setup(newDataset(cubemrdf), selections = "1.0")
+    mrds <- mrdf.setup(flakyRecoverNewDataset(cubemrdf), selections = "1.0")
 
     test_that("univariate multiple response cube", {
         kube <- crtabs(~MR, data = mrds)

diff --git a/tests/testthat/test-cubes.R b/tests/testthat/test-cubes.R
@@ -164,7 +164,7 @@ test_that("margin.table with missing", {
 })
 
 with_test_authentication({
-    ds <- newDataset(cubedf)
+    ds <- flakyRecoverNewDataset(cubedf)
     test_that("cubedf setup", {
         expect_identical(
             names(categories(ds$v7)),
@@ -493,7 +493,7 @@ with_test_authentication({
 
     test_that("scorecard query works", {
         # Setup a dataset for scorecards
-        ds_scorecard <- newDataset(
+        ds_scorecard <- flakyRecoverNewDataset(
             data.frame(
                 x1 = factor(c("a", "b", "c", "a", "c"), letters[1:3]),
                 x2 = factor(c("c", "c", "b", "c", "b"), letters[1:3])

diff --git a/tests/testthat/test-cut.R b/tests/testthat/test-cut.R
@@ -625,7 +625,7 @@ with_mock_crunch({
 })
 
 with_test_authentication({
-    ds <- newDataset(df)
+    ds <- flakyRecoverNewDataset(df)
 
     test_that("cut returns the same thing for Crunch variables and identical vectors", {
         ds$cat_var1 <- cut(ds$v1, 3,

diff --git a/tests/testthat/test-dataset-entity.R b/tests/testthat/test-dataset-entity.R
@@ -548,7 +548,7 @@ if (tolower(Sys.info()[["sysname"]]) != "windows") {
 
     with_test_authentication({
         whereas("When editing dataset metadata", {
-            ds <- createDataset(name = now())
+            ds <- flakyRecoverCreateDataset(name = now())
             test_that("Name and description setters push to server", {
                 d2 <- ds
                 name(ds) <- "Bond. James Bond."

diff --git a/tests/testthat/test-decks.R b/tests/testthat/test-decks.R
@@ -266,7 +266,7 @@ test_that("filter gets pass through method on non-crunch objects", {
 })
 
 with_test_authentication({
-    ds <- newDataset(df)
+    ds <- flakyRecoverNewDataset(df)
     test_that("decks can be created", {
         expect_is(decks(ds), "DeckCatalog")
         expect_equal(length(decks(ds)), 0)

diff --git a/tests/testthat/test-derive.R b/tests/testthat/test-derive.R
@@ -86,7 +86,7 @@ with_mock_crunch({
 })
 
 with_test_authentication({
-    ds <- newDataset(df)
+    ds <- flakyRecoverNewDataset(df)
     ds$v3a <- ds$v3 + 5
     test_that("A derived variable is created on the server", {
         expect_true("v3a" %in% names(allVariables(refresh(ds))))