diff --git a/R/utils_labels.R b/R/utils_labels.R
index a29cdc497..a7f4fa2c3 100644
--- a/R/utils_labels.R
+++ b/R/utils_labels.R
@@ -56,7 +56,7 @@
   value_labels <- c(value_labels, stats::setNames(missing_levels, missing_levels))
 
   # now we can add back levels
-  levels(x) <- names(value_labels)[order(value_labels)]
+  levels(x) <- names(value_labels)[order(as.numeric(value_labels))]
   attr(x, "labels") <- NULL
 
   x