process raw dam water levels data

panukatan · Aug 11, 2024 · ced3306 · ced3306
1 parent 85c90ea
commit ced3306
Show file tree

Hide file tree

Showing 3 changed files with 113 additions and 13 deletions.
diff --git a/R/pagasa_dam.R b/R/pagasa_dam.R
@@ -79,7 +79,7 @@ dam_get_level <- function(.url = "https://www.pagasa.dost.gov.ph/flood#dam-infor
       `Deviation from NHWL (m)` = as.numeric(`Deviation from NHWL (m)`),
       `Rule Curve Elevation (m)` = as.numeric(`Rule Curve Elevation (m)`),
       `Deviation from Rule Curve (m)` = as.numeric(`Deviation from Rule Curve (m)`),
-      `Gate Opening Gates` = as.numeric(`Gate Opening Gates`),                 
+      `Gate Opening Gates` = as.integer(`Gate Opening Gates`),                 
       `Gate Opening Meters` = as.numeric(`Gate Opening Meters`),
       `Estimated (cms) Inflow` = as.numeric(`Estimated (cms) Inflow`),            
       `Estimated (cms) Outflow` = as.numeric(`Estimated (cms) Outflow`),
@@ -108,3 +108,71 @@ dam_archive_raw <- function(dam_level_data, directory = "data-raw") {
 
   paste0(archive_dir, "/dam_level_", Sys.Date(), ".csv")
 }
+
+#'
+#' Process dam water level data
+#' 
+#' @param dam_level_data_files A vector of paths to each of the daily
+#'   dam water level data CSV files
+#'   
+#' @returns A tibble of concatenated daily dam water level data
+#' 
+#' @examples
+#' dam_process_date(dam_level_data_files) 
+#'
+#' @export
+#'
+
+dam_process_data <- function(dam_level_data_files) {
+  lapply(
+    X = dam_level_data_files,
+    FUN = read.csv
+  ) |>
+    dplyr::bind_rows() |>
+    dplyr::rename_with(
+      .fn = function(x) c(
+        "data_retrieval_date", "dam_name", 
+        "observation_date_time", "water_level", 
+        "water_level_deviation_period", "water_level_deviation", 
+        "water_level_high_normal", "water_level_deviation_from_normal",
+        "rule_curve_elevation", "rule_curve_elevation_deviation",
+        "gates_opened", "gates_opening_width", 
+        "inflow_estimated", "outflow_estimated"
+      )
+    ) |>
+    dplyr::filter(
+      as.Date(observation_date_time) == as.Date(data_retrieval_date) |
+        (as.Date(observation_date_time) != as.Date(data_retrieval_date) &
+        !as.Date(observation_date_time) %in% 
+        seq(from = as.Date("2024-04-18"), to = Sys.Date(), by = "day"))
+    ) |>
+    dplyr::mutate(
+      data_retrieval_date = as.Date(data_retrieval_date),
+      observation_date_time = strptime(
+        observation_date_time, format = "%Y-%m-%d %H:%M:%S"
+      ),
+      dplyr::across(
+        .cols = dplyr::contains("estimated"),
+        .fns = ~as.numeric(.x)
+      )
+    ) |>
+    dplyr::arrange(dam_name, observation_date_time) |>
+    tibble::tibble()
+}
+
+#'
+#' 
+#'
+
+dam_archive_processed <- function(dam_level_data_processed, 
+                                  directory = "data") {
+  file_path <- file.path(directory, "dam_water_levels.csv")
+
+  write.csv(
+    x = dam_level_data_processed,
+    file = file_path,
+    row.names = FALSE
+  )
+
+  file_path
+}
diff --git a/_targets_dam.R b/_targets_dam.R
@@ -4,13 +4,13 @@
 #
 ################################################################################
 
-## Setup workflow using project-wide settings ----------------------------------
+# Setup workflow using project-wide settings ----------------------------------
 source("_targets_setup.R")
 
 
-## Create targets and list targets objects -------------------------------------
+# Create targets and list targets objects -------------------------------------
 
-### Data targets
+## Data targets
 data_targets <- tar_plan(
   ### Set PAGASA dam level URL ----
   tar_target(
@@ -23,30 +23,43 @@ data_targets <- tar_plan(
     command = dam_get_level(.url = dam_level_url),
     cue = tar_cue("always")
   ),
-  ### Output dam level data as CSV ----
+  ### Get list of saved/stored dam level CSVs ----
   tar_target(
-    name = dam_level_data_raw_csv,
-    command = dam_archive_raw(dam_level_data),
-    format = "file"
+    name = dam_level_data_files,
+    command = list.files(path = "data-raw/dam", full.names = TRUE)
   )
 )
 
 
-### Processing targets
+## Processing targets
 processing_targets <- tar_plan(
-
+  ### Processing daily dam level data ----
+  tar_target(
+    name = dam_level_data_processed,
+    command = dam_process_data(dam_level_data_files)
+  )
 )
 
 
-### Analysis targets
+## Analysis targets
 analysis_targets <- tar_plan(
 
 )
 
 
-### Output targets
+## Output targets
 output_targets <- tar_plan(
-
+  ### Output dam level data as CSV ----
+  tar_target(
+    name = dam_level_data_raw_csv,
+    command = dam_archive_raw(dam_level_data),
+    format = "file"
+  ),
+  ### Output processed dam level data as CSV ----
+  tar_target(
+    name = dam_level_data_csv,
+    command = dam_archive_processed(dam_level_data_processed)
+  )
 )
 
 

diff --git a/data-raw/dam/dam_level_2024-08-11.csv b/data-raw/dam/dam_level_2024-08-11.csv
@@ -0,0 +1,19 @@
+"Date Retrieved","Dam Name","Observation Time & Date","Reservoir Water Level (RWL) (m)","Water Level Deviation Hr","Water Level Deviation Amount","Normal High Water Level (NHWL) (m)","Deviation from NHWL (m)","Rule Curve Elevation (m)","Deviation from Rule Curve (m)","Gate Opening Gates","Gate Opening Meters","Estimated (cms) Inflow","Estimated (cms) Outflow"
+2024-08-11,"Ambuklao",2024-08-11 08:00:00,751.31,24,-0.07,752,-0.69,745,6.31,NA,NA,NA,NA
+2024-08-11,"Ambuklao",2024-08-10 08:00:00,751.38,24,-0.07,752,-0.62,745,6.38,NA,NA,NA,NA
+2024-08-11,"Angat",2024-08-11 08:00:00,187.24,24,-0.11,210,-22.76,181.24,6,NA,NA,NA,NA
+2024-08-11,"Angat",2024-08-10 08:00:00,187.35,24,-0.11,210,-22.65,181.13,6.22,NA,NA,NA,NA
+2024-08-11,"Binga",2024-08-10 08:00:00,573.08,24,-0.47,575,-1.92,565,8.08,NA,NA,NA,NA
+2024-08-11,"Binga",2024-08-11 08:00:00,572.61,24,-0.47,575,-2.39,565,7.61,NA,NA,NA,NA
+2024-08-11,"Caliraya",2024-08-11 08:00:00,286.83,24,-0.12,0,0,0,0,NA,NA,NA,NA
+2024-08-11,"Caliraya",2024-08-10 08:00:00,286.95,24,-0.12,0,0,0,0,NA,NA,NA,NA
+2024-08-11,"Ipo",2024-08-11 08:00:00,100.52,24,0.13,101.1,-0.58,0,0,NA,NA,NA,NA
+2024-08-11,"Ipo",2024-08-10 08:00:00,100.39,24,0.13,101.1,-0.61,0,0,NA,NA,NA,NA
+2024-08-11,"La Mesa",2024-08-11 08:00:00,79.82,24,0,80.15,-0.33,0,0,NA,NA,NA,NA
+2024-08-11,"La Mesa",2024-08-10 08:00:00,79.82,24,0,80.15,-0.33,0,0,NA,NA,NA,NA
+2024-08-11,"Magat Dam",2024-08-11 08:00:00,184.86,24,-0.32,190,-5.14,180.05,4.81,NA,NA,NA,NA
+2024-08-11,"Magat Dam",2024-08-10 08:00:00,185.18,24,-0.32,190,-4.82,179.93,5.25,NA,NA,NA,NA
+2024-08-11,"Pantabangan",2024-08-11 08:00:00,187.33,24,0.13,216,-28.67,210.02,-22.69,NA,NA,NA,NA
+2024-08-11,"Pantabangan",2024-08-10 08:00:00,187.2,24,0.13,216,-28.8,209.9,-22.7,NA,NA,NA,NA
+2024-08-11,"San Roque",2024-08-11 08:00:00,253.77,24,-0.17,280,-26.23,243.51,10.26,NA,NA,NA,NA
+2024-08-11,"San Roque",2024-08-10 08:00:00,253.94,24,-0.17,280,-26.06,243.03,10.91,NA,NA,NA,NA