diff --git a/R/pagasa_dam.R b/R/pagasa_dam.R new file mode 100644 index 0000000..08a5c95 --- /dev/null +++ b/R/pagasa_dam.R @@ -0,0 +1,90 @@ +#' +#' Get dam level information from PAGASA website +#' +#' @param .url The URL for the dam level information from PAGASA. Currently, +#' this is at https://www.pagasa.dost.gov.ph/flood#dam-information. +#' +#' @returns A tibble of dam level information +#' +#' @examples +#' dam_get_level() +#' +#' @export +#' + +dam_get_level <- function(.url = "https://www.pagasa.dost.gov.ph/flood#dam-information") { + ## Quiet down error on SSL ---- + httr::config(ssl_verifypeer = 0L) |> + httr::set_config() + + ## Initiate an HTML session ---- + pagasa_session <- rvest::session(.url) + + ## Retrieve data ---- + dam_tab <- pagasa_session |> + rvest::html_elements(css = ".dam-table") |> + rvest::html_table() + + dam_tab <- dam_tab[[1]] + + first_row_names <- names(dam_tab) + second_row_names <- dam_tab[1, ] |> unlist() + names(second_row_names) <- NULL + + tab_names <- ifelse( + first_row_names == second_row_names, + first_row_names, + paste(first_row_names, second_row_names) + ) + + names(dam_tab) <- tab_names + + dam_tab <- dam_tab |> + dplyr::slice(2:nrow(dam_tab)) |> + dplyr::mutate( + group = paste(`Dam Name`, ceiling(dplyr::row_number() / 2)), + `Date Retrieved` = Sys.Date(), + .before = `Dam Name` + ) |> + dplyr::group_by(group) |> + dplyr::summarise( + `Date Retrieved` = unique(`Date Retrieved`), + `Dam Name` = unique(`Dam Name`), + `Observation Time & Date` = paste( + paste(`Observation Time & Date`[2], lubridate::year(Sys.Date()), sep = "-"), + `Observation Time & Date`[1] + ), + `Reservoir Water Level (RWL) (m)` = unique(`Reservoir Water Level (RWL) (m)`), + `Water Level Deviation Hr` = unique(`Water Level Deviation Hr`), + `Water Level Deviation Amount` = unique(`Water Level Deviation Amount`), + `Normal High Water Level (NHWL) (m)` = unique(`Normal High Water Level (NHWL) (m)`), + `Deviation from NHWL (m)` = unique(`Deviation from NHWL (m)`), + `Rule Curve Elevation (m)` = unique(`Rule Curve Elevation (m)`), + `Deviation from Rule Curve (m)` = unique(`Deviation from Rule Curve (m)`), + `Gate Opening Gates` = unique(`Gate Opening Gates`), + `Gate Opening Meters` = unique(`Gate Opening Meters`), + `Estimated (cms) Inflow` = unique(`Estimated (cms) Inflow`), + `Estimated (cms) Outflow` = unique(`Estimated (cms) Outflow`), + .groups = "drop" + ) |> + dplyr::select(-group) |> + dplyr::mutate( + `Observation Time & Date` = strptime( + `Observation Time & Date`, format = "%b-%d-%Y %H:%M %p" + ), + `Reservoir Water Level (RWL) (m)` = as.numeric(`Reservoir Water Level (RWL) (m)`), + `Water Level Deviation Hr` = as.integer(`Water Level Deviation Hr`), + `Water Level Deviation Amount` = as.numeric(`Water Level Deviation Amount`), + `Normal High Water Level (NHWL) (m)` = as.numeric(`Normal High Water Level (NHWL) (m)`), + `Deviation from NHWL (m)` = as.numeric(`Deviation from NHWL (m)`), + `Rule Curve Elevation (m)` = as.numeric(`Rule Curve Elevation (m)`), + `Deviation from Rule Curve (m)` = as.numeric(`Deviation from Rule Curve (m)`), + `Gate Opening Gates` = as.numeric(`Gate Opening Gates`), + `Gate Opening Meters` = as.numeric(`Gate Opening Meters`), + `Estimated (cms) Inflow` = as.numeric(`Estimated (cms) Inflow`), + `Estimated (cms) Outflow` = as.numeric(`Estimated (cms) Outflow`), + ) + + ## Return dam_tab + dam_tab +} \ No newline at end of file diff --git a/README.Rmd b/README.Rmd index 925d821..def3d8f 100644 --- a/README.Rmd +++ b/README.Rmd @@ -35,7 +35,7 @@ The `paglaom` project aims to maintain a database of curated datasets on varios The broader and more blue skies vision of the `paglaom` project is to contribute to the increasing interest in science, technology, engineering, and mathematics (STEM) sciences particularly in the Philippines with a collection that showcases topics and data that are homegrown and embedded into the fabric of Philippine life. -Whilst the `paglaom` by its name and the nature of the data it curates has an inherent Filipino audience, it is hoped that those outside of the Philippines will also find the information within useful in similar contexts described above. +Whilst the `paglaom` project by its name and the nature of the data it curates has an inherent Filipino audience, it is hoped that those outside of the Philippines will also find the information within useful in similar contexts described above. ## Repository Structure @@ -85,6 +85,18 @@ paglaom * `_targets.yaml` file defines the different targets sub-projects within this project. +## The workflow + +Currently, the project curates the following datasets: + +1. Tropical cyclones data for various cyclones entering the Philippine area of responsibility since 2017; + +2. Daily heat index data from various data collection points in the Philippines; + +3. Climatological extremes and normals data over time; and, + +4. Daily dam water level data. + ## Reproducibility ### R package dependencies diff --git a/README.md b/README.md index ca832d6..6331501 100644 --- a/README.md +++ b/README.md @@ -65,10 +65,10 @@ engineering, and mathematics (STEM) sciences particularly in the Philippines with a collection that showcases topics and data that are homegrown and embedded into the fabric of Philippine life. -Whilst the `paglaom` by its name and the nature of the data it curates -has an inherent Filipino audience, it is hoped that those outside of the -Philippines will also find the information within useful in similar -contexts described above. +Whilst the `paglaom` project by its name and the nature of the data it +curates has an inherent Filipino audience, it is hoped that those +outside of the Philippines will also find the information within useful +in similar contexts described above. ## Repository Structure @@ -142,6 +142,20 @@ The project repository is structured as follows: - `_targets.yaml` file defines the different targets sub-projects within this project. +## The workflow + +Currently, the project curates the following datasets: + +1. Tropical cyclones data for various cyclones entering the Philippine + area of responsibility since 2017; + +2. Daily heat index data from various data collection points in the + Philippines; + +3. Climatological extremes and normals data over time; and, + +4. Daily dam water level data. + ## Reproducibility ### R package dependencies diff --git a/_targets.yaml b/_targets.yaml index ab90b2e..9005409 100644 --- a/_targets.yaml +++ b/_targets.yaml @@ -7,3 +7,6 @@ heat: climate: script: _targets_climate.R store: _targets +dam: + script: _targets_dam.R + store: _targets diff --git a/_targets_dam.R b/_targets_dam.R new file mode 100644 index 0000000..2442694 --- /dev/null +++ b/_targets_dam.R @@ -0,0 +1,72 @@ +################################################################################ +# +# Targets workflow for dam level data extraction, and processing +# +################################################################################ + +## Setup workflow using project-wide settings ---------------------------------- +source("_targets_setup.R") + + +## Create targets and list targets objects ------------------------------------- + +### Data targets +data_targets <- tar_plan( + ### Set PAGASA dam level URL ---- + tar_target( + name = dam_level_url, + command = "https://www.pagasa.dost.gov.ph/flood#dam-information" + ), + ### Get dam level data ---- + tar_target( + name = dam_level_data, + command = dam_get_level(.url = dam_level_url), + ), + ### Output dam level data as CSV ---- + tar_target( + name = dam_level_data_csv, + command = { + write.csv( + dam_level_data, + file = paste0("data/dam_level_", Sys.Date(), ".csv"), + row.names = FALSE + ) + paste0("data/dam_level_", Sys.Date(), ".csv") + }, + format = "file" + ) +) + + +### Processing targets +processing_targets <- tar_plan( + +) + + +### Analysis targets +analysis_targets <- tar_plan( + +) + + +### Output targets +output_targets <- tar_plan( + +) + + +### Reporting targets +report_targets <- tar_plan( + +) + + +### Deploy targets +deploy_targets <- tar_plan( + +) + + +## List targets +all_targets() diff --git a/renv.lock b/renv.lock index 01bca24..3fbad12 100644 --- a/renv.lock +++ b/renv.lock @@ -256,17 +256,6 @@ ], "Hash": "e85ffbebaad5f70e1a2e2ef4302b4949" }, - "ellipsis": { - "Package": "ellipsis", - "Version": "0.3.2", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "rlang" - ], - "Hash": "bb0eec2fe32e88d9e2836c2f73ea2077" - }, "evaluate": { "Package": "evaluate", "Version": "0.21", @@ -494,20 +483,19 @@ }, "htmltools": { "Package": "htmltools", - "Version": "0.5.6", + "Version": "0.5.8.1", "Source": "Repository", "Repository": "CRAN", "Requirements": [ "R", "base64enc", "digest", - "ellipsis", "fastmap", "grDevices", "rlang", "utils" ], - "Hash": "a2326a66919a3311f7fbb1e3bf568283" + "Hash": "81d371a9cc60640e74e4ab6ac46dcedc" }, "httr": { "Package": "httr",