From 602bdc8c17d6f59ab189aa2dc6533aa1d038ff11 Mon Sep 17 00:00:00 2001 From: Hong Ooi Date: Thu, 30 Sep 2021 22:26:02 +1000 Subject: [PATCH] Implement Onedrive/Sharepoint board (#513) Fixes #498 --- DESCRIPTION | 3 +- NAMESPACE | 9 ++ NEWS.md | 3 + R/board_ms365.R | 226 ++++++++++++++++++++++++++++++ R/pin_versions.R | 4 +- README.Rmd | 4 +- README.md | 16 +-- _pkgdown.yml | 3 +- man/board_ms365.Rd | 74 ++++++++++ man/board_rsconnect.Rd | 4 +- man/pin_versions.Rd | 4 +- tests/testthat/test-board_ms365.R | 9 ++ 12 files changed, 341 insertions(+), 18 deletions(-) create mode 100644 R/board_ms365.R create mode 100644 man/board_ms365.Rd create mode 100644 tests/testthat/test-board_ms365.R diff --git a/DESCRIPTION b/DESCRIPTION index 5cd9ce883..c55e17483 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -39,13 +39,14 @@ Imports: mime, openssl, rappdirs, - rlang, + rlang (>= 0.4.10), tibble, whisker, withr, yaml, zip Suggests: + Microsoft365R, AzureStor, data.table, datasets, diff --git a/NAMESPACE b/NAMESPACE index 557cb7613..b3a2a9a54 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -55,6 +55,7 @@ S3method(pin_delete,pins_board_azure) S3method(pin_delete,pins_board_folder) S3method(pin_delete,pins_board_kaggle_competition) S3method(pin_delete,pins_board_kaggle_dataset) +S3method(pin_delete,pins_board_ms365) S3method(pin_delete,pins_board_rsconnect) S3method(pin_delete,pins_board_s3) S3method(pin_delete,pins_board_url) @@ -62,6 +63,7 @@ S3method(pin_exists,pins_board_azure) S3method(pin_exists,pins_board_folder) S3method(pin_exists,pins_board_kaggle_competition) S3method(pin_exists,pins_board_kaggle_dataset) +S3method(pin_exists,pins_board_ms365) S3method(pin_exists,pins_board_rsconnect) S3method(pin_exists,pins_board_s3) S3method(pin_exists,pins_board_url) @@ -69,6 +71,7 @@ S3method(pin_fetch,pins_board_azure) S3method(pin_fetch,pins_board_folder) S3method(pin_fetch,pins_board_kaggle_competition) S3method(pin_fetch,pins_board_kaggle_dataset) +S3method(pin_fetch,pins_board_ms365) S3method(pin_fetch,pins_board_rsconnect) S3method(pin_fetch,pins_board_s3) S3method(pin_fetch,pins_board_url) @@ -77,6 +80,7 @@ S3method(pin_list,pins_board_folder) S3method(pin_list,pins_board_kaggle_competition) S3method(pin_list,pins_board_kaggle_dataset) S3method(pin_list,pins_board_local) +S3method(pin_list,pins_board_ms365) S3method(pin_list,pins_board_rsconnect) S3method(pin_list,pins_board_s3) S3method(pin_list,pins_board_url) @@ -88,6 +92,7 @@ S3method(pin_meta,pins_board_azure) S3method(pin_meta,pins_board_folder) S3method(pin_meta,pins_board_kaggle_competition) S3method(pin_meta,pins_board_kaggle_dataset) +S3method(pin_meta,pins_board_ms365) S3method(pin_meta,pins_board_rsconnect) S3method(pin_meta,pins_board_s3) S3method(pin_meta,pins_board_url) @@ -102,18 +107,21 @@ S3method(pin_store,pins_board_azure) S3method(pin_store,pins_board_folder) S3method(pin_store,pins_board_kaggle_competition) S3method(pin_store,pins_board_kaggle_dataset) +S3method(pin_store,pins_board_ms365) S3method(pin_store,pins_board_rsconnect) S3method(pin_store,pins_board_s3) S3method(pin_store,pins_board_url) S3method(pin_version_delete,pins_board) S3method(pin_version_delete,pins_board_azure) S3method(pin_version_delete,pins_board_folder) +S3method(pin_version_delete,pins_board_ms365) S3method(pin_version_delete,pins_board_rsconnect) S3method(pin_version_delete,pins_board_s3) S3method(pin_versions,pins_board) S3method(pin_versions,pins_board_azure) S3method(pin_versions,pins_board_folder) S3method(pin_versions,pins_board_kaggle_dataset) +S3method(pin_versions,pins_board_ms365) S3method(pin_versions,pins_board_rsconnect) S3method(pin_versions,pins_board_s3) S3method(print,pin_info) @@ -139,6 +147,7 @@ export(board_kaggle_dataset) export(board_list) export(board_local) export(board_local_storage) +export(board_ms365) export(board_pin_create) export(board_pin_find) export(board_pin_get) diff --git a/NEWS.md b/NEWS.md index 6cc09f77f..d292d9bac 100644 --- a/NEWS.md +++ b/NEWS.md @@ -62,6 +62,9 @@ This version includes the following modern boards: download data from Kaggle. The data is automatically cached so that it's only downloaded when it changes. +* `board_ms365()` allow to pin data to MS One Drive and Sharpoint + (#498, @hongooi73). + * `board_rsconnect()` shares data on [RStudio connect](https://www.rstudio.com/products/connect/). This board supports both modern and legacy APIs, so that you and your colleagues can use diff --git a/R/board_ms365.R b/R/board_ms365.R new file mode 100644 index 000000000..910fcfa98 --- /dev/null +++ b/R/board_ms365.R @@ -0,0 +1,226 @@ +#' Use a OneDrive or Sharepoint document library as a board +#' +#' Pin data to a folder in Onedrive or a SharePoint Online document library +#' using the Microsoft365R package. +#' +#' @inheritParams new_board +#' @param drive A OneDrive or SharePoint document library object, of class +#' [`Microsoft365R::ms_drive`]. +#' @param path Path to directory to store pins. This can be either a string +#' containing the pathname like `"path/to/board"`, or a +#' [`Microsoft365R::ms_drive_item`] object pointing to the board path. +#' @param delete_by_item Whether to handle folder deletions on an item-by-item +#' basis, rather than deleting the entire folder at once. You may need to set +#' this to `TRUE` for a board in SharePoint Online or OneDrive for Business, +#' due to document protection policies that prohibit deleting non-empty +#' folders. +#' @details +#' Sharing a board in OneDrive (personal or business) is a bit complicated, as OneDrive normally allows only the person who owns the drive to access files and folders. First, the drive owner has to set the board folder as shared with other users, using either the OneDrive web interface or Microsoft365R's `ms_drive_item$create_share_link()` method. The other users then call `board_ms365` with a _drive item object_ in the `path` argument, pointing to the shared folder. See the examples below. +#' +#' Sharing a board in SharePoint Online is much more straightforward, assuming all users have access to the document library: in this case, everyone can use the same call `board_ms365(doclib, "path/to/board")`. If you want to share a board with users outside your team, follow the same steps for sharing a board in OneDrive. +#' @export +#' @examples +#' \dontrun{ +#' # A board in your personal OneDrive +#' od <- Microsoft365R::get_personal_onedrive() +#' board <- board_ms365(od, "myboard") +#' board %>% pin_write(iris) +#' +#' # A board in OneDrive for Business +#' odb <- Microsoft365R::get_business_onedrive(tenant = "mytenant") +#' board <- board_ms365(odb, "myproject/board") +#' +#' # A board in a SharePoint Online document library +#' sp <- Microsoft365R::get_sharepoint_site("my site", tenant = "mytenant") +#' doclib <- sp$get_drive() +#' board <- board_ms365(doclib, "general/project1/board") +#' +#' +#' ## Sharing a board in OneDrive: +#' # First, create the board on the drive owner's side +#' board <- board_ms365(od, "myboard") +#' +#' # Next, let other users write to the folder +#' # - set the expiry to NULL if you want the folder to be permanently available +#' od$get_item("myboard")$create_share_link("edit", expiry="30 days") +#' +#' # On the recipient's side: find the shared folder item, then pass it to board_ms365 +#' shared_items <- od$list_shared_items() +#' board_folder <- shared_items$remoteItem[[which(shared_items$name == "myboard")]] +#' board <- board_ms365(od, board_folder) +#' } +board_ms365 <- function(drive, path, versioned = TRUE, cache = NULL, delete_by_item = FALSE) { + check_installed("Microsoft365R") + + if (!inherits(drive, "ms_drive")) { + abort("`drive` must be a OneDrive or SharePoint document library object") + } + if (!inherits(path, c("character", "ms_drive_item"))) { + abort("`path` must be either a string or a drive item object") + } + + if (!inherits(path, "ms_drive_item")) { + # try to create the board folder: ignore error if folder already exists + try(drive$create_folder(path), silent = TRUE) + folder <- drive$get_item(path) + } + else { + folder <- path + # ensure we have the correct properties for a shared item in OneDrive + folder$sync_fields() + path <- NULL + } + + if (!folder$is_folder()) { + abort("Invalid path specified") + } + + cache <- cache %||% board_cache_path(paste0("ms365-", hash(folder$properties$id))) + new_board_v1("pins_board_ms365", + folder = folder, + path = path, + cache = cache, + versioned = versioned, + delete_by_item = delete_by_item + ) +} + +board_ms365_test_charpath <- function(...) { + if (identical(Sys.getenv("PINS_MS365_USE_PERSONAL"), "true")) { + drv <- Microsoft365R::get_personal_onedrive() + } else { + skip_if_missing_envvars("board_ms365()", "PINS_MS365_TEST_DRIVE") + drv <- readRDS(Sys.getenv("PINS_MS365_TEST_DRIVE")) + } + board_ms365(drv, path = "pin_testing", cache = tempfile(), ...) +} + +board_ms365_test_driveitem <- function(...) { + if (identical(Sys.getenv("PINS_MS365_USE_PERSONAL"), "true")) { + drv <- Microsoft365R::get_personal_onedrive() + } else { + skip_if_missing_envvars("board_ms365()", "PINS_MS365_TEST_DRIVE") + drv <- readRDS(Sys.getenv("PINS_MS365_TEST_DRIVE")) + } + folder <- try(drv$create_folder("pin_testing_2"), silent = TRUE) + board_ms365(drv, path = folder, cache = tempfile(), ...) +} + +#' @export +pin_list.pins_board_ms365 <- function(board, ...) { + ms365_list_dirs(board) +} + +#' @export +pin_exists.pins_board_ms365 <- function(board, name, ...) { + name %in% ms365_list_dirs(board) +} + +#' @export +pin_delete.pins_board_ms365 <- function(board, names, ...) { + for (name in names) { + check_pin_exists(board, name) + ms365_delete_dir(board, name) + } + invisible(board) +} + +#' @export +pin_versions.pins_board_ms365 <- function(board, name, ...) { + check_pin_exists(board, name) + version_from_path(ms365_list_dirs(board, name)) +} + +#' @export +pin_version_delete.pins_board_ms365 <- function(board, name, version, ...) { + ms365_delete_dir(board, fs::path(name, version)) +} + +#' @export +pin_meta.pins_board_ms365 <- function(board, name, version = NULL, ...) { + check_pin_exists(board, name) + version <- check_pin_version(board, name, version) + metadata_key <- fs::path(name, version, "data.txt") + + if (!ms365_file_exists(board, metadata_key)) { + abort_pin_version_missing(version) + } + + path_version <- fs::path(board$cache, name, version) + fs::dir_create(path_version) + + ms365_download(board, metadata_key) + local_meta( + read_meta(fs::path(board$cache, name, version)), + dir = path_version, + version = version + ) +} + +#' @export +pin_fetch.pins_board_ms365 <- function(board, name, version = NULL, ...) { + meta <- pin_meta(board, name, version = version) + cache_touch(board, meta) + + for (file in meta$file) { + key <- fs::path(name, meta$local$version, file) + ms365_download(board, key) + } + + meta +} + +#' @export +pin_store.pins_board_ms365 <- function(board, name, paths, metadata, + versioned = NULL, ...) { + check_name(name) + version <- version_setup(board, name, version_name(metadata), versioned = versioned) + + version_dir <- fs::path(name, version) + + # Upload metadata + meta_tmpfile <- tempfile(fileext = ".yml") + on.exit(unlink(meta_tmpfile)) + yaml::write_yaml(metadata, meta_tmpfile) + board$folder$upload(meta_tmpfile, fs::path(version_dir, "data.txt")) + + # Upload files + for (path in paths) { + board$folder$upload(path, fs::path(version_dir, fs::path_file(path))) + } + + name +} + + +# helpers + +# list all the directories inside 'path', which is assumed to live in the board folder +ms365_list_dirs <- function(board, path = "") { + conts <- board$folder$list_files(path) + conts$name[conts$isdir] +} + +# delete directory 'path', which is assumed to live in the board folder +ms365_delete_dir <- function(board, path = "") { + child <- board$folder$get_item(path) + child$delete(confirm = FALSE, by_item = board$delete_by_item) +} + +# check if a file exists and is not a directory +ms365_file_exists <- function(board, key) { + item <- try(board$folder$get_item(key), silent = TRUE) + inherits(item, "ms_drive_item") && !item$is_folder() +} + +# download a specific file from the board, as given by the 'key' path +ms365_download <- function(board, key) { + path <- fs::path(board$cache, key) + + if (!fs::file_exists(path)) { + board$folder$get_item(key)$download(path) + fs::file_chmod(path, "u=r") + } + + path +} diff --git a/R/pin_versions.R b/R/pin_versions.R index 737787d37..6dd59fe8b 100644 --- a/R/pin_versions.R +++ b/R/pin_versions.R @@ -90,7 +90,7 @@ pin_version_delete.pins_board <- function(board, name, version, ...) { #' keep. `n = 3` will keep the last three versions, `days = 14` will #' keep all the versions in the 14 days. Regardless of what values you #' set, `pin_versions_prune()` will never delete the most recent version. -pin_versions_prune <- function(board, name, n = NULL, days = NULL) { +pin_versions_prune <- function(board, name, n = NULL, days = NULL, ...) { versions <- pin_versions(board, name) keep <- versions_keep(versions$created, n = n, days = days) @@ -99,7 +99,7 @@ pin_versions_prune <- function(board, name, n = NULL, days = NULL) { pins_inform(paste0("Deleting versions: ", paste0(to_delete, collapse = ", "))) for (version in to_delete) { - pin_version_delete(board, name, version) + pin_version_delete(board, name, version, ...) } } else { pins_inform("No old versions to delete") diff --git a/README.Rmd b/README.Rmd index 1c0e747c2..57fd8530f 100644 --- a/README.Rmd +++ b/README.Rmd @@ -24,7 +24,7 @@ knitr::opts_chunk$set( The pins package publishes data, models, and other R objects, making it easy to share them across projects and with your colleagues. -You can pin objects to a variety of pin *boards*, including folders (to share on a networked drive or with services like DropBox), RStudio Connect, Amazon S3, Azure blob storage. +You can pin objects to a variety of pin *boards*, including folders (to share on a networked drive or with services like DropBox), RStudio Connect, Amazon S3, Azure storage and Microsoft 365 (OneDrive and SharePoint). Pins can be automatically versioned, making it straightforward to track changes, re-run analyses on historical data, and undo mistakes. pins 1.0.0 includes a new more explicit API and greater support for versioning. @@ -93,5 +93,5 @@ board %>% pin_read("hadley/sales-summary") You can easily control who gets to access the data using the RStudio Connection permissions pane. -The pins package also includes boards that allow you to share data on services like Amazon's S3 (`board_s3()`) and Azure's blob storage (`board_azure()`). +The pins package also includes boards that allow you to share data on services like Amazon's S3 (`board_s3()`), Azure's blob storage (`board_azure()`), and Microsoft SharePoint (`board_ms365()`). Learn more in `vignette("pins")`. diff --git a/README.md b/README.md index 0d69a4d49..18c29c8c7 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,9 @@ The pins package publishes data, models, and other R objects, making it easy to share them across projects and with your colleagues. You can pin objects to a variety of pin *boards*, including folders (to share on a networked drive or with services like DropBox), RStudio Connect, Amazon -S3, Azure blob storage. Pins can be automatically versioned, making it -straightforward to track changes, re-run analyses on historical data, -and undo mistakes. +S3, Azure storage and Microsoft 365 (OneDrive and SharePoint). Pins can +be automatically versioned, making it straightforward to track changes, +re-run analyses on historical data, and undo mistakes. pins 1.0.0 includes a new more explicit API and greater support for versioning. The legacy API (`pin()`, `pin_get()`, and @@ -58,7 +58,7 @@ library(pins) board <- board_temp() board #> Pin board -#> Path: '/tmp/RtmpSqglFq/pins-c3fc3a89fa55' +#> Path: '/tmp/RtmpxQu94x/pins-15f1d4a0c4d71' #> Cache size: 0 ``` @@ -68,8 +68,7 @@ arguments: the board to pin to, an object, and a name: ``` r board %>% pin_write(head(mtcars), "mtcars") #> Guessing `type = 'rds'` -#> Creating new version '20210928T182507Z-f8797' -#> Writing to pin 'mtcars' +#> Creating new version '20210929T184444Z-f8797' ``` As you can see, the data saved as an `.rds` by default, but depending on @@ -115,5 +114,6 @@ You can easily control who gets to access the data using the RStudio Connection permissions pane. The pins package also includes boards that allow you to share data on -services like Amazon’s S3 (`board_s3()`) and Azure’s blob storage -(`board_azure()`). Learn more in `vignette("pins")`. +services like Amazon’s S3 (`board_s3()`), Azure’s blob storage +(`board_azure()`), and Microsoft SharePoint (`board_ms365()`). Learn +more in `vignette("pins")`. diff --git a/_pkgdown.yml b/_pkgdown.yml index 64291fc60..710f9c9fc 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -23,11 +23,12 @@ reference: - title: Boards desc: > Boards abstract over different storage backends, making it easy to - share data in a vareity of ways. + share data in a variety of ways. contents: - board_azure - board_local - board_kaggle_dataset + - board_ms365 - board_rsconnect - board_s3 - board_url diff --git a/man/board_ms365.Rd b/man/board_ms365.Rd new file mode 100644 index 000000000..7f26526bd --- /dev/null +++ b/man/board_ms365.Rd @@ -0,0 +1,74 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/board_ms365.R +\name{board_ms365} +\alias{board_ms365} +\title{Use a OneDrive or Sharepoint document library as a board} +\usage{ +board_ms365( + drive, + path, + versioned = TRUE, + cache = NULL, + delete_by_item = FALSE +) +} +\arguments{ +\item{drive}{A OneDrive or SharePoint document library object, of class +\code{\link[Microsoft365R:ms_drive]{Microsoft365R::ms_drive}}.} + +\item{path}{Path to directory to store pins. This can be either a string +containing the pathname like \code{"path/to/board"}, or a +\code{\link[Microsoft365R:ms_drive_item]{Microsoft365R::ms_drive_item}} object pointing to the board path.} + +\item{versioned}{Should this board be registered with support for versions?} + +\item{cache}{Cache path. Every board requires a local cache to avoid +downloading files multiple times. The default stores in a standard +cache location for your operating system, but you can override if needed.} + +\item{delete_by_item}{Whether to handle folder deletions on an item-by-item +basis, rather than deleting the entire folder at once. You may need to set +this to \code{TRUE} for a board in SharePoint Online or OneDrive for Business, +due to document protection policies that prohibit deleting non-empty +folders.} +} +\description{ +Pin data to a folder in Onedrive or a SharePoint Online document library +using the Microsoft365R package. +} +\details{ +Sharing a board in OneDrive (personal or business) is a bit complicated, as OneDrive normally allows only the person who owns the drive to access files and folders. First, the drive owner has to set the board folder as shared with other users, using either the OneDrive web interface or Microsoft365R's \code{ms_drive_item$create_share_link()} method. The other users then call \code{board_ms365} with a \emph{drive item object} in the \code{path} argument, pointing to the shared folder. See the examples below. + +Sharing a board in SharePoint Online is much more straightforward, assuming all users have access to the document library: in this case, everyone can use the same call \code{board_ms365(doclib, "path/to/board")}. If you want to share a board with users outside your team, follow the same steps for sharing a board in OneDrive. +} +\examples{ +\dontrun{ +# A board in your personal OneDrive +od <- Microsoft365R::get_personal_onedrive() +board <- board_ms365(od, "myboard") +board \%>\% pin_write(iris) + +# A board in OneDrive for Business +odb <- Microsoft365R::get_business_onedrive(tenant = "mytenant") +board <- board_ms365(odb, "myproject/board") + +# A board in a SharePoint Online document library +sp <- Microsoft365R::get_sharepoint_site("my site", tenant = "mytenant") +doclib <- sp$get_drive() +board <- board_ms365(doclib, "general/project1/board") + + +## Sharing a board in OneDrive: +# First, create the board on the drive owner's side +board <- board_ms365(od, "myboard") + +# Next, let other users write to the folder +# - set the expiry to NULL if you want the folder to be permanently available +od$get_item("myboard")$create_share_link("edit", expiry="30 days") + +# On the recipient's side: find the shared folder item, then pass it to board_ms365 +shared_items <- od$list_shared_items() +board_folder <- shared_items$remoteItem[[which(shared_items$name == "myboard")]] +board <- board_ms365(od, board_folder) +} +} diff --git a/man/board_rsconnect.Rd b/man/board_rsconnect.Rd index 417fe228c..97659c10f 100644 --- a/man/board_rsconnect.Rd +++ b/man/board_rsconnect.Rd @@ -71,13 +71,13 @@ After you've shared the pin, it will be automatically available to others. } \section{Public pins}{ If your RSC instance allows it, you can share a pin publicly by setting the -access type to \code{all}:\if{html}{\out{
}}\preformatted{board \%>\% pin_write(my_df, access_type = "all") +access type to \code{all}:\if{html}{\out{
}}\preformatted{board \%>\% pin_write(my_df, access_type = "all") }\if{html}{\out{
}} (You can also do this in RSC by setting "Access" to "Anyone - no login required") -Now anyone can read your pin through \code{\link[=board_url]{board_url()}}:\if{html}{\out{
}}\preformatted{board <- board_url(c( +Now anyone can read your pin through \code{\link[=board_url]{board_url()}}:\if{html}{\out{
}}\preformatted{board <- board_url(c( my_df = "https://connect.rstudioservices.com/content/3004/" )) board \%>\% pin_read("my_df") diff --git a/man/pin_versions.Rd b/man/pin_versions.Rd index a0f5e8b6e..40a489c04 100644 --- a/man/pin_versions.Rd +++ b/man/pin_versions.Rd @@ -10,7 +10,7 @@ pin_versions(board, name, ..., full = deprecated()) pin_version_delete(board, name, version, ...) -pin_versions_prune(board, name, n = NULL, days = NULL) +pin_versions_prune(board, name, n = NULL, days = NULL, ...) } \arguments{ \item{board, name}{A pair of board and pin name. For modern boards, @@ -20,7 +20,7 @@ legacy API, you can also use \code{pin_versions(name)} or \item{...}{Additional arguments passed on to methods for a specific board.} -\item{full}{\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#deprecated}{\figure{lifecycle-deprecated.svg}{options: alt='[Deprecated]'}}}{\strong{[Deprecated]}}} +\item{full}{\ifelse{html}{\figure{lifecycle-deprecated.svg}{options: alt='Deprecated lifecycle'}}{\strong{Deprecated}}} \item{version}{Version identifier.} diff --git a/tests/testthat/test-board_ms365.R b/tests/testthat/test-board_ms365.R new file mode 100644 index 000000000..876bcec9a --- /dev/null +++ b/tests/testthat/test-board_ms365.R @@ -0,0 +1,9 @@ +board <- board_ms365_test_charpath(delete_by_item = TRUE) +test_api_basic(board) +test_api_versioning(board) +test_api_meta(board) + +board2 <- board_ms365_test_driveitem(delete_by_item = TRUE) +test_api_basic(board2) +test_api_versioning(board2) +test_api_meta(board2)