forked from Sage-Bionetworks/data_curator
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #115 from nf-osi/develop
Redo without squash merge.
- Loading branch information
Showing
36 changed files
with
983 additions
and
1,425 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| # During the github workflow to auto deploy the app | ||
| # This config file will be used to overwrite the config.yml in the schematic folder | ||
| # | ||
| # Please modify the configuration values based on your project | ||
|
|
||
| # Do not change the 'definitions' section unless you know what you're doing | ||
| definitions: | ||
| synapse_config: '.synapseConfig' | ||
| creds_path: 'credentials.json' | ||
| token_pickle: 'token.pickle' | ||
| service_acct_creds: 'schematic_service_account_creds.json' | ||
|
|
||
| synapse: | ||
| master_fileview: 'syn16858331' # fileview of project with datasets on Synapse | ||
| manifest_folder: 'manifests' # manifests will be downloaded to this folder | ||
| manifest_filename: 'synapse_storage_manifest.csv' # name of the manifest file in the project dataset | ||
| token_creds: 'syn21088684' # synapse ID of credentials.json file | ||
| service_acct_creds: 'syn25171627' # synapse ID of service_account_creds.json file | ||
|
|
||
| manifest: | ||
| title: 'NF' | ||
| data_type: | ||
| - 'NF' | ||
|
|
||
| model: | ||
| input: | ||
| download_url: 'https://raw.githubusercontent.com/nf-osi/nf-metadata-dictionary/v3.0.2/NF.jsonld' # url to download JSON-LD data model | ||
| location: 'data-models/NF.jsonld' # path to JSON-LD data model | ||
| file_type: 'local' # only type "local" is supported currently | ||
|
|
||
| style: | ||
| google_manifest: | ||
| req_bg_color: | ||
| red: 0.9215 | ||
| green: 0.9725 | ||
| blue: 0.9803 | ||
| opt_bg_color: | ||
| red: 1.0 | ||
| green: 1.0 | ||
| blue: 0.9019 | ||
| master_template_id: '1LYS5qE4nV9jzcYw5sXwCza25slDfRA1CIg3cs-hCdpU' | ||
| strict_validation: false |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| # require github cli package - 'gh': https://cli.github.com/manual/ | ||
|
|
||
| args <- commandArgs(trailingOnly = TRUE) | ||
|
|
||
| repo <- basename(Sys.getenv("PWD")) | ||
|
|
||
| if (repo %in% c("schematic", "data-models")) stop("You are not in the data curator folder !!!") | ||
|
|
||
| secret_lists <- yaml::yaml.load_file(toString(args[1]))$definitions | ||
| message("Assuming all definition files are stored in the schematic folder ...") | ||
|
|
||
| for (f in c("synapse_config", "creds_path", "service_acct_creds")) { | ||
| system( | ||
| sprintf( | ||
| "gh secret set %s < %s", | ||
| toupper(paste0("schematic_", f)), file.path("schematic", secret_lists[f]) | ||
| ) | ||
| ) | ||
| } | ||
|
|
||
| # token.pickle is a binary file | ||
| # If you want to manually add to the secret: | ||
| # 1. copy below code snippet and run it in the terminal | ||
| # 2. copy content from 'token.pickle.b64' to your github secret | ||
| system( | ||
| " | ||
| less schematic/token.pickle | base64 --wrap=0 > schematic/token.pickle.b64; | ||
| gh secret set SCHEMATIC_TOKEN_PICKLE < schematic/token.pickle.b64; | ||
| " | ||
| ) | ||
| unlink("schematic/token.pickle.b64") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,108 @@ | ||
| # Workflow derived from https://github.com/r-lib/actions/tree/master/examples | ||
| # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help | ||
|
|
||
| name: shiny-deploy | ||
|
|
||
| # TODO: remove `shiny-server-develop` when we use shinyapps.io as production site | ||
| # also consider to change shiny-server-main to main since we will not host in shinyServer | ||
| on: | ||
| push: | ||
| branches: | ||
| - develop # add develop for testing for now | ||
| tags: | ||
| - release* | ||
| paths-ignore: | ||
| - .github/ISSUE_TEMPLATE/** | ||
|
|
||
| jobs: | ||
| shiny-deploy: | ||
| runs-on: ubuntu-latest | ||
| # This image seems to be based on rocker/r-ver which in turn is based on debian | ||
| container: rocker/rstudio | ||
| env: | ||
| # This should not be necessary for installing from public repo's however remotes::install_github() fails without it. | ||
| GITHUB_PAT: ${{ secrets.REPO_PAT }} | ||
|
|
||
| steps: | ||
| - name: Install System Dependencies | ||
| run: | | ||
| sudo apt-get update | ||
| sudo apt-get install -y pip python3.8-venv libcurl4-openssl-dev | ||
| - uses: actions/checkout@v2 | ||
|
|
||
| - uses: r-lib/actions/setup-pandoc@v1 | ||
|
|
||
| - name: Create and Activate Python Virtual Environment | ||
| shell: bash | ||
| run: | | ||
| python3 -m venv .venv | ||
| chmod 755 .venv/bin/activate | ||
| source .venv/bin/activate | ||
| - name: Install R Packages Dependencies | ||
| run: | | ||
| R -f install-pkgs.R | ||
| - name: Install Python Packages Dependencies | ||
| shell: bash | ||
| run: | | ||
| source .venv/bin/activate | ||
| pip3 install -r requirements.txt | ||
| - name: Install Schematic | ||
| shell: bash | ||
| run: | | ||
| source .venv/bin/activate | ||
| # use 'poetry' to install schematic from the develop branch | ||
| pip3 install poetry | ||
| git clone --single-branch --branch develop https://github.com/Sage-Bionetworks/schematic.git | ||
| cd schematic | ||
| poetry build | ||
| pip3 install dist/schematicpy-1.0.0-py3-none-any.whl | ||
| - name: Set Configurations for Schematic | ||
| shell: bash | ||
| run: | | ||
| source .venv/bin/activate | ||
| # download data model to the correct location | ||
| R -e ' | ||
| config <- yaml::yaml.load_file(".github/schematic_config.yml"); | ||
| url <- config$model$input$download_url; | ||
| path <- config$model$input$location; | ||
| system(sprintf("mkdir -p %s", dirname(path))); | ||
| system(sprintf("wget %s -O %s", url, path)); | ||
| ' | ||
| # overwrite the config.yml in schematic | ||
| mv -f .github/schematic_config.yml schematic/config.yml | ||
| # write out configuration files using github secrets | ||
| echo "${{ secrets.SCHEMATIC_SYNAPSE_CONFIG }}" > schematic/.synapseConfig | ||
| echo "${{ secrets.SCHEMATIC_SERVICE_ACCT_CREDS }}" > schematic/schematic_service_account_creds.json | ||
| echo "${{ secrets.SCHEMATIC_CREDS_PATH }}" > schematic/credentials.json | ||
| echo "${{ secrets.SCHEMATIC_TOKEN_PICKLE }}" | base64 -d > schematic/token.pickle | ||
| - name: zip virtual env | ||
| shell: bash | ||
| # ShinyApps has a limit of 7000 files, far exceeded by the many Python dependencies | ||
| # that this app' has. As a workaround we zip the virtual environment and later | ||
| # unzip it in 'global.R' | ||
| run: | | ||
| zip -rm .venv.zip .venv | ||
| - name: Authorize and deploy app | ||
| shell: Rscript {0} | ||
| run: | | ||
| branch <- Sys.getenv("GITHUB_REF_NAME") | ||
| repo <- Sys.getenv("GITHUB_REPOSITORY") | ||
| appName <- strsplit(repo, "/")[[1]][2] | ||
| if (!startsWith(branch, "release")) { | ||
| appName <- paste(appName, "staging", sep = "-") | ||
| } | ||
| rsConnectUser <- "${{ secrets.RSCONNECT_USER }}" | ||
| rsConnectToken <- "${{ secrets.RSCONNECT_TOKEN }}" | ||
| rsConnectSecret <- "${{ secrets.RSCONNECT_SECRET }}" | ||
| # create config file | ||
| config <- "CLIENT_ID: ${{ secrets.OAUTH_CLIENT_ID }}" | ||
| config <- c(config, "CLIENT_SECRET: ${{ secrets.OAUTH_CLIENT_SECRET }}") | ||
| appUrl <- sprintf("https://%s.shinyapps.io/%s", rsConnectUser, appName) | ||
| config <- c(config, sprintf("APP_URL: %s", appUrl)) | ||
| configFileConn <- file("oauth_config.yml") | ||
| tryCatch( | ||
| writeLines(config, configFileConn), | ||
| finally=close(configFileConn) | ||
| ) | ||
| rsconnect::setAccountInfo(rsConnectUser, rsConnectToken, rsConnectSecret) | ||
| rsconnect::deployApp(appName = appName) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.