' + + '' + + _("Hide Search Matches") + + "
" + ) + ); + }, + + /** + * helper function to hide the search marks again + */ + hideSearchWords: () => { + document + .querySelectorAll("#searchbox .highlight-link") + .forEach((el) => el.remove()); + document + .querySelectorAll("span.highlighted") + .forEach((el) => el.classList.remove("highlighted")); + localStorage.removeItem("sphinx_highlight_terms") + }, + + initEscapeListener: () => { + // only install a listener if it is really needed + if (!DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS) return; + + document.addEventListener("keydown", (event) => { + // bail for input elements + if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) return; + // bail with special keys + if (event.shiftKey || event.altKey || event.ctrlKey || event.metaKey) return; + if (DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS && (event.key === "Escape")) { + SphinxHighlight.hideSearchWords(); + event.preventDefault(); + } + }); + }, +}; + +_ready(() => { + /* Do not call highlightSearchWords() when we are on the search page. + * It will highlight words from the *previous* search query. + */ + if (typeof Search === "undefined") SphinxHighlight.highlightSearchWords(); + SphinxHighlight.initEscapeListener(); +}); diff --git a/_static/workbench-v2-logo.svg b/_static/workbench-v2-logo.svg new file mode 100644 index 00000000..fb4dd716 --- /dev/null +++ b/_static/workbench-v2-logo.svg @@ -0,0 +1,5 @@ + diff --git a/genindex.html b/genindex.html new file mode 100644 index 00000000..b33d3d06 --- /dev/null +++ b/genindex.html @@ -0,0 +1,120 @@ + + + + + +The following third-party dependencies are required. These can optionally be installed by our Helm chart if they are not alrady provided on your cluster:
+The following third-party dependencies are optional, but recommended:
+CertManager: for automating renewal of Wildcard TLS certificates
The following steps can be used to deploy Workbench to any cluster where you have kubeconfig access.
+For local development, we offer a Makefile to help get everything up and running in the proper order
+For production deployments, we recommend setting up your own custom Helm Configuration values
+To contribute back to the Workbench platform, you can run a local development environment in a few quick steps
+Prerequisites:
+make CLI
Git CLI
Make sure no other application is running on port 80/443
Setup Steps:
+This will run a small Kubernetes cluster on your local machine
git clone https://github.com/nds-org/workbench-helm-chart && cd workbench-helm-chart/
KUBE_CONTEXT=docker-desktop the Kubernetes kubeconfig context to use. This should match the name of a context from kubectl config get-contexts file. See Kubernetes Documentation for more details
NAMESPACE=workbench the Kubernetes namespace where Workbench will run
NAME=workbench the name of the Helm release that will run Workbench
Verify that all dependencies are installed correctly
Update/fetch Helm dependency charts
Clone the webui/apiserver source code locally
Install frontend dependencies and use them to compile the source code
(if REALM_IMPORT=true) Create a configmap called keycloak-realm - this will automatically create a Keycloak realm named workbench-dev with all necessary groups and mappings
Deploy the Workbench Helm chart (with live-reload) with dependencies
Pull and run all required Docker images and bring the platform online
Wait until all Volumes are Bound and all Pods are Running
You should see the Workbench Landing page load in your browser
Click the Login button to go to the Keycloak login page
Click the Register button at the bottom and create a Test User
After registraton, you should now be logged into the Workbench WebUI
+If you’re modifying the Workbench API, it can be helpful to test your changes using Swagger UI +This lets you test the API’s raw request/reponse behavior without involving or changing the WebUI
+To see Swagger UI, go to https://kubernetes.docker.internal/swagger +To use the authenticated endpoints (padlock icon), you can use the POST /authenticate endpoint
+This endpoint takes your Test User username/password combination, and returns a token when successful +After authenticating, this token will be automatically added to all requests from Swagger UI
+You can test this by executing the /me endpoint, which should now return information about the currently logged-in user
+Copy values.yaml to create a new file named values.myworkbench.yaml
Disable any sub-charts that you don’t want to deploy with Workbench (e.g. NFS server, Keycloak, etc)
Edit ingress.api.annotations and ingress.webui.annotations to modify the Ingress annotations for the API / WebUI respectively
Edit config.frontend.customization section to customize the appearance/text of the WebUI
Edit ingress.hostname and config.backend.domain with your desired domain name
Edit config.frontend.signin_url, config.frontend.signout_url, and config.backend.oauth.userinfoUrl
Edit config.backend.namespace and set this to the namespace where your UserApps should launch
Edit config.backend.mongo section and update/match MongoDB credentials throughout
Edit config.backend.keycloak section to point at a third-party Keycloak instance
Edit config.backend.userapps.shared_storage if you want to enable one or more Shared Data Volume
Edit config.backend.userapps.annotations to modify the Ingress annotations for UserApps
Edit mongodb.autoimport section to change which github repo is loaded during the import-specs job
Locate kubernetes.docker.internal throughout and replace with your own domain
Change your kubeconfig context to your desired cluster context: kubectl config use-context <name>
By default, namespace=workbench and name=workbench - you can use any combination here to deploy multiple different Workbench instances to the same cluster
For a full reference of Helm chart configuration values, see https://github.com/nds-org/workbench-helm-chart
+If you need to change the configuration of Keycloak, go to https://kubernetes.docker.internal/auth/ +Default credentials: admin / workbench
+After logging in, choose the workbench-dev realm from the dropdown at the top-left
+ +By default, Keycloak only allows username/password auth. By configuring OIDC, you can allow your users to login to the platform with existing credentials from another system (e.g. GitHub, Google, Azure, etc)
+First, create a new OAuth/OIDC Application within the chosen provider(s) (NOTE: this terminology may vary between providers).
+For example, with CILogon users can fill out https://cilogon.org/oauth2/register to receive these values
+Here, the callback URL will be https://<KEYCLOAKDOMAIN>/auth/realms/<REALM>/broker/<CLIENT>/endpoint
+KEYCLOAKDOMAIN = Keycloak instance domain name
REALM = Keycloak realm name
CLIENT = Keycloak client name (you will create this below)
This should provide you with a ClientID + ClientSecret to use.
+In Keycloak, create a new Client for each provider and specify your ClientID + ClientSecret when requested, and also set:
+First Login Flow = browser
Valid Redirect URLs = https://<APPDOMAIN>/oauth2/callback
Default Scopes = openid profile + any other scopes desired
Set Authorization URL / Token URL / User Info URL / etc according to your chosen provider
For a more detailed example of configuring OAuth2 Proxy authentiating via Keycloak, see an example application
+workbench-users - allows user to login to Workbench (otherwise 403 is returned)
workbench-developers - allows user access to create, modify, and launch custom UserApps
workbench-admin - not currently used. (future: allow user to access admin-only API/UI functions)
If you already have a running Keycloak instance that you would like to use, make sure you configure Mappers needed for OIDC via OAuth2 Proxy
+For more information, see the OAuth2 Proxy documentation
+Configure a dedicated audience mapper for your client by navigating to Clients -> <your client’s id> -> Client scopes.
+Access the dedicated mappers pane by clicking <your client’s id>-dedicated, located under Assigned client scope.
Name ‘aud-mapper-<your client’s id>’
OAuth2 proxy can be set up to pass both the access and ID JWT tokens to your upstream services. If you require additional audience entries, you can use the Included Custom Audience field in addition to the “Included Client Audience” dropdown. Note that the “aud” claim of a JWT token should be limited and only specify its intended recipients.
Add to ID token ‘On’
Save the configuration.
Any subsequent dedicated client mappers can be defined by clicking Dedicated scopes -> Add mapper -> By configuration -> Select mapper
To summarize, the steps required to authorize Keycloak group membership with OAuth2 Proxy are as follows:
+Include a mapper of type Group Membership.
Set the “Token Claim Name” to groups or customize by matching it to the –oidc-groups-claim option of OAuth2 Proxy.
If the “Full group path” option is selected, you need to include a “/” separator in the group names defined in the –allowed-group option of OAuth2 Proxy. Example: “/groupname” or “/groupname/childgroup”.
After creating the Client Scope named groups you will need to attach it to your client. +Clients -> <your client’s id> -> Client scopes -> Add client scope -> Select groups and choose Optional and you should now have a client that maps group memberships into the JWT tokens so that Oauth2 Proxy may evaluate them.
+Contributors and Developers in the system have access to some special functionality.
+To access the following features, ask your administrator to add you to the workbench-developers group in Keycloak.
+Contributors have the ability to “Clone” a spec from the All Apps view
+This will create a copy of the app on the “My Catalog” so that it can be edited or extended
+ +Contributors have the ability to “Edit” envvars for an App instance in the “My Apps” view
+This allows you to set/override custom Environment Variables before launching the app
+ +Most importantly, only developers have access to a custom “My Catalog” view.
+This allows contributors to see their created/cloned applications and edit the spec directly.
+After editing the spec, the App will be displayed in your All Apps view, but will not yet be shown to other users.
+This allows you to create, modify, and test custom applications
+Export JSON spec from Catalog
Clone App to Catalog
Edit App in Catalog
Delete App from Catalog
Import JSON spec to Catalog
Create / Edit App in Catalog +* Key (required) / Label (optional) +* Display: stack / service / none +* Access : external / internal / none +* Docker Image Metadata
++++
+- +
Single repo / image name
- +
List of offered tags
- +
Command / Args
Logo / info URLs
Dependencies
Defaults / Overrides
Passwords
Protocol - TCP / UDP / HTTP(S)
Port Number
Path (HTTP only)
From Path (Host / PVC)
To Path (inside Container)
All Workbench users have the ability to install and launch UserApps
+ +A typical user of the app would likely follow these steps:
+Login to Workbench
Go to “All Apps” to view all AppSpecs offered by the system
Find an AppSpec to install, then click the + button
On the “My Apps” view, your chosen UserApp should now appear
Click the Launch button on your UserApp to start it up
Wait for the status to turn green
Expand the running app - you should see a link appear in the expanded section
Click the link to navigate to your launched UserApp
When you’re finished using the app, click the Shutdown button
Wait for the app to finishing shutting down
Click the Delete button to remove the UserApp from your list
Depending on how your administrator has configured Workbench, there are two ways to log into the platform: username/password or OIDC
+Typically we recommend configuring some form of OIDC for production instances, but this is not always appropriate.
+User navigates to WebUI and clicks the Login button
User is sent to the OAuth2 Proxy’s /oauth2/start endpoint
/oauth2/start begins the OAuth chain, which starts by automatically redirecting the User to Keycloak
User enters their username / password into the form and clicks “Sign In” at the bottom
After login, User is automatically redirected back to the Workbench WebUI with a valid auth token set as a cookie.
+This cookie is automatically sent to the Workbench API server with every subsequent request
+User navigates to WebUI and clicks the login button
User is sent to the OAuth2 Proxy’s /oauth2/start endpoint
If ONLY OIDC is enabled (no username/password), you will be redirected to the provider’s login page
User logs in with provider credentials
After login, User is automatically redirected back to the Workbench WebUI with a valid auth token set as a cookie.
+This cookie is automatically sent to the Workbench API server with every subsequent request
+This view shows the entire catalog of our available AppSpecs
+With this view, we can:
+Search / Filter to find applications that we can create
Create a New Application Instance - this will create a new item under “My Apps”
With this view, we can:
+Launch / Shutdown an Application Instance (App)
Go to a running application
Access the Console of Running App
The growing size and complexity of high-value scientific datasets are pushing +the boundaries of traditional models of data access and discovery. Many large +datasets are only accessible through the systems on which they were created or +require specialized software or computational resources.
+Labs Workbench is a scalable, web-based platform intended to support turn-key +deployment of encapsulated data management and analysis tools to support +exploratory analysis and development near research datasets.
+The following collections are stored in MongoDB:
+AppSpec: Contains all the information needed to create a new UserApp
UserApp: A launchable instance of an AppSpec that is owned by a particular user
Using the Google Cloud UI or gcloud, create a Kubernetes cluster:
+gcloud container clusters create [CLUSTER_NAME] [--zone [COMPUTE_ZONE]]
+
Setup a ClusterRoleBinding for your account:
+gcloud info | grep Account
+Account: [myname@example.org]
+
+kubectl create clusterrolebinding myname-cluster-admin-binding --clusterrole=cluster-admin \
+ --user=myname@example.org
+
On the Cloud Console or your local system, install and configure Helm:
+curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash
+kubectl --namespace kube-system create serviceaccount tiller
+kubectl create clusterrolebinding tiller --clusterrole cluster-admin \
+ --serviceaccount=kube-system:tiller
+helm init --service-account tiller --wait
+
Install the NGINX Ingress Controller:
+helm install --name nginx-ingress stable/nginx-ingress --set rbac.create=true
+
Clone the Workbench chart:
+git clone https:/github.com/nds-org/workbench-helm-chart -b gke
+
Create an in-cluster NFS server to support ReadWriteMany volume claims:
+cd nfs-example
+kubectl create -f deployment.yaml -f rbac.yaml -f class.yaml
+
Configure Workbench values.yaml:
+Set domain to a wildcard DNS entry. Contact us if you need something (e.g., project.ndslabs.org)
Set support_email to a valid Gmail account
Set smtp.gmail_user and smtp.gmail_password to your Gmail account and an app password
Create a self-signed cert using generate-self-signed-cert.sh <domain> and paste into tls.cert and tls.key sections.
Install Workbench chart:
+helm install . --name=workbench --namespace=workbench
+
Once the pods are running, you should be able to access your instance at +https://www.project.ndslabs.org). Click “Sign Up” to register an account. With +the default chart settings, you will be required to verify your email address +and the support_email account will receive a message to approve/deny +registration. After approval, login to your account and launch the File +Manager. Confirm that you can access the File Manager UI.
+Labs Workbench uses the Helm package manager for +Kubernetes. This section describes how to setup Helm on your cluster.
+There a number of ways to install Helm. A simple approach is to +run the Helm installer:
+curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash
+
This will install the latest release of Helm.
+After installing Helm on your system, you must initialize it in your Kubernetes +cluster:
+First, setup a ServiceAccount for use by the tiller:
+kubectl --namespace kube-system create serviceaccount tiller
+
Next, give the ServiceAccount permissions to manage the cluster:
+kubectl create clusterrolebinding tiller --clusterrole cluster-admin \
+ --serviceaccount=kube-system:tiller
+
Finally, initialize Helm in your cluster:
+helm init --service-account tiller --wait
+
This command creates a tiller deployment in the kube-system namespace and +configures your local helm client.
+For more information, see the Helm +<https://docs.helm.sh/using_helm/#installing-helm>_ documentation.
+Labs Workbench uses the Helm package manager for +Kubernetes. This section describes how to install Workbench and +prerequisites on your cluster.
+Labs Workbench has a few prerequisites:
+Kubernetes cluster (can be Minikube or kubeadm)
Helm
ReadWriteMany storage provisioner for home directories
NGINX ingress load balancer
Wildcard DNS and TLS certificate
Labs Workbench uses two types of PersistentVolumeClaims:
+ReadWriteOnce claim for etcd store
ReadWriteMany claim for user volumes
Storage options available to you depend on your cloud provider and Kubernetes +configuration. Since some cloud providers do not support ReadWriteMany storage +options by default, we recommend using a simple NFS provisioner.
+Note, on GKE you’ll need to create a ClusterRoleBinding for your account:
+gcloud info | grep Account
+Account: [myname@example.org]
+
+kubectl create clusterrolebinding you-cluster-admin-binding --clusterrole=cluster-admin \
+ --user=you@gmail.com
+
An example is NFS provider is available: +.. code:: bash
+++git clone https:/github.com/nds-org/workbench-helm-chart +cd nfs-example +kubectl create -f deployment.yaml -f rbac.yaml -f class.yaml
+
This creates an nfs storage class backed by an nfs-provisioner. The +provisioner itself uses a ReadWriteOnce volume from the cloud provider’s +default storage class.
+Labs Workbench uses the NGINX Ingress Controller in both OpenStack and GKE +deployments. To install it:
+helm install --name nginx-ingress stable/nginx-ingress --set rbac.create=true
+
Now we’re ready to install Labs Workbench:
+git clone https://github.com/nds-org/workbench-helm-chart
+cd workbench-helm-chart/
+
You’ll need to edit a few of the settings in values.yaml:
+domain: Base domain for wildcard DNS (e.g., workbench.ndslabs.org)
email_address: Email address used as from address in outgoing emails
Wildcard TLS certificate: Self-signed or valid wildcard certificate for domain
smtp: In-cluster SMTP server or Gmail username/app password
And deploy the chart: +.. code:
+helm install . --name=workbench --namespace=workbench --wait
+
If you change your configuration, you can upgrade the chart: +.. code:
+helm upgrade workbench .
+
The following diagram illustrates the approximate architecture of the Workbench application:
+ +Swagger UI
Redux Store
Websocket use for Console Exec
KubeWatcher
Websocket forwarding for Console Exec
An application specification (AppSpec) contains all of the information necessary to run a containerized application
+The only required pieces are as follows: +* key` is simply unique identifier for this application. This field may only contain alphanumeric characters (no special characters) +* Docker image name/tags of the application
+The information in the AppSpec is used to create a user-specific instance of an app (UserApp)
+Creating a UserApp creates the following resources on the Kubernetes cluster:
+ConfigMap (future: secret?) - environment variables for the application to use at runtime
Ingress (optional) - allows users on the public internet to reach this application
Service (optional) - allows other applications in the cluster to reach this application
Mount: Home Data Volume PVC - each User has a Home volume that will be mounted in the /home/ folder
Mount: Shared Data Volume PVC (optional) - if configured, Workbench will mount one or more shared data volumes
The auth flow presented here is fairly standard, and works with any app or apps that can make an HTTP request with credentials / cookies.
+User navigates to WebUI and clicks the Login button
User is sent to the OAuth2 Proxy’s /oauth2/start endpoint
If local accounts are enabled, you should see a Username/Password field
If one or more OIDC providers are configured, you should also see a button allowing you to login with those providers (e.g. GitHub, Google, CILogon, etc)
If ONLY OIDC is enabled, you will be redirected to the provider’s login page
(optional) if using OIDC, User logs in with provider credentials
After login, User is automatically redirected back to Keycloak’s callback URL
Keycloak sets a few cookies in the User’s browser before automatically redirects the user back to OAuth2 Proxy’s callback URL
OAuth2 Proxy sets an _oauth2_proxy cookie in the User’s browser containing an authorization token
OAuth2 Proxy automatically redirects the User back to the Workbench WebUI
Finally, the User arrives back at the Workbench WebUI with a valid auth token set as a cookie. This cookie is sent to the Workbench API server with every subsequent request
+Anytime the WebUI or API server needs to decode this token, they can make a request to the /oauth2/userinfo endpoint that includes the token cookie.
+The server will respond with a decoded token as a JSON document
+The decoded token includes the following information: +* email +* username +* group membership
+TODO: add more details here
+The following collections are stored in MongoDB:
+AppSpec: Contains all the information needed to create a new UserApp
UserApp: A launchable instance of an AppSpec that is owned by a particular user
Workbench has been used in various different workshops, classes, and environments to provide quick access to runnable app instances with access to real project data for further analysis
+Name |
+Type |
+Description |
+Tools |
+Status |
+
---|---|---|---|---|
IASSIST 2016 Odum Workshop |
+Educational Support |
+Odum Institute workshop at IASSIST to provide hands-on experience integrating Dataverse with iRODS. |
+Dataverse, iRODS Cloudbrowser, iRODS iCAT PostgreSQL, TwoRavens |
+Workshop held on May 31, 2016, 27 registered/active users |
+
NDSC6 Hack Day |
+Educational Support |
+NDS Docker tutorial |
+All Tools |
+Workshop held on October 19, 2016, 20 active users |
+
Phenome 2017 |
+Educational Support |
+TERRA-REF workshop demonstrating use of custom Jupyter/Rstudio environments to analyze data products. |
+Cloud9 Python environment for pyClowder, Docker, File Manager, Hyperspectral/NCO, Jupyter/NetCDF4, Jupyter/PlantCV, JupyterLab for pyClowder, RStudio Geospatial, RStudio Server |
+Workshop held on February 10, 2017, 53 registered users 23 active users |
+
University of Washington iSchool Pilot |
+Educational Support |
+Application of Labs Workbench as sandbox for iSchool Data Curation curriculum allowing students to launch private instances of data management services such as DSpace, Dataverse, etc. |
+CKAN, Dataverse, FIDO, SQL Workbench, Oxygen, EIS Archiver, … |
+Spring 2017, INF598H - Advanced Data Curation |
+
EarthCube |
+Data Analysis Tools |
+Serve as a resource to increase exposure and allow others to utilize EarthCube Building Blocks software/services. |
+CINERGI, SuAVE, GeoDeepDive, OntoSoft, GeoWS, … |
+EarthCube 2017 All Hands Meeting, June 7 - June 9 2017 |
+
PI4 Bootcamp |
+Data Analysis Tools |
+Used for two-week course on the use and analysis of large and complex data based on the TERRA-REF dataset. |
+Jupyter, NetCDF4, PlantCV, OpenRefine, RStudio |
+Ran May 26 - June 9, 2017, 30 students participating |
+
2017 American Einstein Toolkit School + Workshop |
+Educational Support |
+Used initially for 2-day workshop and evolved into NDS pilot project supporting ongoing access |
+Jupyter/Einstein Toolkit |
+Ran August 3 - 4 2017, 30 participants. Pilot service used by 40+ users |
+
ThinkChicago 2017 |
+Educational Support |
+Three day hackathon exploring novel tools addressing city of chicago data centric challenges. |
+Cloud9, Jupyter, RStudio, PHP, MongoDB, Postgres, … |
+Ran August 2 - August 4, 2017, 200 participants |
+
DataDrivenAg 2017 |
+Educational Support |
+Two-day hackathon as part of workshop on big-data approaches to agricultural research in cooperation with TERRA-REF |
+Cloud9, Jupyter, RStudio, Globus Personal Connect, PostgresSQL Studio, and Xpra/OpenBox for desktop apps. Also includes fileserver with over 2TB of sample data, data made available via Geoserver WMS/WCS/WFS, and BETYdb PostGIS database. |
+February 26 - February 27, ~80 participants |
+
TERRA-REF |
+Data Analysis Tools |
+Labs Workbench platform is being extended to replace current “Tool Launcher” service to enable broader container-based analysis of TERRA-REF data. |
+Jupyter, NetCDF4, PlantCV, OpenRefine, RStudio |
+Project concluded, Workbench still running and accessible |
+
Cyber-Human Ecosystem for Engaged Security Education (CHEESE) |
+Educational Support |
+Workbench instance customized with educational cybersecurity lesson plans |
+Jupyter + custom apps based on lesson plan |
+Active 2019-2024, Workbench still running and accessible |
+
Permafrost Discovery Gateway (PDG) Workbench |
+Data Analysis Tools |
+Platform enabling analysis of permafrost data made available by other parts of the platform using the toolset offered by the Workbench |
+All Tools |
+Active 2022-present, Workbench still running and accessible |
+
Metabolomics Workbench |
+Data Analysis Tools |
+Platform enabling analysis of metabolomic data made available by other parts of the platform using the toolset offered by the Workbench |
+All Tools |
+Active 2021-present, Workbench still running and accessible |
+
AMR Dashboard/Workbench |
+Data Analysis Tools |
+Platform enabling analysis of anti-microbial data made available by other parts of the platform using the toolset offered by the Workbench |
+Jupyter, RStudio, … |
+Pending |
+