Merge pull request #107 from uc-cdis/feat/stata-license

Feat/stata license

.github/workflows/build_push_stata_gen3_licensed.yml

@@ -0,0 +1,69 @@
+name: Push Stata Gen3-licensed Image to quay
+
+on:
+  push:
+    paths:
+      - jupyter-pystata-gen3-licensed/**
+      - .github/workflows/build_push_stata_gen3_licensed.yml
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 30000
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+      - uses: actions/checkout@v2
+      - uses: prewk/[email protected]
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          SOURCE: 's3://ctds-stata/Stata17Linux64.tar.gz'
+          DEST: './jupyter-pystata-gen3-licensed/resources/'
+
+      - name: Extract branch name
+        shell: bash
+        run: echo "branch=$(echo $(echo ${GITHUB_REF#refs/*/} | tr / _))" >> $GITHUB_OUTPUT
+        id: extract_branch
+
+      - name: Determine image to build
+        id: parse_image
+        shell: python
+        run: |
+          import os
+
+          build_target = "jupyter-pystata-licensed"
+          print(f"Will trigger build for: {build_target}")
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+            print(f'build_target={build_target}', file=fh)
+
+      - if: ${{ steps.parse_image.outputs.build_target }}
+        name: Sanitize image name
+        id: sanitize_name
+        run: |
+          IMAGE_NAME=$( sed 's/[^[:alnum:]]/_/g' <<< ${{ steps.parse_image.outputs.build_target }} );
+          echo "image_name=$IMAGE_NAME" >> $GITHUB_OUTPUT
+
+      - name: Build Image
+        id: build-image
+        uses: redhat-actions/buildah-build@v2
+        with:
+          image: jupyter-pystata-gen3-licensed
+          tags: ${{ steps.extract_branch.outputs.branch }}
+          dockerfiles: ./jupyter-pystata-gen3-licensed/Dockerfile
+
+      - name: Push To quay.io
+        id: push-to-quay
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/cdis
+          username: ${{ secrets.QUAY_SERVICE_ACCOUNT_USER }}
+          password: ${{ secrets.QUAY_SERVICE_ACCOUNT_PASSWORD }}

.github/workflows/push_stata_image.yml → .github/workflows/build_push_stata_user_licensed.yml

@@ -1,10 +1,10 @@
-name: Push Stata Image to quay
+name: Push Stata User-licensed Image to quay
 
 on:
   push:
     paths:
-      - jupyter-pystata/**
-      - .github/workflows/push_stata_image.yml
+      - jupyter-pystata-user-licensed/**
+      - .github/workflows/build_push_stata_user_licensed.yml
   workflow_dispatch:
 
 jobs:
@@ -25,20 +25,20 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           SOURCE: 's3://ctds-stata/Stata17Linux64.tar.gz'
-          DEST: './jupyter-pystata/resources/'
+          DEST: './jupyter-pystata-user-licensed/resources/'
 
       - name: Extract branch name
         shell: bash
-        run: echo "branch=$(echo $(echo ${GITHUB_REF#refs/heads/} | tr  / _))" >> $GITHUB_OUTPUT
+        run: echo "::set-output name=branch::$(echo $(echo ${GITHUB_REF#refs/*/} | tr / _))"
         id: extract_branch
 
       - name: Build Image
         id: build-image
         uses: redhat-actions/buildah-build@v2
         with:
-          image: stata-heal
-          tags: ${{ steps.extract_branch.outputs.branch }} ${{ github.sha }}
-          dockerfiles: ./jupyter-pystata/Dockerfile
+          image: jupyter-pystata-user-licensed
+          tags: ${{ steps.extract_branch.outputs.branch }}
+          dockerfiles: ./jupyter-pystata-user-licensed/Dockerfile
 
       - name: Push To quay.io
         id: push-to-quay

jupyter-pystata-gen3-licensed/Dockerfile

@@ -0,0 +1,23 @@
+FROM quay.io/cdis/jupyter-pystata-user-licensed:feat_stata-license
+
+USER root
+RUN apt-get update
+RUN apt-get install -y firefox
+RUN wget https://github.com/mozilla/geckodriver/releases/download/v0.30.0/geckodriver-v0.30.0-linux64.tar.gz
+RUN tar -xvzf geckodriver*
+RUN mv geckodriver /bin/
+
+COPY jupyter-pystata-gen3-licensed/resources/wait_for_license.sh /tmp/
+COPY jupyter-pystata-gen3-licensed/resources/setup_licensed_notebook.py /tmp/
+RUN chmod 777 /tmp/wait_for_license.sh /tmp/setup_licensed_notebook.py
+
+USER $NB_USER
+RUN pip3 install selenium
+
+# Pin stata_setup to avoid error on splash parameter
+RUN pip3 uninstall --yes stata-setup
+RUN pip3 install stata-setup==0.1.2
+
+# Remove the notebook created in jupyter-pystata-user-licensed
+RUN rm $HOME/Stata.ipynb
+COPY jupyter-pystata-gen3-licensed/resources/licensed_stata_session.ipynb $HOME

jupyter-pystata-gen3-licensed/README.md

@@ -0,0 +1,29 @@
+## Licensed Stata Workspaces
+
+---
+
+For general information about Gen3 Stata workspaces, see the [Stata workspaces README](https://github.com/uc-cdis/containers/tree/master/jupyter-pystata)
+
+### Licensing
+Stata software requires a license to run.
+This container waits for a license provided by an external job.
+It then runs a script which launces a jupyter notebook, runs its first cell in order to initialize a STATA session, then deletes the license.
+This is to prevent the user from accessing the license directly.
+
+The current external job for license distribution is the [`distribute-licenses-job`](https://github.com/uc-cdis/cloud-automation/blob/master/kube/services/jobs/distribute-licenses-job.yaml) in the [cloud-automation repository](https://github.com/uc-cdis/cloud-automation).
+
+### Local development
+To build, enter the root directory of this repo and run:
+```
+docker build -t stata-licensed -f jupyter-pystata-gen3-licensed/Dockerfile .
+
+docker run --name stata-licensed -p 8888:8888 stata-licensed /tmp/wait_for_license.sh --NotebookApp.base_url=/lw-workspace/proxy/ --NotebookApp.password='' --NotebookApp.token=''
+```
+
+(You will need a local copy of `Stata17Linux64.tar.gz`.)
+
+Then, with your license `stata.lic`,
+
+```
+docker cp stata.lic stata-licensed:/usr/local/stata17/stata.lic
+```

jupyter-pystata/resources/Stata.ipynb → jupyter-pystata-gen3-licensed/resources/licensed_stata_session.ipynb

@@ -1,34 +1,48 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Licensed STATA Notebook Workspace\n",
+    "This notebook runs a licensed STATA MP session, managed by the Gen3 platform.\n",
+    "\n",
+    "The license supplied to this workspace session belongs to the University of Chicago and is not for use outside of the Gen3 platform.\n",
+    "\n",
+    "Because licenses are protected and limited, this workspace is limited to a single running STATA session, which has already been initialized via the cell below.\n",
+    "\n",
+    "Users who have an existing STATA license are encouraged to bring it to a self-supplied STATA workspace."
+   ],
+   "metadata": {}
+  },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
-   "outputs": [],
    "source": [
     "import stata_setup\n",
     "stata_setup.config(\"/usr/local/stata17\", \"mp\")"
-   ]
+   ],
+   "outputs": [],
+   "metadata": {}
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
-   "outputs": [],
    "source": [
     "%%stata\n",
     ". describe"
-   ]
+   ],
+   "outputs": [],
+   "metadata": {}
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
-   "outputs": [],
    "source": [
     "%%mata\n",
     "sqrt(4)"
-   ]
+   ],
+   "outputs": [],
+   "metadata": {}
   }
  ],
  "metadata": {

jupyter-pystata-gen3-licensed/resources/setup_licensed_notebook.py

@@ -0,0 +1,43 @@
+from selenium import webdriver
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.firefox.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.action_chains import ActionChains
+
+
+opts = Options()
+opts.headless = True
+browser = webdriver.Firefox(options=opts)
+
+print("Checking for .lic file")
+with open("/usr/local/stata17/stata.lic", "r") as lic_file:
+    print("Found stata.lic file")
+
+print("Ready to open notebook")
+browser.get("http://127.0.0.1:8888/lw-workspace/proxy/notebooks/licensed_stata_session.ipynb")
+print("Notebook is opened")
+
+actions = ActionChains(browser)
+actions.pause(5)
+
+# Down-arrow to get to the second cell in the notebook
+actions.send_keys(Keys.DOWN)
+actions.pause(1)
+actions.perform()
+
+# Run the stata setup cell
+print("Ready to run stata setup cell")
+actions.key_down(Keys.SHIFT)
+actions.send_keys(Keys.ENTER)
+actions.key_up(Keys.SHIFT)
+actions.perform()
+actions.pause(5)
+
+# Save notebook with output
+print("Ready to save notebook")
+actions.key_down(Keys.CONTROL)
+actions.send_keys("S")
+actions.key_up(Keys.CONTROL)
+actions.pause(1)
+actions.send_keys(Keys.ENTER)
+actions.perform()

jupyter-pystata-gen3-licensed/resources/wait_for_license.sh

@@ -0,0 +1,20 @@
+# Signal to the distributor cron job that we want a license
+touch /tmp/waiting_for_license.flag
+
+while [ ! -f /usr/local/stata17/stata.lic ]; do sleep 1; echo "Waiting for license."; done
+
+echo "Received a license. Starting jupyter."
+
+start-notebook.sh $@ &
+
+sleep 20
+
+echo "Running Stata notebook init script."
+python3 /tmp/setup_licensed_notebook.py
+
+rm geckodriver*
+
+echo "Init script done."
+rm /usr/local/stata17/stata.lic /tmp/waiting_for_license.flag
+
+while true; do sleep 1; done

jupyter-pystata-user-licensed/.gitignore

@@ -0,0 +1 @@
+Stata17Linux64.tar.gz

jupyter-pystata-user-licensed/Dockerfile

@@ -0,0 +1,30 @@
+FROM quay.io/cdis/jupyter-superslim:1.0.5
+
+USER root
+RUN apt-get update
+
+# needed if user wants to run stinit (license validation) in the workspace
+RUN	apt-get install -y libncurses5
+
+RUN mkdir /usr/local/stata17
+COPY jupyter-pystata-user-licensed/resources/Stata17Linux64.tar.gz /tmp/Stata17Linux64.tar.gz
+RUN cd /usr/local/stata17 && tar -xvf /tmp/Stata17Linux64.tar.gz
+
+RUN chown $NB_USER /usr/local/stata17/
+ENV PATH $PATH:/usr/local/stata17
+RUN cd /usr/local/stata17 && \
+	{ echo y; echo y; echo y; } | ./install
+
+COPY jupyter-pystata-user-licensed/resources/Stata.ipynb .
+COPY jupyter-pystata-user-licensed/resources/welcome.html .
+
+USER $NB_USER
+RUN pip install --user stata_setup==0.1.2
+
+COPY jupyter-pystata-user-licensed/dockerstart.sh /usr/local/bin/
+
+RUN mkdir /tmp/custom_api
+COPY jupyter-pystata-user-licensed/resources/custom_api/* /tmp/custom_api/
+ENV PYTHONPATH="${PYTHONPATH}:/tmp/"
+
+CMD /usr/local/bin/dockerstart.sh

jupyter-pystata/README.md → jupyter-pystata-user-licensed/README.md

@@ -12,15 +12,17 @@ Note: While we have received permission from Stata to keep these containers publ
 
 
 ### Licensing
-Stata software requires a license to run. As a one-time step, workspace users should add their license files in their persistant workspace storage as `~/pd/stata.lic` . This file is checked via an iPython startup hook (triggered when a new notebook is opened) and copied to the appropriate location so that Stata can recognize it.
+Stata software requires a license to run. As a one-time step, workspace users should add their license files in their persistant workspace storage as `~/pd/stata.lic`. The notebook has a few lines of python code to check for the license file and copy it to the appropriate location so that Stata can recognize it.
 
 ### Local development
 To build, enter the root directory of this repo and run:
 ```
-docker build -t stata -f jupyter-pystata/Dockerfile .
+docker build -t stata -f jupyter-pystata-user-licensed/Dockerfile .
 ```
 
 and to run this container:
 ```
 docker run -P 8888:8888 stata
 ```
+
+You will need a local copy of `Stata17Linux64.tar.gz` for the build.

jupyter-pystata-user-licensed/dockerstart.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+if [[ -z "${STATA_LICENSE}" ]]; then
+  echo "STATA_LICENSE unset. exiting..."
+  exit 1
+else
+  echo $STATA_LICENSE > /usr/local/stata17/stata.lic
+  unset STATA_LICENSE
+  start-notebook.sh --NotebookApp.nbserver_extensions="{'custom_api.ready_handler':True}" $NOTEBOOK_ARGS $@ &
+  JUPYTER_PID=$!
+  sleep 10
+
+  wait $JUPYTER_PID
+fi

jupyter-pystata-user-licensed/resources/Stata.ipynb

@@ -0,0 +1,45 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import shutil\n",
+    "\n",
+    "if (os.path.isfile(f\"/home/{os.environ['NB_USER']}/pd/stata.lic\")\n",
+    "    and not os.path.isfile(\"/usr/local/stata17/stata.lic\")):\n",
+    "    shutil.copyfile(f\"/home/{os.environ['NB_USER']}/pd/stata.lic\", \"/usr/local/stata17/stata.lic\")\n",
+    "\n",
+    "if (os.path.isfile(\"/usr/local/stata17/stata.lic\")):\n",
+    "    import stata_setup\n",
+    "    stata_setup.config(\"/usr/local/stata17\", \"mp\")\n",
+    "else:\n",
+    "    print(\"Did not find license, please provide it at '/pd/stata.lic'\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}