Hello world! (#2)

Initial Shiny webapp with minimal testing and code generation demo

.coveragerc

@@ -0,0 +1,15 @@
+[run]
+# All files under source are checked, even if not otherwise referenced.
+source = .
+
+omit =
+  # TODO
+  app.py
+
+# More strict: Check transitions between lines, not just individual lines.
+# TODO: branch = True
+
+[report]
+show_missing = True
+skip_covered = True
+fail_under = 100

.flake8

@@ -0,0 +1,12 @@
+[flake8]
+exclude = .git,.venv,__pycache__
+
+# Config recommended by black:
+# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#bugbear
+max-line-length = 80
+extend-select = B950
+extend-ignore = E203,E501,E701
+
+per-file-ignores =
+    # Ignore undefined names
+    */templates/*:F821,F401

.github/workflows/test.yml

@@ -0,0 +1,47 @@
+name: Test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python-version:
+          - '3.9'
+          - '3.12'
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install flit
+        run: pip install flit
+
+      - name: Install package
+        run: flit install
+
+      - name: Check CLI
+        # TODO: This won't catch most missing dependencies.
+        run: dp-creator-ii --help
+
+      - name: Install dev dependencies
+        run: pip install -r requirements-dev.txt
+
+      - name: Install browsers
+        run: playwright install
+
+      - name: Test
+        run: coverage run -m pytest -v
+
+      - name: Check coverage
+        run: coverage report

.mypy.ini

@@ -0,0 +1,5 @@
+[mypy]
+exclude = '\.venv'
+
+# TODO: Ignore undefined names only in templates. 
+disable_error_code = name-defined

.pre-commit-config.yaml

@@ -0,0 +1,17 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+  # Using this mirror lets us use mypyc-compiled black, which is about 2x faster
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 24.8.0
+    hooks:
+      - id: black
+        # It is recommended to specify the latest version of Python
+        # supported by your project here, or alternatively use
+        # pre-commit's default_language_version, see
+        # https://pre-commit.com/#top_level-default_language_version
+        language_version: python3.11

README.md

@@ -1,6 +1,6 @@
 # DP Creator II
 
-** Under Construction **
+**Under Construction**
 
 Building on what we've learned from [DP Creator](https://github.com/opendp/dpcreator), DP Creator II will offer:
 
@@ -10,3 +10,56 @@ Building on what we've learned from [DP Creator](https://github.com/opendp/dpcre
 - Interactive visualization of privacy budget choices
 - UI development in Python with [Shiny](https://shiny.posit.co/py/)
 - Tracking of cumulative privacy consumption between sessions
+
+## Usage
+
+```
+usage: dp-creator-ii [-h] [--csv CSV_PATH] [--unit UNIT_OF_PRIVACY] [--debug]
+
+DP Creator II makes it easier to get started with Differential Privacy.
+
+options:
+  -h, --help            show this help message and exit
+  --csv CSV_PATH        Path to CSV containing private data
+  --unit UNIT_OF_PRIVACY
+                        Unit of privacy: How many rows can an individual
+                        contribute?
+  --debug               Use during development for increased logging and auto-
+                        reload after code changes
+```
+
+
+## Development
+
+### Getting Started
+
+To get started, clone the repo and install dev dependencies in a virtual environment:
+```
+git clone https://github.com/opendp/dp-creator-ii.git
+cd dp-creator-ii
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements-dev.txt
+pre-commit install
+playwright install
+```
+
+Now install the application itself and run it:
+```
+flit install --symlink
+dp-creator-ii
+```
+Your browser should open and connect you to the application.
+
+Tests should pass, and code coverage should be complete (except blocks we explicitly ignore):
+```
+coverage run -m pytest -v
+coverage report
+```
+
+### Conventions
+
+Branch names should be of the form `NNNN-short-description`, where `NNNN` is the issue number being addressed.
+
+Dependencies should be pinned for development, but not pinned when the package is installed.
+New dev dependencies can be added to `requirements-dev.in`, and then run `pip-compile requirements-dev.in` to update `requirements-dev.txt`

dp_creator_ii/.gitignore

@@ -0,0 +1 @@
+config.json

dp_creator_ii/__init__.py

@@ -0,0 +1,62 @@
+"""DP Creator II makes it easier to get started with Differential Privacy."""
+
+import os
+from pathlib import Path
+from argparse import ArgumentParser
+import json
+
+import shiny
+
+
+__version__ = "0.0.1"
+
+
+def get_parser():
+    parser = ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--csv",
+        dest="csv_path",
+        type=Path,
+        help="Path to CSV containing private data",
+    )
+    parser.add_argument(
+        "--unit",
+        dest="unit_of_privacy",
+        type=int,
+        help="Unit of privacy: How many rows can an individual contribute?",
+    )
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="Use during development for increased logging "
+        "and auto-reload after code changes",
+    )
+    return parser
+
+
+def main():  # pragma: no cover
+    parser = get_parser()
+    args = parser.parse_args()
+
+    os.chdir(Path(__file__).parent)  # run_app() depends on the CWD.
+
+    # Just setting variables in a plain python module doesn't work:
+    # The new thread started for the server doesn't see changes.
+    Path("config.json").write_text(
+        json.dumps(
+            {
+                "csv_path": str(args.csv_path),
+                "unit_of_privacy": args.unit_of_privacy,
+            }
+        )
+    )
+
+    run_app_kwargs = (
+        {}
+        if not args.debug
+        else {
+            "reload": True,
+            "log_level": "debug",
+        }
+    )
+    shiny.run_app(launch_browser=True, **run_app_kwargs)

dp_creator_ii/app.py

@@ -0,0 +1,124 @@
+import json
+from pathlib import Path
+
+from shiny import App, ui, reactive, render
+
+from dp_creator_ii.template import make_notebook_py, make_script_py
+from dp_creator_ii.converters import convert_py_to_nb
+
+
+def dataset_panel():
+    return ui.nav_panel(
+        "Select Dataset",
+        "TODO: Pick dataset",
+        ui.output_text("csv_path_text"),
+        ui.output_text("unit_of_privacy_text"),
+        ui.input_action_button("go_to_analysis", "Perform analysis"),
+        value="dataset_panel",
+    )
+
+
+def analysis_panel():
+    return ui.nav_panel(
+        "Perform Analysis",
+        "TODO: Define analysis",
+        ui.input_action_button("go_to_results", "Download results"),
+        value="analysis_panel",
+    )
+
+
+def results_panel():
+    return ui.nav_panel(
+        "Download Results",
+        "TODO: Download Results",
+        ui.download_button("download_script", "Download script"),
+        # TODO: Notebook code is badly formatted
+        # ui.download_button(
+        #     "download_notebook_unexecuted", "Download notebook (unexecuted)"
+        # ),
+        # ui.download_button(
+        #     "download_notebook_executed", "Download notebook (executed)"
+        # )
+        value="results_panel",
+    )
+
+
+app_ui = ui.page_bootstrap(
+    ui.navset_tab(
+        dataset_panel(),
+        analysis_panel(),
+        results_panel(),
+        id="top_level_nav",
+    ),
+    title="DP Creator II",
+)
+
+
+def server(input, output, session):
+    config_path = Path(__file__).parent / "config.json"
+    config = json.loads(config_path.read_text())
+    config_path.unlink()
+
+    csv_path = reactive.value(config["csv_path"])
+    unit_of_privacy = reactive.value(config["unit_of_privacy"])
+
+    @render.text
+    def csv_path_text():
+        return str(csv_path.get())
+
+    @render.text
+    def unit_of_privacy_text():
+        return str(unit_of_privacy.get())
+
+    @reactive.effect
+    @reactive.event(input.go_to_analysis)
+    def go_to_analysis():
+        ui.update_navs("top_level_nav", selected="analysis_panel")
+
+    @reactive.effect
+    @reactive.event(input.go_to_results)
+    def go_to_results():
+        ui.update_navs("top_level_nav", selected="results_panel")
+
+    @render.download(
+        filename="dp-creator-script.py",
+        media_type="text/x-python",
+    )
+    async def download_script():
+        script_py = make_script_py(
+            unit=1,
+            loss=1,
+            weights=[1],
+        )
+        yield script_py
+
+    @render.download(
+        filename="dp-creator-notebook.ipynb",
+        media_type="application/x-ipynb+json",
+    )
+    async def download_notebook_unexecuted():
+        notebook_py = make_notebook_py(
+            csv_path="todo.csv",
+            unit=1,
+            loss=1,
+            weights=[1],
+        )
+        notebook_nb = convert_py_to_nb(notebook_py)
+        yield notebook_nb
+
+    @render.download(
+        filename="dp-creator-notebook-executed.ipynb",
+        media_type="application/x-ipynb+json",
+    )
+    async def download_notebook_executed():
+        notebook_py = make_notebook_py(
+            csv_path="todo.csv",
+            unit=1,
+            loss=1,
+            weights=[1],
+        )
+        notebook_nb = convert_py_to_nb(notebook_py, execute=True)
+        yield notebook_nb
+
+
+app = App(app_ui, server)

dp_creator_ii/converters.py

@@ -0,0 +1,41 @@
+from pathlib import Path
+from tempfile import TemporaryDirectory
+import subprocess
+
+
+def convert_py_to_nb(python_str, execute=False):
+    """
+    Given Python code as a string, returns a notebook as a string.
+    Calls jupytext as a subprocess:
+    Not ideal, but only the CLI is documented well.
+    """
+    with TemporaryDirectory() as temp_dir:
+        temp_dir_path = Path(temp_dir)
+        py_path = temp_dir_path / "input.py"
+        py_path.write_text(python_str)
+        nb_path = temp_dir_path / "output.ipynb"
+        argv = (
+            [
+                "jupytext",
+                "--to",
+                "ipynb",  # Target format
+                "--output",
+                nb_path.absolute(),  # Output
+            ]
+            + (["--execute"] if execute else [])
+            + [py_path.absolute()]  # Input
+        )
+        try:
+            subprocess.run(argv, check=True)
+        except subprocess.CalledProcessError:  # pragma: no cover
+            if not execute:
+                raise
+            # Install kernel if missing
+            # TODO: Is there a better way to do this?
+            subprocess.run(
+                "python -m ipykernel install --name kernel_name --user".split(" "),
+                check=True,
+            )
+            subprocess.run(argv, check=True)
+
+        return nb_path.read_text()