diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..1f2be51 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,15 @@ +[run] +# All files under source are checked, even if not otherwise referenced. +source = . + +omit = + # TODO + app.py + +# More strict: Check transitions between lines, not just individual lines. +# TODO: branch = True + +[report] +show_missing = True +skip_covered = True +fail_under = 100 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..5ecd0d5 --- /dev/null +++ b/.flake8 @@ -0,0 +1,12 @@ +[flake8] +exclude = .git,.venv,__pycache__ + +# Config recommended by black: +# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#bugbear +max-line-length = 80 +extend-select = B950 +extend-ignore = E203,E501,E701 + +per-file-ignores = + # Ignore undefined names + */templates/*:F821,F401 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..ecc48ab --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,47 @@ +name: Test + +on: + push: + branches: + - main + pull_request: + +jobs: + test: + runs-on: ubuntu-22.04 + strategy: + matrix: + python-version: + - '3.9' + - '3.12' + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install flit + run: pip install flit + + - name: Install package + run: flit install + + - name: Check CLI + # TODO: This won't catch most missing dependencies. + run: dp-creator-ii --help + + - name: Install dev dependencies + run: pip install -r requirements-dev.txt + + - name: Install browsers + run: playwright install + + - name: Test + run: coverage run -m pytest -v + + - name: Check coverage + run: coverage report diff --git a/.mypy.ini b/.mypy.ini new file mode 100644 index 0000000..16faf59 --- /dev/null +++ b/.mypy.ini @@ -0,0 +1,5 @@ +[mypy] +exclude = '\.venv' + +# TODO: Ignore undefined names only in templates. +disable_error_code = name-defined diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..d07faf4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,17 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + # Using this mirror lets us use mypyc-compiled black, which is about 2x faster + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.8.0 + hooks: + - id: black + # It is recommended to specify the latest version of Python + # supported by your project here, or alternatively use + # pre-commit's default_language_version, see + # https://pre-commit.com/#top_level-default_language_version + language_version: python3.11 diff --git a/README.md b/README.md index f3ab36a..925c4bd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # DP Creator II -** Under Construction ** +**Under Construction** Building on what we've learned from [DP Creator](https://github.com/opendp/dpcreator), DP Creator II will offer: @@ -10,3 +10,56 @@ Building on what we've learned from [DP Creator](https://github.com/opendp/dpcre - Interactive visualization of privacy budget choices - UI development in Python with [Shiny](https://shiny.posit.co/py/) - Tracking of cumulative privacy consumption between sessions + +## Usage + +``` +usage: dp-creator-ii [-h] [--csv CSV_PATH] [--unit UNIT_OF_PRIVACY] [--debug] + +DP Creator II makes it easier to get started with Differential Privacy. + +options: + -h, --help show this help message and exit + --csv CSV_PATH Path to CSV containing private data + --unit UNIT_OF_PRIVACY + Unit of privacy: How many rows can an individual + contribute? + --debug Use during development for increased logging and auto- + reload after code changes +``` + + +## Development + +### Getting Started + +To get started, clone the repo and install dev dependencies in a virtual environment: +``` +git clone https://github.com/opendp/dp-creator-ii.git +cd dp-creator-ii +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements-dev.txt +pre-commit install +playwright install +``` + +Now install the application itself and run it: +``` +flit install --symlink +dp-creator-ii +``` +Your browser should open and connect you to the application. + +Tests should pass, and code coverage should be complete (except blocks we explicitly ignore): +``` +coverage run -m pytest -v +coverage report +``` + +### Conventions + +Branch names should be of the form `NNNN-short-description`, where `NNNN` is the issue number being addressed. + +Dependencies should be pinned for development, but not pinned when the package is installed. +New dev dependencies can be added to `requirements-dev.in`, and then run `pip-compile requirements-dev.in` to update `requirements-dev.txt` diff --git a/dp_creator_ii/.gitignore b/dp_creator_ii/.gitignore new file mode 100644 index 0000000..0cffcb3 --- /dev/null +++ b/dp_creator_ii/.gitignore @@ -0,0 +1 @@ +config.json \ No newline at end of file diff --git a/dp_creator_ii/__init__.py b/dp_creator_ii/__init__.py new file mode 100644 index 0000000..be09255 --- /dev/null +++ b/dp_creator_ii/__init__.py @@ -0,0 +1,62 @@ +"""DP Creator II makes it easier to get started with Differential Privacy.""" + +import os +from pathlib import Path +from argparse import ArgumentParser +import json + +import shiny + + +__version__ = "0.0.1" + + +def get_parser(): + parser = ArgumentParser(description=__doc__) + parser.add_argument( + "--csv", + dest="csv_path", + type=Path, + help="Path to CSV containing private data", + ) + parser.add_argument( + "--unit", + dest="unit_of_privacy", + type=int, + help="Unit of privacy: How many rows can an individual contribute?", + ) + parser.add_argument( + "--debug", + action="store_true", + help="Use during development for increased logging " + "and auto-reload after code changes", + ) + return parser + + +def main(): # pragma: no cover + parser = get_parser() + args = parser.parse_args() + + os.chdir(Path(__file__).parent) # run_app() depends on the CWD. + + # Just setting variables in a plain python module doesn't work: + # The new thread started for the server doesn't see changes. + Path("config.json").write_text( + json.dumps( + { + "csv_path": str(args.csv_path), + "unit_of_privacy": args.unit_of_privacy, + } + ) + ) + + run_app_kwargs = ( + {} + if not args.debug + else { + "reload": True, + "log_level": "debug", + } + ) + shiny.run_app(launch_browser=True, **run_app_kwargs) diff --git a/dp_creator_ii/app.py b/dp_creator_ii/app.py new file mode 100644 index 0000000..e3bb1ec --- /dev/null +++ b/dp_creator_ii/app.py @@ -0,0 +1,124 @@ +import json +from pathlib import Path + +from shiny import App, ui, reactive, render + +from dp_creator_ii.template import make_notebook_py, make_script_py +from dp_creator_ii.converters import convert_py_to_nb + + +def dataset_panel(): + return ui.nav_panel( + "Select Dataset", + "TODO: Pick dataset", + ui.output_text("csv_path_text"), + ui.output_text("unit_of_privacy_text"), + ui.input_action_button("go_to_analysis", "Perform analysis"), + value="dataset_panel", + ) + + +def analysis_panel(): + return ui.nav_panel( + "Perform Analysis", + "TODO: Define analysis", + ui.input_action_button("go_to_results", "Download results"), + value="analysis_panel", + ) + + +def results_panel(): + return ui.nav_panel( + "Download Results", + "TODO: Download Results", + ui.download_button("download_script", "Download script"), + # TODO: Notebook code is badly formatted + # ui.download_button( + # "download_notebook_unexecuted", "Download notebook (unexecuted)" + # ), + # ui.download_button( + # "download_notebook_executed", "Download notebook (executed)" + # ) + value="results_panel", + ) + + +app_ui = ui.page_bootstrap( + ui.navset_tab( + dataset_panel(), + analysis_panel(), + results_panel(), + id="top_level_nav", + ), + title="DP Creator II", +) + + +def server(input, output, session): + config_path = Path(__file__).parent / "config.json" + config = json.loads(config_path.read_text()) + config_path.unlink() + + csv_path = reactive.value(config["csv_path"]) + unit_of_privacy = reactive.value(config["unit_of_privacy"]) + + @render.text + def csv_path_text(): + return str(csv_path.get()) + + @render.text + def unit_of_privacy_text(): + return str(unit_of_privacy.get()) + + @reactive.effect + @reactive.event(input.go_to_analysis) + def go_to_analysis(): + ui.update_navs("top_level_nav", selected="analysis_panel") + + @reactive.effect + @reactive.event(input.go_to_results) + def go_to_results(): + ui.update_navs("top_level_nav", selected="results_panel") + + @render.download( + filename="dp-creator-script.py", + media_type="text/x-python", + ) + async def download_script(): + script_py = make_script_py( + unit=1, + loss=1, + weights=[1], + ) + yield script_py + + @render.download( + filename="dp-creator-notebook.ipynb", + media_type="application/x-ipynb+json", + ) + async def download_notebook_unexecuted(): + notebook_py = make_notebook_py( + csv_path="todo.csv", + unit=1, + loss=1, + weights=[1], + ) + notebook_nb = convert_py_to_nb(notebook_py) + yield notebook_nb + + @render.download( + filename="dp-creator-notebook-executed.ipynb", + media_type="application/x-ipynb+json", + ) + async def download_notebook_executed(): + notebook_py = make_notebook_py( + csv_path="todo.csv", + unit=1, + loss=1, + weights=[1], + ) + notebook_nb = convert_py_to_nb(notebook_py, execute=True) + yield notebook_nb + + +app = App(app_ui, server) diff --git a/dp_creator_ii/converters.py b/dp_creator_ii/converters.py new file mode 100644 index 0000000..d5dcee1 --- /dev/null +++ b/dp_creator_ii/converters.py @@ -0,0 +1,41 @@ +from pathlib import Path +from tempfile import TemporaryDirectory +import subprocess + + +def convert_py_to_nb(python_str, execute=False): + """ + Given Python code as a string, returns a notebook as a string. + Calls jupytext as a subprocess: + Not ideal, but only the CLI is documented well. + """ + with TemporaryDirectory() as temp_dir: + temp_dir_path = Path(temp_dir) + py_path = temp_dir_path / "input.py" + py_path.write_text(python_str) + nb_path = temp_dir_path / "output.ipynb" + argv = ( + [ + "jupytext", + "--to", + "ipynb", # Target format + "--output", + nb_path.absolute(), # Output + ] + + (["--execute"] if execute else []) + + [py_path.absolute()] # Input + ) + try: + subprocess.run(argv, check=True) + except subprocess.CalledProcessError: # pragma: no cover + if not execute: + raise + # Install kernel if missing + # TODO: Is there a better way to do this? + subprocess.run( + "python -m ipykernel install --name kernel_name --user".split(" "), + check=True, + ) + subprocess.run(argv, check=True) + + return nb_path.read_text() diff --git a/dp_creator_ii/template.py b/dp_creator_ii/template.py new file mode 100644 index 0000000..91ad658 --- /dev/null +++ b/dp_creator_ii/template.py @@ -0,0 +1,112 @@ +from pathlib import Path +import re +from functools import partial + + +def match_indent_maker(v): + # Working around late binding closure in loop: + # https://docs.python-guide.org/writing/gotchas/#late-binding-closures + # Would like a simpler solution. + def match_indent(match): + return "\n".join(match.group(1) + line for line in v.split("\n")) + + return match_indent + + +class _Template: + def __init__(self, path): + self._path = path + template_path = Path(__file__).parent / "templates" / path + self._template = template_path.read_text() + + def fill_expressions(self, map): + for k, v in map.items(): + self._template = self._template.replace(k, v) + return self + + def fill_values(self, map): + for k, v in map.items(): + self._template = self._template.replace(k, repr(v)) + return self + + def fill_blocks(self, map): + for k, v in map.items(): + k_re = re.escape(k) + self._template = re.sub( + rf"^(\s*){k_re}$", + partial(match_indent_maker, v)(), + self._template, + flags=re.MULTILINE, + ) + return self + + def __str__(self): + unfilled = set(re.findall(r"[A-Z][A-Z_]+", self._template)) + if unfilled: + raise Exception( + f"Template {self._path} has unfilled slots: " + f'{", ".join(sorted(unfilled))}\n\n{self._template}' + ) + return self._template + + +def _make_context_for_notebook(csv_path, unit, loss, weights): + return str( + _Template("context.py").fill_values( + { + "CSV_PATH": csv_path, + "UNIT": unit, + "LOSS": loss, + "WEIGHTS": weights, + } + ) + ) + + +def _make_context_for_script(unit, loss, weights): + return str( + _Template("context.py") + .fill_expressions({"CSV_PATH": "csv_path"}) + .fill_values( + { + "UNIT": unit, + "LOSS": loss, + "WEIGHTS": weights, + } + ) + ) + + +def _make_imports(): + return str(_Template("imports.py").fill_values({})) + + +def make_notebook_py(csv_path, unit, loss, weights): + return str( + _Template("notebook.py").fill_blocks( + { + "IMPORTS_BLOCK": _make_imports(), + "CONTEXT_BLOCK": _make_context_for_notebook( + csv_path=csv_path, + unit=unit, + loss=loss, + weights=weights, + ), + } + ) + ) + + +def make_script_py(unit, loss, weights): + return str( + _Template("script.py").fill_blocks( + { + "IMPORTS_BLOCK": _make_imports(), + "CONTEXT_BLOCK": _make_context_for_script( + unit=unit, + loss=loss, + weights=weights, + ), + } + ) + ) diff --git a/dp_creator_ii/templates/README.md b/dp_creator_ii/templates/README.md new file mode 100644 index 0000000..8d6618c --- /dev/null +++ b/dp_creator_ii/templates/README.md @@ -0,0 +1 @@ +Strings of ALL CAPS are replaced in these templates. Keeping them in a format which can actually be parsed as python makes some things easier, but it is also reinventing the wheel. We may revisit this. \ No newline at end of file diff --git a/dp_creator_ii/templates/context.py b/dp_creator_ii/templates/context.py new file mode 100644 index 0000000..b970a75 --- /dev/null +++ b/dp_creator_ii/templates/context.py @@ -0,0 +1,6 @@ +context = dp.Context.compositor( + data=pl.scan_csv(CSV_PATH), + privacy_unit=dp.unit_of(contributions=UNIT), + privacy_loss=dp.loss_of(epsilon=LOSS), + split_by_weights=WEIGHTS, +) diff --git a/dp_creator_ii/templates/imports.py b/dp_creator_ii/templates/imports.py new file mode 100644 index 0000000..5df8d79 --- /dev/null +++ b/dp_creator_ii/templates/imports.py @@ -0,0 +1,4 @@ +import polars as pl +import opendp.prelude as dp + +dp.enable_features("contrib") diff --git a/dp_creator_ii/templates/notebook.py b/dp_creator_ii/templates/notebook.py new file mode 100644 index 0000000..1eb6702 --- /dev/null +++ b/dp_creator_ii/templates/notebook.py @@ -0,0 +1,4 @@ +IMPORTS_BLOCK + +CONTEXT_BLOCK +print(context) diff --git a/dp_creator_ii/templates/script.py b/dp_creator_ii/templates/script.py new file mode 100644 index 0000000..37be6c5 --- /dev/null +++ b/dp_creator_ii/templates/script.py @@ -0,0 +1,18 @@ +from argparse import ArgumentParser + +IMPORTS_BLOCK + + +def get_context(csv_path): + CONTEXT_BLOCK + return context + + +if __name__ == "__main__": + parser = ArgumentParser( + description="Creates a differentially private release from a csv" + ) + parser.add_argument("--csv", help="Path to csv containing private data") + args = parser.parse_args() + context = get_context(csv_path=args.csv) + print(context) diff --git a/dp_creator_ii/tests/fixtures/fake-executed.ipynb b/dp_creator_ii/tests/fixtures/fake-executed.ipynb new file mode 100644 index 0000000..21678c5 --- /dev/null +++ b/dp_creator_ii/tests/fixtures/fake-executed.ipynb @@ -0,0 +1,69 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "d68beaaf", + "metadata": {}, + "source": [ + "Introduction" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "c4a24010", + "metadata": { + "execution": { + "iopub.execute_input": "2024-09-24T22:01:27.184792Z", + "iopub.status.busy": "2024-09-24T22:01:27.184540Z", + "iopub.status.idle": "2024-09-24T22:01:27.188766Z", + "shell.execute_reply": "2024-09-24T22:01:27.188362Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4\n" + ] + } + ], + "source": [ + "print(2 + 2)" + ] + }, + { + "cell_type": "markdown", + "id": "aa6d5643", + "metadata": {}, + "source": [ + "Conclusion" + ] + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "-all" + }, + "kernelspec": { + "display_name": "kernel_name", + "language": "python", + "name": "kernel_name" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/dp_creator_ii/tests/fixtures/fake.csv b/dp_creator_ii/tests/fixtures/fake.csv new file mode 100644 index 0000000..4393e1a --- /dev/null +++ b/dp_creator_ii/tests/fixtures/fake.csv @@ -0,0 +1 @@ +fake-column \ No newline at end of file diff --git a/dp_creator_ii/tests/fixtures/fake.ipynb b/dp_creator_ii/tests/fixtures/fake.ipynb new file mode 100644 index 0000000..a8aff92 --- /dev/null +++ b/dp_creator_ii/tests/fixtures/fake.ipynb @@ -0,0 +1,42 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "3cf946cc", + "metadata": {}, + "source": [ + "Introduction" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5a25047", + "metadata": {}, + "outputs": [], + "source": [ + "print(2 + 2)" + ] + }, + { + "cell_type": "markdown", + "id": "08581071", + "metadata": {}, + "source": [ + "Conclusion" + ] + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "-all", + "main_language": "python", + "notebook_metadata_filter": "-all" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/dp_creator_ii/tests/fixtures/fake.py b/dp_creator_ii/tests/fixtures/fake.py new file mode 100644 index 0000000..64636ab --- /dev/null +++ b/dp_creator_ii/tests/fixtures/fake.py @@ -0,0 +1,5 @@ +# Introduction + +print(2 + 2) + +# Conclusion diff --git a/dp_creator_ii/tests/test_app.py b/dp_creator_ii/tests/test_app.py new file mode 100644 index 0000000..ec1ea58 --- /dev/null +++ b/dp_creator_ii/tests/test_app.py @@ -0,0 +1,13 @@ +from shiny.run import ShinyAppProc +from playwright.sync_api import Page, expect +from shiny.pytest import create_app_fixture + + +app = create_app_fixture("../app.py") + + +# TODO: Why is incomplete coverage reported here? +def test_app(page: Page, app: ShinyAppProc): # pragma: no cover + page.goto(app.url) + expect(page).to_have_title("DP Creator II") + expect(page.locator("body")).to_contain_text("TODO: Pick dataset") diff --git a/dp_creator_ii/tests/test_converters.py b/dp_creator_ii/tests/test_converters.py new file mode 100644 index 0000000..408c267 --- /dev/null +++ b/dp_creator_ii/tests/test_converters.py @@ -0,0 +1,40 @@ +import re +from pathlib import Path +from dp_creator_ii.converters import convert_py_to_nb + + +def norm_nb(nb_str): + normed_nb_str = nb_str + normed_nb_str = re.sub(r'"id": "[^"]+"', '"id": "12345678"', normed_nb_str) + normed_nb_str = re.sub( + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z", + "2024-01-01T00:00:00.000000Z", + normed_nb_str, + ) + # language_info added when saved by VSCode: + normed_nb_str = re.sub(r',\s+"language_info": \{[^}]*\}', "", normed_nb_str) + # version will be different between dev environment and CI: + normed_nb_str = re.sub(r'"version": "[^"]+"', '"version": "3.0.0"', normed_nb_str) + return normed_nb_str + + +def test_convert_py_to_nb(): + fixtures_path = Path("dp_creator_ii/tests/fixtures") + python_str = (fixtures_path / "fake.py").read_text() + actual_nb_str = convert_py_to_nb(python_str) + expected_nb_str = (fixtures_path / "fake.ipynb").read_text() + + normed_actual_nb_str = norm_nb(actual_nb_str) + normed_expected_nb_str = norm_nb(expected_nb_str) + assert normed_actual_nb_str == normed_expected_nb_str + + +def test_convert_py_to_nb_execute(): + fixtures_path = Path("dp_creator_ii/tests/fixtures") + python_str = (fixtures_path / "fake.py").read_text() + actual_nb_str = convert_py_to_nb(python_str, execute=True) + expected_nb_str = (fixtures_path / "fake-executed.ipynb").read_text() + + normed_actual_nb_str = norm_nb(actual_nb_str) + normed_expected_nb_str = norm_nb(expected_nb_str) + assert normed_actual_nb_str == normed_expected_nb_str diff --git a/dp_creator_ii/tests/test_help.py b/dp_creator_ii/tests/test_help.py new file mode 100644 index 0000000..6752325 --- /dev/null +++ b/dp_creator_ii/tests/test_help.py @@ -0,0 +1,20 @@ +from pathlib import Path + +import dp_creator_ii + + +def test_help(): + help = ( + dp_creator_ii.get_parser() + .format_help() + # argparse doesn't actually know the name of the script + # and inserts the name of the running program instead. + .replace("__main__.py", "dp-creator-ii") + .replace("pytest", "dp-creator-ii") + # Text is different under Python 3.9: + .replace("optional arguments:", "options:") + ) + print(help) + + readme_md = (Path(__file__).parent.parent.parent / "README.md").read_text() + assert help in readme_md diff --git a/dp_creator_ii/tests/test_misc.py b/dp_creator_ii/tests/test_misc.py new file mode 100644 index 0000000..d7cbede --- /dev/null +++ b/dp_creator_ii/tests/test_misc.py @@ -0,0 +1,14 @@ +import subprocess +import pytest + + +tests = { + "flake8 linting": "flake8 . --count --show-source --statistics", + "mypy type checking": "mypy .", +} + + +@pytest.mark.parametrize("cmd", tests.values(), ids=tests.keys()) +def test_subprocess(cmd): + result = subprocess.run(cmd, shell=True) + assert result.returncode == 0, f'"{cmd}" failed' diff --git a/dp_creator_ii/tests/test_template.py b/dp_creator_ii/tests/test_template.py new file mode 100644 index 0000000..dd53b55 --- /dev/null +++ b/dp_creator_ii/tests/test_template.py @@ -0,0 +1,60 @@ +from tempfile import NamedTemporaryFile +import subprocess +import re +import pytest +import opendp.prelude as dp +from dp_creator_ii.template import _Template, make_notebook_py, make_script_py + + +fake_csv = "dp_creator_ii/tests/fixtures/fake.csv" + + +def test_fill_template(): + context_template = _Template("context.py") + context_block = str( + context_template.fill_values( + { + "CSV_PATH": fake_csv, + "UNIT": 1, + "LOSS": 1, + "WEIGHTS": [1], + } + ) + ) + assert f"data=pl.scan_csv('{fake_csv}')" in context_block + + +def test_fill_template_unfilled_slots(): + context_template = _Template("context.py") + with pytest.raises( + Exception, + match=re.escape("context.py has unfilled slots: CSV_PATH, LOSS, UNIT, WEIGHTS"), + ): + str(context_template.fill_values({})) + + +def test_make_notebook(): + notebook = make_notebook_py( + csv_path=fake_csv, + unit=1, + loss=1, + weights=[1], + ) + globals = {} + exec(notebook, globals) + assert isinstance(globals["context"], dp.Context) + + +def test_make_script(): + script = make_script_py( + unit=1, + loss=1, + weights=[1], + ) + + with NamedTemporaryFile(mode="w", delete=False) as fp: + fp.write(script) + fp.close() + + result = subprocess.run(["python", fp.name, "--csv", fake_csv]) + assert result.returncode == 0 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..59751d1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ["flit_core >=3.2,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "dp_creator_ii" +authors = [{name = "Chuck McCallum", email = "mccallucc@gmail.com"}] +readme = "README.md" +license = {file = "LICENSE"} +classifiers = ["License :: OSI Approved :: MIT License"] +dynamic = ["version", "description"] +dependencies = [ + "shiny", + "shinywidgets", + "opendp[polars]", + "jupytext", + "jupyter-client", + "nbconvert", + "ipykernel", +] + +[project.scripts] +dp-creator-ii = "dp_creator_ii:main" + +[project.urls] +Home = "https://github.com/opendp/dp-creator-ii" diff --git a/requirements-dev.in b/requirements-dev.in new file mode 100644 index 0000000..f2eb714 --- /dev/null +++ b/requirements-dev.in @@ -0,0 +1,37 @@ +# After making changes here, run: +# pip-compile requirements-dev.in && pip install -r requirements-dev.txt + +# Developer tools: +pip-tools +flit +black +flake8 +flake8-bugbear +pre-commit + +# Testing: +pytest +pytest-playwright +mypy +coverage + + + +# Everything below should also be listed in pyproject.toml: + +# OpenDP: +opendp[polars] +# For Python 3.9: +scipy<1.14 + +# Conversion: +jupytext +jupyter-client +nbconvert +ipykernel +# May also require: +# python -m ipykernel install --name kernel_name --user + +# Shiny: +shiny +shinywidgets diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..7916b16 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,361 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile requirements-dev.in +# +anyio==4.4.0 + # via + # starlette + # watchfiles +appdirs==1.4.4 + # via shiny +appnope==0.1.4 + # via ipykernel +asgiref==3.8.1 + # via shiny +asttokens==2.4.1 + # via stack-data +attrs==24.2.0 + # via + # flake8-bugbear + # jsonschema + # referencing +beautifulsoup4==4.12.3 + # via nbconvert +black==24.8.0 + # via -r requirements-dev.in +bleach==6.1.0 + # via nbconvert +build==1.2.2 + # via pip-tools +certifi==2024.8.30 + # via requests +cfgv==3.4.0 + # via pre-commit +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # black + # pip-tools + # shiny + # uvicorn +comm==0.2.2 + # via + # ipykernel + # ipywidgets +coverage==7.6.1 + # via -r requirements-dev.in +debugpy==1.8.6 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +distlib==0.3.8 + # via virtualenv +docutils==0.21.2 + # via flit +executing==2.1.0 + # via stack-data +fastjsonschema==2.20.0 + # via nbformat +filelock==3.16.1 + # via virtualenv +flake8==7.1.1 + # via + # -r requirements-dev.in + # flake8-bugbear +flake8-bugbear==24.8.19 + # via -r requirements-dev.in +flit==3.9.0 + # via -r requirements-dev.in +flit-core==3.9.0 + # via flit +greenlet==3.0.3 + # via playwright +h11==0.14.0 + # via uvicorn +htmltools==0.5.3 + # via shiny +identify==2.6.1 + # via pre-commit +idna==3.10 + # via + # anyio + # requests +iniconfig==2.0.0 + # via pytest +ipykernel==6.29.5 + # via -r requirements-dev.in +ipython==8.18.0 + # via + # ipykernel + # ipywidgets +ipywidgets==8.1.5 + # via shinywidgets +jedi==0.19.1 + # via ipython +jinja2==3.1.4 + # via nbconvert +joblib==1.4.2 + # via scikit-learn +jsonschema==4.23.0 + # via nbformat +jsonschema-specifications==2023.12.1 + # via jsonschema +jupyter-client==8.6.3 + # via + # -r requirements-dev.in + # ipykernel + # nbclient +jupyter-core==5.7.2 + # via + # ipykernel + # jupyter-client + # nbclient + # nbconvert + # nbformat + # shinywidgets +jupyterlab-pygments==0.3.0 + # via nbconvert +jupyterlab-widgets==3.0.13 + # via ipywidgets +jupytext==1.16.4 + # via -r requirements-dev.in +linkify-it-py==2.0.3 + # via shiny +markdown-it-py==3.0.0 + # via + # jupytext + # mdit-py-plugins + # shiny +markupsafe==2.1.5 + # via + # jinja2 + # nbconvert +matplotlib-inline==0.1.7 + # via + # ipykernel + # ipython +mccabe==0.7.0 + # via flake8 +mdit-py-plugins==0.4.2 + # via + # jupytext + # shiny +mdurl==0.1.2 + # via markdown-it-py +mistune==3.0.2 + # via nbconvert +mypy==1.11.2 + # via -r requirements-dev.in +mypy-extensions==1.0.0 + # via + # black + # mypy +nbclient==0.10.0 + # via nbconvert +nbconvert==7.16.4 + # via -r requirements-dev.in +nbformat==5.10.4 + # via + # jupytext + # nbclient + # nbconvert +nest-asyncio==1.6.0 + # via ipykernel +nodeenv==1.9.1 + # via pre-commit +numpy==1.26.4 + # via + # opendp + # pyarrow + # randomgen + # scikit-learn + # scipy +opendp[polars]==0.11.1 + # via -r requirements-dev.in +packaging==24.1 + # via + # black + # build + # htmltools + # ipykernel + # jupytext + # nbconvert + # pytest + # shiny +pandocfilters==1.5.1 + # via nbconvert +parso==0.8.4 + # via jedi +pathspec==0.12.1 + # via black +pexpect==4.9.0 + # via ipython +pip-tools==7.4.1 + # via -r requirements-dev.in +platformdirs==4.3.6 + # via + # black + # jupyter-core + # virtualenv +playwright==1.47.0 + # via pytest-playwright +pluggy==1.5.0 + # via pytest +polars==1.1.0 + # via opendp +pre-commit==3.8.0 + # via -r requirements-dev.in +prompt-toolkit==3.0.36 + # via + # ipython + # questionary + # shiny +psutil==6.0.0 + # via ipykernel +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.3 + # via stack-data +pyarrow==17.0.0 + # via opendp +pycodestyle==2.12.1 + # via flake8 +pyee==12.0.0 + # via playwright +pyflakes==3.2.0 + # via flake8 +pygments==2.18.0 + # via + # ipython + # nbconvert +pyproject-hooks==1.1.0 + # via + # build + # pip-tools +pytest==8.3.3 + # via + # -r requirements-dev.in + # pytest-base-url + # pytest-playwright +pytest-base-url==2.1.0 + # via pytest-playwright +pytest-playwright==0.5.2 + # via -r requirements-dev.in +python-dateutil==2.9.0.post0 + # via + # jupyter-client + # shinywidgets +python-multipart==0.0.9 + # via shiny +python-slugify==8.0.4 + # via pytest-playwright +pyyaml==6.0.2 + # via + # jupytext + # pre-commit +pyzmq==26.2.0 + # via + # ipykernel + # jupyter-client +questionary==2.0.1 + # via shiny +randomgen==2.0.1 + # via opendp +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +requests==2.32.3 + # via + # flit + # pytest-base-url +rpds-py==0.20.0 + # via + # jsonschema + # referencing +scikit-learn==1.5.2 + # via opendp +scipy==1.13.1 + # via + # -r requirements-dev.in + # scikit-learn +shiny==1.1.0 + # via + # -r requirements-dev.in + # shinywidgets +shinywidgets==0.3.3 + # via -r requirements-dev.in +six==1.16.0 + # via + # asttokens + # bleach + # python-dateutil +sniffio==1.3.1 + # via anyio +soupsieve==2.6 + # via beautifulsoup4 +stack-data==0.6.3 + # via ipython +starlette==0.38.5 + # via shiny +text-unidecode==1.3 + # via python-slugify +threadpoolctl==3.5.0 + # via scikit-learn +tinycss2==1.3.0 + # via nbconvert +tomli-w==1.0.0 + # via flit +tornado==6.4.1 + # via + # ipykernel + # jupyter-client +traitlets==5.14.3 + # via + # comm + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbconvert + # nbformat +typing-extensions==4.12.2 + # via + # htmltools + # mypy + # pyee + # shiny +uc-micro-py==1.0.3 + # via linkify-it-py +urllib3==2.2.3 + # via requests +uvicorn==0.30.6 + # via shiny +virtualenv==20.26.5 + # via pre-commit +watchfiles==0.24.0 + # via shiny +wcwidth==0.2.13 + # via prompt-toolkit +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websockets==13.0.1 + # via shiny +wheel==0.44.0 + # via pip-tools +widgetsnbextension==4.0.13 + # via ipywidgets + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools