diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6d1638a..f16ccb7 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -27,22 +27,29 @@ jobs:
       - name: Install flit
         run: pip install flit
 
-      - name: Install package
-        run: flit install
+      - name: Install production deps
+        run: flit install --deps production
 
-      - name: Check CLI
-        # TODO: This won't catch most missing dependencies.
-        run: dp-wizard --help
-
-      - name: Install dev dependencies
-        run: pip install -r requirements-dev.txt
+      - name: Install subset of test deps
+        run:
+          pip install pytest pytest-playwright
 
       - name: Install browsers
+        # TODO: Why doesn't work to just install chromium now?
         # Install just one browser instead of the default three.
         # https://playwright.dev/python/docs/browsers#managing-browser-binaries
-        run: playwright install chromium
+        run: playwright install
+
+      - name: Run end-to-end test
+        # We want to make sure there aren't hidden dependencies
+        # on dev tools in production code, and that it still works
+        # when dependencies aren't pinned.
+        run: pytest -k test_app
+
+      - name: Install dev dependencies
+        run: pip install -r requirements-dev.txt
 
-      - name: Test
+      - name: Run full tests
         run: ./ci.sh
 
       - uses: actions/upload-artifact@v4
diff --git a/tests/test_app.py b/tests/test_app.py
index 063acdf..76127bb 100644
--- a/tests/test_app.py
+++ b/tests/test_app.py
@@ -23,6 +23,10 @@ def test_demo_app(page: Page, demo_app: ShinyAppProc):  # pragma: no cover
 
 
 def test_default_app(page: Page, default_app: ShinyAppProc):  # pragma: no cover
+    # Default 5s timeout works for me in dev,
+    # but in CI or fresh venv it is slower.
+    expect.set_options(timeout=10_000)
+
     pick_dataset_text = "How many rows of the CSV"
     perform_analysis_text = "Select numeric columns of interest"
     download_results_text = "You can now make a differentially private release"