Merge pull request #1078 from notoraptor/dashboard-new-testing-framework

Dashboard - new testing framework: playwright

.github/workflows/dashboard-src-new-with-playwright.yml

@@ -0,0 +1,74 @@
+name: Playwright Tests
+
+on:
+  push:
+    branches: [ master, develop ]
+  pull_request:
+    branches: [ master, develop ]
+  release:
+    types: [published]
+  workflow_dispatch:
+
+defaults:
+  run:
+    working-directory: dashboard/src
+
+jobs:
+  test-dashboard-src-with-playwright:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    # Playwright requires Node.js 14 or higher.
+    strategy:
+      matrix:
+        node-version: [14.x, 16.x]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+
+    # Launch Orion backend
+
+    - name: Launch Orion server
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install Orion from local copy
+      run: python -m pip install ../../.[profet]
+
+    - name: Setup MongoDB
+      uses: supercharge/[email protected]
+      with:
+        mongodb-version: 4.2
+        mongodb-db: orion_dashboard_test
+    - name: Populate MongoDB
+      run: |
+        cd ../../
+        python .github/workflows/orion/pickle_to_mongodb.py
+        cd dashboard/src/
+
+    - name: Start Orion backend
+      run: |
+        # Start Orion backend in repository root folder.
+        cd ../../
+        mkdir -p gunicorn_tmp_dir
+        orion -vv serve -c .github/workflows/orion/orion_config_mongodb.yaml 2> orion-backend-${{ matrix.node-version }}.log &
+        cd dashboard/src/
+
+    - name: Install dependencies
+      run: yarn
+      # check files formatting using Carbon's `ci-check` script
+    - name: Check files formatting
+      run: yarn ci-check
+    - name: Install Playwright Browsers
+      run: yarn playwright install --with-deps
+
+    # Launch Orion frontend, needed by Playwright
+    - name: Build dashboard
+      run: yarn build
+    - name: Serve build
+      run: ./node_modules/.bin/serve -l 3000 build/ &
+
+    - name: Run Playwright tests
+      run: yarn playwright test --project=chromium --reporter=line

.github/workflows/orion/add_uncompleted_experiment.py

@@ -0,0 +1,173 @@
+"""
+Helper script to add an uncompleted experiment to
+db_dashboard_full_with_uncompleted_experiments.pkl
+built from a copy of db_dashboard_full.pkl
+"""
+from datetime import datetime, timedelta
+
+from orion.core.worker.trial import Trial
+from orion.storage.base import setup_storage
+
+SUBMIT_TIME = datetime(
+    year=2000,
+    month=1,
+    day=1,
+    hour=10,
+    minute=0,
+    second=0,
+    microsecond=123456,
+)
+
+
+NEW_EXPERIMENT_DEFINITIONS = [
+    # Default, max trial 200
+    {
+        "name": "uncompleted_experiment",
+        "count": {
+            "completed": 40,
+            "new": 30,
+            "reserved": 25,
+            "suspended": 20,
+            "interrupted": 15,
+            "broken": 10,
+        },
+        "max_trials": 200,
+    },
+    # Max trial 0
+    {
+        "name": "uncompleted_max_trials_0",
+        "count": {
+            "completed": 6,
+            "new": 5,
+            "reserved": 4,
+            "suspended": 3,
+            "interrupted": 2,
+            "broken": 1,
+        },
+        "max_trials": 0,
+    },
+    # Max trial infinite
+    {
+        "name": "uncompleted_max_trials_infinite",
+        "count": {
+            "completed": 6,
+            "new": 5,
+            "reserved": 4,
+            "suspended": 3,
+            "interrupted": 2,
+            "broken": 1,
+        },
+        "max_trials": float("inf"),
+    },
+    # Completed > max trials
+    {
+        "name": "uncompleted_max_trials_lt_completed_trials",
+        "count": {
+            "completed": 20,
+            "new": 5,
+            "reserved": 4,
+            "suspended": 3,
+            "interrupted": 2,
+            "broken": 1,
+        },
+        "max_trials": 10,
+    },
+    # No completed trials
+    {
+        "name": "uncompleted_no_completed_trials",
+        "count": {
+            "completed": 0,
+            "new": 5,
+            "reserved": 4,
+            "suspended": 3,
+            "interrupted": 2,
+            "broken": 1,
+        },
+        "max_trials": 200,
+    },
+]
+
+
+def main():
+    # Get base experiment model from experiment 2-dim-shape-exp.1 read from db_dashboard_full.pkl
+    input_storage = setup_storage(
+        {
+            "database": {
+                "host": ".github/workflows/orion/db_dashboard_full.pkl",
+                "type": "pickleddb",
+            }
+        }
+    )
+    (base_experiment,) = input_storage._db.read(
+        "experiments", {"name": "2-dim-shape-exp", "version": 1}
+    )
+    del base_experiment["_id"]
+    base_experiment["metadata"]["datetime"] = SUBMIT_TIME
+
+    storage = setup_storage(
+        {
+            "database": {
+                "host": ".github/workflows/orion/db_dashboard_full_with_uncompleted_experiments.pkl",
+                "type": "pickleddb",
+            }
+        }
+    )
+    pickle_db = storage._db
+
+    # Insert new experiments into db_dashboard_full_with_uncompleted_experiments.pkl
+    for new_exp_def in NEW_EXPERIMENT_DEFINITIONS:
+        new_experiment = base_experiment.copy()
+        new_experiment["name"] = new_exp_def["name"]
+        new_experiment["max_trials"] = new_exp_def["max_trials"]
+
+        # Clean database if necessary.
+        for exp_found in pickle_db.read(
+            "experiments", {"name": new_experiment["name"]}
+        ):
+            nb_exps_deleted = pickle_db.remove("experiments", {"_id": exp_found["_id"]})
+            nb_trials_deleted = pickle_db.remove(
+                "trials", {"experiment": exp_found["_id"]}
+            )
+            print(
+                f"[{new_experiment['name']}] Deleted",
+                nb_exps_deleted,
+                "experiment(s),",
+                nb_trials_deleted,
+                "trial(s)",
+            )
+
+        # Write and get new experiment.
+        pickle_db.write(
+            "experiments",
+            new_experiment,
+        )
+        (exp,) = pickle_db.read("experiments", {"name": new_experiment["name"]})
+
+        # Write new experiment trials.
+        x = {"name": "/x", "type": "real", "value": 0.0}
+        results = {"name": "obj", "type": "objective", "value": 0.0}
+        for status, count in sorted(new_exp_def["count"].items()):
+            for i in range(count):
+                trial_kwargs = dict(
+                    experiment=exp["_id"],
+                    params=[x],
+                    status=status,
+                    results=[results],
+                    submit_time=SUBMIT_TIME,
+                )
+                if status != "new":
+                    trial_kwargs.update(
+                        start_time=SUBMIT_TIME + timedelta(minutes=i),
+                    )
+                if status == "completed":
+                    trial_kwargs.update(
+                        end_time=SUBMIT_TIME + timedelta(minutes=(i + 2)),
+                    )
+                pickle_db.write("trials", Trial(**trial_kwargs).to_dict())
+                x["value"] += 1
+                results["value"] += 0.1
+        print(f"[{new_experiment['name']}] written.")
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/orion/pickle_to_mongodb.py

@@ -7,7 +7,7 @@ def main():
     storage = setup_storage(
         {
             "database": {
-                "host": ".github/workflows/orion/db_dashboard_full.pkl",
+                "host": ".github/workflows/orion/db_dashboard_full_with_uncompleted_experiments.pkl",
                 "type": "pickleddb",
             }
         }
@@ -18,6 +18,10 @@ def main():
     with pickle_db.locked_database(write=False) as database:
         for collection_name in database._db.keys():
             print(f"[{collection_name}]")
+            # Clean Mongo DB first
+            for element in mongo_db.read(collection_name):
+                mongo_db.remove(collection_name, {"_id": element["_id"]})
+            # Fill Mongo DB
             data = database.read(collection_name)
             mongo_db.write(collection_name, data)
     print("Pickle to Mongo DB done.")

dashboard/src/.gitignore

@@ -31,3 +31,6 @@ yarn-error.log*
 !.yarn/sdks
 !.yarn/versions
 !yarn.lock
+/test-results/
+/playwright-report/
+/playwright/.cache/

dashboard/src/package.json

@@ -32,6 +32,7 @@
     "react-dnd": "^16.0.1",
     "react-dnd-html5-backend": "^16.0.1",
     "react-dom": "18.2.0",
+    "react-infinite-scroller": "^1.2.6",
     "react-plotly.js": "^2.5.1",
     "react-router-dom": "5.3.3",
     "react-scripts": "5.0.1",
@@ -40,6 +41,7 @@
   "devDependencies": {
     "@commitlint/cli": "17.0.1",
     "@commitlint/config-conventional": "7.6.0",
+    "@playwright/test": "^1.30.0",
     "@testing-library/dom": "^8.17.1",
     "@testing-library/jest-dom": "^5.16.5",
     "@testing-library/react": "^13.4.0",
@@ -48,6 +50,7 @@
     "lint-staged": "12.4.2",
     "prettier": "1.19.1",
     "sass": "1.52.1",
+    "serve": "^14.2.0",
     "wait-for-expect": "3.0.2"
   },
   "resolutions": {