diff --git a/ax/utils/notebook/plotting.py b/ax/utils/notebook/plotting.py index 71047edd1d2..0bccd2462dc 100644 --- a/ax/utils/notebook/plotting.py +++ b/ax/utils/notebook/plotting.py @@ -20,6 +20,13 @@ def init_notebook_plotting(offline: bool = False) -> None: display_bundle = {"text/html": _wrap_js(_js_requires(offline=offline))} display(display_bundle, raw=True) logger.info("Injecting Plotly library into cell. Do not overwrite or delete cell.") + logger.info( + """ + Please see + (https://ax.dev/tutorials/visualizations.html#Fix-for-plots-that-are-not-rendering) + if visualizations are not rendering. + """.strip() + ) init_notebook_mode() diff --git a/tutorials/visualizations.ipynb b/tutorials/visualizations.ipynb index d01a19e8eea..4603bcbb30e 100644 --- a/tutorials/visualizations.ipynb +++ b/tutorials/visualizations.ipynb @@ -1,344 +1,395 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "originalKey": "e23719d9-8a24-4208-8439-34e7b8270c79" - }, - "source": [ - "# Visualizations\n", - "\n", - "This tutorial illustrates the core visualization utilities available in Ax." - ] + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "originalKey": "e23719d9-8a24-4208-8439-34e7b8270c79" + }, + "source": [ + "# Visualizations\n", + "\n", + "This tutorial illustrates the core visualization utilities available in Ax." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "code_folding": [], + "executionStartTime": 1627652821316, + "executionStopTime": 1627652822868, + "hidden_ranges": [], + "originalKey": "101b0e96-5b3d-48c5-bf3c-677b4ddf90c7", + "requestMsgId": "c0dd9aaf-896d-4ea9-912f-1e58d301d114" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "from ax.modelbridge.cross_validation import cross_validate\n", + "from ax.plot.contour import interact_contour\n", + "from ax.plot.diagnostic import interact_cross_validation\n", + "from ax.plot.scatter import interact_fitted, plot_objective_vs_constraints, tile_fitted\n", + "from ax.plot.slice import plot_slice\n", + "from ax.service.ax_client import AxClient, ObjectiveProperties\n", + "from ax.utils.measurement.synthetic_functions import hartmann6\n", + "from ax.utils.notebook.plotting import init_notebook_plotting, render\n", + "\n", + "init_notebook_plotting()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "code_folding": [], + "hidden_ranges": [], + "originalKey": "8449378f-890e-4e76-8d73-ce2aa4120a69", + "showInput": true + }, + "source": [ + "## 1. Create experiment and run optimization\n", + "\n", + "The vizualizations require an experiment object and a model fit on the evaluated data. The routine below is a copy of the Service API tutorial, so the explanation here is omitted. Retrieving the experiment and model objects for each API paradigm is shown in the respective tutorials" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "originalKey": "f7544e06-6c6a-4841-b659-3be6a198a948" + }, + "source": [ + "#### 1a. Define search space and evaluation function" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "code_folding": [], + "executionStartTime": 1627652824829, + "executionStopTime": 1627652824877, + "hidden_ranges": [], + "originalKey": "28f6cb76-828f-445d-bdda-ba057c87dcd0", + "requestMsgId": "7495e7e2-1025-4292-b3aa-e953739cef3e" + }, + "outputs": [], + "source": [ + "noise_sd = 0.1\n", + "param_names = [f\"x{i+1}\" for i in range(6)] # x1, x2, ..., x6\n", + "\n", + "\n", + "def noisy_hartmann_evaluation_function(parameterization):\n", + " x = np.array([parameterization.get(p_name) for p_name in param_names])\n", + " noise1, noise2 = np.random.normal(0, noise_sd, 2)\n", + "\n", + " return {\n", + " \"hartmann6\": (hartmann6(x) + noise1, noise_sd),\n", + " \"l2norm\": (np.sqrt((x**2).sum()) + noise2, noise_sd),\n", + " }" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "originalKey": "17a51543-298e-47d4-bcd9-33459fe1169e" + }, + "source": [ + "#### 1b. Create Experiment" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "code_folding": [], + "executionStartTime": 1627654956712, + "executionStopTime": 1627654956823, + "hidden_ranges": [], + "originalKey": "6fca889c-a4ff-42ef-a669-6eb8803de89c", + "requestMsgId": "905eff52-e649-4bd5-abf0-ff69c1549852" + }, + "outputs": [], + "source": [ + "ax_client = AxClient()\n", + "ax_client.create_experiment(\n", + " name=\"test_visualizations\",\n", + " parameters=[\n", + " {\n", + " \"name\": p_name,\n", + " \"type\": \"range\",\n", + " \"bounds\": [0.0, 1.0],\n", + " }\n", + " for p_name in param_names\n", + " ],\n", + " objectives={\"hartmann6\": ObjectiveProperties(minimize=True)},\n", + " outcome_constraints=[\"l2norm <= 1.25\"],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "code_folding": [], + "hidden_ranges": [], + "originalKey": "ab892f7c-4830-4c1d-b476-ec1078ec3faf", + "showInput": false + }, + "source": [ + "#### 1c. Run the optimization and fit a GP on all data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "code_folding": [], + "executionStartTime": 1627654642967, + "executionStopTime": 1627654862819, + "hidden_ranges": [], + "originalKey": "7269a5ba-45c8-4acf-ac83-a5ea8a52d6c1", + "requestMsgId": "c7a4dea8-fd6d-4e1a-84de-ad973ede0cd7" + }, + "outputs": [], + "source": [ + "for i in range(20):\n", + " parameters, trial_index = ax_client.get_next_trial()\n", + " # Local evaluation here can be replaced with deployment to external system.\n", + " ax_client.complete_trial(\n", + " trial_index=trial_index, raw_data=noisy_hartmann_evaluation_function(parameters)\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "originalKey": "72f4d3e7-fa04-43d0-8451-ded292e705df" + }, + "source": [ + "## 2. Contour plots\n", + "\n", + "The plot below shows the response surface for `hartmann6` metric as a function of the `x1`, `x2` parameters.\n", + "\n", + "The other parameters are fixed in the middle of their respective ranges, which in this example is 0.5 for all of them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "code_folding": [], + "executionStartTime": 1627654870209, + "executionStopTime": 1627654871972, + "hidden_ranges": [], + "originalKey": "843df85c-965d-4a83-9fe1-696225d81c0f", + "requestMsgId": "4a643541-867c-46b6-868d-64337920c2a3" + }, + "outputs": [], + "source": [ + "# this could alternately be done with `ax.plot.contour.plot_contour`\n", + "render(ax_client.get_contour_plot(param_x=\"x1\", param_y=\"x2\", metric_name=\"hartmann6\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "originalKey": "1de0991a-d99b-4d07-acec-4a2eb4a20a73" + }, + "source": [ + "#### 2a. Interactive contour plot\n", + "\n", + "The plot below allows toggling between different pairs of parameters to view the contours." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "code_folding": [], + "executionStartTime": 1627652959076, + "executionStopTime": 1627652982911, + "hidden_ranges": [], + "originalKey": "4af9f166-0163-4ff5-9ecb-f534a69efe3d", + "requestMsgId": "2de6919e-92e1-4425-8d90-b117e9f41855" + }, + "outputs": [], + "source": [ + "model = ax_client.generation_strategy.model\n", + "render(interact_contour(model=model, metric_name=\"hartmann6\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "originalKey": "1ff470bb-5daf-4179-b814-01cc80dafe3e" + }, + "source": [ + "## 3. Tradeoff plots\n", + "This plot illustrates the tradeoffs achievable for 2 different metrics. The plot takes the x-axis metric as input (usually the objective) and allows toggling among all other metrics for the y-axis.\n", + "\n", + "This is useful to get a sense of the pareto frontier (i.e. what is the best objective value achievable for different bounds on the constraint)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "executionStartTime": 1627652996903, + "executionStopTime": 1627652997294, + "originalKey": "57023556-0293-44ef-91d6-81b911ff41d3", + "requestMsgId": "10b72ecc-a019-42a1-8358-18f14927ef75" + }, + "outputs": [], + "source": [ + "render(plot_objective_vs_constraints(model, \"hartmann6\", rel=False))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "originalKey": "f2d4fae3-2140-45d0-8142-49b91548ca59" + }, + "source": [ + "## 4. Cross-validation plots\n", + "\n", + "CV plots are useful to check how well the model predictions calibrate against the actual measurements. If all points are close to the dashed line, then the model is a good predictor of the real data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "executionStartTime": 1627397871181, + "executionStopTime": 1627397871526, + "originalKey": "f770f8a9-466c-4fd2-b268-3a0d166482f3", + "requestMsgId": "d6242810-a316-4e2b-b9dd-dd4c56b725b7" + }, + "outputs": [], + "source": [ + "cv_results = cross_validate(model)\n", + "render(interact_cross_validation(cv_results))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "originalKey": "40f8ee99-fee8-4fd9-9cff-a7aa230dd5ae" + }, + "source": [ + "## 5. Slice plots\n", + "\n", + "Slice plots show the metric outcome as a function of one parameter while fixing the others. They serve a similar function as contour plots." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "executionStartTime": 1627397880415, + "executionStopTime": 1627397880572, + "originalKey": "aed7c789-a024-48c6-86f7-502e571e298f", + "requestMsgId": "a7238d82-f6bb-441d-badc-673dedaa101e" + }, + "outputs": [], + "source": [ + "render(plot_slice(model, \"x2\", \"hartmann6\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "originalKey": "4975848f-31d4-4ed0-976c-39e3b7474fb7" + }, + "source": [ + "## 6. Tile plots\n", + "\n", + "Tile plots are useful for viewing the effect of each arm." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "executionStartTime": 1627397890236, + "executionStopTime": 1627397890496, + "originalKey": "2ed10008-8adf-4ce2-8334-04a4f2a3e895", + "requestMsgId": "33b593e6-2ec8-4bc4-b6e3-6586ddfb15c5" + }, + "outputs": [], + "source": [ + "render(interact_fitted(model, rel=False))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Fix for plots that are not rendering\n", + "\n", + "In certain environments like Google Colab or remote setups, plots may not render. If this is the case, we recommend using the below workaround which overrides the default renderer in plotly. The below cell changes the renderer to \"jupyterlab\" for this tutorial, but you can find the right renderer for your use case by calling `pio.renderers`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import plotly.io as pio\n", + "pio.renderers.default = \"jupyterlab\"\n", + "\n", + "render(ax_client.get_contour_plot(param_x=\"x1\", param_y=\"x2\", metric_name=\"hartmann6\"))" + ] + } + ], + "metadata": { + "custom": { + "cells": [], + "metadata": { + "custom": { + "cells": [], + "metadata": { + "fileHeader": "", + "fileUid": "05c52a71-d835-47cc-a717-85b584211970", + "isAdHoc": false, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 + }, + "fileHeader": "", + "fileUid": "ca754573-88fc-4f06-9300-97ea8ea25f89", + "indentAmount": 2, + "isAdHoc": false, + "kernelspec": { + "display_name": "python3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 + }, + "indentAmount": 2, + "kernelspec": { + "name": "python3", + "display_name": "python3" + } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "code_folding": [], - "executionStartTime": 1627652821316, - "executionStopTime": 1627652822868, - "hidden_ranges": [], - "originalKey": "101b0e96-5b3d-48c5-bf3c-677b4ddf90c7", - "requestMsgId": "c0dd9aaf-896d-4ea9-912f-1e58d301d114" - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "from ax.modelbridge.cross_validation import cross_validate\n", - "from ax.plot.contour import interact_contour\n", - "from ax.plot.diagnostic import interact_cross_validation\n", - "from ax.plot.scatter import interact_fitted, plot_objective_vs_constraints, tile_fitted\n", - "from ax.plot.slice import plot_slice\n", - "from ax.service.ax_client import AxClient, ObjectiveProperties\n", - "from ax.utils.measurement.synthetic_functions import hartmann6\n", - "from ax.utils.notebook.plotting import init_notebook_plotting, render\n", - "\n", - "init_notebook_plotting()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "code_folding": [], - "hidden_ranges": [], - "originalKey": "8449378f-890e-4e76-8d73-ce2aa4120a69", - "showInput": true - }, - "source": [ - "## 1. Create experiment and run optimization\n", - "\n", - "The vizualizations require an experiment object and a model fit on the evaluated data. The routine below is a copy of the Service API tutorial, so the explanation here is omitted. Retrieving the experiment and model objects for each API paradigm is shown in the respective tutorials" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "originalKey": "f7544e06-6c6a-4841-b659-3be6a198a948" - }, - "source": [ - "#### 1a. Define search space and evaluation function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "code_folding": [], - "executionStartTime": 1627652824829, - "executionStopTime": 1627652824877, - "hidden_ranges": [], - "originalKey": "28f6cb76-828f-445d-bdda-ba057c87dcd0", - "requestMsgId": "7495e7e2-1025-4292-b3aa-e953739cef3e" - }, - "outputs": [], - "source": [ - "noise_sd = 0.1\n", - "param_names = [f\"x{i+1}\" for i in range(6)] # x1, x2, ..., x6\n", - "\n", - "\n", - "def noisy_hartmann_evaluation_function(parameterization):\n", - " x = np.array([parameterization.get(p_name) for p_name in param_names])\n", - " noise1, noise2 = np.random.normal(0, noise_sd, 2)\n", - "\n", - " return {\n", - " \"hartmann6\": (hartmann6(x) + noise1, noise_sd),\n", - " \"l2norm\": (np.sqrt((x**2).sum()) + noise2, noise_sd),\n", - " }" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "originalKey": "17a51543-298e-47d4-bcd9-33459fe1169e" - }, - "source": [ - "#### 1b. Create Experiment" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "code_folding": [], - "executionStartTime": 1627654956712, - "executionStopTime": 1627654956823, - "hidden_ranges": [], - "originalKey": "6fca889c-a4ff-42ef-a669-6eb8803de89c", - "requestMsgId": "905eff52-e649-4bd5-abf0-ff69c1549852" - }, - "outputs": [], - "source": [ - "ax_client = AxClient()\n", - "ax_client.create_experiment(\n", - " name=\"test_visualizations\",\n", - " parameters=[\n", - " {\n", - " \"name\": p_name,\n", - " \"type\": \"range\",\n", - " \"bounds\": [0.0, 1.0],\n", - " }\n", - " for p_name in param_names\n", - " ],\n", - " objectives={\"hartmann6\": ObjectiveProperties(minimize=True)},\n", - " outcome_constraints=[\"l2norm <= 1.25\"],\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "code_folding": [], - "hidden_ranges": [], - "originalKey": "ab892f7c-4830-4c1d-b476-ec1078ec3faf", - "showInput": false - }, - "source": [ - "#### 1c. Run the optimization and fit a GP on all data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "code_folding": [], - "executionStartTime": 1627654642967, - "executionStopTime": 1627654862819, - "hidden_ranges": [], - "originalKey": "7269a5ba-45c8-4acf-ac83-a5ea8a52d6c1", - "requestMsgId": "c7a4dea8-fd6d-4e1a-84de-ad973ede0cd7" - }, - "outputs": [], - "source": [ - "for i in range(20):\n", - " parameters, trial_index = ax_client.get_next_trial()\n", - " # Local evaluation here can be replaced with deployment to external system.\n", - " ax_client.complete_trial(\n", - " trial_index=trial_index, raw_data=noisy_hartmann_evaluation_function(parameters)\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "originalKey": "72f4d3e7-fa04-43d0-8451-ded292e705df" - }, - "source": [ - "## 2. Contour plots\n", - "\n", - "The plot below shows the response surface for `hartmann6` metric as a function of the `x1`, `x2` parameters.\n", - "\n", - "The other parameters are fixed in the middle of their respective ranges, which in this example is 0.5 for all of them." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "code_folding": [], - "executionStartTime": 1627654870209, - "executionStopTime": 1627654871972, - "hidden_ranges": [], - "originalKey": "843df85c-965d-4a83-9fe1-696225d81c0f", - "requestMsgId": "4a643541-867c-46b6-868d-64337920c2a3" - }, - "outputs": [], - "source": [ - "# this could alternately be done with `ax.plot.contour.plot_contour`\n", - "render(ax_client.get_contour_plot(param_x=\"x1\", param_y=\"x2\", metric_name=\"hartmann6\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "originalKey": "1de0991a-d99b-4d07-acec-4a2eb4a20a73" - }, - "source": [ - "#### 2a. Interactive contour plot\n", - "\n", - "The plot below allows toggling between different pairs of parameters to view the contours." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "code_folding": [], - "executionStartTime": 1627652959076, - "executionStopTime": 1627652982911, - "hidden_ranges": [], - "originalKey": "4af9f166-0163-4ff5-9ecb-f534a69efe3d", - "requestMsgId": "2de6919e-92e1-4425-8d90-b117e9f41855" - }, - "outputs": [], - "source": [ - "model = ax_client.generation_strategy.model\n", - "render(interact_contour(model=model, metric_name=\"hartmann6\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "originalKey": "1ff470bb-5daf-4179-b814-01cc80dafe3e" - }, - "source": [ - "## 3. Tradeoff plots\n", - "This plot illustrates the tradeoffs achievable for 2 different metrics. The plot takes the x-axis metric as input (usually the objective) and allows toggling among all other metrics for the y-axis.\n", - "\n", - "This is useful to get a sense of the pareto frontier (i.e. what is the best objective value achievable for different bounds on the constraint)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "executionStartTime": 1627652996903, - "executionStopTime": 1627652997294, - "originalKey": "57023556-0293-44ef-91d6-81b911ff41d3", - "requestMsgId": "10b72ecc-a019-42a1-8358-18f14927ef75" - }, - "outputs": [], - "source": [ - "render(plot_objective_vs_constraints(model, \"hartmann6\", rel=False))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "originalKey": "f2d4fae3-2140-45d0-8142-49b91548ca59" - }, - "source": [ - "## 4. Cross-validation plots\n", - "\n", - "CV plots are useful to check how well the model predictions calibrate against the actual measurements. If all points are close to the dashed line, then the model is a good predictor of the real data." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "executionStartTime": 1627397871181, - "executionStopTime": 1627397871526, - "originalKey": "f770f8a9-466c-4fd2-b268-3a0d166482f3", - "requestMsgId": "d6242810-a316-4e2b-b9dd-dd4c56b725b7" - }, - "outputs": [], - "source": [ - "cv_results = cross_validate(model)\n", - "render(interact_cross_validation(cv_results))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "originalKey": "40f8ee99-fee8-4fd9-9cff-a7aa230dd5ae" - }, - "source": [ - "## 5. Slice plots\n", - "\n", - "Slice plots show the metric outcome as a function of one parameter while fixing the others. They serve a similar function as contour plots." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "executionStartTime": 1627397880415, - "executionStopTime": 1627397880572, - "originalKey": "aed7c789-a024-48c6-86f7-502e571e298f", - "requestMsgId": "a7238d82-f6bb-441d-badc-673dedaa101e" - }, - "outputs": [], - "source": [ - "render(plot_slice(model, \"x2\", \"hartmann6\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "originalKey": "4975848f-31d4-4ed0-976c-39e3b7474fb7" - }, - "source": [ - "## 6. Tile plots\n", - "\n", - "Tile plots are useful for viewing the effect of each arm." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "executionStartTime": 1627397890236, - "executionStopTime": 1627397890496, - "originalKey": "2ed10008-8adf-4ce2-8334-04a4f2a3e895", - "requestMsgId": "33b593e6-2ec8-4bc4-b6e3-6586ddfb15c5" - }, - "outputs": [], - "source": [ - "render(interact_fitted(model, rel=False))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.5" - } - }, - "nbformat": 4, - "nbformat_minor": 2 + "nbformat": 4, + "nbformat_minor": 2 }