From e1c82fd99482ce2c1828221e3d399111a96beab9 Mon Sep 17 00:00:00 2001 From: Michael Tiemann <72577720+MichaelTiemannOSC@users.noreply.github.com> Date: Sun, 12 May 2024 07:47:44 -0400 Subject: [PATCH] Clean up ITR-data-production notebook cells. Signed-off-by: Michael Tiemann <72577720+MichaelTiemannOSC@users.noreply.github.com> --- notebooks/ITR-data-production.ipynb | 51 +++++------------------------ 1 file changed, 8 insertions(+), 43 deletions(-) diff --git a/notebooks/ITR-data-production.ipynb b/notebooks/ITR-data-production.ipynb index b1c3b95..f34704e 100644 --- a/notebooks/ITR-data-production.ipynb +++ b/notebooks/ITR-data-production.ipynb @@ -31,7 +31,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "78ba9b8d-2806-4c3a-9c19-0253fd3a8db2", "metadata": {}, "outputs": [], @@ -52,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "cc527cbc-3c9e-429f-bd23-9bb40446d6f8", "metadata": {}, "outputs": [], @@ -65,18 +65,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "ca0ebd26-cdd3-498f-972a-2139a6f76752", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Initializing common units...\n" - ] - } - ], + "outputs": [], "source": [ "# See data-platform-demo/pint-demo.ipynb for quantify/dequantify functions\n", "\n", @@ -148,23 +140,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "0f951a12-1266-4551-868c-ebe04966970e", "metadata": {}, - "outputs": [ - { - "ename": "Skipped", - "evalue": "skipping this notebook", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mSkipped\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[4], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# We'll deal with CI/CD later.\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[43mpytest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mskip\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mskipping this notebook\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mallow_module_level\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;66;03m# Load environment variables from credentials.env\u001b[39;00m\n\u001b[1;32m 5\u001b[0m osc\u001b[38;5;241m.\u001b[39mload_credentials_dotenv()\n", - "File \u001b[0;32m~/virtualenvs/itr-test/lib/python3.11/site-packages/_pytest/outcomes.py:179\u001b[0m, in \u001b[0;36mskip\u001b[0;34m(reason, allow_module_level, msg)\u001b[0m\n\u001b[1;32m 177\u001b[0m __tracebackhide__ \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m 178\u001b[0m reason \u001b[38;5;241m=\u001b[39m _resolve_msg_to_reason(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mskip\u001b[39m\u001b[38;5;124m\"\u001b[39m, reason, msg)\n\u001b[0;32m--> 179\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m Skipped(msg\u001b[38;5;241m=\u001b[39mreason, allow_module_level\u001b[38;5;241m=\u001b[39mallow_module_level)\n", - "\u001b[0;31mSkipped\u001b[0m: skipping this notebook" - ] - } - ], + "outputs": [], "source": [ "# We'll deal with CI/CD later.\n", "pytest.skip(\"skipping this notebook\", allow_module_level=True)\n", @@ -207,24 +186,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "d8a756db-291f-48ad-8ae8-dad3da4af325", "metadata": {}, - "outputs": [ - { - "ename": "KeyError", - "evalue": "'TRINO_USER'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[5], line 15\u001b[0m\n\u001b[1;32m 11\u001b[0m demo_schema \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdemo_dv\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 13\u001b[0m itr_prefix \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mitr_\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m---> 15\u001b[0m engine \u001b[38;5;241m=\u001b[39m \u001b[43mosc\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mattach_trino_engine\u001b[49m\u001b[43m(\u001b[49m\u001b[43mverbose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcatalog\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mingest_catalog\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Dropbox/My Mac (MacBook-Pro.local)/Documents/GitHub/mtiemann-os-climate/osc-ingest-tools/osc_ingest_trino/trino_utils.py:45\u001b[0m, in \u001b[0;36mattach_trino_engine\u001b[0;34m(env_var_prefix, catalog, schema, verbose)\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mattach_trino_engine\u001b[39m(\n\u001b[1;32m 32\u001b[0m env_var_prefix: \u001b[38;5;28mstr\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTRINO\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 33\u001b[0m catalog: Optional[\u001b[38;5;28mstr\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 34\u001b[0m schema: Optional[\u001b[38;5;28mstr\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 35\u001b[0m verbose: Optional[\u001b[38;5;28mbool\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 36\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Engine:\n\u001b[1;32m 37\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Return a SQLAlchemy engine object representing a Trino instance.\u001b[39;00m\n\u001b[1;32m 38\u001b[0m \n\u001b[1;32m 39\u001b[0m \u001b[38;5;124;03m env_var_prefix -- a prefix for all environment variables related to the Trino instance.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;124;03m verbose -- if True, print the full string used to connect.\u001b[39;00m\n\u001b[1;32m 43\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[1;32m 44\u001b[0m sqlstring \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrino://\u001b[39m\u001b[38;5;132;01m{user}\u001b[39;00m\u001b[38;5;124m@\u001b[39m\u001b[38;5;132;01m{host}\u001b[39;00m\u001b[38;5;124m:\u001b[39m\u001b[38;5;132;01m{port}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\n\u001b[0;32m---> 45\u001b[0m user\u001b[38;5;241m=\u001b[39m\u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menviron\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43menv_var_prefix\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m_USER\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m,\n\u001b[1;32m 46\u001b[0m host\u001b[38;5;241m=\u001b[39mos\u001b[38;5;241m.\u001b[39menviron[\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00menv_var_prefix\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m_HOST\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[1;32m 47\u001b[0m port\u001b[38;5;241m=\u001b[39mos\u001b[38;5;241m.\u001b[39menviron[\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00menv_var_prefix\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m_PORT\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[1;32m 48\u001b[0m )\n\u001b[1;32m 49\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m catalog \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 50\u001b[0m sqlstring \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcatalog\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n", - "File \u001b[0;32m:679\u001b[0m, in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n", - "\u001b[0;31mKeyError\u001b[0m: 'TRINO_USER'" - ] - } - ], + "outputs": [], "source": [ "ingest_catalog = \"osc_datacommons_dev\"\n", "ingest_schema = \"mdt_sandbox\"\n",