utils/management_tools/Get_runs_with_most_fields.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utility notebook to get the runs with most fields logged in a particular project\n",
    "\n",
    "<a target=\"_blank\" href=\"https://colab.research.google.com/github/neptune-ai/examples/blob/main/utils/management_tools/Get_runs_with_most_fields.ipynb\">\n",
    "  <img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open in Colab\"/>\n",
    "</a><a target=\"_blank\" href=\"https://github.com/neptune-ai/examples/blob/main/utils/management_tools/Get_runs_with_most_fields.ipynb\">\n",
    "  <img alt=\"Open in GitHub\" src=\"https://img.shields.io/badge/Open_in_GitHub-blue?logo=github&labelColor=black\">\n",
    "</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! pip install -qq -U neptune tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import neptune\n",
    "import pandas as pd\n",
    "from tqdm.auto import tqdm\n",
    "from typing import Optional\n",
    "import logging"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set Neptune API token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if \"NEPTUNE_API_TOKEN\" not in os.environ:\n",
    "    from getpass import getpass\n",
    "\n",
    "    os.environ[\"NEPTUNE_API_TOKEN\"] = getpass(\"Enter the Neptune API token you wish to use: \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Enter project to scan\n",
    "\n",
    "To find the full project name:\n",
    "\n",
    "1. [Log in to Neptune](https://app.neptune.ai/).\n",
    "1. Open the project settings and select **Details & privacy**.\n",
    "\n",
    "For more help, see [Setting Neptune credentials](https://docs.neptune.ai/setup/setting_credentials) in the Neptune docs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"NEPTUNE_PROJECT\"] = input(\"Enter the project to scan in the format workspace/project: \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get all the runs from the project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Suppressing logs\n",
    "logging.getLogger(\"neptune\").setLevel(logging.CRITICAL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "runs_df = pd.DataFrame()\n",
    "\n",
    "with neptune.init_project(mode=\"read-only\") as proj:\n",
    "    runs_df = proj.fetch_runs_table(\n",
    "        columns=[],\n",
    "        progress_bar=False,\n",
    "    ).to_pandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fetch namespaces from all the runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten_namespaces(\n",
    "    dictionary: dict, prefix: Optional[list] = None, result: Optional[list] = None\n",
    ") -> list:\n",
    "    if prefix is None:\n",
    "        prefix = []\n",
    "    if result is None:\n",
    "        result = []\n",
    "\n",
    "    for k, v in dictionary.items():\n",
    "        if isinstance(v, dict):\n",
    "            flatten_namespaces(v, prefix + [k], result)\n",
    "        elif prefix_str := \"/\".join(prefix):\n",
    "            result.append(f\"{prefix_str}/{k}\")\n",
    "        else:\n",
    "            result.append(k)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "namespaces = []\n",
    "\n",
    "\n",
    "for id in tqdm(runs_df[\"sys/id\"].values, total=len(runs_df)):\n",
    "    with neptune.init_run(with_id=id, mode=\"read-only\") as run:\n",
    "        namespaces.append(len(flatten_namespaces(run.get_structure())))\n",
    "\n",
    "runs_df[\"namespaces\"] = namespaces"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Runs with most fields logged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "runs_df.sort_values(by=\"namespaces\", ascending=False)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "private_outputs": true,
   "provenance": [],
   "toc_visible": true
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "py310",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}