02_ChemBERTa2_Embedding_Extraction.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/SylwiaNowakowska/LLM_Fine_Tuning_Molecular_Properties/blob/main/02_ChemBERTa2_Embedding_Extraction.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PvKrQYgdZF_T"
      },
      "source": [
        "# ChemBERTa-2 embeddings extraction\n",
        "Notebook for embeddings extractions from last hidden layer."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Environment setup"
      ],
      "metadata": {
        "id": "P-o-UyMijwkY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JbLXDE8JpZa8",
        "outputId": "04ea9afc-4b78-421a-8467-628d7a22819e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 465
        },
        "id": "BylJ9s3sY09e",
        "outputId": "2c908fd0-0330-4c4f-ddae-4c5afdde3996"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting deepchem\n",
            "  Downloading deepchem-2.7.2.dev20230825195110-py3-none-any.whl (838 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/838.8 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.6/838.8 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m838.8/838.8 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from deepchem) (1.3.2)\n",
            "Requirement already satisfied: numpy>=1.21 in /usr/local/lib/python3.10/dist-packages (from deepchem) (1.23.5)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from deepchem) (1.5.3)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from deepchem) (1.2.2)\n",
            "Requirement already satisfied: scipy>=1.10.1 in /usr/local/lib/python3.10/dist-packages (from deepchem) (1.10.1)\n",
            "Collecting rdkit (from deepchem)\n",
            "  Downloading rdkit-2023.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (29.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m29.7/29.7 MB\u001b[0m \u001b[31m57.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->deepchem) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->deepchem) (2023.3)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from rdkit->deepchem) (9.4.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->deepchem) (3.2.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->deepchem) (1.16.0)\n",
            "Installing collected packages: rdkit, deepchem\n",
            "Successfully installed deepchem-2.7.2.dev20230825195110 rdkit-2023.3.3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:deepchem.feat.molecule_featurizers.rdkit_descriptors:No normalization for AvgIpc. Feature removed!\n",
            "WARNING:deepchem.models:Skipped loading some PyTorch models, missing a dependency. No module named 'torch_geometric'\n",
            "WARNING:deepchem.models:No module named 'torch_geometric'\n",
            "WARNING:deepchem.models:Skipped loading modules with pytorch-geometric dependency, missing a dependency. No module named 'torch_geometric'\n",
            "WARNING:deepchem.models:Skipped loading modules with pytorch-lightning dependency, missing a dependency. No module named 'pytorch_lightning'\n",
            "WARNING:deepchem.models:Skipped loading some Jax models, missing a dependency. No module named 'haiku'\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'2.7.2.dev'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "!pip install --pre deepchem\n",
        "import deepchem\n",
        "deepchem.__version__"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VjDBOn0Wmybe",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ae1cb658-c206-48b9-bb41-1ae759501c53"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers\n",
            "  Downloading transformers-4.32.0-py3-none-any.whl (7.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.5/7.5 MB\u001b[0m \u001b[31m69.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.2)\n",
            "Collecting huggingface-hub<1.0,>=0.15.1 (from transformers)\n",
            "  Downloading huggingface_hub-0.16.4-py3-none-any.whl (268 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m37.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m117.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers)\n",
            "  Downloading safetensors-0.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m74.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.15.1->transformers) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.15.1->transformers) (4.7.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.7.22)\n",
            "Installing collected packages: tokenizers, safetensors, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.16.4 safetensors-0.3.3 tokenizers-0.13.3 transformers-4.32.0\n",
            "Collecting simpletransformers\n",
            "  Downloading simpletransformers-0.64.3-py3-none-any.whl (250 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m250.8/250.8 kB\u001b[0m \u001b[31m698.6 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (1.23.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.47.0 in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (4.66.1)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (2023.6.3)\n",
            "Requirement already satisfied: transformers>=4.31.0 in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (4.32.0)\n",
            "Collecting datasets (from simpletransformers)\n",
            "  Downloading datasets-2.14.4-py3-none-any.whl (519 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m519.3/519.3 kB\u001b[0m \u001b[31m48.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (1.10.1)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (1.2.2)\n",
            "Collecting seqeval (from simpletransformers)\n",
            "  Downloading seqeval-1.2.2.tar.gz (43 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.6/43.6 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tensorboard in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (2.12.3)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (1.5.3)\n",
            "Requirement already satisfied: tokenizers in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (0.13.3)\n",
            "Collecting wandb>=0.10.32 (from simpletransformers)\n",
            "  Downloading wandb-0.15.8-py3-none-any.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m94.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting streamlit (from simpletransformers)\n",
            "  Downloading streamlit-1.26.0-py2.py3-none-any.whl (8.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.1/8.1 MB\u001b[0m \u001b[31m112.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sentencepiece (from simpletransformers)\n",
            "  Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m76.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers>=4.31.0->simpletransformers) (3.12.2)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.15.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.31.0->simpletransformers) (0.16.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.31.0->simpletransformers) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.31.0->simpletransformers) (6.0.1)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.31.0->simpletransformers) (0.3.3)\n",
            "Requirement already satisfied: Click!=8.0.0,>=7.1 in /usr/local/lib/python3.10/dist-packages (from wandb>=0.10.32->simpletransformers) (8.1.7)\n",
            "Collecting GitPython!=3.1.29,>=1.0.0 (from wandb>=0.10.32->simpletransformers)\n",
            "  Downloading GitPython-3.1.32-py3-none-any.whl (188 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m188.5/188.5 kB\u001b[0m \u001b[31m20.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb>=0.10.32->simpletransformers) (5.9.5)\n",
            "Collecting sentry-sdk>=1.0.0 (from wandb>=0.10.32->simpletransformers)\n",
            "  Downloading sentry_sdk-1.29.2-py2.py3-none-any.whl (215 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m215.6/215.6 kB\u001b[0m \u001b[31m27.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting docker-pycreds>=0.4.0 (from wandb>=0.10.32->simpletransformers)\n",
            "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
            "Collecting pathtools (from wandb>=0.10.32->simpletransformers)\n",
            "  Downloading pathtools-0.1.2.tar.gz (11 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting setproctitle (from wandb>=0.10.32->simpletransformers)\n",
            "  Downloading setproctitle-1.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb>=0.10.32->simpletransformers) (67.7.2)\n",
            "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb>=0.10.32->simpletransformers) (1.4.4)\n",
            "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb>=0.10.32->simpletransformers) (3.20.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->simpletransformers) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->simpletransformers) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->simpletransformers) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->simpletransformers) (2023.7.22)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->simpletransformers) (9.0.0)\n",
            "Collecting dill<0.3.8,>=0.3.0 (from datasets->simpletransformers)\n",
            "  Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m17.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting xxhash (from datasets->simpletransformers)\n",
            "  Downloading xxhash-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m25.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess (from datasets->simpletransformers)\n",
            "  Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.10/dist-packages (from datasets->simpletransformers) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->simpletransformers) (3.8.5)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->simpletransformers) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->simpletransformers) (2023.3)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->simpletransformers) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->simpletransformers) (3.2.0)\n",
            "Requirement already satisfied: altair<6,>=4.0 in /usr/local/lib/python3.10/dist-packages (from streamlit->simpletransformers) (4.2.2)\n",
            "Requirement already satisfied: blinker<2,>=1.0.0 in /usr/lib/python3/dist-packages (from streamlit->simpletransformers) (1.4)\n",
            "Requirement already satisfied: cachetools<6,>=4.0 in /usr/local/lib/python3.10/dist-packages (from streamlit->simpletransformers) (5.3.1)\n",
            "Requirement already satisfied: importlib-metadata<7,>=1.4 in /usr/local/lib/python3.10/dist-packages (from streamlit->simpletransformers) (6.8.0)\n",
            "Requirement already satisfied: pillow<10,>=7.1.0 in /usr/local/lib/python3.10/dist-packages (from streamlit->simpletransformers) (9.4.0)\n",
            "Collecting pympler<2,>=0.9 (from streamlit->simpletransformers)\n",
            "  Downloading Pympler-1.0.1-py3-none-any.whl (164 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m164.8/164.8 kB\u001b[0m \u001b[31m23.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: rich<14,>=10.14.0 in /usr/local/lib/python3.10/dist-packages (from streamlit->simpletransformers) (13.5.2)\n",
            "Requirement already satisfied: tenacity<9,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from streamlit->simpletransformers) (8.2.3)\n",
            "Requirement already satisfied: toml<2,>=0.10.1 in /usr/local/lib/python3.10/dist-packages (from streamlit->simpletransformers) (0.10.2)\n",
            "Requirement already satisfied: typing-extensions<5,>=4.1.0 in /usr/local/lib/python3.10/dist-packages (from streamlit->simpletransformers) (4.7.1)\n",
            "Collecting tzlocal<5,>=1.1 (from streamlit->simpletransformers)\n",
            "  Downloading tzlocal-4.3.1-py3-none-any.whl (20 kB)\n",
            "Collecting validators<1,>=0.2 (from streamlit->simpletransformers)\n",
            "  Downloading validators-0.21.2-py3-none-any.whl (25 kB)\n",
            "Collecting pydeck<1,>=0.8 (from streamlit->simpletransformers)\n",
            "  Downloading pydeck-0.8.0-py2.py3-none-any.whl (4.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.7/4.7 MB\u001b[0m \u001b[31m94.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tornado<7,>=6.0.3 in /usr/local/lib/python3.10/dist-packages (from streamlit->simpletransformers) (6.3.2)\n",
            "Collecting watchdog>=2.1.5 (from streamlit->simpletransformers)\n",
            "  Downloading watchdog-3.0.0-py3-none-manylinux2014_x86_64.whl (82 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.1/82.1 kB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.10/dist-packages (from tensorboard->simpletransformers) (1.4.0)\n",
            "Requirement already satisfied: grpcio>=1.48.2 in /usr/local/lib/python3.10/dist-packages (from tensorboard->simpletransformers) (1.57.0)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.10/dist-packages (from tensorboard->simpletransformers) (2.17.3)\n",
            "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /usr/local/lib/python3.10/dist-packages (from tensorboard->simpletransformers) (1.0.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard->simpletransformers) (3.4.4)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard->simpletransformers) (0.7.1)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard->simpletransformers) (2.3.7)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.10/dist-packages (from tensorboard->simpletransformers) (0.41.2)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.10/dist-packages (from altair<6,>=4.0->streamlit->simpletransformers) (0.4)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from altair<6,>=4.0->streamlit->simpletransformers) (3.1.2)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair<6,>=4.0->streamlit->simpletransformers) (4.19.0)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.10/dist-packages (from altair<6,>=4.0->streamlit->simpletransformers) (0.12.0)\n",
            "Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from docker-pycreds>=0.4.0->wandb>=0.10.32->simpletransformers) (1.16.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->simpletransformers) (23.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->simpletransformers) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->simpletransformers) (4.0.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->simpletransformers) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->simpletransformers) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->simpletransformers) (1.3.1)\n",
            "Collecting gitdb<5,>=4.0.1 (from GitPython!=3.1.29,>=1.0.0->wandb>=0.10.32->simpletransformers)\n",
            "  Downloading gitdb-4.0.10-py3-none-any.whl (62 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard->simpletransformers) (0.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard->simpletransformers) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard->simpletransformers) (1.3.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<7,>=1.4->streamlit->simpletransformers) (3.16.2)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich<14,>=10.14.0->streamlit->simpletransformers) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich<14,>=10.14.0->streamlit->simpletransformers) (2.16.1)\n",
            "Collecting pytz-deprecation-shim (from tzlocal<5,>=1.1->streamlit->simpletransformers)\n",
            "  Downloading pytz_deprecation_shim-0.1.0.post0-py2.py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.10/dist-packages (from werkzeug>=1.0.1->tensorboard->simpletransformers) (2.1.3)\n",
            "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb>=0.10.32->simpletransformers)\n",
            "  Downloading smmap-5.0.0-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit->simpletransformers) (2023.7.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit->simpletransformers) (0.30.2)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit->simpletransformers) (0.9.2)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich<14,>=10.14.0->streamlit->simpletransformers) (0.1.2)\n",
            "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard->simpletransformers) (0.5.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard->simpletransformers) (3.2.2)\n",
            "Collecting tzdata (from pytz-deprecation-shim->tzlocal<5,>=1.1->streamlit->simpletransformers)\n",
            "  Downloading tzdata-2023.3-py2.py3-none-any.whl (341 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.8/341.8 kB\u001b[0m \u001b[31m39.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: seqeval, pathtools\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for seqeval: filename=seqeval-1.2.2-py3-none-any.whl size=16162 sha256=cdd07ea1d630c996751db8862f4c3cc9d6a60484faa03bec0fb1824aaf41a257\n",
            "  Stored in directory: /root/.cache/pip/wheels/1a/67/4a/ad4082dd7dfc30f2abfe4d80a2ed5926a506eb8a972b4767fa\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8791 sha256=e2bd8af1ea1249c809671b95c0fe0af656c2c288d465dc918f3bf0fb3bb6ee91\n",
            "  Stored in directory: /root/.cache/pip/wheels/e7/f3/22/152153d6eb222ee7a56ff8617d80ee5207207a8c00a7aab794\n",
            "Successfully built seqeval pathtools\n",
            "Installing collected packages: sentencepiece, pathtools, xxhash, watchdog, validators, tzdata, smmap, setproctitle, sentry-sdk, pympler, docker-pycreds, dill, pytz-deprecation-shim, pydeck, multiprocess, gitdb, tzlocal, seqeval, GitPython, wandb, datasets, streamlit, simpletransformers\n",
            "  Attempting uninstall: tzlocal\n",
            "    Found existing installation: tzlocal 5.0.1\n",
            "    Uninstalling tzlocal-5.0.1:\n",
            "      Successfully uninstalled tzlocal-5.0.1\n",
            "Successfully installed GitPython-3.1.32 datasets-2.14.4 dill-0.3.7 docker-pycreds-0.4.0 gitdb-4.0.10 multiprocess-0.70.15 pathtools-0.1.2 pydeck-0.8.0 pympler-1.0.1 pytz-deprecation-shim-0.1.0.post0 sentencepiece-0.1.99 sentry-sdk-1.29.2 seqeval-1.2.2 setproctitle-1.3.2 simpletransformers-0.64.3 smmap-5.0.0 streamlit-1.26.0 tzdata-2023.3 tzlocal-4.3.1 validators-0.21.2 wandb-0.15.8 watchdog-3.0.0 xxhash-3.3.0\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "!pip install simpletransformers\n",
        "!cd .."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZE1C_baibNUh",
        "outputId": "7316e292-c5ad-476c-9d72-97d268d163fe"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'bertviz_repo'...\n",
            "remote: Enumerating objects: 1633, done.\u001b[K\n",
            "remote: Counting objects: 100% (329/329), done.\u001b[K\n",
            "remote: Compressing objects: 100% (118/118), done.\u001b[K\n",
            "remote: Total 1633 (delta 231), reused 225 (delta 211), pack-reused 1304\u001b[K\n",
            "Receiving objects: 100% (1633/1633), 198.37 MiB | 16.68 MiB/s, done.\n",
            "Resolving deltas: 100% (1073/1073), done.\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (2023.6.3)\n"
          ]
        }
      ],
      "source": [
        "import sys\n",
        "!test -d bertviz_repo && echo \"FYI: bertviz_repo directory already exists, to pull latest version uncomment this line: !rm -r bertviz_repo\"\n",
        "# !rm -r bertviz_repo # Uncomment if you need a clean pull from repo\n",
        "!test -d bertviz_repo || git clone https://github.com/jessevig/bertviz bertviz_repo\n",
        "if not 'bertviz_repo' in sys.path:\n",
        "  sys.path += ['bertviz_repo']\n",
        "!pip install regex"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nhuOe5aP0la4",
        "outputId": "c6e4d0d4-6856-4628-b9f8-9363a87bcabb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'bert-loves-chemistry'...\n",
            "remote: Enumerating objects: 1566, done.\u001b[K\n",
            "remote: Counting objects: 100% (202/202), done.\u001b[K\n",
            "remote: Compressing objects: 100% (111/111), done.\u001b[K\n",
            "remote: Total 1566 (delta 96), reused 91 (delta 91), pack-reused 1364\u001b[K\n",
            "Receiving objects: 100% (1566/1566), 55.35 MiB | 15.78 MiB/s, done.\n",
            "Resolving deltas: 100% (1000/1000), done.\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/seyonechithrananda/bert-loves-chemistry.git"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Imports and parameters setting"
      ],
      "metadata": {
        "id": "qKxh4VdYj7vz"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7x4trKiL3MPd"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import sklearn\n",
        "import torch\n",
        "\n",
        "from rdkit import Chem\n",
        "from transformers import AutoModelForMaskedLM, AutoTokenizer, pipeline, RobertaModel, RobertaTokenizer\n",
        "from typing import List\n",
        "\n",
        "# import MoleculeNet loaders from DeepChem\n",
        "from deepchem.molnet import load_bbbp, load_clearance, load_clintox, load_delaney, load_hiv, load_qm7, load_tox21\n",
        "\n",
        "# import MoleculeNet dataloder from bert-loves-chemistry fork\n",
        "%cd /content/bert-loves-chemistry\n",
        "from chemberta.utils.molnet_dataloader import load_molnet_dataset, write_molnet_dataset_for_chemprop"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# enter the location of the model\n",
        "project_name = \"ChemBERTA_HIV\"\n",
        "output_path = '/content/drive/MyDrive/COLAB_NOTEBOOKS/LLM_Fine_Tuning_Molecular_Properties_output'\n",
        "model_name = 'model_36'\n",
        "\n",
        "model_folder = os.path.join(output_path, model_name)\n",
        "evaluation_folder = os.path.join(output_path, model_name + '_evaluation')\n",
        "\n",
        "embeddings_folder = os.path.join(output_path, model_name + '_embeddings')\n",
        "if not os.path.exists(embeddings_folder):\n",
        "    os.makedirs(embeddings_folder)\n",
        "\n",
        "print(model_folder)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "boJmajpjkiAN",
        "outputId": "0770a05f-331f-467a-e9e1-964dfc8ecbf9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive/COLAB_NOTEBOOKS/LLM_Fine_Tuning_Molecular_Properties_output/model_36\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Loading predictions"
      ],
      "metadata": {
        "id": "noCr3UFTk02d"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# load the dataframe with predictions\n",
        "predictions_df_path = os.path.join(evaluation_folder, 'predictions_df.csv')\n",
        "predictions_df = pd.read_csv(predictions_df_path, index_col=0)\n",
        "predictions_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "R1__g6VE1LCE",
        "outputId": "6445239e-d87c-44ce-ec33-5db3a534648d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                text  labels  \\\n",
              "0  CN=C1OC2(c3ccccc3)C3=C(OC(=NC)N2C)C(=O)OC3(c2c...       0   \n",
              "1                      c1ccc2c(c1)[nH]c1c3ccccc3sc21       0   \n",
              "2              CN1C(=O)N(C)P(=O)(c2ccccc2)P1c1ccccc1       0   \n",
              "3                       O=C(c1ccccc1)N1OCCCc2ccccc21       0   \n",
              "4                 COC(=O)c1c2c(cc3nc4ccccc4nc13)OCO2       0   \n",
              "\n",
              "   class_0_probability  class_1_probability  y_pred  \n",
              "0             1.991554            -2.474177       0  \n",
              "1             1.659338            -2.097368       0  \n",
              "2             1.971669            -2.436386       0  \n",
              "3             1.928813            -2.344722       0  \n",
              "4             1.767895            -2.166682       0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-30dffe4b-a7a7-4eb3-9a09-1adf243c3614\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>labels</th>\n",
              "      <th>class_0_probability</th>\n",
              "      <th>class_1_probability</th>\n",
              "      <th>y_pred</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CN=C1OC2(c3ccccc3)C3=C(OC(=NC)N2C)C(=O)OC3(c2c...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.991554</td>\n",
              "      <td>-2.474177</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>c1ccc2c(c1)[nH]c1c3ccccc3sc21</td>\n",
              "      <td>0</td>\n",
              "      <td>1.659338</td>\n",
              "      <td>-2.097368</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>CN1C(=O)N(C)P(=O)(c2ccccc2)P1c1ccccc1</td>\n",
              "      <td>0</td>\n",
              "      <td>1.971669</td>\n",
              "      <td>-2.436386</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>O=C(c1ccccc1)N1OCCCc2ccccc21</td>\n",
              "      <td>0</td>\n",
              "      <td>1.928813</td>\n",
              "      <td>-2.344722</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>COC(=O)c1c2c(cc3nc4ccccc4nc13)OCO2</td>\n",
              "      <td>0</td>\n",
              "      <td>1.767895</td>\n",
              "      <td>-2.166682</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-30dffe4b-a7a7-4eb3-9a09-1adf243c3614')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-30dffe4b-a7a7-4eb3-9a09-1adf243c3614 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-30dffe4b-a7a7-4eb3-9a09-1adf243c3614');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-16aed1fa-a1af-4af3-a484-eb13ed954301\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-16aed1fa-a1af-4af3-a484-eb13ed954301')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const charts = await google.colab.kernel.invokeFunction(\n",
              "          'suggestCharts', [key], {});\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-16aed1fa-a1af-4af3-a484-eb13ed954301 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Pre-trained & fine-tined model: calculate the embeddings"
      ],
      "metadata": {
        "id": "JLepWHmTlKxP"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LCoYYv1DHllo"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "from simpletransformers.classification import ClassificationModel, ClassificationArgs\n",
        "from transformers import AutoTokenizer, AutoModel\n",
        "\n",
        "# Load the tokenizers and models\n",
        "tokenizer = AutoTokenizer.from_pretrained('DeepChem/ChemBERTa-77M-MLM')\n",
        "model_before = AutoModel.from_pretrained('DeepChem/ChemBERTa-77M-MLM')\n",
        "model_after = AutoModel.from_pretrained(model_folder)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def calculate_embeddings(df, model, tokenizer, output_path=None):\n",
        "    text_data = df['text'].tolist()\n",
        "    labels = df['labels'].tolist()\n",
        "    inputs = tokenizer(text_data, padding=True, truncation=True, return_tensors='pt', max_length=128)\n",
        "\n",
        "    with torch.no_grad():\n",
        "        embeddings = model(**inputs).last_hidden_state[:, 0, :].numpy()\n",
        "\n",
        "    if output_path:\n",
        "        np.save(output_path, embeddings)\n",
        "\n",
        "    return embeddings"
      ],
      "metadata": {
        "id": "gPvHn10Bn8tc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "id_string = 'test'\n",
        "embeddings_before_path = os.path.join(embeddings_folder, id_string + '_embeddings_before.npy')\n",
        "embeddings_after_path = os.path.join(embeddings_folder, id_string + '_embeddings_after.npy')\n",
        "\n",
        "test_embeddings_before = calculate_embeddings(predictions_df, model_before, tokenizer, output_path=embeddings_before_path)\n",
        "test_embeddings_after = calculate_embeddings(predictions_df, model_after, tokenizer, output_path=embeddings_after_path)"
      ],
      "metadata": {
        "id": "zquY23swrIXu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Embeddings: dimensionality reduction"
      ],
      "metadata": {
        "id": "nZMLe7-w187-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "test_embeddings_before = np.load(embeddings_before_path)\n",
        "test_embeddings_after = np.load(embeddings_after_path)\n",
        "\n",
        "test_embeddings_pca_before_path = os.path.join(embeddings_folder, 'test_embeddings_pca_before.npy')\n",
        "test_embeddings_pca_after_path = os.path.join(embeddings_folder, 'test_embeddings_pca_after.npy')"
      ],
      "metadata": {
        "id": "0Jv_861SxmKC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.decomposition import PCA\n",
        "\n",
        "# Perform PCA for dimensionality reduction\n",
        "pca = PCA()\n",
        "test_embeddings_pca_before = pca.fit_transform(test_embeddings_before)\n",
        "np.save(test_embeddings_pca_before_path, test_embeddings_pca_before)\n",
        "test_embeddings_pca_after = pca.fit_transform(test_embeddings_after)\n",
        "np.save(test_embeddings_pca_after_path, test_embeddings_pca_after)"
      ],
      "metadata": {
        "id": "63_TdIdnxOI3"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}