IDEFICS_2_Visual_q_and_a.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "Introducing Idefics2: A Powerful 8B Vision-Language Model for the community"
      ],
      "metadata": {
        "id": "Bly6ap6MluEq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%pip install -U transformers \\\n",
        "datasets==2.14.4 \\\n",
        "diffusers==0.20.0 \\\n",
        "accelerate==0.21.0 \\\n",
        "torch==2.0.1 \\\n",
        "torchvision==0.15.2 \\\n",
        "sentencepiece==0.1.99"
      ],
      "metadata": {
        "id": "DSsPzzc0lJ35"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Load the model"
      ],
      "metadata": {
        "id": "WfEBfqgilV_9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from transformers import IdeficsForVisionText2Text, AutoProcessor\n",
        "\n",
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "\n",
        "model_name = \"HuggingFaceM4/idefics-9b-instruct\"\n",
        "\n",
        "model = IdeficsForVisionText2Text.from_pretrained(model_name, torch_dtype=torch.bfloat16).to(device)\n",
        "\n",
        "\n",
        "processor = AutoProcessor.from_pretrained(model_name)\n",
        "\n",
        "# Generation args\n",
        "exit_condition = processor.tokenizer(\"<end_of_utterance>\", add_special_tokens=False).input_ids\n",
        "bad_words_ids = processor.tokenizer([\"<image>\", \"<fake_token_around_image>\"], add_special_tokens=False).input_ids"
      ],
      "metadata": {
        "id": "pfEt9tl2lFFW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Load the image"
      ],
      "metadata": {
        "id": "uaA56ZrcldCi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from PIL import Image\n",
        "img = Image.open(\"dog.jpg\")\n",
        "img"
      ],
      "metadata": {
        "id": "Uo1v_YGWlFLF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Provide the Prompt and generate the response"
      ],
      "metadata": {
        "id": "Gic4lINIllrA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = [\n",
        "   \"User:\",\n",
        "   img,\n",
        "   \"Describe this image.\"\n",
        "   \"Assistant:\",\n",
        "\n",
        "]\n",
        "\n",
        "inputs = processor(prompt, return_tensors=\"pt\").to(\"cuda\")\n",
        "generated_ids = model.generate(**inputs, eos_token_id = exit_condition, max_new_tokens=100, bad_words_ids=bad_words_ids)\n",
        "\n",
        "generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]\n",
        "print(generated_text)"
      ],
      "metadata": {
        "id": "gal5KpFFlFN3"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}