remove unused code in nanollava notebook (openvinotoolkit#2538)

notebooks/nano-llava-multimodal-chatbot/nano-llava-multimodal-chatbot.ipynb

@@ -15,7 +15,6 @@
     "\n",
     "- [Prerequisites](#Prerequisites)\n",
     "- [Select Model](#Select-Model)\n",
-    "- [Download PyTorch model](#Download-PyTorch-model)\n",
     "- [Convert and Optimize model](#Convert-and-Optimize-model)\n",
     "    - [Convert model to OpenVINO IR format](#Convert-model-to-OpenVINO-IR-format)\n",
     "    - [Compress Model weights to 4 and 8 bits using NNCF](#Compress-Model-weights-to-4-and-8-bits-using-NNCF)\n",
@@ -56,7 +55,7 @@
    },
    "outputs": [],
    "source": [
-    "%pip install -q \"torch>=2.1\" \"transformers>=4.40\" \"accelerate\" \"pillow\" \"gradio>=4.26\" \"tqdm\" --extra-index-url https://download.pytorch.org/whl/cpu\n",
+    "%pip install -q \"torch>=2.1\" \"transformers>=4.45\" \"accelerate\" \"pillow\" \"gradio>=4.26\" \"tqdm\" --extra-index-url https://download.pytorch.org/whl/cpu\n",
     "%pip install -q \"nncf>=2.13\"\n",
     "%pip install -q -U --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly \"openvino-tokenizers[transformers]\" \"openvino>=2024.4.0\"\n",
     "%pip install -q \"git+https://github.com/huggingface/optimum-intel.git\""
@@ -75,12 +74,17 @@
     "import requests\n",
     "\n",
     "helper_file = Path(\"ov_nano_llava_helper.py\")\n",
+    "cmd_helper_file = Path(\"cmd_helper.py\")\n",
     "\n",
     "if not helper_file.exists():\n",
     "    r = requests.get(\n",
     "        url=f\"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/notebooks/nano-llava-multimodal-chatbot/{helper_file.name}\"\n",
     "    )\n",
-    "    helper_file.open(\"w\").write(r.text)"
+    "    helper_file.open(\"w\").write(r.text)\n",
+    "\n",
+    "if not cmd_helper_file.exists():\n",
+    "    r = requests.get(url=f\"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/{cmd_helper_file.name}\")\n",
+    "    cmd_helper_file.open(\"w\").write(r.text)"
    ]
   },
   {
@@ -155,14 +159,11 @@
    },
    "outputs": [],
    "source": [
-    "from ov_nano_llava_helper import download_original_model, converted_model_exists, copy_model_files\n",
+    "from ov_nano_llava_helper import converted_model_exists, copy_model_files\n",
     "\n",
     "model_id = model_dropdown.value\n",
     "model_dir = Path(model_id.split(\"/\")[-1])\n",
-    "ov_model_dir = Path(\"ov_\" + model_dir.name) / \"FP16\"\n",
-    "\n",
-    "if not converted_model_exists(ov_model_dir):\n",
-    "    download_original_model(model_id, model_dir)"
+    "ov_model_dir = Path(\"ov_\" + model_dir.name) / \"FP16\""
    ]
   },
   {
@@ -207,8 +208,10 @@
    },
    "outputs": [],
    "source": [
+    "from cmd_helper import optimum_cli\n",
+    "\n",
     "if not converted_model_exists(ov_model_dir):\n",
-    "    !optimum-cli export openvino --model {model_id} --task image-text-to-text --trust-remote-code --weight-format fp16 {ov_model_dir}"
+    "    optimum_cli(model_id, ov_model_dir, additional_args={\"task\": \"image-text-to-text\", \"trust-remote-code\": \"\", \"weight-format\": \"fp16\"})"
    ]
   },
   {
@@ -573,10 +576,11 @@
    "source": [
     "import requests\n",
     "\n",
-    "r = requests.get(\n",
-    "    url=\"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py\",\n",
-    ")\n",
-    "open(\"notebook_utils.py\", \"w\").write(r.text)\n",
+    "if not Path(\"notebook_utils.py\").exists():\n",
+    "    r = requests.get(\n",
+    "        url=\"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py\",\n",
+    "    )\n",
+    "    open(\"notebook_utils.py\", \"w\").write(r.text)\n",
     "\n",
     "from notebook_utils import device_widget\n",
     "\n",
@@ -661,8 +665,14 @@
     "\n",
     "messages = [{\"role\": \"user\", \"content\": f\"<image>\\n{prompt}\"}]\n",
     "text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n",
-    "url = \"https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/8bf7d9f2-018a-4498-bec4-55f17c273ecc\"\n",
-    "image = Image.open(requests.get(url, stream=True).raw)\n",
+    "test_image = Path(\"nanollava.png\")\n",
+    "\n",
+    "if not test_image.exists():\n",
+    "    url = \"https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/8bf7d9f2-018a-4498-bec4-55f17c273ecc\"\n",
+    "    image = Image.open(requests.get(url, stream=True).raw)\n",
+    "    image.save(test_image)\n",
+    "else:\n",
+    "    image = Image.open(test_image)\n",
     "image_tensor = process_images(image, None, processor)\n",
     "input_ids, attention_mask = process_text_input(text, tokenizer)\n",
     "\n",

notebooks/nano-llava-multimodal-chatbot/ov_nano_llava_helper.py

@@ -1,35 +1,9 @@
 from pathlib import Path
 import shutil
-from huggingface_hub import snapshot_download
 import torch
 from PIL import Image
 
 
-def download_original_model(model_id, model_local_dir):
-    if not model_local_dir.exists():
-        snapshot_download(repo_id=model_id, local_dir=model_local_dir)
-
-    modeling_file = model_local_dir / "modeling_llava_qwen2.py"
-    orig_modeling_file = model_local_dir / f"orig_{modeling_file.name}"
-
-    # model code depends from flash_attn package that may be problematic to load. Patch model code for avoiding import of this package
-    if not orig_modeling_file.exists():
-        modeling_file.rename(orig_modeling_file)
-    with orig_modeling_file.open("r") as f:
-        content = f.read()
-    replacement_lines = [
-        ("from flash_attn import flash_attn_func, flash_attn_varlen_func", ""),
-        ("from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input", ""),
-        (' _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters)', "pass"),
-    ]
-
-    for replace_pair in replacement_lines:
-        content = content.replace(*replace_pair)
-
-    with modeling_file.open("w") as f:
-        f.write(content)
-
-
 def converted_model_exists(model_dir):
     for file_name in ["openvino_language_model.xml", "openvino_text_embeddings_model.xml", "openvino_vision_embeddings_model.xml"]:
         if not (Path(model_dir) / file_name).exists() or not (Path(model_dir) / file_name.replace(".bin")).exists():