experiments/create_benchmark_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from SPARQLWrapper import SPARQLWrapper, JSON\n",
    "from IPython.display import display, Markdown\n",
    "import base64\n",
    "from PIL import Image\n",
    "import io\n",
    "import json\n",
    "import subprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_video_entities(offset=0, limit=10):\n",
    "    endpoint = \"http://localhost:7200/repositories/kgrc4si\"\n",
    "    sparql = \"\"\"\n",
    "    PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>\n",
    "    PREFIX mssn: <http://mssn.sigappfr.org/mssn/>\n",
    "    PREFIX vh2kg: <http://kgrc4si.home.kg/virtualhome2kg/ontology/>\n",
    "    PREFIX ex: <http://kgrc4si.home.kg/virtualhome2kg/instance/>\n",
    "    PREFIX ac: <http://kgrc4si.home.kg/virtualhome2kg/ontology/action/>\n",
    "    PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>\n",
    "    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>\n",
    "    PREFIX sosa: <http://www.w3.org/ns/sosa/>\n",
    "    select distinct ?activity ?event ?prevObjectName ?action ?object ?frame ?targetObject\n",
    "    where {\n",
    "        ?prevEvent vh2kg:hasVideoSegment ?visualSegment .\n",
    "        ?visualSegment mssn:hasMediaDescriptor ?frame_r ;\n",
    "            vh2kg:hasEndFrame ?endFrame .\n",
    "        ?frame_r vh2kg:frameNumber ?frame .\n",
    "        filter(?frame < ?endFrame && ?frame >= (?endFrame - 5))\n",
    "        ?frame_r mssn:hasMediaDescriptor ?bbox2d_1 .\n",
    "        ?bbox2d_1 vh2kg:is2DbboxOf ?prevObject ;\n",
    "                rdfs:label ?prevObjectName .\n",
    "        \n",
    "        ?activity vh2kg:hasEvent ?event .\n",
    "        ?prevEvent vh2kg:action ac:grab ;\n",
    "            vh2kg:mainObject ?prevObject ;\n",
    "            vh2kg:nextEvent ?prevEvent2 .\n",
    "        ?prevEvent2 vh2kg:action ac:walk ;\n",
    "            vh2kg:nextEvent ?event .\n",
    "        ?event vh2kg:action ?action ;\n",
    "            vh2kg:mainObject ?object .\n",
    "        optional { ?event vh2kg:targetObject ?targetObject }\n",
    "        \n",
    "        BIND(RAND() AS ?rand)\n",
    "    } order by ?rand offset \"\"\" + str(offset) +  \"\"\" limit \"\"\" + str(limit) + \"\"\"\n",
    "    \"\"\"\n",
    "    wrapper = SPARQLWrapper(endpoint)\n",
    "    wrapper.setQuery(sparql)\n",
    "    wrapper.setReturnFormat(JSON)\n",
    "    results = wrapper.query().convert()\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_video_base64(activity_ent):\n",
    "    endpoint = \"http://localhost:7200/repositories/kgrc4si\"\n",
    "    sparql = \"\"\"\n",
    "        PREFIX vh2kg: <http://kgrc4si.home.kg/virtualhome2kg/ontology/>\n",
    "        PREFIX sosa: <http://www.w3.org/ns/sosa/>\n",
    "        PREFIX ex: <http://kgrc4si.home.kg/virtualhome2kg/instance/>\n",
    "        SELECT ?base64 ?frameRate WHERE {\n",
    "            ?video vh2kg:isVideoOf <\"\"\" + activity_ent + \"\"\"> ;\n",
    "        sosa:madeBySensor ?camera .\n",
    "    ?video vh2kg:video ?base64 ;\n",
    "           vh2kg:frameRate ?frameRate .\n",
    "    filter(?camera != ex:camera2_scene1 && ?camera != ex:camera2_scene2 && ?camera != ex:camera2_scene3 && ?camera != ex:camera2_scene4 && ?camera != ex:camera2_scene5 && ?camera != ex:camera2_scene6 && ?camera != ex:camera2_scene7)\n",
    "        } order by ?base64 limit 1\n",
    "    \"\"\"\n",
    "    wrapper = SPARQLWrapper(endpoint)\n",
    "    wrapper.setQuery(sparql)\n",
    "    wrapper.setReturnFormat(JSON)\n",
    "    results = wrapper.query().convert()\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def remove_url_and_id(object_ent):\n",
    "    object_name = object_ent.replace(\"http://kgrc4si.home.kg/virtualhome2kg/instance/\", \"\")\n",
    "    object_name = re.sub(r'\\d+_scene\\d', '', object_name)\n",
    "    return object_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "few_shot_size = 10\n",
    "test_size = 100\n",
    "\n",
    "results = get_video_entities(offset=0, limit=few_shot_size+test_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "bindings = results[\"results\"][\"bindings\"]\n",
    "image_list = []\n",
    "\n",
    "few_shot_samples = []\n",
    "ground_truth = []\n",
    "i = 0\n",
    "for result in bindings:\n",
    "    activity_ent = result[\"activity\"][\"value\"]\n",
    "    # video_ent = result[\"video\"][\"value\"]\n",
    "    prev_object_name = result[\"prevObjectName\"][\"value\"]\n",
    "    action_ent = result[\"action\"][\"value\"]\n",
    "    action_name = action_ent.replace(\"http://kgrc4si.home.kg/virtualhome2kg/ontology/action/\", \"\")\n",
    "    object_ent = result[\"object\"][\"value\"]\n",
    "    object_name = remove_url_and_id(object_ent)\n",
    "    target_object_ent = None\n",
    "    target_object_name = None\n",
    "    if \"targetObject\" in result:\n",
    "        target_object_ent = result[\"targetObject\"][\"value\"]\n",
    "        target_object_name = remove_url_and_id(target_object_ent)\n",
    "    \n",
    "    answer = \"The person will \" + action_name\n",
    "    if action_name == \"sit\":\n",
    "        answer += \" on\"\n",
    "    elif action_name == \"walk\":\n",
    "        answer += \" to\"\n",
    "    answer += \" the \" + object_name\n",
    "    if target_object_ent is not None:\n",
    "        if action_name == \"put\":\n",
    "            answer += \" on the \" + target_object_name\n",
    "        elif action_name == \"pour\":\n",
    "            answer += \" into the \" + target_object_name\n",
    "        else:\n",
    "            answer += \" to the \" + target_object_name\n",
    "    answer += \".\"\n",
    "    \n",
    "    # action_ent = result[\"action\"][\"value\"]\n",
    "    frame = result[\"frame\"][\"value\"]\n",
    "    video_results = get_video_base64(activity_ent)\n",
    "    video_base64 = video_results[\"results\"][\"bindings\"][0][\"base64\"][\"value\"]\n",
    "    frame_rate = video_results[\"results\"][\"bindings\"][0][\"frameRate\"][\"value\"]\n",
    "    time = (float(frame))/float(frame_rate)\n",
    "\n",
    "\n",
    "    # base64デコードしてバイナリデータを取得\n",
    "    decoded_data = base64.b64decode(video_base64)\n",
    "\n",
    "    # ファイルに書き込み\n",
    "    with open(\"./tmp/decoded\" + str(i) + \".mp4\", \"wb\") as file:\n",
    "        file.write(decoded_data)\n",
    "\n",
    "    # Command to execute ffmpeg\n",
    "    print(time)\n",
    "    command = \"ffmpeg -i ./tmp/decoded\" + str(i) + \".mp4 -r 30 -ss \" + str(time) + \" -frames:v 1  ./test/\" + str(i) + \".png\"\n",
    "\n",
    "    # Execute the command\n",
    "    subprocess.run(command, shell=True)\n",
    "\n",
    "\n",
    "    if i >= test_size:\n",
    "         few_shot_samples.append({\n",
    "            \"activity_entity\": activity_ent,\n",
    "            \"prev_object_name\": prev_object_name, \n",
    "            \"action\": action_name,\n",
    "            \"object\": object_name,\n",
    "            \"target_object\": target_object_name,\n",
    "            \"frame\": frame,\n",
    "            \"answer\": answer\n",
    "        })\n",
    "    else:\n",
    "        ground_truth.append({\n",
    "            \"activity_entity\": activity_ent,\n",
    "            \"prev_object_name\": prev_object_name, \n",
    "            \"action\": action_name,\n",
    "            \"object\": object_name,\n",
    "            \"target_object\": target_object_name,\n",
    "            \"frame\": frame,\n",
    "            \"answer\": answer\n",
    "        })\n",
    "\n",
    "    # Convert the ground_truth variable to a JSON list\n",
    "    few_shot_samples_json = json.dumps(few_shot_samples)\n",
    "    ground_truth_json = json.dumps(ground_truth)\n",
    "\n",
    "    # Save the JSON list to a file\n",
    "    with open('test/few_shot_samples.json', 'w') as file:\n",
    "        file.write(few_shot_samples_json)\n",
    "\n",
    "    with open('test/ground_truth.json', 'w') as file:\n",
    "        file.write(ground_truth_json)\n",
    "\n",
    "    i += 1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}