img_db_hash.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ddcb8a80-6125-47d9-9e03-e614ea8e17fc",
   "metadata": {},
   "source": [
    "### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "65846a34-2ffd-4c07-aff3-c42df0fbf30d",
   "metadata": {},
   "outputs": [],
   "source": [
    "work_dir=\"/scratch/cse/phd/anz198717/XC\"\n",
    "dataset=\"MM-AmazonTitles-300K\"\n",
    "corpus_dset=f\"{work_dir}/data/{dataset}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "da0cbb83-d18a-470d-8136-a16c9440e339",
   "metadata": {},
   "outputs": [],
   "source": [
    "import site\n",
    "site.addsitedir(f\"{work_dir}/programs/ExtremeMethods\")\n",
    "import xc.tools.build_from_msr as msr"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9cca8fc-b8d0-48ac-af6d-c35b155a72c1",
   "metadata": {},
   "source": [
    "## Only for MSR internal datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ed1e2a97-7ef3-425c-9bac-424bc430f8b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# args=f\"--in_dir {corpus_dset} --ot_dir {corpus_dset} \\\n",
    "#     --docs_input corpus_data.txt --lbls_input corpus_x_y.txt\"\n",
    "# sys.argv = f\"TOKEN {args}\".split()\n",
    "# print(args)\n",
    "# args = msr.setup()\n",
    "# lines = msr.build_docs(args)\n",
    "# msr.build_lbls(args, lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4619b8f7-c45d-4c4c-912c-93710345acd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir=f\"{corpus_dset}/images\"\n",
    "data_txt=f\"{corpus_dset}/raw_data\"\n",
    "img_path=f\"{corpus_dset}/img.bin\"\n",
    "os.makedirs(f\"{data_dir}\", exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bb1edb1-3154-40a2-9faa-22149abe08e7",
   "metadata": {},
   "source": [
    "## Building image database hash map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3092198e-94f5-4ed8-b6bb-021e812fc329",
   "metadata": {},
   "outputs": [],
   "source": [
    "from xc.libs.utils import pbar\n",
    "import scipy.sparse as sp\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "\n",
    "def read_ptrs(file):\n",
    "    ptrs = {}\n",
    "    curr = 0\n",
    "    if os.path.exists(file):\n",
    "        with open(file, \"rb\") as f:\n",
    "            for line in pbar(f):\n",
    "                uid, _ = line.split(b\"\\t\", 1)\n",
    "                uid = uid.decode('utf-8')\n",
    "                _ptrs = ptrs.get(uid, [])\n",
    "                _ptrs.append(curr)\n",
    "                ptrs[uid] = _ptrs\n",
    "                curr = f.tell()\n",
    "    return ptrs\n",
    "\n",
    "\n",
    "def build_sparse_mat(doc_map, dict_ptrs):\n",
    "    uids = list(map(lambda x: x.split(\"->\", 1)[0], pbar(open(doc_map,\"r\", encoding=\"latin1\"))))\n",
    "    ptrs, cols, rows, num_cols, num_rows = [], [], [], 0, 0\n",
    "    for row, uid in pbar(enumerate(uids), desc=\"buildling\"):\n",
    "        uid = uid.split(\",\")\n",
    "        # NOTE offesting it with 1 for sparse matrix\n",
    "        sub_ptrs = np.concatenate(\n",
    "            list(map(lambda x: dict_ptrs.get(x, [-1]), uid))) + 1\n",
    "        ptrs.append(sub_ptrs)\n",
    "        cols.append(np.arange(sub_ptrs.size) + num_cols)\n",
    "        rows.append(np.ones(sub_ptrs.size)*row)\n",
    "        num_cols += sub_ptrs.size\n",
    "    image_mat = sp.lil_matrix((len(uids), num_cols))\n",
    "    rows = np.concatenate(rows)\n",
    "    cols = np.concatenate(cols)\n",
    "    ptrs = np.concatenate(ptrs)\n",
    "    image_mat[rows, cols] = ptrs\n",
    "    image_mat = image_mat.tocsr()\n",
    "    return image_mat\n",
    "\n",
    "\n",
    "def save(data_path, img):\n",
    "    if img.nnz >0:\n",
    "        sp.save_npz(f\"{data_path}.img.bin.npz\", img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ce44e770-fb44-46f3-b170-1bf65d60b703",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "docs: 970237it [00:00, 1087165.72it/s]\n",
      "docs: 1305265it [00:01, 1074220.80it/s]\n",
      "970237it [00:00, 1296265.58it/s]\n",
      "buildling: 970237it [00:15, 64480.39it/s]\n",
      "1305265it [00:00, 1336204.46it/s]\n",
      "buildling: 1305265it [00:20, 63698.07it/s]\n"
     ]
    }
   ],
   "source": [
    "dict_ptrs = read_ptrs(img_path)\n",
    "\n",
    "tst_img = build_sparse_mat(f\"{data_txt}/test.raw.txt\", dict_ptrs)\n",
    "trn_img = build_sparse_mat(f\"{data_txt}/train.raw.txt\", dict_ptrs)\n",
    "lbl_img = build_sparse_mat(f\"{data_txt}/label.raw.txt\", dict_ptrs)\n",
    "\n",
    "save(f\"{data_dir}/test\", tst_img)\n",
    "save(f\"{data_dir}/label\", lbl_img)\n",
    "save(f\"{data_dir}/train\", trn_img)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}