tutorials/tutorial-query-performance-prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IR Lab Tutorial (Research Oriented): Query Performance Prediction\n",
    "\n",
    "This tutorial shows the concept of qpp."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparation: Install dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is only needed in Google Colab, in a dev container, everything should be installed already\n",
    "!pip3 install python-terrier tira"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyterrier as pt\n",
    "from tira.third_party_integrations import ensure_pyterrier_is_loaded\n",
    "from tira.rest_api_client import Client\n",
    "\n",
    "ensure_pyterrier_is_loaded()\n",
    "tira = Client()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Our Scenario\n",
    "\n",
    "We want to build a search engine to support web developers working with CSS.\n",
    "\n",
    "Our search engine has the following 3 documents:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def qpp_correlation_to_ground_truth(bm25, qpp, dataset, eval_metrics):\n",
    "    import pandas as pd\n",
    "    topics = dataset.get_topics()\n",
    "    df_eval = pt.Experiment([bm25], topics=topics, qrels=dataset.get_qrels(), eval_metrics=eval_metrics, perquery=True, names=['BM25'])\n",
    "    df_predictions = qpp(topics)\n",
    "    df_joined = pd.merge(df_eval, df_predictions, on=['qid'])\n",
    "    ret = []\n",
    "\n",
    "    for q in ['max-idf', 'avg-idf', 'scq', 'max-scq', 'avg-scq', 'var', 'max-var', 'avg-var', 'wig+10', 'nqc+100', 'smv+100']:\n",
    "        ret += [{'QPP Method': q, 'Pearson Correlation': df_joined[['value', q]].corr(method='pearson').iloc[0].to_dict()[q], 'Kendall': df_joined[['value', q]].corr(method='kendall').iloc[0].to_dict()[q], 'Spearman': df_joined[['value', q]].corr(method='spearman').iloc[0].to_dict()[q]}]\n",
    "\n",
    "    return pd.DataFrame(ret).sort_values('Pearson Correlation', ascending=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are multiple query fields available: ('title', 'description', 'narrative'). To use with pyterrier, provide variant or modify dataframe to add query column.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/pyterrier/pipelines.py:129: UserWarning: 1 topic(s) not found in qrels. Scores for these topics are given as NaN and should not contribute to averages.\n",
      "  warn(f'{backfill_count} topic(s) not found in qrels. Scores for these topics are given as NaN and should not contribute to averages.')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>QPP Method</th>\n",
       "      <th>Pearson Correlation</th>\n",
       "      <th>Kendall</th>\n",
       "      <th>Spearman</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>nqc+100</td>\n",
       "      <td>0.411153</td>\n",
       "      <td>0.317898</td>\n",
       "      <td>0.451018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>smv+100</td>\n",
       "      <td>0.391963</td>\n",
       "      <td>0.302243</td>\n",
       "      <td>0.429123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>wig+10</td>\n",
       "      <td>0.342660</td>\n",
       "      <td>0.240992</td>\n",
       "      <td>0.345846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>avg-var</td>\n",
       "      <td>0.302321</td>\n",
       "      <td>0.256582</td>\n",
       "      <td>0.369490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max-var</td>\n",
       "      <td>0.276160</td>\n",
       "      <td>0.233053</td>\n",
       "      <td>0.338973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max-scq</td>\n",
       "      <td>0.264181</td>\n",
       "      <td>0.221495</td>\n",
       "      <td>0.323365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>avg-idf</td>\n",
       "      <td>0.248902</td>\n",
       "      <td>0.168261</td>\n",
       "      <td>0.243340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>avg-scq</td>\n",
       "      <td>0.220149</td>\n",
       "      <td>0.157302</td>\n",
       "      <td>0.229591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max-idf</td>\n",
       "      <td>0.214187</td>\n",
       "      <td>0.175756</td>\n",
       "      <td>0.255930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>var</td>\n",
       "      <td>0.163394</td>\n",
       "      <td>0.163238</td>\n",
       "      <td>0.235878</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>scq</td>\n",
       "      <td>0.015893</td>\n",
       "      <td>0.011904</td>\n",
       "      <td>0.018180</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   QPP Method  Pearson Correlation   Kendall  Spearman\n",
       "9     nqc+100             0.411153  0.317898  0.451018\n",
       "10    smv+100             0.391963  0.302243  0.429123\n",
       "8      wig+10             0.342660  0.240992  0.345846\n",
       "7     avg-var             0.302321  0.256582  0.369490\n",
       "6     max-var             0.276160  0.233053  0.338973\n",
       "3     max-scq             0.264181  0.221495  0.323365\n",
       "1     avg-idf             0.248902  0.168261  0.243340\n",
       "4     avg-scq             0.220149  0.157302  0.229591\n",
       "0     max-idf             0.214187  0.175756  0.255930\n",
       "5         var             0.163394  0.163238  0.235878\n",
       "2         scq             0.015893  0.011904  0.018180"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = pt.get_dataset(\"irds:disks45/nocr/trec-robust-2004\")\n",
    "qpp = tira.pt.transform_queries('ir-benchmarks/qpptk/all-predictors', dataset)\n",
    "bm25 = tira.pt.from_submission('ir-benchmarks/tira-ir-starter/BM25 Re-Rank (tira-ir-starter-pyterrier)', dataset)\n",
    "\n",
    "qpp_correlation_to_ground_truth(bm25, qpp, dataset, ['ndcg_cut_10'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Download: 199kiB [00:00, 1.94MiB/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Download finished. Extract...\n",
      "Extraction finished:  /root/.tira/extracted_datasets/ir-benchmarks/msmarco-passage-trec-dl-2019-judged-20230107-training/\n",
      "There are multiple query fields available: ('text', 'title', 'query', 'description', 'narrative'). To use with pyterrier, provide variant or modify dataframe to add query column.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>QPP Method</th>\n",
       "      <th>Pearson Correlation</th>\n",
       "      <th>Kendall</th>\n",
       "      <th>Spearman</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max-var</td>\n",
       "      <td>0.466222</td>\n",
       "      <td>0.291485</td>\n",
       "      <td>0.418178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>nqc+100</td>\n",
       "      <td>0.373691</td>\n",
       "      <td>0.260822</td>\n",
       "      <td>0.370728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>wig+10</td>\n",
       "      <td>0.371033</td>\n",
       "      <td>0.229745</td>\n",
       "      <td>0.328437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max-idf</td>\n",
       "      <td>0.370412</td>\n",
       "      <td>0.231163</td>\n",
       "      <td>0.322145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>smv+100</td>\n",
       "      <td>0.363132</td>\n",
       "      <td>0.300778</td>\n",
       "      <td>0.432806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>avg-idf</td>\n",
       "      <td>0.328585</td>\n",
       "      <td>0.207548</td>\n",
       "      <td>0.314164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>avg-var</td>\n",
       "      <td>0.325339</td>\n",
       "      <td>0.189790</td>\n",
       "      <td>0.296492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max-scq</td>\n",
       "      <td>0.197687</td>\n",
       "      <td>0.134762</td>\n",
       "      <td>0.203420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>var</td>\n",
       "      <td>0.172750</td>\n",
       "      <td>0.067703</td>\n",
       "      <td>0.134124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>scq</td>\n",
       "      <td>0.025106</td>\n",
       "      <td>0.027747</td>\n",
       "      <td>0.038213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>avg-scq</td>\n",
       "      <td>-0.198025</td>\n",
       "      <td>-0.103219</td>\n",
       "      <td>-0.118642</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   QPP Method  Pearson Correlation   Kendall  Spearman\n",
       "6     max-var             0.466222  0.291485  0.418178\n",
       "9     nqc+100             0.373691  0.260822  0.370728\n",
       "8      wig+10             0.371033  0.229745  0.328437\n",
       "0     max-idf             0.370412  0.231163  0.322145\n",
       "10    smv+100             0.363132  0.300778  0.432806\n",
       "1     avg-idf             0.328585  0.207548  0.314164\n",
       "7     avg-var             0.325339  0.189790  0.296492\n",
       "3     max-scq             0.197687  0.134762  0.203420\n",
       "5         var             0.172750  0.067703  0.134124\n",
       "2         scq             0.025106  0.027747  0.038213\n",
       "4     avg-scq            -0.198025 -0.103219 -0.118642"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = pt.get_dataset(\"irds:ir-benchmarks/msmarco-passage-trec-dl-2019-judged-20230107-training\")\n",
    "qpp = tira.pt.transform_queries('ir-benchmarks/qpptk/all-predictors', 'msmarco-passage-trec-dl-2019-judged-20230107-training')\n",
    "bm25 = tira.pt.from_submission('ir-benchmarks/tira-ir-starter/BM25 Re-Rank (tira-ir-starter-pyterrier)', 'msmarco-passage-trec-dl-2019-judged-20230107-training')\n",
    "\n",
    "qpp_correlation_to_ground_truth(bm25, qpp, dataset, ['ndcg_cut_10'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Download from the Incubator: https://files.webis.de/data-in-production/data-research/tira-zenodo-dump-preparation/query-processors-in-progress/qpptk-all-predictors-trec-recent.zip\n",
      "\tThis is only used for last spot checks before archival to Zenodo.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Download: 100%|██████████| 223k/223k [00:00<00:00, 2.41MiB/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Download finished. Extract...\n",
      "Extraction finished:  /root/.tira/extracted_runs/ir-benchmarks/msmarco-passage-trec-dl-2020-judged-20230107-training/qpptk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Download: 642kiB [00:00, 4.76MiB/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Download finished. Extract...\n",
      "Extraction finished:  /root/.tira/extracted_runs/ir-benchmarks/msmarco-passage-trec-dl-2020-judged-20230107-training/tira-ir-starter\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Download: 235kiB [00:00, 2.30MiB/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Download finished. Extract...\n",
      "Extraction finished:  /root/.tira/extracted_datasets/ir-benchmarks/msmarco-passage-trec-dl-2020-judged-20230107-training/\n",
      "There are multiple query fields available: ('text', 'title', 'query', 'description', 'narrative'). To use with pyterrier, provide variant or modify dataframe to add query column.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>QPP Method</th>\n",
       "      <th>Pearson Correlation</th>\n",
       "      <th>Kendall</th>\n",
       "      <th>Spearman</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>wig+10</td>\n",
       "      <td>0.474703</td>\n",
       "      <td>0.313177</td>\n",
       "      <td>0.446778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>avg-var</td>\n",
       "      <td>0.213855</td>\n",
       "      <td>0.128626</td>\n",
       "      <td>0.225047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>avg-idf</td>\n",
       "      <td>0.203385</td>\n",
       "      <td>0.132821</td>\n",
       "      <td>0.181974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max-var</td>\n",
       "      <td>0.201037</td>\n",
       "      <td>0.103597</td>\n",
       "      <td>0.182545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>avg-scq</td>\n",
       "      <td>0.154104</td>\n",
       "      <td>0.104858</td>\n",
       "      <td>0.173512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>nqc+100</td>\n",
       "      <td>0.149108</td>\n",
       "      <td>0.131423</td>\n",
       "      <td>0.192456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>smv+100</td>\n",
       "      <td>0.105101</td>\n",
       "      <td>0.090877</td>\n",
       "      <td>0.135890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max-idf</td>\n",
       "      <td>0.074536</td>\n",
       "      <td>0.068116</td>\n",
       "      <td>0.111758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>var</td>\n",
       "      <td>0.067241</td>\n",
       "      <td>0.055925</td>\n",
       "      <td>0.101241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>scq</td>\n",
       "      <td>0.019864</td>\n",
       "      <td>0.019574</td>\n",
       "      <td>0.026949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max-scq</td>\n",
       "      <td>-0.131996</td>\n",
       "      <td>-0.108241</td>\n",
       "      <td>-0.150106</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   QPP Method  Pearson Correlation   Kendall  Spearman\n",
       "8      wig+10             0.474703  0.313177  0.446778\n",
       "7     avg-var             0.213855  0.128626  0.225047\n",
       "1     avg-idf             0.203385  0.132821  0.181974\n",
       "6     max-var             0.201037  0.103597  0.182545\n",
       "4     avg-scq             0.154104  0.104858  0.173512\n",
       "9     nqc+100             0.149108  0.131423  0.192456\n",
       "10    smv+100             0.105101  0.090877  0.135890\n",
       "0     max-idf             0.074536  0.068116  0.111758\n",
       "5         var             0.067241  0.055925  0.101241\n",
       "2         scq             0.019864  0.019574  0.026949\n",
       "3     max-scq            -0.131996 -0.108241 -0.150106"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = pt.get_dataset(\"irds:ir-benchmarks/msmarco-passage-trec-dl-2020-judged-20230107-training\")\n",
    "qpp = tira.pt.transform_queries('ir-benchmarks/qpptk/all-predictors', 'msmarco-passage-trec-dl-2020-judged-20230107-training')\n",
    "bm25 = tira.pt.from_submission('ir-benchmarks/tira-ir-starter/BM25 Re-Rank (tira-ir-starter-pyterrier)', 'msmarco-passage-trec-dl-2020-judged-20230107-training')\n",
    "\n",
    "qpp_correlation_to_ground_truth(bm25, qpp, dataset, ['ndcg_cut_10'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are multiple query fields available: ('title', 'description', 'narrative'). To use with pyterrier, provide variant or modify dataframe to add query column.\n",
      "/root/.ir_datasets/touche/2020/task-1/qrels.qrels\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>QPP Method</th>\n",
       "      <th>Pearson Correlation</th>\n",
       "      <th>Kendall</th>\n",
       "      <th>Spearman</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>avg-idf</td>\n",
       "      <td>0.396869</td>\n",
       "      <td>0.291332</td>\n",
       "      <td>0.433549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>smv+100</td>\n",
       "      <td>0.303901</td>\n",
       "      <td>0.159763</td>\n",
       "      <td>0.226448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max-scq</td>\n",
       "      <td>0.261866</td>\n",
       "      <td>0.145881</td>\n",
       "      <td>0.204320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>avg-scq</td>\n",
       "      <td>0.226422</td>\n",
       "      <td>0.178558</td>\n",
       "      <td>0.250134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>nqc+100</td>\n",
       "      <td>0.213075</td>\n",
       "      <td>0.113628</td>\n",
       "      <td>0.166977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>wig+10</td>\n",
       "      <td>0.172826</td>\n",
       "      <td>0.113628</td>\n",
       "      <td>0.169223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max-idf</td>\n",
       "      <td>0.134577</td>\n",
       "      <td>0.139077</td>\n",
       "      <td>0.174775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max-var</td>\n",
       "      <td>0.048378</td>\n",
       "      <td>0.051513</td>\n",
       "      <td>0.084354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>scq</td>\n",
       "      <td>0.016478</td>\n",
       "      <td>0.009398</td>\n",
       "      <td>0.037010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>avg-var</td>\n",
       "      <td>0.001100</td>\n",
       "      <td>0.041863</td>\n",
       "      <td>0.057020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>var</td>\n",
       "      <td>-0.028511</td>\n",
       "      <td>-0.026485</td>\n",
       "      <td>-0.022410</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   QPP Method  Pearson Correlation   Kendall  Spearman\n",
       "1     avg-idf             0.396869  0.291332  0.433549\n",
       "10    smv+100             0.303901  0.159763  0.226448\n",
       "3     max-scq             0.261866  0.145881  0.204320\n",
       "4     avg-scq             0.226422  0.178558  0.250134\n",
       "9     nqc+100             0.213075  0.113628  0.166977\n",
       "8      wig+10             0.172826  0.113628  0.169223\n",
       "0     max-idf             0.134577  0.139077  0.174775\n",
       "6     max-var             0.048378  0.051513  0.084354\n",
       "2         scq             0.016478  0.009398  0.037010\n",
       "7     avg-var             0.001100  0.041863  0.057020\n",
       "5         var            -0.028511 -0.026485 -0.022410"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = pt.get_dataset(\"irds:argsme/2020-04-01/touche-2020-task-1\")\n",
    "qpp = tira.pt.transform_queries('ir-benchmarks/qpptk/all-predictors', dataset)\n",
    "bm25 = tira.pt.from_submission('ir-benchmarks/tira-ir-starter/BM25 Re-Rank (tira-ir-starter-pyterrier)', dataset)\n",
    "\n",
    "qpp_correlation_to_ground_truth(bm25, qpp, dataset, ['ndcg_cut_10'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Download from the Incubator: https://files.webis.de/data-in-production/data-research/tira-zenodo-dump-preparation/query-processors-in-progress/qpptk-all-predictors-clef-labs.zip\n",
      "\tThis is only used for last spot checks before archival to Zenodo.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Download: 100%|██████████| 969k/969k [00:00<00:00, 6.39MiB/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Download finished. Extract...\n",
      "Extraction finished:  /root/.tira/extracted_runs/ir-benchmarks/argsme-touche-2021-task-1-20230209-training/qpptk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Download: 1.03MiB [00:00, 1.16MiB/s]\n",
      "[INFO] [starting] opening zip file\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Download finished. Extract...\n",
      "Extraction finished:  /root/.tira/extracted_runs/ir-benchmarks/argsme-touche-2021-task-1-20230209-training/tira-ir-starter\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[INFO] [starting] https://zenodo.org/record/6798216/files/topics-task-1-only-titles-2021.zip\n",
      "[INFO] [finished] https://zenodo.org/record/6798216/files/topics-task-1-only-titles-2021.zip: [00:00] [1.35kB] [4.40MB/s]\n",
      "[INFO] [finished] opening zip file [344ms]                                                            \n",
      "[INFO] [starting] https://zenodo.org/record/6798216/files/touche-task1-51-100-relevance.qrels\n",
      "[INFO] [finished] https://zenodo.org/record/6798216/files/touche-task1-51-100-relevance.qrels: [00:00] [100kB] [1.02MB/s]\n",
      "[INFO] [starting] https://zenodo.org/record/6798216/files/touche-task1-51-100-quality.qrels            \n",
      "[INFO] [finished] https://zenodo.org/record/6798216/files/touche-task1-51-100-quality.qrels: [00:00] [99.7kB] [828kB/s]\n",
      "                                                                                                     \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are multiple qrel fields available: ['relevance', 'quality']. Defaulting to \"relevance\", but to use a different one, supply variant\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>QPP Method</th>\n",
       "      <th>Pearson Correlation</th>\n",
       "      <th>Kendall</th>\n",
       "      <th>Spearman</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max-var</td>\n",
       "      <td>0.270399</td>\n",
       "      <td>0.111834</td>\n",
       "      <td>0.171912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>wig+10</td>\n",
       "      <td>0.263305</td>\n",
       "      <td>0.226122</td>\n",
       "      <td>0.345882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>avg-scq</td>\n",
       "      <td>0.237657</td>\n",
       "      <td>0.168980</td>\n",
       "      <td>0.259256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max-scq</td>\n",
       "      <td>0.227286</td>\n",
       "      <td>0.161212</td>\n",
       "      <td>0.248397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>avg-var</td>\n",
       "      <td>0.219205</td>\n",
       "      <td>0.128163</td>\n",
       "      <td>0.199232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>nqc+100</td>\n",
       "      <td>0.152029</td>\n",
       "      <td>0.067755</td>\n",
       "      <td>0.104346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>smv+100</td>\n",
       "      <td>0.145271</td>\n",
       "      <td>0.062857</td>\n",
       "      <td>0.084658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>var</td>\n",
       "      <td>0.088366</td>\n",
       "      <td>0.044898</td>\n",
       "      <td>0.064106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>scq</td>\n",
       "      <td>0.000595</td>\n",
       "      <td>-0.041633</td>\n",
       "      <td>-0.060168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>avg-idf</td>\n",
       "      <td>-0.017020</td>\n",
       "      <td>-0.023673</td>\n",
       "      <td>-0.038271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max-idf</td>\n",
       "      <td>-0.201394</td>\n",
       "      <td>-0.153051</td>\n",
       "      <td>-0.202617</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   QPP Method  Pearson Correlation   Kendall  Spearman\n",
       "6     max-var             0.270399  0.111834  0.171912\n",
       "8      wig+10             0.263305  0.226122  0.345882\n",
       "4     avg-scq             0.237657  0.168980  0.259256\n",
       "3     max-scq             0.227286  0.161212  0.248397\n",
       "7     avg-var             0.219205  0.128163  0.199232\n",
       "9     nqc+100             0.152029  0.067755  0.104346\n",
       "10    smv+100             0.145271  0.062857  0.084658\n",
       "5         var             0.088366  0.044898  0.064106\n",
       "2         scq             0.000595 -0.041633 -0.060168\n",
       "1     avg-idf            -0.017020 -0.023673 -0.038271\n",
       "0     max-idf            -0.201394 -0.153051 -0.202617"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = pt.get_dataset(\"irds:argsme/2020-04-01/touche-2021-task-1\")\n",
    "qpp = tira.pt.transform_queries('ir-benchmarks/qpptk/all-predictors', dataset)\n",
    "bm25 = tira.pt.from_submission('ir-benchmarks/tira-ir-starter/BM25 Re-Rank (tira-ir-starter-pyterrier)', dataset)\n",
    "\n",
    "qpp_correlation_to_ground_truth(bm25, qpp, dataset, ['ndcg_cut_10'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}