pytroch-llm-science-exam.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"source":"<a href=\"https://www.kaggle.com/code/averma111/pytroch-llm-science-exam?scriptVersionId=140045608\" target=\"_blank\"><img align=\"left\" alt=\"Kaggle\" title=\"Open in Kaggle\" src=\"https://kaggle.com/static/images/open-in-kaggle.svg\"></a>","metadata":{},"cell_type":"markdown"},{"cell_type":"code","source":"from transformers import pipeline, set_seed\nfrom datasets import load_dataset, load_from_disk\nimport matplotlib.pyplot as plt\nfrom datasets import load_dataset\nimport pandas as pd\nfrom datasets import load_dataset, load_metric\nfrom datasets import Dataset\nfrom transformers import AutoTokenizer\nfrom transformers.tokenization_utils_base import PreTrainedTokenizerBase, PaddingStrategy\nfrom transformers import AutoTokenizer, DebertaV2ForMultipleChoice,TrainingArguments, Trainer\nfrom transformers import AutoModelForMultipleChoice, TrainingArguments, Trainer\nfrom typing import Optional, Union\nfrom tqdm import tqdm\nimport torch,os\nfrom dataclasses import dataclass\nfrom transformers import AutoTokenizer, DebertaV2Model\nfrom typing import Optional, Union\nimport numpy as np\nfrom colorama import Fore, Back, Style\nfrom tqdm.notebook import tqdm\nimport gc\nfrom transformers.tokenization_utils_base import PreTrainedTokenizerBase, PaddingStrategy\nfrom transformers import AutoModelForMultipleChoice, TrainingArguments, Trainer, AutoModel, EarlyStoppingCallback\nfrom sklearn.model_selection import KFold\nfrom IPython.display import IFrame\n\nimport warnings\nwarnings.filterwarnings('ignore')\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \n ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-16T03:53:06.757845Z","iopub.execute_input":"2023-08-16T03:53:06.758306Z","iopub.status.idle":"2023-08-16T03:53:06.777687Z","shell.execute_reply.started":"2023-08-16T03:53:06.758259Z","shell.execute_reply":"2023-08-16T03:53:06.776603Z"},"trusted":true},"execution_count":22,"outputs":[{"name":"stdout","text":"/kaggle/input/kaggle-llm-science-exam/sample_submission.csv\n/kaggle/input/kaggle-llm-science-exam/train.csv\n/kaggle/input/kaggle-llm-science-exam/test.csv\n/kaggle/input/additional-train-data-for-llm-science-exam/6000_train_examples.csv\n/kaggle/input/additional-train-data-for-llm-science-exam/extra_train_set.csv\n","output_type":"stream"}]},{"cell_type":"code","source":"class DataAcquisition:\n    \n    def __init__(self):\n        pass\n    \n    def prepare_data(self):\n        path = '/kaggle/input/kaggle-llm-science-exam/train.csv'\n        df = pd.read_csv(\n            path, \n            sep=',', \n            low_memory=False\n        )\n        \n        X = df.copy()\n        return X\n    \n    def prepare_extra_data(self):\n        path = '/kaggle/input/additional-train-data-for-llm-science-exam/extra_train_set.csv'\n        df = pd.read_csv(\n            path, \n            sep=',', \n            low_memory=False\n        )\n        \n        X = df.copy()\n        return X\n    \n    \n    def prepare_data_test(self):\n        path = '/kaggle/input/kaggle-llm-science-exam/test.csv'\n        df = pd.read_csv(\n            path, \n            sep=',', \n            low_memory=False\n        )\n        \n        X = df.copy()\n        return X\n    \n    \n    def new_features(self,df):\n        pass\n    \n    \n    \n    def preprocessing_dataset(self,df):\n        return  Dataset.from_pandas(df)\n        \n    def preprocessing_test(self,df):\n        for idx in tqdm(range(df.shape[0]) , total = df.shape[0]):\n            df[\"answer\"][idx] = df[df[\"answer\"][idx]][idx]\n        return df\n    \n        \nacq = DataAcquisition()\n\ntrain_df = acq.prepare_data()\nextra_df = acq.prepare_extra_data()\ntest_df = acq.prepare_data_test()","metadata":{"execution":{"iopub.status.busy":"2023-08-16T03:53:06.780154Z","iopub.execute_input":"2023-08-16T03:53:06.780862Z","iopub.status.idle":"2023-08-16T03:53:06.826665Z","shell.execute_reply.started":"2023-08-16T03:53:06.780828Z","shell.execute_reply":"2023-08-16T03:53:06.825049Z"},"trusted":true},"execution_count":23,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-16T03:53:06.827997Z","iopub.execute_input":"2023-08-16T03:53:06.828374Z","iopub.status.idle":"2023-08-16T03:53:06.844853Z","shell.execute_reply.started":"2023-08-16T03:53:06.828343Z","shell.execute_reply":"2023-08-16T03:53:06.843385Z"},"trusted":true},"execution_count":24,"outputs":[{"execution_count":24,"output_type":"execute_result","data":{"text/plain":"   id                                             prompt  \\\n0   0  Which of the following statements accurately d...   \n1   1  Which of the following is an accurate definiti...   \n2   2  Which of the following statements accurately d...   \n3   3  What is the significance of regularization in ...   \n4   4  Which of the following statements accurately d...   \n\n                                                   A  \\\n0  MOND is a theory that reduces the observed mis...   \n1  Dynamic scaling refers to the evolution of sel...   \n2  The triskeles symbol was reconstructed as a fe...   \n3  Regularizing the mass-energy of an electron wi...   \n4  The angular spacing of features in the diffrac...   \n\n                                                   B  \\\n0  MOND is a theory that increases the discrepanc...   \n1  Dynamic scaling refers to the non-evolution of...   \n2  The triskeles symbol is a representation of th...   \n3  Regularizing the mass-energy of an electron wi...   \n4  The angular spacing of features in the diffrac...   \n\n                                                   C  \\\n0  MOND is a theory that explains the missing bar...   \n1  Dynamic scaling refers to the evolution of sel...   \n2  The triskeles symbol is a representation of a ...   \n3  Regularizing the mass-energy of an electron wi...   \n4  The angular spacing of features in the diffrac...   \n\n                                                   D  \\\n0  MOND is a theory that reduces the discrepancy ...   \n1  Dynamic scaling refers to the non-evolution of...   \n2  The triskeles symbol represents three interloc...   \n3  Regularizing the mass-energy of an electron wi...   \n4  The angular spacing of features in the diffrac...   \n\n                                                   E answer  \n0  MOND is a theory that eliminates the observed ...      D  \n1  Dynamic scaling refers to the evolution of sel...      A  \n2  The triskeles symbol is a representation of th...      A  \n3  Regularizing the mass-energy of an electron wi...      C  \n4  The angular spacing of features in the diffrac...      D  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>prompt</th>\n      <th>A</th>\n      <th>B</th>\n      <th>C</th>\n      <th>D</th>\n      <th>E</th>\n      <th>answer</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>Which of the following statements accurately d...</td>\n      <td>MOND is a theory that reduces the observed mis...</td>\n      <td>MOND is a theory that increases the discrepanc...</td>\n      <td>MOND is a theory that explains the missing bar...</td>\n      <td>MOND is a theory that reduces the discrepancy ...</td>\n      <td>MOND is a theory that eliminates the observed ...</td>\n      <td>D</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>Which of the following is an accurate definiti...</td>\n      <td>Dynamic scaling refers to the evolution of sel...</td>\n      <td>Dynamic scaling refers to the non-evolution of...</td>\n      <td>Dynamic scaling refers to the evolution of sel...</td>\n      <td>Dynamic scaling refers to the non-evolution of...</td>\n      <td>Dynamic scaling refers to the evolution of sel...</td>\n      <td>A</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>Which of the following statements accurately d...</td>\n      <td>The triskeles symbol was reconstructed as a fe...</td>\n      <td>The triskeles symbol is a representation of th...</td>\n      <td>The triskeles symbol is a representation of a ...</td>\n      <td>The triskeles symbol represents three interloc...</td>\n      <td>The triskeles symbol is a representation of th...</td>\n      <td>A</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>What is the significance of regularization in ...</td>\n      <td>Regularizing the mass-energy of an electron wi...</td>\n      <td>Regularizing the mass-energy of an electron wi...</td>\n      <td>Regularizing the mass-energy of an electron wi...</td>\n      <td>Regularizing the mass-energy of an electron wi...</td>\n      <td>Regularizing the mass-energy of an electron wi...</td>\n      <td>C</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>Which of the following statements accurately d...</td>\n      <td>The angular spacing of features in the diffrac...</td>\n      <td>The angular spacing of features in the diffrac...</td>\n      <td>The angular spacing of features in the diffrac...</td>\n      <td>The angular spacing of features in the diffrac...</td>\n      <td>The angular spacing of features in the diffrac...</td>\n      <td>D</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"extra_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-16T03:53:06.846473Z","iopub.execute_input":"2023-08-16T03:53:06.846825Z","iopub.status.idle":"2023-08-16T03:53:06.864065Z","shell.execute_reply.started":"2023-08-16T03:53:06.846796Z","shell.execute_reply":"2023-08-16T03:53:06.862816Z"},"trusted":true},"execution_count":25,"outputs":[{"execution_count":25,"output_type":"execute_result","data":{"text/plain":"                                              prompt  \\\n0  In relation to Eunice Fay McKenzie's career, w...   \n1  How does Modified Newtonian Dynamics (MOND) im...   \n2  Which of the following statements accurately d...   \n3  What is the significance of the Museum of the ...   \n4  What was the previous name of the Christian Sc...   \n\n                                                   C  \\\n0  McKenzie gained recognition for her role as a ...   \n1  MOND is a theory that reduces the observed mis...   \n2  Ray Montgomerie is a former footballer who pla...   \n3  The Museum of the Occupation of Latvia was est...   \n4  The Evangelical School and Chapel for the Deaf...   \n\n                                                   E  \\\n0  McKenzie's successful career in sound films co...   \n1  MOND's impact on the observed missing baryonic...   \n2  Ray Montgomerie is a former footballer who pla...   \n3  The Museum of the Occupation of Latvia is a mu...   \n4         The Evangelical School for the Blind (ESB)   \n\n                                                   D  \\\n0  McKenzie's collaborations with director Blake ...   \n1  MOND is a theory that eliminates the observed ...   \n2  Ray Montgomerie is a former footballer who pla...   \n3  The Museum of the Occupation of Latvia primari...   \n4          The Evangelical School for the Deaf (ESD)   \n\n                                                   B  \\\n0  McKenzie is primarily remembered for her starr...   \n1  MOND explains the missing baryonic mass in gal...   \n2  Ray Montgomerie is a former footballer who pla...   \n3  The Museum of the Occupation of Latvia showcas...   \n4           The Christian School for the Blind (CSB)   \n\n                                                   A answer  \n0  McKenzie showcased her singing talents in nume...      B  \n1  MOND is a theory that increases the discrepanc...      E  \n2  Ray Montgomerie is a former footballer who pla...      B  \n3  The Museum of the Occupation of Latvia is a me...      C  \n4            The Christian School for the Deaf (CSD)      D  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>prompt</th>\n      <th>C</th>\n      <th>E</th>\n      <th>D</th>\n      <th>B</th>\n      <th>A</th>\n      <th>answer</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>In relation to Eunice Fay McKenzie's career, w...</td>\n      <td>McKenzie gained recognition for her role as a ...</td>\n      <td>McKenzie's successful career in sound films co...</td>\n      <td>McKenzie's collaborations with director Blake ...</td>\n      <td>McKenzie is primarily remembered for her starr...</td>\n      <td>McKenzie showcased her singing talents in nume...</td>\n      <td>B</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>How does Modified Newtonian Dynamics (MOND) im...</td>\n      <td>MOND is a theory that reduces the observed mis...</td>\n      <td>MOND's impact on the observed missing baryonic...</td>\n      <td>MOND is a theory that eliminates the observed ...</td>\n      <td>MOND explains the missing baryonic mass in gal...</td>\n      <td>MOND is a theory that increases the discrepanc...</td>\n      <td>E</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Which of the following statements accurately d...</td>\n      <td>Ray Montgomerie is a former footballer who pla...</td>\n      <td>Ray Montgomerie is a former footballer who pla...</td>\n      <td>Ray Montgomerie is a former footballer who pla...</td>\n      <td>Ray Montgomerie is a former footballer who pla...</td>\n      <td>Ray Montgomerie is a former footballer who pla...</td>\n      <td>B</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>What is the significance of the Museum of the ...</td>\n      <td>The Museum of the Occupation of Latvia was est...</td>\n      <td>The Museum of the Occupation of Latvia is a mu...</td>\n      <td>The Museum of the Occupation of Latvia primari...</td>\n      <td>The Museum of the Occupation of Latvia showcas...</td>\n      <td>The Museum of the Occupation of Latvia is a me...</td>\n      <td>C</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>What was the previous name of the Christian Sc...</td>\n      <td>The Evangelical School and Chapel for the Deaf...</td>\n      <td>The Evangelical School for the Blind (ESB)</td>\n      <td>The Evangelical School for the Deaf (ESD)</td>\n      <td>The Christian School for the Blind (CSB)</td>\n      <td>The Christian School for the Deaf (CSD)</td>\n      <td>D</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"concat_df = pd.concat([\n    train_df,\n    extra_df\n])\ntrain_df.reset_index(inplace=True, drop=True)\nconcat_df.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-16T03:53:06.867167Z","iopub.execute_input":"2023-08-16T03:53:06.867569Z","iopub.status.idle":"2023-08-16T03:53:06.879771Z","shell.execute_reply.started":"2023-08-16T03:53:06.867537Z","shell.execute_reply":"2023-08-16T03:53:06.878328Z"},"trusted":true},"execution_count":26,"outputs":[{"execution_count":26,"output_type":"execute_result","data":{"text/plain":"(700, 8)"},"metadata":{}}]},{"cell_type":"code","source":"train_ds=acq.preprocessing_dataset(concat_df)\n","metadata":{"execution":{"iopub.status.busy":"2023-08-16T03:53:06.881105Z","iopub.execute_input":"2023-08-16T03:53:06.881504Z","iopub.status.idle":"2023-08-16T03:53:06.899651Z","shell.execute_reply.started":"2023-08-16T03:53:06.881474Z","shell.execute_reply":"2023-08-16T03:53:06.898369Z"},"trusted":true},"execution_count":27,"outputs":[]},{"cell_type":"code","source":"def map3(y_true, y_pred):\n    m = (y_true.reshape((-1,1)) == y_pred)\n    return np.mean(np.where(m.any(axis=1), m.argmax(axis=1)+1, np.inf)**(-1))","metadata":{"execution":{"iopub.status.busy":"2023-08-16T03:53:06.900831Z","iopub.execute_input":"2023-08-16T03:53:06.901377Z","iopub.status.idle":"2023-08-16T03:53:06.908901Z","shell.execute_reply.started":"2023-08-16T03:53:06.901347Z","shell.execute_reply":"2023-08-16T03:53:06.90772Z"},"trusted":true},"execution_count":28,"outputs":[]},{"cell_type":"code","source":"# Load model directly\nfrom transformers import AutoModel\nmodel = AutoModel.from_pretrained(\"microsoft/deberta-v2-xxlarge\")\ntokenizer = AutoTokenizer.from_pretrained(\"microsoft/deberta-v2-xxlarge\")","metadata":{"execution":{"iopub.status.busy":"2023-08-16T04:00:58.991618Z","iopub.execute_input":"2023-08-16T04:00:58.992049Z","iopub.status.idle":"2023-08-16T04:01:19.719183Z","shell.execute_reply.started":"2023-08-16T04:00:58.992015Z","shell.execute_reply":"2023-08-16T04:01:19.717601Z"},"trusted":true},"execution_count":36,"outputs":[{"name":"stderr","text":"Some weights of the model checkpoint at microsoft/deberta-v2-xxlarge were not used when initializing DebertaV2Model: ['lm_predictions.lm_head.LayerNorm.weight', 'lm_predictions.lm_head.bias', 'lm_predictions.lm_head.LayerNorm.bias', 'lm_predictions.lm_head.dense.weight', 'lm_predictions.lm_head.dense.bias']\n- This IS expected if you are initializing DebertaV2Model from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing DebertaV2Model from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Downloading (…)okenizer_config.json:   0%|          | 0.00/52.0 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"89389d14dd4a41a9844bb0e1ab9187d3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading spm.model:   0%|          | 0.00/2.45M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f0ee5215457943bd909ed88d81a11c25"}},"metadata":{}},{"name":"stderr","text":"Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\nSpecial tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n","output_type":"stream"}]},{"cell_type":"code","source":"options = 'ABCDE'\nindices = list(range(5))\noption_to_index = {option: index for option, index in zip(options, indices)}\nindex_to_option = {index: option for option, index in zip(options, indices)}\n\ndef preprocess(example):\n    first_sentence = [example['prompt']] * 5\n    second_sentence = [example[option] for option in options]\n    tokenized_example = tokenizer(first_sentence, second_sentence, truncation=True)\n    tokenized_example['label'] = option_to_index[example['answer']]\n    return tokenized_example\n","metadata":{"execution":{"iopub.status.busy":"2023-08-16T04:01:35.46118Z","iopub.execute_input":"2023-08-16T04:01:35.461999Z","iopub.status.idle":"2023-08-16T04:01:35.471474Z","shell.execute_reply.started":"2023-08-16T04:01:35.461954Z","shell.execute_reply":"2023-08-16T04:01:35.469904Z"},"trusted":true},"execution_count":37,"outputs":[]},{"cell_type":"code","source":"tokenized_train_ds = train_ds.map(preprocess, batched=False, remove_columns=['prompt', 'A', 'B', 'C', 'D', 'E', 'answer'])","metadata":{"execution":{"iopub.status.busy":"2023-08-16T04:01:37.669933Z","iopub.execute_input":"2023-08-16T04:01:37.670367Z","iopub.status.idle":"2023-08-16T04:01:38.798977Z","shell.execute_reply.started":"2023-08-16T04:01:37.670334Z","shell.execute_reply":"2023-08-16T04:01:38.797529Z"},"trusted":true},"execution_count":38,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/700 [00:00<?, ?ex/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6952728b923c48beab8d94a4df5db060"}},"metadata":{}}]},{"cell_type":"code","source":"tokenized_train_ds","metadata":{"execution":{"iopub.status.busy":"2023-08-16T04:01:49.074074Z","iopub.execute_input":"2023-08-16T04:01:49.074663Z","iopub.status.idle":"2023-08-16T04:01:49.082918Z","shell.execute_reply.started":"2023-08-16T04:01:49.074626Z","shell.execute_reply":"2023-08-16T04:01:49.081652Z"},"trusted":true},"execution_count":39,"outputs":[{"execution_count":39,"output_type":"execute_result","data":{"text/plain":"Dataset({\n    features: ['id', '__index_level_0__', 'input_ids', 'token_type_ids', 'attention_mask', 'label'],\n    num_rows: 700\n})"},"metadata":{}}]},{"cell_type":"code","source":"@dataclass\nclass DataCollatorForMultipleChoice:\n    tokenizer: PreTrainedTokenizerBase\n    padding: Union[bool, str, PaddingStrategy] = True\n    max_length: Optional[int] = None\n    pad_to_multiple_of: Optional[int] = None\n    \n    def __call__(self, features):\n        label_name = \"label\" if 'label' in features[0].keys() else 'labels'\n        labels = [feature.pop(label_name) for feature in features]\n        batch_size = len(features)\n        num_choices = len(features[0]['input_ids'])\n        flattened_features = [\n            [{k: v[i] for k, v in feature.items()} for i in range(num_choices)] for feature in features\n        ]\n        flattened_features = sum(flattened_features, [])\n        \n        batch = self.tokenizer.pad(\n            flattened_features,\n            padding=self.padding,\n            max_length=self.max_length,\n            pad_to_multiple_of=self.pad_to_multiple_of,\n            return_tensors='pt',\n        )\n        batch = {k: v.view(batch_size, num_choices, -1) for k, v in batch.items()}\n        batch['labels'] = torch.tensor(labels, dtype=torch.int64)\n        return batch","metadata":{"execution":{"iopub.status.busy":"2023-08-16T04:01:59.890416Z","iopub.execute_input":"2023-08-16T04:01:59.89086Z","iopub.status.idle":"2023-08-16T04:01:59.903405Z","shell.execute_reply.started":"2023-08-16T04:01:59.890825Z","shell.execute_reply":"2023-08-16T04:01:59.902236Z"},"trusted":true},"execution_count":40,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfinal_dfs = pd.DataFrame()\nkf = KFold(n_splits=4, shuffle=True, random_state=42)\ncv_list = []\nfor fold, (idx_tr, idx_va) in enumerate(kf.split(concat_df)):\n    \n    train_set = concat_df.loc[idx_tr, ['prompt', 'A', 'B', 'C', 'D', 'E', 'answer']]\n    valid_set = concat_df.loc[idx_va, ['prompt', 'A', 'B', 'C', 'D', 'E', 'answer']]\n    valid_label = concat_df.loc[idx_va, 'answer'].values\n    train_set = Dataset.from_pandas(train_set)\n    tokenized_train = train_set.map(preprocess, remove_columns=['prompt', 'A', 'B', 'C', 'D', 'E', 'answer'])\n    valid_set = Dataset.from_pandas(valid_set)\n    tokenized_valid = valid_set.map(preprocess, remove_columns=['prompt', 'A', 'B', 'C', 'D', 'E', 'answer'])\n    \n    training_args = TrainingArguments(\n        output_dir='./',\n        overwrite_output_dir=True,\n        load_best_model_at_end=True,\n        save_total_limit=1,\n        evaluation_strategy=\"epoch\",\n        save_strategy=\"epoch\",\n        warmup_ratio=0.8,\n        learning_rate=2e-6,\n        per_device_train_batch_size=1,\n        per_device_eval_batch_size=2,\n        num_train_epochs=6,\n        report_to='none',\n        seed=42\n    )\n    model = AutoModelForMultipleChoice.from_pretrained(deberta_v3_large)\n\n    trainer = Trainer(\n        model=model,\n        args=training_args,\n        tokenizer=tokenizer,\n        data_collator=DataCollatorForMultipleChoice(tokenizer=tokenizer),\n        train_dataset=tokenized_train,\n        eval_dataset=tokenized_valid,\n        callbacks=[EarlyStoppingCallback(early_stopping_patience=1)]\n    )\n    trainer.train()\n    \n    valid_pred = trainer.predict(tokenized_valid).predictions\n    valid_pred_ids = np.argsort(-valid_pred, 1)\n    valid_pred_letters = np.array(list('ABCDE'))[valid_pred_ids][:, :3]\n    valid_map3 = map3(valid_label, valid_pred_letters)\n    print(f\"{Fore.RED}{Style.BRIGHT}Fold {fold}: MAP@3 = {valid_map3:.5f}{Style.RESET_ALL}\")\n    cv_list.append(valid_map3)\n    \n    test_predictions = trainer.predict(tokenized_test_ds).predictions\n    fold_predict_df = pd.DataFrame(test_predictions, columns=[f'{x}{fold}' for x in ['A','B','C','D','E']])\n    final_dfs = pd.concat([final_dfs, fold_predict_df], axis=1)\n    \n    del model, trainer, tokenized_train, tokenized_valid, train_set, valid_set\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-08-16T04:03:12.499704Z","iopub.execute_input":"2023-08-16T04:03:12.50012Z","iopub.status.idle":"2023-08-16T04:03:16.316571Z","shell.execute_reply.started":"2023-08-16T04:03:12.500089Z","shell.execute_reply":"2023-08-16T04:03:16.315021Z"},"trusted":true},"execution_count":44,"outputs":[{"output_type":"display_data","data":{"text/plain":"\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n\u001b[31m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.10/site-packages/IPython/core/magics/\u001b[0m\u001b[1;33mexecution.py\u001b[0m:\u001b[94m1325\u001b[0m in \u001b[92mtime\u001b[0m            \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1322 \u001b[0m\u001b[2m│   │   \u001b[0m\u001b[94melse\u001b[0m:                                                                             \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1323 \u001b[0m\u001b[2m│   │   │   \u001b[0mst = clock2()                                                                 \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1324 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[94mtry\u001b[0m:                                                                          \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1325 \u001b[2m│   │   │   │   \u001b[0mexec(code, glob, local_ns)                                                \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1326 \u001b[0m\u001b[2m│   │   │   │   \u001b[0mout=\u001b[94mNone\u001b[0m                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1327 \u001b[0m\u001b[2m│   │   │   │   \u001b[0m\u001b[2m# multi-line %%time case\u001b[0m                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1328 \u001b[0m\u001b[2m│   │   │   │   \u001b[0m\u001b[94mif\u001b[0m expr_val \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m:                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m in \u001b[92m<module>\u001b[0m:\u001b[94m6\u001b[0m                                                                                    \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.10/site-packages/pandas/core/\u001b[0m\u001b[1;33mindexing.py\u001b[0m:\u001b[94m1067\u001b[0m in \u001b[92m__getitem__\u001b[0m              \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1064 \u001b[0m\u001b[2m│   │   │   \u001b[0mkey = \u001b[96mtuple\u001b[0m(com.apply_if_callable(x, \u001b[96mself\u001b[0m.obj) \u001b[94mfor\u001b[0m x \u001b[95min\u001b[0m key)                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1065 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[94mif\u001b[0m \u001b[96mself\u001b[0m._is_scalar_access(key):                                               \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1066 \u001b[0m\u001b[2m│   │   │   │   \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m.obj._get_value(*key, takeable=\u001b[96mself\u001b[0m._takeable)                 \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1067 \u001b[2m│   │   │   \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m._getitem_tuple(key)                                               \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1068 \u001b[0m\u001b[2m│   │   \u001b[0m\u001b[94melse\u001b[0m:                                                                             \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1069 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[2m# we by definition only have the 0th axis\u001b[0m                                     \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1070 \u001b[0m\u001b[2m│   │   │   \u001b[0maxis = \u001b[96mself\u001b[0m.axis \u001b[95mor\u001b[0m \u001b[94m0\u001b[0m                                                         \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.10/site-packages/pandas/core/\u001b[0m\u001b[1;33mindexing.py\u001b[0m:\u001b[94m1254\u001b[0m in \u001b[92m_getitem_tuple\u001b[0m           \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1251 \u001b[0m\u001b[2m│   │   \u001b[0m                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1252 \u001b[0m\u001b[2m│   │   \u001b[0m\u001b[2m# ugly hack for GH #836\u001b[0m                                                           \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1253 \u001b[0m\u001b[2m│   │   \u001b[0m\u001b[94mif\u001b[0m \u001b[96mself\u001b[0m._multi_take_opportunity(tup):                                             \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1254 \u001b[2m│   │   │   \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m._multi_take(tup)                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1255 \u001b[0m\u001b[2m│   │   \u001b[0m                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1256 \u001b[0m\u001b[2m│   │   \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m._getitem_tuple_same_dim(tup)                                          \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1257 \u001b[0m                                                                                          \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.10/site-packages/pandas/core/\u001b[0m\u001b[1;33mindexing.py\u001b[0m:\u001b[94m1205\u001b[0m in \u001b[92m_multi_take\u001b[0m              \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1202 \u001b[0m\u001b[2;33m│   │   \u001b[0m\u001b[33mvalues: same type as the object being indexed\u001b[0m                                     \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1203 \u001b[0m\u001b[2;33m│   │   \u001b[0m\u001b[33m\"\"\"\u001b[0m                                                                               \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1204 \u001b[0m\u001b[2m│   │   \u001b[0m\u001b[2m# GH 836\u001b[0m                                                                          \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1205 \u001b[2m│   │   \u001b[0md = {                                                                             \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1206 \u001b[0m\u001b[2m│   │   │   \u001b[0maxis: \u001b[96mself\u001b[0m._get_listlike_indexer(key, axis)                                   \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1207 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[94mfor\u001b[0m (key, axis) \u001b[95min\u001b[0m \u001b[96mzip\u001b[0m(tup, \u001b[96mself\u001b[0m.obj._AXIS_ORDERS)                            \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1208 \u001b[0m\u001b[2m│   │   \u001b[0m}                                                                                 \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.10/site-packages/pandas/core/\u001b[0m\u001b[1;33mindexing.py\u001b[0m:\u001b[94m1206\u001b[0m in \u001b[92m<dictcomp>\u001b[0m               \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1203 \u001b[0m\u001b[2;33m│   │   \u001b[0m\u001b[33m\"\"\"\u001b[0m                                                                               \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1204 \u001b[0m\u001b[2m│   │   \u001b[0m\u001b[2m# GH 836\u001b[0m                                                                          \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1205 \u001b[0m\u001b[2m│   │   \u001b[0md = {                                                                             \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1206 \u001b[2m│   │   │   \u001b[0maxis: \u001b[96mself\u001b[0m._get_listlike_indexer(key, axis)                                   \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1207 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[94mfor\u001b[0m (key, axis) \u001b[95min\u001b[0m \u001b[96mzip\u001b[0m(tup, \u001b[96mself\u001b[0m.obj._AXIS_ORDERS)                            \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1208 \u001b[0m\u001b[2m│   │   \u001b[0m}                                                                                 \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1209 \u001b[0m\u001b[2m│   │   \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m.obj._reindex_with_indexers(d, copy=\u001b[94mTrue\u001b[0m, allow_dups=\u001b[94mTrue\u001b[0m)             \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.10/site-packages/pandas/core/\u001b[0m\u001b[1;33mindexing.py\u001b[0m:\u001b[94m1432\u001b[0m in \u001b[92m_get_listlike_indexer\u001b[0m    \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1429 \u001b[0m\u001b[2m│   │   \u001b[0max = \u001b[96mself\u001b[0m.obj._get_axis(axis)                                                     \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1430 \u001b[0m\u001b[2m│   │   \u001b[0maxis_name = \u001b[96mself\u001b[0m.obj._get_axis_name(axis)                                         \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1431 \u001b[0m\u001b[2m│   │   \u001b[0m                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1432 \u001b[2m│   │   \u001b[0mkeyarr, indexer = ax._get_indexer_strict(key, axis_name)                          \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1433 \u001b[0m\u001b[2m│   │   \u001b[0m                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1434 \u001b[0m\u001b[2m│   │   \u001b[0m\u001b[94mreturn\u001b[0m keyarr, indexer                                                            \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m1435 \u001b[0m                                                                                          \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.10/site-packages/pandas/core/indexes/\u001b[0m\u001b[1;33mbase.py\u001b[0m:\u001b[94m6070\u001b[0m in \u001b[92m_get_indexer_strict\u001b[0m  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m6067 \u001b[0m\u001b[2m│   │   \u001b[0m\u001b[94melse\u001b[0m:                                                                             \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m6068 \u001b[0m\u001b[2m│   │   │   \u001b[0mkeyarr, indexer, new_indexer = \u001b[96mself\u001b[0m._reindex_non_unique(keyarr)               \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m6069 \u001b[0m\u001b[2m│   │   \u001b[0m                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m6070 \u001b[2m│   │   \u001b[0m\u001b[96mself\u001b[0m._raise_if_missing(keyarr, indexer, axis_name)                                \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m6071 \u001b[0m\u001b[2m│   │   \u001b[0m                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m6072 \u001b[0m\u001b[2m│   │   \u001b[0mkeyarr = \u001b[96mself\u001b[0m.take(indexer)                                                       \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m6073 \u001b[0m\u001b[2m│   │   \u001b[0m\u001b[94mif\u001b[0m \u001b[96misinstance\u001b[0m(key, Index):                                                        \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.10/site-packages/pandas/core/indexes/\u001b[0m\u001b[1;33mbase.py\u001b[0m:\u001b[94m6133\u001b[0m in \u001b[92m_raise_if_missing\u001b[0m    \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m6130 \u001b[0m\u001b[2m│   │   │   │   \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mKeyError\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mNone of [\u001b[0m\u001b[33m{\u001b[0mkey\u001b[33m}\u001b[0m\u001b[33m] are in the [\u001b[0m\u001b[33m{\u001b[0maxis_name\u001b[33m}\u001b[0m\u001b[33m]\u001b[0m\u001b[33m\"\u001b[0m)               \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m6131 \u001b[0m\u001b[2m│   │   │   \u001b[0m                                                                              \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m6132 \u001b[0m\u001b[2m│   │   │   \u001b[0mnot_found = \u001b[96mlist\u001b[0m(ensure_index(key)[missing_mask.nonzero()[\u001b[94m0\u001b[0m]].unique())       \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m6133 \u001b[2m│   │   │   \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mKeyError\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33m{\u001b[0mnot_found\u001b[33m}\u001b[0m\u001b[33m not in index\u001b[0m\u001b[33m\"\u001b[0m)                                   \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m6134 \u001b[0m\u001b[2m│   \u001b[0m                                                                                      \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m6135 \u001b[0m\u001b[2m│   \u001b[0m\u001b[1;95m@overload\u001b[0m                                                                             \u001b[31m│\u001b[0m\n\u001b[31m│\u001b[0m   \u001b[2m6136 \u001b[0m\u001b[2m│   \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m_get_indexer_non_comparable\u001b[0m(                                                      \u001b[31m│\u001b[0m\n\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n\u001b[1;91mKeyError: \u001b[0m\u001b[32m'\u001b[0m\u001b[32m[\u001b[0m\u001b[32m501, 502, 503, 504, 505, 506, 508, 509, 510, 512, 513, 514, 518, 519, 520, 521, 522, 523, 524, 526, \u001b[0m\n\u001b[32m527, 528, 529, 530, 531, 532, 534, 535, 536, 537, 538, 539, 540, 542, 544, 546, 547, 548, 550, 551, 552, 553, 554, \u001b[0m\n\u001b[32m555, 556, 558, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 571, 572, 573, 574, 575, 576, 581, 583, 584, 586, \u001b[0m\n\u001b[32m587, 588, 589, 590, 591, 592, 593, 594, 595, 597, 598, 600, 602, 603, 604, 606, 607, 608, 609, 610, 611, 612, 613, \u001b[0m\n\u001b[32m614, 615, 616, 617, 619, 620, 621, 622, 623, 624, 625, 628, 629, 630, 632, 633, 635, 637, 638, 639, 640, 642, 643, \u001b[0m\n\u001b[32m644, 645, 646, 647, 648, 649, 651, 652, 653, 654, 657, 658, 659, 660, 661, 662, 663, 664, 665, 670, 671, 674, 675, \u001b[0m\n\u001b[32m676, 677, 678, 679, 680, 682, 683, 684, 685, 686, 688, 689, 690, 692, 693, 694, 697, 698, 699\u001b[0m\u001b[32m]\u001b[0m\u001b[32m not in index'\u001b[0m\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800000; text-decoration-color: #800000\">╭─────────────────────────────── </span><span style=\"color: #800000; text-decoration-color: #800000; font-weight: bold\">Traceback </span><span style=\"color: #bf7f7f; text-decoration-color: #bf7f7f; font-weight: bold\">(most recent call last)</span><span style=\"color: #800000; text-decoration-color: #800000\"> ────────────────────────────────╮</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/conda/lib/python3.10/site-packages/IPython/core/magics/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">execution.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">1325</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">time</span>            <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1322 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">else</span>:                                                                             <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1323 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span>st = clock2()                                                                 <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1324 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">try</span>:                                                                          <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>1325 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   │   </span>exec(code, glob, local_ns)                                                <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1326 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   │   </span>out=<span style=\"color: #0000ff; text-decoration-color: #0000ff\">None</span>                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1327 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   │   </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"># multi-line %%time case</span>                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1328 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">if</span> expr_val <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">is</span> <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">not</span> <span style=\"color: #0000ff; text-decoration-color: #0000ff\">None</span>:                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">&lt;module&gt;</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">6</span>                                                                                    <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/conda/lib/python3.10/site-packages/pandas/core/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">indexing.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">1067</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">__getitem__</span>              <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1064 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span>key = <span style=\"color: #00ffff; text-decoration-color: #00ffff\">tuple</span>(com.apply_if_callable(x, <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>.obj) <span style=\"color: #0000ff; text-decoration-color: #0000ff\">for</span> x <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">in</span> key)                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1065 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">if</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._is_scalar_access(key):                                               <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1066 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>.obj._get_value(*key, takeable=<span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._takeable)                 <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>1067 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._getitem_tuple(key)                                               <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1068 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">else</span>:                                                                             <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1069 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"># we by definition only have the 0th axis</span>                                     <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1070 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span>axis = <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>.axis <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">or</span> <span style=\"color: #0000ff; text-decoration-color: #0000ff\">0</span>                                                         <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/conda/lib/python3.10/site-packages/pandas/core/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">indexing.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">1254</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">_getitem_tuple</span>           <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1251 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span>                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1252 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"># ugly hack for GH #836</span>                                                           <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1253 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">if</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._multi_take_opportunity(tup):                                             <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>1254 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._multi_take(tup)                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1255 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span>                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1256 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._getitem_tuple_same_dim(tup)                                          <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1257 </span>                                                                                          <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/conda/lib/python3.10/site-packages/pandas/core/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">indexing.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">1205</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">_multi_take</span>              <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1202 </span><span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">│   │   </span><span style=\"color: #808000; text-decoration-color: #808000\">values: same type as the object being indexed</span>                                     <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1203 </span><span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">│   │   </span><span style=\"color: #808000; text-decoration-color: #808000\">\"\"\"</span>                                                                               <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1204 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"># GH 836</span>                                                                          <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>1205 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span>d = {                                                                             <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1206 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span>axis: <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._get_listlike_indexer(key, axis)                                   <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1207 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">for</span> (key, axis) <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">in</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">zip</span>(tup, <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>.obj._AXIS_ORDERS)                            <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1208 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span>}                                                                                 <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/conda/lib/python3.10/site-packages/pandas/core/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">indexing.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">1206</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">&lt;dictcomp&gt;</span>               <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1203 </span><span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">│   │   </span><span style=\"color: #808000; text-decoration-color: #808000\">\"\"\"</span>                                                                               <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1204 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"># GH 836</span>                                                                          <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1205 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span>d = {                                                                             <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>1206 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span>axis: <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._get_listlike_indexer(key, axis)                                   <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1207 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">for</span> (key, axis) <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">in</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">zip</span>(tup, <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>.obj._AXIS_ORDERS)                            <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1208 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span>}                                                                                 <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1209 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>.obj._reindex_with_indexers(d, copy=<span style=\"color: #0000ff; text-decoration-color: #0000ff\">True</span>, allow_dups=<span style=\"color: #0000ff; text-decoration-color: #0000ff\">True</span>)             <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/conda/lib/python3.10/site-packages/pandas/core/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">indexing.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">1432</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">_get_listlike_indexer</span>    <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1429 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span>ax = <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>.obj._get_axis(axis)                                                     <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1430 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span>axis_name = <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>.obj._get_axis_name(axis)                                         <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1431 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span>                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>1432 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span>keyarr, indexer = ax._get_indexer_strict(key, axis_name)                          <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1433 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span>                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1434 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> keyarr, indexer                                                            <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1435 </span>                                                                                          <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/conda/lib/python3.10/site-packages/pandas/core/indexes/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">base.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">6070</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">_get_indexer_strict</span>  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">6067 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">else</span>:                                                                             <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">6068 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span>keyarr, indexer, new_indexer = <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._reindex_non_unique(keyarr)               <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">6069 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span>                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>6070 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span><span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._raise_if_missing(keyarr, indexer, axis_name)                                <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">6071 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span>                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">6072 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span>keyarr = <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>.take(indexer)                                                       <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">6073 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">if</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">isinstance</span>(key, Index):                                                        <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/conda/lib/python3.10/site-packages/pandas/core/indexes/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">base.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">6133</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">_raise_if_missing</span>    <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">6130 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">raise</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">KeyError</span>(<span style=\"color: #808000; text-decoration-color: #808000\">f\"None of [{</span>key<span style=\"color: #808000; text-decoration-color: #808000\">}] are in the [{</span>axis_name<span style=\"color: #808000; text-decoration-color: #808000\">}]\"</span>)               <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">6131 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span>                                                                              <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">6132 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span>not_found = <span style=\"color: #00ffff; text-decoration-color: #00ffff\">list</span>(ensure_index(key)[missing_mask.nonzero()[<span style=\"color: #0000ff; text-decoration-color: #0000ff\">0</span>]].unique())       <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>6133 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">raise</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">KeyError</span>(<span style=\"color: #808000; text-decoration-color: #808000\">f\"{</span>not_found<span style=\"color: #808000; text-decoration-color: #808000\">} not in index\"</span>)                                   <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">6134 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   </span>                                                                                      <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">6135 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   </span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">@overload</span>                                                                             <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">6136 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">def</span> <span style=\"color: #00ff00; text-decoration-color: #00ff00\">_get_indexer_non_comparable</span>(                                                      <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n<span style=\"color: #800000; text-decoration-color: #800000\">╰──────────────────────────────────────────────────────────────────────────────────────────────────╯</span>\n<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-weight: bold\">KeyError: </span><span style=\"color: #008000; text-decoration-color: #008000\">'[501, 502, 503, 504, 505, 506, 508, 509, 510, 512, 513, 514, 518, 519, 520, 521, 522, 523, 524, 526, </span>\n<span style=\"color: #008000; text-decoration-color: #008000\">527, 528, 529, 530, 531, 532, 534, 535, 536, 537, 538, 539, 540, 542, 544, 546, 547, 548, 550, 551, 552, 553, 554, </span>\n<span style=\"color: #008000; text-decoration-color: #008000\">555, 556, 558, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 571, 572, 573, 574, 575, 576, 581, 583, 584, 586, </span>\n<span style=\"color: #008000; text-decoration-color: #008000\">587, 588, 589, 590, 591, 592, 593, 594, 595, 597, 598, 600, 602, 603, 604, 606, 607, 608, 609, 610, 611, 612, 613, </span>\n<span style=\"color: #008000; text-decoration-color: #008000\">614, 615, 616, 617, 619, 620, 621, 622, 623, 624, 625, 628, 629, 630, 632, 633, 635, 637, 638, 639, 640, 642, 643, </span>\n<span style=\"color: #008000; text-decoration-color: #008000\">644, 645, 646, 647, 648, 649, 651, 652, 653, 654, 657, 658, 659, 660, 661, 662, 663, 664, 665, 670, 671, 674, 675, </span>\n<span style=\"color: #008000; text-decoration-color: #008000\">676, 677, 678, 679, 680, 682, 683, 684, 685, 686, 688, 689, 690, 692, 693, 694, 697, 698, 699] not in index'</span>\n</pre>\n"},"metadata":{}}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}