final_wt_train.ipynb

{"cells":[{"cell_type":"code","execution_count":6,"metadata":{"collapsed":true,"execution":{"iopub.execute_input":"2023-03-12T18:11:59.357895Z","iopub.status.busy":"2023-03-12T18:11:59.357193Z","iopub.status.idle":"2023-03-12T18:14:06.404335Z","shell.execute_reply":"2023-03-12T18:14:06.403046Z","shell.execute_reply.started":"2023-03-12T18:11:59.357858Z"},"jupyter":{"outputs_hidden":true},"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (1.21.6)\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0mRequirement already satisfied: pandas in /opt/conda/lib/python3.7/site-packages (1.3.5)\n","Requirement already satisfied: python-dateutil>=2.7.3 in /opt/conda/lib/python3.7/site-packages (from pandas) (2.8.2)\n","Requirement already satisfied: pytz>=2017.3 in /opt/conda/lib/python3.7/site-packages (from pandas) (2022.7.1)\n","Requirement already satisfied: numpy>=1.17.3 in /opt/conda/lib/python3.7/site-packages (from pandas) (1.21.6)\n","Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil>=2.7.3->pandas) (1.16.0)\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0mCollecting decord\n","  Downloading decord-0.6.0-py3-none-manylinux2010_x86_64.whl (13.6 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.6/13.6 MB\u001b[0m \u001b[31m77.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n","\u001b[?25hRequirement already satisfied: numpy>=1.14.0 in /opt/conda/lib/python3.7/site-packages (from decord) (1.21.6)\n","Installing collected packages: decord\n","Successfully installed decord-0.6.0\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0mLooking in indexes: https://pypi.org/simple, https://download.pytorch.org/whl/cu117\n","Requirement already satisfied: torch in /opt/conda/lib/python3.7/site-packages (1.13.0)\n","Requirement already satisfied: torchvision in /opt/conda/lib/python3.7/site-packages (0.14.0)\n","Requirement already satisfied: torchaudio in /opt/conda/lib/python3.7/site-packages (0.13.0)\n","Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from torch) (4.4.0)\n","Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from torchvision) (1.21.6)\n","Requirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from torchvision) (2.28.2)\n","Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /opt/conda/lib/python3.7/site-packages (from torchvision) (9.4.0)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.7/site-packages (from requests->torchvision) (2.1.1)\n","Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->torchvision) (3.4)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->torchvision) (1.26.14)\n","Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->torchvision) (2022.12.7)\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0mRequirement already satisfied: pytorch-lightning in /opt/conda/lib/python3.7/site-packages (1.9.3)\n","Requirement already satisfied: numpy>=1.17.2 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (1.21.6)\n","Requirement already satisfied: tqdm>=4.57.0 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (4.64.1)\n","Requirement already satisfied: packaging>=17.1 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (23.0)\n","Requirement already satisfied: lightning-utilities>=0.6.0.post0 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (0.7.1)\n","Requirement already satisfied: torchmetrics>=0.7.0 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (0.11.1)\n","Requirement already satisfied: typing-extensions>=4.0.0 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (4.4.0)\n","Requirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (1.13.0)\n","Requirement already satisfied: PyYAML>=5.4 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (6.0)\n","Requirement already satisfied: fsspec[http]>2021.06.0 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (2023.1.0)\n","Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /opt/conda/lib/python3.7/site-packages (from fsspec[http]>2021.06.0->pytorch-lightning) (3.8.3)\n","Requirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from fsspec[http]>2021.06.0->pytorch-lightning) (2.28.2)\n","Requirement already satisfied: importlib-metadata>=4.0.0 in /opt/conda/lib/python3.7/site-packages (from lightning-utilities>=0.6.0.post0->pytorch-lightning) (4.11.4)\n","Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.7/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning) (4.0.2)\n","Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.7/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning) (1.3.1)\n","Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.7/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning) (1.8.2)\n","Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.7/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning) (22.2.0)\n","Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.7/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning) (1.3.3)\n","Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /opt/conda/lib/python3.7/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning) (2.1.1)\n","Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.7/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning) (6.0.4)\n","Requirement already satisfied: asynctest==0.13.0 in /opt/conda/lib/python3.7/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning) (0.13.0)\n","Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata>=4.0.0->lightning-utilities>=0.6.0.post0->pytorch-lightning) (3.11.0)\n","Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->fsspec[http]>2021.06.0->pytorch-lightning) (2022.12.7)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->fsspec[http]>2021.06.0->pytorch-lightning) (1.26.14)\n","Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->fsspec[http]>2021.06.0->pytorch-lightning) (3.4)\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0mCollecting pytorchvideo\n","  Downloading pytorchvideo-0.1.5.tar.gz (132 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.7/132.7 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n","\u001b[?25hCollecting fvcore\n","  Downloading fvcore-0.1.5.post20221221.tar.gz (50 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.2/50.2 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n","\u001b[?25hCollecting av\n","  Downloading av-10.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30.7 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m30.7/30.7 MB\u001b[0m \u001b[31m34.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n","\u001b[?25hCollecting parameterized\n","  Downloading parameterized-0.8.1-py2.py3-none-any.whl (26 kB)\n","Collecting iopath\n","  Downloading iopath-0.1.10.tar.gz (42 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.2/42.2 kB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n","\u001b[?25hRequirement already satisfied: networkx in /opt/conda/lib/python3.7/site-packages (from pytorchvideo) (2.6.3)\n","Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from fvcore->pytorchvideo) (1.21.6)\n","Requirement already satisfied: yacs>=0.1.6 in /opt/conda/lib/python3.7/site-packages (from fvcore->pytorchvideo) (0.1.8)\n","Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.7/site-packages (from fvcore->pytorchvideo) (6.0)\n","Requirement already satisfied: tqdm in /opt/conda/lib/python3.7/site-packages (from fvcore->pytorchvideo) (4.64.1)\n","Requirement already satisfied: termcolor>=1.1 in /opt/conda/lib/python3.7/site-packages (from fvcore->pytorchvideo) (2.2.0)\n","Requirement already satisfied: Pillow in /opt/conda/lib/python3.7/site-packages (from fvcore->pytorchvideo) (9.4.0)\n","Requirement already satisfied: tabulate in /opt/conda/lib/python3.7/site-packages (from fvcore->pytorchvideo) (0.9.0)\n","Requirement already satisfied: typing_extensions in /opt/conda/lib/python3.7/site-packages (from iopath->pytorchvideo) (4.4.0)\n","Requirement already satisfied: portalocker in /opt/conda/lib/python3.7/site-packages (from iopath->pytorchvideo) (2.7.0)\n","Building wheels for collected packages: pytorchvideo, fvcore, iopath\n","  Building wheel for pytorchvideo (setup.py) ... \u001b[?25ldone\n","\u001b[?25h  Created wheel for pytorchvideo: filename=pytorchvideo-0.1.5-py3-none-any.whl size=188715 sha256=00be207953b20ca157b9b54c6af80ce56ed3461ff8846071cc15aec4017ea1c4\n","  Stored in directory: /root/.cache/pip/wheels/71/3c/ab/bfe50bbf6cfeea284d6179c59b31c463d55c68b6a10728ba20\n","  Building wheel for fvcore (setup.py) ... \u001b[?25ldone\n","\u001b[?25h  Created wheel for fvcore: filename=fvcore-0.1.5.post20221221-py3-none-any.whl size=61431 sha256=4cbeaf8189dcf01a1847b805c24cf831b530010624a40bed094ecac2ac32bae7\n","  Stored in directory: /root/.cache/pip/wheels/12/a2/36/21b9bde5f8deeeb6312efe88ddde26a51facbd2089f32917b3\n","  Building wheel for iopath (setup.py) ... \u001b[?25ldone\n","\u001b[?25h  Created wheel for iopath: filename=iopath-0.1.10-py3-none-any.whl size=31547 sha256=61e3dbc01d028f41edea7c1666a6a7fbb586d5898ac376f8555ffb7df83d38e2\n","  Stored in directory: /root/.cache/pip/wheels/96/9a/78/61eeeec98da40f44085da9ba3fec952b4ab7224f5c5be75126\n","Successfully built pytorchvideo fvcore iopath\n","Installing collected packages: parameterized, av, iopath, fvcore, pytorchvideo\n","Successfully installed av-10.0.0 fvcore-0.1.5.post20221221 iopath-0.1.10 parameterized-0.8.1 pytorchvideo-0.1.5\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0mRequirement already satisfied: scikit-learn in /opt/conda/lib/python3.7/site-packages (1.0.2)\n","Requirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.7/site-packages (from scikit-learn) (1.2.0)\n","Requirement already satisfied: scipy>=1.1.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn) (1.7.3)\n","Requirement already satisfied: numpy>=1.14.6 in /opt/conda/lib/python3.7/site-packages (from scikit-learn) (1.21.6)\n","Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn) (3.1.0)\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0mRequirement already satisfied: scikit-multilearn in /opt/conda/lib/python3.7/site-packages (0.2.0)\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0mCollecting segmentation-models-pytorch\n","  Downloading segmentation_models_pytorch-0.3.2-py3-none-any.whl (106 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m106.7/106.7 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: torchvision>=0.5.0 in /opt/conda/lib/python3.7/site-packages (from segmentation-models-pytorch) (0.14.0)\n","Requirement already satisfied: tqdm in /opt/conda/lib/python3.7/site-packages (from segmentation-models-pytorch) (4.64.1)\n","Collecting pretrainedmodels==0.7.4\n","  Downloading pretrainedmodels-0.7.4.tar.gz (58 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.8/58.8 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n","\u001b[?25hCollecting efficientnet-pytorch==0.7.1\n","  Downloading efficientnet_pytorch-0.7.1.tar.gz (21 kB)\n","  Preparing metadata (setup.py) ... \u001b[?25ldone\n","\u001b[?25hRequirement already satisfied: timm==0.6.12 in /opt/conda/lib/python3.7/site-packages (from segmentation-models-pytorch) (0.6.12)\n","Requirement already satisfied: pillow in /opt/conda/lib/python3.7/site-packages (from segmentation-models-pytorch) (9.4.0)\n","Requirement already satisfied: torch in /opt/conda/lib/python3.7/site-packages (from efficientnet-pytorch==0.7.1->segmentation-models-pytorch) (1.13.0)\n","Requirement already satisfied: munch in /opt/conda/lib/python3.7/site-packages (from pretrainedmodels==0.7.4->segmentation-models-pytorch) (2.5.0)\n","Requirement already satisfied: huggingface-hub in /opt/conda/lib/python3.7/site-packages (from timm==0.6.12->segmentation-models-pytorch) (0.12.1)\n","Requirement already satisfied: pyyaml in /opt/conda/lib/python3.7/site-packages (from timm==0.6.12->segmentation-models-pytorch) (6.0)\n","Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from torchvision>=0.5.0->segmentation-models-pytorch) (1.21.6)\n","Requirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from torchvision>=0.5.0->segmentation-models-pytorch) (2.28.2)\n","Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from torchvision>=0.5.0->segmentation-models-pytorch) (4.4.0)\n","Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from huggingface-hub->timm==0.6.12->segmentation-models-pytorch) (4.11.4)\n","Requirement already satisfied: packaging>=20.9 in /opt/conda/lib/python3.7/site-packages (from huggingface-hub->timm==0.6.12->segmentation-models-pytorch) (23.0)\n","Requirement already satisfied: filelock in /opt/conda/lib/python3.7/site-packages (from huggingface-hub->timm==0.6.12->segmentation-models-pytorch) (3.9.0)\n","Requirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from munch->pretrainedmodels==0.7.4->segmentation-models-pytorch) (1.16.0)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.7/site-packages (from requests->torchvision>=0.5.0->segmentation-models-pytorch) (2.1.1)\n","Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->torchvision>=0.5.0->segmentation-models-pytorch) (3.4)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->torchvision>=0.5.0->segmentation-models-pytorch) (1.26.14)\n","Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->torchvision>=0.5.0->segmentation-models-pytorch) (2022.12.7)\n","Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->huggingface-hub->timm==0.6.12->segmentation-models-pytorch) (3.11.0)\n","Building wheels for collected packages: efficientnet-pytorch, pretrainedmodels\n","  Building wheel for efficientnet-pytorch (setup.py) ... \u001b[?25ldone\n","\u001b[?25h  Created wheel for efficientnet-pytorch: filename=efficientnet_pytorch-0.7.1-py3-none-any.whl size=16446 sha256=38f3f81fe00223a03f90276a5add6e5b5fb2b0fe8b7397496926088dba0367f1\n","  Stored in directory: /root/.cache/pip/wheels/96/3f/5f/13976445f67f3b4e77b054e65f7f4c39016e92e8358fe088db\n","  Building wheel for pretrainedmodels (setup.py) ... \u001b[?25ldone\n","\u001b[?25h  Created wheel for pretrainedmodels: filename=pretrainedmodels-0.7.4-py3-none-any.whl size=60966 sha256=83410e77e80973534fa38ffce6e1a7c99759e28fe5a8ae2e28899254be11b631\n","  Stored in directory: /root/.cache/pip/wheels/4f/89/a3/5cf59e30a8a75c917c313f14da0f6209be2d147e3160b985d6\n","Successfully built efficientnet-pytorch pretrainedmodels\n","Installing collected packages: efficientnet-pytorch, pretrainedmodels, segmentation-models-pytorch\n","Successfully installed efficientnet-pytorch-0.7.1 pretrainedmodels-0.7.4 segmentation-models-pytorch-0.3.2\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0mRequirement already satisfied: transformers in /opt/conda/lib/python3.7/site-packages (4.26.1)\n","Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from transformers) (4.11.4)\n","Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.7/site-packages (from transformers) (2021.11.10)\n","Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.7/site-packages (from transformers) (0.13.2)\n","Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.7/site-packages (from transformers) (4.64.1)\n","Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.7/site-packages (from transformers) (6.0)\n","Requirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from transformers) (2.28.2)\n","Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.7/site-packages (from transformers) (1.21.6)\n","Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /opt/conda/lib/python3.7/site-packages (from transformers) (0.12.1)\n","Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.7/site-packages (from transformers) (23.0)\n","Requirement already satisfied: filelock in /opt/conda/lib/python3.7/site-packages (from transformers) (3.9.0)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.7/site-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.4.0)\n","Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->transformers) (3.11.0)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (1.26.14)\n","Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (3.4)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (2.1.1)\n","Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (2022.12.7)\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0mCollecting einops\n","  Downloading einops-0.6.0-py3-none-any.whl (41 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.6/41.6 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hInstalling collected packages: einops\n","Successfully installed einops-0.6.0\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0mRequirement already satisfied: tqdm in /opt/conda/lib/python3.7/site-packages (4.64.1)\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0m"]}],"source":["!pip install numpy\n","!pip install pandas\n","!pip install decord\n","!pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu117\n","!pip install pytorch-lightning\n","!pip install pytorchvideo\n","!pip install scikit-learn\n","!pip install scikit-multilearn\n","!pip install segmentation-models-pytorch\n","!pip install transformers\n","!pip install einops\n","!pip install tqdm"]},{"cell_type":"code","execution_count":7,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:26.595657Z","iopub.status.busy":"2023-03-12T18:14:26.595181Z","iopub.status.idle":"2023-03-12T18:14:26.623683Z","shell.execute_reply":"2023-03-12T18:14:26.622618Z","shell.execute_reply.started":"2023-03-12T18:14:26.595615Z"},"tags":[],"trusted":true},"outputs":[],"source":["#https://github.com/XuezheMax/apollo/blob/master/optim/apollo.py\n","import numpy as np\n","import torch\n","from torch.optim.optimizer import Optimizer\n","\n","\n","class Apollo(Optimizer):\n","    r\"\"\"Implements Atom algorithm.\n","        Arguments:\n","            params (iterable): iterable of parameters to optimize or dicts defining\n","                parameter groups\n","            lr (float): learning rate\n","            beta (float, optional): coefficient used for computing running averages of gradient (default: 0.9)\n","            eps (float, optional): term added to the denominator to improve numerical stability (default: 1e-4)\n","            rebound (str, optional): recified bound for diagonal hessian:\n","                ``'constant'`` | ``'belief'`` (default: None)\n","            warmup (int, optional): number of warmup steps (default: 500)\n","            init_lr (float, optional): initial learning rate for warmup (default: lr/1000)\n","            weight_decay (float, optional): weight decay coefficient (default: 0)\n","            weight_decay_type (str, optional): type of weight decay:\n","                ``'L2'`` | ``'decoupled'`` | ``'stable'`` (default: 'L2')\n","        \"\"\"\n","\n","    def __init__(self, params, lr, beta=0.9, eps=1e-4, rebound='constant', warmup=500, init_lr=None, weight_decay=0, weight_decay_type=None):\n","        if not 0.0 < lr:\n","            raise ValueError(\"Invalid learning rate value: {}\".format(lr))\n","        if not 0.0 <= eps:\n","            raise ValueError(\"Invalid epsilon value: {}\".format(eps))\n","        if not 0.0 <= beta < 1.0:\n","            raise ValueError(\"Invalid beta parameter at index 0: {}\".format(beta))\n","        if rebound not in ['constant', 'belief']:\n","            raise ValueError(\"Invalid recitifed bound: {}\".format(rebound))\n","        if not 0.0 <= warmup:\n","            raise ValueError(\"Invalid warmup updates: {}\".format(warmup))\n","        if init_lr is None:\n","            init_lr = lr / 1000\n","        if not 0.0 <= init_lr <= lr:\n","            raise ValueError(\"Invalid initial learning rate: {}\".format(init_lr))\n","        if not 0.0 <= weight_decay:\n","            raise ValueError(\"Invalid weight_decay value: {}\".format(weight_decay))\n","        if weight_decay_type is None:\n","            weight_decay_type = 'L2' if rebound == 'constant' else 'decoupled'\n","        if weight_decay_type not in ['L2', 'decoupled', 'stable']:\n","            raise ValueError(\"Invalid weight decay type: {}\".format(weight_decay_type))\n","\n","        defaults = dict(lr=lr, beta=beta, eps=eps, rebound=rebound,\n","                        warmup=warmup, init_lr=init_lr, base_lr=lr,\n","                        weight_decay=weight_decay, weight_decay_type=weight_decay_type)\n","        super(Apollo, self).__init__(params, defaults)\n","\n","    def __setstate__(self, state):\n","        super(Apollo, self).__setstate__(state)\n","\n","    @torch.no_grad()\n","    def step(self, closure=None):\n","        \"\"\"Performs a single optimization step.\n","        Arguments:\n","            closure (callable, optional): A closure that reevaluates the model\n","                and returns the loss.\n","        \"\"\"\n","        loss = None\n","        if closure is not None:\n","            with torch.enable_grad():\n","                loss = closure()\n","\n","        for group in self.param_groups:\n","            for p in group['params']:\n","                if p.grad is None:\n","                    continue\n","\n","                state = self.state[p]\n","\n","                # State initialization\n","                if len(state) == 0:\n","                    state['step'] = 0\n","                    # Exponential moving average of gradient values\n","                    state['exp_avg_grad'] = torch.zeros_like(p, memory_format=torch.preserve_format)\n","                    # Exponential moving average of squared gradient values\n","                    state['approx_hessian'] = torch.zeros_like(p, memory_format=torch.preserve_format)\n","                    # Previous update direction\n","                    state['update'] = torch.zeros_like(p, memory_format=torch.preserve_format)\n","\n","                # Calculate current lr\n","                if state['step'] < group['warmup']:\n","                    curr_lr = (group['base_lr'] - group['init_lr']) * state['step'] / group['warmup'] + group['init_lr']\n","                else:\n","                    curr_lr = group['lr']\n","\n","                # Perform optimization step\n","                grad = p.grad\n","                if grad.is_sparse:\n","                    raise RuntimeError('Atom does not support sparse gradients.')\n","\n","                # Perform step weight decay\n","                if group['weight_decay'] != 0 and group['weight_decay_type'] == 'L2':\n","                    grad = grad.add(p, alpha=group['weight_decay'])\n","\n","                beta = group['beta']\n","                eps = group['eps']\n","                exp_avg_grad = state['exp_avg_grad']\n","                B = state['approx_hessian']\n","                d_p = state['update']\n","\n","                state['step'] += 1\n","                bias_correction = 1 - beta ** state['step']\n","                alpha = (1 - beta) / bias_correction\n","\n","                # calc the diff grad\n","                delta_grad = grad - exp_avg_grad\n","                if group['rebound'] == 'belief':\n","                    rebound = delta_grad.norm(p=np.inf)\n","                else:\n","                    rebound = 0.01\n","                    eps = eps / rebound\n","\n","                # Update the running average grad\n","                exp_avg_grad.add_(delta_grad, alpha=alpha)\n","\n","                denom = d_p.norm(p=4).add(eps)\n","                d_p.div_(denom)\n","                v_sq = d_p.mul(d_p)\n","                delta = delta_grad.div_(denom).mul_(d_p).sum().mul(-alpha) - B.mul(v_sq).sum()\n","\n","                # Update B\n","                B.addcmul_(v_sq, delta)\n","\n","                # calc direction of parameter updates\n","                if group['rebound'] == 'belief':\n","                    denom = torch.max(B.abs(), rebound).add_(eps / alpha)\n","                else:\n","                    denom = B.abs().clamp_(min=rebound)\n","\n","                d_p.copy_(exp_avg_grad.div(denom))\n","\n","                # Perform step weight decay\n","                if group['weight_decay'] != 0 and group['weight_decay_type'] != 'L2':\n","                    if group['weight_decay_type'] == 'stable':\n","                        weight_decay = group['weight_decay'] / denom.mean().item()\n","                    else:\n","                        weight_decay = group['weight_decay']\n","                    d_p.add_(p, alpha=weight_decay)\n","\n","                p.add_(d_p, alpha=-curr_lr)\n","\n","        return "]},{"cell_type":"code","execution_count":8,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:26.974088Z","iopub.status.busy":"2023-03-12T18:14:26.973358Z","iopub.status.idle":"2023-03-12T18:14:26.988967Z","shell.execute_reply":"2023-03-12T18:14:26.987898Z","shell.execute_reply.started":"2023-03-12T18:14:26.974036Z"},"tags":[],"trusted":true},"outputs":[],"source":["#https://github.com/issamemari/pytorch-multilabel-balanced-sampler/blob/master/sampler.py\n","import random\n","import numpy as np\n","\n","from torch.utils.data.sampler import Sampler\n","\n","\n","class MultilabelBalancedRandomSampler(Sampler):\n","    \"\"\"\n","    MultilabelBalancedRandomSampler: Given a multilabel dataset of length n_samples and\n","    number of classes n_classes, samples from the data with equal probability per class\n","    effectively oversampling minority classes and undersampling majority classes at the\n","    same time. Note that using this sampler does not guarantee that the distribution of\n","    classes in the output samples will be uniform, since the dataset is multilabel and\n","    sampling is based on a single class. This does however guarantee that all classes\n","    will have at least batch_size / n_classes samples as batch_size approaches infinity\n","    \"\"\"\n","\n","    def __init__(self, labels, indices=None, class_choice=\"least_sampled\"):\n","        \"\"\"\n","        Parameters:\n","        -----------\n","            labels: a multi-hot encoding numpy array of shape (n_samples, n_classes)\n","            indices: an arbitrary-length 1-dimensional numpy array representing a list\n","            of indices to sample only from\n","            class_choice: a string indicating how class will be selected for every\n","            sample:\n","                \"least_sampled\": class with the least number of sampled labels so far\n","                \"random\": class is chosen uniformly at random\n","                \"cycle\": the sampler cycles through the classes sequentially\n","        \"\"\"\n","        self.labels = labels\n","        self.indices = indices\n","        if self.indices is None:\n","            self.indices = range(len(labels))\n","\n","        self.num_classes = self.labels.shape[1]\n","\n","        # List of lists of example indices per class\n","        self.class_indices = []\n","        for class_ in range(self.num_classes):\n","            lst = np.where(self.labels[:, class_] == 1)[0]\n","            lst = lst[np.isin(lst, self.indices)]\n","            self.class_indices.append(lst)\n","\n","        self.counts = [0] * self.num_classes\n","\n","        assert class_choice in [\"least_sampled\", \"random\", \"cycle\"]\n","        self.class_choice = class_choice\n","        self.current_class = 0\n","\n","    def __iter__(self):\n","        self.count = 0\n","        return self\n","\n","    def __next__(self):\n","        if self.count >= len(self.indices):\n","            raise StopIteration\n","        self.count += 1\n","        return self.sample()\n","\n","    def sample(self):\n","        class_ = self.get_class()\n","        class_indices = self.class_indices[class_]\n","        chosen_index = np.random.choice(class_indices)\n","        if self.class_choice == \"least_sampled\":\n","            for class_, indicator in enumerate(self.labels[chosen_index]):\n","                if indicator == 1:\n","                    self.counts[class_] += 1\n","        return chosen_index\n","\n","    def get_class(self):\n","        if self.class_choice == \"random\":\n","            class_ = random.randint(0, self.labels.shape[1] - 1)\n","        elif self.class_choice == \"cycle\":\n","            class_ = self.current_class\n","            self.current_class = (self.current_class + 1) % self.labels.shape[1]\n","        elif self.class_choice == \"least_sampled\":\n","            min_count = self.counts[0]\n","            min_classes = [0]\n","            for class_ in range(1, self.num_classes):\n","                if self.counts[class_] < min_count:\n","                    min_count = self.counts[class_]\n","                    min_classes = [class_]\n","                if self.counts[class_] == min_count:\n","                    min_classes.append(class_)\n","            try:\n","                class_ = np.random.choice(min_classes)\n","            except: pass\n","        return class_\n","\n","    def __len__(self):\n","        return len(self.indices)"]},{"cell_type":"code","execution_count":9,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:27.303739Z","iopub.status.busy":"2023-03-12T18:14:27.303416Z","iopub.status.idle":"2023-03-12T18:14:29.290170Z","shell.execute_reply":"2023-03-12T18:14:29.289161Z","shell.execute_reply.started":"2023-03-12T18:14:27.303712Z"},"tags":[],"trusted":true},"outputs":[],"source":["import numpy as np\n","import pandas as pd\n","import torch\n","import torch.nn as nn\n","import torch.nn.functional as F\n","import pytorch_lightning as pl\n","\n","from einops import rearrange\n","from decord import VideoReader\n","from sklearn.metrics import f1_score\n","from torch.utils.data import Dataset, DataLoader\n","from segmentation_models_pytorch.losses import FocalLoss\n","from transformers import AutoModel, AutoImageProcessor, AutoConfig\n","from skmultilearn.model_selection import iterative_train_test_split\n","from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping\n","from pytorchvideo.transforms.transforms_factory import create_video_transform"]},{"cell_type":"code","execution_count":10,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:29.292938Z","iopub.status.busy":"2023-03-12T18:14:29.292368Z","iopub.status.idle":"2023-03-12T18:14:29.302210Z","shell.execute_reply":"2023-03-12T18:14:29.301144Z","shell.execute_reply.started":"2023-03-12T18:14:29.292899Z"},"tags":[],"trusted":true},"outputs":[],"source":["config = {\n","    \"seed\":1203,\n","    \"model_name\":\"facebook/timesformer-base-finetuned-k400\",\n","    \"batch_size\":8,\n","    \"learning_rate\":1e-5,\n","    \"data_dir\":'/kaggle/input/dacon-car-crash',\n","    \"checkpoint_dir\":'./checkpoint',\n","    \"submission_dir\":'./submission',\n","    \"n_classes\":(3,2),\n","    \"label_dict\":{\n","        -1:[-1,-1],\n","        0:[0,0],\n","        1:[0,1],\n","        2:[1,0],\n","        3:[1,1],\n","        4:[2,0],\n","        5:[2,1],\n","    },\n","    \"label_reverse_dict\":{\n","        (0,0):0,\n","        (0,1):1,\n","        (1,0):2,\n","        (1,1):3,\n","        (2,0):4,\n","        (2,1):5,\n","    }\n","}\n"]},{"cell_type":"code","execution_count":11,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:29.304736Z","iopub.status.busy":"2023-03-12T18:14:29.303666Z","iopub.status.idle":"2023-03-12T18:14:29.319536Z","shell.execute_reply":"2023-03-12T18:14:29.318449Z","shell.execute_reply.started":"2023-03-12T18:14:29.304695Z"},"tags":[],"trusted":true},"outputs":[{"data":{"text/plain":["1203"]},"execution_count":11,"metadata":{},"output_type":"execute_result"}],"source":["pl.seed_everything(config['seed'])"]},{"cell_type":"code","execution_count":12,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:29.322531Z","iopub.status.busy":"2023-03-12T18:14:29.322155Z","iopub.status.idle":"2023-03-12T18:14:29.358568Z","shell.execute_reply":"2023-03-12T18:14:29.357677Z","shell.execute_reply.started":"2023-03-12T18:14:29.322495Z"},"tags":[],"trusted":true},"outputs":[],"source":["train_df = pd.read_csv(f\"{config['data_dir']}/train.csv\")\n","test_df = pd.read_csv(f\"{config['data_dir']}/test.csv\")"]},{"cell_type":"code","execution_count":13,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:29.360569Z","iopub.status.busy":"2023-03-12T18:14:29.360173Z","iopub.status.idle":"2023-03-12T18:14:29.401758Z","shell.execute_reply":"2023-03-12T18:14:29.400826Z","shell.execute_reply.started":"2023-03-12T18:14:29.360534Z"},"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["       sample_id              video_path  label\n","0     TRAIN_0000  ./train/TRAIN_0000.mp4      7\n","1     TRAIN_0001  ./train/TRAIN_0001.mp4      7\n","2     TRAIN_0002  ./train/TRAIN_0002.mp4      0\n","3     TRAIN_0003  ./train/TRAIN_0003.mp4      0\n","4     TRAIN_0004  ./train/TRAIN_0004.mp4      1\n","...          ...                     ...    ...\n","2693  TRAIN_2693  ./train/TRAIN_2693.mp4      3\n","2694  TRAIN_2694  ./train/TRAIN_2694.mp4      5\n","2695  TRAIN_2695  ./train/TRAIN_2695.mp4      0\n","2696  TRAIN_2696  ./train/TRAIN_2696.mp4      0\n","2697  TRAIN_2697  ./train/TRAIN_2697.mp4      0\n","\n","[2672 rows x 3 columns]\n"]}],"source":["train_df.loc[train_df['sample_id'].isin(['TRAIN_2236', 'TRAIN_2596']), 'label'] = 0\n","train_df.loc[train_df['sample_id'].isin(['TRAIN_0061', 'TRAIN_0107', 'TRAIN_0123', 'TRAIN_0294',\n","    'TRAIN_0800', 'TRAIN_1280', 'TRAIN_1590', 'TRAIN_2302', 'TRAIN_2548']), 'label'] = 1\n","train_df.loc[train_df['sample_id'].isin(['TRAIN_0056', 'TRAIN_0129', 'TRAIN_0149', 'TRAIN_0242',\n","    'TRAIN_0263', 'TRAIN_0728', 'TRAIN_0861', 'TRAIN_0889', 'TRAIN_0896', 'TRAIN_0920', 'TRAIN_1098',\n","    'TRAIN_1169', 'TRAIN_1251', 'TRAIN_1605', 'TRAIN_1654', 'TRAIN_1656', 'TRAIN_1698', 'TRAIN_1795',\n","    'TRAIN_1839', 'TRAIN_1955', 'TRAIN_2249', 'TRAIN_2388', 'TRAIN_2647']), 'label'] = 3\n","train_df.loc[train_df['sample_id'].isin(['TRAIN_0221', 'TRAIN_0856', 'TRAIN_1081', 'TRAIN_1263',\n","    'TRAIN_1488', 'TRAIN_1492', 'TRAIN_1874', 'TRAIN_2166', 'TRAIN_2555', 'TRAIN_2595', 'TRAIN_2622']),\n","    'label'] = 4\n","train_df.loc[train_df['sample_id'].isin(['TRAIN_0017', 'TRAIN_0225', 'TRAIN_0306', 'TRAIN_1193',\n","    'TRAIN_1771', 'TRAIN_1848', 'TRAIN_2140', 'TRAIN_2298', 'TRAIN_2532', 'TRAIN_2570']), 'label'] = 5\n","train_df.loc[train_df['sample_id'].isin(['TRAIN_0809']), 'label'] = 6\n","train_df.loc[train_df['sample_id'].isin(['TRAIN_0020', 'TRAIN_0507', 'TRAIN_0617', 'TRAIN_1023',\n","    'TRAIN_1420', 'TRAIN_1531', 'TRAIN_2033', 'TRAIN_2063']), 'label'] = 7\n","train_df.loc[train_df['sample_id'].isin(['TRAIN_0332', 'TRAIN_0674', 'TRAIN_0720', 'TRAIN_0917',\n","    'TRAIN_1287', 'TRAIN_1699', 'TRAIN_1923', 'TRAIN_1949', 'TRAIN_2239', 'TRAIN_2491', 'TRAIN_2534', 'TRAIN_2615']), 'label'] = 9\n","train_df.loc[train_df['sample_id'].isin(['TRAIN_0877', 'TRAIN_1728', 'TRAIN_2328', 'TRAIN_2685']), 'label'] = 10\n","train_df.loc[train_df['sample_id'].isin(['TRAIN_0341', 'TRAIN_1041', 'TRAIN_1581', 'TRAIN_1727', 'TRAIN_2607']), 'label'] = 11\n","train_df.loc[train_df['sample_id'].isin(['TRAIN_2571']), 'label'] = 12\n","\n","# 삭제할 sample_id 리스트\n","del_list = ['TRAIN_0048', 'TRAIN_0234', 'TRAIN_0238', 'TRAIN_0325', 'TRAIN_0528', 'TRAIN_0554', 'TRAIN_0668',\n","                  'TRAIN_0705', 'TRAIN_0875', 'TRAIN_1082', 'TRAIN_1151', 'TRAIN_1337', 'TRAIN_1362', 'TRAIN_1506',\n","                  'TRAIN_1674', 'TRAIN_1681', 'TRAIN_1753', 'TRAIN_1838', 'TRAIN_2191', 'TRAIN_2356', 'TRAIN_2360',\n","                  'TRAIN_2428', 'TRAIN_2451', 'TRAIN_2486', 'TRAIN_2558', 'TRAIN_2658']\n","\n","# sample_id가 삭제할 리스트에 포함되지 않는 경우만 추출하여 새로운 데이터프레임 생성\n","train_df = train_df[~train_df['sample_id'].isin(del_list)]\n","\n","# 결과 출력\n","print(train_df)"]},{"cell_type":"code","execution_count":14,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:30.381849Z","iopub.status.busy":"2023-03-12T18:14:30.381395Z","iopub.status.idle":"2023-03-12T18:14:30.413618Z","shell.execute_reply":"2023-03-12T18:14:30.412596Z","shell.execute_reply.started":"2023-03-12T18:14:30.381810Z"},"trusted":true},"outputs":[{"data":{"text/plain":["label\n","0     1784\n","7      287\n","1      284\n","3       95\n","9       45\n","2       41\n","5       36\n","11      36\n","4       23\n","8       22\n","10       8\n","12       7\n","6        4\n","dtype: int64"]},"execution_count":14,"metadata":{},"output_type":"execute_result"}],"source":["train_df.value_counts('label')"]},{"cell_type":"code","execution_count":15,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:30.923983Z","iopub.status.busy":"2023-03-12T18:14:30.923558Z","iopub.status.idle":"2023-03-12T18:14:30.946612Z","shell.execute_reply":"2023-03-12T18:14:30.945668Z","shell.execute_reply.started":"2023-03-12T18:14:30.923947Z"},"trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>sample_id</th>\n","      <th>video_path</th>\n","      <th>label</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>TRAIN_0000</td>\n","      <td>./train/TRAIN_0000.mp4</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>TRAIN_0001</td>\n","      <td>./train/TRAIN_0001.mp4</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>TRAIN_0004</td>\n","      <td>./train/TRAIN_0004.mp4</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>TRAIN_0006</td>\n","      <td>./train/TRAIN_0006.mp4</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>TRAIN_0007</td>\n","      <td>./train/TRAIN_0007.mp4</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>2685</th>\n","      <td>TRAIN_2685</td>\n","      <td>./train/TRAIN_2685.mp4</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>2689</th>\n","      <td>TRAIN_2689</td>\n","      <td>./train/TRAIN_2689.mp4</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2692</th>\n","      <td>TRAIN_2692</td>\n","      <td>./train/TRAIN_2692.mp4</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2693</th>\n","      <td>TRAIN_2693</td>\n","      <td>./train/TRAIN_2693.mp4</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>2694</th>\n","      <td>TRAIN_2694</td>\n","      <td>./train/TRAIN_2694.mp4</td>\n","      <td>4</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>888 rows × 3 columns</p>\n","</div>"],"text/plain":["       sample_id              video_path  label\n","0     TRAIN_0000  ./train/TRAIN_0000.mp4      0\n","1     TRAIN_0001  ./train/TRAIN_0001.mp4      0\n","4     TRAIN_0004  ./train/TRAIN_0004.mp4      0\n","6     TRAIN_0006  ./train/TRAIN_0006.mp4      2\n","7     TRAIN_0007  ./train/TRAIN_0007.mp4      0\n","...          ...                     ...    ...\n","2685  TRAIN_2685  ./train/TRAIN_2685.mp4      3\n","2689  TRAIN_2689  ./train/TRAIN_2689.mp4      0\n","2692  TRAIN_2692  ./train/TRAIN_2692.mp4      0\n","2693  TRAIN_2693  ./train/TRAIN_2693.mp4      2\n","2694  TRAIN_2694  ./train/TRAIN_2694.mp4      4\n","\n","[888 rows x 3 columns]"]},"execution_count":15,"metadata":{},"output_type":"execute_result"}],"source":["train_df = train_df[train_df['label'] != 0] # 0인라벨 제거\n","train_df['label'].replace({1: 0, 7: 0,\n","                           2: 1, 8: 1,\n","                           3: 2, 9: 2,\n","                           4: 3,10: 3,\n","                           5: 4,11: 4,\n","                           6: 5,12: 5}, inplace=True)\n","train_df"]},{"cell_type":"code","execution_count":16,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:31.726928Z","iopub.status.busy":"2023-03-12T18:14:31.726043Z","iopub.status.idle":"2023-03-12T18:14:31.738263Z","shell.execute_reply":"2023-03-12T18:14:31.736919Z","shell.execute_reply.started":"2023-03-12T18:14:31.726887Z"},"trusted":true},"outputs":[{"data":{"text/plain":["label\n","0    571\n","2    140\n","4     72\n","1     63\n","3     31\n","5     11\n","dtype: int64"]},"execution_count":16,"metadata":{},"output_type":"execute_result"}],"source":["train_df.value_counts('label')"]},{"cell_type":"code","execution_count":17,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:32.432629Z","iopub.status.busy":"2023-03-12T18:14:32.432006Z","iopub.status.idle":"2023-03-12T18:14:32.441922Z","shell.execute_reply":"2023-03-12T18:14:32.440722Z","shell.execute_reply.started":"2023-03-12T18:14:32.432590Z"},"tags":[],"trusted":true},"outputs":[],"source":["train_df['sample_id'] = train_df['sample_id'].apply(lambda x: int(x.split('_')[1]))\n","test_df['sample_id'] = test_df['sample_id'].apply(lambda x: int(x.split('_')[1]))"]},{"cell_type":"code","execution_count":18,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:33.082676Z","iopub.status.busy":"2023-03-12T18:14:33.081671Z","iopub.status.idle":"2023-03-12T18:14:33.090666Z","shell.execute_reply":"2023-03-12T18:14:33.089522Z","shell.execute_reply.started":"2023-03-12T18:14:33.082627Z"},"tags":[],"trusted":true},"outputs":[],"source":["train_df['video_path'] = train_df['video_path'].apply(lambda x: config['data_dir'] + x[1:])\n","test_df['video_path'] = test_df['video_path'].apply(lambda x: config['data_dir'] + x[1:])"]},{"cell_type":"code","execution_count":19,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:33.659662Z","iopub.status.busy":"2023-03-12T18:14:33.657113Z","iopub.status.idle":"2023-03-12T18:14:33.666638Z","shell.execute_reply":"2023-03-12T18:14:33.665467Z","shell.execute_reply.started":"2023-03-12T18:14:33.659622Z"},"tags":[],"trusted":true},"outputs":[],"source":["test_df['label']=-1\n","test_df['label_split'] = test_df['label'].apply(config['label_dict'].get)"]},{"cell_type":"code","execution_count":20,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:37.064592Z","iopub.status.busy":"2023-03-12T18:14:37.063394Z","iopub.status.idle":"2023-03-12T18:14:37.071966Z","shell.execute_reply":"2023-03-12T18:14:37.070818Z","shell.execute_reply.started":"2023-03-12T18:14:37.064533Z"},"tags":[],"trusted":true},"outputs":[],"source":["train_df['label_split'] = train_df['label'].apply(config['label_dict'].get)\n","train_label_split = np.array(train_df['label_split'].tolist())"]},{"cell_type":"code","execution_count":21,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:37.574982Z","iopub.status.busy":"2023-03-12T18:14:37.573944Z","iopub.status.idle":"2023-03-12T18:14:37.582891Z","shell.execute_reply":"2023-03-12T18:14:37.581897Z","shell.execute_reply.started":"2023-03-12T18:14:37.574933Z"},"tags":[],"trusted":true},"outputs":[],"source":["train_label_multi_hot = np.hstack([np.eye(n_class, dtype=np.int32)[train_label_split[:,idx]] for idx, n_class in enumerate(config['n_classes'])])\n","train_df['label_multi_hot'] = train_label_multi_hot.tolist()"]},{"cell_type":"code","execution_count":22,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:38.027200Z","iopub.status.busy":"2023-03-12T18:14:38.026159Z","iopub.status.idle":"2023-03-12T18:14:38.048636Z","shell.execute_reply":"2023-03-12T18:14:38.047510Z","shell.execute_reply.started":"2023-03-12T18:14:38.027148Z"},"tags":[],"trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>sample_id</th>\n","      <th>video_path</th>\n","      <th>label</th>\n","      <th>label_split</th>\n","      <th>label_multi_hot</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>/kaggle/input/dacon-car-crash/train/TRAIN_0000...</td>\n","      <td>0</td>\n","      <td>[0, 0]</td>\n","      <td>[1, 0, 0, 1, 0]</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1</td>\n","      <td>/kaggle/input/dacon-car-crash/train/TRAIN_0001...</td>\n","      <td>0</td>\n","      <td>[0, 0]</td>\n","      <td>[1, 0, 0, 1, 0]</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>4</td>\n","      <td>/kaggle/input/dacon-car-crash/train/TRAIN_0004...</td>\n","      <td>0</td>\n","      <td>[0, 0]</td>\n","      <td>[1, 0, 0, 1, 0]</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>6</td>\n","      <td>/kaggle/input/dacon-car-crash/train/TRAIN_0006...</td>\n","      <td>2</td>\n","      <td>[1, 0]</td>\n","      <td>[0, 1, 0, 1, 0]</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>7</td>\n","      <td>/kaggle/input/dacon-car-crash/train/TRAIN_0007...</td>\n","      <td>0</td>\n","      <td>[0, 0]</td>\n","      <td>[1, 0, 0, 1, 0]</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>2685</th>\n","      <td>2685</td>\n","      <td>/kaggle/input/dacon-car-crash/train/TRAIN_2685...</td>\n","      <td>3</td>\n","      <td>[1, 1]</td>\n","      <td>[0, 1, 0, 0, 1]</td>\n","    </tr>\n","    <tr>\n","      <th>2689</th>\n","      <td>2689</td>\n","      <td>/kaggle/input/dacon-car-crash/train/TRAIN_2689...</td>\n","      <td>0</td>\n","      <td>[0, 0]</td>\n","      <td>[1, 0, 0, 1, 0]</td>\n","    </tr>\n","    <tr>\n","      <th>2692</th>\n","      <td>2692</td>\n","      <td>/kaggle/input/dacon-car-crash/train/TRAIN_2692...</td>\n","      <td>0</td>\n","      <td>[0, 0]</td>\n","      <td>[1, 0, 0, 1, 0]</td>\n","    </tr>\n","    <tr>\n","      <th>2693</th>\n","      <td>2693</td>\n","      <td>/kaggle/input/dacon-car-crash/train/TRAIN_2693...</td>\n","      <td>2</td>\n","      <td>[1, 0]</td>\n","      <td>[0, 1, 0, 1, 0]</td>\n","    </tr>\n","    <tr>\n","      <th>2694</th>\n","      <td>2694</td>\n","      <td>/kaggle/input/dacon-car-crash/train/TRAIN_2694...</td>\n","      <td>4</td>\n","      <td>[2, 0]</td>\n","      <td>[0, 0, 1, 1, 0]</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>888 rows × 5 columns</p>\n","</div>"],"text/plain":["      sample_id                                         video_path  label  \\\n","0             0  /kaggle/input/dacon-car-crash/train/TRAIN_0000...      0   \n","1             1  /kaggle/input/dacon-car-crash/train/TRAIN_0001...      0   \n","4             4  /kaggle/input/dacon-car-crash/train/TRAIN_0004...      0   \n","6             6  /kaggle/input/dacon-car-crash/train/TRAIN_0006...      2   \n","7             7  /kaggle/input/dacon-car-crash/train/TRAIN_0007...      0   \n","...         ...                                                ...    ...   \n","2685       2685  /kaggle/input/dacon-car-crash/train/TRAIN_2685...      3   \n","2689       2689  /kaggle/input/dacon-car-crash/train/TRAIN_2689...      0   \n","2692       2692  /kaggle/input/dacon-car-crash/train/TRAIN_2692...      0   \n","2693       2693  /kaggle/input/dacon-car-crash/train/TRAIN_2693...      2   \n","2694       2694  /kaggle/input/dacon-car-crash/train/TRAIN_2694...      4   \n","\n","     label_split  label_multi_hot  \n","0         [0, 0]  [1, 0, 0, 1, 0]  \n","1         [0, 0]  [1, 0, 0, 1, 0]  \n","4         [0, 0]  [1, 0, 0, 1, 0]  \n","6         [1, 0]  [0, 1, 0, 1, 0]  \n","7         [0, 0]  [1, 0, 0, 1, 0]  \n","...          ...              ...  \n","2685      [1, 1]  [0, 1, 0, 0, 1]  \n","2689      [0, 0]  [1, 0, 0, 1, 0]  \n","2692      [0, 0]  [1, 0, 0, 1, 0]  \n","2693      [1, 0]  [0, 1, 0, 1, 0]  \n","2694      [2, 0]  [0, 0, 1, 1, 0]  \n","\n","[888 rows x 5 columns]"]},"execution_count":22,"metadata":{},"output_type":"execute_result"}],"source":["train_df"]},{"cell_type":"code","execution_count":23,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:38.415414Z","iopub.status.busy":"2023-03-12T18:14:38.415074Z","iopub.status.idle":"2023-03-12T18:14:38.472613Z","shell.execute_reply":"2023-03-12T18:14:38.471656Z","shell.execute_reply.started":"2023-03-12T18:14:38.415384Z"},"tags":[],"trusted":true},"outputs":[],"source":["train_df_for_dataset, _ , val_df_for_dataset, _  = iterative_train_test_split(X=train_df.values, y=train_label_multi_hot, test_size=0.2)\n","test_df_for_dataset = test_df.values"]},{"cell_type":"code","execution_count":24,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:39.091557Z","iopub.status.busy":"2023-03-12T18:14:39.090900Z","iopub.status.idle":"2023-03-12T18:14:39.097202Z","shell.execute_reply":"2023-03-12T18:14:39.096002Z","shell.execute_reply.started":"2023-03-12T18:14:39.091511Z"},"tags":[],"trusted":true},"outputs":[],"source":["train_multi_hot_for_sampler = np.array(train_df_for_dataset[:,4].tolist())"]},{"cell_type":"code","execution_count":25,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:39.602923Z","iopub.status.busy":"2023-03-12T18:14:39.602205Z","iopub.status.idle":"2023-03-12T18:14:39.613528Z","shell.execute_reply":"2023-03-12T18:14:39.610543Z","shell.execute_reply.started":"2023-03-12T18:14:39.602882Z"},"tags":[],"trusted":true},"outputs":[],"source":["class VideoDataset(Dataset):\n","    def __init__(self, df_for_dataset, transform=None):\n","        self.sample_id = df_for_dataset[:,0]\n","        self.video_path = df_for_dataset[:,1]\n","        self.label = df_for_dataset[:,2]\n","        self.label_split = np.array(df_for_dataset[:,3].tolist())\n","        self.transform = transform\n","\n","    def __len__(self):\n","        return len(self.sample_id)\n","\n","    def __getitem__(self, idx):\n","        sample_id = self.sample_id[idx]\n","        video_path = self.video_path[idx]\n","        vr = VideoReader(video_path)\n","        video = torch.from_numpy(vr.get_batch(range(50)).asnumpy())\n","        video = rearrange(video, 't h w c -> c t h w')\n","        label = self.label[idx]\n","        label_split = self.label_split[idx]\n","        \n","        if self.transform:\n","            video = self.transform(video)\n","        video = rearrange(video, 'c t h w -> t c h w')\n","\n","        sample = {\n","            'sample_id':sample_id,\n","            'video':video,\n","            'label':label,\n","            'label_split':label_split\n","        }\n","        \n","        return sample"]},{"cell_type":"code","execution_count":26,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:40.194984Z","iopub.status.busy":"2023-03-12T18:14:40.194271Z","iopub.status.idle":"2023-03-12T18:14:41.398384Z","shell.execute_reply":"2023-03-12T18:14:41.397457Z","shell.execute_reply.started":"2023-03-12T18:14:40.194944Z"},"tags":[],"trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"3219835366ab499baa5afb31f9bd3536","version_major":2,"version_minor":0},"text/plain":["Downloading (…)lve/main/config.json:   0%|          | 0.00/22.7k [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"b1283a74828446888a274eefa9831690","version_major":2,"version_minor":0},"text/plain":["Downloading (…)rocessor_config.json:   0%|          | 0.00/412 [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"}],"source":["model_config = AutoConfig.from_pretrained(config['model_name'])\n","image_processor_config = AutoImageProcessor.from_pretrained(config['model_name'])"]},{"cell_type":"code","execution_count":27,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:41.401255Z","iopub.status.busy":"2023-03-12T18:14:41.400586Z","iopub.status.idle":"2023-03-12T18:14:41.409997Z","shell.execute_reply":"2023-03-12T18:14:41.408886Z","shell.execute_reply.started":"2023-03-12T18:14:41.401216Z"},"tags":[],"trusted":true},"outputs":[],"source":["from torchvision.transforms import Compose, RandomHorizontalFlip, RandomResizedCrop\n","\n","train_transform = Compose([\n","    create_video_transform(\n","        mode='train',\n","        num_samples=model_config.num_frames,\n","        video_mean=tuple(image_processor_config.image_mean),\n","        video_std=tuple(image_processor_config.image_std),\n","        crop_size=tuple(image_processor_config.crop_size.values())\n","    ),\n","    RandomHorizontalFlip(p=0.5),\n","    RandomResizedCrop(size=(224, 224), scale=(0.5, 1.0))\n","])\n","\n","val_transform = create_video_transform(\n","    mode='val',\n","    num_samples=model_config.num_frames,\n","    video_mean=tuple(image_processor_config.image_mean),\n","    video_std=tuple(image_processor_config.image_std),\n","    crop_size=tuple(image_processor_config.crop_size.values())\n",")\n"]},{"cell_type":"code","execution_count":28,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:41.412303Z","iopub.status.busy":"2023-03-12T18:14:41.411605Z","iopub.status.idle":"2023-03-12T18:14:41.424478Z","shell.execute_reply":"2023-03-12T18:14:41.423534Z","shell.execute_reply.started":"2023-03-12T18:14:41.412266Z"},"tags":[],"trusted":true},"outputs":[],"source":["train_dataset = VideoDataset(train_df_for_dataset, transform=train_transform)\n","val_dataset = VideoDataset(val_df_for_dataset, transform=val_transform)\n","test_dataset = VideoDataset(test_df_for_dataset, transform=val_transform)"]},{"cell_type":"code","execution_count":29,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:42.117604Z","iopub.status.busy":"2023-03-12T18:14:42.116664Z","iopub.status.idle":"2023-03-12T18:14:42.126847Z","shell.execute_reply":"2023-03-12T18:14:42.125627Z","shell.execute_reply.started":"2023-03-12T18:14:42.117551Z"},"tags":[],"trusted":true},"outputs":[],"source":["train_sampler = MultilabelBalancedRandomSampler(train_multi_hot_for_sampler)\n","train_dataloader = DataLoader(train_dataset, batch_size= config['batch_size'], sampler=train_sampler)\n","val_dataloader = DataLoader(val_dataset, batch_size = config['batch_size']*2)\n","test_dataloader = DataLoader(test_dataset, batch_size = config['batch_size']*2)"]},{"cell_type":"code","execution_count":30,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:43.859861Z","iopub.status.busy":"2023-03-12T18:14:43.859419Z","iopub.status.idle":"2023-03-12T18:14:43.878325Z","shell.execute_reply":"2023-03-12T18:14:43.877293Z","shell.execute_reply.started":"2023-03-12T18:14:43.859826Z"},"tags":[],"trusted":true},"outputs":[],"source":["class PLVideoModel(pl.LightningModule):\n","    def __init__(self, config):\n","        super().__init__()\n","        self.config = config\n","        self.learning_rate = config['learning_rate']\n","        self.model = AutoModel.from_pretrained(config['model_name'])\n","        self.classifiers = nn.ModuleList([\n","            nn.LazyLinear(n_class) for n_class in config['n_classes']\n","        ])\n","        self.loss = FocalLoss('multiclass')\n","\n","    def forward(self, x):\n","        x = self.model(x).last_hidden_state.mean(dim=1)\n","        x_out = [classifier(x) for classifier in self.classifiers]\n","        return x_out\n","\n","\n","    def training_step(self, batch, batch_idx):\n","        video, label, label_split = batch['video'], batch['label'], batch['label_split']\n","        y_hats = self.forward(batch[\"video\"])\n","        loss = sum([self.loss(y_hats[i], batch[\"label_split\"][:,i]) for i in range(len(self.config['n_classes']))])\n","        loss = loss/len(self.config['n_classes'])\n","        self.log(\"train_loss\", loss)\n","        return loss\n","\n","    def validation_step(self, batch, batch_idx):\n","        video, label, label_split = batch['video'], batch['label'], batch['label_split']\n","        y_hats = self.forward(batch[\"video\"])\n","        step_output = [*y_hats, label]\n","        return step_output\n","\n","    def predict_step(self, batch, batch_idx):\n","        video, _, _ = batch['video'], batch['label'], batch['label_split']\n","        y_hats = self.forward(batch[\"video\"])\n","        step_output = y_hats\n","        return step_output\n","\n","    def validation_epoch_end(self, step_outputs):\n","        pred1, pred2, label = [], [], []\n","        for step_output in step_outputs:\n","            pred1.append(step_output[0])\n","            pred2.append(step_output[1])\n","            label.append(step_output[2])\n","\n","        pred1 = torch.cat(pred1).argmax(1)\n","        pred2 = torch.cat(pred2).argmax(1)\n","        label = torch.cat(label).tolist()\n","\n","        pred = torch.stack([pred1,pred2],dim=1).cpu().detach().numpy().tolist()\n","        pred = list(map(lambda x: self.config['label_reverse_dict'].get(tuple(x),0),pred))\n","\n","        score = f1_score(label,pred, average='macro')\n","        self.log(\"val_score\", score)\n","        return score\n","\n","    def post_preproc(self, step_outputs):\n","        pred1, pred2 = [], []\n","        for step_output in step_outputs:\n","            pred1.append(step_output[0])\n","            pred2.append(step_output[1])\n","\n","        pred1 = torch.cat(pred1).argmax(1)\n","        pred2 = torch.cat(pred2).argmax(1)\n","\n","        pred = torch.stack([pred1,pred2],dim=1).cpu().detach().numpy().tolist()\n","        pred = list(map(lambda x: self.config['label_reverse_dict'].get(tuple(x),0),pred))\n","\n","        return pred\n","\n","    def configure_optimizers(self):\n","        optimizer = Apollo(self.parameters(), lr=self.learning_rate)\n","        return [optimizer]"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T18:14:48.343677Z","iopub.status.busy":"2023-03-12T18:14:48.342859Z"},"tags":[],"trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"b5b34af742894819a62c147b2065cf8c","version_major":2,"version_minor":0},"text/plain":["Downloading (…)\"pytorch_model.bin\";:   0%|          | 0.00/486M [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stderr","output_type":"stream","text":["Some weights of the model checkpoint at facebook/timesformer-base-finetuned-k400 were not used when initializing TimesformerModel: ['classifier.weight', 'classifier.bias']\n","- This IS expected if you are initializing TimesformerModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n","- This IS NOT expected if you are initializing TimesformerModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n","/opt/conda/lib/python3.7/site-packages/torch/nn/modules/lazy.py:180: UserWarning: Lazy modules are a new feature under heavy development so changes to the API or functionality can happen at any moment.\n","  warnings.warn('Lazy modules are a new feature under heavy development '\n","/opt/conda/lib/python3.7/site-packages/pytorch_lightning/utilities/model_summary/model_summary.py:412: UserWarning: A layer with UninitializedParameter was found. Thus, the total number of parameters detected may be inaccurate.\n","  \"A layer with UninitializedParameter was found. \"\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Sanity Checking: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"09822515b01b4b2abf697d3da01d76f2","version_major":2,"version_minor":0},"text/plain":["Training: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"}],"source":["checkpoint_callback = ModelCheckpoint(\n","    monitor='val_score',\n","    dirpath=config['checkpoint_dir'],\n","    filename=f'{config[\"model_name\"]}'+'-{epoch:02d}-{train_loss:.4f}-{val_score:.4f}',\n","    mode='max',\n","    save_last=True\n",")\n","\n","early_stop_callback = EarlyStopping(\n","    monitor=\"train_loss\",\n","    patience=3,\n","    verbose=False,\n","    mode=\"min\"\n",")\n","\n","pl_video_model = PLVideoModel(config)\n","\n","trainer = pl.Trainer(\n","    max_epochs=50,\n","    accelerator='auto', \n","    precision=16,\n","#     callbacks=[early_stop_callback, checkpoint_callback]  \n","    callbacks=[checkpoint_callback]\n",")\n","trainer.fit(pl_video_model, train_dataloader, val_dataloader)"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["cd checkpoint/facebook"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["ls"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["cd ../.."]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["pred = trainer.predict(pl_video_model, test_dataloader)"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["pred_post_proc = pl_video_model.post_preproc(pred)\n","submit = pd.read_csv(f\"{config['data_dir']}/sample_submission.csv\")\n","submit['label'] = pred_post_proc\n","submit.to_csv(f\"./final_wt.csv\", index=False)"]},{"cell_type":"code","execution_count":34,"metadata":{"execution":{"iopub.execute_input":"2023-03-12T14:24:30.957016Z","iopub.status.busy":"2023-03-12T14:24:30.956611Z","iopub.status.idle":"2023-03-12T15:42:32.384425Z","shell.execute_reply":"2023-03-12T15:42:32.383358Z","shell.execute_reply.started":"2023-03-12T14:24:30.956979Z"},"tags":[],"trusted":true},"outputs":[{"name":"stderr","output_type":"stream","text":["Some weights of the model checkpoint at facebook/timesformer-base-finetuned-k400 were not used when initializing TimesformerModel: ['classifier.bias', 'classifier.weight']\n","- This IS expected if you are initializing TimesformerModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n","- This IS NOT expected if you are initializing TimesformerModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n","/opt/conda/lib/python3.7/site-packages/torch/nn/modules/lazy.py:180: UserWarning: Lazy modules are a new feature under heavy development so changes to the API or functionality can happen at any moment.\n","  warnings.warn('Lazy modules are a new feature under heavy development '\n","/opt/conda/lib/python3.7/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:613: UserWarning: Checkpoint directory /kaggle/working/checkpoint exists and is not empty.\n","  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Sanity Checking: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"160519e8f5f64031804c601301bfbaf0","version_major":2,"version_minor":0},"text/plain":["Training: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Validation: 0it [00:00, ?it/s]"]},"metadata":{},"output_type":"display_data"}],"source":["# # 8ep pretrained + 10 epochs\n","# pl_video_model_pretrained = PLVideoModel.load_from_checkpoint(\n","#     \"./checkpoint/facebook/timesformer-base-finetuned-k400-epoch=08-train_loss=0.2507-val_score=0.5843.ckpt\",\n","#     config=config\n","# )\n","\n","# trainer = pl.Trainer(\n","#     max_epochs=10,\n","#     accelerator='auto', \n","#     precision=16,\n","#     callbacks=[early_stop_callback, checkpoint_callback]  \n","# #     callbacks=[checkpoint_callback]\n","# )\n","# trainer.fit(pl_video_model_pretrained, train_dataloader, val_dataloader)"]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.12"}},"nbformat":4,"nbformat_minor":4}