train_bianry_demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chong.tian/.conda/envs/py38/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import cv2\n",
    "import wandb\n",
    "from datetime import datetime\n",
    "from tqdm import tqdm\n",
    "import argparse\n",
    "import random\n",
    "import json\n",
    "import collections\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.utils.data import ConcatDataset\n",
    "\n",
    "from sklearn.metrics import f1_score, accuracy_score\n",
    "\n",
    "sys.path.append('/home/hong/hc701/HC701-PROJECT')\n",
    "from VAL import test\n",
    "\n",
    "import torch.backends.cudnn as cudnn\n",
    "\n",
    "import timm\n",
    "from timm.models import create_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "horizontal_flip {'p': 0}\n"
     ]
    }
   ],
   "source": [
    "from hc701fed.dataset.dataset_list_transform import (\n",
    "    MESSIDOR_binary_pairs_train,\n",
    "    MESSIDOR_binary_pairs_test,\n",
    "    MESSIDOR_binary_Etienne_train,\n",
    "    MESSIDOR_binary_Etienne_test,\n",
    "    MESSIDOR_binary_Brest_train,\n",
    "    MESSIDOR_binary_Brest_test\n",
    ")\n",
    "\n",
    "centerlized_train = ConcatDataset([MESSIDOR_binary_pairs_train, MESSIDOR_binary_Etienne_train, MESSIDOR_binary_Brest_train])\n",
    "centerlized_test = ConcatDataset([MESSIDOR_binary_pairs_test, MESSIDOR_binary_Etienne_test, MESSIDOR_binary_Brest_train])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "def test(model_, test_loader, device):\n",
    "    model_test = copy.deepcopy(model_)\n",
    "    model_test.to(device)\n",
    "    model_test.eval()\n",
    "    y_pred = []\n",
    "    y_true = []\n",
    "    with torch.no_grad():\n",
    "        for images, labels in test_loader:\n",
    "            images = images.to(device)\n",
    "            labels = labels.to(device)\n",
    "            outputs = model_test(images)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            y_pred.extend(predicted.cpu().numpy())\n",
    "            y_true.extend(labels.cpu().numpy())\n",
    "    return y_true, y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "batch_size = 8\n",
    "# Data loader\n",
    "train_loader_pairs = DataLoader(MESSIDOR_binary_pairs_train, batch_size=batch_size, shuffle=True)\n",
    "train_loader_Etienne = DataLoader(MESSIDOR_binary_Etienne_train, batch_size=batch_size, shuffle=True)\n",
    "train_loader_Brest = DataLoader(MESSIDOR_binary_Brest_train, batch_size=batch_size, shuffle=True)\n",
    "train_loader = DataLoader(centerlized_train, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "test_loader_pairs = DataLoader(MESSIDOR_binary_pairs_test, batch_size=1, shuffle=False)\n",
    "test_loader_Etienne = DataLoader(MESSIDOR_binary_Etienne_test, batch_size=1, shuffle=False)\n",
    "test_loader_Brest = DataLoader(MESSIDOR_binary_Brest_test, batch_size=1, shuffle=False)\n",
    "test_loader = DataLoader(centerlized_test, batch_size=1, shuffle=False)\n",
    "\n",
    "train_list = [train_loader_pairs, train_loader_Etienne, train_loader_Brest]\n",
    "test_list = [test_loader_pairs, test_loader_Etienne, test_loader_Brest]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/vgg16-397923af.pth\" to /home/chong.tian/.cache/torch/hub/checkpoints/vgg16-397923af.pth\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0, Loss: 0.6704, Accuracy+f1/2: 0.6363\n",
      "Epoch: 1, Loss: 0.4029, Accuracy+f1/2: 0.6125\n",
      "Epoch: 2, Loss: 0.5121, Accuracy+f1/2: 0.6029\n",
      "Epoch: 3, Loss: 0.9235, Accuracy+f1/2: 0.5306\n",
      "Epoch: 4, Loss: 0.1746, Accuracy+f1/2: 0.6632\n",
      "Epoch: 5, Loss: 0.6526, Accuracy+f1/2: 0.2836\n",
      "Epoch: 6, Loss: 0.8615, Accuracy+f1/2: 0.6971\n",
      "Epoch: 7, Loss: 0.7641, Accuracy+f1/2: 0.5498\n",
      "Epoch: 8, Loss: 0.4540, Accuracy+f1/2: 0.8097\n",
      "Epoch: 9, Loss: 0.6398, Accuracy+f1/2: 0.7534\n"
     ]
    }
   ],
   "source": [
    "# centerlized train\n",
    "model = create_model('vgg16', pretrained=True, num_classes=2)\n",
    "\n",
    "model.to(device)\n",
    "cudnn.benchmark = True\n",
    "\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=0.01)\n",
    "\n",
    "for epoch in range(10):\n",
    "    for X, y in train_loader:\n",
    "        X = X.to(device)\n",
    "        y = y.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        y_pred = model(X)\n",
    "        loss = criterion(y_pred, y)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "    y_pred, y_true = test(model, test_loader, device)\n",
    "\n",
    "    print('Epoch: {}, Loss: {:.4f}, Accuracy+f1/2: {:.4f}'.format(epoch, loss.item(), (accuracy_score(y_true, y_pred) + f1_score(y_true, y_pred, average='binary'))/2))\n",
    "\n",
    "y_pred, y_true = test(model, test_loader, device)\n",
    "\n",
    "print('Epoch: {}, Loss: {:.4f}, Accuracy+f1/2: {:.4f}'.format(epoch, loss.item(), (accuracy_score(y_true, y_pred) + f1_score(y_true, y_pred, average='binary'))/2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 9, Loss: 0.6398, Accuracy+f1/2: 0.7534\n"
     ]
    }
   ],
   "source": [
    "y_pred, y_true = test(model, test_loader, device)\n",
    "\n",
    "print('Epoch: {}, Loss: {:.4f}, Accuracy+f1/2: {:.4f}'.format(epoch, loss.item(), (accuracy_score(y_true, y_pred) + f1_score(y_true, y_pred, average='binary'))/2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed: 42, noise_scale: 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "38it [00:03, 10.16it/s]\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu!",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[5], line 109\u001b[0m\n\u001b[1;32m    107\u001b[0m global_models\u001b[39m=\u001b[39m[copy\u001b[39m.\u001b[39mdeepcopy(model) \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39m3\u001b[39m)]\n\u001b[1;32m    108\u001b[0m \u001b[39mfor\u001b[39;00m rounds \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(com_round):\n\u001b[0;32m--> 109\u001b[0m     models\u001b[39m=\u001b[39mlocal_step(train_list,\u001b[39m1000\u001b[39;49m,global_models[\u001b[39m0\u001b[39;49m],com_round)\n\u001b[1;32m    110\u001b[0m y_true,y_pred\u001b[39m=\u001b[39mtest(models[\u001b[39m0\u001b[39m],test_loader,device)\n\u001b[1;32m    111\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m'\u001b[39m\u001b[39mRound: \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m Accuracy: \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m.\u001b[39mformat(com_round,accuracy_score(y_true,y_pred)))\n",
      "Cell \u001b[0;32mIn[5], line 90\u001b[0m, in \u001b[0;36mlocal_step\u001b[0;34m(training_dataloaders_list, num_updates, model_init, rounds)\u001b[0m\n\u001b[1;32m     88\u001b[0m MODEL_LIST\u001b[39m=\u001b[39m[model_init \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39m3\u001b[39m)]\n\u001b[1;32m     89\u001b[0m \u001b[39mfor\u001b[39;00m i,j \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(training_dataloaders_list,\u001b[39mrange\u001b[39m(\u001b[39m3\u001b[39m)):\n\u001b[0;32m---> 90\u001b[0m     MODEL_LIST[j]\u001b[39m=\u001b[39mlocal_dp_train(trains_loader_item\u001b[39m=\u001b[39;49mi,num_updates\u001b[39m=\u001b[39;49mnum_updates,model_init\u001b[39m=\u001b[39;49mmodel_init,client_id\u001b[39m=\u001b[39;49mj,rounds\u001b[39m=\u001b[39;49mrounds)\n\u001b[1;32m     91\u001b[0m \u001b[39mreturn\u001b[39;00m MODEL_LIST\n",
      "Cell \u001b[0;32mIn[5], line 84\u001b[0m, in \u001b[0;36mlocal_dp_train\u001b[0;34m(trains_loader_item, num_updates, model_init, client_id, rounds)\u001b[0m\n\u001b[1;32m     82\u001b[0m dp_update_tensor\u001b[39m=\u001b[39mapply_dp(update_tensor\u001b[39m=\u001b[39mmodel_update,total_round\u001b[39m=\u001b[39mrounds)\n\u001b[1;32m     83\u001b[0m \u001b[39m# print(dp_update_tensor)\u001b[39;00m\n\u001b[0;32m---> 84\u001b[0m model_t_dp\u001b[39m=\u001b[39mupdate_model(dp_update_tensor\u001b[39m=\u001b[39;49mdp_update_tensor,model_pre\u001b[39m=\u001b[39;49mmodel_pre,model_t\u001b[39m=\u001b[39;49mmodel_t)\n\u001b[1;32m     85\u001b[0m \u001b[39mreturn\u001b[39;00m model_t_dp\n",
      "Cell \u001b[0;32mIn[5], line 73\u001b[0m, in \u001b[0;36mupdate_model\u001b[0;34m(dp_update_tensor, model_pre, model_t)\u001b[0m\n\u001b[1;32m     71\u001b[0m     length_paramter\u001b[39m=\u001b[39m\u001b[39mint\u001b[39m((torch\u001b[39m.\u001b[39mflatten(param)\u001b[39m.\u001b[39mshape)[\u001b[39m0\u001b[39m]) \u001b[39m# Get the length of of paramter\u001b[39;00m\n\u001b[1;32m     72\u001b[0m     delta\u001b[39m=\u001b[39mdp_update_tensor[:length_paramter]\u001b[39m.\u001b[39mreshape(param\u001b[39m.\u001b[39mshape)  \u001b[39m# Delta is the parameter grad times the lr\u001b[39;00m\n\u001b[0;32m---> 73\u001b[0m     t_dict_model[name]\u001b[39m=\u001b[39mpre_dict_model[name]\u001b[39m+\u001b[39;49mdelta  \u001b[39m# update model\u001b[39;00m\n\u001b[1;32m     74\u001b[0m     dp_update_tensor\u001b[39m=\u001b[39mdp_update_tensor[length_paramter:] \u001b[39m# remove used\u001b[39;00m\n\u001b[1;32m     75\u001b[0m model_t\u001b[39m.\u001b[39mload_state_dict(t_dict_model) \u001b[39m# update model\u001b[39;00m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu!"
     ]
    }
   ],
   "source": [
    "for noise_scale in [0.1]:\n",
    "    for seed in [42,43,44]:\n",
    "        print('seed: {}, noise_scale: {}'.format(seed, noise_scale))\n",
    "        cudnn.deterministic = True\n",
    "        cudnn.benchmark = True\n",
    "        torch.cuda.manual_seed(seed)\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "        random.seed(seed)\n",
    "        np.random.seed(seed)\n",
    "        os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "        torch.manual_seed(seed)\n",
    "        \n",
    "\n",
    "        # use vgg16 as the base model\n",
    "        model = create_model('vgg16', pretrained=False, num_classes=2)\n",
    "\n",
    "        # load the fix model\n",
    "        model_name = 'vgg16'\n",
    "        model_path = '/home/hong/hc701/random_init/{}.pth'.format(model_name)\n",
    "        model.load_state_dict(torch.load(model_path))\n",
    "        mean = torch.tensor(0, dtype=torch.float)\n",
    "        std = torch.tensor(0.0948*noise_scale, dtype=torch.float)\n",
    "        lr=0.01\n",
    "        clip_value=30\n",
    "\n",
    "        def local_train(trains_loader_item, num_updates,model_init,client_id,rounds,lr=lr):\n",
    "            __model=copy.deepcopy(model_init)\n",
    "            __model.to(device)\n",
    "            # Local train\n",
    "            loss = torch.nn.CrossEntropyLoss()\n",
    "            optimizer=torch.optim.SGD(__model.parameters(),lr=lr)\n",
    "            __model.train()\n",
    "            update_count = 0\n",
    "            for i, (X, y) in tqdm(enumerate(trains_loader_item)):\n",
    "                update_count += 1\n",
    "                X, y = X.to(device), y.to(device)\n",
    "                # Compute prediction and loss\n",
    "                pred = __model(X)\n",
    "                _loss = loss(pred,y)\n",
    "                # Backpropagation\n",
    "                _loss.backward()\n",
    "                optimizer.step()\n",
    "                optimizer.zero_grad()\n",
    "                if update_count >= num_updates:\n",
    "                    break\n",
    "            return __model,model_init\n",
    "        \n",
    "        def get_update(model_previous: torch.nn.modules,model_t: torch.nn.modules):\n",
    "            updates=torch.tensor([]).to('cuda').reshape(1,-1)\n",
    "            for p_t,p_pre in zip(model_t.parameters(),model_previous.parameters()):\n",
    "                p_pre=p_pre.to('cuda')\n",
    "                updates=torch.cat((updates,torch.flatten(p_t-p_pre).to('cuda').reshape(1,-1)),1)\n",
    "            return updates\n",
    "        \n",
    "        def apply_dp(update_tensor,total_round,clip_threshold=clip_value,learn_rate=lr,dp_std=std,dp_mean=mean):\n",
    "            # clip gradient\n",
    "            updates_norm=torch.linalg.vector_norm(update_tensor)\n",
    "            clip_threshold=learn_rate*clip_threshold\n",
    "            update_tensor_clip=update_tensor/torch.max(torch.tensor([1]).to(device),updates_norm/clip_threshold)\n",
    "            # add noise\n",
    "            update_tensor_clip+=torch.normal(mean=dp_mean, std=learn_rate*dp_std*clip_threshold*total_round,size=update_tensor_clip.shape).to('cuda')\n",
    "            return update_tensor_clip\n",
    "        \n",
    "        def update_model(dp_update_tensor: torch.Tensor(), model_pre: torch.nn.modules,model_t: torch.nn.modules):\n",
    "            dp_update_tensor=torch.flatten(dp_update_tensor)  # Flatten update make it to 1D\n",
    "            # Save model dict\n",
    "            pre_dict_model=model_pre.state_dict()\n",
    "            t_dict_model=model_t.state_dict()\n",
    "            #update by the DP guarantee delta\n",
    "            for name, param in model_t.named_parameters():\n",
    "                length_paramter=int((torch.flatten(param).shape)[0]) # Get the length of of paramter\n",
    "                delta=dp_update_tensor[:length_paramter].reshape(param.shape)  # Delta is the parameter grad times the lr\n",
    "                t_dict_model[name]=pre_dict_model[name]+delta  # update model\n",
    "                dp_update_tensor=dp_update_tensor[length_paramter:] # remove used\n",
    "            model_t.load_state_dict(t_dict_model) # update model\n",
    "            return model_t\n",
    "        \n",
    "        def local_dp_train(trains_loader_item, num_updates,model_init,client_id,rounds):\n",
    "            model_t,model_pre=local_train(trains_loader_item=trains_loader_item, num_updates=num_updates,model_init=model_init,client_id=client_id,rounds=rounds)\n",
    "            model_update=get_update(model_pre,model_t)\n",
    "            # print(model_update)\n",
    "            dp_update_tensor=apply_dp(update_tensor=model_update,total_round=rounds)\n",
    "            # print(dp_update_tensor)\n",
    "            model_t_dp=update_model(dp_update_tensor=dp_update_tensor,model_pre=model_pre,model_t=model_t)\n",
    "            return model_t_dp\n",
    "        \n",
    "        def local_step(training_dataloaders_list,num_updates,model_init,rounds):\n",
    "            MODEL_LIST=[model_init for i in range(3)]\n",
    "            for i,j in zip(training_dataloaders_list,range(3)):\n",
    "                MODEL_LIST[j]=local_dp_train(trains_loader_item=i,num_updates=num_updates,model_init=model_init,client_id=j,rounds=rounds)\n",
    "            return MODEL_LIST\n",
    "        \n",
    "        def aggregation_model(models_list):\n",
    "            _models_list=copy.deepcopy(models_list)\n",
    "            fed_state_dict=collections.OrderedDict()\n",
    "            weight_keys=models_list[0].state_dict().keys()\n",
    "            for key in weight_keys:\n",
    "                key_avg=0\n",
    "                for _model in _models_list:\n",
    "                    key_avg=key_avg+_model.state_dict()[key]*1/3\n",
    "                fed_state_dict[key]=key_avg\n",
    "            for _model in _models_list:\n",
    "                _model.load_state_dict(fed_state_dict)\n",
    "            return _models_list\n",
    "        \n",
    "        for com_round in [1]:\n",
    "            global_models=[copy.deepcopy(model) for i in range(3)]\n",
    "            for rounds in range(com_round):\n",
    "                models=local_step(train_list,1000,global_models[0],com_round)\n",
    "            y_true,y_pred=test(models[0],test_loader,device)\n",
    "            print('Round: {} Accuracy: {}'.format(com_round,accuracy_score(y_true,y_pred)))\n",
    "            print('Round:',com_round,'f1_score:',f1_score(y_true,y_pred,average='macro'))\n",
    "            print('Round:',com_round,'average_Acc_f1:',(accuracy_score(y_true,y_pred)+f1_score(y_true,y_pred,average='macro'))/2)\n",
    "\n",
    "            y_true2,y_pred2=test(models[1],test_loader,device)\n",
    "            print('Round: {} Accuracy: {}'.format(com_round,accuracy_score(y_true2,y_pred2)))\n",
    "            print('Round:',com_round,'f1_score:',f1_score(y_true2,y_pred2,average='macro'))\n",
    "            print('Round:',com_round,'average_Acc_f1:',(accuracy_score(y_true2,y_pred2)+f1_score(y_true2,y_pred2,average='macro'))/2)\n",
    "\n",
    "            y_true3,y_pred3=test(models[2],test_loader,device)\n",
    "            print('Round: {} Accuracy: {}'.format(com_round,accuracy_score(y_true3,y_pred3)))\n",
    "            print('Round:',com_round,'f1_score:',f1_score(y_true3,y_pred3,average='macro'))\n",
    "            print('Round:',com_round,'average_Acc_f1:',(accuracy_score(y_true3,y_pred3)+f1_score(y_true3,y_pred3,average='macro'))/2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.844805262605389"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "np.sqrt(2*np.log(1.25/10**(-5)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py38",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}