PCGRL.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Install Dependencis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9Lv9SkgJSPX8",
    "outputId": "07d785f9-9f69-4b8f-95cd-476213884372",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!pip install tensorflow_version==1.15.0\n",
    "!pip install \"gym==0.19\"\n",
    "!pip install stable-baselines[mpi]==2.9.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clone PCGRL Repository "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! git clone https://github.com/maryamkheirkhah/gym-pcgrl.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.chdir('gym-pcgrl')\n",
    "!pip install -e ."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check Your System Number Of CPUs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of CPUs: 8\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "print(\"Number of CPUs:\", os.cpu_count())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#pip install tensorflow==1.15\n",
    "#Install stable-baselines as described in the documentation\n",
    "print(\"first\")\n",
    "import model\n",
    "from model import FullyConvPolicyBigMap, FullyConvPolicySmallMap, CustomPolicyBigMap, CustomPolicySmallMap\n",
    "from utils import get_exp_name, max_exp_idx, load_model, make_vec_envs\n",
    "from stable_baselines import PPO2\n",
    "from stable_baselines.results_plotter import load_results, ts2xy\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "n_steps = 0\n",
    "log_dir = './'\n",
    "best_mean_reward, n_steps = -np.inf, 0\n",
    "\n",
    "def callback(_locals, _globals):\n",
    "    print(\"in callback\")\n",
    "    \"\"\"\n",
    "    Callback called at each step (for DQN an others) or after n steps (see ACER or PPO2)\n",
    "    :param _locals: (dict)\n",
    "    :param _globals: (dict)\n",
    "    \"\"\"\n",
    "    global n_steps, best_mean_reward\n",
    "    # Print stats every 1000 calls\n",
    "    if (n_steps + 1) % 10 == 0:\n",
    "        x, y = ts2xy(load_results(log_dir), 'timesteps')\n",
    "        if len(x) > 100:\n",
    "           #pdb.set_trace()\n",
    "            mean_reward = np.mean(y[-100:])\n",
    "            print(x[-1], 'timesteps')\n",
    "            print(\"Best mean reward: {:.2f} - Last mean reward per episode: {:.2f}\".format(best_mean_reward, mean_reward))\n",
    "\n",
    "            # New best model, we save the agent here\n",
    "            if mean_reward > best_mean_reward:\n",
    "                best_mean_reward = mean_reward\n",
    "                # Example for saving best model\n",
    "                print(\"Saving new best model\")\n",
    "                _locals['self'].save(os.path.join(log_dir, 'best_model.pkl'))\n",
    "            else:\n",
    "                print(\"Saving latest model\")\n",
    "                _locals['self'].save(os.path.join(log_dir, 'latest_model.pkl'))\n",
    "        else:\n",
    "            print('{} monitor entries'.format(len(x)))\n",
    "            pass\n",
    "    n_steps += 1\n",
    "    # Returning False will stop training early\n",
    "    return True\n",
    "\n",
    "\n",
    "def main(game, representation, experiment, steps, n_cpu, render, logging, **kwargs):\n",
    "    print(\"in main\")\n",
    "    env_name = '{}-{}-v0'.format(game, representation)\n",
    "    exp_name = get_exp_name(game, representation, experiment, **kwargs)\n",
    "    resume = kwargs.get('resume', False)\n",
    "    if representation == 'wide':\n",
    "        policy = FullyConvPolicyBigMap\n",
    "        if game == \"sokoban\":\n",
    "            policy = FullyConvPolicySmallMap\n",
    "    else:\n",
    "        policy = CustomPolicyBigMap\n",
    "        if game == \"sokoban\":\n",
    "            policy = CustomPolicySmallMap\n",
    "    if game == \"binary\":\n",
    "        kwargs['cropped_size'] = 28\n",
    "    elif game == \"zelda\":\n",
    "        kwargs['cropped_size'] = 22\n",
    "    elif game == \"sokoban\":\n",
    "        kwargs['cropped_size'] = 10\n",
    "    n = max_exp_idx(exp_name)\n",
    "    global log_dir\n",
    "    if not resume:\n",
    "        print(\"resume\")\n",
    "        n = n + 1\n",
    "    log_dir = 'runs/{}_{}_{}'.format(exp_name, n, 'log')\n",
    "    if not resume:\n",
    "        print(\"make resume\")\n",
    "        os.makedirs(log_dir)\n",
    "    else:\n",
    "        print(\"load model\")\n",
    "        model = load_model(log_dir)\n",
    "    kwargs = {\n",
    "        **kwargs,\n",
    "        'render_rank': 0,\n",
    "        'render': render,\n",
    "    }\n",
    "    used_dir = log_dir\n",
    "    if not logging:\n",
    "        print(\"1\")\n",
    "        used_dir = None\n",
    "    env = make_vec_envs(env_name, representation, log_dir, n_cpu, **kwargs)\n",
    "    if not resume or model is None:\n",
    "        print(\"2\")\n",
    "        model = PPO2(policy, env, verbose=1, tensorboard_log=\"./runs\")\n",
    "    else:\n",
    "        print(\"3\")\n",
    "        model.set_env(env)\n",
    "    if not logging:\n",
    "        print(\"4\")\n",
    "        model.learn(total_timesteps=int(steps), tb_log_name=exp_name)\n",
    "    else:\n",
    "        print(\"5\")\n",
    "        model.learn(total_timesteps=int(steps), tb_log_name=exp_name, callback=callback)\n",
    "\n",
    "################################## MAIN ########################################\n",
    "games =['binary','zelda','sokoban']\n",
    "representations = ['narrow','turtle','wide']\n",
    "# Fill these parameters according to your equipments and wishes \n",
    "experiment = None\n",
    "steps = 10000\n",
    "render = True\n",
    "logging = True\n",
    "n_cpu = 1\n",
    "kwargs = {\n",
    "    'resume': False\n",
    "}\n",
    "for g in games:\n",
    "    for r in representations:\n",
    "        if __name__ == '__main__':\n",
    "            main(g, r, experiment, steps, n_cpu, render, logging, **kwargs)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plotting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I used tensorboard for plotting trained models "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'f:\\\\learning\\\\master\\\\PCGRL\\\\gym-pcgrl'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "'kill' is not recognized as an internal or external command,\n",
      "operable program or batch file.\n"
     ]
    }
   ],
   "source": [
    "!kill 6020"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Display Tensorboard inline\n",
    "Run & Display tensorboard   \n",
    "**PS.** *sometimes it does not show up at all, then test to uncomment the reload code, or jusrt run cell again*\n",
    "\n",
    "It's correctly loaded when you see this view\n",
    "![Tensorboard](https://nextgrid.ai/wp-content/uploads/2019/12/Screenshot-2019-12-27-at-16.40.02.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "logs_base_dir = './real-100-runs/zelda/' # Log DIR\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.11.2\n"
     ]
    }
   ],
   "source": [
    "import tensorboard\n",
    "print(tensorboard.__version__)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "'lsof' is not recognized as an internal or external command,\n",
      "operable program or batch file.\n"
     ]
    }
   ],
   "source": [
    "!lsof -i :6006\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "'kill' is not recognized as an internal or external command,\n",
      "operable program or batch file.\n"
     ]
    }
   ],
   "source": [
    "!kill 6020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Launching TensorBoard..."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Often not loading on first try, run again until u see the screen\n",
    "%load_ext tensorboard\n",
    "\n",
    "%tensorboard --logdir {logs_base_dir}/ --host localhost --port 8084\n",
    "#%tensorboard --logdir=data/ --host localhost --port 808\n",
    "\n",
    "%reload_ext tensorboard\n",
    "#!kill 6020\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Maa7y1fDhdo6"
   },
   "source": [
    "# inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import model\n",
    "from stable_baselines import PPO2\n",
    "\n",
    "import time\n",
    "from utils import make_vec_envs\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython import display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_game_env(game, representation,change_percentage):\n",
    "    global model_path, kwargs,agent\n",
    "    model_path = 'runs/{}_{}_1_log/best_model.pkl'.format(game, representation)\n",
    "    kwargs = {\n",
    "        'change_percentage': change_percentage,\n",
    "        'verbose': True\n",
    "    }\n",
    "\n",
    "    if game == \"binary\":\n",
    "        model.FullyConvPolicy = model.FullyConvPolicyBigMap\n",
    "    elif game == \"zelda\":\n",
    "        model.FullyConvPolicy = model.FullyConvPolicyBigMap\n",
    "    elif game == \"sokoban\":\n",
    "        model.FullyConvPolicy = model.FullyConvPolicySmallMap\n",
    "\n",
    "    agent = PPO2.load(model_path)\n",
    "\n",
    "    env_name = '{}-{}-v0'.format(game, representation)\n",
    "    if game == \"binary\":\n",
    "        kwargs['cropped_size'] = 28\n",
    "    elif game == \"zelda\":\n",
    "        kwargs['cropped_size'] = 22\n",
    "    elif game == \"sokoban\":\n",
    "        kwargs['cropped_size'] = 10\n",
    "        \n",
    "    env = make_vec_envs(env_name, representation, None, 1, **kwargs)\n",
    "    return env\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "AJ3dogXjhdJ2"
   },
   "outputs": [],
   "source": [
    "def show_state(env, step=0, changes=0, total_reward=0, name=\"\"):\n",
    "    fig = plt.figure(10)\n",
    "    plt.clf()\n",
    "    plt.title(\"{} | Step: {} Changes: {} Total Reward: {}\".format(name, step, changes, total_reward))\n",
    "    plt.axis('off')\n",
    "    plt.imshow(env.render(mode='rgb_array'))\n",
    "    display.clear_output(wait=True)\n",
    "    display.display(plt.gcf())\n",
    "    \n",
    "def infer(env, agent, **kwargs):\n",
    "    obs = env.reset()\n",
    "    dones = False\n",
    "    total_rewards = 0\n",
    "    while not dones:\n",
    "        action, _ = agent.predict(obs)\n",
    "        obs, rewards, dones, info = env.step(action)\n",
    "        total_rewards += rewards\n",
    "        if dones:\n",
    "            break\n",
    "        show_state(env, info[0]['iterations'], info[0]['changes'], total_rewards)\n",
    "    if kwargs.get('verbose', False):\n",
    "        print(info[0])\n",
    "    change_percentage = kwargs.get('change_percentage')\n",
    "    return info[0]['iterations'], info[0]['changes'], total_rewards[0], change_percentage\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "games =['zelda','sokoban']\n",
    "representations = ['turtle','wide']\n",
    "arr_all_game = []\n",
    "for game in games:\n",
    "    for representation in representations:\n",
    "        for c in range(0,11):\n",
    "            print(game, representation)\n",
    "            env = make_game_env(game, representation,c/10)\n",
    "            total_rewards_arr=[]\n",
    "            for i in range(40):\n",
    "                r = infer(env, agent, **kwargs)\n",
    "                print(r)\n",
    "                total_rewards_arr.append(r)\n",
    "            arr_all_game.append(total_rewards_arr)\n",
    "            with open('result-{}.txt'.format(game), 'w') as f:\n",
    "                for line in arr_all_game:\n",
    "                    f.write(f\"{representation} \")\n",
    "                    f.write(f\"{line}\\n\")\n",
    "            \n",
    "       \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Zelda Wide"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2436.]\n",
      "[array([34.], dtype=float32), array([18.], dtype=float32), array([53.], dtype=float32), array([53.], dtype=float32), array([58.], dtype=float32), array([47.], dtype=float32), array([35.], dtype=float32), array([42.], dtype=float32), array([55.], dtype=float32), array([38.], dtype=float32), array([49.], dtype=float32), array([80.], dtype=float32), array([63.], dtype=float32), array([52.], dtype=float32), array([33.], dtype=float32), array([51.], dtype=float32), array([41.], dtype=float32), array([56.], dtype=float32), array([45.], dtype=float32), array([49.], dtype=float32), array([71.], dtype=float32), array([31.], dtype=float32), array([46.], dtype=float32), array([69.], dtype=float32), array([59.], dtype=float32), array([86.], dtype=float32), array([61.], dtype=float32), array([51.], dtype=float32), array([67.], dtype=float32), array([48.], dtype=float32), array([53.], dtype=float32), array([47.], dtype=float32), array([46.], dtype=float32), array([57.], dtype=float32), array([56.], dtype=float32), array([62.], dtype=float32), array([34.], dtype=float32), array([34.], dtype=float32), array([60.], dtype=float32), array([9.], dtype=float32), array([47.], dtype=float32), array([32.], dtype=float32), array([39.], dtype=float32), array([65.], dtype=float32), array([40.], dtype=float32), array([49.], dtype=float32), array([8.], dtype=float32), array([46.], dtype=float32), array([33.], dtype=float32), array([78.], dtype=float32)]\n",
      "[48.72]\n"
     ]
    }
   ],
   "source": [
    "print(sum)\n",
    "rewards_wide_zelda = total_rewards_arr\n",
    "mean_rewards_wide_zelda=sum/50\n",
    "print(rewards_wide_zelda)\n",
    "print(mean_rewards_wide_zelda)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sokoban Wide"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1061.]\n",
      "[array([8.], dtype=float32), array([22.], dtype=float32), array([23.], dtype=float32), array([25.], dtype=float32), array([21.], dtype=float32), array([26.], dtype=float32), array([-1.], dtype=float32), array([22.], dtype=float32), array([17.], dtype=float32), array([12.], dtype=float32), array([6.], dtype=float32), array([28.], dtype=float32), array([4.], dtype=float32), array([11.], dtype=float32), array([38.], dtype=float32), array([21.], dtype=float32), array([25.], dtype=float32), array([14.], dtype=float32), array([27.], dtype=float32), array([22.], dtype=float32), array([21.], dtype=float32), array([23.], dtype=float32), array([11.], dtype=float32), array([40.], dtype=float32), array([16.], dtype=float32), array([19.], dtype=float32), array([16.], dtype=float32), array([22.], dtype=float32), array([43.], dtype=float32), array([35.], dtype=float32), array([15.], dtype=float32), array([25.], dtype=float32), array([34.], dtype=float32), array([20.], dtype=float32), array([25.], dtype=float32), array([8.], dtype=float32), array([30.], dtype=float32), array([18.], dtype=float32), array([21.], dtype=float32), array([26.], dtype=float32), array([5.], dtype=float32), array([21.], dtype=float32), array([15.], dtype=float32), array([44.], dtype=float32), array([22.], dtype=float32), array([28.], dtype=float32), array([30.], dtype=float32), array([7.], dtype=float32), array([21.], dtype=float32), array([29.], dtype=float32)]\n",
      "[21.22]\n"
     ]
    }
   ],
   "source": [
    "print(sum)\n",
    "rewards_wide_sokoban = total_rewards_arr\n",
    "mean_rewards_wide_sokoban=sum/50\n",
    "print(rewards_wide_sokoban)\n",
    "print(mean_rewards_wide_sokoban)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sokoban Narrow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1016.]\n",
      "[array([32.], dtype=float32), array([10.], dtype=float32), array([32.], dtype=float32), array([35.], dtype=float32), array([13.], dtype=float32), array([42.], dtype=float32), array([34.], dtype=float32), array([19.], dtype=float32), array([15.], dtype=float32), array([12.], dtype=float32), array([22.], dtype=float32), array([7.], dtype=float32), array([26.], dtype=float32), array([23.], dtype=float32), array([28.], dtype=float32), array([16.], dtype=float32), array([17.], dtype=float32), array([13.], dtype=float32), array([17.], dtype=float32), array([7.], dtype=float32), array([22.], dtype=float32), array([5.], dtype=float32), array([20.], dtype=float32), array([22.], dtype=float32), array([14.], dtype=float32), array([30.], dtype=float32), array([7.], dtype=float32), array([30.], dtype=float32), array([20.], dtype=float32), array([12.], dtype=float32), array([25.], dtype=float32), array([37.], dtype=float32), array([42.], dtype=float32), array([30.], dtype=float32), array([18.], dtype=float32), array([17.], dtype=float32), array([24.], dtype=float32), array([11.], dtype=float32), array([23.], dtype=float32), array([7.], dtype=float32), array([15.], dtype=float32), array([24.], dtype=float32), array([25.], dtype=float32), array([7.], dtype=float32), array([37.], dtype=float32), array([26.], dtype=float32), array([9.], dtype=float32), array([6.], dtype=float32), array([10.], dtype=float32), array([21.], dtype=float32)]\n",
      "[20.32]\n"
     ]
    }
   ],
   "source": [
    "print(sum)\n",
    "rewards_narrow_sokoban = total_rewards_arr\n",
    "mean_rewards_narrow_sokoban=sum/50\n",
    "print(rewards_narrow_sokoban)\n",
    "print(mean_rewards_narrow_sokoban)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# binary turtle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[array([75.], dtype=float32), array([54.], dtype=float32), array([145.], dtype=float32), array([20.], dtype=float32), array([144.], dtype=float32), array([-1.], dtype=float32), array([141.], dtype=float32), array([102.], dtype=float32), array([70.], dtype=float32), array([7.], dtype=float32), array([27.], dtype=float32), array([153.], dtype=float32), array([22.], dtype=float32), array([219.], dtype=float32), array([20.], dtype=float32), array([126.], dtype=float32), array([57.], dtype=float32), array([154.], dtype=float32), array([80.], dtype=float32), array([155.], dtype=float32), array([143.], dtype=float32), array([143.], dtype=float32), array([43.], dtype=float32), array([172.], dtype=float32), array([6.], dtype=float32), array([23.], dtype=float32), array([20.], dtype=float32), array([56.], dtype=float32), array([150.], dtype=float32), array([119.], dtype=float32), array([53.], dtype=float32), array([149.], dtype=float32), array([141.], dtype=float32), array([201.], dtype=float32), array([118.], dtype=float32), array([20.], dtype=float32), array([101.], dtype=float32), array([26.], dtype=float32), array([155.], dtype=float32), array([24.], dtype=float32), array([24.], dtype=float32), array([115.], dtype=float32), array([41.], dtype=float32), array([51.], dtype=float32), array([143.], dtype=float32), array([44.], dtype=float32), array([24.], dtype=float32), array([23.], dtype=float32), array([25.], dtype=float32), array([37.], dtype=float32)]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "54.0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(total_rewards_arr)\n",
    "total_rewards_arr[1][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "sokoban turtle:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[array([23.], dtype=float32), array([21.], dtype=float32), array([13.], dtype=float32), array([24.], dtype=float32), array([19.], dtype=float32), array([10.], dtype=float32), array([24.], dtype=float32), array([15.], dtype=float32), array([35.], dtype=float32), array([29.], dtype=float32), array([7.], dtype=float32), array([9.], dtype=float32), array([16.], dtype=float32), array([24.], dtype=float32), array([32.], dtype=float32), array([0.], dtype=float32), array([15.], dtype=float32), array([19.], dtype=float32), array([30.], dtype=float32), array([40.], dtype=float32), array([20.], dtype=float32), array([32.], dtype=float32), array([20.], dtype=float32), array([25.], dtype=float32), array([14.], dtype=float32), array([26.], dtype=float32), array([7.], dtype=float32), array([11.], dtype=float32), array([45.], dtype=float32), array([8.], dtype=float32), array([24.], dtype=float32), array([34.], dtype=float32), array([49.], dtype=float32), array([24.], dtype=float32), array([16.], dtype=float32), array([14.], dtype=float32), array([33.], dtype=float32), array([34.], dtype=float32), array([22.], dtype=float32), array([11.], dtype=float32), array([19.], dtype=float32), array([19.], dtype=float32), array([52.], dtype=float32), array([35.], dtype=float32), array([33.], dtype=float32), array([24.], dtype=float32), array([29.], dtype=float32), array([13.], dtype=float32), array([29.], dtype=float32), array([20.], dtype=float32)]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "21.0"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(total_rewards_arr)\n",
    "total_rewards_arr[1][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sokoban Turtle Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1147.]\n",
      "[22.94]\n"
     ]
    }
   ],
   "source": [
    "print(sum)\n",
    "rewards_turtle_sokoban = total_rewards_arr\n",
    "mean_rewards_turtle_sokoban=sum/50\n",
    "print(mean_rewards_turtle_sokoban)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Zelda Turtle Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(sum)\n",
    "rewards_turtle_zelda = total_rewards_arr\n",
    "mean_rewards_turtle_zelda=sum/50\n",
    "print(mean_rewards_turtle_zelda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4160.]\n"
     ]
    }
   ],
   "source": [
    "print(sum)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Binary Turtle Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4160.]\n",
      "[83.2]\n"
     ]
    }
   ],
   "source": [
    "print(sum)\n",
    "rewards_turtle_binary = total_rewards_arr\n",
    "mean_rewards_turtle_binary=sum/50\n",
    "print(mean_rewards_turtle_binary)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Binary Wide Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[175.0], [34.0], [106.0], [169.0], [171.0], [38.0], [24.0], [25.0], [155.0], [51.0], [162.0], [36.0], [22.0], [25.0], [156.0], [121.0], [24.0], [34.0], [192.0], [96.0], [151.0], [100.0], [27.0], [170.0], [22.0], [127.0], [42.0], [107.0], [163.0], [124.0], [209.0], [113.0], [150.0], [22.0], [103.0], [30.0], [79.0], [30.0], [50.0], [92.0], [63.0], [175.0], [32.0], [161.0], [22.0], [191.0], [154.0], [22.0], [105.0], [29.0]]\n",
      "93.62\n"
     ]
    }
   ],
   "source": [
    "rewards_wide_binary = [[175.],[34.],[106.],[169.],[171.],[38.],[24.],[25.],[155.],[51.],[162.],[36.],[22.],[25.],[156.],[121.],[24.],[34.],[192.],[96.],[151.],[100.],[27.],[170.],[22.],[127.],[42.],[107.],[163.], [124.], [209.], [113.], [150.], [22.], [103.], [30.], [79.], [30.], [50.],[92.],[63.],[175.], [32.], [161.], [22.], [191.], [154.], [22.], [105.], [29.]]\n",
    "print(rewards_wide_binary)\n",
    "mean_rewards_wide_binary = 4681./50\n",
    "print(mean_rewards_wide_binary)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# binary Narrow inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4140.]\n",
      "[82.8]\n"
     ]
    }
   ],
   "source": [
    "print(sum)\n",
    "rewards_narrow_binary \n",
    "mean_rewards_narrow_binary\n",
    "print(mean_rewards_narrow_binary)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot Base On Change Percentage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('result-{}.txt'.format(game), 'w') as f:\n",
    "                for line in arr_all_game:\n",
    "                    f.write(f\"{representation} \")\n",
    "                    f.write(f\"{line}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readFile(game):\n",
    "    file1 = open('result-{}.txt'.format(game), 'r')\n",
    "    Lines = file1.readlines()\n",
    "    narrow= []\n",
    "    for i in range(0,11):\n",
    "        narrow.append(lines[i])\n",
    "    turtle=[]\n",
    "        for i in range(12,23):\n",
    "        turtle.append(lines[i])\n",
    "    wide = []\n",
    "    for i in range(13,34):\n",
    "        wide.append(lines[i])\n",
    "    return narrow ,turtle,wide"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Plot():\n",
    "    for game in games:\n",
    "        readFile(game)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reward(game,arr):\n",
    "    cropped_size = 0\n",
    "    if game == \"binary\":\n",
    "        cropped_size = 28\n",
    "    elif game == \"zelda\":\n",
    "        cropped_size = 22\n",
    "    elif game == \"sokoban\":\n",
    "        cropped_size= 10\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "binarys = [rewards_turtle_binary,rewards_wide_binary]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import gym_pcgrl\n",
    "\n",
    "env = gym.make('sokoban-narrow-v0')\n",
    "obs = env.reset()\n",
    "for t in range(1000):\n",
    "  action = env.action_space.sample()\n",
    "  obs, reward, done, info = env.step(action)\n",
    "  env.render('human')\n",
    "  if done:\n",
    "    print(\"Episode finished after {} timesteps\".format(t+1))\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "x = np.array([1, 2, 3], dtype='f')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# get - if didn't get any positive reward get - reward\n",
    "توی ترتل هیچ کاری نکردن هیچ مزیتی نداره!فقط ضرره پس باید منفی بگیره "
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "PCGRL.ipynb",
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}