Evacuation_RL_1006_One_Class_126_60_10timeslarge.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Evacuation_RL_1006_One_Class_126_60_10timeslarge.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNHjaLYOFK4FFCK0AYRr2GE",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bravoila/MachineLearning/blob/master/Evacuation_RL_1006_One_Class_126_60_10timeslarge.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BXfVWLMxxEI0",
        "outputId": "ffeed305-bf6c-49c6-cc76-98b674252f45",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "pip install rlcard"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: rlcard in /usr/local/lib/python3.6/dist-packages (0.2.5)\n",
            "Requirement already satisfied: pillow>=5.2.0 in /usr/local/lib/python3.6/dist-packages (from rlcard) (7.0.0)\n",
            "Requirement already satisfied: matplotlib>=3.0 in /usr/local/lib/python3.6/dist-packages (from rlcard) (3.2.2)\n",
            "Requirement already satisfied: numpy>=1.16.3 in /usr/local/lib/python3.6/dist-packages (from rlcard) (1.18.5)\n",
            "Requirement already satisfied: termcolor in /usr/local/lib/python3.6/dist-packages (from rlcard) (1.1.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from rlcard) (20.4)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->rlcard) (1.2.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->rlcard) (2.4.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->rlcard) (0.10.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->rlcard) (2.8.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->rlcard) (1.15.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jhGcGdwdxICT",
        "outputId": "f4ae41f4-f7f5-401f-d3f8-2fd84764940d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "pip install rlcard[tensorflow]"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: rlcard[tensorflow] in /usr/local/lib/python3.6/dist-packages (0.2.5)\n",
            "Requirement already satisfied: matplotlib>=3.0 in /usr/local/lib/python3.6/dist-packages (from rlcard[tensorflow]) (3.2.2)\n",
            "Requirement already satisfied: pillow>=5.2.0 in /usr/local/lib/python3.6/dist-packages (from rlcard[tensorflow]) (7.0.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from rlcard[tensorflow]) (20.4)\n",
            "Requirement already satisfied: numpy>=1.16.3 in /usr/local/lib/python3.6/dist-packages (from rlcard[tensorflow]) (1.18.5)\n",
            "Requirement already satisfied: termcolor in /usr/local/lib/python3.6/dist-packages (from rlcard[tensorflow]) (1.1.0)\n",
            "Requirement already satisfied: tensorflow<2.0,>=1.14; extra == \"tensorflow\" in /usr/local/lib/python3.6/dist-packages (from rlcard[tensorflow]) (1.15.4)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->rlcard[tensorflow]) (2.8.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->rlcard[tensorflow]) (1.2.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->rlcard[tensorflow]) (0.10.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->rlcard[tensorflow]) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->rlcard[tensorflow]) (1.15.0)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (0.8.1)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (1.12.1)\n",
            "Requirement already satisfied: keras-applications>=1.0.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (1.0.8)\n",
            "Requirement already satisfied: tensorflow-estimator==1.15.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (1.15.1)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (3.12.4)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (0.10.0)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (1.32.0)\n",
            "Requirement already satisfied: gast==0.2.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (0.2.2)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (1.1.2)\n",
            "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (0.35.1)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (3.3.0)\n",
            "Requirement already satisfied: tensorboard<1.16.0,>=1.15.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (1.15.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (0.2.0)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.8->tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (2.10.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (50.3.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (1.0.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (3.2.2)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tensorboard<1.16.0,>=1.15.0->tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (2.0.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<1.16.0,>=1.15.0->tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (3.2.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rCm-mXNVJt4U",
        "outputId": "58dc8c37-c9d9-45f5-bb4e-0e87abc5cbe7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# Stable Baselines only supports tensorflow 1.x for now\n",
        "%tensorflow_version 1.x\n",
        "!apt install swig cmake libopenmpi-dev zlib1g-dev\n",
        "!pip install stable-baselines[mpi]==2.10.0"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 1.x selected.\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "zlib1g-dev is already the newest version (1:1.2.11.dfsg-0ubuntu2).\n",
            "libopenmpi-dev is already the newest version (2.1.1-8).\n",
            "swig is already the newest version (3.0.12-1).\n",
            "cmake is already the newest version (3.10.2-1ubuntu2.18.04.1).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 6 not upgraded.\n",
            "Requirement already satisfied: stable-baselines[mpi]==2.10.0 in /usr/local/lib/python3.6/dist-packages (2.10.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from stable-baselines[mpi]==2.10.0) (0.16.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from stable-baselines[mpi]==2.10.0) (1.4.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from stable-baselines[mpi]==2.10.0) (1.1.2)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from stable-baselines[mpi]==2.10.0) (3.2.2)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from stable-baselines[mpi]==2.10.0) (1.18.5)\n",
            "Requirement already satisfied: opencv-python in /usr/local/lib/python3.6/dist-packages (from stable-baselines[mpi]==2.10.0) (4.1.2.30)\n",
            "Requirement already satisfied: gym[atari,classic_control]>=0.11 in /usr/local/lib/python3.6/dist-packages (from stable-baselines[mpi]==2.10.0) (0.17.2)\n",
            "Requirement already satisfied: cloudpickle>=0.5.5 in /usr/local/lib/python3.6/dist-packages (from stable-baselines[mpi]==2.10.0) (1.3.0)\n",
            "Requirement already satisfied: mpi4py; extra == \"mpi\" in /tensorflow-1.15.2/python3.6 (from stable-baselines[mpi]==2.10.0) (3.0.3)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->stable-baselines[mpi]==2.10.0) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.6/dist-packages (from pandas->stable-baselines[mpi]==2.10.0) (2.8.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->stable-baselines[mpi]==2.10.0) (1.2.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->stable-baselines[mpi]==2.10.0) (2.4.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->stable-baselines[mpi]==2.10.0) (0.10.0)\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.6/dist-packages (from gym[atari,classic_control]>=0.11->stable-baselines[mpi]==2.10.0) (1.5.0)\n",
            "Requirement already satisfied: atari-py~=0.2.0; extra == \"atari\" in /usr/local/lib/python3.6/dist-packages (from gym[atari,classic_control]>=0.11->stable-baselines[mpi]==2.10.0) (0.2.6)\n",
            "Requirement already satisfied: Pillow; extra == \"atari\" in /usr/local/lib/python3.6/dist-packages (from gym[atari,classic_control]>=0.11->stable-baselines[mpi]==2.10.0) (7.0.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.7.3->pandas->stable-baselines[mpi]==2.10.0) (1.15.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym[atari,classic_control]>=0.11->stable-baselines[mpi]==2.10.0) (0.16.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Grxcnf0WJ7VX"
      },
      "source": [
        "Code for environment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fj7EwdXCJ4Hs"
      },
      "source": [
        "#from google.colab import files\n",
        "#import io\n",
        "#uploaded = files.upload()"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P1BceU4TKZ2O",
        "cellView": "both"
      },
      "source": [
        "#import io\n",
        "#io.BytesIO(uploaded['patient_generattion2.xlsx'])\n",
        "#@title Code for Environment"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZUxRL33-4CWG"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QBIND4ZhJ6f7",
        "cellView": "form"
      },
      "source": [
        "#@title Code for Environment\n",
        "#!/usr/bin/env python3\n",
        "# -*- coding: utf-8 -*-\n",
        "\"\"\"\n",
        "Created on Tue Dec 17 11:46:46 2020\n",
        "\n",
        "@author: machaolun\n",
        "\"\"\"\n",
        "\"\"\"\n",
        "Unit: Minute\n",
        "\"\"\"\n",
        "import gym\n",
        "from gym import spaces\n",
        "\n",
        "# import tkinter as tk\n",
        "import random\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "# from collections import deque\n",
        "from heapq import heappush,heappop\n",
        "import csv\n",
        "import copy\n",
        "from collections import deque\n",
        "\n",
        "random.seed(2020)\n",
        "# This class is to generate patients\n",
        "total_simulation_time = 10000\n",
        "\n",
        "\n",
        "def SURV_PROB_CAL(my_facility, my_patype, my_total_time, surv_num):\n",
        "        # survival probablity\n",
        "    # Resource-Based Patient Prioritization in Mass-Casualty Incidents\n",
        "    # first is the worst case and the fifth is the optimistic case    \n",
        "    # Immediate         Delayed\n",
        "    # β0,I β1,I β2,I    β0,D β1,D β2,D\n",
        "    # 0.09,17  ,1.01    0.57,61   ,2.03\n",
        "    # 0.15,28  ,1.38    0.65,86   ,2.11\n",
        "    # 0.24,47  ,1.30    0.76,138  ,2.17\n",
        "    # 0.40,59  ,1.47    0.77,140  ,2.29\n",
        "    # 0.56,91  ,1.58    0.81,160  ,1.41\n",
        "    if surv_num == 1:\n",
        "        # set para\n",
        "        # Senerio 1, assume no different between ACF and hospital\n",
        "        if my_facility == 1 and my_patype == 0:\n",
        "            \"\"\"Hospital, immediate\"\"\"\n",
        "            Beta = [0.09,17  ,1.01]\n",
        "        elif my_facility == 1 and my_patype == 1:\n",
        "            \"\"\"Hospital, delayed\"\"\"\n",
        "            Beta = [0.57,61 ,2.03]\n",
        "        elif my_facility == 2 and my_patype == 0:\n",
        "            \"\"\"ACF, immediate\"\"\"\n",
        "            Beta = [0.09,17  ,1.01]\n",
        "        elif my_facility == 2 and my_patype == 1:\n",
        "            \"\"\"ACF, delayed\"\"\"\n",
        "            Beta = [0.57,61   ,2.03]\n",
        "    elif surv_num == 2:       \n",
        "        # # Senerio 2, assume no different between ACF and hospital\n",
        "        if my_facility == 1 and my_patype == 0:\n",
        "            \"\"\"Hospital, immediate\"\"\"\n",
        "            Beta = [0.15,28  ,1.38]\n",
        "        elif my_facility == 1 and my_patype == 1:\n",
        "            \"\"\"Hospital, delayed\"\"\"\n",
        "            Beta = [0.65,86   ,2.11]\n",
        "        elif my_facility == 2 and my_patype == 0:\n",
        "            \"\"\"ACF, immediate\"\"\"\n",
        "            Beta = [0.15,28  ,1.38]\n",
        "        elif my_facility == 2 and my_patype == 1:\n",
        "            \"\"\"ACF, delayed\"\"\"\n",
        "            Beta = [0.65,86   ,2.11]\n",
        "            \n",
        "    elif surv_num == 3:\n",
        "        \n",
        "        # # Senerio 3, assume no different between ACF and hospital\n",
        "        if my_facility == 1 and my_patype == 0:\n",
        "            \"\"\"Hospital, immediate\"\"\"\n",
        "            Beta = [0.24,47  ,1.30]\n",
        "        elif my_facility == 1 and my_patype == 1:\n",
        "            \"\"\"Hospital, delayed\"\"\"\n",
        "            Beta = [0.76,138  ,2.17]\n",
        "        elif my_facility == 2 and my_patype == 0:\n",
        "            \"\"\"ACF, immediate\"\"\"\n",
        "            Beta = [0.24,47  ,1.30]\n",
        "        elif my_facility == 2 and my_patype == 1:\n",
        "            \"\"\"ACF, delayed\"\"\"\n",
        "            Beta = [0.76,138  ,2.17]\n",
        "    elif surv_num == 4:\n",
        "            \n",
        "        # # Senerio 4, assume no different between ACF and hospital\n",
        "        if my_facility == 1 and my_patype == 0:\n",
        "            \"\"\"Hospital, immediate\"\"\"\n",
        "            Beta = [0.40,59  ,1.47]\n",
        "        elif my_facility == 1 and my_patype == 1:\n",
        "            \"\"\"Hospital, delayed\"\"\"\n",
        "            Beta = [0.77,140  ,2.29]\n",
        "        elif my_facility == 2 and my_patype == 0:\n",
        "            \"\"\"ACF, immediate\"\"\"\n",
        "            Beta = [0.40,59  ,1.47]\n",
        "        elif my_facility == 2 and my_patype == 1:\n",
        "            \"\"\"ACF, delayed\"\"\"\n",
        "            Beta = [0.77,140  ,2.29]            \n",
        "    elif surv_num == 5:\n",
        "        # Senerio 5, assume no different between ACF and hospital            \n",
        "        if my_facility == 1 and my_patype == 0:\n",
        "            \"\"\"Hospital, immediate\"\"\"\n",
        "            Beta = [0.56,91  ,1.58]\n",
        "        elif my_facility == 1 and my_patype == 1:\n",
        "            \"\"\"Hospital, delayed\"\"\"\n",
        "            Beta = [0.81,160  ,1.41]\n",
        "        elif my_facility == 2 and my_patype == 0:\n",
        "            \"\"\"ACF, immediate\"\"\"\n",
        "            Beta = [0.56,91  ,1.58]\n",
        "        elif my_facility == 2 and my_patype == 1:\n",
        "            \"\"\"ACF, delayed\"\"\"\n",
        "            Beta = [0.81,160  ,1.41]        \n",
        "    \n",
        "    # 0.9 is ACF discount factor\n",
        "    if my_facility == 1:\n",
        "        SP = 100*Beta[0]/((my_total_time/Beta[1]) ** Beta[2] + 1.0)\n",
        "    else:\n",
        "        SP = 100* 0.8 * Beta[0]/((my_total_time/Beta[1]) ** Beta[2] + 1.0)\n",
        "    return SP\n",
        "\n",
        "\n",
        "def eva_perform(env, num):\n",
        "    ''' Evaluate he performance of the agents in the environment\n",
        "    Args:\n",
        "        env (Env class): The environment to be evaluated.\n",
        "        num (int): The number of games to play.\n",
        "    Returns:\n",
        "        A list of avrage payoffs for each player\n",
        "    '''\n",
        "    payoffs = 0\n",
        "    counter = 0\n",
        "\n",
        "    while counter < num:\n",
        "        tj = env.run(1, is_training=False)\n",
        "        return_ = pd.DataFrame(tj)\n",
        "        payoffs += sum(return_[2])\n",
        "        counter += 1\n",
        "\n",
        "    payoffs /= counter\n",
        "    \n",
        "    return payoffs\n",
        "\n",
        "class Event(object):\n",
        "    # num_of_events = 0\n",
        "    def __init__(self,generate_time, ptype, surv_num):\n",
        "        self.generate_time = generate_time\n",
        "        self.trans_time = 0.2\n",
        "        self.arrival_time = 0.3\n",
        "        self.start_service_time = 0.4\n",
        "        self.service_time = 0.5\n",
        "        self.departure_time = 0.6\n",
        "        self.survival_prob_res = 0\n",
        "        self.facility = 0.7\n",
        "        self.patype = ptype\n",
        "        self.status = \"default\"\n",
        "        self.surv_num = surv_num\n",
        "        # severe , minor\n",
        "        # Event.num_of_events += 1\n",
        "    def total_time(self):\n",
        "        return self.departure_time - self.generate_time\n",
        "    \n",
        "    def sojourn_time(self):\n",
        "        return self.departure_time - self.arrival_time\n",
        "\n",
        "    def waiting_time(self):\n",
        "        # waiting time in queue\n",
        "        return self.sojourn_time() - self.service_time\n",
        "    \n",
        "    def survival_prob(self):\n",
        "        my_total_time = self.total_time()\n",
        "        SP = SURV_PROB_CAL(self.facility, self.patype, my_total_time, self.surv_num)\n",
        "        self.survival_prob_res = SP\n",
        "        return SP\n",
        "\n",
        "    def __repr__(self):\n",
        "        return f\"{self.generate_time}, {self.trans_time},{self.arrival_time}, {self.start_service_time}, {self.service_time}, {self.departure_time}, {self.survival_prob_res}, {self.facility}, {self.patype} \\n\"\n",
        "    def get_info(self):\n",
        "        #print([self.generate_time, self.trans_time, self.arrival_time, self.start_service_time, self.service_time, self.departure_time, self.facility, self.patype, self.status])\n",
        "        return [self.generate_time, self.trans_time, self.arrival_time, self.start_service_time, self.service_time, self.departure_time, self.survival_prob_res, self.facility, self.patype, self.status]\n",
        "    \n",
        "\"\"\"########################################################################\"\"\"\n",
        "\n",
        "class Server(object):\n",
        "    def __init__(self, c, status, mu0, mu1, tt):\n",
        "        # self.jobs = jobs\n",
        "        self.c = c\n",
        "        self.status = status\n",
        "        self.mu0 = mu0\n",
        "        self.mu1 = mu1\n",
        "        self.tt = tt\n",
        "        self.server_tnext = 0\n",
        "\n",
        "        self.num_busy = 0  # number of busy servers\n",
        "        self.stack = []    # store the patient info in the server\n",
        "        self.queue = []\n",
        "        self.served_jobs = [] \n",
        "\n",
        "    def handle_arrival(self, time, job):\n",
        "        # time here refers to arrival time\n",
        "        # print(\"arrival_job\", job)\n",
        "        # input(\"arrival\")\n",
        "    \n",
        "        if self.num_busy < self.c:\n",
        "            if len(self.queue) == 0:\n",
        "                # print(\"handle_arrival1 \")\n",
        "                self.start_service(time, job)\n",
        "        else:\n",
        "            job.status = \"queue\"\n",
        "            heappush(self.queue, ((job.patype, job.arrival_time), job))\n",
        "            \n",
        "        # print(\"handle_arrival_done \")\n",
        "\n",
        "    def start_service(self, time, job):\n",
        "        # time here refers to start service time\n",
        "        self.num_busy += 1  # server becomes busy.\n",
        "        job.start_service_time = time\n",
        "        job.departure_time = time + job.service_time\n",
        "        \n",
        "        job.status = \"stack\"\n",
        "        heappush(self.stack, (job.departure_time, job))\n",
        "        \n",
        "        # print(\"service_job \", job)\n",
        "        # input(\"service\")\n",
        "        \n",
        "        if job.departure_time <= self.server_tnext:\n",
        "            # print(\"start_service1 \")\n",
        "            self.handle_departure()\n",
        "            \n",
        "        \n",
        "    def handle_departure(self):\n",
        "        # time here refers to departure time \n",
        "        # self.stack.sort(reverse=True)\n",
        "        try:\n",
        "            while self.stack[0][0] <= self.server_tnext:\n",
        "                # print(\"while \", self.stack[0][0], self.server_tnext)\n",
        "                prev_depart = copy.deepcopy(self.stack[0][0])\n",
        "                self.num_busy -= 1\n",
        "                self.stack[0][1].status = \"released\"\n",
        "                next_done = heappop(self.stack)\n",
        "                next_done[1].survival_prob()\n",
        "                self.served_jobs.append(next_done)\n",
        "                \n",
        "                # print(\"next_done \",next_done)\n",
        "                # input(\"depature\")\n",
        "                \n",
        "                if self.queue and self.num_busy < self.c:\n",
        "                    if self.queue[0][1].arrival_time < self.server_tnext:\n",
        "                        # print(\"queue\")\n",
        "                        # print(\"self.server_tnext \", self.server_tnext)\n",
        "                        # print(\"prev_depart \", prev_depart)\n",
        "                        next_job = heappop(self.queue)\n",
        "                        # print(\"next_job! \", next_job)\n",
        "                        \n",
        "                        # input(\"here1\")\n",
        "                        # print(\"arr!\", next_job[1].arrival_time)\n",
        "                        # print(\"prev!\",prev_depart)\n",
        "                        # input(\"here2\")\n",
        "                        self.start_service(max(next_job[1].arrival_time, prev_depart),next_job[1])\n",
        "        except:\n",
        "            pass\n",
        "        \n",
        "    def travel_time(self):\n",
        "        return self.tt\n",
        "        # return random.expovariate(self.tt)\n",
        "        \n",
        "        # return self.tt\n",
        "        \n",
        "    def service_time(self, patient_type):\n",
        "        if patient_type == 0:\n",
        "            return random.expovariate(self.mu0)\n",
        "        else:\n",
        "            return random.expovariate(self.mu1)\n",
        "            \n",
        "    def getSize(self, tnow):\n",
        "        try:\n",
        "            count_queue0 = 0\n",
        "            count_queue1 = 0\n",
        "            copy_queue = copy.deepcopy(self.queue)\n",
        "            for item in copy_queue:\n",
        "                if item[1].arrival_time <= tnow: \n",
        "                    if item[1].patype == 0:\n",
        "                        count_queue0 += 1\n",
        "                    else:\n",
        "                        count_queue1 += 1      \n",
        "                    \n",
        "            return [count_queue0, count_queue1]\n",
        "        \n",
        "        except:\n",
        "            return [0,0]\n",
        "    \n",
        "    \n",
        "    \"\"\"need to optimize code \"\"\"\n",
        "    def getBusyServer(self, tnow):\n",
        "        count_busyserver = 0\n",
        "        copy_stack = copy.deepcopy(self.stack) + copy.deepcopy(self.served_jobs) \n",
        "        # print(\"copy_stack \", copy_stack)\n",
        "        # print(\"stack \", self.stack\n",
        "        \n",
        "        try:\n",
        "            for item in copy_stack:\n",
        "                # print(\"item \",item[1])\n",
        "                if item[1].departure_time >= tnow and item[1].start_service_time <= tnow and count_busyserver < self.c:\n",
        "                    count_busyserver += 1\n",
        "                # print(\"count_busyserver\", count_busyserver)\n",
        "                \n",
        "            return [count_busyserver]\n",
        "        \n",
        "        except:\n",
        "            return [0]\n",
        "    \n",
        "    \n",
        "\"\"\"########################################################################\"\"\"\n",
        "\n",
        "def AssignType():\n",
        "    # set para\n",
        "    p = 0\n",
        "    myassign = np.random.binomial(1,p)\n",
        "    if myassign == 1:\n",
        "        patype = 0\n",
        "    else:\n",
        "        patype = 1\n",
        "    \n",
        "    return patype\n",
        "\n",
        "\"\"\"########################################################################\"\"\"\n",
        "\n",
        "class Env(gym.Env):\n",
        "    def __init__(self, tnow):\n",
        "        self.tnow = tnow\n",
        "        self.gen_patient_time_list = []\n",
        "        self.tgen = 0\n",
        "        self.tgen_next = 0\n",
        "        self.arrival_list1 = []\n",
        "        self.arrival_list2 = []\n",
        "        self.arrival_time = self.tnow\n",
        "        self.surv_num = 5\n",
        "        # Define constants for clearer code\n",
        "        self.S1 = 0\n",
        "        self.S2 = 1\n",
        "        self.action_num = 2\n",
        "        self.action_space = spaces.Discrete(self.action_num)\n",
        "\n",
        "        \n",
        "        \"\"\"['gen_time', 'num_to_hos', 'num_to_acf', \n",
        "        'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2','served_1', 'served_2', 'rewards'])\"\"\"    \n",
        "        \"\"\" obs space\"\"\"\n",
        "        \"\"\" ['num_to_hos', 'num_to_acf', 'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2', \n",
        "        'waiting time at hos', ' service at hos', 'waiting time at acf', ' service at acf'])\"\"\"\n",
        "        \n",
        "        self.obs_low = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
        "        self.obs_high = np.array([15, 15, 20, 20, 20, 20, 12, 6, 1000, 1000, 1000, 1000])\n",
        "        # self.observation_space = spaces.Box(self.obs_low, self.obs_high, dtype=np.int32)\n",
        "        \n",
        "        self.tgen_list = []\n",
        "        # pd.DataFrame(columns = ['gen_time', 'num_to_hos', 'num_to_acf', \n",
        "        # 'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2','served_1', 'served_2', 'rewards'])\n",
        "        self.rewards_records = {}\n",
        "        self.obs_records = {}\n",
        "        self.action_records = {}\n",
        "        self.wait_records1 = {}  # queuing_time\n",
        "        self.service_records1 = {} #service_time\n",
        "        self.wait_records2 = {}  # queuing_time\n",
        "        self.service_records2 = {}\n",
        "\n",
        "        \"\"\"Server Settings\"\"\"\n",
        "        \"\"\" mu_tt = log(x) \"\"\"\n",
        "        \"\"\"pdf = (np.exp(-(np.log(x) - mu)**2 / (2 * sigma**2)) / (x * sigma * np.sqrt(2 * np.pi)))\"\"\"\n",
        "\n",
        "        # set para server\n",
        "\n",
        "        self.server1_rate0 = 1/90  \n",
        "        self.server1_rate1 = 1/30\n",
        "        \"\"\" mean value, not mu\"\"\"\n",
        "        self.server1_tt = 60\n",
        "        \n",
        "        self.server2_rate0 = 1/90\n",
        "        self.server2_rate1 = 1/30\n",
        "        \n",
        "        \"\"\" mean value, not mu\"\"\"\n",
        "        self.server2_tt = 10\n",
        "        \n",
        "        # __init__(self, c, status, mu0, mu1, tt):\n",
        "        self.server1 = Server(12, 'open',self.server1_rate0,self.server1_rate1,self.server1_tt)\n",
        "        self.server2 = Server(6, 'open', self.server2_rate0, self.server2_rate1,self.server2_tt)\n",
        "        self.s2_isopen = True   \n",
        "        self.simulation_time = total_simulation_time\n",
        "        \n",
        "        \n",
        "    #     \"\"\" use pre-generated data\"\"\"\n",
        "    # def Get_Time(self, myi):\n",
        "    #     nlength = 50\n",
        "    #     a = pd.read_excel(\"/Users/machaolun/Documents/GitHub/EvacuationMDP/Evacuation_code/RL/patient_generattion2.xlsx\",\n",
        "    #               index_col=0)\n",
        "    #     b = list(a.iloc[myi,:nlength])\n",
        "    #     self.gen_patient_time_list = deque(b)\n",
        "        \n",
        "        \n",
        "        \"\"\" use generated data\"\"\"\n",
        "    def Get_Time(self, myi):\n",
        "        total_num_patient = 50\n",
        "        arrive_lambda = 1/5.0\n",
        "        mylist = []\n",
        "        t= 0\n",
        "        \"\"\"Generate Casulties\"\"\"\n",
        "        for j in range(total_num_patient):\n",
        "            t+= random.expovariate(arrive_lambda)\n",
        "            mylist.append(t)   \n",
        "            \n",
        "        self.gen_patient_time_list = deque(mylist)   \n",
        "    # def GenerateTime(self):\n",
        "    #     \"\"\"\n",
        "    #     Arrival rate per min\n",
        "    #     Set Arrival Rate (time inhomogeneous) Here\n",
        "    #     \"\"\"\n",
        "    #     # ref: https://zhuanlan.zhihu.com/p/28785318\n",
        "    #     t = 0\n",
        "    #     I = 0\n",
        "    #     s = []\n",
        "        \n",
        "    #     max_simulate_time = 1000\n",
        "        \n",
        "    #     while t <= max_simulate_time:\n",
        "    #         u = np.random.uniform(1e-8,1)\n",
        "    #         t = t - np.log(u)\n",
        "    #         if t <= max_simulate_time:\n",
        "    #             u1 = np.random.uniform(1e-8,1)\n",
        "    #             if u1 <= inhop(t):\n",
        "    #                 I = I + 1\n",
        "    #                 s.append(t)\n",
        "                    \n",
        "    #     # time when event happens\n",
        "    #     self.gen_patient_time_list = deque(s)        \n",
        "    \n",
        "    \"\"\" Open new ACF, the resource comes from hospital\"\"\"        \n",
        "    # def take_action(self, Action):\n",
        "    #     if (not self.s2_isopen) and Action == \"open\":\n",
        "    #         original_num_server = copy.deepcopy(self.server1.c)\n",
        "    #         self.s2_isopen = True\n",
        "            \n",
        "    #         # set para\n",
        "    #         precent_move = 0.2\n",
        "    #         self.server1.c = round((1 - precent_move)*original_num_server)\n",
        "    #         self.server2.c = int(precent_move * original_num_server) + 2 \n",
        "    #         self.server2.status = 'open'\n",
        "            \n",
        "    #         print(\"!!!!!!!!!!!!!!!\\n\",self.server1.c, self.server2.c)\n",
        "    #     else:\n",
        "    #         pass \n",
        "        \n",
        "    def step(self, Action):\n",
        "        \n",
        "        try:\n",
        "            self.tgen = self.gen_patient_time_list.popleft()\n",
        "            try:\n",
        "                self.tgen_next = self.gen_patient_time_list[0]\n",
        "            except:\n",
        "                self.tgen_next = 100000\n",
        "            # action records\n",
        "            self.action_records[self.tgen] = Action\n",
        "            \n",
        "            patient = Event(self.tgen, AssignType(), self.surv_num)\n",
        "\n",
        "            #  self.s2_isopen and         \n",
        "            if Action == self.S2:    \n",
        "                patient.trans_time = self.server2.travel_time()\n",
        "                # self.server2.tnext = min(self.tgen + patient.travel_time , self.simulation_time)\n",
        "                patient.facility = 2\n",
        "                patient.arrival_time = patient.generate_time + patient.trans_time\n",
        "                patient.service_time = self.server2.service_time(patient.patype)\n",
        "                heappush(self.arrival_list2, ((patient.arrival_time), patient))\n",
        "                \n",
        "            elif Action == self.S1:\n",
        "                patient.trans_time = self.server1.travel_time()\n",
        "                # self.server1.tnext = min(self.tgen + patient.travel_time, self.simulation_time)\n",
        "                patient.facility = 1\n",
        "                patient.arrival_time = patient.generate_time + patient.trans_time\n",
        "                patient.service_time = self.server1.service_time(patient.patype) \n",
        "                heappush(self.arrival_list1, ((patient.arrival_time), patient))\n",
        "                                \n",
        "            else:\n",
        "                raise ValueError(\"Received invalid action={} which is not part of the action space\".format(Action))\n",
        "        \n",
        "        except:\n",
        "            self.tgen_next = 100001\n",
        "            \n",
        "        try:\n",
        "            while self.arrival_list1[0][0] <= self.tgen_next:\n",
        "                try:\n",
        "                    self.server1.server_tnext = min(self.arrival_list1[0][0], self.tgen_next)\n",
        "                except:\n",
        "                    self.server1.server_tnext = self.tgen_next\n",
        "                    \n",
        "                self.server1.handle_departure()\n",
        "                mypatient_s1 = heappop(self.arrival_list1)\n",
        "                self.server1.handle_arrival(mypatient_s1[1].arrival_time, mypatient_s1[1])\n",
        "                \n",
        "        except:\n",
        "            pass\n",
        "        \n",
        "        self.server1.server_tnext = self.tgen_next    \n",
        "        self.server1.handle_departure()\n",
        "        \n",
        "        if self.s2_isopen:\n",
        "            try:\n",
        "                while self.arrival_list2[0][0] <= self.tgen_next:\n",
        "    \n",
        "                    try:\n",
        "                        self.server2.server_tnext = min(self.arrival_list2[0][0], self.tgen_next)\n",
        "                    except:\n",
        "                        self.server2.server_tnext = self.tgen_next\n",
        "                        \n",
        "                    self.server2.handle_departure()\n",
        "                    \n",
        "                    mypatient_s2 = heappop(self.arrival_list2)\n",
        "                    self.server2.handle_arrival(mypatient_s2[1].arrival_time, mypatient_s2[1])\n",
        "                    \n",
        "            except:\n",
        "                pass\n",
        "            \n",
        "            self.server2.server_tnext = self.tgen_next\n",
        "            self.server2.handle_departure()\n",
        "                \n",
        "        \"\"\" gen_time, num to hos, num to acf, \n",
        "        [ql10,ql11], [ql20, ql21],\n",
        "        busy server1, busy 2 \"\"\"        \n",
        "        \"\"\"['gen_time', 'num_to_hos', 'num_to_acf', \n",
        "        'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2','served_1', 'served_2', 'rewards'])\"\"\"    \n",
        "        \n",
        "        # print(\"self.tgen_next\", self.tgen_next)\n",
        "        if self.s2_isopen:\n",
        "            Legal_Action = [0,1]\n",
        "        else:\n",
        "            Legal_Action = [0,1]\n",
        "\n",
        "        # for the first one\n",
        "        if len(self.gen_patient_time_list) == 49:\n",
        "            self.tgen_list.append([self.tgen, Legal_Action])\n",
        "            self.obs_records[self.tgen] = np.array([self.tgen, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0 ]) \n",
        "\n",
        "            #action record is above \n",
        "            \n",
        "        all_served = self.server1.served_jobs + self.server2.served_jobs\n",
        "        \n",
        "       \n",
        "        for jobs in all_served:\n",
        "            self.rewards_records[jobs[1].generate_time] = jobs[1].survival_prob_res\n",
        "  \n",
        "        # add obs to records\n",
        "        if self.tgen_next < 100000:\n",
        "            self.tgen_list.append([self.tgen_next, Legal_Action])\n",
        "            i = 0\n",
        "            j = 0\n",
        "            w_record1 = 0\n",
        "            s_record1 = 0\n",
        "            w_record2 = 0\n",
        "            s_record2 = 0\n",
        "            \n",
        "            for jobs2 in all_served:\n",
        "\n",
        "                # time window -- future consideration\n",
        "                if jobs2[1].departure_time < self.tgen_next and jobs2[1].facility  == 1: # 1hos 2acf\n",
        "                    i += 1\n",
        "                    w_record1 += jobs2[1].waiting_time()            \n",
        "                    s_record1 += jobs2[1].service_time\n",
        "                    \n",
        "                if jobs2[1].departure_time < self.tgen_next and jobs2[1].facility  == 2: # 1hos 2acf\n",
        "                    j += 1\n",
        "                    w_record2 += jobs2[1].waiting_time()            \n",
        "                    s_record2 += jobs2[1].service_time                    \n",
        "\n",
        "            self.wait_records1[self.tgen_next] = 0 if i == 0 else int(round(w_record1/i))   # queuing_time\n",
        "            self.service_records1[self.tgen_next] = 0 if i == 0 else int(round(s_record1/i)) #service_time\n",
        "\n",
        "            self.wait_records2[self.tgen_next] = 0 if j == 0 else int(round(w_record2/j))   # queuing_time\n",
        "            self.service_records2[self.tgen_next] = 0 if j == 0 else int(round(s_record2/j))  #service_time\n",
        "                   \n",
        "            # observation records\n",
        "            self.obs_records[self.tgen_next] = np.array([self.tgen_next, len(self.arrival_list1), len(self.arrival_list2)] \n",
        "                        + self.server1.getSize(self.tgen_next)\n",
        "                        + self.server2.getSize(self.tgen_next)\n",
        "                        + self.server1.getBusyServer(self.tgen_next)\n",
        "                        + self.server2.getBusyServer(self.tgen_next)\n",
        "                        + [self.wait_records1[self.tgen_next],\n",
        "                           self.service_records1[self.tgen_next],\n",
        "                           self.wait_records2[self.tgen_next],\n",
        "                           self.service_records2[self.tgen_next]])\n",
        "                                                        \n",
        "                        # + [len(self.server1.served_jobs)]\n",
        "                        # + [len(self.server2.served_jobs)])\n",
        "            states = {'obs': np.delete(self.obs_records[self.tgen_next],0), \n",
        "                      'legal_actions': Legal_Action}\n",
        "        else:\n",
        "            states = {'obs': self.obs_low,\n",
        "                      'legal_actions': [0,1]}\n",
        "        \n",
        "\n",
        "            \n",
        "             \n",
        "        returns = []\n",
        "\n",
        "        for everyone in self.tgen_list[:]:\n",
        "            try:\n",
        "                rewards = self.rewards_records[everyone[0]]\n",
        "                \n",
        "                obs = self.obs_records[everyone[0]]\n",
        "                \n",
        "                actions = self.action_records[everyone[0]]\n",
        "                \n",
        "                returns.append({'obs': obs,'legal_actions': everyone[1],'action': actions ,'reward': rewards})\n",
        "                self.tgen_list.remove(everyone)\n",
        "            except:\n",
        "                continue\n",
        "\n",
        "        done = bool(self.tgen_next == 100000)\n",
        "        # obs = np.array([1,1,1,1,1,1,1,1])\n",
        "        # reward = 1\n",
        "        info = {}\n",
        "        \n",
        "        return states, returns, done, info\n",
        "        \n",
        "    def set_agents(self, agents):\n",
        "        ''' Set the agents that will interact with the environment\n",
        "        Args:\n",
        "            agents (list): List of Agent classes\n",
        "        '''\n",
        "        self.agents = agents\n",
        "\n",
        "\n",
        "    def run(self, myj, is_training=False):\n",
        "        '''\n",
        "        Run a complete game, either for evaluation or training RL agent.\n",
        "        Args:\n",
        "            is_training (boolean): True if for training purpose.\n",
        "        Returns:\n",
        "            (tuple) Tuple containing:\n",
        "                (list): A list of trajectories generated from the environment.\n",
        "                (list): A list payoffs. Each entry corresponds to one player.\n",
        "        Note: The trajectories are 3-dimension list. The first dimension is for different players.\n",
        "              The second dimension is for different transitions. The third dimension is for the contents of each transiton\n",
        "        '''\n",
        "        return_summary = []\n",
        "        done = False\n",
        "        \n",
        "        states, returns, done, info = self.reset()\n",
        "        # read arrival data \n",
        "        self.Get_Time(myj)\n",
        "        \n",
        "        while not done:\n",
        "            # action, _states = model.predict(states)\n",
        "            if not is_training:\n",
        "                action, probs_ = self.agents.eval_step(states)\n",
        "                # print(\"best action \", action)\n",
        "                # print(\"probs \", probs_)\n",
        "            else:\n",
        "                action = self.agents.step(states)\n",
        "            \n",
        "            # if random.random() > 0.9:\n",
        "            #     action = 1   \n",
        "            # else:\n",
        "            #     action = 0\n",
        "            # action = 0\n",
        "            \n",
        "            states, returns, done, info = self.step(action)\n",
        "            \n",
        "            return_summary = np.append(return_summary, returns)\n",
        "        \n",
        "        # served_result = self.get_stat()\n",
        "        \n",
        "        # print(served_result)\n",
        "        \n",
        "        list2 = [x for x in return_summary if x != []]\n",
        "        sorted_ts = sorted(list2, key = lambda i: i['obs'][0]) \n",
        "        \n",
        "        trajectories = []\n",
        "\n",
        "        for i in range(len(sorted_ts) - 1):\n",
        "            #np.delete(sorted_ts[i]['obs'],0)\n",
        "            trajectories.append([{'obs': np.delete(sorted_ts[i]['obs'],0), 'legal_actions': sorted_ts[i]['legal_actions']}, sorted_ts[i]['action'],sorted_ts[i]['reward'],\n",
        "          {'obs': np.delete(sorted_ts[i+1]['obs'],0), 'legal_actions': sorted_ts[i+1]['legal_actions']}, bool(i == len(sorted_ts) - 2)])\n",
        "        \n",
        "        return trajectories\n",
        "    \n",
        "    def get_Reward(self):\n",
        "    # Playing Atari with Deep Reinforcement Learning    \n",
        "    # receives a reward rt representing the change in game score.\n",
        "    # survival probablity\n",
        "    # Resource-Based Patient Prioritization in Mass-Casualty Incidents\n",
        "        all_served = self.server1.served_jobs + self.server2.served_jobs\n",
        "        \n",
        "        for jobs in all_served:\n",
        "            # if jobs[1].generate_time not in list(self.rewards_records.keys()):\n",
        "            self.rewards_records[jobs[1].generate_time] = jobs[1].survival_prob_res\n",
        "\n",
        "\n",
        "    def reset(self):\n",
        "        # return observation\n",
        "        self.tnow = 0\n",
        "        self.gen_patient_time_list = []\n",
        "        self.tgen = 0\n",
        "        self.tnext = 0\n",
        "        self.arrival_list1 = []\n",
        "        self.arrival_list2 = []\n",
        "        self.arrival_time = self.tnow\n",
        "        self.surv_num = 5\n",
        "\n",
        "        \n",
        "        \"\"\"['gen_time', 'num_to_hos', 'num_to_acf', \n",
        "        'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2','served_1', 'served_2', 'rewards'])\"\"\"    \n",
        "        \"\"\" obs space\"\"\"\n",
        "        \"\"\" ['num_to_hos', 'num_to_acf', 'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2', 'waiting time ', ' service'])\"\"\"\n",
        "        \n",
        "        # self.observation_space = spaces.Box(self.obs_low, self.obs_high, dtype=np.int32)\n",
        "        \n",
        "        self.tgen_list = []\n",
        "        # pd.DataFrame(columns = ['gen_time', 'num_to_hos', 'num_to_acf', \n",
        "        # 'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2','served_1', 'served_2', 'rewards'])\n",
        "        self.rewards_records = {}\n",
        "        self.obs_records = {}\n",
        "        self.action_records = {}\n",
        "        self.wait_records1 = {}  # queuing_time\n",
        "        self.service_records1 = {} #service_time\n",
        "        self.wait_records2 = {}  # queuing_time\n",
        "        self.service_records2 = {}\n",
        "        \n",
        "        \n",
        "        self.server1 = Server(12, 'open',self.server1_rate0,self.server1_rate1,self.server1_tt)\n",
        "        self.server2 = Server(6, 'not_open', self.server2_rate0, self.server2_rate1,self.server2_tt)\n",
        "        self.s2_isopen = True\n",
        "        self.simulation_time = 100000\n",
        "        \n",
        "        done = False\n",
        "        info = {}\n",
        "        returns = []\n",
        "        states = {'obs': self.obs_low,'legal_actions': [0,1]}\n",
        "        \n",
        "        return states, returns, done, info\n",
        "\n",
        "    \n",
        "    def get_stat(self):\n",
        "        # Avg_SojournTime, Avg_WaitingTime, Avg_QueueLength\n",
        "        SojournTime1 = 0.0\n",
        "        WaitingTime1 = 0.0\n",
        "        SojournTime2 = 0.0\n",
        "        WaitingTime2 = 0.0\n",
        "        \n",
        "        mydata = []\n",
        "        \n",
        "        for jobs in self.server1.served_jobs:\n",
        "            SojournTime1 += jobs[1].sojourn_time()\n",
        "            WaitingTime1 += jobs[1].waiting_time()\n",
        "            \"\"\" f\"{self.arrival_time}, {self.start_service_time}, {self.service_time}, \n",
        "            {self.departure_time}, {self.facility}, {self.patype} \\n\" \"\"\"\n",
        "\n",
        "            # heappush(mydata, (jobs[1].arrival_time, jobs[1].get_info()))\n",
        "            mydata.append(jobs[1].get_info())\n",
        "            \n",
        "            \"\"\"also attach patient in queue and not release\"\"\"\n",
        "            \n",
        "        for not_complete_job in self.server1.stack:\n",
        "            mydata.append(not_complete_job[1].get_info())\n",
        "            \n",
        "        for not_complete_job in self.server1.queue:\n",
        "            mydata.append(not_complete_job[1].get_info())\n",
        "\n",
        "        with open('Server1.csv', 'w', newline='') as file:\n",
        "            writer = csv.writer(file, delimiter=',')\n",
        "            writer.writerows(mydata)        \n",
        "        \n",
        "        mydata = []\n",
        "        \n",
        "        for jobs in self.server2.served_jobs:\n",
        "            SojournTime2 += jobs[1].sojourn_time()\n",
        "            WaitingTime2 += jobs[1].waiting_time()\n",
        "            mydata.append(jobs[1].get_info())\n",
        "            # mydata.append([jobs[1].arrival_time,jobs[1].start_service_time,jobs[1].service_time, jobs[1].departure_time])\n",
        "            # print(\"{0:.8f}\".format(jobs[1].arrival_time), \"{0:.8f}\".format(jobs[1].start_service_time),\\\n",
        "            #       \"{0:.8f}\".format(jobs[1].service_time), \"{0:.8f}\".format(jobs[1].departure_time))\n",
        "            # print(jobs[1].get_info())\n",
        "            \n",
        "        for not_complete_job in self.server2.stack:\n",
        "            mydata.append(not_complete_job[1].get_info())\n",
        "            \n",
        "        for not_complete_job in self.server2.queue:\n",
        "            mydata.append(not_complete_job[1].get_info())            \n",
        "        \n",
        "        with open('Server2.csv', 'w', newline='') as file:\n",
        "            writer = csv.writer(file, delimiter=',')\n",
        "            writer.writerows(mydata) \n",
        "            \n",
        "            \n",
        "        if len(self.server1.served_jobs) != 0:\n",
        "            Avg_SojournTime1 = SojournTime1/len(self.server1.served_jobs)\n",
        "            Avg_WaitingTime1 = WaitingTime1/len(self.server1.served_jobs)\n",
        "        else:    \n",
        "            Avg_SojournTime1 = None\n",
        "            Avg_WaitingTime1 = None\n",
        "\n",
        "        if len(self.server2.served_jobs) != 0:\n",
        "            Avg_SojournTime2 = SojournTime2/len(self.server2.served_jobs)             \n",
        "            Avg_WaitingTime2 = WaitingTime2/len(self.server2.served_jobs)\n",
        "        else:\n",
        "            Avg_SojournTime2 = None\n",
        "            Avg_WaitingTime2 = None\n",
        "            \n",
        "        \n",
        "        return  Avg_SojournTime1, Avg_WaitingTime1, \\\n",
        "            Avg_SojournTime2, Avg_WaitingTime2\n",
        "                # Avg_QueueLength1, Avg_QueueLength2\n",
        "        \n",
        "    def debug(self):\n",
        "        print(\"==============\")\n",
        "        # self.server1.queue,\n",
        "        # print(\"server1  \", self.server1.c, self.server1.action, \\\n",
        "              # self.server1.getSize(),  self.server1.stack)\n",
        "        # print(\"served job \", self.server1.served_jobs)\n",
        "        # print(\"server2  \", self.server2.c, self.server2.action, \\\n",
        "        # self.server2.getSize(), self.server2.stack)\n",
        "\n",
        "\n",
        " \n"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8AkpLa7UxOYj"
      },
      "source": [
        "import tensorflow as tf\n",
        "import os\n",
        "import pandas as pd\n",
        "import rlcard\n",
        "\n",
        "from rlcard.utils import set_global_seed\n",
        "from rlcard.utils import Logger"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "au65qNEZ1MMZ",
        "cellView": "form"
      },
      "source": [
        "#@title Code for Agent\n",
        "''' DQN agent\n",
        "The code is derived from https://github.com/dennybritz/reinforcement-learning/blob/master/DQN/dqn.py\n",
        "Copyright (c) 2019 DATA Lab at Texas A&M University\n",
        "Copyright (c) 2016 Denny Britz\n",
        "Permission is hereby granted, free of charge, to any person obtaining a copy\n",
        "of this software and associated documentation files (the \"Software\"), to deal\n",
        "in the Software without restriction, including without limitation the rights\n",
        "to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n",
        "copies of the Software, and to permit persons to whom the Software is\n",
        "furnished to do so, subject to the following conditions:\n",
        "The above copyright notice and this permission notice shall be included in all\n",
        "copies or substantial portions of the Software.\n",
        "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n",
        "IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n",
        "FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n",
        "AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n",
        "LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n",
        "OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n",
        "SOFTWARE.\n",
        "'''\n",
        "\n",
        "import random\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from collections import namedtuple\n",
        "\n",
        "from rlcard.utils.utils import remove_illegal\n",
        "\n",
        "Transition = namedtuple('Transition', ['state', 'action', 'reward', 'next_state', 'done'])\n",
        "\n",
        "\n",
        "class DQNAgent(object):\n",
        "\n",
        "    def __init__(self,\n",
        "                 sess,\n",
        "                 scope,\n",
        "                 replay_memory_size=10000,\n",
        "                 replay_memory_init_size=400,\n",
        "                 update_target_estimator_every=500,\n",
        "                 discount_factor=0.999,\n",
        "                 epsilon_start=1.0,\n",
        "                 epsilon_end=0.1,\n",
        "                 epsilon_decay_steps=10000,\n",
        "                 batch_size=32,\n",
        "                 action_num=2,\n",
        "                 state_shape=None,\n",
        "                 train_every=1,\n",
        "                 mlp_layers=None,\n",
        "                 learning_rate=0.00002):\n",
        "\n",
        "        '''\n",
        "        Q-Learning algorithm for off-policy TD control using Function Approximation.\n",
        "        Finds the optimal greedy policy while following an epsilon-greedy policy.\n",
        "        Args:\n",
        "            sess (tf.Session): Tensorflow Session object.\n",
        "            scope (string): The name scope of the DQN agent.\n",
        "            replay_memory_size (int): Size of the replay memory\n",
        "            replay_memory_init_size (int): Number of random experiences to sampel when initializing\n",
        "              the reply memory.\n",
        "            train_every (int): Train the agent every X steps.\n",
        "            update_target_estimator_every (int): Copy parameters from the Q estimator to the\n",
        "              target estimator every N steps\n",
        "            discount_factor (float): Gamma discount factor\n",
        "            epsilon_start (int): Chance to sample a random action when taking an action.\n",
        "              Epsilon is decayed over time and this is the start value\n",
        "            epsilon_end (int): The final minimum value of epsilon after decaying is done\n",
        "            epsilon_decay_steps (int): Number of steps to decay epsilon over\n",
        "            batch_size (int): Size of batches to sample from the replay memory\n",
        "            evaluate_every (int): Evaluate every N steps\n",
        "            action_num (int): The number of the actions\n",
        "            state_space (list): The space of the state vector\n",
        "            train_every (int): Train the network every X steps.\n",
        "            mlp_layers (list): The layer number and the dimension of each layer in MLP\n",
        "            learning_rate (float): The learning rate of the DQN agent.\n",
        "        '''\n",
        "        self.use_raw = False\n",
        "        self.sess = sess\n",
        "        self.scope = scope\n",
        "        self.replay_memory_init_size = replay_memory_init_size\n",
        "        self.update_target_estimator_every = update_target_estimator_every\n",
        "        self.discount_factor = discount_factor\n",
        "        self.epsilon_decay_steps = epsilon_decay_steps\n",
        "        self.batch_size = batch_size\n",
        "        self.action_num = action_num\n",
        "        self.train_every = train_every\n",
        "\n",
        "        # Total timesteps\n",
        "        self.total_t = 0\n",
        "\n",
        "        # Total training step\n",
        "        self.train_t = 0\n",
        "\n",
        "        # The epsilon decay scheduler\n",
        "        self.epsilons = np.linspace(epsilon_start, epsilon_end, epsilon_decay_steps)\n",
        "\n",
        "        # Create estimators\n",
        "        self.q_estimator = Estimator(scope=self.scope+\"_q\", action_num=action_num, learning_rate=learning_rate, state_shape=state_shape, mlp_layers=mlp_layers)\n",
        "        self.target_estimator = Estimator(scope=self.scope+\"_target_q\", action_num=action_num, learning_rate=learning_rate, state_shape=state_shape, mlp_layers=mlp_layers)\n",
        "\n",
        "        # Create replay memory\n",
        "        self.memory = Memory(replay_memory_size, batch_size)\n",
        "\n",
        "    def feed(self, ts):\n",
        "        ''' Store data in to replay buffer and train the agent. There are two stages.\n",
        "            In stage 1, populate the memory without training\n",
        "            In stage 2, train the agent every several timesteps\n",
        "        Args:\n",
        "            ts (list): a list of 5 elements that represent the transition\n",
        "        '''\n",
        "        (state, action, reward, next_state, done) = tuple(ts)\n",
        "        self.feed_memory(state['obs'], action, reward, next_state['obs'], done)\n",
        "        self.total_t += 1\n",
        "        tmp = self.total_t - self.replay_memory_init_size\n",
        "        if tmp>=0 and tmp%self.train_every == 0:\n",
        "            self.train()\n",
        "\n",
        "    def step(self, state):\n",
        "        ''' Predict the action for generating training data\n",
        "        Args:\n",
        "            state (numpy.array): current state\n",
        "        Returns:\n",
        "            action (int): an action id\n",
        "        '''\n",
        "        A = self.predict(state['obs'])\n",
        "        A = remove_illegal(A, state['legal_actions'])\n",
        "        action = np.random.choice(np.arange(len(A)), p=A)\n",
        "        return action\n",
        "\n",
        "    def eval_step(self, state):\n",
        "        ''' Predict the action for evaluation purpose.\n",
        "        Args:\n",
        "            state (numpy.array): current state\n",
        "        Returns:\n",
        "            action (int): an action id\n",
        "            probs (list): a list of probabilies\n",
        "        '''\n",
        "        q_values = self.q_estimator.predict(self.sess, np.expand_dims(state['obs'], 0))[0]\n",
        "        probs = remove_illegal(np.exp(q_values), state['legal_actions'])\n",
        "        best_action = np.argmax(probs)\n",
        "\n",
        "        # print(\"q_values \", q_values)\n",
        "        # print(\"eval_step \", state, best_action, probs)\n",
        "        return best_action, probs\n",
        "\n",
        "    def predict(self, state):\n",
        "        ''' Predict the action probabilities\n",
        "        Args:\n",
        "            state (numpy.array): current state\n",
        "        Returns:\n",
        "            q_values (numpy.array): a 1-d array where each entry represents a Q value\n",
        "        '''\n",
        "        epsilon = self.epsilons[min(self.total_t, self.epsilon_decay_steps-1)]\n",
        "        A = np.ones(self.action_num, dtype=float) * epsilon / self.action_num\n",
        "        q_values = self.q_estimator.predict(self.sess, np.expand_dims(state, 0))[0]\n",
        "        best_action = np.argmax(q_values)\n",
        "        A[best_action] += (1.0 - epsilon)\n",
        "        return A\n",
        "\n",
        "    def train(self):\n",
        "        ''' Train the network\n",
        "        Returns:\n",
        "            loss (float): The loss of the current batch.\n",
        "        '''\n",
        "        state_batch, action_batch, reward_batch, next_state_batch, done_batch = self.memory.sample()\n",
        "        # Calculate q values and targets (Double DQN)\n",
        "        q_values_next = self.q_estimator.predict(self.sess, next_state_batch)\n",
        "        best_actions = np.argmax(q_values_next, axis=1)\n",
        "        q_values_next_target = self.target_estimator.predict(self.sess, next_state_batch)\n",
        "        target_batch = reward_batch + np.invert(done_batch).astype(np.float32) * \\\n",
        "            self.discount_factor * q_values_next_target[np.arange(self.batch_size), best_actions]\n",
        "\n",
        "        # Perform gradient descent update\n",
        "        state_batch = np.array(state_batch)\n",
        "        loss = self.q_estimator.update(self.sess, state_batch, action_batch, target_batch)\n",
        "        print('\\rINFO - Agent {}, step {}, rl-loss: {}'.format(self.scope, self.total_t, loss), end='')\n",
        "\n",
        "\n",
        "        # Update the target estimator\n",
        "        if self.train_t % self.update_target_estimator_every == 0:\n",
        "            copy_model_parameters(self.sess, self.q_estimator, self.target_estimator)\n",
        "            print(\"\\nINFO - Copied model parameters to target network.\")\n",
        "\n",
        "        self.train_t += 1\n",
        "\n",
        "    def feed_memory(self, state, action, reward, next_state, done):\n",
        "        ''' Feed transition to memory\n",
        "        Args:\n",
        "            state (numpy.array): the current state\n",
        "            action (int): the performed action ID\n",
        "            reward (float): the reward received\n",
        "            next_state (numpy.array): the next state after performing the action\n",
        "            done (boolean): whether the episode is finished\n",
        "        '''\n",
        "        self.memory.save(state, action, reward, next_state, done)\n",
        "\n",
        "    def copy_params_op(self, global_vars):\n",
        "        ''' Copys the variables of two estimator to others.\n",
        "        Args:\n",
        "            global_vars (list): A list of tensor\n",
        "        '''\n",
        "        self_vars = tf.contrib.slim.get_variables(scope=self.scope, collection=tf.GraphKeys.TRAINABLE_VARIABLES)\n",
        "        update_ops = []\n",
        "        for v1, v2 in zip(global_vars, self_vars):\n",
        "            op = v2.assign(v1)\n",
        "            update_ops.append(op)\n",
        "        self.sess.run(update_ops)\n",
        "\n",
        "class Estimator():\n",
        "    ''' Q-Value Estimator neural network.\n",
        "        This network is used for both the Q-Network and the Target Network.\n",
        "    '''\n",
        "\n",
        "    def __init__(self, scope=\"estimator\", action_num=2, learning_rate=0.001, state_shape=None, mlp_layers=None):\n",
        "        ''' Initilalize an Estimator object.\n",
        "        Args:\n",
        "            action_num (int): the number output actions\n",
        "            state_shap (list): the shape of the state space\n",
        "        '''\n",
        "        self.scope = scope\n",
        "        self.action_num = action_num\n",
        "        self.learning_rate=learning_rate\n",
        "        self.state_shape = state_shape if isinstance(state_shape, list) else [state_shape]\n",
        "        self.mlp_layers = map(int, mlp_layers)\n",
        "\n",
        "        with tf.variable_scope(scope):\n",
        "            # Build the graph\n",
        "            self._build_model()\n",
        "            update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope=tf.get_variable_scope().name)\n",
        "        # Optimizer Parameters from original paper\n",
        "        self.optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate, name='dqn_adam')\n",
        "\n",
        "        with tf.control_dependencies(update_ops):\n",
        "            self.train_op = self.optimizer.minimize(self.loss, global_step=tf.contrib.framework.get_global_step())\n",
        "\n",
        "    def _build_model(self):\n",
        "        ''' Build an MLP model.\n",
        "        '''\n",
        "        # Placeholders for our input\n",
        "        # Our input are 4 RGB frames of shape 160, 160 each\n",
        "        input_shape = [None]\n",
        "        input_shape.extend(self.state_shape)\n",
        "        self.X_pl = tf.placeholder(shape=input_shape, dtype=tf.float32, name=\"X\")\n",
        "        # The TD target value\n",
        "        self.y_pl = tf.placeholder(shape=[None], dtype=tf.float32, name=\"y\")\n",
        "        # Integer id of which action was selected\n",
        "        self.actions_pl = tf.placeholder(shape=[None], dtype=tf.int32, name=\"actions\")\n",
        "        # Boolean to indicate whether is training or not\n",
        "        self.is_train = tf.placeholder(tf.bool, name=\"is_train\")\n",
        "\n",
        "        batch_size = tf.shape(self.X_pl)[0]\n",
        "\n",
        "        # Batch Normalization\n",
        "        X = tf.layers.batch_normalization(self.X_pl, training=self.is_train)\n",
        "\n",
        "        # Fully connected layers\n",
        "        fc = tf.contrib.layers.flatten(X)\n",
        "        for dim in self.mlp_layers:\n",
        "            fc = tf.contrib.layers.fully_connected(fc, dim, activation_fn=tf.tanh)\n",
        "        self.predictions = tf.contrib.layers.fully_connected(fc, self.action_num, activation_fn=None)\n",
        "\n",
        "        # Get the predictions for the chosen actions only\n",
        "        gather_indices = tf.range(batch_size) * tf.shape(self.predictions)[1] + self.actions_pl\n",
        "        self.action_predictions = tf.gather(tf.reshape(self.predictions, [-1]), gather_indices)\n",
        "\n",
        "        # Calculate the loss\n",
        "        self.losses = tf.squared_difference(self.y_pl, self.action_predictions)\n",
        "        self.loss = tf.reduce_mean(self.losses)\n",
        "\n",
        "    def predict(self, sess, s):\n",
        "        ''' Predicts action values.\n",
        "        Args:\n",
        "          sess (tf.Session): Tensorflow Session object\n",
        "          s (numpy.array): State input of shape [batch_size, 4, 160, 160, 3]\n",
        "          is_train (boolean): True if is training\n",
        "        Returns:\n",
        "          Tensor of shape [batch_size, NUM_VALID_ACTIONS] containing the estimated\n",
        "          action values.\n",
        "        '''\n",
        "        return sess.run(self.predictions, { self.X_pl: s, self.is_train:False})\n",
        "\n",
        "    def update(self, sess, s, a, y):\n",
        "        ''' Updates the estimator towards the given targets.\n",
        "        Args:\n",
        "          sess (tf.Session): Tensorflow Session object\n",
        "          s (list): State input of shape [batch_size, 4, 160, 160, 3]\n",
        "          a (list): Chosen actions of shape [batch_size]\n",
        "          y (list): Targets of shape [batch_size]\n",
        "        Returns:\n",
        "          The calculated loss on the batch.\n",
        "        '''\n",
        "        feed_dict = { self.X_pl: s, self.y_pl: y, self.actions_pl: a, self.is_train: True}\n",
        "        _, _, loss = sess.run(\n",
        "                [tf.contrib.framework.get_global_step(), self.train_op, self.loss],\n",
        "                feed_dict)\n",
        "        return loss\n",
        "\n",
        "class Memory(object):\n",
        "    ''' Memory for saving transitions\n",
        "    '''\n",
        "\n",
        "    def __init__(self, memory_size, batch_size):\n",
        "        ''' Initialize\n",
        "        Args:\n",
        "            memory_size (int): the size of the memroy buffer\n",
        "        '''\n",
        "        self.memory_size = memory_size\n",
        "        self.batch_size = batch_size\n",
        "        self.memory = []\n",
        "\n",
        "    def save(self, state, action, reward, next_state, done):\n",
        "        ''' Save transition into memory\n",
        "        Args:\n",
        "            state (numpy.array): the current state\n",
        "            action (int): the performed action ID\n",
        "            reward (float): the reward received\n",
        "            next_state (numpy.array): the next state after performing the action\n",
        "            done (boolean): whether the episode is finished\n",
        "        '''\n",
        "        if len(self.memory) == self.memory_size:\n",
        "            self.memory.pop(0)\n",
        "        transition = Transition(state, action, reward, next_state, done)\n",
        "        self.memory.append(transition)\n",
        "\n",
        "    def sample(self):\n",
        "        ''' Sample a minibatch from the replay memory\n",
        "        Returns:\n",
        "            state_batch (list): a batch of states\n",
        "            action_batch (list): a batch of actions\n",
        "            reward_batch (list): a batch of rewards\n",
        "            next_state_batch (list): a batch of states\n",
        "            done_batch (list): a batch of dones\n",
        "        '''\n",
        "        samples = random.sample(self.memory, self.batch_size)\n",
        "        return map(np.array, zip(*samples))\n",
        "\n",
        "def copy_model_parameters(sess, estimator1, estimator2):\n",
        "    ''' Copys the model parameters of one estimator to another.\n",
        "    Args:\n",
        "        sess (tf.Session): Tensorflow Session object\n",
        "        estimator1 (Estimator): Estimator to copy the paramters from\n",
        "        estimator2 (Estimator): Estimator to copy the parameters to\n",
        "    '''\n",
        "    e1_params = [t for t in tf.trainable_variables() if t.name.startswith(estimator1.scope)]\n",
        "    e1_params = sorted(e1_params, key=lambda v: v.name)\n",
        "    e2_params = [t for t in tf.trainable_variables() if t.name.startswith(estimator2.scope)]\n",
        "    e2_params = sorted(e2_params, key=lambda v: v.name)\n",
        "\n",
        "    update_ops = []\n",
        "    for e1_v, e2_v in zip(e1_params, e2_params):\n",
        "        op = e2_v.assign(e1_v)\n",
        "        update_ops.append(op)\n",
        "\n",
        "    sess.run(update_ops)\n",
        "\n",
        "#if __name__ == \"__main__\":\n",
        "#    with tf.Session() as sess:\n",
        "#        agent = DQNAgent(sess,\n",
        "#                         scope='dqn',\n",
        "#                         action_num=4,\n",
        "#                         replay_memory_init_size=100,\n",
        "#                         norm_step=100,\n",
        "#                         state_shape=[2],\n",
        "#                         mlp_layers=[10,10])\n",
        "#\n",
        "#        for a in tf.global_variables():\n",
        "#            print(a)\n",
        "\n"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JdXItFuWahIb"
      },
      "source": [
        "# Make environment\n",
        "env = Env(0)\n",
        "eval_env = Env(0.1)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WagAZhdAYcmG"
      },
      "source": [
        "# Set the iterations numbers and how frequently we evaluate/save plot\n",
        "evaluate_every = 200\n",
        "evaluate_num = 500\n",
        "episode_num = 10000\n",
        "\n",
        "# The intial memory size\n",
        "memory_init_size = 300\n",
        "\n",
        "# Train the agent every X steps\n",
        "train_every = 1\n",
        "\n",
        "# The paths for saving the logs and learning curves\n",
        "log_dir = './experiments/blackjack_dqn_result/'\n",
        "\n",
        "# Set a global seed\n",
        "tf.compat.v1.set_random_seed(0)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j2W2O1hOaL6P",
        "outputId": "7750c64c-bf92-4ff0-8bdf-0acb5fbb1c73",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "sess = tf.Session()\n",
        "# Initialize a global step\n",
        "global_step = tf.Variable(0, name='global_step', trainable=False)\n",
        "\n",
        "# Set up the agents\n",
        "agent = DQNAgent(sess,\n",
        "                  scope='dqn',\n",
        "                  action_num=env.action_num,\n",
        "                  replay_memory_init_size=memory_init_size,\n",
        "                  train_every=train_every,\n",
        "                  state_shape=[12],\n",
        "                  mlp_layers=[12,12])\n",
        "env.set_agents(agent)\n",
        "eval_env.set_agents(agent)\n",
        "\n",
        "# Initialize global variables\n",
        "sess.run(tf.global_variables_initializer())\n",
        "\n",
        "# Init a Logger to plot the learning curve\n",
        "logger = Logger(log_dir)\n",
        "\n",
        "for episode in range(episode_num):\n",
        "    # print('\\n episode1 ', episode)\n",
        "    # Generate data from the environment\n",
        "    trajectories = env.run(myj = 1, is_training=True)\n",
        "    # print(\"trajectories \", trajectories)\n",
        "\n",
        "    # Feed transitions into agent memory, and train the agent\n",
        "    for ts in trajectories:\n",
        "        # print(\"ts \", ts)\n",
        "        agent.feed(ts)\n",
        "\n",
        "    # Evaluate the performance. Play with random agents.\n",
        "    #if episode % evaluate_every == 0:\n",
        "    #    logger.log_performance(env.timestep, tournament(eval_env, evaluate_num)[0])\n",
        "\n",
        "    # Evaluate the performance. Play with random agents.\n",
        "    if episode % evaluate_every == 0:\n",
        "      # print('episode2 ', episode)\n",
        "      # print(\"logger \", eva_perform(eval_env, evaluate_num))\n",
        "      logger.log_performance(episode, eva_perform(eval_env, evaluate_num))        "
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-8-9043f8fa3f87>:253: batch_normalization (from tensorflow.python.layers.normalization) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.BatchNormalization instead.  In particular, `tf.control_dependencies(tf.GraphKeys.UPDATE_OPS)` should not be used (consult the `tf.keras.layers.batch_normalization` documentation).\n",
            "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/layers/normalization.py:327: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `layer.__call__` method instead.\n",
            "WARNING:tensorflow:\n",
            "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
            "For more information, please see:\n",
            "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
            "  * https://github.com/tensorflow/addons\n",
            "  * https://github.com/tensorflow/io (for I/O related ops)\n",
            "If you depend on functionality not listed there, please file an issue.\n",
            "\n",
            "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/contrib/layers/python/layers/layers.py:1634: flatten (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.flatten instead.\n",
            "WARNING:tensorflow:From <ipython-input-8-9043f8fa3f87>:233: get_global_step (from tensorflow.contrib.framework.python.ops.variables) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please switch to tf.train.get_global_step\n",
            "\n",
            "----------------------------------------\n",
            "  timestep     |  0\n",
            "  reward       |  2754.6499996615507\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 300, rl-loss: 3302.699462890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 800, rl-loss: 3299.55322265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 1300, rl-loss: 3138.96435546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 1800, rl-loss: 3376.800048828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 2300, rl-loss: 3474.321533203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 2800, rl-loss: 3059.790283203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 3300, rl-loss: 2983.5859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 3800, rl-loss: 3376.24560546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 4300, rl-loss: 3097.808349609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 4800, rl-loss: 3438.61962890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 5300, rl-loss: 3199.40771484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 5800, rl-loss: 3209.4892578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 6300, rl-loss: 3159.30810546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 6800, rl-loss: 3028.52734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 7300, rl-loss: 3361.866943359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 7800, rl-loss: 3090.368896484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 8300, rl-loss: 3131.6845703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 8800, rl-loss: 2874.6201171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 9300, rl-loss: 3023.66943359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 9800, rl-loss: 3093.059326171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 9849, rl-loss: 2976.705078125\n",
            "----------------------------------------\n",
            "  timestep     |  200\n",
            "  reward       |  2777.782832800822\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 10300, rl-loss: 3221.916259765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 10800, rl-loss: 3211.607177734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 11300, rl-loss: 2988.583740234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 11800, rl-loss: 2878.394287109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 12300, rl-loss: 3218.375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 12800, rl-loss: 3116.130615234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 13300, rl-loss: 2945.283203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 13800, rl-loss: 2975.6884765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 14300, rl-loss: 2886.43310546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 14800, rl-loss: 2879.024169921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 15300, rl-loss: 2898.76025390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 15800, rl-loss: 2946.70361328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 16300, rl-loss: 2943.1484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 16800, rl-loss: 2735.09423828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 17300, rl-loss: 3033.37841796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 17800, rl-loss: 3020.37109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 18300, rl-loss: 2965.2900390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 18800, rl-loss: 3171.886474609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 19300, rl-loss: 2766.466552734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 19649, rl-loss: 2828.13330078125\n",
            "----------------------------------------\n",
            "  timestep     |  400\n",
            "  reward       |  2774.416151647387\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 19800, rl-loss: 2849.09130859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 20300, rl-loss: 2851.0302734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 20800, rl-loss: 3051.94287109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 21300, rl-loss: 2901.015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 21800, rl-loss: 2732.05859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 22300, rl-loss: 2908.22412109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 22800, rl-loss: 2876.67529296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 23300, rl-loss: 2631.7421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 23800, rl-loss: 3107.733642578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 24300, rl-loss: 3190.4853515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 24800, rl-loss: 2798.12353515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 25300, rl-loss: 2927.001220703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 25800, rl-loss: 2994.6787109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 26300, rl-loss: 2875.16650390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 26800, rl-loss: 3032.50244140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 27300, rl-loss: 2802.187255859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 27800, rl-loss: 2860.052734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 28300, rl-loss: 3142.050048828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 28800, rl-loss: 3157.826171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 29300, rl-loss: 3163.05615234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 29449, rl-loss: 3187.798828125\n",
            "----------------------------------------\n",
            "  timestep     |  600\n",
            "  reward       |  2779.9239185716533\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 29800, rl-loss: 3003.4462890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 30300, rl-loss: 3161.874267578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 30800, rl-loss: 3052.52587890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 31300, rl-loss: 3129.1591796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 31800, rl-loss: 3342.76025390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 32300, rl-loss: 3261.21044921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 32800, rl-loss: 2877.909912109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 33300, rl-loss: 3044.225341796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 33800, rl-loss: 2914.26708984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 34300, rl-loss: 3182.408935546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 34800, rl-loss: 3222.02880859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 35300, rl-loss: 3336.592041015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 35800, rl-loss: 3210.728271484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 36300, rl-loss: 3010.311767578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 36800, rl-loss: 3175.0869140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 37300, rl-loss: 3342.268798828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 37800, rl-loss: 3027.36083984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 38300, rl-loss: 3238.911865234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 38800, rl-loss: 3117.03857421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 39249, rl-loss: 3257.220458984375\n",
            "----------------------------------------\n",
            "  timestep     |  800\n",
            "  reward       |  2776.9538814089833\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 39300, rl-loss: 3283.34228515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 39800, rl-loss: 3273.45703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 40300, rl-loss: 2932.067138671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 40800, rl-loss: 3219.3720703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 41300, rl-loss: 3317.5048828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 41800, rl-loss: 3335.020751953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 42300, rl-loss: 3035.032470703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 42800, rl-loss: 3096.865966796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 43300, rl-loss: 3357.03125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 43800, rl-loss: 3142.7392578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 44300, rl-loss: 3260.9228515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 44800, rl-loss: 3196.80419921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 45300, rl-loss: 3038.132080078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 45800, rl-loss: 2838.33740234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 46300, rl-loss: 3224.045654296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 46800, rl-loss: 3151.7783203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 47300, rl-loss: 3045.193359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 47800, rl-loss: 3136.90185546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 48300, rl-loss: 3067.371826171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 48800, rl-loss: 3249.366943359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 49049, rl-loss: 3368.97119140625\n",
            "----------------------------------------\n",
            "  timestep     |  1000\n",
            "  reward       |  2777.759104234168\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 49300, rl-loss: 3268.7646484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 49800, rl-loss: 3319.17041015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 50300, rl-loss: 3206.959716796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 50800, rl-loss: 3432.098388671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 51300, rl-loss: 3291.12939453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 51800, rl-loss: 3372.2080078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 52300, rl-loss: 3081.432373046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 52800, rl-loss: 3076.012451171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 53300, rl-loss: 3064.821044921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 53800, rl-loss: 2934.75341796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 54300, rl-loss: 3462.33203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 54800, rl-loss: 3292.54345703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 55300, rl-loss: 3163.364990234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 55800, rl-loss: 3359.572265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 56300, rl-loss: 3098.6484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 56800, rl-loss: 3102.012939453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 57300, rl-loss: 3170.49072265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 57800, rl-loss: 3167.92822265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 58300, rl-loss: 3123.78125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 58800, rl-loss: 3241.08447265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 58849, rl-loss: 3258.732421875\n",
            "----------------------------------------\n",
            "  timestep     |  1200\n",
            "  reward       |  2775.4120560430574\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 59300, rl-loss: 3046.807373046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 59800, rl-loss: 3197.871337890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 60300, rl-loss: 3153.69970703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 60800, rl-loss: 3192.126953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 61300, rl-loss: 3358.03857421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 61800, rl-loss: 3437.305419921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 62300, rl-loss: 3183.4345703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 62800, rl-loss: 3248.836669921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 63300, rl-loss: 3043.879638671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 63800, rl-loss: 3033.691162109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 64300, rl-loss: 3114.4033203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 64800, rl-loss: 3448.36328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 65300, rl-loss: 3354.7265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 65800, rl-loss: 3200.8232421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 66300, rl-loss: 3050.0703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 66800, rl-loss: 3356.431640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 67300, rl-loss: 3249.091064453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 67800, rl-loss: 3356.04638671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 68300, rl-loss: 3289.83984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 68649, rl-loss: 3322.598876953125\n",
            "----------------------------------------\n",
            "  timestep     |  1400\n",
            "  reward       |  2781.181419490371\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 68800, rl-loss: 3023.0302734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 69300, rl-loss: 3113.012451171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 69800, rl-loss: 2939.572265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 70300, rl-loss: 3272.28125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 70800, rl-loss: 3135.989501953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 71300, rl-loss: 3285.505859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 71800, rl-loss: 3180.495361328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 72300, rl-loss: 3232.61767578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 72800, rl-loss: 3122.3232421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 73300, rl-loss: 3615.775390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 73800, rl-loss: 3208.986572265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 74300, rl-loss: 3254.60107421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 74800, rl-loss: 3325.668701171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 75300, rl-loss: 3219.4609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 75800, rl-loss: 3269.07763671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 76300, rl-loss: 3470.994873046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 76800, rl-loss: 2892.671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 77300, rl-loss: 3165.431640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 77800, rl-loss: 3226.181640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 78300, rl-loss: 3008.249755859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 78449, rl-loss: 3326.225830078125\n",
            "----------------------------------------\n",
            "  timestep     |  1600\n",
            "  reward       |  2776.671120726432\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 78800, rl-loss: 3271.47216796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 79300, rl-loss: 3368.2626953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 79800, rl-loss: 3258.66162109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 80300, rl-loss: 3176.36279296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 80800, rl-loss: 3120.622802734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 81300, rl-loss: 3315.802734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 81800, rl-loss: 3278.80908203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 82300, rl-loss: 3183.716796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 82800, rl-loss: 3441.1533203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 83300, rl-loss: 3149.25\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 83800, rl-loss: 3238.142822265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 84300, rl-loss: 3348.2353515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 84800, rl-loss: 3272.131103515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 85300, rl-loss: 3178.986328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 85800, rl-loss: 3466.892822265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 86300, rl-loss: 3305.45556640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 86800, rl-loss: 3135.60400390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 87300, rl-loss: 3148.34521484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 87800, rl-loss: 3287.80322265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 88249, rl-loss: 3033.008544921875\n",
            "----------------------------------------\n",
            "  timestep     |  1800\n",
            "  reward       |  2776.352850030278\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 88300, rl-loss: 3173.9365234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 88800, rl-loss: 3373.35595703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 89300, rl-loss: 2960.856689453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 89800, rl-loss: 3242.548828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 90300, rl-loss: 3033.764404296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 90800, rl-loss: 3141.7373046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 91300, rl-loss: 2985.2744140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 91800, rl-loss: 3366.671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 92300, rl-loss: 3379.79833984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 92800, rl-loss: 3548.9794921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 93300, rl-loss: 3263.26025390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 93800, rl-loss: 3382.1298828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 94300, rl-loss: 3260.70263671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 94800, rl-loss: 2987.43017578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 95300, rl-loss: 3231.607421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 95800, rl-loss: 3060.185546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 96300, rl-loss: 3239.83203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 96800, rl-loss: 3213.69384765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 97300, rl-loss: 2972.8427734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 97800, rl-loss: 3156.5791015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 98049, rl-loss: 2885.6337890625\n",
            "----------------------------------------\n",
            "  timestep     |  2000\n",
            "  reward       |  2778.1350379755236\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 98300, rl-loss: 3074.3369140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 98800, rl-loss: 3253.11572265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 99300, rl-loss: 3117.447265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 99800, rl-loss: 3375.551025390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 100300, rl-loss: 3350.997314453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 100800, rl-loss: 3452.92431640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 101300, rl-loss: 3367.67529296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 101800, rl-loss: 3344.7509765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 102300, rl-loss: 3255.513671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 102800, rl-loss: 3102.95751953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 103300, rl-loss: 3448.79150390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 103800, rl-loss: 3213.161376953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 104300, rl-loss: 3143.96435546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 104800, rl-loss: 3402.57177734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 105300, rl-loss: 3447.743408203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 105800, rl-loss: 3146.08154296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 106300, rl-loss: 3023.93359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 106800, rl-loss: 3270.7314453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 107300, rl-loss: 3318.2958984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 107800, rl-loss: 3155.54931640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 107849, rl-loss: 3165.614013671875\n",
            "----------------------------------------\n",
            "  timestep     |  2200\n",
            "  reward       |  2778.596219836865\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 108300, rl-loss: 3416.408447265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 108800, rl-loss: 3016.5263671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 109300, rl-loss: 2988.49365234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 109800, rl-loss: 3161.03125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 110300, rl-loss: 3248.65234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 110800, rl-loss: 3514.394775390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 111300, rl-loss: 3271.143798828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 111800, rl-loss: 3203.307861328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 112300, rl-loss: 3019.7705078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 112800, rl-loss: 3150.36083984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 113300, rl-loss: 3426.450927734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 113800, rl-loss: 3210.509033203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 114300, rl-loss: 3475.880126953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 114800, rl-loss: 3297.47216796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 115300, rl-loss: 3382.778564453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 115800, rl-loss: 3317.31005859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 116300, rl-loss: 3203.005126953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 116800, rl-loss: 3239.462158203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 117300, rl-loss: 3108.587890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 117649, rl-loss: 3162.6689453125\n",
            "----------------------------------------\n",
            "  timestep     |  2400\n",
            "  reward       |  2779.5163967374388\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 117800, rl-loss: 3227.9267578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 118300, rl-loss: 3116.4189453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 118800, rl-loss: 2874.499755859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 119300, rl-loss: 3088.65087890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 119800, rl-loss: 3157.076171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 120300, rl-loss: 3401.951171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 120800, rl-loss: 3081.568359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 121300, rl-loss: 3081.882568359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 121800, rl-loss: 3319.5498046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 122300, rl-loss: 3322.65478515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 122800, rl-loss: 3242.91748046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 123300, rl-loss: 3400.250244140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 123800, rl-loss: 3124.22900390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 124300, rl-loss: 3163.97705078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 124800, rl-loss: 3332.6591796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 125300, rl-loss: 3308.609130859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 125800, rl-loss: 3259.767578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 126300, rl-loss: 3205.97412109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 126800, rl-loss: 3225.060546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 127300, rl-loss: 3322.0078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 127449, rl-loss: 3135.86669921875\n",
            "----------------------------------------\n",
            "  timestep     |  2600\n",
            "  reward       |  2777.73829829802\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 127800, rl-loss: 3413.990234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 128300, rl-loss: 3374.159912109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 128800, rl-loss: 3076.4853515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 129300, rl-loss: 3076.756103515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 129800, rl-loss: 3110.814697265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 130300, rl-loss: 3265.663330078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 130800, rl-loss: 3218.12255859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 131300, rl-loss: 3212.8173828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 131800, rl-loss: 3177.126953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 132300, rl-loss: 3138.0380859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 132800, rl-loss: 3112.720703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 133300, rl-loss: 3306.95068359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 133800, rl-loss: 3116.427490234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 134300, rl-loss: 3219.9296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 134800, rl-loss: 3452.16064453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 135300, rl-loss: 3166.040283203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 135800, rl-loss: 3159.490966796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 136300, rl-loss: 3005.103515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 136800, rl-loss: 3477.2353515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 137249, rl-loss: 3063.50439453125\n",
            "----------------------------------------\n",
            "  timestep     |  2800\n",
            "  reward       |  2776.2562911089763\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 137300, rl-loss: 3182.0419921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 137800, rl-loss: 3211.24560546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 138300, rl-loss: 3056.16162109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 138800, rl-loss: 2990.957275390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 139300, rl-loss: 3449.4697265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 139800, rl-loss: 3315.169921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 140300, rl-loss: 3509.66064453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 140800, rl-loss: 3390.30029296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 141300, rl-loss: 3366.41796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 141800, rl-loss: 3226.099365234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 142300, rl-loss: 3222.33740234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 142800, rl-loss: 3144.34814453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 143300, rl-loss: 3272.7470703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 143800, rl-loss: 3285.48876953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 144300, rl-loss: 3194.2587890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 144800, rl-loss: 3323.87451171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 145300, rl-loss: 3284.68994140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 145800, rl-loss: 3244.5810546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 146300, rl-loss: 3136.072998046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 146800, rl-loss: 3107.32568359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 147049, rl-loss: 3423.36962890625\n",
            "----------------------------------------\n",
            "  timestep     |  3000\n",
            "  reward       |  2773.9336220714367\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 147300, rl-loss: 3407.87890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 147800, rl-loss: 3337.8095703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 148300, rl-loss: 3338.8701171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 148800, rl-loss: 3221.13232421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 149300, rl-loss: 3185.66943359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 149800, rl-loss: 3007.46142578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 150300, rl-loss: 3190.1259765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 150800, rl-loss: 3406.436767578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 151300, rl-loss: 3282.821533203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 151800, rl-loss: 3077.467529296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 152300, rl-loss: 3238.963623046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 152800, rl-loss: 3192.265869140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 153300, rl-loss: 3393.49755859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 153800, rl-loss: 3497.223876953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 154300, rl-loss: 3233.711669921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 154800, rl-loss: 3404.5966796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 155300, rl-loss: 3193.959228515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 155800, rl-loss: 3325.472412109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 156300, rl-loss: 3024.323974609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 156800, rl-loss: 3266.101806640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 156849, rl-loss: 3612.6591796875\n",
            "----------------------------------------\n",
            "  timestep     |  3200\n",
            "  reward       |  2777.4737732697067\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 157300, rl-loss: 3456.423828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 157800, rl-loss: 3215.9228515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 158300, rl-loss: 3517.8857421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 158800, rl-loss: 2984.806640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 159300, rl-loss: 3367.911865234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 159800, rl-loss: 3220.62890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 160300, rl-loss: 3255.82470703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 160800, rl-loss: 3300.88671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 161300, rl-loss: 3189.4833984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 161800, rl-loss: 3142.40283203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 162300, rl-loss: 3345.23291015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 162800, rl-loss: 3367.06103515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 163300, rl-loss: 3452.11279296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 163800, rl-loss: 3055.4814453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 164300, rl-loss: 3222.041015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 164800, rl-loss: 3259.92041015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 165300, rl-loss: 3469.253662109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 165800, rl-loss: 3404.123046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 166300, rl-loss: 3254.08154296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 166649, rl-loss: 2839.25146484375\n",
            "----------------------------------------\n",
            "  timestep     |  3400\n",
            "  reward       |  2777.4162829282955\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 166800, rl-loss: 3277.314453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 167300, rl-loss: 3366.419921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 167800, rl-loss: 3118.38671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 168300, rl-loss: 3130.1123046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 168800, rl-loss: 3218.522705078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 169300, rl-loss: 3075.5966796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 169800, rl-loss: 3285.923828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 170300, rl-loss: 3080.61865234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 170800, rl-loss: 3246.63623046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 171300, rl-loss: 3404.1923828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 171800, rl-loss: 3540.2470703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 172300, rl-loss: 3089.9052734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 172800, rl-loss: 3023.732421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 173300, rl-loss: 3398.6669921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 173800, rl-loss: 3392.858642578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 174300, rl-loss: 3343.169677734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 174800, rl-loss: 3080.761962890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 175300, rl-loss: 3161.97119140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 175800, rl-loss: 3175.829345703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 176300, rl-loss: 3311.60400390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 176449, rl-loss: 3135.439697265625\n",
            "----------------------------------------\n",
            "  timestep     |  3600\n",
            "  reward       |  2774.260587111904\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 176800, rl-loss: 3254.55712890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 177300, rl-loss: 3223.921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 177800, rl-loss: 3231.82861328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 178300, rl-loss: 3134.2587890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 178800, rl-loss: 3081.638427734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 179300, rl-loss: 3268.40625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 179800, rl-loss: 3344.996826171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 180300, rl-loss: 3004.12255859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 180800, rl-loss: 3341.568359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 181300, rl-loss: 3166.6318359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 181800, rl-loss: 3382.2626953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 182300, rl-loss: 3403.373046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 182800, rl-loss: 3188.016357421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 183300, rl-loss: 3339.401611328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 183800, rl-loss: 3543.73291015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 184300, rl-loss: 3292.7392578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 184800, rl-loss: 3245.3388671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 185300, rl-loss: 3355.074951171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 185800, rl-loss: 3154.6884765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 186249, rl-loss: 2981.23779296875\n",
            "----------------------------------------\n",
            "  timestep     |  3800\n",
            "  reward       |  2778.5094244181914\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 186300, rl-loss: 3325.492919921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 186800, rl-loss: 3342.906494140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 187300, rl-loss: 3309.360595703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 187800, rl-loss: 3198.11767578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 188300, rl-loss: 3044.606689453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 188800, rl-loss: 3418.13671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 189300, rl-loss: 3616.04736328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 189800, rl-loss: 3076.554931640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 190300, rl-loss: 3326.06884765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 190800, rl-loss: 3304.727294921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 191300, rl-loss: 3179.7373046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 191800, rl-loss: 3370.0849609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 192300, rl-loss: 3259.408935546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 192800, rl-loss: 3326.715576171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 193300, rl-loss: 3045.23779296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 193800, rl-loss: 3157.31884765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 194300, rl-loss: 3652.82861328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 194800, rl-loss: 2948.106201171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 195300, rl-loss: 3312.8525390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 195800, rl-loss: 3630.124755859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 196049, rl-loss: 3470.84130859375\n",
            "----------------------------------------\n",
            "  timestep     |  4000\n",
            "  reward       |  2774.2412469775777\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 196300, rl-loss: 3106.771484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 196800, rl-loss: 3279.23828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 197300, rl-loss: 3402.582275390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 197800, rl-loss: 3242.9345703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 198300, rl-loss: 2990.8173828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 198800, rl-loss: 3437.95556640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 199300, rl-loss: 3166.6025390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 199800, rl-loss: 3161.852294921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 200300, rl-loss: 3337.1455078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 200800, rl-loss: 3427.89453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 201300, rl-loss: 2971.789306640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 201800, rl-loss: 3577.60546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 202300, rl-loss: 3387.22021484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 202800, rl-loss: 3015.849609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 203300, rl-loss: 3250.79345703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 203800, rl-loss: 3172.396728515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 204300, rl-loss: 2981.066650390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 204800, rl-loss: 3417.4873046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 205300, rl-loss: 2899.717529296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 205800, rl-loss: 3189.646240234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 205849, rl-loss: 3235.62890625\n",
            "----------------------------------------\n",
            "  timestep     |  4200\n",
            "  reward       |  2772.405019343199\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 206300, rl-loss: 3305.330322265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 206800, rl-loss: 3350.4345703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 207300, rl-loss: 3357.71044921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 207800, rl-loss: 2909.77001953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 208300, rl-loss: 3525.3876953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 208800, rl-loss: 3185.560302734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 209300, rl-loss: 3306.27880859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 209800, rl-loss: 3296.77734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 210300, rl-loss: 3186.75732421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 210800, rl-loss: 3020.93212890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 211300, rl-loss: 3346.0791015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 211800, rl-loss: 3505.782470703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 212300, rl-loss: 3357.17822265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 212800, rl-loss: 3511.303955078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 213300, rl-loss: 3169.01708984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 213800, rl-loss: 3238.4375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 214300, rl-loss: 3184.650390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 214800, rl-loss: 3320.880615234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 215300, rl-loss: 3251.333984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 215649, rl-loss: 3240.185546875\n",
            "----------------------------------------\n",
            "  timestep     |  4400\n",
            "  reward       |  2776.348906948748\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 215800, rl-loss: 3171.635986328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 216300, rl-loss: 3392.596435546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 216800, rl-loss: 3291.6591796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 217300, rl-loss: 3015.233154296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 217800, rl-loss: 2947.611328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 218300, rl-loss: 3403.49365234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 218800, rl-loss: 3245.956787109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 219300, rl-loss: 3368.66357421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 219800, rl-loss: 3220.0439453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 220300, rl-loss: 3040.4951171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 220800, rl-loss: 3321.31591796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 221300, rl-loss: 3081.04833984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 221800, rl-loss: 3171.08642578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 222300, rl-loss: 3129.6005859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 222800, rl-loss: 2965.676025390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 223300, rl-loss: 3289.051513671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 223800, rl-loss: 3035.795654296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 224300, rl-loss: 3461.82470703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 224800, rl-loss: 3005.21240234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 225300, rl-loss: 3185.2900390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 225449, rl-loss: 3261.311279296875\n",
            "----------------------------------------\n",
            "  timestep     |  4600\n",
            "  reward       |  2776.126130069945\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 225800, rl-loss: 3242.140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 226300, rl-loss: 3458.52099609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 226800, rl-loss: 3259.2900390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 227300, rl-loss: 3503.814453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 227800, rl-loss: 3498.59716796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 228300, rl-loss: 3347.679931640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 228800, rl-loss: 3360.3193359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 229300, rl-loss: 3321.604736328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 229800, rl-loss: 3397.70703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 230300, rl-loss: 3259.646484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 230800, rl-loss: 2995.2255859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 231300, rl-loss: 3125.62646484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 231800, rl-loss: 3495.396484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 232300, rl-loss: 3237.9072265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 232800, rl-loss: 3708.6005859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 233300, rl-loss: 3311.382568359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 233800, rl-loss: 3016.67578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 234300, rl-loss: 3166.83740234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 234800, rl-loss: 3235.0927734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 235249, rl-loss: 3330.00390625\n",
            "----------------------------------------\n",
            "  timestep     |  4800\n",
            "  reward       |  2780.094494002658\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 235300, rl-loss: 3248.8271484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 235800, rl-loss: 3227.30712890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 236300, rl-loss: 3243.818115234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 236800, rl-loss: 3229.55615234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 237300, rl-loss: 3018.63037109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 237800, rl-loss: 3514.755615234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 238300, rl-loss: 3196.583984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 238800, rl-loss: 3426.206298828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 239300, rl-loss: 3002.9052734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 239800, rl-loss: 3212.3095703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 240300, rl-loss: 3415.97119140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 240800, rl-loss: 3570.87939453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 241300, rl-loss: 3477.68310546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 241800, rl-loss: 3160.9326171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 242300, rl-loss: 3262.827880859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 242800, rl-loss: 3602.8974609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 243300, rl-loss: 3268.9638671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 243800, rl-loss: 3083.30126953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 244300, rl-loss: 3276.869384765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 244800, rl-loss: 3372.983154296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 245049, rl-loss: 3367.8115234375\n",
            "----------------------------------------\n",
            "  timestep     |  5000\n",
            "  reward       |  2774.704461707864\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 245300, rl-loss: 3535.68359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 245800, rl-loss: 3251.969482421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 246300, rl-loss: 3156.49462890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 246800, rl-loss: 3329.64111328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 247300, rl-loss: 3143.889404296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 247800, rl-loss: 3247.18359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 248300, rl-loss: 3314.23681640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 248800, rl-loss: 3424.61376953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 249300, rl-loss: 3211.2451171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 249800, rl-loss: 3160.5830078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 250300, rl-loss: 3399.911865234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 250800, rl-loss: 3573.017822265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 251300, rl-loss: 3328.416015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 251800, rl-loss: 3336.814453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 252300, rl-loss: 3382.34619140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 252800, rl-loss: 2925.414794921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 253300, rl-loss: 3254.5791015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 253800, rl-loss: 3623.2099609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 254300, rl-loss: 3282.924072265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 254800, rl-loss: 3154.214599609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 254849, rl-loss: 3295.09619140625\n",
            "----------------------------------------\n",
            "  timestep     |  5200\n",
            "  reward       |  2776.7060971296914\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 255300, rl-loss: 3022.41796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 255800, rl-loss: 3322.55517578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 256300, rl-loss: 3164.56201171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 256800, rl-loss: 3207.559814453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 257300, rl-loss: 2853.145751953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 257800, rl-loss: 3244.71337890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 258300, rl-loss: 3307.534912109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 258800, rl-loss: 3294.15478515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 259300, rl-loss: 3366.37744140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 259800, rl-loss: 3103.9345703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 260300, rl-loss: 3240.807373046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 260800, rl-loss: 3328.27587890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 261300, rl-loss: 3473.93505859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 261800, rl-loss: 3122.9130859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 262300, rl-loss: 3273.138671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 262800, rl-loss: 3108.86865234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 263300, rl-loss: 3197.634765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 263800, rl-loss: 3457.296630859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 264300, rl-loss: 3480.9306640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 264649, rl-loss: 3320.4267578125\n",
            "----------------------------------------\n",
            "  timestep     |  5400\n",
            "  reward       |  2773.9510013889503\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 264800, rl-loss: 3586.137451171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 265300, rl-loss: 2907.23681640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 265800, rl-loss: 3448.4892578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 266300, rl-loss: 3499.544921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 266800, rl-loss: 3238.135498046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 267300, rl-loss: 3098.997802734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 267800, rl-loss: 3554.99755859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 268300, rl-loss: 3363.3896484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 268800, rl-loss: 3431.466796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 269300, rl-loss: 3244.67333984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 269800, rl-loss: 3134.985595703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 270300, rl-loss: 3061.14990234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 270800, rl-loss: 3138.66650390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 271300, rl-loss: 3246.01611328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 271800, rl-loss: 3471.650146484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 272300, rl-loss: 3501.0283203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 272800, rl-loss: 3387.740234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 273300, rl-loss: 3565.63037109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 273800, rl-loss: 3414.06884765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 274300, rl-loss: 3143.75\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 274449, rl-loss: 2988.88037109375\n",
            "----------------------------------------\n",
            "  timestep     |  5600\n",
            "  reward       |  2778.246285109214\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 274800, rl-loss: 3600.615966796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 275300, rl-loss: 3355.86474609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 275800, rl-loss: 3478.330078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 276300, rl-loss: 3499.8447265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 276800, rl-loss: 3319.9365234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 277300, rl-loss: 3070.665283203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 277800, rl-loss: 3386.128173828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 278300, rl-loss: 3569.55224609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 278800, rl-loss: 3449.07177734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 279300, rl-loss: 3083.5908203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 279800, rl-loss: 2992.6181640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 280300, rl-loss: 3369.03759765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 280800, rl-loss: 3385.051025390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 281300, rl-loss: 3370.783203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 281800, rl-loss: 3425.23095703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 282300, rl-loss: 3186.183349609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 282800, rl-loss: 3081.601318359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 283300, rl-loss: 3249.16064453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 283800, rl-loss: 3299.4208984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 284249, rl-loss: 3374.2216796875\n",
            "----------------------------------------\n",
            "  timestep     |  5800\n",
            "  reward       |  2772.937303266677\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 284300, rl-loss: 3174.823974609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 284800, rl-loss: 3233.282958984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 285300, rl-loss: 3345.531494140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 285800, rl-loss: 3239.10107421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 286300, rl-loss: 3078.94775390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 286800, rl-loss: 3051.30908203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 287300, rl-loss: 3295.38916015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 287800, rl-loss: 3405.403076171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 288300, rl-loss: 3229.96826171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 288800, rl-loss: 3490.09716796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 289300, rl-loss: 3299.485595703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 289800, rl-loss: 3175.0634765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 290300, rl-loss: 3405.834716796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 290800, rl-loss: 3778.76611328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 291300, rl-loss: 3085.931396484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 291800, rl-loss: 3183.697509765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 292300, rl-loss: 3318.75439453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 292800, rl-loss: 3201.9052734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 293300, rl-loss: 2995.093017578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 293800, rl-loss: 3400.966796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 294049, rl-loss: 3349.21435546875\n",
            "----------------------------------------\n",
            "  timestep     |  6000\n",
            "  reward       |  2774.713350117633\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 294300, rl-loss: 3456.283447265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 294800, rl-loss: 3061.916259765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 295300, rl-loss: 3224.17138671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 295800, rl-loss: 3402.8837890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 296300, rl-loss: 3064.82080078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 296800, rl-loss: 3238.130859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 297300, rl-loss: 3317.52783203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 297800, rl-loss: 3391.301513671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 298300, rl-loss: 3350.380859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 298800, rl-loss: 3213.3935546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 299300, rl-loss: 3245.94921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 299800, rl-loss: 3276.3740234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 300300, rl-loss: 3260.34375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 300800, rl-loss: 3461.82373046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 301300, rl-loss: 3261.72216796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 301800, rl-loss: 3302.93701171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 302300, rl-loss: 3411.18408203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 302800, rl-loss: 3106.166015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 303300, rl-loss: 3544.939453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 303800, rl-loss: 3137.333984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 303849, rl-loss: 3192.12841796875\n",
            "----------------------------------------\n",
            "  timestep     |  6200\n",
            "  reward       |  2775.316626067324\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 304300, rl-loss: 3201.9140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 304800, rl-loss: 3347.7333984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 305300, rl-loss: 3318.095703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 305800, rl-loss: 3209.85693359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 306300, rl-loss: 3222.49267578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 306800, rl-loss: 3371.67041015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 307300, rl-loss: 3270.284423828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 307800, rl-loss: 3094.85546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 308300, rl-loss: 3372.77392578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 308800, rl-loss: 3281.47998046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 309300, rl-loss: 3288.163818359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 309800, rl-loss: 3222.038818359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 310300, rl-loss: 3136.64453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 310800, rl-loss: 3142.681640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 311300, rl-loss: 3161.663818359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 311800, rl-loss: 3315.64404296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 312300, rl-loss: 3138.5302734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 312800, rl-loss: 3464.9375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 313300, rl-loss: 3217.2822265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 313649, rl-loss: 3262.06494140625\n",
            "----------------------------------------\n",
            "  timestep     |  6400\n",
            "  reward       |  2769.5981051032936\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 313800, rl-loss: 3097.92431640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 314300, rl-loss: 2859.76025390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 314800, rl-loss: 3577.54833984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 315300, rl-loss: 3181.8291015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 315800, rl-loss: 3192.22802734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 316300, rl-loss: 3164.59814453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 316800, rl-loss: 3064.42333984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 317300, rl-loss: 3301.8134765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 317800, rl-loss: 3173.397705078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 318300, rl-loss: 3319.2529296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 318800, rl-loss: 3115.384521484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 319300, rl-loss: 3269.236328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 319800, rl-loss: 3286.2236328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 320300, rl-loss: 3174.227783203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 320800, rl-loss: 3145.204833984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 321300, rl-loss: 3271.24951171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 321800, rl-loss: 3324.8994140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 322300, rl-loss: 3324.2451171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 322800, rl-loss: 3220.835693359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 323300, rl-loss: 3358.99560546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 323449, rl-loss: 3398.583251953125\n",
            "----------------------------------------\n",
            "  timestep     |  6600\n",
            "  reward       |  2775.2452048417376\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 323800, rl-loss: 3401.47607421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 324300, rl-loss: 3312.2822265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 324800, rl-loss: 3345.06103515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 325300, rl-loss: 3191.775146484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 325800, rl-loss: 3429.5361328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 326300, rl-loss: 3593.74267578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 326800, rl-loss: 3055.06103515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 327300, rl-loss: 3137.57666015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 327800, rl-loss: 3562.661376953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 328300, rl-loss: 3186.177734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 328800, rl-loss: 3438.10400390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 329300, rl-loss: 3264.274658203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 329800, rl-loss: 3342.7158203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 330300, rl-loss: 3459.284912109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 330800, rl-loss: 3565.93603515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 331300, rl-loss: 3301.09375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 331800, rl-loss: 3123.74169921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 332300, rl-loss: 3267.7587890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 332800, rl-loss: 3483.882568359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 333249, rl-loss: 3648.88818359375"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:139: RuntimeWarning: overflow encountered in exp\n",
            "/usr/local/lib/python3.6/dist-packages/rlcard/utils/utils.py:356: RuntimeWarning: invalid value encountered in true_divide\n",
            "  probs /= sum(probs)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "----------------------------------------\n",
            "  timestep     |  6800\n",
            "  reward       |  2776.7923989512865\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 333300, rl-loss: 2940.81982421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 333800, rl-loss: 3685.442138671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 334300, rl-loss: 3581.081298828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 334800, rl-loss: 3763.4345703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 335300, rl-loss: 3217.56201171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 335800, rl-loss: 3451.618896484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 336300, rl-loss: 3053.9873046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 336800, rl-loss: 3141.2919921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 337300, rl-loss: 3214.14990234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 337800, rl-loss: 3549.66455078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 338300, rl-loss: 3466.795654296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 338800, rl-loss: 3739.5244140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 339300, rl-loss: 3536.4423828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 339800, rl-loss: 3252.62548828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 340300, rl-loss: 3400.3955078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 340800, rl-loss: 3332.110107421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 341300, rl-loss: 3227.344482421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 341800, rl-loss: 3278.365966796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 342300, rl-loss: 3375.43115234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 342800, rl-loss: 3158.162109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 343049, rl-loss: 3116.248046875\n",
            "----------------------------------------\n",
            "  timestep     |  7000\n",
            "  reward       |  2780.0093876996093\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 343300, rl-loss: 3384.599365234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 343800, rl-loss: 3282.4462890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 344300, rl-loss: 3236.823974609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 344800, rl-loss: 3309.8994140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 345300, rl-loss: 3318.18701171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 345800, rl-loss: 3527.61767578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 346300, rl-loss: 3180.11669921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 346800, rl-loss: 3270.99169921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 347300, rl-loss: 3737.20849609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 347800, rl-loss: 3041.744140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 348300, rl-loss: 3245.28466796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 348800, rl-loss: 3668.12060546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 349300, rl-loss: 3259.9833984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 349800, rl-loss: 3507.4150390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 350300, rl-loss: 3490.021484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 350800, rl-loss: 3184.730712890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 351300, rl-loss: 3140.64892578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 351800, rl-loss: 3183.90771484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 352300, rl-loss: 3319.572265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 352800, rl-loss: 3119.469482421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 352849, rl-loss: 3298.02587890625\n",
            "----------------------------------------\n",
            "  timestep     |  7200\n",
            "  reward       |  2775.363456962536\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 353300, rl-loss: 3291.26513671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 353800, rl-loss: 3129.907470703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 354300, rl-loss: 3286.11865234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 354800, rl-loss: 3083.43408203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 355300, rl-loss: 3295.83251953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 355800, rl-loss: 3544.135498046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 356300, rl-loss: 3293.007568359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 356800, rl-loss: 3247.7509765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 357300, rl-loss: 3400.5546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 357800, rl-loss: 3133.8125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 358300, rl-loss: 3446.590087890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 358800, rl-loss: 3555.974853515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 359300, rl-loss: 3581.51806640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 359800, rl-loss: 3378.38330078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 360300, rl-loss: 3275.08349609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 360800, rl-loss: 3343.275146484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 361300, rl-loss: 3072.08837890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 361800, rl-loss: 3374.77880859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 362300, rl-loss: 3471.854248046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 362649, rl-loss: 3550.4521484375\n",
            "----------------------------------------\n",
            "  timestep     |  7400\n",
            "  reward       |  2774.644318602989\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 362800, rl-loss: 3269.45849609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 363300, rl-loss: 3479.3759765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 363800, rl-loss: 3186.375732421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 364300, rl-loss: 3107.633544921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 364800, rl-loss: 3761.443359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 365300, rl-loss: 3475.999267578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 365800, rl-loss: 3517.12939453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 366300, rl-loss: 3474.019287109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 366800, rl-loss: 3179.0986328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 367300, rl-loss: 3170.43359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 367800, rl-loss: 3723.189453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 368300, rl-loss: 3314.25634765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 368800, rl-loss: 3253.4521484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 369300, rl-loss: 3239.684326171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 369800, rl-loss: 3364.583251953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 370300, rl-loss: 3381.21533203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 370800, rl-loss: 3168.48095703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 371300, rl-loss: 3583.168212890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 371800, rl-loss: 3297.86376953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 372300, rl-loss: 3258.110595703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 372449, rl-loss: 3555.66064453125\n",
            "----------------------------------------\n",
            "  timestep     |  7600\n",
            "  reward       |  2773.9615404645547\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 372800, rl-loss: 3386.39501953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 373300, rl-loss: 3079.82421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 373800, rl-loss: 3474.9501953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 374300, rl-loss: 3694.8125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 374800, rl-loss: 3119.357666015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 375300, rl-loss: 3728.813232421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 375800, rl-loss: 3107.1474609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 376300, rl-loss: 3679.40771484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 376800, rl-loss: 3559.328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 377300, rl-loss: 3132.115234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 377800, rl-loss: 3353.21240234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 378300, rl-loss: 3163.25732421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 378800, rl-loss: 3173.969970703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 379300, rl-loss: 3309.860107421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 379800, rl-loss: 3506.538330078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 380300, rl-loss: 3691.751953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 380800, rl-loss: 3422.159423828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 381300, rl-loss: 3023.89208984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 381800, rl-loss: 3175.18603515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 382249, rl-loss: 3335.438720703125\n",
            "----------------------------------------\n",
            "  timestep     |  7800\n",
            "  reward       |  2776.0766286947483\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 382300, rl-loss: 3248.044189453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 382800, rl-loss: 3503.787109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 383300, rl-loss: 2956.8427734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 383800, rl-loss: 3469.24169921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 384300, rl-loss: 3146.3701171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 384800, rl-loss: 3073.24072265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 385300, rl-loss: 3146.698974609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 385800, rl-loss: 3421.187255859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 386300, rl-loss: 3566.00341796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 386800, rl-loss: 3462.12158203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 387300, rl-loss: 3410.66552734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 387800, rl-loss: 3219.97021484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 388300, rl-loss: 3458.83056640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 388800, rl-loss: 3091.0224609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 389300, rl-loss: 3602.32080078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 389800, rl-loss: 3316.209716796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 390300, rl-loss: 3363.78955078125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 390800, rl-loss: 3218.16650390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 391300, rl-loss: 3152.196533203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 391800, rl-loss: 3587.11328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 392049, rl-loss: 3119.560302734375\n",
            "----------------------------------------\n",
            "  timestep     |  8000\n",
            "  reward       |  2769.5300859103804\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 392300, rl-loss: 3293.373046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 392800, rl-loss: 3414.82666015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 393300, rl-loss: 3515.38037109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 393800, rl-loss: 3483.492919921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 394300, rl-loss: 3153.123291015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 394800, rl-loss: 3518.685791015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 395300, rl-loss: 3326.76904296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 395800, rl-loss: 3361.7119140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 396300, rl-loss: 3443.689453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 396800, rl-loss: 3611.258056640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 397300, rl-loss: 3343.388671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 397800, rl-loss: 3398.38134765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 398300, rl-loss: 3776.447021484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 398800, rl-loss: 3648.72119140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 399300, rl-loss: 3111.547119140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 399800, rl-loss: 3242.4775390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 400300, rl-loss: 3233.1357421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 400800, rl-loss: 3269.49755859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 401300, rl-loss: 3476.065185546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 401800, rl-loss: 3519.703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 401849, rl-loss: 3245.9140625\n",
            "----------------------------------------\n",
            "  timestep     |  8200\n",
            "  reward       |  2776.154342445085\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 402300, rl-loss: 3264.084716796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 402800, rl-loss: 3316.24755859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 403300, rl-loss: 3429.54833984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 403800, rl-loss: 3132.21240234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 404300, rl-loss: 3110.22900390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 404800, rl-loss: 3293.15380859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 405300, rl-loss: 3423.8212890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 405800, rl-loss: 3376.123046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 406300, rl-loss: 3461.49658203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 406800, rl-loss: 3127.08154296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 407300, rl-loss: 3284.483642578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 407800, rl-loss: 3182.1630859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 408300, rl-loss: 3107.333740234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 408800, rl-loss: 3328.07861328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 409300, rl-loss: 3374.258544921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 409800, rl-loss: 3299.950439453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 410300, rl-loss: 3505.504150390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 410800, rl-loss: 3234.431640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 411300, rl-loss: 3257.095703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 411649, rl-loss: 3414.79736328125\n",
            "----------------------------------------\n",
            "  timestep     |  8400\n",
            "  reward       |  2778.232949710084\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 411800, rl-loss: 3488.369140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 412300, rl-loss: 3551.845703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 412800, rl-loss: 3145.3154296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 413300, rl-loss: 3393.73291015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 413800, rl-loss: 3765.7763671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 414300, rl-loss: 3236.4091796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 414800, rl-loss: 3450.60498046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 415300, rl-loss: 3606.921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 415800, rl-loss: 3472.6708984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 416300, rl-loss: 3516.87939453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 416800, rl-loss: 3476.888671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 417300, rl-loss: 3477.345458984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 417800, rl-loss: 3430.1025390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 418300, rl-loss: 3244.22509765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 418800, rl-loss: 3409.98193359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 419300, rl-loss: 3328.820556640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 419800, rl-loss: 3267.47314453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 420300, rl-loss: 3561.5185546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 420800, rl-loss: 3385.4970703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 421300, rl-loss: 3477.739501953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 421449, rl-loss: 3348.041259765625\n",
            "----------------------------------------\n",
            "  timestep     |  8600\n",
            "  reward       |  2775.7619915885066\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 421800, rl-loss: 3482.07421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 422300, rl-loss: 3305.35791015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 422800, rl-loss: 3159.4130859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 423300, rl-loss: 3356.75244140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 423800, rl-loss: 3317.52685546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 424300, rl-loss: 3750.46533203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 424800, rl-loss: 3266.7626953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 425300, rl-loss: 3260.170166015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 425800, rl-loss: 3276.13525390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 426300, rl-loss: 3227.8916015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 426800, rl-loss: 3469.271484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 427300, rl-loss: 3441.002685546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 427800, rl-loss: 3563.86083984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 428300, rl-loss: 3308.86767578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 428800, rl-loss: 3270.010986328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 429300, rl-loss: 3226.728515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 429800, rl-loss: 3661.466796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 430300, rl-loss: 3630.23486328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 430800, rl-loss: 3383.805419921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 431249, rl-loss: 3565.845703125\n",
            "----------------------------------------\n",
            "  timestep     |  8800\n",
            "  reward       |  2781.89774720598\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 431300, rl-loss: 3413.04052734375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 431800, rl-loss: 3172.5947265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 432300, rl-loss: 3503.657470703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 432800, rl-loss: 3452.358154296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 433300, rl-loss: 3312.22412109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 433800, rl-loss: 3298.3056640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 434300, rl-loss: 3408.060791015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 434800, rl-loss: 3849.552001953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 435300, rl-loss: 3408.965576171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 435800, rl-loss: 3508.31103515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 436300, rl-loss: 3660.687255859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 436800, rl-loss: 3517.10107421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 437300, rl-loss: 3666.0546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 437800, rl-loss: 3342.65576171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 438300, rl-loss: 3709.7958984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 438800, rl-loss: 3170.51123046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 439300, rl-loss: 3220.166015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 439800, rl-loss: 3399.3837890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 440300, rl-loss: 3274.3828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 440800, rl-loss: 3555.32373046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 441049, rl-loss: 2941.2841796875\n",
            "----------------------------------------\n",
            "  timestep     |  9000\n",
            "  reward       |  2778.577174095636\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 441300, rl-loss: 3394.19384765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 441800, rl-loss: 3430.081787109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 442300, rl-loss: 3224.59130859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 442800, rl-loss: 3373.63134765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 443300, rl-loss: 3295.79150390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 443800, rl-loss: 3265.5390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 444300, rl-loss: 3474.96923828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 444800, rl-loss: 2995.127685546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 445300, rl-loss: 3300.242431640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 445800, rl-loss: 3221.93603515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 446300, rl-loss: 3359.12646484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 446800, rl-loss: 3558.181884765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 447300, rl-loss: 2845.646240234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 447800, rl-loss: 3742.80908203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 448300, rl-loss: 3343.2001953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 448800, rl-loss: 3206.280029296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 449300, rl-loss: 3524.97021484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 449800, rl-loss: 3406.9130859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 450300, rl-loss: 3083.431396484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 450800, rl-loss: 3359.91357421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 450849, rl-loss: 3142.17822265625\n",
            "----------------------------------------\n",
            "  timestep     |  9200\n",
            "  reward       |  2777.5538264969537\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 451300, rl-loss: 3311.384765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 451800, rl-loss: 3514.876953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 452300, rl-loss: 3473.619140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 452800, rl-loss: 3458.099365234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 453300, rl-loss: 3510.46630859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 453800, rl-loss: 3823.180419921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 454300, rl-loss: 3519.98876953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 454800, rl-loss: 3127.661376953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 455300, rl-loss: 3721.87744140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 455800, rl-loss: 3403.78369140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 456300, rl-loss: 3406.1982421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 456800, rl-loss: 3468.70166015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 457300, rl-loss: 3538.24609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 457800, rl-loss: 3808.460693359375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 458300, rl-loss: 3315.6103515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 458800, rl-loss: 3339.38037109375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 459300, rl-loss: 3795.23095703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 459800, rl-loss: 3195.209716796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 460300, rl-loss: 3547.795166015625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 460649, rl-loss: 3408.93798828125\n",
            "----------------------------------------\n",
            "  timestep     |  9400\n",
            "  reward       |  2778.412733777741\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 460800, rl-loss: 3413.01708984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 461300, rl-loss: 3681.494140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 461800, rl-loss: 3256.945556640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 462300, rl-loss: 3484.714599609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 462800, rl-loss: 3422.0419921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 463300, rl-loss: 3513.05126953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 463800, rl-loss: 3695.6572265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 464300, rl-loss: 3389.7158203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 464800, rl-loss: 3527.218505859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 465300, rl-loss: 2999.1953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 465800, rl-loss: 3070.51904296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 466300, rl-loss: 3414.832275390625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 466800, rl-loss: 3391.494140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 467300, rl-loss: 3449.080810546875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 467800, rl-loss: 3299.528564453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 468300, rl-loss: 3469.540283203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 468800, rl-loss: 3542.64306640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 469300, rl-loss: 3154.983642578125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 469800, rl-loss: 3568.470703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 470300, rl-loss: 2930.246337890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 470449, rl-loss: 3757.91748046875\n",
            "----------------------------------------\n",
            "  timestep     |  9600\n",
            "  reward       |  2777.2306571754084\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 470800, rl-loss: 3500.839111328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 471300, rl-loss: 3506.1376953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 471800, rl-loss: 3585.61962890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 472300, rl-loss: 3432.88720703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 472800, rl-loss: 3381.281005859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 473300, rl-loss: 3300.7099609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 473800, rl-loss: 3276.3076171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 474300, rl-loss: 3588.1884765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 474800, rl-loss: 3485.001708984375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 475300, rl-loss: 3509.1484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 475800, rl-loss: 3184.2490234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 476300, rl-loss: 3648.578857421875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 476800, rl-loss: 3603.68115234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 477300, rl-loss: 3490.215576171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 477800, rl-loss: 3513.44140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 478300, rl-loss: 3171.32470703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 478800, rl-loss: 3258.012939453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 479300, rl-loss: 3382.32470703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 479800, rl-loss: 3139.159423828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 480249, rl-loss: 3622.367919921875\n",
            "----------------------------------------\n",
            "  timestep     |  9800\n",
            "  reward       |  2777.2995324248586\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 480300, rl-loss: 3578.451171875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 480800, rl-loss: 3398.751953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 481300, rl-loss: 3577.230224609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 481800, rl-loss: 3727.42138671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 482300, rl-loss: 3979.072509765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 482800, rl-loss: 3239.90771484375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 483300, rl-loss: 3578.401611328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 483800, rl-loss: 3686.9462890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 484300, rl-loss: 3298.34130859375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 484800, rl-loss: 3772.846923828125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 485300, rl-loss: 3334.54736328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 485800, rl-loss: 3581.44921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 486300, rl-loss: 3545.795654296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 486800, rl-loss: 3345.912353515625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 487300, rl-loss: 3658.748046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 487800, rl-loss: 3553.101806640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 488300, rl-loss: 3836.3115234375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 488800, rl-loss: 3080.35009765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 489300, rl-loss: 3466.85009765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 489800, rl-loss: 3587.17431640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 490000, rl-loss: 3591.783203125"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DRzmdZr-VL5A",
        "outputId": "3e1100df-1160-4ad5-fa55-5282e3ac8e70",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        }
      },
      "source": [
        "logger.close_files()\n",
        "logger.plot('DQN')"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "./experiments/blackjack_dqn_result/performance.csv\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEGCAYAAABPdROvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXhb1Zn48e+RJcu7E6/xlthZSOIkZCUEQsCEQCmlUChlLYVutKWdlk477XQ6LXSm0/lNl0wX2lKmMG2HQKAFulCWQokhC9ljsockduIldrzvliVL5/fHvXJkW7JkW/L6fp7Hj6V7patzJfu+Ouc9i9JaI4QQQgyXZawLIIQQYmKTQCKEEGJEJJAIIYQYEQkkQgghRkQCiRBCiBGxjnUBIiUtLU3n5+cP67kdHR3Ex8eHt0ATgJz31CLnPbWEet779u2r11qnD+XYkzaQ5Ofns3fv3mE9t7i4mKKiovAWaAKQ855a5LynllDPWyl1dqjHlqYtIYQQIyKBRAghxIhIIBFCCDEiEkiEEEKMiAQSIYQQIyKBRAghxIhIIBFCCDEiEkiEECKMKps6+duRmrEuxqiSQCKEEGH0661lfOapfTR1OMe6KKNGAokQQoTRmYYOtIZdZQ1jXZRRI4FECCHC6GxDJwA7TksgEUIIMUQ9bg8VjRJIxATi9miOnGsZ62IIIUzVLQ56PJo56fGcqm2nttUx1kUaFRJIJrCXD1XzgZ9u41Rt+1gXRQiBkR8BuGv1TADeKZ0atRIJJBPY8ZpWAA5WNo9xSYQQAGfM/Mj1i2eQHGtjxykJJGKcK6s3vv0cOdc6xiURQgCcre/AbrWQnRzLmtkp7CitH+sijQoJJBNYaZ0RSI5KIBFiXDjb2Mms1DgsFsXlc9KoaOzqTb5PZhJIJiiPR/e2xx4514LWeoxLJIQ429DBzBRjOdvL56QC8M4U6L0lgWSCqml14HB5KMxKotXRQ1Vz11gXSYgpzePRnG3oJD81DoC5GQmkJdjZcXryN29FLJAopfKUUluUUkeVUkeUUl8ytz+rlCoxf84opUrM7Tal1G+VUoeUUseUUt/wOdb1SqkTSqlTSql/jlSZJxJvfuQDF2cBkicRYqzVtnXT3eNhVppRI1FKcfmcVHacbpj0LQaRrJH0AF/RWhcCa4DPK6UKtdZ3aK2Xaa2XAc8DL5iP/whg11ovAVYCn1FK5SulooCfA+8HCoG7lFKFESz3hFBqBpIblmRhURJIxrOT59s4Vds21sUQEeZtavbWSMBo3qpt6+a0mc+crCIWSLTW1Vrr/ebtNuAYkOPdr5RSwO3AM96nAPFKKSsQCziBVmA1cEprXaq1dgKbgZsjVe5wqm/vjtixy+o6iLVFkZ8aR0FavCTcx7GHni3h688fGutijBvvVjTzhaf343J7xrooYXXWDCSzzBwJwOVz0gB4Z5I3b1lH40WUUvnAcmCXz+Z1wHmt9Unz/h8wAkQ1EAd8WWvdqJTKASp8nlcJXBrgdR4AHgDIzMykuLh4WOVtb28f9nO9zrV7+Oa2Lv5xpZ0l6eF/m/e+5yA9RvPWW2+RbnWwv6xzxGUOx3lPRJE87+4ezdFzncRYYcuWLRjfn8aHsfq8nzzczduVPVyS0MSspKhRf/1InffbJ5xEKTj57i5KLcbnrLUmNUbxp53Hyes+E/bXHIpIft4RDyRKqQSMJqyHtNa+X5vv4kJtBIyahxvIBqYDW5VSbwzltbTWjwOPA6xatUoXFRUNq8zFxcUM97lefyqpQlPCuagM/qHo4hEdy59H9mxhSUEyRUUrOKFOs/OV4yy95HKmx0cP+5jhOO+JKJLnvbO0Ac1Ounpg0crLyEiKicjrDMdYfN5aa765cwvQQ3zOfIpW5o7q60Pkzvv3VfuZmdrKNev7Hnt9/bu8fuw8V155FRbL2H2RiOTnHdFeW0opG0YQ2aS1fsFnuxW4FXjW5+F3A69qrV1a61pgO7AKqALyfB6Xa24b106b05ZsOV4X9kSbs8dDRVMXs82kXmF2EgBHq0eveUtrTU3L1JhHaCRKKi7MOiBT2UB5Y2dvD8Njo/j3OhrONHQwyyc/4nX53FSaO12j+v852iLZa0sBTwDHtNYb++3eABzXWlf6bCsH1pvPjcdI0B8H9gDzlFIFSqlo4E7gz5Eqd7h4k2s1rQ6O14Q30VrR1InboynwBpIsI5CM1gSOWmu++9djrPnPv7P1ZN2ovOZEVVLeTFKMUfE/XSeBZNspI1eQlhA9qQKJ1kbX31kpAwPJZbO9eZLJO54kkjWStcC9wHqf7r43mPvupG+zFhg9sxKUUkcwgsf/aq0Paq17gC8Ar2Ek7J/TWh+JYLnD4nRdO0tykgEoPhHei22ZGaS8gSQ1wc6MpJhRS7g/+uYpnthWRpRF8dO/nwz+hCmspKKZqxdkkGC3So0E2H6qnuzkGDYszORYdeuk6Rbb0OGkvbuHWanxA/bNSI5hdnr8pB5PErEcidZ6G+C3QVBrfb+fbe0YXYD9Pf5l4OVwli+S3B5NaX0H91+ej0drtpyo5XNFc8J2fO8YEm8gAViUnTQqXYB/984ZfvT6e9y6PIclucl85y9H2VXawKWzUyP+2hNNdUsXNa0OluVN40x9B6emeI3E7dHsON3AtQszWZiVxOY9FZxv7WZG8vjJGw2XdzGr/LSBNRIwugG/uL8Kl9uDLWryjQOffGc0Dpxr7sLZ42FOejxXz89g39kmWrpcYTt+aX0HKfHRTIu7kFhflJ3E6bp2HC532F6nvz8eqOLbfzrChoWZ/NdtF3PX6pmkJUTz6JZTEXvNiayk3MiPLMubxpyMBE7XTu6xBMEcPddKc6eLK+alsTDLm9ebHOvp9Hb99VMjAaMbcIfTzcHKyXG+/UkgiQDvN8/Z6QkUzU/H7dFsOxm+am1ZfXuf2ggYCXePJuz5GK+/HzvPV37/Lmtmp/Do3cuxRVmIsUXxqXWz2XqynncrZCr7/koqmomOslCYncTcjARqWh20OcL3hWKi2XrKaOK9fE4aC7ISAThWPTkGap5p6EQpyJ0e63f/mtneebcmZ/OWBJII8PbYmpOewLK8aSTH2thyojZsxy+r7xgQSBZlG/mYSCTcd5U28OCm/SzKTuJ/PraKGNuFvv8fXTOL5Fib1Er8OFDRTGF2EnZrFHPSEwAm/QjnwWw/Vc+CGYmkJ9pJirGRlxI7aXoylTd0kJ0ci93qf1xMSnw0C7OSJu3yuxJIIuB0XQfT42ykxEdjjbJw5UXpvPVeHR7PyBOLHd09nG/tHhBIcqfHkhhjDXvCfevJOj71273kTo/lNx9fTWKMrc/+BLuV+y/P5/Wj53sX2hLG2t2HKltYljcNMCbwgwtfMqYah8vNnjNNXDE3rXfbwhlJk6bn1pmGzoD5Ea/L56Sy92xTRJufx4oEkgg4Xdfe+w0U4Or56dS1dYfl25c30T67XyBRSlGYFb6E++GqFu59Yhf3PrGb1IRonvrUpaQEGOz48bX5xEdH8fMtp8Py2pHwrT8e5uejWGs6cb6NLpeb5TONQDIzJQ5blJqyCfe9Z5pw9nhYO88nkGQlUVbfQaezZwxLFh5nGzoC5ke81i/IwNnj4QevnQj5uPXt3RMi8EggiYDSuo4+geTKi9IB2HJ85M1bvT220gf+0S7KTuZ4TSvuEdR8yhs6+dLmA9z4s20cqmrhXz+wkFcfupKsZP9tvwDT4qL56GWz+OvBc73lG0/cHs0f9lXy4oHRG8fqHYjorZHYoizMSo2fsl2At52qxxalWJ2f0rttYVYSWsOJCOX1RktLl4umTpffMSS+1s5N4/7L83liWxnP7C4Pety9Zxq56vtbuO/J3SP6nx4NEkjCrKXTRX17N7N9LvRpCXaW5iaHJU/ivVDn+/n2U5idhMPloax+6Berxg4nm451c83GYl47UsODRXN465+u5lPrZvfJiQTyqStmY4uy8Mvi8ZcrOV3XTpfLzem6dtq7R+fbb0l5Mynx0cz0ubjMTU+IWNPWHb96h/95uzQixw6H7afqWT5zOvH2CyMOvANpx3PC3e3R3PvELp7bWxHwMeVm199gNRKAf/3AQtbNS+Nbfzw86ADFvWcaue/J3cTYothV1jiqtenhkEASZqfrLyTafRXNz+BARTNNHc4RHb+svoPs5Bi/F/dF2d4R7kNr3tJa86nf7uGNsz3ctjKX4q9ezdeuX0ByrC34k03piXbuvCSPF/ZXjbtFtg6ZXS61Hr1liQ9UNLMsb1qfSRrnZiRwtrETZ094Z71taO9mV1kjb4/TWQaaOpwcPtfSJz8CRl4vwW4d13mSt9+rY+vJen69NXCQ7p0+PkiOBMAaZeHRu1cwKzWOz23axxk/Nfh9Z40gkpEUw8tfWseHlmXzk7+fZO+ZxuGfSIRJIAmz3h5bGX0DydULMtCaEf+zl9Z3+G3WAuNCFR1lGfLFcvupBvaXN/PRwmj+89aLhz1A7IGrjEGXv3prfOVKDlW1EG0OAjtUFfl+/K0OF6fr2nubtbzmZMTj9ujeMQfh4j2n8dps9k5pA1obTTu+LBbFwqzEcR1Intp5FoD3zrcH7Ezi/TxnBmna8kqOtfHk/ZcA8Mnf7ukzxswIInvISIph8wNryEyK4d8/tJicabF8aXNJWMejhZMEkhDtLmsMqV3zdF0HtihFXr/+5BfnJJMaHz3odCl7zzQO2n1Xa01Z3cAxJF62KAsXzUgYco3k0S0nyUi0sy5nZBMd5EyL5cMrctm8p4LatvEzoePBSqN2kJlk51Bl5Me7HKxoQWsGBJK56cbYiXBf8A+bgaS6xTFqTXdDsfVkPYl2K0tzkwfsW5iVxPGatrD0aAy3isZO3jxRy12rZxJlUfzl3XN+H3emoZOMRDtx0aH//8xKjeexj67kbEMnX3h6Pz1uD/vONnHfk3tIT7TzzKeNIAKQGGPjp3ct53yrg3954dC4nFZGAkmIntldzsN/OhK0B0VpXTv5qfFY+02DYLGo3m7A/hJnz+2p4PZfvcPnN+0P+IfS2OGk1dFDQVqC3/0Ai7KSOTqEOYz2nW1kZ2kjD1w5m+iokU9x/bmiObg9mp/9fXy06fa4PRytbmVJbjJLcpJHpUZSUtEEwNJ+gcSbNwv35I2+5zQeuxdvP1XPmjmpA/4nwAgk7d09VDR1jkHJBvfM7nIU8A/r53L5nFT+8m613/+r8oZOvznLYNbMTuU/blnM1pP1fOHpA9z35G7SEqJ55tNrBrQKLMubxleum89fD1UPmq8ZKxJIQtTldON0e9h/tmnQx52ua++TaPdVND+dxg4nB/t9K35iWxlfe/4gM5JiONPQyf5y/9+aA3X99bUoJ4nGDic1raHVCB598xTT42zcfenMkB4fTH5aPB+9dCabdp0d8biSlk4Xz+2t4KWD59h6so6Dlc2UN3TS0ukK+Rvsqbp2HC4PS3KSWZIzjdL6joh/az9Q3syc9PgBOaZ4u5Xs5JgI1EhaexPXJ8dZIClv6KS8sXNAfsRrYW/CfXw1b3X3uHl2TwXXLMwke1osNy3Npryxk3f9THESaPr4UNxxyUw+dUUBrx6pMYLIAwODiNdnrpzN2rmpPPLno+OuGVMCSYi6zJrIYCNTXW4PZxs6ByTava6cl45FwRazeUtrzY/feI9/f+ko7188g5e+uI4Ym4UXD1T6fX6pn8ka++udUr4q+D/m4aoWtpyo4xNrC4ZULQ/my9deRFKsjX/7y9ERVcOf2FbK1/5wkC88fYB7n9jNTY9u58ofbGHpv/2NwodfDSlQeRPtS3KTWZJrdDc9EsFaidaakopmluVN97t/TkZCWMeSNHY4qWru4salWcY4lXF2gdluTgnSPz/iNT8zEYuCo+Os59arh2to6HDy0TWzALhu0Qyioyz8uaRv81ans4fatm7yB/mfDOYbNyzkvz68hOc+c9mg3ewtFsXG25cRY7PwxWcO0N3Tt3Wkx+2hts3Be+dH/70claV2JwNvk9Y7pYEDSXljJz0eHTCQTI+PZvnM6bx1opYvb5jHv790jCe3l3Hbylz+361LsEZZuK5wBn95t5pv3Vg4YLqFsvoOrBYVcD4fgAVZSShlLHK1oTBz0HP6RfEpEu1WPnZ5/qCPG6ppcdF8ecNFPPznI/zt6Hnet2jGsI6zs6yRwqwk/vuOZbR0uXp/mjqcfO+VY7x2+DwLZiQNeoxDVS0k2K0UpMaTaK4LcqiqJWKzFVc2ddHQ4WTZzGl+98/NSGDz7go8Hh2W1fK8zVrL8qaRPw7HqWw7Vc+MpBjmBKilx0ZHkZ8WP+5qJE/tPMus1DjWmQEwOdZG0fx0Xjp4jm9+YCFR5mdX3mg0yYWaaPcnyqK445LQWgQyk2L44UeW8snf7uW2X75DtNVCY4eTxg5nn0T8ie9eH3C6lkiQGkmIvIHk3YpmOgI0jQTqseWr6KJ03q1s4R+eOcCT28u4//J8vv/hi3vbj29dkUNLl4stxwcm5cvqOpiZGue3rdkrwW4lPzU+6Jxbp2rbeOVwDfdeNmtI3XxDdc+lM7koM4H/+OuxYY3M7e5xU1LRzOVzUpk/I5HVBSlcW5jJbStz+fSVs1mSkxzSoloHK1tYlJ2ExaLISIxhRlJMb3I6Eg6YAxGX5/kPJHPSE+hyuakOsekxGO+5LM5JZm5GwrhaPMvj0ew4Vc/auWmDrlVfmDW+pko5XtPKnjNN3HPpzD7B/qZl2dS2dbO77EI33DP15vTxw8iRDNc1CzP5+vULcLk92K0WFmUncfOybB7aMI9/v3kRj969HOV/BY+IkUASIofLQ2KMlR6PZm+APIl3Qr5AORIwugEDvHSwmi9eM4+HP1jY54/1irlppCXY/TZvldV3DJof8SrMTgo6Hcsvik9jt1r45BUFQY83HNYoC9++cRHljZ08ub1syM8/WNmCs8fD6oIUv/uvmJvGgYrmQWfTdbk9HKtu5WKf3kKLc5I5GMFAUlLeTIzNwoIZiX73e+fcClfN4VBlC/mpcSTF2JiXkcDZho5xM6XG0epWmjpdXDFv8NrfwqwkKpu6xk3X1qd2niXaauEjK/P6bL9mQSZx0VH82af3Vm/X32HmSIbrc0VzePWhK3n602t49O4V/NvNi3low0Xce1k+N16cTbR1dC/tEkhC1OVyc9nsVGxRKuCI1NK6djLMmU0DKcxK4qal2XznpkX847UXDfimZo2y8KFl2bx5vLbP4EWPR1PWMHDWX38WZSdR0Rj4H7OisZM/lZzj7tWzSE2wBz3ecF0xL41rCzN59M1T1A7xG7j3W98l+f4Dybp5xvT8O0sDD9I6eb6d7h4Pi3MuBJKLc5Mpi2DC/UBFE0tykgPWGsM9eeOhqpbe85uTkYBHXxggN9a2m8vqrp3jPz/i5c3rHR8HtZL27h5e3F/FjRdnMb3f3HKx0VFcW5jJK4ereweVnmnoJCU+OiK1+olEAkmIHC43KfHRLM2dFjBPMliPLS+LRfHTu5Zz3yB5iVtW5OBya146VN277VyLsVjWYF1/vbz/mC/ur/Q7ivqxt04TpRQPXDk76LFG6ps3LKTHrfn+ECaqA9hV1sj8zMQB/8xeK2ZNI9YWNWjzlrfZ5+LcC81MS3KSI5Zwd/Z4OHKudcD4EV+p5kVnsIT78ZpW7vn1Tpo7B58FoclMtHuXdA53bWckXG4Pm/dUsDgniYykwQe4jqeeWy8eqKLD6e5Nsvd309JsmjtdvUGyvLFjRPmRyUICSYi6XG5ibFFcNieVw1UtA5pUtNac7jdZ43AVZiWxYEYiL+y/0Lzlb3ndQFbMmk5+ahyP/OUoa/7z7zzy5yMcrmpBa835Vge/31vJh1fmjsoSp/lp8XziigL+sK8y5MWvetwe9p1pDNisBWC3RrFmdsqgC4YdrGom0W7tM5me99t7JMaTHKtuxdnjYflM/z22wJileW5GwqAX+0ffPMX2Uw28dqRm0NfznoM3kMxJT0ApoyY21p7eVU5ZfQdfuXZ+0MdmJtmZHmcb8zm3tNZs2nmWRdlJAXNc6+alkxxr623eOlPfSf4oN2uNRxJIQtTt8vQGErdHs6ffvDcNZq+JcAQSpRS3LM/hQHlzbwDpHUMSpMYDkBRj441/vIon71/FZbNTeXpXOTf+bBvX/3grX362BLfWfO6q8K0hH8wX1s8lPdHOI385ElJ34KPVrXQ43YMGEoAr5qVTWt9BZYDBbIcqjWYf3xxUeqKdrOSYEQWSk+fb/OZm+s/4G8jc9ARKA9RIalocvHrYCCCvHz0/6HG857DIDCQxtijypseN+VT1rQ4XP37jPS6fk0rR/PSgj1dKsTAriWNjvJ7N3rNNHK9p46NrZgXsHBBttfD+xTP425EaWrpcnGvpCmmyxslOAkkI3B6N0+0hxmZhxczpRFst7DjVt3krlB5bQ3HzshyUonfq89K6DuKio8hIDC2nYY2ysH5BJj+/ZwV7vrmB735oMXH2KHacbuBDy3JGNTmYYLfytffN50B5M38q8T/NhC9vfiRYILnSXNvCX63E2ePhWE0bS/xMy7E4J7l3fMlQHa9p5dr/fptl//Y6t/xiOz987QQ7TtXjcLk5UN5EhhmoBjMnI576dqffpqtNu87i1poNCzPYerJ+0LU6Dle1MCs1rk/7/NyMyM0wHKpfbDlNc5eLf7lh4aC9tXx5p0rpcYd3QsuheGrnWRLtVm5elj3o425amk2H083vdpxB69Ama5zsJJCEwNsLJtYWRYwtihUzB+ZJvIMFA/WXH6oZyTFcMTeNFw9UGnNsmcvrhvqP6Ss5zsZH18zixQfXsv2f1/MftywOSxmH4sMrclkwI5EntgXvwbWrrJH81LjeuYYCmZuRQGaSna1+Asl759tw9nh6m318LclJprS+Y1jrp+8xg9zHLjPa0H/51mnu/vUuLv7O33jlcM2AGX8DlRsG5jK6e9w8vaucaxZk8IkrCuju8fD2e4Gb7nwT7b7HLq3rGLMLclVzF09uL+OW5TkDyjaYwqwknD2esK1n09HdE7Cm6k9zp5OXD1Xz4ZW5QQfnXjo7lfREe29vxJkpUiORQBIC76h279Ttl81O42h1a59vlKdr24mxWcgeZGTqUN2yPIeKxi72nm3yu077cORMiw1pfZFws1gUd1ySx6GqlkFH3nrMZsNgtREwmkSumJvO9tP1A+YvO9SbaPcTSHK969sPvSnlQHkz6Yl2vn1jIS8+uJaSb1/LE/et4t41s1iYlcStK3KCHsM7eWP/MR8vvVtNQ4eT+y8vYHV+CsmxNv521H+epKnDSWVT14BAOTcjAafbQ0XT2Ezl/8PXTqCAr14XPDfiy5twD9ca7t988RC3/fKdkB9/vKYNl1uz3uyeP5goi+IDS7Jo6jS+iEiORAJJSHxrJACXzUlFa+Obs9fpunYK0hLCMlrZ632LZhAXHcXm3RVUNnWGNIZkPPvg0mysFsXz+/1PAQPGXFHNnS5WF4Q28vzKi9Jo7nQNGGR4qKqFpBir3x413ovvcJq3SiqaWe5T60iMsXHNwky+dWMhf/z8Wq5fnBX0GDnTY4m2WvrUSLTW/GbHGeZmJLB2rjHB4TULMnjzeK3f2sXhc30T7V5j2XPrUGULLx6o4pNXFJA9bWhfqOZmJGCLUmFJuNe3d/PXQ9XUtDpoDbHWWTHEEeo3mc1fiXZrwCWopxIJJCHwBhK7zXi7luYlE2Oz9BlPYvTYCu+FPt5u5fpFM/hjSRUe7X953YkkLcFO0fx0/nigKuDSobvLjPf00hBqJHBhDqdtp/o2AR2qbGFJbrLfZqa0BDvZw0i4N3U4Ka3vCDj9SaiiLIrZaX2nM9lf3syhqhbuuzy/t8zXLcqkudPFnjMDB8B6y744e3wEEq01//HyUVLjo/lc0dA7ckRbLcxJTwhLF+Df763E5Tb+vrwBIpiKpi6UIuQAuDxvGnkpscxKixtWc/NkI4EkBA6X8Y3QWyOxW6NYNSuFnWaexOFyU9EUeLLGkbh1RW7vRTeUMSTj3YdX5HK+tbu3H35/u8oamZEUM+h8Yr7SEuwUZiXx9nsXxpN097g5XtM6aBv94pzkIU+VUlLpnf4kcPfeUBnTmVzIB/x2xxkSY6zcuvxC09i6eelEWy1+m7cOV7UwMyWO5Li+A+GSYmxkJtk5WTu6XWnfPF7LztJGHtowj8RBBuQOpjAr+IwMwXg8mmd2l5NmDrStaAytia+ysZOspJiQR4Qrpfjpncv5t5tHP984HkkgCYGjX44EjOat4zVtNLR3c7ahE63D12PL12VzUslMMv4pCiZBN8P1CzNIirH2GSPjpbVmd5mRHxnKt7x189LYX97UOwfaezXtuNyai3MC1xy8CfdQmz7AmP7EovznXYZqTnoCFU2dOFxuzrc6ePlQNbevyuuzpnm83cq6uWm8fvT8gG7Th6pa/HYkgNHvudXj9vC9l48xOy2eO1cPfzmChVlJ1LV1U9/ePexjbDtVT3ljJ1/aMA8YSo2kk9whDixcPnM6KwYZMzSVRCyQKKXylFJblFJHlVJHlFJfMrc/q5QqMX/OKKVKzO33+GwvUUp5lFLLzH3FSqkTPvuCZ8TCyJtsj42+EEjWmLPH7ipr7E2aRiKHEWVRfOyyfAqzkgZ8+5yI7NYoPrg0m1eP1AzoNVXe2EltW3dIiXZf6+al43Lr3m7DB6uMmkOgCy34JNxDmG7f60BFMxdlJva52A/X3IwEtDa6dW/aVY5b696eYL6uLcyksqmrT+6gqcNJRWNXwBrX3HRjwONwpvDfcbqeLz9bMqQAu3lPBafrOvjn9y/ANsiEosEsyjES7o8Vnx72iombdp0lJT6a21flkhRjDXnBrIrGLhmhPgKRrJH0AF/RWhcCa4DPK6UKtdZ3aK2Xaa2XAc8DLwBorTf5bL8XKNNal/gc7x7vfq11bQTLPYC3aSvGZ1rmi3OTiYuOYsfp+t5vf6EMFhyOB4vm8PKX1kXk2GPhwytzcbg8vHK4b5ONt/NCqPkRr1X507FbLbxtTpdyuKqF5FgbeSmBm8e8QSbU5i2PR/NuRTPLR5gf8fLmMo5Vt/L0rrOsn5/hd2DbNQszUarv4MRAifbeY2cm0h/oqnYAACAASURBVOF0U90S+vxm3T1uvvfyMe759S5ePFDFzkHW3envyW1lrJw1nWuDLFsQzJqCVO5ancevt5Xx4Kb9g46h8ed8q4M3jtXykVW52K1R5KXE9U7zPpjuHjfn2xzkTZdAMlwRCyRa62qt9X7zdhtwDOhtAFZG28XtwDN+nn4XsDlSZRuqCzWSC2+XLcrCJfkpvHO6gdN17eRMiw3r4lC+Jlsyb3neNArS4nl+X9/mrd1ljaTER/deZEMVY4tidcGF6VIOVrZwcYBEu1eqmXAPdSbgsoYOWrpcYcmPAOaYIGNNmPp2Z8C519IT7ayYOb1PnqQ30Z7jfy2WuelDS7i/d76ND/18B4+/XcqHV+QChDyeo7vHzZmGjqBTxYfCYlF875Yl/OsHFvLa0Rpu/9U71AwhGD67pwK3R3OXubbHzJS4kJq2qpq60JpBv3iIwY3KwlZKqXxgObDLZ/M64LzW+qSfp9wB3Nxv2/8qpdwYtZjvaj/1dqXUA8ADAJmZmRQXFw+rvO3t7X2e+26lUc0/sHcPlXEXgkkmTt6qc9HU1kl2vGXYrzde9D/vSFo+3ckLJzv4/ctvkm6+p28d7aQgycJbb7015ONlW1xsrXXy3Mtvcry6i+vzbUHPJSvGxe6T1bwv0RP0sduqjL8B1/mTFBefHnL5/EmLUZyu6yArXuGuOkzxOf8X4jkxTp476+L5V94kNdbCmwccpMcqSnbv8Pv4lm7jX+PVHSV4zgVuDm1ta+ebv3mdZ99zEmuFh1bYWZbexN+iYfuhU8zXwdcGr2rz4NHQXVdOcXHwWQtCMRf40nI7j73byvUb3+ShFXbykwcf++TRmt9s7WJRqoUzh/dwBtDtTs42uHhzyxYsPkGu/9/5oTqj5lN/9gTFrafCcg7jUST/vyMeSJRSCRgX/4e01r4N0nfhpzailLoU6NRaH/bZfI/WukoplWge617gd/2fq7V+HHgcYNWqVbqoqGhYZS4uLsb3ueXvnIHDRyhat5Z0nylKUuY289x722l0aG5akUdR0aJhvd540f+8I2nu0k5e+K8t1Njz+EjRPKpbuqh79U0+e818ioaxRkrm/FaePbGVQ84M3PosN16+hKIlg4/pOOw5yQ//9h4We3zQ837jj4dItJ/jrhuuDttYoSVn9vDm8Vo+t6GQqy/LD/i4WYs7eO5EMW1JBXx4bQHf2v0ml8xNpqhopd/Ha6359s7XIXkGRUVL/D6mob2b+x7bwuF6J+sXZPBfH7649297wfF3cABFRZcFPYdXD9fA9n3ceOUlfqejGa4i4PorW/nkb/by//Z2s/H2ZdwwyOf592PnaXTs5Xu3LafIHMtTYT/Dq2eOsGjlZX1mSej/d16x8yzsO8xN69eOykSmYyWS/98R7bWllLJhXPg3aa1f8NluBW4FnvXztDvpF2C01lXm7zbgaWB1pMrsj8NPsh1gUXZy7/Kt4R5DMtnlTo9jzewUXjhQ1dtbC4aeH/FaMCORtAQ7z+41vkWHMj3HEnN6+bOtwacTOVDezNK8aWEdcLo0dxop8dHcajYnBVKQFs/cjAReP3ae5s7BE+1gNIXOy0jg1CCzAP/7S0c53ujmux9azBP3rerzBakgLb53yp9gSuuN14jEGKcFM5L40xfWUpiVxIOb9vPjN94LmITftKuc9EQ71yy8kKfJM5PnwfIklY2dRFstIc9jJwaKZK8tBTwBHNNab+y3ewNwXGtd2e85Foy8yWafbValVJp52wbcCPjWViKuy+lNtvd9u6IsqvfCF4kxJJPdrStyKavvYH95M7vLGkmwW3unyhgqY7qUVJw9HqbH2UIah+JNVp8JEki6nG6O17QFndV3qB68eg5bvlpEQgi9wK4rzGRnaSPbzclCB+uRBkYyP9AswGcbOvjzu+fYMNPqd6bbgvR46tu7Q+q5VVrXQUaiPaRzGI60BDtPf3oNt67I4cdvnOQzT+0b0NuvsqmTLSdqufOSvD69xryBJFiepKKpk9xpsWH9kjDVRLJGshajCWq9T7fdG8x9A2odpiuBCq11qc82O/CaUuogUAJUAf8TwXIP4OhxY4tSfle9u+qidKwWxbxM/0urisBuWJJFjM3CC/sr2V3WyKr86USN4J953TxjyvLFOYMn2r1S4qPJmRZLWcvgS9MeqmrB7dFh67HlZYuyhLyy3rWFmbg9mp+9aaQU+49o729uRgKNHU4a/IzJeOyt01ijLFyf7/+1vd3Yy+qC10pKQ1jMbaRibFH86CNLefiDhbx5vJabf769T0eCZ/cYtdA7Lum7NG7OtFiUCl4jqWjsGvIYEtFXxHIkWutt4H8Feq31/QG2F2N0Ffbd1gH4bwweJV1Od5+uv77uvnQWa+em9WkaEKFJMKeA+VPJOdq7e7glhAkPB7NuXhpKBV8PxNeSnGT2nD6Px6MDfiM9UG5MURLuGslQLM2dRkaineM1beROjw24cqSX71Qpvsspn2vu4g/7KrnzkplMi/E/u4A3MJTVd7A0yDmX1ncMmrsIF6UUH19bwMKsJD6/aT8f+vl2Nt6+lKsXZLB5TwVXz88gt1/33RhbFJmJMUFHt1c0dYZlkOlUJiPbQ9Dd4yYm2n8gibIoZkuz1rDduiK3d/301QHWZw9VRlIMzz5wGZ8ewhLC718ygwaH5o1jgReRKqloZmZKXETXtw/GYlG94zSCNWuBTyDp17z1+NulaA2fuSrwe5SXEodFETRP0tjhpLnTNaqTia6Zncpf/uEKZqfH88D/7eMz/7ePurZu7rnU/4j6YF2A2xwumjtdvc1gYngkkISgy+kmxiZvVSSsnZtGZpIdu9USll4/qwtSSBrCXE8fWJJFWqzisbdOBxwJfqC8eUxrI17eQBJKR4Ls5FhibVF9moDq2rrZvKecW5bnDPj27ss7mC/QKo5eZWaifbTzg9nTYnnuM5dx28pc3jxeS3ZyDEXz/U92kZsSO+jodm9tRQYjjsyojCOZ6BwuT++EjSK8oiyKr1+/gHPNXdgDNB9GkjdX8NSxZvacaRowPUt1Sxc1rY6w50eGY+3cNL5w9dzeQYODsVgGrg3/xLYynD2ekGbnLUiLDzoo0TvpZDjWyRmqGFsUP7jtYq66KJ2MRHvA3NrMlDhePFBFd4/b79+XN8jI9CgjI4EkBF0u95gsBjVVBOv+Gmnrcq28XA6/LD7F6oK+PctLys0Zf8fB5Hy2KAtffV/oC0bNzUjonaG6pdPFUzvPcsOSrJCaYgvS4tld1ojWOmDHhdK6DmxRKuSZmsNNKcUHlw6+LG7e9Di0Nkav+ztvb7OXjGofGWmvCYFDAsmkZo9SfPzyfLacqON4Td9JHA9UNBMdZWFh1sTrlTc3I4HqFgft3T38ZscZ2rt7+PzVc0N67uz0BDqdbs63Bp6Jt7SunVmp8X57M44XM1MHH0tS2dRFot0acu854d/4/QsYRySQTH73XjaLuOgofvVWaZ/tJeXNLMpJGpNmt5Hy5i4OVjTz5PYyNizMDHmcjjeB7h1w6E9pfce4X7XTm/sItPRwRaMxffxkm89utEkgCYGRI5G3ajKbFhfNXatn8ud3z/U2d7jcHg5WjY9E+3DMyzQCyXf/eoyWLhdfWB9abQQu5D0C5UncHs3Zho5xv2pnRqKdaKslYM+t8sZO8saoaW4ykatjCCRHMjV8al0BFmUkpQFO1LThcHnGRX5kOGalxGGLUhytbmXdvLQhBcQZSTHE2qIoDTAosbKpE5dbM2ecr9ppsRg5HH+BRGtNZVOXdP0NAwkkIXC43NJrawrISo7l5mU5bN5TTmOHkwMV3qV1J2aNxBplId9c4+QLIeZGvCwWRf4gPbe8ASbSo9rDYWaAdUnq2510udxSIwkDCSQhkBzJ1PHZq2bjcHn47Y4zHChvIi0hesx6JYXD1QsyuLYwk0vNFT2HYnZafMCxJL2rgk6Awbh50/0PSvR2/ZUaychJ998QOFweCSRTxNyMRK4tzOS375whwW5lWd60CZ2I/ZcbFg77ubPT43n1SA3OHg/R/SYsLa3vYFqcjZQgU7WMBzNT4mh19NDS6eqzXPWFrr8SSEZKaiRBuD0ap9sjI9unkM9eNYfmTheVTV0TNj8SDgVp8bg92u/I8LK6jjEZiDgc3jEi/c+j0uzJNZFrnOOFXB2D6F2LRGokU8bKWdN7R7hP1PxIOHgDhb+Ee2l9O7PHeaLdyzsdTP88SUVjJ2kJ0RFbInsqkUAShDeQSNPW1PL16+ezdm7qlK6ReANFWb+xJO3dPZxv7Z4QiXa4MCixf56koqlTmrXCRAJJEF1SI5mSVs5KYdOn1gxYFXMqSY6zkRofPaDnlnedkomyKmhSjI3kWJufGkmXTNYYJhJIgnC4jNXz7JIjEVNQQVp87+SMXt7R7hOhx5bXzJS4PqPb3R7NueYumWMrTOTqGIQ0bYmpzN8swKfrOlBqYs2Ym5fSd1BidUsXPR4tNZIwkUAShCTbxVQ2Oz2BurbuPuukl9V3kDs9dkJ9ucpLiaOqqQu3x1hzply6/oaVBJIguqRGIqYwf3NuldZNnB5bXnnT43C6PZxvdQBQKQtahZUEkiC8ORKpkYipyHf9djDmpyqr75gwPba8vM1w3uatiqZOLAqypsWMZbEmDQkkQVyokchbJaaeWalxKHVhLElNq4NOp3tCJdrhQhOWt0mrorGTrORYbON4LZWJRN7FICTZLqYyuzWK3OmxlJo1kt7JGifIqHavnGmxKHVhXZKKJumxFU4SSIKQQCKmuoK0hN5Bid6AMtGatqKtFrKSYi40bTV2Sn4kjCSQBNHba2sKD0wTU9vstHjK6jrQWlNa105cdBQzkiZebiEvxZgF2OnW1LZ1T6juy+OdBJIgupxGsj3GKm+VmJpmp8fT4XRT29ZNqTlZ40ScETnPXJekvkv33hfhIVfHIBw9bmxRCqsk5cQU5Tt5Y2l9+4SZ9be/mSlx1LZ1U91hfDmUHEn4yNUxiC6nmxirNGuJqcvbQ+t4TSuVTV0TrseWlzdwHGswmqslRxI+EQskSqk8pdQWpdRRpdQRpdSXzO3PKqVKzJ8zSqkSc/s9PttLlFIepdQyc99KpdQhpdQppdRP1SjWq7t73MRIfkRMYVlJMditFracqEPriTNZY3/enMjRRjd2q4X0RPsYl2jyiORE/D3AV7TW+5VSicA+pdTrWus7vA9QSv0IaAHQWm8CNpnblwB/1FqXmA/9JfBpYBfwMnA98EoEy97LWB1RKm5i6rJYFAVp8ew83QAw4Ua1e3lrIOfaNXPS4yZknme8GjSQKKX+AuhA+7XWNw2yrxqoNm+3KaWOATnAUfPYCrgdWO/n6XcBm83HZQFJWuud5v3fAR9ilAJJl9Mto9rFlDc7PZ7jNW0AFEzQGkl6oh271UJ3j0cS7WEWrEbyQ/P3rcAM4Cnz/l3A+VBfRCmVDyzHqFF4rQPOa61P+nnKHcDN5u0coNJnX6W5bVQ4etwyhkRMed4Ee0ainQT7xFxRUClFXkocp2rbJT8SZoP+RWit3wKjCUprvcpn11+UUntDeQGlVALwPPCQ1rrVZ9ddwDN+Hn8p0Km1PhzK8fs99wHgAYDMzEyKi4uHeggA2tvbe59bXWuMhB3usSYS3/OeSuS8g+uuN2b/TbG5JvR7FaeNSRudTdUUF9ePcWlGVyT/zkP9ahGvlJqttS4FUEoVAEHrt0opG0YQ2aS1fsFnuxWjlrPSz9PupG+AqQJyfe7nmtsG0Fo/DjwOsGrVKl1UVBSsiH4VFxfjfe5/H9lOcqyNoqLVwzrWROJ73lOJnHdwSeVN/PrQDlbMy6WoaElkCxZBW1oOc7DuLEWrFlO0JGusizOqIvl3HmogeQgoVkqVAgqYhfnNPxAzB/IEcExrvbHf7g3Aca11Zb/nWDDyJuu827TW1UqpVqXUGoymsY8BPwux3CPmcLqZkSS9O8TUNic9gRibhSU5yWNdlBHx5kYkRxJeQQOJeXFPBuYBC8zNx7XW3UGeuha4Fzjk7eIL/IvW+mUG1jq8rgQqvDUfHw8CvwFiMZLso5JoB8mRCAGQHGvjrX+6mrSEif2l6vrFM3jn0Enmz0gc66JMKkEDidbao5T6mtb6OeDdUA+std6GUXvxt+/+ANuLgTV+tu8FFof62uEkAxKFMGROwPm1+sudHse9hXaZPj7MQn0331BKfdUcZJji/YloycYJh8stEzYKIcQgQs2ReAcRft5nmwZmh7c444/D5cEuAxKFECKgkAKJ1rog0gUZj9wejdPtkQGJQggxiJBHFimlFgOFQG9Dqdb6d5Eo1Hghi1oJIURwIQUSpdTDQBFGIHkZeD+wDZgSgURqJEIIEViojf+3AdcANVrrjwNLMboET2pdvTUSyZEIIUQgoV4hu7TWHqBHKZUE1AJ5kSvW+OBwmasjSo1ECCECCjVHslcpNQ34H2Af0A68E7FSjROSIxFCiOBC7bX1oHnzMaXUqxjTuh+MXLHGB8mRCCFEcKEm2/8PeBvYqrU+HtkijR9dUiMRQoigQs2RPAlkAT9TSpUqpZ73Lp07mXlzJFIjEUKIwEJt2tqilHobuAS4GvgssAj4SQTLNuYc0mtLCCGCCrVp6+8Y64+8A2wFLtFa10ayYOOBNG0JIURwoX7VPgg4MWbgvRhYrJSKjVipxoluCSRCCBFUqE1bXwZQSiUC9wP/i7GG+8RenCAIb41EZv8VQojAQm3a+gLGqoUrgTMYyfetkSvW+NA7INEqORIhhAgk1AGJMcBGYJ/WuieC5RlXulxubFEKqyyCI4QQAYV0hdRa/xCwYSydi1IqXSk16aeWd7hkdUQhhAgmpEBizv77deAb5iYb8FSkCjVeOFxuYiQ/IoQQgwq1zeYW4CagA0BrfQ5IjFShxguHyyNjSIQQIohQr5JOrbXGWF4XpVR85Io0fnQ5pWlLCCGCCRpIlFIKeEkp9StgmlLq08AbGDMBT2qOHrd0/RVCiCCC9trSWmul1EeAfwRagfnAt7XWr0e6cGNNaiRCCBFcqN1/9wPNWut/imRhxhtHj4fkWNtYF0MIIca1UAPJpcA9SqmzmAl3AK31xREp1TjhcLrJTJzUg/eFEGLEQg0k74toKcYpyZEIIURwoc61dTbSBRmPJEcihBDBySCJQThcUiMRQohgQm3aGjKlVB7wOyATY/zJ41rrnyilnsXo+QUwDSOJv8x8zsXAr4AkwIOx7olDKVWMsUJjl/m860ZjPRSHy4NdBiQKIcSgIhZIgB7gK1rr/eb08/uUUq9rre/wPkAp9SOgxbxtxZh25V6t9btKqVTA5XO8e7TWeyNY3j7cHo3T7ZFldoUQIoiIBRKtdTVQbd5uU0odA3KAo9A70PF2YL35lOuAg1rrd83nNESqbKHo7pFFrYQQIhTKmPkkwi+iVD7wNrBYa91qbrsS2Ki1XmXefwhjvZMMIB3YrLX+vrmvGEgF3MDzwHe1n4IrpR4AHgDIzMxcuXnz5mGVt729HU90PF98s5OPLoxmw6ypMZakvb2dhISEsS7GqJPznlrkvAd39dVX7/Nel0OmtY7oD5AA7ANu7bf9lxhNX977XwXKgDQgDmN9+GvMfTnm70Tgb8DHgr3uypUr9XBt2bJFVzZ16llff0lv3n122MeZaLZs2TLWRRgTct5Ti5z34IC9eojX+YhmkpVSNowaxCat9Qs+263ArcCzPg+vBN7WWtdrrTuBl4EVAFrrKvN3G/A0sDqS5Qaj6y9I05YQQgQTsUBi5kCeAI5prTf2270BOK61rvTZ9hqwRCkVZwaaq4CjSimrUirNPKYNuBE4HKlyezlcEkiEECIUkey1tRZjRcVDSqkSc9u/aK1fBu4EnvF9sNa6SSm1EdiD0V34Za31X80p618zg0gUozTzsDeQSK8tIYQYXCR7bW0DVIB99wfY/hT9Vl7UWndgJOFHlcPlAaRGIoQQwchouwC6pEYihBAhkUASwIUcibxFQggxGLlKBtAlyXYhhAiJBJIAuiWQCCFESCSQBNAlTVtCCBESuUoGIL22hBAiNBJIAuhyubFaFLYoeYuEEGIwcpUMwOFyS9dfIYQIgQSSABwuN3YJJEIIEZQEkgAcLg+x0fL2CCFEMHKlDKDL6SbGKjUSIYQIRgJJAI4eN7HREkiEECIYCSQBOFxSIxFCiFBIIAmgy+UhRmokQggRlASSALpdbmKs8vYIIUQwcqUMoMslORIhhAiFBJIAJEcihBChkUASQJdTaiRCCBEKCSQBOHo82GXmXyGECEqulH54tMbZ45G5toQQIgQSSPwwlyKRKeSFECIEEkj86DaWIpEaiRBChEACiR8utwZkdUQhhAiFXCn96JamLSGECJkEEj9cHm+NRAKJEEIEI4HED6fUSIQQImQSSPzwBhJJtgshRHARCyRKqTyl1Bal1FGl1BGl1JfM7c8qpUrMnzNKqRKf51yslHrHfPwhpVSMuX2lef+UUuqnSikVqXIDOD2SbBdCiFBZI3jsHuArWuv9SqlEYJ9S6nWt9R3eByilfgS0mLetwFPAvVrrd5VSqYDLfOgvgU8Du4CXgeuBVyJVcKmRCCFE6CL2lVtrXa213m/ebgOOATne/Wat4nbgGXPTdcBBrfW75nMatNZupVQWkKS13qm11sDvgA9FqtwATrck24UQIlSRrJH0UkrlA8sxahRe64DzWuuT5v2LAK2Ueg1IBzZrrb+PEXwqfZ5XiU9A6vc6DwAPAGRmZlJcXDys8rZ1dQOK/Xt2cdoe0Va0caW9vX3Y79lEJuc9tch5h1/EA4lSKgF4HnhIa93qs+suLtRGvGW5ArgE6AT+rpTah9n0FQqt9ePA4wCrVq3SRUVFwyrzK2WvA07WX3UFiTG2YR1jIiouLma479lEJuc9tch5h19Es8lKKRtGENmktX7BZ7sVuBV41ufhlcDbWut6rXUnRi5kBVAF5Po8LtfcFjHStCWEEKGLZK8tBTwBHNNab+y3ewNwXGvt22T1GrBEKRVnBpqrgKNa62qgVSm1xjzmx4A/RarcAC4PWC0KW5T02hJCiGAieaVcC9wLrPfp7nuDue9O+jZrobVuAjYCe4ASYL/W+q/m7geBXwOngNNEsMcWQLdbS48tIYQIUcRyJFrrbYDfTLXW+v4A25/C6ALcf/teYHE4yzcYlxvsEkiEECIk0nbjR7dHExstb40QQoRCrpZ+uNwQY5UaiRBChEICiR9ON8RGSyARQohQSCDxw+nRUiMRQogQSSDxw+mGGKmRCCFESCSQ+OF0a2Ks8tYIIUQo5Grph9MjORIhhAiVBBI/nNJrSwghQiaBxA+nW8uiVkIIESK5Wvrh9EiyXQghQiWBpB+3R9PjkaYtIYQIlQSSfrp7jHV2JdkuhBChkUDST5e5YLt0/xVCiNDI1bIfR48HkBqJEEKESgJJPw6XWSORaeSFECIkEkj66W3akkAihBAhkUDSjzfZLoFECCFCI4Gkny6nmSORQCKEECGRQNLPhRyJvDVCCBEKuVr202UGEqmRCCFEaKxjXYDxRnptCTG1uVwuKisrcTgcY12UsEpOTubYsWO992NiYsjNzcVms4342BJI+pFAIsTUVllZSWJiIvn5+Silxro4YdPW1kZiYiIAWmsaGhqorKykoKBgxMeWpq1+HC4j2S45EiGmJofDQWpq6qQKIv0ppUhNTQ1brUuulv10SY1EiClvMgcRr3CeowSSfhwuN1EKbFHy1gghRCjkatlPl8uNTLMlhBhLUVFRLFu2jEWLFrF06VJ+9KMf4fF4evdv27aN1atXs2DBAubPn88vfvGL3n2PPPIIcXFx1NbW9m5LSEiIaHklkPTjcHmwWSZ/tVYIMX7FxsZSUlLCkSNHeP3113nllVf4zne+A0BNTQ133303jz32GMePH2f79u088cQTvPjii73PT0tL40c/+tGolTdivbaUUnnA74BMQAOPa61/opR6FphvPmwa0Ky1XqaUygeOASfMfTu11p81j1UMZAFd5r7rtNYXwm0YOVxu7FIjEUIA3/nLEY6eaw3rMQuzk3j4g4tCfnxGRgaPP/44l1xyCY888gg///nPuf/++1mxYgVgBI3vf//7fOtb3+KWW24B4BOf+AS/+c1v+PrXv05KSkpYy+9PJGskPcBXtNaFwBrg80qpQq31HVrrZVrrZcDzwAs+zznt3ecNIj7u8dkXkSACRiCRPLsQYjyZPXs2breb2tpajhw5wsqVK/vsX7VqFUePHu29n5CQwCc+8Ql+8pOfjEr5IlYj0VpXA9Xm7Tal1DEgBzgKoIwuA7cD6yNVhuHocrmJlqYtIQQMqeYw3nzxi19k2bJlfPWrX434a43KgESz2Wo5sMtn8zrgvNb6pM+2AqXUAaAV+Fet9Vafff+rlHJj1GK+q7XWfl7nAeABgMzMTIqLi4dc1pq6LqJwD+u5E117e7uc9xQi5+1fcnIybW1to1egAHzLUFZWhsViITY2ljlz5rBjxw7Wr7/wHXzr1q0sX76ctrY2uru7sdlsREVFcdttt7Fx40YA3G73gPNyOBzh+RvQWkf0B0gA9gG39tv+S4ymL+99O5Bq3l4JVABJ5v0c83ci8DfgY8Fed+XKlXo4bnp0m/7A918Z1nMnui1btox1EcaEnPfUEuy8jx49OjoFGUR8fHzv7draWn3ttdfqb3/721prrc+dO6fz8vL0gQMHtNZa19fX6zVr1uji4mKttdYPP/yw/sEPfqC11rqurk7n5+dru92uW1tbB7yOv3MF9uohXucj2mtLKWXDqEFs0lq/4LPdCtwKPOsT0Lq11g3m7X3AaeAi836V+bsNeBpYHakyd0v3XyHEGOvq6urt/rthwwauu+46Hn74YQCysrJ46qmneOCBB5g/fz7Z2dl88Ytf5KqrrhpwnLS0NG655Ra6u7sjWt5I9tpSzDUtTAAACJtJREFUwBPAMa31xn67NwDHtdaVPo9PBxq11m6l1GxgHlBqBp1pWut6MzDdCLwRqXKvnZtGR11VpA4vhBBBud3uQfdfeeWV7N69G4Bf/OIXfO973+P6669n+vTpPPLII30eu3HjRjZu3BjR5rpI1kjWAvcC65VSJebPDea+O4Fn+j3+SuCgUqoE+APwWa11I0aT12tKqYNACVAF/E+kCv2tGwu5vmDks2EKIcRoePDBBzl06BDTp08fszJEstfWNsBv9yet9f1+tj2P0QzWf3sHRs5ECCHEOCQj24UQoh89sFPopBPOc5RAIoQQPmJiYmhoaJjUwUSb65HExMSE5XiysJUQQvjIzc2lsrKSurq6sS5KWDkcjj6Bw7tCYjhIIBFCCB82my0sqwaON8XFxSxfvjwix5amLSGEECMigUQIIcSISCARQggxImqy9kxQStUBZ4f59DSgPozFmSjkvKcWOe+pJdTznqW1Th/KgSdtIBkJpdRerfWqsS7HaJPznlrkvKeWSJ63NG0JIYQYEQkkQgghRkQCiX+Pj3UBxoic99Qi5z21ROy8JUcihBBiRKRGIoQQYkQkkAghhBgRCSQ+lFLXK6VOKKVOKaX+eazLM1JKqTyl1Bal1FGl1BGl1JfM7SlKqdeVUifN39PN7Uop9VPz/A8qpVb4HOs+8/EnlVL3jdU5DYVSKkopdUAp9ZJ5v0Aptcs8v2eVUtHmdrt5/5S5P9/nGN8wt59QSr1vbM4kdEqpaUqpPyiljiuljimlLptCn/eXzb/zw0qpZ5RSMZPxM1dKPamUqlVKHfbZFrbPWCm1Uil1yHzOT83Vbgc31EXeJ+sPEIWxTvxsIBp4Fygc63KN8JyygBXm7UTgPaAQ+D7wz+b2fwb+y7x9A/AKxoJka4Bd5vYUoNT8Pd28PX2szy+E8/9H4GngJfP+c8Cd5u3HgM+Ztx8EHjNv3wk8a94uNP8O7ECB+fcRNdbnFeScfwt8yrwdDUybCp83kAOUAbE+n/X9k/Ezx1hNdgVw2Gdb2D5jYLf5WGU+9/1ByzTWb8p4+QEuA17zuf8N4BtjXa4wn+OfgGuBE0CWuS0LOGHe/hVwl8/jT5j77wJ+5bO9z+PG4w+QC/wdWA+8ZP5T1APW/p838BpwmXnbaj5O9f8b8H3cePwBks2Lqeq3fSp83jlAhXlhtJqf+fsm62cO5PcLJGH5jM19x32293lcoB9p2rrA+4foVWlumxTMqvtyYBeQqbWuNnfVAJnm7UDvwUR8b34MfA3wmPdTgWatdY953/cces/P3N9iPn6inXcBUAf8r9mk92ulVDxT4PPWWlcBPwTKgWqMz3Afk/8z9wrXZ5xj3u6/fVASSKYApVQC8DzwkNa61XefNr52TKo+4EqpG4FarfW+sS7LKLNiNHn8Umu9HOjAaOboNRk/bwAzJ3AzRjDNBuKB68e0UGNkLD5jCSQXVAF5PvdzzW0TmlLKhhFENmmtXzA3n1dKZZn7s4Bac3ug92CivTdrgZuUUmeAzRjNWz8BpimlvIu5+Z5D7/mZ+5OBBibeeVcClVrrXeb9P2AElsn+eQNsAMq01nVaaxfwAsbfwWT/zL3C9RlXmbf7bx+UBJIL9gDzzF4e0RgJuD+PcZlGxOxt8QRwTGu90WfXnwFvL437MHIn3u0fM3t6rAFazOrya8B1Sqnp5je/68xt45LW+hta61ytdT7G5/im1voeYAtwm/mw/uftfT9uMx+vze13mj18CoB5GInIcUlrXQNUKKXmm5uuAY4yyT9vUzmwRikVZ/7de899Un/mPsLyGZv7WpVSa8z38WM+xwpsrJNG4+kHo4fDexg9Nb451uUJw/lcgVHFPQiUmD83YLQF/x04CbwBpJiPV8DPzfM/BKzyOdYngFPmz8fH+tyG8B4UcaHX1myMi8Ip4PeA3dweY94/Ze6f7fP8b5rvxwlC6L0y1j/AMmCv+Zn/EaNHzpT4vIHvAMeBw8D/YfS8mnSfOfAMRh7IhVEL/WQ4P2NglfkengYepV/nDX8/MkWKEEKIEZGmLSGEECMigUQIIcSISCARQggxIhJIhBBCjIgEEiGEECMigUSIAMyZdB80b2crpf4QwddappS6IVLHFyKSJJAIEdg0jFli0Vqf01rfFuTxI7EMY4yPEBOOjCMRIgCl1GaM+ZtOYAz0Wqi1XqyUuh/4EMZ8TvMwJguMBu4FuoEbtNaNSqk5GIPB0oFO/n979xNiUxjGcfz7E5nZzV5s7DQxk6Ks/VlYSbKwIzULdorVRYrFLChlYWVhRZSMhahZXWVQzJgiCytLC00pyvws3vfqmu6M3GO65PfZ3PPnnvPeTt2e3vOc8zxw3PYbSYeAs8A3SrHA3ZSXwoYp5SguUarXXgVGgXXAOdv36tgHKCU9NgA3bZ9f5UsRsaK1v/5KxH/rDDBqe6xWT57q2jdKqaY8RAkCp22PS7pMKStxBbgOTNh+J2kncI1S96sF7LP9QdKI7a+SWpS3jk8ASLpIKdtxVNIIMCPpcR17Rx3/M/BM0gPbz1fzQkSsJIEkoj/TtheABUmfgPt1+xywtVZc3gXc7mowt75+toEbkm5Rigv2spdSePJUXR8CNtXlR7Y/Aki6SymFk0ASA5NAEtGfL13Li13ri5T/1RpKL4yxpQfanqgzlP3AC0nbe5xfwEHbb3/aWI5bej8696djoJJsj1jeAqVF8W9z6fvyvuZDOr2zt9Xlzbaf2m5RGlFt7DHWQ+Bkp1+2pPGufXtqj+5hSq6m3c9vjPhTEkgillFvH7UlvQYm+zjFEeCYpFfAPCVxDzApaa6e9wmlR/g0sEXSS0mHgQuUJPuspPm63jFD6TEzC9xJfiQGLU9tRfxD6lNbP5LyEX+DzEgiIqKRzEgiIqKRzEgiIqKRBJKIiGgkgSQiIhpJIImIiEYSSCIiopHvgVzZxPtBcE8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "awlrQQCUpEJd",
        "outputId": "72032d67-13c4-41d3-8a79-7f8f8a3b0f11",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        }
      },
      "source": [
        "from google.colab import files\n",
        "files.download('./experiments/blackjack_dqn_result/performance.csv') \n",
        "\n",
        "# sess.close()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_7917f53a-9609-4bd3-94d3-2f4c17e96fac\", \"performance.csv\", 1233)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vdFyx_OpxE7H",
        "outputId": "28daa3eb-4887-4f48-f4db-f3dbce596bdd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uBYS0WXIU5zQ",
        "outputId": "53e32e34-da5e-4470-feab-518e6dbae909",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# Save model\n",
        "save_dir = '/content/drive/My Drive/RL_evacuation/model/blackjack_dqn_60'\n",
        "# if not os.path.exists(save_dir):\n",
        "#     os.makedirs(save_dir)\n",
        "saver = tf.train.Saver()\n",
        "saver.save(sess, os.path.join(save_dir, 'model')) "
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content/drive/My Drive/RL_evacuation/model/blackjack_dqn_60/model'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RTRbTsUkaDug"
      },
      "source": [
        "# self.obs_high = np.array([15, 15, 20, 20, 20, 20, 8, 4, 1000, 1000, 1000, 1000])\n",
        "# # self.observation_space = spaces.Box(self.obs_low, self.obs_high, dtype=np.int32)\n",
        "\n",
        "# self.tgen_list = []\n",
        "# pd.DataFrame(columns = ['num_to_hos', 'num_to_acf', \n",
        "# 'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2','avg_waiting_1', 'avg_serv_1','avg_waiting_2', 'avg_serv_2'])\n"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mjqw9cFIVpTq",
        "outputId": "4933c1c6-b66a-4bae-dbd7-c9f0d18b1c20",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "agent.predict([0, 1, \n",
        "               0, 0, 0, 0, \n",
        "               3, 6, 0, 10, 10, 10])"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.05, 0.95])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IAfEjnPL_e75"
      },
      "source": [
        "Test for the  pregenerated dataset (1000 runs)\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OM7C78yg_ebn",
        "cellView": "form"
      },
      "source": [
        "#@title Code for Test Environment\n",
        "#!/usr/bin/env python3\n",
        "# -*- coding: utf-8 -*-\n",
        "\"\"\"\n",
        "Created on Tue Dec 17 11:46:46 2020\n",
        "\n",
        "@author: machaolun\n",
        "\"\"\"\n",
        "\"\"\"\n",
        "Unit: Minute\n",
        "\"\"\"\n",
        "import gym\n",
        "from gym import spaces\n",
        "\n",
        "# import tkinter as tk\n",
        "import random\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "# from collections import deque\n",
        "from heapq import heappush,heappop\n",
        "import csv\n",
        "import copy\n",
        "from collections import deque\n",
        "\n",
        "random.seed(2020)\n",
        "# This class is to generate patients\n",
        "total_simulation_time = 10000\n",
        "\n",
        "\n",
        "def SURV_PROB_CAL(my_facility, my_patype, my_total_time, surv_num):\n",
        "        # survival probablity\n",
        "    # Resource-Based Patient Prioritization in Mass-Casualty Incidents\n",
        "    # first is the worst case and the fifth is the optimistic case    \n",
        "    # Immediate         Delayed\n",
        "    # β0,I β1,I β2,I    β0,D β1,D β2,D\n",
        "    # 0.09,17  ,1.01    0.57,61   ,2.03\n",
        "    # 0.15,28  ,1.38    0.65,86   ,2.11\n",
        "    # 0.24,47  ,1.30    0.76,138  ,2.17\n",
        "    # 0.40,59  ,1.47    0.77,140  ,2.29\n",
        "    # 0.56,91  ,1.58    0.81,160  ,1.41\n",
        "    if surv_num == 1:\n",
        "        # set para\n",
        "        # Senerio 1, assume no different between ACF and hospital\n",
        "        if my_facility == 1 and my_patype == 0:\n",
        "            \"\"\"Hospital, immediate\"\"\"\n",
        "            Beta = [0.09,17  ,1.01]\n",
        "        elif my_facility == 1 and my_patype == 1:\n",
        "            \"\"\"Hospital, delayed\"\"\"\n",
        "            Beta = [0.57,61 ,2.03]\n",
        "        elif my_facility == 2 and my_patype == 0:\n",
        "            \"\"\"ACF, immediate\"\"\"\n",
        "            Beta = [0.09,17  ,1.01]\n",
        "        elif my_facility == 2 and my_patype == 1:\n",
        "            \"\"\"ACF, delayed\"\"\"\n",
        "            Beta = [0.57,61   ,2.03]\n",
        "    elif surv_num == 2:       \n",
        "        # # Senerio 2, assume no different between ACF and hospital\n",
        "        if my_facility == 1 and my_patype == 0:\n",
        "            \"\"\"Hospital, immediate\"\"\"\n",
        "            Beta = [0.15,28  ,1.38]\n",
        "        elif my_facility == 1 and my_patype == 1:\n",
        "            \"\"\"Hospital, delayed\"\"\"\n",
        "            Beta = [0.65,86   ,2.11]\n",
        "        elif my_facility == 2 and my_patype == 0:\n",
        "            \"\"\"ACF, immediate\"\"\"\n",
        "            Beta = [0.15,28  ,1.38]\n",
        "        elif my_facility == 2 and my_patype == 1:\n",
        "            \"\"\"ACF, delayed\"\"\"\n",
        "            Beta = [0.65,86   ,2.11]\n",
        "            \n",
        "    elif surv_num == 3:\n",
        "        \n",
        "        # # Senerio 3, assume no different between ACF and hospital\n",
        "        if my_facility == 1 and my_patype == 0:\n",
        "            \"\"\"Hospital, immediate\"\"\"\n",
        "            Beta = [0.24,47  ,1.30]\n",
        "        elif my_facility == 1 and my_patype == 1:\n",
        "            \"\"\"Hospital, delayed\"\"\"\n",
        "            Beta = [0.76,138  ,2.17]\n",
        "        elif my_facility == 2 and my_patype == 0:\n",
        "            \"\"\"ACF, immediate\"\"\"\n",
        "            Beta = [0.24,47  ,1.30]\n",
        "        elif my_facility == 2 and my_patype == 1:\n",
        "            \"\"\"ACF, delayed\"\"\"\n",
        "            Beta = [0.76,138  ,2.17]\n",
        "    elif surv_num == 4:\n",
        "            \n",
        "        # # Senerio 4, assume no different between ACF and hospital\n",
        "        if my_facility == 1 and my_patype == 0:\n",
        "            \"\"\"Hospital, immediate\"\"\"\n",
        "            Beta = [0.40,59  ,1.47]\n",
        "        elif my_facility == 1 and my_patype == 1:\n",
        "            \"\"\"Hospital, delayed\"\"\"\n",
        "            Beta = [0.77,140  ,2.29]\n",
        "        elif my_facility == 2 and my_patype == 0:\n",
        "            \"\"\"ACF, immediate\"\"\"\n",
        "            Beta = [0.40,59  ,1.47]\n",
        "        elif my_facility == 2 and my_patype == 1:\n",
        "            \"\"\"ACF, delayed\"\"\"\n",
        "            Beta = [0.77,140  ,2.29]            \n",
        "    elif surv_num == 5:\n",
        "        # Senerio 5, assume no different between ACF and hospital            \n",
        "        if my_facility == 1 and my_patype == 0:\n",
        "            \"\"\"Hospital, immediate\"\"\"\n",
        "            Beta = [0.56,91  ,1.58]\n",
        "        elif my_facility == 1 and my_patype == 1:\n",
        "            \"\"\"Hospital, delayed\"\"\"\n",
        "            Beta = [0.81,160  ,1.41]\n",
        "        elif my_facility == 2 and my_patype == 0:\n",
        "            \"\"\"ACF, immediate\"\"\"\n",
        "            Beta = [0.56,91  ,1.58]\n",
        "        elif my_facility == 2 and my_patype == 1:\n",
        "            \"\"\"ACF, delayed\"\"\"\n",
        "            Beta = [0.81,160  ,1.41]        \n",
        "    \n",
        "    # 0.9 is ACF discount factor\n",
        "    if my_facility == 1:\n",
        "        SP = Beta[0]/((my_total_time/Beta[1]) ** Beta[2] + 1.0)\n",
        "    else:\n",
        "        SP = 0.8 * Beta[0]/((my_total_time/Beta[1]) ** Beta[2] + 1.0)\n",
        "    return SP\n",
        "\n",
        "\n",
        "def eva_perform(env, num):\n",
        "    ''' Evaluate he performance of the agents in the environment\n",
        "    Args:\n",
        "        env (Env class): The environment to be evaluated.\n",
        "        num (int): The number of games to play.\n",
        "    Returns:\n",
        "        A list of avrage payoffs for each player\n",
        "    '''\n",
        "    payoffs = 0\n",
        "    counter = 0\n",
        "\n",
        "    while counter < num:\n",
        "        tj = env.run(1, is_training=False)\n",
        "        return_ = pd.DataFrame(tj)\n",
        "        payoffs += sum(return_[2])\n",
        "        counter += 1\n",
        "\n",
        "    payoffs /= counter\n",
        "    \n",
        "    return payoffs\n",
        "\n",
        "class Event(object):\n",
        "    # num_of_events = 0\n",
        "    def __init__(self,generate_time, ptype, surv_num):\n",
        "        self.generate_time = generate_time\n",
        "        self.trans_time = 0.2\n",
        "        self.arrival_time = 0.3\n",
        "        self.start_service_time = 0.4\n",
        "        self.service_time = 0.5\n",
        "        self.departure_time = 0.6\n",
        "        self.survival_prob_res = 0\n",
        "        self.facility = 0.7\n",
        "        self.patype = ptype\n",
        "        self.status = \"default\"\n",
        "        self.surv_num = surv_num\n",
        "        # severe , minor\n",
        "        # Event.num_of_events += 1\n",
        "    def total_time(self):\n",
        "        return self.departure_time - self.generate_time\n",
        "    \n",
        "    def sojourn_time(self):\n",
        "        return self.departure_time - self.arrival_time\n",
        "\n",
        "    def waiting_time(self):\n",
        "        # waiting time in queue\n",
        "        return self.sojourn_time() - self.service_time\n",
        "    \n",
        "    def survival_prob(self):\n",
        "        my_total_time = self.total_time()\n",
        "        SP = SURV_PROB_CAL(self.facility, self.patype, my_total_time, self.surv_num)\n",
        "        self.survival_prob_res = SP\n",
        "        return SP\n",
        "\n",
        "    def __repr__(self):\n",
        "        return f\"{self.generate_time}, {self.trans_time},{self.arrival_time}, {self.start_service_time}, {self.service_time}, {self.departure_time}, {self.survival_prob_res}, {self.facility}, {self.patype} \\n\"\n",
        "    def get_info(self):\n",
        "        #print([self.generate_time, self.trans_time, self.arrival_time, self.start_service_time, self.service_time, self.departure_time, self.facility, self.patype, self.status])\n",
        "        return [self.generate_time, self.trans_time, self.arrival_time, self.start_service_time, self.service_time, self.departure_time, self.survival_prob_res, self.facility, self.patype, self.status]\n",
        "    \n",
        "\"\"\"########################################################################\"\"\"\n",
        "\n",
        "class Server(object):\n",
        "    def __init__(self, c, status, mu0, mu1, tt):\n",
        "        # self.jobs = jobs\n",
        "        self.c = c\n",
        "        self.status = status\n",
        "        self.mu0 = mu0\n",
        "        self.mu1 = mu1\n",
        "        self.tt = tt\n",
        "        self.server_tnext = 0\n",
        "\n",
        "        self.num_busy = 0  # number of busy servers\n",
        "        self.stack = []    # store the patient info in the server\n",
        "        self.queue = []\n",
        "        self.served_jobs = [] \n",
        "\n",
        "    def handle_arrival(self, time, job):\n",
        "        # time here refers to arrival time\n",
        "        # print(\"arrival_job\", job)\n",
        "        # input(\"arrival\")\n",
        "    \n",
        "        if self.num_busy < self.c:\n",
        "            if len(self.queue) == 0:\n",
        "                # print(\"handle_arrival1 \")\n",
        "                self.start_service(time, job)\n",
        "        else:\n",
        "            job.status = \"queue\"\n",
        "            heappush(self.queue, ((job.patype, job.arrival_time), job))\n",
        "            \n",
        "        # print(\"handle_arrival_done \")\n",
        "\n",
        "    def start_service(self, time, job):\n",
        "        # time here refers to start service time\n",
        "        self.num_busy += 1  # server becomes busy.\n",
        "        job.start_service_time = time\n",
        "        job.departure_time = time + job.service_time\n",
        "        \n",
        "        job.status = \"stack\"\n",
        "        heappush(self.stack, (job.departure_time, job))\n",
        "        \n",
        "        # print(\"service_job \", job)\n",
        "        # input(\"service\")\n",
        "        \n",
        "        if job.departure_time <= self.server_tnext:\n",
        "            # print(\"start_service1 \")\n",
        "            self.handle_departure()\n",
        "            \n",
        "        \n",
        "    def handle_departure(self):\n",
        "        # time here refers to departure time \n",
        "        # self.stack.sort(reverse=True)\n",
        "        try:\n",
        "            while self.stack[0][0] <= self.server_tnext:\n",
        "                # print(\"while \", self.stack[0][0], self.server_tnext)\n",
        "                prev_depart = copy.deepcopy(self.stack[0][0])\n",
        "                self.num_busy -= 1\n",
        "                self.stack[0][1].status = \"released\"\n",
        "                next_done = heappop(self.stack)\n",
        "                next_done[1].survival_prob()\n",
        "                self.served_jobs.append(next_done)\n",
        "                \n",
        "                # print(\"next_done \",next_done)\n",
        "                # input(\"depature\")\n",
        "                \n",
        "                if self.queue and self.num_busy < self.c:\n",
        "                    if self.queue[0][1].arrival_time < self.server_tnext:\n",
        "                        # print(\"queue\")\n",
        "                        # print(\"self.server_tnext \", self.server_tnext)\n",
        "                        # print(\"prev_depart \", prev_depart)\n",
        "                        next_job = heappop(self.queue)\n",
        "                        # print(\"next_job! \", next_job)\n",
        "                        \n",
        "                        # input(\"here1\")\n",
        "                        # print(\"arr!\", next_job[1].arrival_time)\n",
        "                        # print(\"prev!\",prev_depart)\n",
        "                        # input(\"here2\")\n",
        "                        self.start_service(max(next_job[1].arrival_time, prev_depart),next_job[1])\n",
        "        except:\n",
        "            pass\n",
        "        \n",
        "    def travel_time(self):\n",
        "        return self.tt\n",
        "        # return random.expovariate(self.tt)\n",
        "        \n",
        "        # return self.tt\n",
        "        \n",
        "    def service_time(self, patient_type):\n",
        "        if patient_type == 0:\n",
        "            return random.expovariate(self.mu0)\n",
        "        else:\n",
        "            return random.expovariate(self.mu1)\n",
        "            \n",
        "    def getSize(self, tnow):\n",
        "        try:\n",
        "            count_queue0 = 0\n",
        "            count_queue1 = 0\n",
        "            copy_queue = copy.deepcopy(self.queue)\n",
        "            for item in copy_queue:\n",
        "                if item[1].arrival_time <= tnow: \n",
        "                    if item[1].patype == 0:\n",
        "                        count_queue0 += 1\n",
        "                    else:\n",
        "                        count_queue1 += 1      \n",
        "                    \n",
        "            return [count_queue0, count_queue1]\n",
        "        \n",
        "        except:\n",
        "            return [0,0]\n",
        "    \n",
        "    \n",
        "    \"\"\"need to optimize code \"\"\"\n",
        "    def getBusyServer(self, tnow):\n",
        "        count_busyserver = 0\n",
        "        copy_stack = copy.deepcopy(self.stack) + copy.deepcopy(self.served_jobs) \n",
        "        # print(\"copy_stack \", copy_stack)\n",
        "        # print(\"stack \", self.stack\n",
        "        \n",
        "        try:\n",
        "            for item in copy_stack:\n",
        "                # print(\"item \",item[1])\n",
        "                if item[1].departure_time >= tnow and item[1].start_service_time <= tnow and count_busyserver < self.c:\n",
        "                    count_busyserver += 1\n",
        "                # print(\"count_busyserver\", count_busyserver)\n",
        "                \n",
        "            return [count_busyserver]\n",
        "        \n",
        "        except:\n",
        "            return [0]\n",
        "    \n",
        "    \n",
        "\"\"\"########################################################################\"\"\"\n",
        "\n",
        "def AssignType():\n",
        "    # set para\n",
        "    p = 0\n",
        "    myassign = np.random.binomial(1,p)\n",
        "    if myassign == 1:\n",
        "        patype = 0\n",
        "    else:\n",
        "        patype = 1\n",
        "    \n",
        "    return patype\n",
        "\n",
        "\"\"\"########################################################################\"\"\"\n",
        "\n",
        "class TestEnv(gym.Env):\n",
        "    def __init__(self, tnow):\n",
        "        self.tnow = tnow\n",
        "        self.gen_patient_time_list = []\n",
        "        self.tgen = 0\n",
        "        self.tgen_next = 0\n",
        "        self.arrival_list1 = []\n",
        "        self.arrival_list2 = []\n",
        "        self.arrival_time = self.tnow\n",
        "        self.surv_num = 5\n",
        "        # Define constants for clearer code\n",
        "        self.S1 = 0\n",
        "        self.S2 = 1\n",
        "        self.action_num = 2\n",
        "        self.action_space = spaces.Discrete(self.action_num)\n",
        "\n",
        "        \n",
        "        \"\"\"['gen_time', 'num_to_hos', 'num_to_acf', \n",
        "        'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2','served_1', 'served_2', 'rewards'])\"\"\"    \n",
        "        \"\"\" obs space\"\"\"\n",
        "        \"\"\" ['num_to_hos', 'num_to_acf', 'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2', \n",
        "        'waiting time at hos', ' service at hos', 'waiting time at acf', ' service at acf'])\"\"\"\n",
        "        \n",
        "        self.obs_low = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
        "        self.obs_high = np.array([15, 15, 20, 20, 20, 20, 12, 6, 1000, 1000, 1000, 1000])\n",
        "        # self.observation_space = spaces.Box(self.obs_low, self.obs_high, dtype=np.int32)\n",
        "        \n",
        "        self.tgen_list = []\n",
        "        # pd.DataFrame(columns = ['gen_time', 'num_to_hos', 'num_to_acf', \n",
        "        # 'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2','served_1', 'served_2', 'rewards'])\n",
        "        self.rewards_records = {}\n",
        "        self.obs_records = {}\n",
        "        self.action_records = {}\n",
        "        self.wait_records1 = {}  # queuing_time\n",
        "        self.service_records1 = {} #service_time\n",
        "        self.wait_records2 = {}  # queuing_time\n",
        "        self.service_records2 = {}\n",
        "\n",
        "        \"\"\"Server Settings\"\"\"\n",
        "        \"\"\" mu_tt = log(x) \"\"\"\n",
        "        \"\"\"pdf = (np.exp(-(np.log(x) - mu)**2 / (2 * sigma**2)) / (x * sigma * np.sqrt(2 * np.pi)))\"\"\"\n",
        "\n",
        "        # set para server\n",
        "\n",
        "        self.server1_rate0 = 1/90  \n",
        "        self.server1_rate1 = 1/30\n",
        "        \"\"\" mean value, not mu\"\"\"\n",
        "        self.server1_tt = 60\n",
        "        \n",
        "        self.server2_rate0 = 1/90\n",
        "        self.server2_rate1 = 1/30\n",
        "        \n",
        "        \"\"\" mean value, not mu\"\"\"\n",
        "        self.server2_tt = 10\n",
        "        \n",
        "        # __init__(self, c, status, mu0, mu1, tt):\n",
        "        self.server1 = Server(12, 'open',self.server1_rate0,self.server1_rate1,self.server1_tt)\n",
        "        self.server2 = Server(6, 'open', self.server2_rate0, self.server2_rate1,self.server2_tt)\n",
        "        self.s2_isopen = True   \n",
        "        self.simulation_time = total_simulation_time\n",
        "        \n",
        "        \n",
        "        \"\"\" use pre-generated data\"\"\"\n",
        "    def Get_Time(self, myi):\n",
        "        nlength = 50\n",
        "        a = pd.read_excel(\"/content/drive/My Drive/RL_evacuation/data/patient_generattion.xlsx\",\n",
        "                  index_col=0)\n",
        "        b = list(a.iloc[myi,:nlength])\n",
        "        self.gen_patient_time_list = deque(b)\n",
        "        \n",
        "        \n",
        "        \"\"\" use generated data\"\"\"\n",
        "    # def Get_Time(self, myi):\n",
        "    #     total_num_patient = 50\n",
        "    #     arrive_lambda = 1/5.0\n",
        "    #     mylist = []\n",
        "    #     t= 0\n",
        "    #     \"\"\"Generate Casulties\"\"\"\n",
        "    #     for j in range(total_num_patient):\n",
        "    #         t+= random.expovariate(arrive_lambda)\n",
        "    #         mylist.append(t)   \n",
        "            \n",
        "    #     self.gen_patient_time_list = deque(mylist)   \n",
        "    # def GenerateTime(self):\n",
        "    #     \"\"\"\n",
        "    #     Arrival rate per min\n",
        "    #     Set Arrival Rate (time inhomogeneous) Here\n",
        "    #     \"\"\"\n",
        "    #     # ref: https://zhuanlan.zhihu.com/p/28785318\n",
        "    #     t = 0\n",
        "    #     I = 0\n",
        "    #     s = []\n",
        "        \n",
        "    #     max_simulate_time = 1000\n",
        "        \n",
        "    #     while t <= max_simulate_time:\n",
        "    #         u = np.random.uniform(1e-8,1)\n",
        "    #         t = t - np.log(u)\n",
        "    #         if t <= max_simulate_time:\n",
        "    #             u1 = np.random.uniform(1e-8,1)\n",
        "    #             if u1 <= inhop(t):\n",
        "    #                 I = I + 1\n",
        "    #                 s.append(t)\n",
        "                    \n",
        "    #     # time when event happens\n",
        "    #     self.gen_patient_time_list = deque(s)        \n",
        "    \n",
        "    \"\"\" Open new ACF, the resource comes from hospital\"\"\"        \n",
        "    # def take_action(self, Action):\n",
        "    #     if (not self.s2_isopen) and Action == \"open\":\n",
        "    #         original_num_server = copy.deepcopy(self.server1.c)\n",
        "    #         self.s2_isopen = True\n",
        "            \n",
        "    #         # set para\n",
        "    #         precent_move = 0.2\n",
        "    #         self.server1.c = round((1 - precent_move)*original_num_server)\n",
        "    #         self.server2.c = int(precent_move * original_num_server) + 2 \n",
        "    #         self.server2.status = 'open'\n",
        "            \n",
        "    #         print(\"!!!!!!!!!!!!!!!\\n\",self.server1.c, self.server2.c)\n",
        "    #     else:\n",
        "    #         pass \n",
        "        \n",
        "    def step(self, Action):\n",
        "        \n",
        "        try:\n",
        "            self.tgen = self.gen_patient_time_list.popleft()\n",
        "            try:\n",
        "                self.tgen_next = self.gen_patient_time_list[0]\n",
        "            except:\n",
        "                self.tgen_next = 100000\n",
        "            # action records\n",
        "            self.action_records[self.tgen] = Action\n",
        "            \n",
        "            patient = Event(self.tgen, AssignType(), self.surv_num)\n",
        "\n",
        "            #  self.s2_isopen and         \n",
        "            if Action == self.S2:    \n",
        "                patient.trans_time = self.server2.travel_time()\n",
        "                # self.server2.tnext = min(self.tgen + patient.travel_time , self.simulation_time)\n",
        "                patient.facility = 2\n",
        "                patient.arrival_time = patient.generate_time + patient.trans_time\n",
        "                patient.service_time = self.server2.service_time(patient.patype)\n",
        "                heappush(self.arrival_list2, ((patient.arrival_time), patient))\n",
        "                \n",
        "            elif Action == self.S1:\n",
        "                patient.trans_time = self.server1.travel_time()\n",
        "                # self.server1.tnext = min(self.tgen + patient.travel_time, self.simulation_time)\n",
        "                patient.facility = 1\n",
        "                patient.arrival_time = patient.generate_time + patient.trans_time\n",
        "                patient.service_time = self.server1.service_time(patient.patype) \n",
        "                heappush(self.arrival_list1, ((patient.arrival_time), patient))\n",
        "                                \n",
        "            else:\n",
        "                raise ValueError(\"Received invalid action={} which is not part of the action space\".format(Action))\n",
        "        \n",
        "        except:\n",
        "            self.tgen_next = 100001\n",
        "            \n",
        "        try:\n",
        "            while self.arrival_list1[0][0] <= self.tgen_next:\n",
        "                try:\n",
        "                    self.server1.server_tnext = min(self.arrival_list1[0][0], self.tgen_next)\n",
        "                except:\n",
        "                    self.server1.server_tnext = self.tgen_next\n",
        "                    \n",
        "                self.server1.handle_departure()\n",
        "                mypatient_s1 = heappop(self.arrival_list1)\n",
        "                self.server1.handle_arrival(mypatient_s1[1].arrival_time, mypatient_s1[1])\n",
        "                \n",
        "        except:\n",
        "            pass\n",
        "        \n",
        "        self.server1.server_tnext = self.tgen_next    \n",
        "        self.server1.handle_departure()\n",
        "        \n",
        "        if self.s2_isopen:\n",
        "            try:\n",
        "                while self.arrival_list2[0][0] <= self.tgen_next:\n",
        "    \n",
        "                    try:\n",
        "                        self.server2.server_tnext = min(self.arrival_list2[0][0], self.tgen_next)\n",
        "                    except:\n",
        "                        self.server2.server_tnext = self.tgen_next\n",
        "                        \n",
        "                    self.server2.handle_departure()\n",
        "                    \n",
        "                    mypatient_s2 = heappop(self.arrival_list2)\n",
        "                    self.server2.handle_arrival(mypatient_s2[1].arrival_time, mypatient_s2[1])\n",
        "                    \n",
        "            except:\n",
        "                pass\n",
        "            \n",
        "            self.server2.server_tnext = self.tgen_next\n",
        "            self.server2.handle_departure()\n",
        "                \n",
        "        \"\"\" gen_time, num to hos, num to acf, \n",
        "        [ql10,ql11], [ql20, ql21],\n",
        "        busy server1, busy 2 \"\"\"        \n",
        "        \"\"\"['gen_time', 'num_to_hos', 'num_to_acf', \n",
        "        'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2','served_1', 'served_2', 'rewards'])\"\"\"    \n",
        "        \n",
        "        # print(\"self.tgen_next\", self.tgen_next)\n",
        "        if self.s2_isopen:\n",
        "            Legal_Action = [0,1]\n",
        "        else:\n",
        "            Legal_Action = [0,1]\n",
        "\n",
        "        # for the first one\n",
        "        if len(self.gen_patient_time_list) == 49:\n",
        "            self.tgen_list.append([self.tgen, Legal_Action])\n",
        "            self.obs_records[self.tgen] = np.array([self.tgen, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0 ]) \n",
        "\n",
        "            #action record is above \n",
        "            \n",
        "        all_served = self.server1.served_jobs + self.server2.served_jobs\n",
        "        \n",
        "       \n",
        "        for jobs in all_served:\n",
        "            self.rewards_records[jobs[1].generate_time] = jobs[1].survival_prob_res\n",
        "  \n",
        "        # add obs to records\n",
        "        if self.tgen_next < 100000:\n",
        "            self.tgen_list.append([self.tgen_next, Legal_Action])\n",
        "            i = 0\n",
        "            j = 0\n",
        "            w_record1 = 0\n",
        "            s_record1 = 0\n",
        "            w_record2 = 0\n",
        "            s_record2 = 0\n",
        "            \n",
        "            for jobs2 in all_served:\n",
        "\n",
        "                # time window -- future consideration\n",
        "                if jobs2[1].departure_time < self.tgen_next and jobs2[1].facility  == 1: # 1hos 2acf\n",
        "                    i += 1\n",
        "                    w_record1 += jobs2[1].waiting_time()            \n",
        "                    s_record1 += jobs2[1].service_time\n",
        "                    \n",
        "                if jobs2[1].departure_time < self.tgen_next and jobs2[1].facility  == 2: # 1hos 2acf\n",
        "                    j += 1\n",
        "                    w_record2 += jobs2[1].waiting_time()            \n",
        "                    s_record2 += jobs2[1].service_time                    \n",
        "\n",
        "            self.wait_records1[self.tgen_next] = 0 if i == 0 else int(round(w_record1/i))   # queuing_time\n",
        "            self.service_records1[self.tgen_next] = 0 if i == 0 else int(round(s_record1/i)) #service_time\n",
        "\n",
        "            self.wait_records2[self.tgen_next] = 0 if j == 0 else int(round(w_record2/j))   # queuing_time\n",
        "            self.service_records2[self.tgen_next] = 0 if j == 0 else int(round(s_record2/j))  #service_time\n",
        "                   \n",
        "            # observation records\n",
        "            self.obs_records[self.tgen_next] = np.array([self.tgen_next, len(self.arrival_list1), len(self.arrival_list2)] \n",
        "                        + self.server1.getSize(self.tgen_next)\n",
        "                        + self.server2.getSize(self.tgen_next)\n",
        "                        + self.server1.getBusyServer(self.tgen_next)\n",
        "                        + self.server2.getBusyServer(self.tgen_next)\n",
        "                        + [self.wait_records1[self.tgen_next],\n",
        "                           self.service_records1[self.tgen_next],\n",
        "                           self.wait_records2[self.tgen_next],\n",
        "                           self.service_records2[self.tgen_next]])\n",
        "                                                        \n",
        "                        # + [len(self.server1.served_jobs)]\n",
        "                        # + [len(self.server2.served_jobs)])\n",
        "            states = {'obs': np.delete(self.obs_records[self.tgen_next],0), \n",
        "                      'legal_actions': Legal_Action}\n",
        "        else:\n",
        "            states = {'obs': self.obs_low,\n",
        "                      'legal_actions': [0,1]}\n",
        "        \n",
        "\n",
        "            \n",
        "             \n",
        "        returns = []\n",
        "\n",
        "        for everyone in self.tgen_list[:]:\n",
        "            try:\n",
        "                rewards = self.rewards_records[everyone[0]]\n",
        "                \n",
        "                obs = self.obs_records[everyone[0]]\n",
        "                \n",
        "                actions = self.action_records[everyone[0]]\n",
        "                \n",
        "                returns.append({'obs': obs,'legal_actions': everyone[1],'action': actions ,'reward': rewards})\n",
        "                self.tgen_list.remove(everyone)\n",
        "            except:\n",
        "                continue\n",
        "\n",
        "        done = bool(self.tgen_next == 100000)\n",
        "        # obs = np.array([1,1,1,1,1,1,1,1])\n",
        "        # reward = 1\n",
        "        info = {}\n",
        "        \n",
        "        return states, returns, done, info\n",
        "        \n",
        "    def set_agents(self, agents):\n",
        "        ''' Set the agents that will interact with the environment\n",
        "        Args:\n",
        "            agents (list): List of Agent classes\n",
        "        '''\n",
        "        self.agents = agents\n",
        "\n",
        "\n",
        "    def run(self, myj, is_training=False):\n",
        "        '''\n",
        "        Run a complete game, either for evaluation or training RL agent.\n",
        "        Args:\n",
        "            is_training (boolean): True if for training purpose.\n",
        "        Returns:\n",
        "            (tuple) Tuple containing:\n",
        "                (list): A list of trajectories generated from the environment.\n",
        "                (list): A list payoffs. Each entry corresponds to one player.\n",
        "        Note: The trajectories are 3-dimension list. The first dimension is for different players.\n",
        "              The second dimension is for different transitions. The third dimension is for the contents of each transiton\n",
        "        '''\n",
        "        return_summary = []\n",
        "        done = False\n",
        "        \n",
        "        states, returns, done, info = self.reset()\n",
        "        # read arrival data \n",
        "        self.Get_Time(myj)\n",
        "        \n",
        "        while not done:\n",
        "            # action, _states = model.predict(states)\n",
        "            if not is_training:\n",
        "                action, probs_ = self.agents.eval_step(states)\n",
        "                # print(\"best action \", action)\n",
        "                # print(\"probs \", probs_)\n",
        "            else:\n",
        "                action = self.agents.step(states)\n",
        "            \n",
        "            # if random.random() > 0.9:\n",
        "            #     action = 1   \n",
        "            # else:\n",
        "            #     action = 0\n",
        "            # action = 0\n",
        "            \n",
        "            states, returns, done, info = self.step(action)\n",
        "            \n",
        "            return_summary = np.append(return_summary, returns)\n",
        "        \n",
        "        # served_result = self.get_stat()\n",
        "        \n",
        "        # print(served_result)\n",
        "        \n",
        "        list2 = [x for x in return_summary if x != []]\n",
        "        sorted_ts = sorted(list2, key = lambda i: i['obs'][0]) \n",
        "        \n",
        "        trajectories = []\n",
        "\n",
        "        for i in range(len(sorted_ts) - 1):\n",
        "            #np.delete(sorted_ts[i]['obs'],0)\n",
        "            trajectories.append([{'obs': np.delete(sorted_ts[i]['obs'],0), 'legal_actions': sorted_ts[i]['legal_actions']}, sorted_ts[i]['action'],sorted_ts[i]['reward'],\n",
        "          {'obs': np.delete(sorted_ts[i+1]['obs'],0), 'legal_actions': sorted_ts[i+1]['legal_actions']}, bool(i == len(sorted_ts) - 2)])\n",
        "        \n",
        "        return trajectories\n",
        "    \n",
        "    def get_Reward(self):\n",
        "    # Playing Atari with Deep Reinforcement Learning    \n",
        "    # receives a reward rt representing the change in game score.\n",
        "    # survival probablity\n",
        "    # Resource-Based Patient Prioritization in Mass-Casualty Incidents\n",
        "        all_served = self.server1.served_jobs + self.server2.served_jobs\n",
        "        \n",
        "        for jobs in all_served:\n",
        "            # if jobs[1].generate_time not in list(self.rewards_records.keys()):\n",
        "            self.rewards_records[jobs[1].generate_time] = jobs[1].survival_prob_res\n",
        "\n",
        "\n",
        "    def reset(self):\n",
        "        # return observation\n",
        "        self.tnow = 0\n",
        "        self.gen_patient_time_list = []\n",
        "        self.tgen = 0\n",
        "        self.tnext = 0\n",
        "        self.arrival_list1 = []\n",
        "        self.arrival_list2 = []\n",
        "        self.arrival_time = self.tnow\n",
        "        self.surv_num = 5\n",
        "\n",
        "        \n",
        "        \"\"\"['gen_time', 'num_to_hos', 'num_to_acf', \n",
        "        'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2','served_1', 'served_2', 'rewards'])\"\"\"    \n",
        "        \"\"\" obs space\"\"\"\n",
        "        \"\"\" ['num_to_hos', 'num_to_acf', 'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2', 'waiting time ', ' service'])\"\"\"\n",
        "        \n",
        "        # self.observation_space = spaces.Box(self.obs_low, self.obs_high, dtype=np.int32)\n",
        "        \n",
        "        self.tgen_list = []\n",
        "        # pd.DataFrame(columns = ['gen_time', 'num_to_hos', 'num_to_acf', \n",
        "        # 'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2','served_1', 'served_2', 'rewards'])\n",
        "        self.rewards_records = {}\n",
        "        self.obs_records = {}\n",
        "        self.action_records = {}\n",
        "        self.wait_records1 = {}  # queuing_time\n",
        "        self.service_records1 = {} #service_time\n",
        "        self.wait_records2 = {}  # queuing_time\n",
        "        self.service_records2 = {}\n",
        "        \n",
        "        \n",
        "        self.server1 = Server(12, 'open',self.server1_rate0,self.server1_rate1,self.server1_tt)\n",
        "        self.server2 = Server(6, 'not_open', self.server2_rate0, self.server2_rate1,self.server2_tt)\n",
        "        self.s2_isopen = True\n",
        "        self.simulation_time = 100000\n",
        "        \n",
        "        done = False\n",
        "        info = {}\n",
        "        returns = []\n",
        "        states = {'obs': self.obs_low,'legal_actions': [0,1]}\n",
        "        \n",
        "        return states, returns, done, info\n",
        "\n",
        "    \n",
        "    def get_stat(self):\n",
        "        # Avg_SojournTime, Avg_WaitingTime, Avg_QueueLength\n",
        "        SojournTime1 = 0.0\n",
        "        WaitingTime1 = 0.0\n",
        "        SojournTime2 = 0.0\n",
        "        WaitingTime2 = 0.0\n",
        "        \n",
        "        mydata = []\n",
        "        \n",
        "        for jobs in self.server1.served_jobs:\n",
        "            SojournTime1 += jobs[1].sojourn_time()\n",
        "            WaitingTime1 += jobs[1].waiting_time()\n",
        "            \"\"\" f\"{self.arrival_time}, {self.start_service_time}, {self.service_time}, \n",
        "            {self.departure_time}, {self.facility}, {self.patype} \\n\" \"\"\"\n",
        "\n",
        "            # heappush(mydata, (jobs[1].arrival_time, jobs[1].get_info()))\n",
        "            mydata.append(jobs[1].get_info())\n",
        "            \n",
        "            \"\"\"also attach patient in queue and not release\"\"\"\n",
        "            \n",
        "        for not_complete_job in self.server1.stack:\n",
        "            mydata.append(not_complete_job[1].get_info())\n",
        "            \n",
        "        for not_complete_job in self.server1.queue:\n",
        "            mydata.append(not_complete_job[1].get_info())\n",
        "\n",
        "        with open('Server1.csv', 'w', newline='') as file:\n",
        "            writer = csv.writer(file, delimiter=',')\n",
        "            writer.writerows(mydata)        \n",
        "        \n",
        "        mydata = []\n",
        "        \n",
        "        for jobs in self.server2.served_jobs:\n",
        "            SojournTime2 += jobs[1].sojourn_time()\n",
        "            WaitingTime2 += jobs[1].waiting_time()\n",
        "            mydata.append(jobs[1].get_info())\n",
        "            # mydata.append([jobs[1].arrival_time,jobs[1].start_service_time,jobs[1].service_time, jobs[1].departure_time])\n",
        "            # print(\"{0:.8f}\".format(jobs[1].arrival_time), \"{0:.8f}\".format(jobs[1].start_service_time),\\\n",
        "            #       \"{0:.8f}\".format(jobs[1].service_time), \"{0:.8f}\".format(jobs[1].departure_time))\n",
        "            # print(jobs[1].get_info())\n",
        "            \n",
        "        for not_complete_job in self.server2.stack:\n",
        "            mydata.append(not_complete_job[1].get_info())\n",
        "            \n",
        "        for not_complete_job in self.server2.queue:\n",
        "            mydata.append(not_complete_job[1].get_info())            \n",
        "        \n",
        "        with open('Server2.csv', 'w', newline='') as file:\n",
        "            writer = csv.writer(file, delimiter=',')\n",
        "            writer.writerows(mydata) \n",
        "            \n",
        "            \n",
        "        if len(self.server1.served_jobs) != 0:\n",
        "            Avg_SojournTime1 = SojournTime1/len(self.server1.served_jobs)\n",
        "            Avg_WaitingTime1 = WaitingTime1/len(self.server1.served_jobs)\n",
        "        else:    \n",
        "            Avg_SojournTime1 = None\n",
        "            Avg_WaitingTime1 = None\n",
        "\n",
        "        if len(self.server2.served_jobs) != 0:\n",
        "            Avg_SojournTime2 = SojournTime2/len(self.server2.served_jobs)             \n",
        "            Avg_WaitingTime2 = WaitingTime2/len(self.server2.served_jobs)\n",
        "        else:\n",
        "            Avg_SojournTime2 = None\n",
        "            Avg_WaitingTime2 = None\n",
        "            \n",
        "        \n",
        "        return  Avg_SojournTime1, Avg_WaitingTime1, \\\n",
        "            Avg_SojournTime2, Avg_WaitingTime2\n",
        "                # Avg_QueueLength1, Avg_QueueLength2\n",
        "        \n",
        "    def debug(self):\n",
        "        print(\"==============\")\n",
        "        # self.server1.queue,\n",
        "        # print(\"server1  \", self.server1.c, self.server1.action, \\\n",
        "              # self.server1.getSize(),  self.server1.stack)\n",
        "        # print(\"served job \", self.server1.served_jobs)\n",
        "        # print(\"server2  \", self.server2.c, self.server2.action, \\\n",
        "        # self.server2.getSize(), self.server2.stack)\n",
        "\n",
        "\n",
        " \n"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mo1xLebQCQmb"
      },
      "source": [
        "test_env = TestEnv(0)\n",
        "test_env.set_agents(agent)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JhEGUOlu5wgy",
        "outputId": "70204700-f80a-4f84-d2b0-58e4a3ba5dc2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# sess = tf.Session()\n",
        "\n",
        "sess = tf.Session()\n",
        "load_dir = '/content/drive/My Drive/RL_evacuation/model/blackjack_dqn_60'\n",
        "\n",
        "total_rewards = 0\n",
        "\n",
        "with sess.as_default():\n",
        "    saver.restore(sess, tf.train.latest_checkpoint(load_dir))\n",
        "    # Evaluate the performance. Play with random agents.\n",
        "    for episode in range(999):\n",
        "    # print('\\n episode1 ', episode)\n",
        "    # Generate data from the environment\n",
        "      trajectories = test_env.run(myj = episode, is_training=False)\n",
        "      # print(\"trajectories \", trajectories)\n",
        "      # print(\"episode \", episode)\n",
        "      \n",
        "      eps_reward = 0\n",
        "      # Evaluate the performance. Play with random agents.\n",
        "      for i in range(len(trajectories)):\n",
        "        if trajectories[i][1] == 1:\n",
        "          print(\"TO ACF!\")\n",
        "        eps_reward  += trajectories[i][2] \n",
        "      print(\"eps_reward \", eps_reward)\n",
        "\n",
        "      total_rewards += eps_reward\n",
        "    print(\"total_rewards \", total_rewards/50)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Restoring parameters from /content/drive/My Drive/RL_evacuation/model/blackjack_dqn_60/model\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:139: RuntimeWarning: overflow encountered in exp\n",
            "/usr/local/lib/python3.6/dist-packages/rlcard/utils/utils.py:356: RuntimeWarning: invalid value encountered in true_divide\n",
            "  probs /= sum(probs)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "eps_reward  27.323058532670018\n",
            "eps_reward  27.263049602015755\n",
            "eps_reward  27.97078725272867\n",
            "eps_reward  28.279547896192213\n",
            "eps_reward  28.127147552409454\n",
            "eps_reward  28.416365056128775\n",
            "eps_reward  28.201659677746143\n",
            "eps_reward  28.626139696982147\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  27.483358702398338\n",
            "eps_reward  27.11784883963126\n",
            "eps_reward  27.016080543586046\n",
            "eps_reward  27.368307375331266\n",
            "eps_reward  28.364210952541235\n",
            "eps_reward  28.041273707682937\n",
            "eps_reward  27.904600185383718\n",
            "eps_reward  26.999877493439524\n",
            "eps_reward  28.21161565724326\n",
            "eps_reward  27.910780760723803\n",
            "eps_reward  27.748868541809514\n",
            "eps_reward  27.750379049935933\n",
            "eps_reward  28.254643507377438\n",
            "eps_reward  27.687956314927156\n",
            "eps_reward  27.97799882640463\n",
            "eps_reward  27.46090493270881\n",
            "eps_reward  27.696591388901638\n",
            "eps_reward  27.420977905137168\n",
            "eps_reward  27.90439391098478\n",
            "eps_reward  27.66517293887945\n",
            "eps_reward  27.372999660265553\n",
            "eps_reward  27.770118025485356\n",
            "eps_reward  28.606579475193616\n",
            "eps_reward  28.133794793936765\n",
            "eps_reward  27.981261572867883\n",
            "eps_reward  28.15699251700041\n",
            "eps_reward  28.013682202182775\n",
            "eps_reward  27.367528496044248\n",
            "eps_reward  27.374767118473507\n",
            "eps_reward  27.733297363957462\n",
            "eps_reward  27.995634372329352\n",
            "eps_reward  27.1980845776436\n",
            "eps_reward  27.011812329593027\n",
            "eps_reward  27.832868333802747\n",
            "eps_reward  27.95930340733761\n",
            "eps_reward  27.733098849337942\n",
            "eps_reward  27.44135731578122\n",
            "eps_reward  27.561123962602007\n",
            "eps_reward  28.028557751966062\n",
            "eps_reward  28.03767440121545\n",
            "eps_reward  27.70739162304072\n",
            "eps_reward  28.257389440834302\n",
            "eps_reward  27.76705856660947\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  27.234435113095312\n",
            "eps_reward  27.752594278375327\n",
            "eps_reward  28.30970539359629\n",
            "eps_reward  27.85839486743136\n",
            "eps_reward  27.92214470638852\n",
            "eps_reward  28.20545125311604\n",
            "eps_reward  27.427400712174258\n",
            "eps_reward  27.458836116000068\n",
            "eps_reward  28.175308520656014\n",
            "eps_reward  28.176101766779887\n",
            "eps_reward  27.782893819288844\n",
            "eps_reward  27.240933700333752\n",
            "eps_reward  27.610117505099705\n",
            "eps_reward  28.084716862559365\n",
            "eps_reward  27.252162946766056\n",
            "eps_reward  27.902411775222284\n",
            "eps_reward  27.64914844583379\n",
            "eps_reward  28.466633563009957\n",
            "eps_reward  27.50747714938298\n",
            "eps_reward  28.04577851529475\n",
            "eps_reward  27.330077567910678\n",
            "eps_reward  28.70066988435223\n",
            "eps_reward  28.160224335511813\n",
            "eps_reward  27.80818692329744\n",
            "eps_reward  27.92671893444131\n",
            "eps_reward  26.93635732313025\n",
            "eps_reward  27.21976021988786\n",
            "eps_reward  28.081461822017754\n",
            "eps_reward  27.337624084906032\n",
            "eps_reward  27.741871054781168\n",
            "eps_reward  28.382813528731038\n",
            "eps_reward  28.059467381126048\n",
            "eps_reward  27.161516727258498\n",
            "eps_reward  27.238390459812006\n",
            "eps_reward  27.487200228072854\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  25.75853982936132\n",
            "eps_reward  27.68586633750193\n",
            "eps_reward  28.674764453955273\n",
            "eps_reward  27.830518903604887\n",
            "eps_reward  28.69478030693298\n",
            "eps_reward  28.430025792713142\n",
            "eps_reward  27.81930216665347\n",
            "eps_reward  27.848271895439304\n",
            "eps_reward  27.668901914890135\n",
            "eps_reward  28.38405359315485\n",
            "eps_reward  27.75963161940884\n",
            "eps_reward  28.176488695207496\n",
            "eps_reward  28.35379642046302\n",
            "eps_reward  28.273895133655554\n",
            "eps_reward  28.32347348653235\n",
            "eps_reward  27.699932098737136\n",
            "eps_reward  27.989218614869763\n",
            "eps_reward  28.41537466613305\n",
            "eps_reward  28.13998701915691\n",
            "eps_reward  29.029300912753133\n",
            "eps_reward  27.668726779387868\n",
            "eps_reward  27.58437880184768\n",
            "eps_reward  28.771841834613323\n",
            "eps_reward  28.15517594271326\n",
            "eps_reward  28.190383158915676\n",
            "eps_reward  27.311905476827096\n",
            "eps_reward  27.509715010155084\n",
            "eps_reward  27.543403024443155\n",
            "eps_reward  28.144517033358294\n",
            "eps_reward  26.809157222411194\n",
            "eps_reward  27.79961753567141\n",
            "eps_reward  27.670208568390365\n",
            "eps_reward  27.970776721612086\n",
            "eps_reward  27.77217632704076\n",
            "eps_reward  27.699318992939226\n",
            "eps_reward  27.924186350725833\n",
            "eps_reward  26.58527618233867\n",
            "eps_reward  28.47384513338367\n",
            "eps_reward  26.508083972350505\n",
            "eps_reward  27.540400530911832\n",
            "eps_reward  28.047643273322837\n",
            "eps_reward  28.544798663369082\n",
            "eps_reward  27.99440336559468\n",
            "eps_reward  28.21612756210118\n",
            "eps_reward  27.721890206548924\n",
            "eps_reward  27.55432787945311\n",
            "eps_reward  28.10389472505272\n",
            "eps_reward  27.49126469693803\n",
            "eps_reward  27.42160411403073\n",
            "eps_reward  27.80046616293332\n",
            "eps_reward  28.088639636178016\n",
            "eps_reward  28.032044217889823\n",
            "eps_reward  27.132924612367756\n",
            "eps_reward  27.391429284814862\n",
            "eps_reward  27.660003911734606\n",
            "eps_reward  28.01118448719614\n",
            "eps_reward  28.03591050178343\n",
            "eps_reward  27.71955744967698\n",
            "eps_reward  27.885206374566305\n",
            "eps_reward  27.825697657368714\n",
            "eps_reward  27.657794620727536\n",
            "eps_reward  27.351205942904958\n",
            "eps_reward  27.538002119734852\n",
            "eps_reward  26.539765915197155\n",
            "eps_reward  27.062502695602046\n",
            "eps_reward  28.257765233320583\n",
            "eps_reward  27.575239141099644\n",
            "eps_reward  27.857200865102897\n",
            "eps_reward  28.054180964665004\n",
            "eps_reward  27.831047310007623\n",
            "eps_reward  28.103974044701744\n",
            "eps_reward  27.72941973272547\n",
            "eps_reward  27.771401988824127\n",
            "eps_reward  26.663126542693092\n",
            "eps_reward  27.61266124283585\n",
            "eps_reward  27.91464263396648\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  27.191556607635427\n",
            "eps_reward  27.78588453855717\n",
            "eps_reward  27.52051660347935\n",
            "eps_reward  27.971386775441758\n",
            "eps_reward  27.78770090545663\n",
            "eps_reward  27.52553312525578\n",
            "eps_reward  27.757994317152733\n",
            "eps_reward  28.114675177836588\n",
            "eps_reward  28.05368007180818\n",
            "eps_reward  27.486788328827483\n",
            "eps_reward  27.705155883841012\n",
            "eps_reward  27.453794260931655\n",
            "eps_reward  27.68510543084917\n",
            "eps_reward  27.386312559269744\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  27.282051946332903\n",
            "eps_reward  27.463889674379075\n",
            "eps_reward  27.670133767985114\n",
            "eps_reward  28.277811718087307\n",
            "eps_reward  28.193486298381014\n",
            "eps_reward  27.62493666606205\n",
            "eps_reward  27.49856414364984\n",
            "eps_reward  27.886623616778746\n",
            "eps_reward  26.702514056753262\n",
            "eps_reward  26.58842121266983\n",
            "eps_reward  27.549927383027114\n",
            "eps_reward  26.843550584046547\n",
            "eps_reward  26.982176651043055\n",
            "eps_reward  27.77883067941873\n",
            "eps_reward  29.06411311851642\n",
            "eps_reward  27.28442615914356\n",
            "eps_reward  28.04232550125076\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  27.96162470346741\n",
            "eps_reward  28.390144773188815\n",
            "eps_reward  27.87575181090714\n",
            "eps_reward  27.195808620220237\n",
            "eps_reward  27.83058285713725\n",
            "eps_reward  28.312951803756057\n",
            "eps_reward  27.671974496081088\n",
            "eps_reward  27.71544804512879\n",
            "eps_reward  27.96790213710003\n",
            "eps_reward  28.07875347388612\n",
            "eps_reward  28.904170711094192\n",
            "eps_reward  28.158790508552457\n",
            "eps_reward  27.60687492398138\n",
            "eps_reward  27.987088418686707\n",
            "eps_reward  27.56586003739291\n",
            "eps_reward  27.873078004287088\n",
            "eps_reward  27.610943628394\n",
            "TO ACF!\n",
            "eps_reward  27.735554413737052\n",
            "eps_reward  28.189911305636898\n",
            "eps_reward  28.038058680256867\n",
            "eps_reward  28.468403355214914\n",
            "eps_reward  27.980637587612165\n",
            "eps_reward  27.453634929284515\n",
            "eps_reward  27.703160677177785\n",
            "eps_reward  28.514243887319314\n",
            "eps_reward  27.661851009027522\n",
            "eps_reward  27.739657031997712\n",
            "eps_reward  28.296925342346878\n",
            "eps_reward  26.937495536298936\n",
            "eps_reward  27.10458463257292\n",
            "eps_reward  27.793057142244738\n",
            "eps_reward  27.303172794634396\n",
            "eps_reward  28.592033667761\n",
            "eps_reward  27.842235741194806\n",
            "eps_reward  27.64336715107659\n",
            "eps_reward  27.947411012325457\n",
            "eps_reward  27.899025386116104\n",
            "eps_reward  27.52249941802416\n",
            "eps_reward  27.18469575549765\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  27.22161006003956\n",
            "eps_reward  27.525911351446812\n",
            "eps_reward  27.09879049579922\n",
            "eps_reward  27.95930264030037\n",
            "eps_reward  27.801071196816782\n",
            "eps_reward  27.91274303181383\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  25.927560657410492\n",
            "eps_reward  27.988569229588542\n",
            "eps_reward  27.268870739262073\n",
            "eps_reward  27.753167368627327\n",
            "eps_reward  27.858932744756753\n",
            "eps_reward  27.274878048243348\n",
            "eps_reward  27.925097342909204\n",
            "eps_reward  27.904991273494982\n",
            "eps_reward  27.764227624046526\n",
            "eps_reward  27.814819974325694\n",
            "eps_reward  27.01001793663959\n",
            "eps_reward  28.211842362668214\n",
            "eps_reward  27.026517987162997\n",
            "eps_reward  27.373786347814466\n",
            "eps_reward  27.16798695991178\n",
            "eps_reward  28.113374679631505\n",
            "eps_reward  27.70957565693976\n",
            "eps_reward  27.302996984032777\n",
            "eps_reward  28.199744847622554\n",
            "eps_reward  28.209286759366023\n",
            "eps_reward  27.298659127133035\n",
            "eps_reward  28.44214087288048\n",
            "eps_reward  28.007424828603252\n",
            "eps_reward  27.904589433709138\n",
            "eps_reward  28.437824908704656\n",
            "eps_reward  27.91941100413268\n",
            "eps_reward  27.78374744128536\n",
            "eps_reward  28.782305845134957\n",
            "eps_reward  27.45248979624923\n",
            "eps_reward  27.87951298880345\n",
            "eps_reward  28.080753260587958\n",
            "eps_reward  27.78136482446413\n",
            "eps_reward  27.013336415489675\n",
            "eps_reward  28.09799288975141\n",
            "eps_reward  27.867240520182815\n",
            "eps_reward  27.722033199423162\n",
            "eps_reward  27.899945208190953\n",
            "eps_reward  27.288406071687618\n",
            "eps_reward  28.099324546333186\n",
            "eps_reward  28.23616543689194\n",
            "eps_reward  27.145911072259807\n",
            "eps_reward  27.64375385697605\n",
            "eps_reward  28.761149914640455\n",
            "eps_reward  27.410879013121292\n",
            "eps_reward  26.797926638887162\n",
            "eps_reward  27.917048881991626\n",
            "eps_reward  28.26241074255353\n",
            "eps_reward  28.201358271031758\n",
            "eps_reward  27.875413733831657\n",
            "eps_reward  27.85339630594373\n",
            "eps_reward  28.222994409186487\n",
            "eps_reward  28.39389633923838\n",
            "eps_reward  27.99436172443914\n",
            "eps_reward  27.57402031022211\n",
            "eps_reward  28.36992592695403\n",
            "eps_reward  28.25588187253492\n",
            "eps_reward  27.7183964670705\n",
            "eps_reward  28.252616413611644\n",
            "eps_reward  28.610083718983297\n",
            "eps_reward  27.991295380966783\n",
            "eps_reward  27.374092210415725\n",
            "eps_reward  27.145463364732812\n",
            "eps_reward  27.986715679178438\n",
            "eps_reward  27.721284009499644\n",
            "eps_reward  27.822907736835834\n",
            "eps_reward  28.061691550629593\n",
            "eps_reward  27.919983008761076\n",
            "eps_reward  28.157960229813092\n",
            "eps_reward  27.621773934189147\n",
            "eps_reward  28.62496947778875\n",
            "eps_reward  27.46740768889444\n",
            "eps_reward  27.568781546262674\n",
            "eps_reward  28.231541771745565\n",
            "eps_reward  27.654432448655573\n",
            "eps_reward  27.65770093045765\n",
            "eps_reward  27.753080574916005\n",
            "eps_reward  27.87020406402994\n",
            "eps_reward  28.09840675932254\n",
            "eps_reward  28.02977497698657\n",
            "eps_reward  27.771845083539482\n",
            "eps_reward  27.914577572257674\n",
            "eps_reward  28.12267825375169\n",
            "eps_reward  27.09795866115424\n",
            "eps_reward  27.283378013601105\n",
            "eps_reward  27.979540378441996\n",
            "eps_reward  28.31983760159588\n",
            "eps_reward  28.033534959340635\n",
            "eps_reward  27.86139410063567\n",
            "eps_reward  27.662260565873176\n",
            "eps_reward  28.5009705607079\n",
            "eps_reward  27.971220933056184\n",
            "eps_reward  28.587125504636422\n",
            "eps_reward  28.29165895538137\n",
            "eps_reward  27.732960786175244\n",
            "eps_reward  28.026584275787524\n",
            "TO ACF!\n",
            "eps_reward  27.088338205704733\n",
            "eps_reward  27.86371791149336\n",
            "eps_reward  28.71166713041447\n",
            "eps_reward  28.102258350955992\n",
            "eps_reward  26.967158652973396\n",
            "eps_reward  28.331946409416705\n",
            "eps_reward  27.27008078754951\n",
            "eps_reward  28.245025014691468\n",
            "eps_reward  28.068862303979714\n",
            "eps_reward  28.35516334409456\n",
            "eps_reward  28.051983806370785\n",
            "eps_reward  27.358685879318127\n",
            "eps_reward  27.66874608939079\n",
            "eps_reward  27.402958773612596\n",
            "eps_reward  28.062300893942673\n",
            "eps_reward  28.26667287446792\n",
            "eps_reward  27.692370009952338\n",
            "eps_reward  27.968850722057855\n",
            "eps_reward  28.240726300561484\n",
            "eps_reward  28.127024682362173\n",
            "eps_reward  27.814322175234928\n",
            "eps_reward  27.617547377590036\n",
            "eps_reward  27.872801257486394\n",
            "eps_reward  28.17006180959302\n",
            "eps_reward  28.525902079826075\n",
            "eps_reward  27.62848516711321\n",
            "eps_reward  27.60846736530165\n",
            "eps_reward  27.710365445120214\n",
            "eps_reward  28.233394643421935\n",
            "eps_reward  27.40956767609024\n",
            "eps_reward  27.37666260317507\n",
            "eps_reward  28.19596923638546\n",
            "eps_reward  27.389990820221584\n",
            "eps_reward  26.43715998812926\n",
            "eps_reward  28.09085857900228\n",
            "eps_reward  28.563074500363214\n",
            "eps_reward  27.759466092043514\n",
            "eps_reward  27.727712230804155\n",
            "eps_reward  28.168434977895412\n",
            "eps_reward  28.168057653877977\n",
            "eps_reward  28.3926415354197\n",
            "eps_reward  27.237361377983767\n",
            "eps_reward  27.596603000138522\n",
            "eps_reward  27.0392620444343\n",
            "eps_reward  27.12061637941757\n",
            "eps_reward  27.677796043717425\n",
            "eps_reward  27.193638857333386\n",
            "eps_reward  28.207838087014203\n",
            "eps_reward  27.87092361209316\n",
            "eps_reward  27.516334217112167\n",
            "eps_reward  27.229669977282175\n",
            "eps_reward  28.734063813330984\n",
            "eps_reward  28.233425240226698\n",
            "eps_reward  27.663885646265232\n",
            "eps_reward  27.491702072810234\n",
            "eps_reward  27.580340821452033\n",
            "eps_reward  27.85545515583275\n",
            "eps_reward  27.559103164241613\n",
            "eps_reward  27.812234392320654\n",
            "eps_reward  28.026226046635323\n",
            "eps_reward  27.829819480900134\n",
            "eps_reward  28.358324412553902\n",
            "eps_reward  27.695438425788943\n",
            "eps_reward  28.59901273549105\n",
            "eps_reward  27.727249530767725\n",
            "eps_reward  27.566187321051647\n",
            "eps_reward  27.393114009462803\n",
            "eps_reward  28.05726505253376\n",
            "eps_reward  28.878215601171917\n",
            "eps_reward  28.12194250964689\n",
            "eps_reward  28.187812721586113\n",
            "eps_reward  27.203617193438717\n",
            "eps_reward  27.607928824087608\n",
            "eps_reward  27.631493769602113\n",
            "eps_reward  28.2591818549214\n",
            "eps_reward  28.46246638626966\n",
            "eps_reward  28.436961786837948\n",
            "eps_reward  27.71443632506341\n",
            "eps_reward  27.908058992765163\n",
            "eps_reward  28.019659159493187\n",
            "eps_reward  27.33019895182434\n",
            "eps_reward  27.289211153855344\n",
            "eps_reward  27.87442719566297\n",
            "eps_reward  28.538561114475172\n",
            "eps_reward  27.38347833174981\n",
            "eps_reward  27.974382436257837\n",
            "eps_reward  27.736254198946696\n",
            "eps_reward  26.807879323725114\n",
            "eps_reward  27.64844285550689\n",
            "eps_reward  27.79902238979986\n",
            "eps_reward  28.615350733761243\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  26.53751708632705\n",
            "eps_reward  27.54467662434762\n",
            "eps_reward  28.181882586607752\n",
            "eps_reward  27.621752282629895\n",
            "eps_reward  28.064919644494445\n",
            "eps_reward  27.812670655697918\n",
            "eps_reward  27.324833903164162\n",
            "eps_reward  27.599669802935775\n",
            "eps_reward  26.256461986016106\n",
            "eps_reward  27.759137602180644\n",
            "eps_reward  27.80870246746138\n",
            "eps_reward  27.657150505227627\n",
            "eps_reward  28.93453093210967\n",
            "eps_reward  28.284964228150635\n",
            "eps_reward  27.18322872644499\n",
            "eps_reward  26.921211333558976\n",
            "eps_reward  27.225751212929058\n",
            "eps_reward  27.2448390018775\n",
            "eps_reward  27.418399486325917\n",
            "eps_reward  27.74384733382594\n",
            "eps_reward  28.297646867857672\n",
            "eps_reward  27.478157410233194\n",
            "eps_reward  28.69993477978094\n",
            "eps_reward  27.092296515083525\n",
            "eps_reward  27.881186833574066\n",
            "eps_reward  28.138398750976346\n",
            "eps_reward  28.597616264942324\n",
            "eps_reward  28.53288150257126\n",
            "eps_reward  28.118688706048026\n",
            "eps_reward  27.235903916705915\n",
            "eps_reward  27.498038311679718\n",
            "eps_reward  28.577967091805327\n",
            "eps_reward  28.765952126357107\n",
            "eps_reward  28.10410194268989\n",
            "eps_reward  27.01755789442296\n",
            "eps_reward  27.837373156265016\n",
            "eps_reward  27.83356562480585\n",
            "eps_reward  28.4425354196302\n",
            "eps_reward  27.766266862091257\n",
            "eps_reward  27.058790817648976\n",
            "eps_reward  27.131670377368422\n",
            "eps_reward  27.603862721144264\n",
            "eps_reward  27.297404269623108\n",
            "eps_reward  27.71536818601245\n",
            "eps_reward  28.339109635948045\n",
            "eps_reward  27.864832662928986\n",
            "eps_reward  27.68567976908181\n",
            "eps_reward  28.046978379717025\n",
            "eps_reward  27.61652232234233\n",
            "eps_reward  28.52745648336101\n",
            "eps_reward  26.93812339549895\n",
            "eps_reward  27.755307210665617\n",
            "eps_reward  28.120347619651056\n",
            "eps_reward  28.203865393427535\n",
            "eps_reward  28.0895608929559\n",
            "eps_reward  28.685951885971896\n",
            "eps_reward  27.366403590968787\n",
            "eps_reward  27.685121069977924\n",
            "eps_reward  26.954946272697377\n",
            "eps_reward  28.32070822975879\n",
            "eps_reward  27.386122552294616\n",
            "eps_reward  27.998489733054733\n",
            "eps_reward  27.79438024565508\n",
            "eps_reward  27.42515740047927\n",
            "eps_reward  26.96156571858098\n",
            "eps_reward  27.56880753288262\n",
            "eps_reward  27.3855556916036\n",
            "eps_reward  28.480186402575296\n",
            "eps_reward  27.70464564384457\n",
            "eps_reward  26.923029800818906\n",
            "eps_reward  27.666932224722455\n",
            "eps_reward  28.371609828345978\n",
            "eps_reward  27.410942589803586\n",
            "eps_reward  27.23085643295081\n",
            "eps_reward  27.332489004764\n",
            "eps_reward  28.399952075091786\n",
            "TO ACF!\n",
            "eps_reward  26.669073446923132\n",
            "eps_reward  27.926535375912554\n",
            "eps_reward  27.730986967022258\n",
            "eps_reward  27.43676002603655\n",
            "eps_reward  27.182025869990678\n",
            "eps_reward  27.520038243853683\n",
            "eps_reward  28.46230358453646\n",
            "eps_reward  27.87420731636917\n",
            "eps_reward  26.9395857246409\n",
            "eps_reward  28.399135416413397\n",
            "eps_reward  27.067762034843803\n",
            "eps_reward  27.920959134456027\n",
            "eps_reward  28.21025423782821\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  25.46953400807743\n",
            "eps_reward  27.572189319721787\n",
            "eps_reward  27.93666823495903\n",
            "eps_reward  27.18374194799646\n",
            "eps_reward  27.704726236587042\n",
            "eps_reward  27.407612714601623\n",
            "eps_reward  27.696983195191784\n",
            "eps_reward  28.002793773222432\n",
            "eps_reward  28.13162722108011\n",
            "eps_reward  28.875743504834247\n",
            "eps_reward  26.8228904754724\n",
            "eps_reward  28.423764191120757\n",
            "eps_reward  28.399791442615857\n",
            "eps_reward  27.383969820580806\n",
            "eps_reward  27.87442025752523\n",
            "eps_reward  28.073253594694684\n",
            "eps_reward  27.390019136886274\n",
            "eps_reward  28.498754079896646\n",
            "eps_reward  27.236093029915725\n",
            "eps_reward  27.824603476548948\n",
            "eps_reward  28.119571248100286\n",
            "eps_reward  27.215284234112197\n",
            "eps_reward  27.769538076131063\n",
            "eps_reward  27.59010973689718\n",
            "eps_reward  28.16735468313358\n",
            "eps_reward  28.085678598374443\n",
            "eps_reward  27.727795848872375\n",
            "eps_reward  27.260811103970674\n",
            "eps_reward  28.746983087801503\n",
            "eps_reward  27.886245279139395\n",
            "eps_reward  27.580627296223472\n",
            "eps_reward  28.092060740222962\n",
            "eps_reward  27.210757597162896\n",
            "eps_reward  26.874048854256387\n",
            "eps_reward  28.138339782056704\n",
            "eps_reward  28.135748628396502\n",
            "eps_reward  27.498527152587236\n",
            "eps_reward  27.67085718849123\n",
            "eps_reward  27.738950859851794\n",
            "eps_reward  27.61756156686696\n",
            "eps_reward  28.09063510412059\n",
            "eps_reward  27.623206104258045\n",
            "eps_reward  27.797088834674238\n",
            "eps_reward  27.416566483083848\n",
            "eps_reward  27.819379533687258\n",
            "eps_reward  28.03684827807243\n",
            "eps_reward  27.78801367240812\n",
            "eps_reward  27.627636469381937\n",
            "eps_reward  28.284288682697017\n",
            "eps_reward  27.875259202593362\n",
            "eps_reward  27.501144398111784\n",
            "eps_reward  27.73043437183881\n",
            "eps_reward  28.03053405127334\n",
            "eps_reward  27.91654526521544\n",
            "eps_reward  27.161314955966112\n",
            "eps_reward  27.737551115990314\n",
            "eps_reward  27.506697570784656\n",
            "eps_reward  27.65030151039136\n",
            "eps_reward  27.335294508928715\n",
            "eps_reward  27.332290008443643\n",
            "eps_reward  27.197451459120696\n",
            "eps_reward  27.346913996761792\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  26.018468200247785\n",
            "eps_reward  28.053634049163207\n",
            "eps_reward  27.15633164469198\n",
            "eps_reward  27.38579794523139\n",
            "eps_reward  27.628450972871928\n",
            "eps_reward  28.10058146054479\n",
            "eps_reward  27.598631570153035\n",
            "eps_reward  28.192041588610632\n",
            "eps_reward  27.157239727559816\n",
            "eps_reward  27.212043399366923\n",
            "eps_reward  27.295390754673168\n",
            "eps_reward  27.77612608861335\n",
            "eps_reward  27.483236781517324\n",
            "eps_reward  27.202141708525833\n",
            "eps_reward  26.48423911813217\n",
            "eps_reward  27.566256026592843\n",
            "eps_reward  27.62513882913064\n",
            "eps_reward  27.60321529379286\n",
            "eps_reward  27.873607048353062\n",
            "eps_reward  27.926911216372527\n",
            "eps_reward  27.056291892973388\n",
            "eps_reward  28.35910267880546\n",
            "eps_reward  27.669946764736284\n",
            "eps_reward  27.544865362569638\n",
            "eps_reward  27.67424587710712\n",
            "eps_reward  27.469411401511483\n",
            "eps_reward  27.673813051974303\n",
            "eps_reward  28.081314637064906\n",
            "eps_reward  25.928425423777558\n",
            "eps_reward  27.65806066412297\n",
            "eps_reward  28.503365135884522\n",
            "eps_reward  27.124754316190597\n",
            "eps_reward  28.34038943938202\n",
            "eps_reward  27.910629569819164\n",
            "eps_reward  27.34227302364198\n",
            "eps_reward  27.777989605122112\n",
            "eps_reward  27.89717112470643\n",
            "eps_reward  27.402338756577425\n",
            "eps_reward  27.356058549174048\n",
            "eps_reward  27.76824576939906\n",
            "eps_reward  27.87979956749091\n",
            "eps_reward  27.60239934325735\n",
            "eps_reward  28.130950403178574\n",
            "eps_reward  28.345420647568016\n",
            "eps_reward  27.482863721545694\n",
            "eps_reward  27.192733181276548\n",
            "eps_reward  26.893813678428042\n",
            "eps_reward  27.555428881473205\n",
            "eps_reward  28.281165224922\n",
            "eps_reward  27.579697505742892\n",
            "eps_reward  27.065663393446844\n",
            "eps_reward  27.52284416275542\n",
            "eps_reward  27.173777548688786\n",
            "eps_reward  28.398005012282848\n",
            "eps_reward  27.99155370181053\n",
            "TO ACF!\n",
            "eps_reward  27.00990139383594\n",
            "eps_reward  27.52865832994578\n",
            "eps_reward  28.391686113729087\n",
            "eps_reward  27.323456329435768\n",
            "eps_reward  28.1926109486484\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  27.543294044519097\n",
            "eps_reward  27.59026568921849\n",
            "eps_reward  28.03765447405102\n",
            "eps_reward  27.721918253370145\n",
            "eps_reward  27.683591247075896\n",
            "eps_reward  27.49838429256581\n",
            "eps_reward  27.874781270816946\n",
            "eps_reward  28.270266367277262\n",
            "eps_reward  27.69514052873185\n",
            "eps_reward  27.89271700608057\n",
            "eps_reward  27.87118732997216\n",
            "eps_reward  27.333634363083718\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  26.768171609106965\n",
            "eps_reward  27.068895427924943\n",
            "TO ACF!\n",
            "eps_reward  28.0246251442194\n",
            "eps_reward  27.926875596083843\n",
            "eps_reward  27.741649616062745\n",
            "eps_reward  27.009189880438218\n",
            "eps_reward  28.403544746870157\n",
            "eps_reward  27.302616245716905\n",
            "eps_reward  28.43262609927335\n",
            "eps_reward  28.1046908125289\n",
            "eps_reward  27.280062065472112\n",
            "eps_reward  27.03320094631373\n",
            "eps_reward  27.51441416371429\n",
            "eps_reward  27.588670666625553\n",
            "eps_reward  28.254511505741633\n",
            "eps_reward  27.194799542307365\n",
            "eps_reward  28.204150848515052\n",
            "eps_reward  27.297168939763175\n",
            "eps_reward  27.60473193881628\n",
            "eps_reward  28.12357656284429\n",
            "eps_reward  28.276393907236354\n",
            "eps_reward  28.32072474850552\n",
            "eps_reward  27.63019396106695\n",
            "eps_reward  27.49207745463812\n",
            "eps_reward  27.215429190812678\n",
            "eps_reward  27.84702598099905\n",
            "eps_reward  27.81560903461417\n",
            "eps_reward  27.23133838248176\n",
            "eps_reward  28.023827435618877\n",
            "eps_reward  27.68944528774591\n",
            "eps_reward  27.402019160337677\n",
            "eps_reward  27.428969926285376\n",
            "eps_reward  27.65049649522663\n",
            "eps_reward  28.5192860385474\n",
            "eps_reward  28.536892670247905\n",
            "eps_reward  28.40490136770105\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  27.10109965590175\n",
            "eps_reward  28.91578380707427\n",
            "eps_reward  27.92275839260386\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  27.188465136448237\n",
            "eps_reward  27.875549309650435\n",
            "eps_reward  27.18137599838072\n",
            "eps_reward  28.459737715791615\n",
            "eps_reward  28.126558484434117\n",
            "eps_reward  28.177949744217685\n",
            "eps_reward  28.0485838961938\n",
            "eps_reward  28.507751357783096\n",
            "eps_reward  28.037782149101094\n",
            "eps_reward  27.981332713229563\n",
            "eps_reward  27.705315475120624\n",
            "eps_reward  28.36914228855322\n",
            "eps_reward  27.214875872243653\n",
            "eps_reward  27.254551592265184\n",
            "eps_reward  28.014127355097415\n",
            "eps_reward  27.18480523007067\n",
            "eps_reward  26.906553000521765\n",
            "TO ACF!\n",
            "eps_reward  26.396044837943982\n",
            "eps_reward  28.246551215260244\n",
            "eps_reward  28.57107549542191\n",
            "eps_reward  27.51640202246647\n",
            "eps_reward  27.46188414006573\n",
            "eps_reward  26.86180631000441\n",
            "eps_reward  28.05345943290341\n",
            "eps_reward  28.029333453877694\n",
            "eps_reward  28.850196006552622\n",
            "eps_reward  28.291790263147533\n",
            "eps_reward  27.916043027065076\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  26.89808944245282\n",
            "eps_reward  27.737711593045994\n",
            "eps_reward  28.501191386022676\n",
            "eps_reward  27.645871187366843\n",
            "eps_reward  27.909731407769723\n",
            "eps_reward  27.99514508631285\n",
            "eps_reward  26.919893546000033\n",
            "eps_reward  27.39252457386068\n",
            "eps_reward  27.793419400798893\n",
            "eps_reward  27.18961789338759\n",
            "eps_reward  27.56658452626767\n",
            "eps_reward  28.73338471505079\n",
            "eps_reward  28.189140828189203\n",
            "eps_reward  28.22527107418922\n",
            "eps_reward  27.988412260768946\n",
            "eps_reward  27.839739969564157\n",
            "eps_reward  27.239267972062958\n",
            "eps_reward  27.727922253384047\n",
            "eps_reward  27.70243369143416\n",
            "eps_reward  27.395482164946007\n",
            "eps_reward  27.979319008904362\n",
            "eps_reward  28.01482113071407\n",
            "eps_reward  27.811078634836033\n",
            "eps_reward  27.653126692863456\n",
            "eps_reward  28.834586559870246\n",
            "eps_reward  28.22580713345939\n",
            "eps_reward  27.671878277358477\n",
            "eps_reward  27.954511355578763\n",
            "eps_reward  27.639350164370114\n",
            "eps_reward  27.249191627927758\n",
            "eps_reward  27.92320051367527\n",
            "eps_reward  27.838689944130497\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  26.69110641523021\n",
            "eps_reward  28.00097938003807\n",
            "TO ACF!\n",
            "eps_reward  27.024498506064756\n",
            "eps_reward  28.935885262875303\n",
            "eps_reward  28.010276779931843\n",
            "eps_reward  28.127440311568037\n",
            "eps_reward  27.26484631836618\n",
            "eps_reward  28.231545697281522\n",
            "eps_reward  28.163867319144412\n",
            "eps_reward  28.04689450245244\n",
            "eps_reward  27.681984679961435\n",
            "eps_reward  27.771669409380635\n",
            "eps_reward  27.81416666979916\n",
            "eps_reward  28.115461595724213\n",
            "eps_reward  27.60305014536361\n",
            "eps_reward  26.38800116142081\n",
            "eps_reward  27.874317441929215\n",
            "eps_reward  28.489485276256843\n",
            "eps_reward  28.07166260045077\n",
            "eps_reward  28.079004776967317\n",
            "eps_reward  27.806016898354205\n",
            "eps_reward  27.782286583426277\n",
            "eps_reward  27.92524334618666\n",
            "eps_reward  28.147332114669133\n",
            "eps_reward  27.347478551984363\n",
            "eps_reward  27.330055497154387\n",
            "eps_reward  28.273625236078733\n",
            "eps_reward  27.666130082052668\n",
            "eps_reward  27.349156226403252\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  27.640981804692448\n",
            "eps_reward  27.87375578614694\n",
            "eps_reward  27.413685000463005\n",
            "eps_reward  27.36313739744464\n",
            "eps_reward  28.68341766074161\n",
            "eps_reward  26.90537725909789\n",
            "eps_reward  28.262658474650664\n",
            "eps_reward  28.35441316021254\n",
            "eps_reward  28.47547509987999\n",
            "eps_reward  28.115054974084664\n",
            "eps_reward  27.62627898903307\n",
            "eps_reward  28.17878878718635\n",
            "eps_reward  27.412755721996064\n",
            "TO ACF!\n",
            "eps_reward  27.498496984805087\n",
            "eps_reward  27.35774327067464\n",
            "eps_reward  27.302201625187884\n",
            "eps_reward  26.962517891479976\n",
            "eps_reward  27.70843381384078\n",
            "eps_reward  27.7787314409729\n",
            "eps_reward  28.005076720467663\n",
            "eps_reward  28.274597905629843\n",
            "eps_reward  27.39961864391698\n",
            "eps_reward  28.078156458173144\n",
            "eps_reward  27.441785450156793\n",
            "eps_reward  27.472037310592416\n",
            "eps_reward  28.394692670685913\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  27.50602246126249\n",
            "eps_reward  27.25756278687317\n",
            "eps_reward  27.457113671121892\n",
            "eps_reward  28.161154154810784\n",
            "eps_reward  27.988889411177706\n",
            "eps_reward  27.851561289388563\n",
            "eps_reward  27.67545742759682\n",
            "eps_reward  28.764309221567707\n",
            "eps_reward  27.431085276062184\n",
            "eps_reward  28.00475222926034\n",
            "eps_reward  27.921693129686425\n",
            "eps_reward  27.77097693416662\n",
            "eps_reward  27.992975865052628\n",
            "eps_reward  28.7040313863881\n",
            "TO ACF!\n",
            "eps_reward  26.83520877672016\n",
            "eps_reward  26.138016365008767\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  26.71979775302099\n",
            "eps_reward  28.745242322574637\n",
            "eps_reward  27.308371552221693\n",
            "eps_reward  27.769980482486826\n",
            "eps_reward  27.90351914150318\n",
            "eps_reward  28.11837572704313\n",
            "eps_reward  28.000997240836394\n",
            "eps_reward  28.19177214901315\n",
            "eps_reward  26.87907510370416\n",
            "eps_reward  27.416376131950678\n",
            "eps_reward  28.264753100986717\n",
            "eps_reward  28.706097248685207\n",
            "eps_reward  27.502900432107616\n",
            "eps_reward  27.523667526182106\n",
            "eps_reward  27.767555643549468\n",
            "eps_reward  27.237482506291894\n",
            "eps_reward  28.5531516923991\n",
            "eps_reward  27.4941153438026\n",
            "eps_reward  26.87077445141547\n",
            "eps_reward  27.66691221211781\n",
            "eps_reward  27.912195892840703\n",
            "eps_reward  28.739171952243904\n",
            "eps_reward  27.56204478140254\n",
            "eps_reward  27.41357859748627\n",
            "eps_reward  28.140240759640303\n",
            "eps_reward  28.20378747887984\n",
            "eps_reward  27.338051010842275\n",
            "eps_reward  27.927886740721213\n",
            "eps_reward  28.202433289915703\n",
            "eps_reward  27.72829750651003\n",
            "eps_reward  26.981718191392982\n",
            "eps_reward  28.446887246614374\n",
            "eps_reward  27.8980532483918\n",
            "eps_reward  27.231767875212455\n",
            "eps_reward  27.911581459683433\n",
            "eps_reward  27.962022006926137\n",
            "eps_reward  28.761820805241324\n",
            "eps_reward  27.790414803261815\n",
            "eps_reward  27.928146093930234\n",
            "eps_reward  27.346352483592394\n",
            "eps_reward  27.577452804121926\n",
            "eps_reward  27.166001461883248\n",
            "eps_reward  27.23566772353822\n",
            "eps_reward  27.78951189783825\n",
            "eps_reward  28.071611618317263\n",
            "eps_reward  28.037837014091277\n",
            "eps_reward  28.16850987829504\n",
            "eps_reward  27.49791198834883\n",
            "eps_reward  28.487452411425124\n",
            "eps_reward  28.17921421637687\n",
            "eps_reward  27.543163155403306\n",
            "eps_reward  27.90815857924693\n",
            "eps_reward  27.94272831647966\n",
            "eps_reward  28.384365316816236\n",
            "eps_reward  27.19796265990353\n",
            "eps_reward  27.463115002283917\n",
            "eps_reward  27.442625035015425\n",
            "eps_reward  27.92966905177084\n",
            "eps_reward  28.056703662715737\n",
            "eps_reward  27.043818597106842\n",
            "eps_reward  28.500279647107963\n",
            "eps_reward  26.828761532967263\n",
            "eps_reward  27.53708111435673\n",
            "eps_reward  27.675318110035825\n",
            "eps_reward  26.13574592604095\n",
            "eps_reward  28.579323399040607\n",
            "eps_reward  27.589893084843006\n",
            "eps_reward  27.663062611508202\n",
            "eps_reward  28.38957562391645\n",
            "eps_reward  27.266556729396687\n",
            "eps_reward  27.48272668525601\n",
            "eps_reward  27.824377885579846\n",
            "eps_reward  27.05936030437738\n",
            "eps_reward  27.342405977443942\n",
            "eps_reward  27.65519058694832\n",
            "eps_reward  28.38645666519674\n",
            "eps_reward  27.58347306133078\n",
            "eps_reward  27.787648037248267\n",
            "eps_reward  27.106384932711084\n",
            "eps_reward  27.94817447903049\n",
            "eps_reward  27.37133527560338\n",
            "eps_reward  27.441781616067296\n",
            "eps_reward  27.879764684197905\n",
            "eps_reward  27.643760827492706\n",
            "eps_reward  28.768324409325235\n",
            "eps_reward  27.426223319854362\n",
            "eps_reward  27.75835504489418\n",
            "TO ACF!\n",
            "eps_reward  28.05161207728506\n",
            "eps_reward  26.80689353633413\n",
            "eps_reward  27.233358600362276\n",
            "eps_reward  27.961997206610715\n",
            "eps_reward  26.865506280706544\n",
            "eps_reward  28.23886013957558\n",
            "eps_reward  27.69624434187763\n",
            "eps_reward  27.779462607882113\n",
            "eps_reward  27.7024882775653\n",
            "eps_reward  27.578874551904864\n",
            "eps_reward  27.376770434780056\n",
            "eps_reward  28.509253491663546\n",
            "eps_reward  27.800953589522805\n",
            "eps_reward  28.455872457502107\n",
            "eps_reward  27.706714592837155\n",
            "eps_reward  28.674004865245067\n",
            "eps_reward  28.350372518318867\n",
            "eps_reward  27.523456636789223\n",
            "eps_reward  27.977020604753353\n",
            "eps_reward  28.64702894673733\n",
            "eps_reward  27.048173469505628\n",
            "eps_reward  27.279323673179462\n",
            "eps_reward  27.546351137346466\n",
            "eps_reward  27.991733842620533\n",
            "eps_reward  27.965088521867155\n",
            "eps_reward  28.100693575516893\n",
            "eps_reward  28.27019476458311\n",
            "eps_reward  28.428564982914697\n",
            "eps_reward  27.306457037232214\n",
            "eps_reward  27.750658942043408\n",
            "eps_reward  28.191176483128498\n",
            "eps_reward  27.610457506585796\n",
            "eps_reward  27.945993264983407\n",
            "eps_reward  27.157402658934803\n",
            "eps_reward  28.017896708522983\n",
            "eps_reward  27.694665002190703\n",
            "eps_reward  28.138584745865597\n",
            "eps_reward  27.3428689804933\n",
            "eps_reward  27.18863865347629\n",
            "eps_reward  27.309605827126546\n",
            "eps_reward  27.262736766023266\n",
            "eps_reward  27.915008916965\n",
            "eps_reward  27.963813174252838\n",
            "eps_reward  27.863803086273006\n",
            "eps_reward  28.056289888760283\n",
            "eps_reward  27.766839899001532\n",
            "eps_reward  27.878981606779387\n",
            "eps_reward  27.815691800360696\n",
            "eps_reward  28.301305557388574\n",
            "eps_reward  27.769104204470988\n",
            "eps_reward  27.891354607276895\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  27.499622405415963\n",
            "eps_reward  27.565174460499527\n",
            "eps_reward  27.749349407878363\n",
            "eps_reward  27.001649252169237\n",
            "eps_reward  27.124094167286255\n",
            "eps_reward  27.044871928846753\n",
            "eps_reward  27.622221032049577\n",
            "eps_reward  28.53058169902909\n",
            "eps_reward  28.31619207516739\n",
            "eps_reward  27.28768546820854\n",
            "eps_reward  27.73421857801965\n",
            "TO ACF!\n",
            "TO ACF!\n",
            "eps_reward  28.19046584909572\n",
            "eps_reward  27.66155399170932\n",
            "eps_reward  27.961083994446334\n",
            "eps_reward  27.010798718323453\n",
            "eps_reward  28.55792255633077\n",
            "eps_reward  27.834222300431367\n",
            "eps_reward  27.749192896624983\n",
            "eps_reward  27.927579441211364\n",
            "eps_reward  27.52031259042604\n",
            "eps_reward  28.07399006878232\n",
            "eps_reward  27.46234235436874\n",
            "eps_reward  28.456645269076233\n",
            "eps_reward  27.758406966250526\n",
            "eps_reward  27.187756062035014\n",
            "eps_reward  27.87834163350229\n",
            "eps_reward  27.457582303967936\n",
            "eps_reward  26.809157412018926\n",
            "eps_reward  27.5754241186695\n",
            "eps_reward  27.732836738501916\n",
            "eps_reward  27.921967908832244\n",
            "eps_reward  27.470053061422867\n",
            "eps_reward  27.806058131044537\n",
            "eps_reward  27.46985325023405\n",
            "eps_reward  27.337340939345943\n",
            "eps_reward  28.20940547196665\n",
            "eps_reward  27.700780801270422\n",
            "eps_reward  27.83036741726771\n",
            "eps_reward  28.134927814371235\n",
            "eps_reward  28.436059524680807\n",
            "eps_reward  28.49750471995672\n",
            "eps_reward  28.01857845678825\n",
            "eps_reward  27.42190193595406\n",
            "eps_reward  27.182845989874462\n",
            "total_rewards  554.682185570104\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cYmz09nTFFRX",
        "outputId": "e332b5e2-b727-44e0-a266-ab2b26e783dc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "trajectories[1]"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'legal_actions': [0, 1],\n",
              "  'obs': array([1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])},\n",
              " 0,\n",
              " 0.6160369580718652,\n",
              " {'legal_actions': [0, 1],\n",
              "  'obs': array([2., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])},\n",
              " False]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    }
  ]
}