TPU运行Cifar10.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "欢迎使用 Colaboratory",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "fo0v18No2MNj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        },
        "outputId": "a92a273d-1461-4399-a712-0dea9b83a245"
      },
      "source": [
        "#代码来自：\n",
        "#https://blog.shikoan.com/colab-tpu-resnet-benchmarks/\n",
        "!pip install tensorflow==1.13.1"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tensorflow==1.13.1 in /usr/local/lib/python3.6/dist-packages (1.13.1)\n",
            "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (0.2.2)\n",
            "Requirement already satisfied: tensorboard<1.14.0,>=1.13.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.13.1)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.16.5)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.12.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (0.33.6)\n",
            "Requirement already satisfied: tensorflow-estimator<1.14.0rc0,>=1.13.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.13.0)\n",
            "Requirement already satisfied: absl-py>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (0.8.0)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.0.8)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (3.7.1)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.15.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.1.0)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (0.8.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.14.0,>=1.13.0->tensorflow==1.13.1) (0.15.6)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.14.0,>=1.13.0->tensorflow==1.13.1) (3.1.1)\n",
            "Requirement already satisfied: mock>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-estimator<1.14.0rc0,>=1.13.0->tensorflow==1.13.1) (3.0.5)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow==1.13.1) (2.8.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow==1.13.1) (41.2.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "64koeuutzN66",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "1501bbcc-63b6-4ef3-82ad-30b815506a95"
      },
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow.keras.backend as K\n",
        "from tensorflow.keras.layers import Conv2D, Activation, BatchNormalization, Add, Input, GlobalAveragePooling2D, Dense\n",
        "from tensorflow.keras.models import Model\n",
        "from keras.datasets import cifar10\n",
        "from keras.utils import to_categorical\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "from tensorflow.keras.callbacks import Callback, LearningRateScheduler\n",
        "from tensorflow.keras.optimizers import SGD\n",
        "from keras.regularizers import l2\n",
        "from tensorflow.contrib.tpu.python.tpu import keras_support\n",
        "import time, pickle, os\n",
        "\n",
        "# 経過時間用のコールバック\n",
        "class TimeHistory(Callback):\n",
        "    def on_train_begin(self, logs={}):\n",
        "        self.times = []\n",
        "\n",
        "    def on_epoch_begin(self, epoch, logs={}):\n",
        "        self.epoch_start_time = time.time()\n",
        "\n",
        "    def on_epoch_end(self, epoch, logs={}):\n",
        "        self.times.append(time.time() - self.epoch_start_time)\n",
        "\n",
        "class ResNet:\n",
        "    def __init__(self, n, framework, initial_lr=0.01, nb_epochs=100):\n",
        "        self.n = n\n",
        "        # バッチサイズがかなり違うので学習率は要調整\n",
        "        self.framework = framework\n",
        "        self.initial_lr = initial_lr\n",
        "        self.nb_epochs = nb_epochs\n",
        "        self.weight_decay = 0.0005\n",
        "        # Make model\n",
        "        self.model = self.make_model()\n",
        "\n",
        "    # オリジナルの論文に従って、サブサンプリングにPoolingではなくstride=2のConvを使う\n",
        "    def subsumpling(self, output_channels, input_tensor):\n",
        "        return Conv2D(output_channels, kernel_size=1, strides=(2,2), kernel_regularizer=l2(self.weight_decay))(input_tensor)\n",
        "\n",
        "    # BN->ReLU->Conv->BN->ReLU->Conv をショートカットさせる(Kaimingらの研究による)\n",
        "    # https://www.slideshare.net/KotaNagasato/resnet-82940994\n",
        "    def block(self, channles, input_tensor):\n",
        "        # ショートカット元\n",
        "        shortcut = input_tensor\n",
        "        # メイン側\n",
        "        x = BatchNormalization()(input_tensor)\n",
        "        x = Activation(\"relu\")(x)\n",
        "        x = Conv2D(channles, kernel_size=3, padding=\"same\", kernel_regularizer=l2(self.weight_decay))(x)\n",
        "        x = BatchNormalization()(x)\n",
        "        x = Activation(\"relu\")(x)\n",
        "        x = Conv2D(channles, kernel_size=3, padding=\"same\", kernel_regularizer=l2(self.weight_decay))(x)\n",
        "        # 結合\n",
        "        return Add()([x, shortcut])\n",
        "\n",
        "    def make_model(self):\n",
        "        input = Input(shape=(32, 32, 3))\n",
        "        # 3->16にチャンネル数を増やす\n",
        "        x = Conv2D(16, kernel_size=3, padding=\"same\", kernel_regularizer=l2(self.weight_decay))(input)\n",
        "        # 32x32x16のブロックをn回\n",
        "        for i in range(self.n):\n",
        "            x = self.block(16, x)\n",
        "        # 16x16x32\n",
        "        x = self.subsumpling(32, x)\n",
        "        for i in range(self.n):\n",
        "            x = self.block(32, x)\n",
        "        # 8x8x64\n",
        "        x = self.subsumpling(64, x)\n",
        "        for i in range(self.n):\n",
        "            x = self.block(64, x)\n",
        "        # Global Average Pooling\n",
        "        x = GlobalAveragePooling2D()(x)\n",
        "        x = Dense(10, activation=\"softmax\")(x)\n",
        "        # model\n",
        "        model = Model(input, x)\n",
        "        return model\n",
        "\n",
        "    def lr_schduler(self, epoch):\n",
        "        x = self.initial_lr\n",
        "        if epoch >= self.nb_epochs * 0.5: x /= 10.0\n",
        "        if epoch >= self.nb_epochs * 0.75: x /= 10.0\n",
        "        return x\n",
        "\n",
        "    def train(self, X_train, y_train, X_val, y_val):\n",
        "        # コンパイル\n",
        "        self.model.compile(optimizer=SGD(lr=self.initial_lr, momentum=0.9),\n",
        "                          loss=\"categorical_crossentropy\", metrics=[\"acc\"])\n",
        "        # Data Augmentation\n",
        "        traingen = ImageDataGenerator(\n",
        "            rescale=1./255,\n",
        "            width_shift_range=4./32,\n",
        "            height_shift_range=4./32,\n",
        "            horizontal_flip=True)\n",
        "        valgen = ImageDataGenerator(\n",
        "            rescale=1./255)\n",
        "        # TPU\n",
        "        tpu_grpc_url = \"grpc://\"+os.environ[\"COLAB_TPU_ADDR\"]\n",
        "        tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(tpu_grpc_url)\n",
        "        strategy = keras_support.TPUDistributionStrategy(tpu_cluster_resolver)\n",
        "        self.model = tf.contrib.tpu.keras_to_tpu_model(self.model, strategy=strategy)\n",
        "\n",
        "        # Callback\n",
        "        time_cb = TimeHistory()\n",
        "        lr_cb = LearningRateScheduler(self.lr_schduler)\n",
        "        # Train\n",
        "        history = self.model.fit_generator(traingen.flow(X_train, y_train, batch_size=1024), epochs=self.nb_epochs,\n",
        "                                           steps_per_epoch=len(X_train)/1024, validation_data=valgen.flow(X_val, y_val, batch_size=1024),\n",
        "                                           validation_steps=len(X_val)/1024,\n",
        "                                           callbacks=[time_cb, lr_cb]).history\n",
        "        history[\"time\"] = time_cb.times\n",
        "        # Save history\n",
        "        file_name = f\"{self.framework}_n{self.n}.dat\"\n",
        "        with open(file_name, \"wb\") as fp:\n",
        "            pickle.dump(history, fp)\n",
        "\n",
        "# Main function\n",
        "def main(n, framework):\n",
        "    K.clear_session()\n",
        "    # layers = 6n+2\n",
        "    net = ResNet(n, framework, nb_epochs=100)\n",
        "    # CIFAR\n",
        "    (X_train, y_train), (X_test, y_test) = cifar10.load_data()\n",
        "    y_train, y_test = to_categorical(y_train), to_categorical(y_test)\n",
        "    # train\n",
        "    net.train(X_train, y_train, X_test, y_test)\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    main(3, \"tf_keras_tpu\")\n"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Colocations handled automatically by placer.\n",
            "INFO:tensorflow:Querying Tensorflow master (grpc://10.94.17.170:8470) for TPU system metadata.\n",
            "INFO:tensorflow:Found TPU system:\n",
            "INFO:tensorflow:*** Num TPU Cores: 8\n",
            "INFO:tensorflow:*** Num TPU Workers: 1\n",
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 5318427542844900690)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, 5972651299852305070)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 16868474098292505710)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, 7432862407376987853)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, 8452046943043630676)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 4894794154746251569)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 16242358405714188922)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 15231011154896667063)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 1938731572797274520)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, 9964760283833988398)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 8589934592, 7014322032663729690)\n",
            "WARNING:tensorflow:tpu_model (from tensorflow.contrib.tpu.python.tpu.keras_support) is experimental and may change or be removed at any time, and without warning.\n",
            "INFO:tensorflow:Cloning SGD {'lr': 0.009999999776482582, 'momentum': 0.8999999761581421, 'decay': 0.0, 'nesterov': False}\n",
            "INFO:tensorflow:Cloning SGD {'lr': 0.009999999776482582, 'momentum': 0.8999999761581421, 'decay': 0.0, 'nesterov': False}\n",
            "Epoch 1/100\n",
            "INFO:tensorflow:New input shapes; (re-)compiling: mode=train (# of cores 8), [TensorSpec(shape=(128,), dtype=tf.int32, name='core_id0'), TensorSpec(shape=(128, 32, 32, 3), dtype=tf.float32, name='input_1_10'), TensorSpec(shape=(128, 10), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Cloning SGD {'lr': 0.009999999776482582, 'momentum': 0.8999999761581421, 'decay': 0.0, 'nesterov': False}\n",
            "INFO:tensorflow:Remapping placeholder for input_1\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/tpu/python/tpu/keras_support.py:302: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.cast instead.\n",
            "INFO:tensorflow:KerasCrossShard: <tensorflow.python.keras.optimizers.SGD object at 0x7f8e2b748550> []\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.cast instead.\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 11.385817527770996 secs\n",
            "INFO:tensorflow:Setting weights on TPU model.\n",
            "INFO:tensorflow:CPU -> TPU lr: 0.009999999776482582 {0.01}\n",
            "INFO:tensorflow:CPU -> TPU momentum: 0.8999999761581421 {0.9}\n",
            "INFO:tensorflow:CPU -> TPU decay: 0.0 {0.0}\n",
            "WARNING:tensorflow:Cannot update non-variable config: nesterov\n",
            "21/49 [===========>..................] - ETA: 1:05 - loss: 2.5399 - acc: 0.2125INFO:tensorflow:New input shapes; (re-)compiling: mode=train (# of cores 8), [TensorSpec(shape=(106,), dtype=tf.int32, name='core_id0'), TensorSpec(shape=(106, 32, 32, 3), dtype=tf.float32, name='input_1_10'), TensorSpec(shape=(106, 10), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for input_1\n",
            "INFO:tensorflow:KerasCrossShard: <tensorflow.python.keras.optimizers.SGD object at 0x7f8e2b748550> [<tf.Variable 'tpu_140248638363800/SGD/iterations:0' shape=() dtype=int64>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e265c1be0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e265c1eb8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e265e1630>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e2658b668>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e264fccf8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e26491550>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e26491e48>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e26432c18>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e26403da0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e26372940>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e2633f4e0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e2630c780>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e262afd30>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e262476a0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e2620f1d0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e2626beb8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e26180cf8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e26118828>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e260be710>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e260e1c88>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e26088e48>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e2601dc18>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25f31d68>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25f556a0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25f238d0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25e3b208>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25e3bb00>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25e06160>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25dce518>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25d41da0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25d64e80>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25c79dd8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25c9e978>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25c43e48>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25c11f28>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25ba68d0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25b16ba8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25abccc0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25af2390>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e259f81d0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e259caf28>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e259e9cc0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25932c18>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25927f28>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25893fd0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25839860>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25870c88>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e2579dd30>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25746908>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e257460f0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e256aef60>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e256a4e10>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25614be0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e255d9550>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e2552c128>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e2552c080>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e254bfba8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e2542ddd8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e254542b0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e2541ea90>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e253c4d30>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e252cac88>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e252cada0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e2532ada0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25273860>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e251afe80>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e251d1d30>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25178128>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25141dd8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e250da3c8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e250a5b38>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25049cf8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e25014f60>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e24f83c50>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e24faadd8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e24ef5780>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e24ee6fd0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e24e58ef0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e24e249e8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7f8e24dc8080>]\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 12.1472327709198 secs\n",
            "48/49 [============================>.] - ETA: 1s - loss: 2.3087 - acc: 0.2831INFO:tensorflow:New input shapes; (re-)compiling: mode=eval (# of cores 8), [TensorSpec(shape=(128,), dtype=tf.int32, name='core_id_10'), TensorSpec(shape=(128, 32, 32, 3), dtype=tf.float32, name='input_1_10'), TensorSpec(shape=(128, 10), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Cloning SGD {'lr': 0.009999999776482582, 'momentum': 0.8999999761581421, 'decay': 0.0, 'nesterov': False}\n",
            "INFO:tensorflow:Remapping placeholder for input_1\n",
            "INFO:tensorflow:KerasCrossShard: <tensorflow.python.keras.optimizers.SGD object at 0x7f8e1c2441d0> []\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 8.65952754020691 secs\n",
            " 9/10 [==========================>...] - ETA: 1s - loss: 2.6732 - acc: 0.1459INFO:tensorflow:New input shapes; (re-)compiling: mode=eval (# of cores 8), [TensorSpec(shape=(98,), dtype=tf.int32, name='core_id_10'), TensorSpec(shape=(98, 32, 32, 3), dtype=tf.float32, name='input_1_10'), TensorSpec(shape=(98, 10), dtype=tf.float32, name='dense_target_30')]\n",
            "INFO:tensorflow:Overriding default placeholder.\n",
            "INFO:tensorflow:Remapping placeholder for input_1\n",
            "INFO:tensorflow:KerasCrossShard: <tensorflow.python.keras.optimizers.SGD object at 0x7f8e1c2441d0> []\n",
            "INFO:tensorflow:Started compiling\n",
            "INFO:tensorflow:Finished compiling. Time elapsed: 5.899410247802734 secs\n",
            "10/10 [==============================] - 23s 2s/step - loss: 2.6726 - acc: 0.1459\n",
            "49/49 [==============================] - 108s 2s/step - loss: 2.3035 - acc: 0.2845 - val_loss: 2.6726 - val_acc: 0.1459\n",
            "Epoch 2/100\n",
            "10/10 [==============================] - 1s 133ms/step - loss: 2.8799 - acc: 0.1693\n",
            "49/49 [==============================] - 17s 337ms/step - loss: 1.9664 - acc: 0.4042 - val_loss: 2.8799 - val_acc: 0.1693\n",
            "Epoch 3/100\n",
            "10/10 [==============================] - 1s 132ms/step - loss: 3.0991 - acc: 0.1638\n",
            "49/49 [==============================] - 20s 407ms/step - loss: 1.8401 - acc: 0.4598 - val_loss: 3.0991 - val_acc: 0.1638\n",
            "Epoch 4/100\n",
            "10/10 [==============================] - 1s 124ms/step - loss: 2.8439 - acc: 0.2025\n",
            "49/49 [==============================] - 20s 402ms/step - loss: 1.7443 - acc: 0.4951 - val_loss: 2.8439 - val_acc: 0.2025\n",
            "Epoch 5/100\n",
            "10/10 [==============================] - 1s 131ms/step - loss: 3.1338 - acc: 0.1962\n",
            "49/49 [==============================] - 20s 408ms/step - loss: 1.6749 - acc: 0.5223 - val_loss: 3.1338 - val_acc: 0.1962\n",
            "Epoch 6/100\n",
            "10/10 [==============================] - 1s 127ms/step - loss: 2.6096 - acc: 0.2802\n",
            "49/49 [==============================] - 20s 403ms/step - loss: 1.6104 - acc: 0.5474 - val_loss: 2.6096 - val_acc: 0.2802\n",
            "Epoch 7/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 1.8398 - acc: 0.4465\n",
            "49/49 [==============================] - 20s 406ms/step - loss: 1.5409 - acc: 0.5715 - val_loss: 1.8398 - val_acc: 0.4465\n",
            "Epoch 8/100\n",
            "10/10 [==============================] - 1s 132ms/step - loss: 1.7142 - acc: 0.4913\n",
            "49/49 [==============================] - 20s 405ms/step - loss: 1.4927 - acc: 0.5890 - val_loss: 1.7142 - val_acc: 0.4913\n",
            "Epoch 9/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 1.6611 - acc: 0.5267\n",
            "49/49 [==============================] - 20s 404ms/step - loss: 1.4430 - acc: 0.6066 - val_loss: 1.6611 - val_acc: 0.5267\n",
            "Epoch 10/100\n",
            "10/10 [==============================] - 1s 131ms/step - loss: 1.5727 - acc: 0.5596\n",
            "49/49 [==============================] - 20s 403ms/step - loss: 1.3989 - acc: 0.6202 - val_loss: 1.5727 - val_acc: 0.5596\n",
            "Epoch 11/100\n",
            "10/10 [==============================] - 1s 133ms/step - loss: 1.4498 - acc: 0.6066\n",
            "49/49 [==============================] - 20s 407ms/step - loss: 1.3583 - acc: 0.6347 - val_loss: 1.4498 - val_acc: 0.6066\n",
            "Epoch 12/100\n",
            "10/10 [==============================] - 1s 136ms/step - loss: 1.5785 - acc: 0.5598\n",
            "49/49 [==============================] - 20s 406ms/step - loss: 1.3157 - acc: 0.6483 - val_loss: 1.5785 - val_acc: 0.5598\n",
            "Epoch 13/100\n",
            "10/10 [==============================] - 1s 134ms/step - loss: 1.3907 - acc: 0.6330\n",
            "49/49 [==============================] - 20s 408ms/step - loss: 1.2840 - acc: 0.6607 - val_loss: 1.3907 - val_acc: 0.6330\n",
            "Epoch 14/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 1.4163 - acc: 0.6184\n",
            "49/49 [==============================] - 20s 404ms/step - loss: 1.2530 - acc: 0.6725 - val_loss: 1.4163 - val_acc: 0.6184\n",
            "Epoch 15/100\n",
            "10/10 [==============================] - 1s 124ms/step - loss: 1.3498 - acc: 0.6367\n",
            "49/49 [==============================] - 20s 401ms/step - loss: 1.2241 - acc: 0.6820 - val_loss: 1.3498 - val_acc: 0.6367\n",
            "Epoch 16/100\n",
            "10/10 [==============================] - 1s 125ms/step - loss: 2.3644 - acc: 0.4371\n",
            "49/49 [==============================] - 20s 409ms/step - loss: 1.1885 - acc: 0.6934 - val_loss: 2.3644 - val_acc: 0.4371\n",
            "Epoch 17/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 1.2765 - acc: 0.6628\n",
            "49/49 [==============================] - 20s 410ms/step - loss: 1.1605 - acc: 0.7048 - val_loss: 1.2765 - val_acc: 0.6628\n",
            "Epoch 18/100\n",
            "10/10 [==============================] - 1s 140ms/step - loss: 1.6145 - acc: 0.6053\n",
            "49/49 [==============================] - 20s 404ms/step - loss: 1.1338 - acc: 0.7154 - val_loss: 1.6145 - val_acc: 0.6053\n",
            "Epoch 19/100\n",
            "10/10 [==============================] - 1s 136ms/step - loss: 1.3215 - acc: 0.6441\n",
            "49/49 [==============================] - 20s 408ms/step - loss: 1.1139 - acc: 0.7182 - val_loss: 1.3215 - val_acc: 0.6441\n",
            "Epoch 20/100\n",
            "10/10 [==============================] - 1s 134ms/step - loss: 1.4224 - acc: 0.6276\n",
            "49/49 [==============================] - 20s 410ms/step - loss: 1.0826 - acc: 0.7305 - val_loss: 1.4224 - val_acc: 0.6276\n",
            "Epoch 21/100\n",
            "10/10 [==============================] - 1s 132ms/step - loss: 1.1921 - acc: 0.6980\n",
            "49/49 [==============================] - 20s 407ms/step - loss: 1.0667 - acc: 0.7344 - val_loss: 1.1921 - val_acc: 0.6980\n",
            "Epoch 22/100\n",
            "10/10 [==============================] - 1s 127ms/step - loss: 1.2557 - acc: 0.6831\n",
            "49/49 [==============================] - 20s 405ms/step - loss: 1.0452 - acc: 0.7428 - val_loss: 1.2557 - val_acc: 0.6831\n",
            "Epoch 23/100\n",
            "10/10 [==============================] - 1s 125ms/step - loss: 1.1666 - acc: 0.6996\n",
            "49/49 [==============================] - 20s 410ms/step - loss: 1.0274 - acc: 0.7460 - val_loss: 1.1666 - val_acc: 0.6996\n",
            "Epoch 24/100\n",
            "10/10 [==============================] - 1s 134ms/step - loss: 1.2801 - acc: 0.6730\n",
            "49/49 [==============================] - 20s 409ms/step - loss: 1.0111 - acc: 0.7524 - val_loss: 1.2801 - val_acc: 0.6730\n",
            "Epoch 25/100\n",
            "10/10 [==============================] - 1s 136ms/step - loss: 1.1383 - acc: 0.7120\n",
            "49/49 [==============================] - 20s 408ms/step - loss: 0.9939 - acc: 0.7574 - val_loss: 1.1383 - val_acc: 0.7120\n",
            "Epoch 26/100\n",
            "10/10 [==============================] - 1s 132ms/step - loss: 1.1603 - acc: 0.7167\n",
            "49/49 [==============================] - 20s 413ms/step - loss: 0.9659 - acc: 0.7654 - val_loss: 1.1603 - val_acc: 0.7167\n",
            "Epoch 27/100\n",
            "10/10 [==============================] - 1s 132ms/step - loss: 1.0836 - acc: 0.7310\n",
            "49/49 [==============================] - 20s 404ms/step - loss: 0.9560 - acc: 0.7710 - val_loss: 1.0836 - val_acc: 0.7310\n",
            "Epoch 28/100\n",
            "10/10 [==============================] - 1s 133ms/step - loss: 1.1530 - acc: 0.7166\n",
            "49/49 [==============================] - 20s 412ms/step - loss: 0.9386 - acc: 0.7753 - val_loss: 1.1530 - val_acc: 0.7166\n",
            "Epoch 29/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 1.1922 - acc: 0.7094\n",
            "49/49 [==============================] - 20s 403ms/step - loss: 0.9239 - acc: 0.7826 - val_loss: 1.1922 - val_acc: 0.7094\n",
            "Epoch 30/100\n",
            "10/10 [==============================] - 1s 125ms/step - loss: 1.3473 - acc: 0.6672\n",
            "49/49 [==============================] - 20s 404ms/step - loss: 0.9113 - acc: 0.7829 - val_loss: 1.3473 - val_acc: 0.6672\n",
            "Epoch 31/100\n",
            "10/10 [==============================] - 1s 132ms/step - loss: 1.0207 - acc: 0.7525\n",
            "49/49 [==============================] - 20s 406ms/step - loss: 0.8905 - acc: 0.7913 - val_loss: 1.0207 - val_acc: 0.7525\n",
            "Epoch 32/100\n",
            "10/10 [==============================] - 1s 135ms/step - loss: 1.1512 - acc: 0.7148\n",
            "49/49 [==============================] - 20s 410ms/step - loss: 0.8854 - acc: 0.7916 - val_loss: 1.1512 - val_acc: 0.7148\n",
            "Epoch 33/100\n",
            "10/10 [==============================] - 1s 133ms/step - loss: 0.9648 - acc: 0.7658\n",
            "49/49 [==============================] - 20s 412ms/step - loss: 0.8753 - acc: 0.7945 - val_loss: 0.9648 - val_acc: 0.7658\n",
            "Epoch 34/100\n",
            "10/10 [==============================] - 1s 134ms/step - loss: 1.2502 - acc: 0.7095\n",
            "49/49 [==============================] - 20s 413ms/step - loss: 0.8590 - acc: 0.7986 - val_loss: 1.2502 - val_acc: 0.7095\n",
            "Epoch 35/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 1.2458 - acc: 0.6907\n",
            "49/49 [==============================] - 20s 405ms/step - loss: 0.8419 - acc: 0.8026 - val_loss: 1.2458 - val_acc: 0.6907\n",
            "Epoch 36/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 0.9138 - acc: 0.7827\n",
            "49/49 [==============================] - 20s 417ms/step - loss: 0.8349 - acc: 0.8048 - val_loss: 0.9138 - val_acc: 0.7827\n",
            "Epoch 37/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 1.0212 - acc: 0.7513\n",
            "49/49 [==============================] - 20s 409ms/step - loss: 0.8277 - acc: 0.8065 - val_loss: 1.0212 - val_acc: 0.7513\n",
            "Epoch 38/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 1.0976 - acc: 0.7358\n",
            "49/49 [==============================] - 20s 408ms/step - loss: 0.8173 - acc: 0.8093 - val_loss: 1.0976 - val_acc: 0.7358\n",
            "Epoch 39/100\n",
            "10/10 [==============================] - 1s 138ms/step - loss: 1.0074 - acc: 0.7567\n",
            "49/49 [==============================] - 20s 409ms/step - loss: 0.8058 - acc: 0.8141 - val_loss: 1.0074 - val_acc: 0.7567\n",
            "Epoch 40/100\n",
            "10/10 [==============================] - 1s 134ms/step - loss: 1.0679 - acc: 0.7363\n",
            "49/49 [==============================] - 20s 411ms/step - loss: 0.7910 - acc: 0.8180 - val_loss: 1.0679 - val_acc: 0.7363\n",
            "Epoch 41/100\n",
            "10/10 [==============================] - 1s 133ms/step - loss: 0.8873 - acc: 0.7915\n",
            "49/49 [==============================] - 20s 407ms/step - loss: 0.7808 - acc: 0.8215 - val_loss: 0.8873 - val_acc: 0.7915\n",
            "Epoch 42/100\n",
            "10/10 [==============================] - 1s 141ms/step - loss: 1.0063 - acc: 0.7543\n",
            "49/49 [==============================] - 20s 408ms/step - loss: 0.7766 - acc: 0.8212 - val_loss: 1.0063 - val_acc: 0.7543\n",
            "Epoch 43/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 1.0038 - acc: 0.7595\n",
            "49/49 [==============================] - 20s 405ms/step - loss: 0.7689 - acc: 0.8232 - val_loss: 1.0038 - val_acc: 0.7595\n",
            "Epoch 44/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 0.9407 - acc: 0.7785\n",
            "49/49 [==============================] - 20s 407ms/step - loss: 0.7552 - acc: 0.8289 - val_loss: 0.9407 - val_acc: 0.7785\n",
            "Epoch 45/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 0.9503 - acc: 0.7723\n",
            "49/49 [==============================] - 20s 406ms/step - loss: 0.7461 - acc: 0.8315 - val_loss: 0.9503 - val_acc: 0.7723\n",
            "Epoch 46/100\n",
            "10/10 [==============================] - 1s 133ms/step - loss: 0.8446 - acc: 0.8058\n",
            "49/49 [==============================] - 20s 409ms/step - loss: 0.7398 - acc: 0.8315 - val_loss: 0.8446 - val_acc: 0.8058\n",
            "Epoch 47/100\n",
            "10/10 [==============================] - 1s 128ms/step - loss: 0.8945 - acc: 0.7907\n",
            "49/49 [==============================] - 20s 406ms/step - loss: 0.7404 - acc: 0.8314 - val_loss: 0.8945 - val_acc: 0.7907\n",
            "Epoch 48/100\n",
            "10/10 [==============================] - 1s 122ms/step - loss: 0.9399 - acc: 0.7784\n",
            "49/49 [==============================] - 20s 403ms/step - loss: 0.7235 - acc: 0.8376 - val_loss: 0.9399 - val_acc: 0.7784\n",
            "Epoch 49/100\n",
            "10/10 [==============================] - 1s 126ms/step - loss: 1.0394 - acc: 0.7523\n",
            "49/49 [==============================] - 20s 404ms/step - loss: 0.7103 - acc: 0.8408 - val_loss: 1.0394 - val_acc: 0.7523\n",
            "Epoch 50/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 0.9245 - acc: 0.7724\n",
            "49/49 [==============================] - 20s 402ms/step - loss: 0.7075 - acc: 0.8399 - val_loss: 0.9245 - val_acc: 0.7724\n",
            "Epoch 51/100\n",
            "10/10 [==============================] - 1s 137ms/step - loss: 0.8313 - acc: 0.8059\n",
            "49/49 [==============================] - 20s 410ms/step - loss: 0.7105 - acc: 0.8402 - val_loss: 0.8313 - val_acc: 0.8059\n",
            "Epoch 52/100\n",
            "10/10 [==============================] - 1s 128ms/step - loss: 0.7599 - acc: 0.8295\n",
            "49/49 [==============================] - 20s 405ms/step - loss: 0.6551 - acc: 0.8598 - val_loss: 0.7599 - val_acc: 0.8295\n",
            "Epoch 53/100\n",
            "10/10 [==============================] - 1s 135ms/step - loss: 0.7518 - acc: 0.8318\n",
            "49/49 [==============================] - 20s 405ms/step - loss: 0.6464 - acc: 0.8619 - val_loss: 0.7518 - val_acc: 0.8318\n",
            "Epoch 54/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 0.7371 - acc: 0.8390\n",
            "49/49 [==============================] - 20s 400ms/step - loss: 0.6431 - acc: 0.8637 - val_loss: 0.7371 - val_acc: 0.8390\n",
            "Epoch 55/100\n",
            "10/10 [==============================] - 1s 130ms/step - loss: 0.7453 - acc: 0.8348\n",
            "49/49 [==============================] - 20s 404ms/step - loss: 0.6406 - acc: 0.8653 - val_loss: 0.7453 - val_acc: 0.8348\n",
            "Epoch 56/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 0.7685 - acc: 0.8288\n",
            "49/49 [==============================] - 20s 405ms/step - loss: 0.6370 - acc: 0.8663 - val_loss: 0.7685 - val_acc: 0.8288\n",
            "Epoch 57/100\n",
            "10/10 [==============================] - 1s 124ms/step - loss: 0.7342 - acc: 0.8380\n",
            "49/49 [==============================] - 20s 404ms/step - loss: 0.6348 - acc: 0.8647 - val_loss: 0.7342 - val_acc: 0.8380\n",
            "Epoch 58/100\n",
            "10/10 [==============================] - 1s 128ms/step - loss: 0.7682 - acc: 0.8261\n",
            "49/49 [==============================] - 20s 406ms/step - loss: 0.6333 - acc: 0.8638 - val_loss: 0.7682 - val_acc: 0.8261\n",
            "Epoch 59/100\n",
            "10/10 [==============================] - 1s 127ms/step - loss: 0.7439 - acc: 0.8352\n",
            "49/49 [==============================] - 20s 409ms/step - loss: 0.6325 - acc: 0.8653 - val_loss: 0.7439 - val_acc: 0.8352\n",
            "Epoch 60/100\n",
            "10/10 [==============================] - 1s 132ms/step - loss: 0.7630 - acc: 0.8302\n",
            "49/49 [==============================] - 20s 406ms/step - loss: 0.6290 - acc: 0.8681 - val_loss: 0.7630 - val_acc: 0.8302\n",
            "Epoch 61/100\n",
            "10/10 [==============================] - 1s 133ms/step - loss: 0.7497 - acc: 0.8335\n",
            "49/49 [==============================] - 20s 411ms/step - loss: 0.6251 - acc: 0.8702 - val_loss: 0.7497 - val_acc: 0.8335\n",
            "Epoch 62/100\n",
            "10/10 [==============================] - 1s 135ms/step - loss: 0.7398 - acc: 0.8365\n",
            "49/49 [==============================] - 20s 410ms/step - loss: 0.6283 - acc: 0.8676 - val_loss: 0.7398 - val_acc: 0.8365\n",
            "Epoch 63/100\n",
            "10/10 [==============================] - 1s 130ms/step - loss: 0.7568 - acc: 0.8276\n",
            "49/49 [==============================] - 20s 405ms/step - loss: 0.6238 - acc: 0.8689 - val_loss: 0.7568 - val_acc: 0.8276\n",
            "Epoch 64/100\n",
            "10/10 [==============================] - 1s 130ms/step - loss: 0.7588 - acc: 0.8304\n",
            "49/49 [==============================] - 20s 408ms/step - loss: 0.6249 - acc: 0.8691 - val_loss: 0.7588 - val_acc: 0.8304\n",
            "Epoch 65/100\n",
            "10/10 [==============================] - 1s 131ms/step - loss: 0.7549 - acc: 0.8324\n",
            "49/49 [==============================] - 20s 406ms/step - loss: 0.6238 - acc: 0.8686 - val_loss: 0.7549 - val_acc: 0.8324\n",
            "Epoch 66/100\n",
            "10/10 [==============================] - 1s 130ms/step - loss: 0.7465 - acc: 0.8361\n",
            "49/49 [==============================] - 20s 406ms/step - loss: 0.6223 - acc: 0.8698 - val_loss: 0.7465 - val_acc: 0.8361\n",
            "Epoch 67/100\n",
            "10/10 [==============================] - 1s 148ms/step - loss: 0.7475 - acc: 0.8357\n",
            "49/49 [==============================] - 20s 418ms/step - loss: 0.6199 - acc: 0.8705 - val_loss: 0.7475 - val_acc: 0.8357\n",
            "Epoch 68/100\n",
            "10/10 [==============================] - 1s 131ms/step - loss: 0.7376 - acc: 0.8360\n",
            "49/49 [==============================] - 20s 405ms/step - loss: 0.6196 - acc: 0.8719 - val_loss: 0.7376 - val_acc: 0.8360\n",
            "Epoch 69/100\n",
            "10/10 [==============================] - 1s 130ms/step - loss: 0.7572 - acc: 0.8321\n",
            "49/49 [==============================] - 20s 408ms/step - loss: 0.6154 - acc: 0.8714 - val_loss: 0.7572 - val_acc: 0.8321\n",
            "Epoch 70/100\n",
            "10/10 [==============================] - 1s 130ms/step - loss: 0.7442 - acc: 0.8333\n",
            "49/49 [==============================] - 20s 407ms/step - loss: 0.6173 - acc: 0.8699 - val_loss: 0.7442 - val_acc: 0.8333\n",
            "Epoch 71/100\n",
            "10/10 [==============================] - 1s 128ms/step - loss: 0.7300 - acc: 0.8386\n",
            "49/49 [==============================] - 20s 407ms/step - loss: 0.6161 - acc: 0.8720 - val_loss: 0.7300 - val_acc: 0.8386\n",
            "Epoch 72/100\n",
            "10/10 [==============================] - 1s 126ms/step - loss: 0.7484 - acc: 0.8352\n",
            "49/49 [==============================] - 20s 407ms/step - loss: 0.6166 - acc: 0.8699 - val_loss: 0.7484 - val_acc: 0.8352\n",
            "Epoch 73/100\n",
            "10/10 [==============================] - 1s 130ms/step - loss: 0.7372 - acc: 0.8363\n",
            "49/49 [==============================] - 20s 403ms/step - loss: 0.6151 - acc: 0.8714 - val_loss: 0.7372 - val_acc: 0.8363\n",
            "Epoch 74/100\n",
            "10/10 [==============================] - 1s 125ms/step - loss: 0.7481 - acc: 0.8340\n",
            "49/49 [==============================] - 20s 405ms/step - loss: 0.6123 - acc: 0.8731 - val_loss: 0.7481 - val_acc: 0.8340\n",
            "Epoch 75/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 0.7466 - acc: 0.8311\n",
            "49/49 [==============================] - 20s 402ms/step - loss: 0.6108 - acc: 0.8731 - val_loss: 0.7466 - val_acc: 0.8311\n",
            "Epoch 76/100\n",
            "10/10 [==============================] - 1s 128ms/step - loss: 0.7349 - acc: 0.8355\n",
            "49/49 [==============================] - 20s 402ms/step - loss: 0.6089 - acc: 0.8738 - val_loss: 0.7349 - val_acc: 0.8355\n",
            "Epoch 77/100\n",
            "10/10 [==============================] - 1s 128ms/step - loss: 0.7366 - acc: 0.8354\n",
            "49/49 [==============================] - 20s 404ms/step - loss: 0.6061 - acc: 0.8741 - val_loss: 0.7366 - val_acc: 0.8354\n",
            "Epoch 78/100\n",
            "10/10 [==============================] - 1s 130ms/step - loss: 0.7325 - acc: 0.8373\n",
            "49/49 [==============================] - 20s 406ms/step - loss: 0.6075 - acc: 0.8732 - val_loss: 0.7325 - val_acc: 0.8373\n",
            "Epoch 79/100\n",
            "10/10 [==============================] - 1s 132ms/step - loss: 0.7387 - acc: 0.8356\n",
            "49/49 [==============================] - 20s 408ms/step - loss: 0.6051 - acc: 0.8742 - val_loss: 0.7387 - val_acc: 0.8356\n",
            "Epoch 80/100\n",
            "10/10 [==============================] - 1s 125ms/step - loss: 0.7358 - acc: 0.8369\n",
            "49/49 [==============================] - 20s 405ms/step - loss: 0.6035 - acc: 0.8743 - val_loss: 0.7358 - val_acc: 0.8369\n",
            "Epoch 81/100\n",
            "10/10 [==============================] - 1s 133ms/step - loss: 0.7356 - acc: 0.8369\n",
            "49/49 [==============================] - 20s 406ms/step - loss: 0.6029 - acc: 0.8754 - val_loss: 0.7356 - val_acc: 0.8369\n",
            "Epoch 82/100\n",
            "10/10 [==============================] - 1s 132ms/step - loss: 0.7388 - acc: 0.8369\n",
            "49/49 [==============================] - 20s 402ms/step - loss: 0.6050 - acc: 0.8749 - val_loss: 0.7388 - val_acc: 0.8369\n",
            "Epoch 83/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 0.7358 - acc: 0.8361\n",
            "49/49 [==============================] - 20s 406ms/step - loss: 0.6021 - acc: 0.8746 - val_loss: 0.7358 - val_acc: 0.8361\n",
            "Epoch 84/100\n",
            "10/10 [==============================] - 1s 126ms/step - loss: 0.7367 - acc: 0.8370\n",
            "49/49 [==============================] - 20s 401ms/step - loss: 0.6013 - acc: 0.8755 - val_loss: 0.7367 - val_acc: 0.8370\n",
            "Epoch 85/100\n",
            "10/10 [==============================] - 1s 132ms/step - loss: 0.7377 - acc: 0.8375\n",
            "49/49 [==============================] - 20s 405ms/step - loss: 0.6063 - acc: 0.8728 - val_loss: 0.7377 - val_acc: 0.8375\n",
            "Epoch 86/100\n",
            "10/10 [==============================] - 1s 127ms/step - loss: 0.7350 - acc: 0.8371\n",
            "49/49 [==============================] - 20s 403ms/step - loss: 0.6044 - acc: 0.8753 - val_loss: 0.7350 - val_acc: 0.8371\n",
            "Epoch 87/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 0.7364 - acc: 0.8375\n",
            "49/49 [==============================] - 20s 403ms/step - loss: 0.6052 - acc: 0.8732 - val_loss: 0.7364 - val_acc: 0.8375\n",
            "Epoch 88/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 0.7332 - acc: 0.8370\n",
            "49/49 [==============================] - 20s 405ms/step - loss: 0.6029 - acc: 0.8739 - val_loss: 0.7332 - val_acc: 0.8370\n",
            "Epoch 89/100\n",
            "10/10 [==============================] - 1s 125ms/step - loss: 0.7347 - acc: 0.8383\n",
            "49/49 [==============================] - 20s 406ms/step - loss: 0.6002 - acc: 0.8758 - val_loss: 0.7347 - val_acc: 0.8383\n",
            "Epoch 90/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 0.7350 - acc: 0.8374\n",
            "49/49 [==============================] - 20s 402ms/step - loss: 0.6037 - acc: 0.8753 - val_loss: 0.7350 - val_acc: 0.8374\n",
            "Epoch 91/100\n",
            "10/10 [==============================] - 1s 134ms/step - loss: 0.7340 - acc: 0.8369\n",
            "49/49 [==============================] - 20s 404ms/step - loss: 0.6033 - acc: 0.8757 - val_loss: 0.7340 - val_acc: 0.8369\n",
            "Epoch 92/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 0.7359 - acc: 0.8379\n",
            "49/49 [==============================] - 20s 402ms/step - loss: 0.6016 - acc: 0.8774 - val_loss: 0.7359 - val_acc: 0.8379\n",
            "Epoch 93/100\n",
            "10/10 [==============================] - 1s 129ms/step - loss: 0.7345 - acc: 0.8378\n",
            "49/49 [==============================] - 20s 401ms/step - loss: 0.6038 - acc: 0.8756 - val_loss: 0.7345 - val_acc: 0.8378\n",
            "Epoch 94/100\n",
            "10/10 [==============================] - 1s 130ms/step - loss: 0.7411 - acc: 0.8351\n",
            "49/49 [==============================] - 20s 404ms/step - loss: 0.6018 - acc: 0.8764 - val_loss: 0.7411 - val_acc: 0.8351\n",
            "Epoch 95/100\n",
            "10/10 [==============================] - 1s 137ms/step - loss: 0.7336 - acc: 0.8387\n",
            "49/49 [==============================] - 20s 406ms/step - loss: 0.6023 - acc: 0.8756 - val_loss: 0.7336 - val_acc: 0.8387\n",
            "Epoch 96/100\n",
            "10/10 [==============================] - 1s 130ms/step - loss: 0.7333 - acc: 0.8376\n",
            "49/49 [==============================] - 20s 408ms/step - loss: 0.6020 - acc: 0.8759 - val_loss: 0.7333 - val_acc: 0.8376\n",
            "Epoch 97/100\n",
            "10/10 [==============================] - 1s 131ms/step - loss: 0.7317 - acc: 0.8376\n",
            "49/49 [==============================] - 20s 409ms/step - loss: 0.6033 - acc: 0.8734 - val_loss: 0.7317 - val_acc: 0.8376\n",
            "Epoch 98/100\n",
            "10/10 [==============================] - 1s 138ms/step - loss: 0.7357 - acc: 0.8377\n",
            "49/49 [==============================] - 20s 412ms/step - loss: 0.6026 - acc: 0.8768 - val_loss: 0.7357 - val_acc: 0.8377\n",
            "Epoch 99/100\n",
            "10/10 [==============================] - 1s 131ms/step - loss: 0.7318 - acc: 0.8374\n",
            "49/49 [==============================] - 20s 413ms/step - loss: 0.6005 - acc: 0.8768 - val_loss: 0.7318 - val_acc: 0.8374\n",
            "Epoch 100/100\n",
            "10/10 [==============================] - 1s 128ms/step - loss: 0.7328 - acc: 0.8369\n",
            "49/49 [==============================] - 20s 405ms/step - loss: 0.6018 - acc: 0.8765 - val_loss: 0.7328 - val_acc: 0.8369\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}