mnist_cnn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MNIST CNN\n",
    "\n",
    "A Convolutional Network implementation example using TensorFlow library.\n",
    "This example is using the MNIST database of handwritten digits\n",
    "(http://yann.lecun.com/exdb/mnist/)\n",
    "\n",
    "Author: Aymeric Damien\n",
    "Project: https://github.com/aymericdamien/TensorFlow-Examples/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import shutil\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU available.\n",
      "Specs: \n",
      "\n",
      "Fri Sep  7 22:13:43 2018       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 396.54                 Driver Version: 396.54                    |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  Tesla K80           Off  | 00000000:00:1E.0 Off |                    0 |\n",
      "| N/A   61C    P0    85W / 149W |  10876MiB / 11441MiB |      5%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                       GPU Memory |\n",
      "|  GPU       PID   Type   Process name                             Usage      |\n",
      "|=============================================================================|\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "if tf.test.is_gpu_available():\n",
    "    print(\"GPU available.\\nSpecs: \\n\")\n",
    "    !nvidia-smi\n",
    "else:\n",
    "    print(\"No GPU found... Defaulting to CPU.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /floyd/input/mnist/train-images-idx3-ubyte.gz\n",
      "Extracting /floyd/input/mnist/train-labels-idx1-ubyte.gz\n",
      "Extracting /floyd/input/mnist/t10k-images-idx3-ubyte.gz\n",
      "Extracting /floyd/input/mnist/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import MNIST data\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/floyd/input/mnist\", one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "learning_rate = 0.001\n",
    "training_iters = 200000\n",
    "batch_size = 128\n",
    "display_step = 10\n",
    "\n",
    "# Network Parameters\n",
    "n_input = 784 # MNIST data input (img shape: 28*28)\n",
    "n_classes = 10 # MNIST total classes (0-9 digits)\n",
    "dropout = 0.75 # Dropout, probability to keep units\n",
    "\n",
    "logs_path = './logs/'\n",
    "\n",
    "# tf Graph input\n",
    "x = tf.placeholder(tf.float32, [None, n_input], name=\"x\")\n",
    "y = tf.placeholder(tf.float32, [None, n_classes], name=\"y\")\n",
    "keep_prob = tf.placeholder(tf.float32, name=\"keep_prob\") #dropout (keep probability)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create some wrappers for simplicity\n",
    "def conv2d(x, W, b, strides=1):\n",
    "    # Conv2D wrapper, with bias and relu activation\n",
    "    x = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding='SAME')\n",
    "    x = tf.nn.bias_add(x, b)\n",
    "    return tf.nn.relu(x)\n",
    "\n",
    "\n",
    "def maxpool2d(x, k=2):\n",
    "    # MaxPool2D wrapper\n",
    "    return tf.nn.max_pool(x, ksize=[1, k, k, 1], strides=[1, k, k, 1],\n",
    "                          padding='SAME')\n",
    "\n",
    "\n",
    "# Create model\n",
    "def conv_net(x, weights, biases, dropout):\n",
    "    # Reshape input picture\n",
    "    x = tf.reshape(x, shape=[-1, 28, 28, 1])\n",
    "\n",
    "    # Convolution Layer\n",
    "    conv1 = conv2d(x, weights['wc1'], biases['bc1'])\n",
    "    # Max Pooling (down-sampling)\n",
    "    conv1 = maxpool2d(conv1, k=2)\n",
    "\n",
    "    # Convolution Layer\n",
    "    conv2 = conv2d(conv1, weights['wc2'], biases['bc2'])\n",
    "    # Max Pooling (down-sampling)\n",
    "    conv2 = maxpool2d(conv2, k=2)\n",
    "\n",
    "    # Fully connected layer\n",
    "    # Reshape conv2 output to fit fully connected layer input\n",
    "    fc1 = tf.reshape(conv2, [-1, weights['wd1'].get_shape().as_list()[0]])\n",
    "    fc1 = tf.add(tf.matmul(fc1, weights['wd1']), biases['bd1'])\n",
    "    fc1 = tf.nn.relu(fc1)\n",
    "    # Apply Dropout\n",
    "    fc1 = tf.nn.dropout(fc1, dropout)\n",
    "\n",
    "    # Output, class prediction\n",
    "    out = tf.add(tf.matmul(fc1, weights['out']), biases['out'])\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-6-0077e4435775>:24: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Store layers weight & bias\n",
    "weights = {\n",
    "    # 5x5 conv, 1 input, 32 outputs\n",
    "    'wc1': tf.Variable(tf.random_normal([5, 5, 1, 32])),\n",
    "    # 5x5 conv, 32 inputs, 64 outputs\n",
    "    'wc2': tf.Variable(tf.random_normal([5, 5, 32, 64])),\n",
    "    # fully connected, 7*7*64 inputs, 1024 outputs\n",
    "    'wd1': tf.Variable(tf.random_normal([7*7*64, 1024])),\n",
    "    # 1024 inputs, 10 outputs (class prediction)\n",
    "    'out': tf.Variable(tf.random_normal([1024, n_classes]))\n",
    "}\n",
    "\n",
    "biases = {\n",
    "    'bc1': tf.Variable(tf.random_normal([32])),\n",
    "    'bc2': tf.Variable(tf.random_normal([64])),\n",
    "    'bd1': tf.Variable(tf.random_normal([1024])),\n",
    "    'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "}\n",
    "\n",
    "# Construct model\n",
    "pred = conv_net(x, weights, biases, keep_prob)\n",
    "\n",
    "# Define loss and optimizer\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n",
    "\n",
    "# Evaluate model\n",
    "correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32), name=\"accuracy\")\n",
    "\n",
    "# Create a summary to monitor cost tensor\n",
    "tf.summary.scalar(\"loss\", cost)\n",
    "# Create a summary to monitor accuracy tensor\n",
    "tf.summary.scalar(\"accuracy\", accuracy)\n",
    "# Merge all summaries into a single op\n",
    "merged_summary_op = tf.summary.merge_all()\n",
    "\n",
    "# Delete ./model dir if exists to avoid SavedModelBuilder error\n",
    "if os.path.exists('./model'):\n",
    "    shutil.rmtree('./model', ignore_errors=True)\n",
    "    \n",
    "# Initializing the variables\n",
    "init = tf.global_variables_initializer()\n",
    "builder = tf.saved_model.builder.SavedModelBuilder(\"./model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"metric\": \"Minibatch loss\", \"value\": 14612.599609}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.39062}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 9827.154297}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.60938}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 3431.417969}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.75781}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 5153.578613}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.75781}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 2701.952393}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.85156}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 4444.277344}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.75781}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 2107.426758}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.85156}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 2121.080811}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.84375}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 1686.530273}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.87500}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 2615.851562}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.86719}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 2157.554688}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.89062}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 2353.160156}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.87500}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 1957.043091}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.92188}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 1128.000732}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.90625}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 1702.037354}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.90625}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 1243.490845}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.87500}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 533.521240}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.93750}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 1668.980713}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.87500}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 1143.786255}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.92188}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 470.046692}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.92969}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 829.994873}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.92969}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 1141.459717}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.91406}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 1353.867554}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.91406}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 343.723602}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.93750}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 591.427979}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 754.989990}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.94531}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 468.836426}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.93750}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 859.823059}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.93750}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 1214.445435}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 1561.179688}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.90625}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 695.835571}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.92969}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 326.790039}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.97656}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 575.396790}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.93750}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 1183.162109}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.93750}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 367.617310}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.94531}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 733.932739}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 897.733887}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.93750}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 510.103485}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 1474.493896}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.92188}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 724.748901}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 368.277252}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.93750}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 464.268616}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 381.701294}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 369.268433}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.92969}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 861.415161}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.93750}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 545.133179}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.93750}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 560.594604}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.94531}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 1016.210510}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.93750}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 160.677246}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 58.048431}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.98438}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 233.581833}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 589.006531}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.92188}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 589.754395}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.94531}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 634.583923}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.93750}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 77.011154}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.98438}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 313.237579}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 790.818726}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.94531}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 239.733597}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 512.488281}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 277.907043}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.92969}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 528.972351}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.94531}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 47.223167}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.98438}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 857.834839}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.92969}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 414.263763}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 210.532333}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 486.151184}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.94531}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 119.391052}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 389.152985}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 574.559204}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.93750}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 295.246643}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96094}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 235.132141}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.97656}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 617.503113}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.92188}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 34.002609}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.98438}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 354.495392}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 294.217285}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96094}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 350.603577}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.94531}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 451.157990}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.92969}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 245.821075}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 854.932983}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.92969}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 291.048309}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 128.316635}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 318.904663}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.97656}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 73.215683}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 549.928711}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 397.128235}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 67.854401}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 58.947678}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.98438}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 237.991959}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.94531}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 418.569153}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.93750}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 390.227264}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 342.484802}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96094}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 308.402100}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 402.612976}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.94531}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 113.961357}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.98438}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 227.303543}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96094}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 142.668823}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.97656}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 163.173233}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 222.267212}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 167.343567}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.97656}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 52.208576}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.97656}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 296.749847}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96094}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 262.259155}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.97656}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 239.454758}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 472.043671}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96094}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 24.822357}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.99219}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 248.111115}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.97656}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 370.727600}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 67.517258}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.98438}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 114.689056}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.97656}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 129.667618}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96094}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 364.676147}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.94531}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 174.290710}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 342.932739}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 13.420685}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.99219}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 124.369690}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.97656}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 20.484772}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.99219}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 500.841553}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 351.654419}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 125.683792}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.97656}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 263.697906}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 12.862526}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.98438}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 229.709000}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 285.007263}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 178.308243}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 11.162933}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.99219}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 343.702362}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.94531}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 94.929268}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 142.603760}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.97656}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 246.369812}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 55.173767}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.99219}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 0.000000}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 1.00000}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 69.841583}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.98438}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 43.601883}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 10.909302}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.98438}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 330.245056}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 82.224190}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.97656}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 231.812256}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 121.648491}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 113.121933}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.98438}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 87.285255}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96094}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 54.510612}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.98438}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 17.214996}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.99219}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 70.142883}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.99219}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 152.893066}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.98438}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 121.996819}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.97656}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 86.053932}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.97656}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 87.382935}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 302.879456}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96094}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 196.001877}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 235.883408}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 67.857117}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.97656}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 273.333832}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96094}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 137.479309}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 442.456177}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.95312}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 57.409439}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "{\"metric\": \"Minibatch loss\", \"value\": 188.089813}\n",
      "{\"metric\": \"Training accuracy\", \"value\": 0.96875}\n",
      "Optimization Finished!\n",
      "INFO:tensorflow:No assets to save.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: b'./model/saved_model.pb'\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "b'./model/saved_model.pb'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Launch the graph\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    \n",
    "    # op to write logs to Tensorboard\n",
    "    summary_writer = tf.summary.FileWriter(logs_path, graph=tf.get_default_graph())\n",
    "    \n",
    "    step = 1\n",
    "    # Keep training until reach max iterations\n",
    "    while step * batch_size < training_iters:\n",
    "        batch_x, batch_y = mnist.train.next_batch(batch_size)\n",
    "        \n",
    "        # Run optimization op (backprop), cost op (to get loss value) and summary nodes\n",
    "        _, c, summary = sess.run([optimizer, cost, merged_summary_op], \n",
    "                                    feed_dict={x: batch_x, y: batch_y, keep_prob: dropout})\n",
    "        \n",
    "        # Write logs at every iteration\n",
    "        summary_writer.add_summary(summary, step * batch_size)\n",
    "        \n",
    "        if step % display_step == 0:\n",
    "            # Calculate batch loss and accuracy\n",
    "            loss, acc = sess.run([cost, accuracy], feed_dict={x: batch_x,\n",
    "                                                              y: batch_y,\n",
    "                                                              keep_prob: 1.})\n",
    "            # print(\"Iter \" + str(step*batch_size) + \", Minibatch Loss= \" + \\\n",
    "            #      \"{:.6f}\".format(loss) + \", Training Accuracy= \" + \\\n",
    "            #      \"{:.5f}\".format(acc))\n",
    "            print('{{\"metric\": \"Minibatch loss\", \"value\": {:.6f}}}'.format(loss))\n",
    "            print('{{\"metric\": \"Training accuracy\", \"value\": {:.5f}}}'.format(acc))\n",
    "        step += 1\n",
    "    print(\"Optimization Finished!\")\n",
    "\n",
    "    builder.add_meta_graph_and_variables(sess, [\"EVALUATING\"])\n",
    "\n",
    "builder.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Accuracy: 0.1015625\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    \n",
    "    # Calculate accuracy for 256 mnist test images\n",
    "    print(\"Testing Accuracy:\", sess.run(accuracy, feed_dict={x: mnist.test.images[:256], y: mnist.test.labels[:256], keep_prob: 1.}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}