11_deep_learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPython 3.6.6\n",
      "IPython 6.5.0\n",
      "\n",
      "numpy 1.15.2\n",
      "sklearn 0.20.0\n",
      "scipy 1.1.0\n",
      "matplotlib 2.2.3\n",
      "tensorflow 1.12.0\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -v -p numpy,sklearn,scipy,matplotlib,tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**11장 – 심층 신경망 훈련**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_이 노트북은 11장에 있는 모든 샘플 코드를 가지고 있습니다._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 설정"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "파이썬 2와 3을 모두 지원합니다. 공통 모듈을 임포트하고 맷플롯립 그림이 노트북 안에 포함되도록 설정하고 생성한 그림을 저장하기 위한 함수를 준비합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 파이썬 2와 파이썬 3 지원\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# 공통\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# 일관된 출력을 위해 유사난수 초기화\n",
    "def reset_graph(seed=42):\n",
    "    tf.reset_default_graph()\n",
    "    tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "# 맷플롯립 설정\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# 한글출력\n",
    "plt.rcParams['font.family'] = 'NanumBarunGothic'\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "\n",
    "# 그림을 저장할 폴더\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"deep\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 그래디언트 소실/폭주 문제"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def logit(z):\n",
    "    return 1 / (1 + np.exp(-z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzs3Xd4VEXbwOHfpGeTkGCA0KsIojRpUhNqIHQUsKBiAwQV+F54lSLSVCxgwQZIeaVYKIpIFUgAAUWq9BKpEjohpCeb+f44m5CyKbBJNuW5r2sus2fmnPPsusnDzJkzR2mtEUIIIQoaB3sHIIQQQlgjCUoIIUSBJAlKCCFEgSQJSgghRIEkCUoIIUSBJAlKFAtKKZXq5xCl1Kg8Pl+W51BKjVVKnUlVvkxV97lSasFdnu+u9lFKLVBKTcuivr1SKjxVuZSqrqpSSiulSmWxv1JKRSmlnszxmxAiHUlQosBTSu22/EHMrDRO1fa/SqnEdEUDb97F+Z5I/QfZSv3ATOL4PQfHrqGUOgMMSlcVZElUj2az/5icnMfSdmO6+DJNSOlprTdprX1SlbI53deiM2ACDt/lfkKkcLJ3AELkQACZf1dvpns9HfjM8rMGzMBV4PhdnK8GcDmbNqFa6/vv4phGQFqHAlWVUj5AS8AHiAB2aq2vASilBmRxiLLAWUs7B6CEZbtLcgOlVBWgCTAXWJBq30Sl1OPAzqxiVEqFZ1E9jzufb2b7VwS+Aq4Anymlemqtb2W1jxDWSIISBZ7WOjKzulQjd8ltzRhJKbn+UcAD2HIXp+wLPKyUelRr/UcmbaorpWLTbZuqtZ6a3cGVUvWA7cAl4F+gjOV4HbTWyb0jF0sSi9Fax1n2cwS6ArFKKU+gKnAw1aH/Z/lvSeDhLEI4klV8Wmsfy5CoP1AbI4Fu01qft8RR1dK0hFIqEbilLXf8WxLg18A6YDCwCPhbKTUe+E5rnZjVuYVITYb4RIFnGeIzWxm6S8RIRlkth/I6sEJrfT2H5/oPUAoYCXxn6Q1Y84/W2i1dyTY5WZTH6Pk10loHAI2BfzASTrInLW1Gptr2FnALI7ktAo5prZXWWgFfJDfSWu/XWk8EYoCngFeBARif02StdZYJSinlCoQAb2P00OoDu5VSz6ZrGmqJ0U8p1V8pdQL4EhivtR6gtY7SWvcGJgPvAJesHEOITEkPShQWw7XWn9/NDkqpZkB/ICgHbV2AscAIIEhrvcMyVLZFKfWk1npXFvu6A96AH1APOJ3VubTW65RSzwMTLf/9DuMa2SqllMnS7H9a64GW4zsB44BhQCuMxLAU2KqUGqq13m8lpoEYybmd1vq4UqoGsBFj2O0rSzNHpZQbkJiuZ+NveR+lk7crpUItMX6bql3pVMOS+zGS0DKtdVS69ztXKfU/jKHaDLEKkRnpQYnCwkEp5ZRJyfA9VkqVxvjDfwqYZvlDbJVSajBGUukFtNFa7wDQWv8Ho2eyWSm1xPJHPlmN5MkHwHXgAMYf70Ag7bhj2nN9qJQ6BHyKMVznA3TDuHb2N/Cxld1mAe0welzHtdaJlp7JN5ZjWFMH+EtrfdzyXkIxel6ph/5GYfSyPkq37z+AG9BDKeWmlCqHkeSPZfa+LHH9L31ySlWfqLXemJzQhMgRrbUUKQW6ALsxhqcyKwvStXcDdmD0MtyAvcBiwMlSHwKMStW+JdAPcMzk/FUwejCmu4h5JvCEle1eGEOIvhjDZ85W2jQAGqd6bQIcsjnf88ArqV43A6KB8RhJ7E0gCvC31C8ApmVxvEDL5/QvcAKj1+VrqStrqfO2vJ6Yzf+f9GWIvb9TUgpHUVrLauai8FBKjQBe0lpbnQSglPIDlltedtRaxyil7ge2YiSqvsBa4Fetdfqew93G4g44ZtEkTmudkMm+r2Ikj8z8oLUebmW/AcDLQC2gNEbS2QfM0lovSdf2YWAg8B+MXtjX2jIcqIx7pi5prTOdfm+ZiOHInWt8af6rLZNXLD3Y9L1YDyAc4/ragXR1Zi1/eEQOyBCfKDIs12/+AM5hSU4AWutTQBvL9sR0+zhmMXSYoaQ75U7gdhYlQ4JJZS7GcJu18jPGNa307+91jJ7ZfIxeljtGoloIfK2UeiV1e631ISB54sYc4KpSqr5SqkUWcaW2EbiBMREiHGOCRkTy+1OWG3W11knaGMJLKdyZSWlOXyfJSeSUTJIQBZZSKgTjgr21Omt/5B4EmmutM9xka0lSQy37pq7alNk5Mjnvg1rrY5ZjNsgm9qwMx5gll9n9VsusbAsCFmmtF6TaFgZ8o5RqYqn/Sin1C9DJUp/cw9uBkWiuANuyiS1NnDrd5BTLNPMsJ4IIkRskQYmCrBN318uPu4d/nbcni0kN6encvY9ni9a68120XwtMUErtwujdhGPc8xSIMVtxgqXdk4AzxlBcEhCbfqhR5XxZJHfL/ViplbDaUohcJglKFFha6/h8OIc5+1Z5poNSKrObkC9rrVPPGkRr/alllYeXgQ8wJltEYsz+e01rvdDSzupMunv0gaUIke9kkoQQeUAp1Rwjyfxj71iEKKxsniShlHJWSo1SSiUopZ7IpI2fUmqWUuqoUmqXUmqbUqqurecWoqDSWu+U5CSEbXJjFt/LGGPdma1ZBvAIsF5r/aDWuinGLKXpuXBuIYQQRZTN16C01l8CKKW6Z9FmbbpNYZmdWyk1CMujCNzd3RtVqlTJ1hBzVVJSEg4OMjs/J+Szyrnz58+jtaZy5cr2DqVQsNd3K9ocjcnRlH3DAqQg/h6eOHHimta6dHbt8n2ShOVGysnAi9bqtdazgdkAjRs31rt3787H6LIXEhJCQECAvcMoFOSzyrmAgADCw8PZv1+WqsuJ/Pxuzd07l7bV2lK9ZPV8OV9uK4i/h0qpszlpl69pVSnlC6wBJmqt7+bxB0IIke8WHljIS6te4qMdNi06Iu5RviUoy4KTm4DpWutF+XVeIYS4F2tPruWFX16gXbV2fBxobQ1fkdfyJEEppXyVUtuVUjUtr6tgJKcp6dcLE0KIgubPC3/y+NLHqedXj5/6/4Srk6u9QyqW8qoHZcJYATp5PbHpGM/KGa2U+sNSZIhPCFEgTdoyiXKe5Vjz1BpKuMrCGfaSa5MktPFk0OSfzwMVU71+PLfOI4QQee3Hvj9yPfo6fp5+9g6lWCtYcw+FEMJObsTc4NU1rxIZH4mniydVfKrYO6RiTxKUEKLYi06Ipsd3PZizdw4HLx+0dzjCQhaLFUIUa4lJifRf1p8d53fwY98faV6pub1DEhaSoIQQxZbWmkGrBvHriV/5qutXPF5HLpcXJDLEJ4Qoti7evsjaU2t52/9thjQeYu9wRDrSgxJCFFsVSlTgwJADlDZluyycsAPpQQkhip3vDn7H6A2jSdJJlPEog1I5fqiyyEeSoIQQxcqG0A089/Nz/HXxLxLMCfYOR2RBEpQQotj469+/6PNDH+qUrsPKJ1bKEkYFnCQoIUSxcOL6CYKWBFHGowxrn16Lt5t39jsJu5IEJYQoFo5dO4a7kzvrB6ynnFc5e4cjckBm8QkhijStNUopetTqQacanXBzcrN3SCKHpAclhCiyYhJi6LiwI0sOGk/5keRUuEiCEkIUSYlJiTy14ik2n96Mo3K0dzjiHsgQnxCiyNFaM3T1UH4+9jOfdf6M/g/3t3dI4h5ID0oIUeRMCJ7AnL1zGNd6HK81e83e4Yh7JAlKCFGkaK1J0km81PAlprSdYu9whA1kiE8IUWTEJsbi5uTGO+3fSZm9Jwov6UEJIYqEjf9spObMmikPHJTkVPhJghJCFHp7Lu6h9w+9KelWkkrelewdjsglkqCEEIXayesn6bK4C77uvqwbsA4fNx97hyRyiSQoIUShdSnyEoGLAtFoNjyzgfJe5e0dkshFkqCEEIVWCdcSNKnQhNVPreYB3wfsHY7IZTKLTwhR6MQnxRMRF0EJ1xL88PgP9g5H5BGbe1BKKWel1CilVIJS6olM2iil1BSl1HGl1BGl1CKllIet5xZCFD/mJDNTj06l7f/aEm+Ot3c4Ig/lRg/qZUADf2TR5jkgCGigtY5RSs0H3gdezerAx48fJyAgIM22fv36MXToUKKjowkKCsqwz8CBAxk4cCDXrl3j8ccfz1D/yiuv0L9/f86fP88zzzyTof4///kP3bt35/jx4wwePDhDfffu3QkICGD//v2MGDEiQ/27775LixYt2LFjB2PHjs1Q/8knn9CgQQM2btzI1KlTM9TPmjWLWrVqsWrVKqZPn56hfuHChVSqVIkffviBr776KkP9smXLKFWqFAsWLGDBggUZ6tesWYPJZOLLL7/kxx9/zFAfEhICwEcffcSvv/6aps7d3Z21a9cCMGXKFDZt2pSm3tfXl+XLlwMwZswY1q5di4/PnQvWFStWZNGiRQCMGDGC/fv3p9n/gQceYPbs2QAMGjSIEydOpKlv0KABn3zyCQADBgzgwoULaeqbN2/Oe++9B8Bjjz3G9evX09S3b9+et956C4AuXboQExOTpr5bt26MGjUKIMP3DvL2u7d//37uu+8+gEy/e+PHj6dDhw7F+runtebRKY+ym93UOFmDTks6ARm/ezt37kyzf3H+7oWHhzNmzBib/u7l9XcvMzYnKK31lwBKqe5ZNOsPzNJaJ/9f+RTYhJUEpZQaBAwCcHZ2Jjw8PE39iRMnCAkJITY2NkMdwLFjxwgJCeHWrVtW6w8fPkxISAhXrlyxWn/w4EG8vLw4d+6c1fqYmBhCQkI4deqU1fq9e/cSHx/PoUOHrNbv3r2b8PBwDhw4YLX+zz//JCwsjIMHD1qt37lzJ6GhoRw+fNhq/fbt2/H29ubYsWNW67du3YqbmxsnTpywWp/8RyI0NDRDffJ7Bzh9+nSG+qSkpJT6c+fOYTab07RxdnZOqb9w4UKG/S9evJhSf/HixQz1Fy5cSKm/fPlyhvpz586l1F+9epWIiIg09adPn06pv3HjBnFxcWnqQ0NDU+qtfTZ5+d1LTExM+fwy++4dOHAAJyenYv3dW3BmAbvZTaljpfA86kk4Rrv03730+xfn757ZbLb5715W3z2tYdeu/dy4YebAgZNcvuyB1m4kJblaihtz5lzlvvuOcOKEmVOnAoEtGc5hjdJa56hhtgdSKgT4Wmv9vZW648BQrfUmy2svIALw0VrfyuyYjRs31rt3786V+HJLSEiI1X/hiIzks8q5gIAAwsPDM/zLXtwxf998XvjlBZ5v8DzPlHiGtm3b2jukQsHa76HWEBkJN24Y5dYtuH3bKBER2f98+7axf0yMUZKS7jYqtUdr3Ti7Vvk1SUIB5lSvEy3/lVmEQogcaV+9Pa83fZ3pgdP5fevv9g6nQImOhkuXjHL5slGuXzeSz5EjtZgx404ySi4JCbl3ficncHc3isl052drr93dwcoIsfXj5l6IWboAVE71ujIQCWTsSwohRCrHrh3jAd8HqOxdmU+7fGrvcPJVYiJcvAjnzt0p//6bNhldumT0aDJn/fH2JhPcd59RvL2hRAnw8rpTUr+2VufhcSfhODtnfvaEhASc0zWwa4JSSvkCvwADtdYngYXAS0qp77XW8cBrwAqdW+OLQogiaV/YPvwX+PNq01d5t/279g4n1yUlwYULcOIEnDoFZ89mTEZmc/bHcXGBsmWN4udnlFKljORz5coxWrasnZKMkotbHj1c2Gw2s2fPHtasWcPy5cs5evQooaGhVKlS5a6PlVc9KBNQBfC2vP4WuB/YpZRKBI6QzQw+IUTxFnojlC6Lu+Dj5sPQJkPtHY5Nbt2CQ4fg6FE4edJISCdPQmgoxMZmvW+5clC5slGqVIEKFe4ko+Ti7Q2ZrY0bEnKJgIDauf+mUrl48SIbNmxg2bJlhISE4ODgQGxsLAkJCVSqVInKlStnfxArci1Baa0DUv18HqiY6rUZGGcpQgiRpcuRlwlcFEhCUgIhA0OoWKJi9jsVAPHxcOwYHDyYtpw/n/k+fn5Qs6ZRqlXLmIxcXfMv/pyKi4vj999/Z9WqVaxcuZKwsDCcnJyIiopK087d3Z3XXnvtnleWl5UkhBAFitaaPj/2ISwyjM3PbqZ2qbz91/+9Mpvh+HHYtetO+ftv65MPXF2hTh146CF44IE7CalmTeN6TmGxbt063n33XXbt2oWrqyuRkZEkWabwpZ86D8b0/4EDB97z+SRBCSEKFKUU77V/j+iEaJpVbGbvcFJcvw7btsEffxjJaPfujJMTlIL774e6ddOWGjWMmW6F3T///MOOHTswm81WE1J67dq1o3Tp0vd8viLwkQkhigJzkpktZ7fQrlo72lRpY+9wuHIFtm6FLVuMcvBgxjaVK0PTpkZp0gQaNTJmuRVVQ4cO5eTJk8yePZvo6Ogs23p5eTF8+HCbzicJSghhd1prhq8bzhd/fcGul3bRpEKTfI8hPh62b4d162D9ejhwIG29qys8+ii0agXNmhkJqWzZfA/T7mbMmMG///7L6tWrs0xSLi4udOjQwaZzSYISQtjdO9ve4Yu/vmB0i9H5mpwuXoRffoG1a2HzZmN1hGTu7tCiBfj7G6Vp07ybml2Y3Lx5k+PHj6dce7LG1dWVIUOG4OjoaNO5JEEJIexq9p7ZvBX8Fs/Wf5ZpHabl+flOnYKffoIVK4zrSak9/DB07myUVq0K5gw6ezp//jytWrUiLCyMhCyWolBKMWjQIJvPJwlKCGE3x68d55XVrxBUM4hvun+Dg8qb1c/++QcWL4alS9NeS3JzM5JRt24QGAgVC8dsdrs4fPgw/v7+3Lx5M8veE0CjRo3u+d6n1CRBCSHsplapWvzw+A90ub8Lzo5ZrJdzD27cgB9/hIULYceOO9tLlIDu3aF3byM5eciT6bK1fft2unTpwm0rayq5u7vj4uLCrVvGut9eXl6MHDkyV84rCUoIke/+vvw3MQkxNKvYjMfrZHx21r0ym43rSXPnwurVd+5JMpmgTx946ilo395YGkjkzMqVK3nyySczPMMKwMPDg19//RVfX19atGhBpOUiXvfuWT19KeckQQkh8tXpm6cJXBSIt6s3h4YewsnB9j9Dly4ZSWn2bGMNOwAHB+jUCZ55Bnr1Ak9Pm09T7MyePZsRI0ZkSE4ODg54e3sTHBxM/fr1AeOBlO3atWPgwIG45NK/ACRBCSHyzZWoKwQuCiQuMY7lzyy3KTlpbdyn9OWXxoSHRMtDfGrUgEGDjMRUzvpC3iIbWmsmTZrEBx98kCE5OTk54efnx7Zt26hWrVrK9tatW7Njxw6qV6+ea3FIghJC5IvbcbfpuqQrFyIusPHZjTxU5qF7Ok5iImzeXJpRo2DPHmObo6PRS3rlFejQweg9iXtjNpsZMmQIS5YsyZCcXF1dqV69Olu2bLG6QkSTJrl7i4AkKCFEvvjsz8/YF7aPn5/4mRaVWtz1/lFRMH8+zJgBp08bya1MGSMpvfSSzMDLDXFxcTz22GMEBwdnuAnX3d2dRo0asXbtWjzzabxUEpQQIl+80eoN/Kv606pyq7va79Yt+PRTo9y4YWyrWDGat94y8eyzcvNsbomIiKBTp078/fffGXpOJpOJzp0789133+Xa9aWckI6wECLPaK35aMdHXLx9EScHp7tKTlFRMG2a8QiKt982klOzZrB8OSxYsItBgyQ55ZZLly7RuHFj9u/fbzU5Pf/88yxdujRfkxNIghJC5KH3t7/P6N9GM2/fvBzvExMDH38M1avDmDFw8ya0bg3BwbBzpzFd3MYVdEQqJ0+epEGDBpw+fTrDCuXu7u6MHz+ezz//HAc7XNiTIT4hRJ6Yt28eYzaN4em6TzO29dhs25vN8L//wVtvGWvkgbH+3dSpxsSHe3zmncjC7t276dChAxEREWit09S5u7vz5Zdf2vQ8J1tJghJC5Lpfjv/Cy6teJrBGIPN6zst2CaOtW2HECNi3z3jdoAFMmQJdu0piyisbNmygd+/eVlckN5lMLF26lKCgIDtEdocM8QkhclWSTmLq1qk0KteIZf2W4eKY+XWL06ehb19jtfB9+4yZeIsWGdPHu3WT5JRXFi9eTK9evTIkJ6UUJUqUYPPmzXZPTiA9KCFELnNQDqwfsB6zNuPpYn06cnQ0vPMOTJ8OcXHGoy3eeANGjzaWJRJ556OPPmLChAkZJkM4Ojri6+vL1q1bqVWrlp2iS0t6UEKIXHE2/CxDVw8lNjGWku4lKWUqZbXdhg3GYy3efddITgMGwIkTxkw9SU55R2vNyJEjefvttzMkJxcXF6pUqcK+ffsKTHICSVBCiFxwLfoagYsC+e7Qd5wNP2u1zdWrxvJDgYHG0F69esasvIUL5SbbvJaQkMATTzxh9VHtbm5u1K1blz179lC+fHk7RWidDPEJIWwSGR9J1yVdOXvrLBsGbKBWqbT/Atcavv0W/u//jHuZ3Nxg4kTjtXPuPmFDWBETE0PHjh3ZtWuX1XucWrduzc8//4xbAbypTBKUEOKeJZgTePzHx9l9cTcr+q2gdZXWaer//RdeeMEY1gNjuvjXXxsLuoq8d/36dYYNG8bFixcz3ONkMpno27cvc+fOtfnR7HnF5iE+pZS/UmqvUupvpdRupdSjVtr4KaWWKqX2KaV2KaV+V0rd3XonQogCJ/RmKH9d/ItZ3WbRs3bPNHU//AB16xrJydfX6EVt2CDJKb+cO3eOhg0bcv78eavJacSIEcyfP7/AJiewsQellPIBVgDdtNY7lVIBwEqlVDWtdeqBzneBa0A/rbVWSvUBfgAq2HJ+IYR91S5VmxOvnsDX5JuyLTwchg2DJUuM1127wjffQNmydgqyGIqKiuKRRx6x+nh2d3d3PvjgA4YNG2an6HLO1h5UIHBca70TQGsdAoQB7dO1+xfwAVwtr0tbtgkhCqEPtn/AlC1T0FqnSU6bNxu9piVLjBl5X38Nq1ZJcspv7u7uPProo7i6uqbZbjKZWLhwYaFITmD7NajqQGi6baGW7am9DcwGriilwoGrQFdrB1RKDQIGAfj5+RESEmJjiLkrMjKywMVUUMlnlXPh4eGYzeZC8Xmtu7SO94+/T7vS7WiZ1BIH5YDZrJg3rypLllQB4MEHIxg79igVK8awZUvuxyDfrey9/vrrhIaGEhoaSkJCAu7u7rzzzjv4+voWns9Oa33PBRgLLEi37QdgpJV2awBvy+uXgd2AY1bHb9SokS5ogoOD7R1CoSGfVc75+/vr+vXr2zuMbP16/FftOMlRd/i2g45LjNNaa/3vv1q3aaM1aO3oqPWkSVonJORtHPLdypnw8HBdsWJFXbJkSX3gwAF7h5MC2K1zkGNsHeK7AFROt62yZXtqTwGfaq1vWZLiHIzrT/VtPL8QIp/sOL+Dvkv70rBcQ1b0W4GLowubNkHDhsZaeuXKGUN8EyaAk8wPLhC8vb356quvOHbsGPXq1bN3OHfN1gS1EqinlKoLoJRqCtQGNiultiulalranQD6KGWsGKmUagOUAM7ZeH4hRD4JvRFKVZ+qrH5qNR7OXkyZAh07wpUr0L69sZZemzb2jlKk5+npSZkyZewdxj2x6d85WutbSqm+wDyllAYSgSDABFQBvC1NhwIfA3uVUsnzHR/TWl+z5fxCiLyXpJNwUA48U/8Z+j/cn8hbLgQ9BuvXG4u5TphglAI8W1kUUjZ3xLXWwUATK1UVU7W5BDxp67mEEPnrevR1Oi3qxET/iXSv1Z3QEy706AGnTkGpUrB4MXTqZO8oRVEla/EJIayKio+i23fdOHzlMCVcS7BmjfHI9VOnjOtOe/ZIcrLFn3/+iVKKsLCwe9o/ICCAjz76KJejKlgkQQkhMkgwJ9BvWT92/buLJX2+Y9dSf7p1g9u3jec3bdsGldNPjxJ3ZcWKFfj6+lLWyk1ib775JkqpNKVq1ar5H6SdSYISQqShtealVS+x5uQaPuswm5+n9ea//zUWfZ082VjCyMPD3lEWbhs3buTTTz9FKcWoUaMyrPYwbdq0NNOtly5dmlJXtWpVlFJsyYsbzAoYmQwqhEhDo/Fy8eKNBh+yZPSL7NhhrAqxcCH06WPv6Aq3+Ph43n33XaZNm8aMGTPo0qULQUFB7Nq1i6lTp+Lv7w/A1q1b2bp1a8p+hw8fTvn5zJkzAHTo0CFfY7cH6UEJIVJEJ0TjoBwYWWsmP/33P+zYAZUqwY4dkpxsNW7cOMqXL8/y5ctZv349Q4cOpVq1auzZs4dWrVrRtWtXatasyYEDB9i8eTOLFy/G09MTT09PmjVrxoQJE4A7PahNmzbZ+R3lPelBCSEAWPT3IsZuGsvM+jsZ9GQFrlyB+vVhzRooYM+xK5Rat25N69at6dSpEw4Od/oGJpOJ9957j7FjxxIcHEy9evX46aefqFSpEkOGDCEpKYn4+HjCw8PZvn07n376KT179iQgIMB+byafSIISQrD25FqeX/k8dcJH8dT48kRHGzfhLlsGJUrYO7qioXPnzlnWe3l50aNHj5TXv/32G+7u7ri7u+Pl5YWPjw9Vq1alXbt29OzZM4sjFR2SoIQo5v688CePL32ccscncPj78ZjNimefhTlzwMXF3tEVDQEBAXc1qeHo0aNMnDgxyzbDhw+nWrVqNkZWsMk1KCGKsRPXTxC0uCuu26dyfvFbmM2KceNgwQJJTrlp06ZNJCQkpCkrV67E29s7w/aEhARq164NwMSJEzNMN08uffr0YePGjXZ+Z3lLelBCFGNlTH6U2j6fE2u6oxR8+SUMGWLvqIoea0+tTb4O5ZTNyrqBgYF8//33GbZ369Ytd4IrwCRBCVEMhceG44Qro1/35sSq7jg5GcsW9etn78hEek5OTvj4+FjdXtQV/XcohEgjOiGaoG97ETrvba782RY3N1i+HIKC7B2ZsCYxMZHw8HCr24s6SVBCFCOJSYk8vuQZdn40Gk62xcvLeCS75f5QUQCtX7+ekiVLWq3r1atXPkeTvyRBCVFMaK15/sfXWPv2a3A2AF9fWLcOGje2d2TFU7du3az2jFKbOHFitrP5ijJJUEIUExPWf8iiN56Cc60pVw42boQ6dewdlRCZkwQlRDEQEQGrJ7wG59ypWFETHKznOk1+AAAgAElEQVS4/357RyVE1uQ+KCGKuB2nDtG5s2bfX+5UqgQhIZKcROEgPSghirCf9m/mse4m9AVF5coQHAzVq9s7KiFyRhKUEEXU5iN7eby7F/pCEypVTiIkxIEivjKOKGJkiE+IImjP6ZMEdkki6UITKlVJZOsWSU6i8JEEJUQRczMinlYdw0k815iKlRPYtsWJYvi0cFEESIISogiJjYV+j7kQG9qEMmXj2RLsTJUq9o5KiHsjCUqIIuJWVAwBQdfYuBHKlIGtIS4yIUIUapKghCgCYuMTqdN+L38Gl8KnpJmNG6FWLXtHJYRtbE5QSil/pdRepdTfSqndSqlHM2lXSim1TCl1SCm1Ryn1nq3nFkKA2aypG/gXF/9siZtHHBt/c6RuXXtHJYTtbJpmrpTyAVYA3bTWO5VSAcBKpVQ1rXV0qnauwK/AaK31Nss2X1vOLYQAraFxz92cCmmOs1scmza40qiRvaMSInfY2oMKBI5rrXcCaK1DgDCgfbp2zwJ/AK9belkLAWcbzy1EsaY19H3hX/avboKjSzzrVrvQooW9oxIi99h6o251IDTdtlDL9tTaAA2A7sA5YDKwmIyJDKXUIGAQgJ+fHyEhITaGmLsiIyMLXEwFlXxWORceHo7ZbL6rz2vevKosX1gVB0czUyYfxsHhFsXl45bvVs4V5s/K1gSlAHO6bYlk7JmVARZorc8AKKXeB24ppTy11pGpG2qtZwOzARo3bqwDAgJsDDF3hYSEUNBiKqjks8o5Hx8fwsPDc/x5vTbxBAsXVsXBAZYtdaR374Z5G2ABI9+tnCvMn5WtQ3wXgMrptlW2bE/tChCR6nVSqiKEuAvvfPkPn096AIDZszW9e9s5ICHyiK0JaiVQTylVF0Ap1RSoDWxWSm1XStW0tFsBDFJKeVlejwA2p55IIYTI3twf/2X8a5UAGDspghdfVHaOSIi8Y9MQn9b6llKqLzBPKaUxhveCABNQBfC2tPtJKXU/8JdSKgY4Cwy05dxCFDe/brrGy8+UhCRnXhh6g6lv3WfvkITIUzavZq61DgaaWKmqmK7dh8CHtp5PiOLoyBF4vLc7Ot5E0OPXmDOzFEo6T6KIk5UkhCjgzp2DwECIu+1B6w63+HlJKRzkN1cUA/I1F6IAu3TZzCOtrnLhArRqBetWeuMsdxCKYkISlBAFVESEpl6r81w/X5qK999g1SowmewdlRD5RxKUEAVQXBw0bPcPV09VxbvsDXZtvQ8fH3tHJUT+kgQlRAFjNkOLrqf4Z08N3Lxv8dfWkpQrZ++ohMh/kqCEKEC0hqHDEtm76X6c3KPYusmDmjVlup4oniRBCVGATJoEs2c54eqqWfWLI00a2XwniBCFliQoIQqIi5F9mDQJHBw033+v6NzBzd4hCWFXkqCEKAAu3GzN1dDxAHz4WQS9etk5ICEKAElQQtjZ9z/fJPTQBMCBkeOv8H/DvO0dkhAFgiQoIexo87Yonu7vBknO+FScy/TJZewdkhAFhiQoIezk6FHo3cOFpHh3fCqtpPJ9n8v6ekKkIlOEhLCDc+egUyeICHcmsEsC0ZGfERGh7R2WEAWKJCgh8tnVq5oGrS5z80JZWraEFcucCQpK/2Bqw++//86pU6coX758SilZsiRKulqiGJAEJUQ+ioyER/wvcvN8BUpVu8iqVeWzXF9v1apVTJ8+HU9PT5KSkoiLiyMpKYmSJUtSunRpKlasSLVq1ahatSoVKlRISWI1atTAxcUl/96YEHlAEpQQ+SQuDpq2v8CFoxXxKHOVfdvKUrJk1vuMHz+eb775hhs3bqTZfvXqVa5evcqRI0cAcHJyws3NDUdHRyIjIxk7diyTJ0/Oq7ciRL6QSRJC5AOzGdr2+JejuyriUuImf231oWKF7H/9vLy8+PLLL/Hw8MiyXWJiIpGRkdy6dQt3d3eGDRuWW6ELYTeSoITIY1rDK6/Azg0VcHSPJGSjGw/WyvlDnfr160fdunVzdN3Jw8ODGTNm4OfnZ0vIQhQIkqCEyGNjxiYxZw64ucFva9xo3sT9rvZXSjF//nzc3LJf+sjT05Nnn332XkMVokCRBCVEHho35TrvT3PAwVGzdCm0Dbi3y761a9fmlVdewd096+R2+/Ztateuzc6dO+/pPEIUJJKghMgjn82K4N0JvgC889m/dOtm2/EmT56c7bWo6Ohozpw5Q/v27XnxxRe5deuWbScVwo4kQQmRB75fFsPwocb88RET/+HNoRVtPqaHhwdff/11tkkKICYmhsWLF1O9enV++uknm88thD1IghIil/22KYGnn3KEJCf6Dz3Bx29Xz7Vj9+nTh4YNG+LgcOdX193d3erQX1xcHDdu3GDAgAEEBgby77//5locQuQHSVBC5KK9e+Gx3k4kJbjQtu9Rvvv8gVw9vlKKefPm4erqCoCrqyv9+/dn+/bt1K5dG5OVu36jo6PZvHkztWrVYubMmSQlJeVqTELkFZsTlFLKXym1Vyn1t1Jqt1Lq0Wzav62USlBKVbX13EIUJMePawI7J3H7tuKJJzQbv38wTxZ/rVmzJq+99hrOzs6YTCY+/vhjGjZsyMGDB5k0aRImkwlHR8c0+yQmJhIVFcWYMWNo0KABhw4dyv3AhMhlNiUopZQPsAIYprWuB4wCViqlrC7eopTqDpQDZKxBFCmnT0Oz1re5dtWBdh3j+d//FA55OD7x9ttvU6lSJebMmYOPjw9grCYxatQojhw5QqtWraxeq4qKiuLQoUM0bdqU//73v8TGxuZdkELYyNZfoUDguNZ6J4DWOgQIA9qnb6iUqgWMAIbbeE4hCpQLF6Bp6whuXS1B6QeP8fMKJ/J6GTyTycSpU6d47LHHMtRVqVKF4OBg5s6di7e3d8pwYDKtNTExMXz++efUqFGD4ODgvA1WiHtk61p81YHQdNtCLdtTKKVKAHOBp7XWcVndEa+UGgQMAvDz8yMkJMTGEHNXZGRkgYupoCoOn9WNG84MfrU218J8MVU+zOz3L7Nn96W7Pk54eDhmszlXPy8/Pz8WLlzIzJkz2bp1K3FxcWnqY2JiiImJoUuXLrRs2ZLXX38db+/C8TTf4vDdyi2F+rPSWt9zAcYCC9Jt+wEYmeq1An4CAlNtOwNUze74jRo10gVNcHCwvUMoNIr6Z3XtmtbVa0Vp0Nq94nF9Nuz2PR/L399f169fPxejS2vLli26YsWK2mQyaSBDcXFx0d7e3nrRokU6KSkpz+LILUX9u5WbCuJnBezWOcgxtg7xXQAqp9tW2bI9mRfQAJiklPpDKfUHxnWon5RSA208vxB2ER5uPHDwn+MmSlS8wO6t91G5rKe9w8pUmzZtOHXqFMOHD8fd3T3Dun7x8fHcunWLwYMH4+/vz+nTp+0UqRB32JqgVgL1lFJ1AZRSTYHawGal1HalVE2tdYTWuprW+tHkgnGdqrfWeoGN5xci392+DR0C49m7F2rUgKN/VqROtVL2Ditbrq6uvPvuu+zevZt69eplOolix44dPPzww0ybNo3ExEQ7RCqEwaYEpbW+BfQF5imldgGfAEGACagCFI4BbSFyKDoaunSLZ88uF0ylrrJpE5Qvb++o7k6dOnXYu3cvH3zwAR4eHjg5pb0UbTabiY6OZsqUKdSpU4c9e/bYKVJR3Nk8EVZrHay1bqK1bqq1bqG13qm1Pq+1rqi13p3JPlW11mdsPbcQ+Sk2Fnr2SmT7VhfwCuObZWeoUsXeUd0bBwcHhg4dysmTJ+nYsWOmN/iePHmS1q1b8+qrrxIZGWmHSEVxJitJ5LE///wTpRRhYWH3tH9AQAAfffRRLkcl7paRnJLY+JsTeFzh8++P8qR/E3uHZbNy5cqxZs0alixZgq+vr9VHesTExDB37lyqV6/O6tWr7RClKK4kQeWxFStW4OvrS9myZTPUvfnmmyil0pSqVavmf5AiSzEx0LMnbFjvAKarvD33d4YFtbN3WLmqZ8+enD59mmeffdbqun6xsbFcvXqVfv360bNnTy5fvmyHKEVxIwkqD23cuJFPP/0UpRSjRo3KsAbatGnT0kypXLp0aUpd1apVUUqxZcuW/A5bpBIdDT16wIYNUNI3gTfmrGdi/z72DitPeHl5MWvWLEJCQqhevXqmw35r167l/vvvZ86cOcm3jQiRJyRB5YH4+HgmTpxIt27dmDFjBrt27WLNmjX4+/unSThbt25l6tSpKWX58uUpdWfOnEFrTfv2GRblEPkkOTlt3AhlysC2Lc5Me2qAvcPKc02bNuXYsWOMGTMGd3f3NCunAyQkJBAZGcnIkSNp2rQpx48ft1OkoqiTBJXLxo0bR/ny5Vm+fDnr169n6NChVKtWjT179tCqVSu6du1KzZo1OXDgAJs3b2bx4sV4enri6elJs2bNmDBhAnCnB7Vp0yY7v6PiKSoKunWDTZsAj0v8Z9avPPSQvaPKP87OzowfP56DBw/StGnTTKek79mzh4YNGzJhwgTi4+PtEKkoyiRB5bLWrVuzaNEiDhw4gL+/f8p2k8nEe++9R1hYGNOnT6devXoAVKpUiSFDhjBo0CAGDhxIu3bt2L59O59++ila6zTHEPkjIgK6doXgYMAzjOZvjWV4t472DssuatSowY4dO/jiiy8oUaIELukWGdSWdf2mT5/OAw88wPbt2+0UqSiKJEHlss6dO9O5c+cMwyLJvLy86NGjR8qd/L/99hvu7u6UKlWKWrVqERgYyOTJkzl27Fh+hi0srl+H9u1hyxbA6yIPjX6V9SM+xdXJNdt9iyqlFM899xyhoaH06NEj02tTZ8+epWPHjjz//POEh4fbIVJR1EiCykUBAQEZZuVlVZ544omUCRLR0dFcvnyZ48ePs379et544w0Ahg8fTocOHez8zoqHixehTRvYvRsc7jtNlf97ms3/+QovVy97h1YglCpViqVLl7Jy5Ur8/PyszvaLiYnhu+++o3r16ixbtkwmUQibSILKRZs2bSIhISFNWblyJd7e3hm2JyQkULt2bQAmTpyYaRLr06cPGzdutPM7K/r++QdatYIjR6BOHZi14ijBw+dRxqOMvUMrcDp06MA///zDkCFDMn3U/M2bNxk4cCAdO3bk/PnzdohSFAWSoHKRo6MjTk5OaUryUF/67emXlwkMDOTmzZsZSsuWLe3xVoqVI0egdWvjoYO16kWwZQu85B9EtZLV7B1agWUymZgxYwY7d+7kwQcftDrsFxUVxZYtW6hduzZ79+61Q5SisJMEVUA4OTnh4+OToaRPZCJ3/fWXMax38SJ4PbCPC71qo0zX7R1WoVG/fn0OHjzI1KlTM33UvJOTE5Urp3/ogRDZkwRVQCQmJhIeHp6hyGrSeWf1aggIMCZGlG7wF5H9WvHtE5/ja/K1d2iFiqOjIyNHjuTo0aO0adMmzZR0Dw8PZs6cSalSBX+1d1HwSIIqINavX0/JkiUzFJm2mzfmzDFuwo2Ohhptt3G1ewu+6jmDPg8WzVUi8kPlypXZtGkT8+fPx8fHB2dnZ+rXr88zzzxj79BEISUJKo9169Yt2ym3EydOzPKpkqNGjcqnaIs+rWHCBBg0CJKS4LEhRwht04ZJ7d9icOPB9g6v0FNK0bdvX06fPs3IkSNZvHhxhocjCpFTcoFDFBsJCUZiWrAAHBzgq6/g5ZcfZNWJlXR/oLu9wytSfHx8eP/99+0dhijkJEGJYuHWLejXz1j01WSC1z/cTrvH/VDqfnrU6mHv8IQQVsgQnyjyTp6EZs2M5FS6NLz77Q4+uh7AuM3j7B2aSOfgwYMopbh69Wqa7SEhIXh6eqa8Xrt2bZrZrm3btk3TXp6jVjRID0oUab/9ZvScwsPh4Ydh8qz9DAjpSN0ydZnTfY69wxPphIeH4+joiLe3t9X6qKgoZs6cyc2bN3nzzTfT1E2bNo2uXbtSt27d/AhV5ANJUDkUHR0tF3sLEa3hs8/g//7PmAzRsye89ckJAn/sQDnPcqx9ei0lXEvYO0yRztGjR/Hz80tZlPb27duYzeaUx807OTlRu3ZtEhMTKVu2LKdPn6ZMmTJUqFABINPEJgonGeLLgZs3b1K/fn0aNGjArVu37B2OyEZcHLz8MowYYSSnceNgxQp4/6/xODk4sX7Aevw8/ewdprDixx9/5OLFi/zyyy8ANGzYkJIlS9K9uzGJxdXVlV69erF//37eeustjhw5wqxZs5g+fTotW7akTZs28qDPIkR6UNmIiIigdevWnD17FoDBgwfTtGlTypUrZ+fIhDXnz0P//rBzJ7i5GTP2+vc36ub3nM/5iPPUuK+GXWMU1m3ZsoUtW7Ywfvx4Bg8eTOPGjTl16hRgXIPq1q0bAOvWrWP9+vWcOXMGX19ftNZ07tyZyZMnc+bMGQBZYLmIkB5UFqKiomjbti2nTp1KWeD12rVrNG7cmEuXLtk7PJHOunXQsKGRnCpWhN9/h+59ohm9YTS3427j4eJB7VK17R2msCIsLIxnnnmGIUOGMGXKFDp37kxAQEBKgkrt6tWrlCxZEl9fY8UPpRQPPvggV69exWw2k5iYKKuoFxGSoDIRGxtLx44dOXLkCHFxcSnbzWYzN2/e5Pp1Wa+toDCb4a23ICjIWLYoMBD27YP6DRN5YtkTTN85ne3nZUWOgmrXrl00btyYOnXqpMy8mz17Nh07duSRRx4hLCwsTftu3bqRkJDAxIkT2bNnD0uWLGHJkiUMHjyYGjVq4OzszObNm+3xVkQukwRlRXx8PEFBQezbt4/Y2Ng0da6urvz66688VJye/12AXboEHTvC1KmgFEyZAmvWgK+vZvCqwaw6sYovgr6g8/2d7R2qyISTkxMvv/wyq1atwtXVeDCks7MzX3zxBTt27MgwnF6yZEnmzp2Ln58fS5Ys4dy5c2zfvp22bdty5swZeRJ1EWJzglJK+Sul9iql/lZK7VZKPWqljZ9SapZS6qhSapdSaptSqkDOBU1MTKR379788ccfGZKTu7s7kyZNol27dnaKTqS2cSM88ojxaHY/P2NK+fjxxioR4zaPY97+eUxoM4FXmrxi71BFFh555BEmTpyIs7NzhrqHH37Y6j4mk4nnn3+eEiVK8MMPP9CoUSPc3NwoX748PXr04IUXXuD555/P69BFHrMpQSmlfIAVwDCtdT1gFLBSKZX+4TCPAOu11g9qrZsCPwPTbTl3XkhKSuLJJ58kODiYmJiYNHXu7u4sXryYZs2a2Sk6kSwmBoYPN3pOYWHG4zL27YPkfzdcj77O/w78j8GNBjMxYKJdYxV355FHHuGnn37KsL1hw4b89ttvabb16dOHDRs2MHPmTM6fP09ERAR//PEHrVu3ZtCgQRw5ciS/whZ5xNYeVCBwXGu9E0BrHQKEAe1TN9Jar9Var0i1KYwCNoNQa83AgQNZs2aN1eQ0Z84cevfubafoRLK9e6FRI+MeJycnY2hv0yZIPQrka/Jl98u7+SLoC7l3rZC5ceNGmmu+yby9vWnevHnKa7PZzNq1a5kwYQKtWrXC29sbFxcXKleuzOjRo2nQoAEbNmzIz9BFHrA1SVQHQtNtC7Vst0op5QdMBl7MpH4QMAjAz8+PkJAQG0PMntaajz/+mA0bNmT45XB1dWXo0KFUqFCBkJAQIiMj8yWmoiA3PyuzWfHdd5VYsKAqZrMDlStHMXbsUWrViuT33402u2/s5q+bfzG4+mAclAPHOZ4r584P4eHhmM3mYv/dio2NZf/+/ZnecOvq6oqDgwMxMTG0atWKESNGMGTIEO6//35cXFy4fv06mzZtYt++fTz77LPF/vOE3P09zHdZPeYhuwKMBRak2/YDMDKT9r7AHmBATo7fqFEjnR9GjRqlTSaTBtIUd3d3/cknn6RpGxwcnC8xFQW59VkdPqx18+ZaG+tDaP3aa1pHRaVts+vCLu3xjoeu/1V9HREbkSvnzU/+/v66fv369g7D7qpUqZLh9zB12bZtm9ba+G7FxsbqqVOn6saNG+uSJUtqDw8PXaVKFd2vXz/9xx9/2PmdFBwF8W8WsFvnIAfY2oO6AKS/I64ysCx9Q6VUOWAt8IHWeomN5801EydO5MsvvyQ6OjrNdpPJxPjx4xk+fLidIhOxsfDee0ZJSIDy5WH+fOjUKW2749eOE7QkiDIeZVj79Fq8XL3sE7CwWfKNtjnh6urKuHHjGDdOFv0tqmy9BrUSqJc8I08p1RSoDWxWSm1XStW0bK8CbAKmFKTk9MEHH/Dhhx9aTU4jR45kzJgxdopMbN0KDRrA5Ml3nuN06FDG5HTx9kUCFwWiUKwfsJ5yXrLChxBFhU09KK31LaVUX2CeUkoDiUAQYAKqAMkDydMBP2C0Umq0ZVuc1tpuNyt88cUXTJo0yWpyGjRoEFOmTLFTZMXbzZvw3//CN98Yr2vXhtmzoXVr6+0PXj5IVEIU655eR03fmvkXqBAiz9k8k05rHQw0sVJVMVWbx209T26aP38+o0ePzjBbz2Qy8fTTTzNjxgyZ/ZXPEhONRDRhgrEahIsLjB0Lb74Jlns309Bao5Qi8P5ATg8/jaeLZ8ZGQohCrditJPH9998zbNgwq8mpV69efP3115Kc8tlvvxnDecOGGcnJ3x/274e337aenBKTEum3rB/fHvgWQJKTEEVUsUpQK1eu5IUXXrCanAIDA/n2229xcChWH4ldnTgBPXoY15UOH4Zq1WD5cmNliAcftL6P1ppXfn2FZUeWEREXkb8BCyHyVbH5a7x+/XqefPJJqzfhtm7dmh9//BFHR0c7RVe8XLgAr7wCDz0Eq1aBpydMmwZHjkCfPsaaepmZEDyBb/Z9w/jW43m16av5F7QQIt8VqNUc8srWrVvp06dPhuTk5uZG06ZN+eWXX3ByKhYfhV1dvmxMGf/6a+OhgkrBiy8aq0GULZv9/jP/nMnUbVN5qeFLTG47Oe8DFkLYVZH/q/znn38SFBSUYbaeq6sr9evXZ+3atSmPlxZ54/p1+PBDmDkTkv839OsHEydmPpRnzY2YG/Sq3Yuvun0l1wmFKAaKdILav38/HTp0ICoqKs12FxcXateuzcaNG3F3d7dTdEXfpUuujBwJc+ZA8v+Cnj1h0iSoXz/nx0kwJ+Ds6MzbAW9jTjLj6CBDsUIUB0X2GtSRI0fw9/cnMjIyzXZnZ2eqV6/Oli1b8PSU2V95Yf9+ePppePrpR/nkEyM5de4Mu3bBzz/fXXLac3EPtT6vxd6wvQCSnIQoRopkD+rkyZO0atWKiIi0s7ycnJyoVKkSv//+e6aLUYp7k5QE69fDJ59A8iLSDg5Goho9+u6SUrKT10/SZXEXTM4mynrm4CKVEKJIKdQJKjY2Fjc3tzTbzp49S4sWLQgPD0+z3dHRkbJly7Jjxw58fX3zM8wi7coVmDcPZs2C5GXUPDzg5ZehWbM/eOKJ5lnun5mw22EELgpEo9nwzAbKe5XPvaCFEIVCoR3i27p1K6VKlWLnzp0p2y5evEjz5s25ceNG8urpADg4OFC6dGn++OMP/Pz87BFukaK1sVbek09CxYowZoyRnKpWNWbpnTsHH38MZctmfK5PTtyKvUWXxV24EnWFNU+t4QHfB3I1fiFE4VBoe1CLFy8mOjqajh078uuvv1KnTh2aN2/OlStXSEpKSmmnlOK+++5j586dVKhQwY4RF34nTsDChbB4MZw+bWxzcDButh0yxLjhNjduJXNxdKGmb00+6PgBTSpYW0VLCFEcFMoEpbVm+fLlaK2JiooiKCiI0qVLc/HiRcxmc5q23t7ebN++napVq9on2ELuyhX44QdYtMiY5JCsQgV4/nljKK9y5dw5lznJTFRCFCVcS7C079LcOagQotAqlAlq//79xMbGpryOiYnhwoULaXpOACVKlGDbtm088IAMEd2Nc+fgp5+Msm2bMQECwMsLHn8cBgww1svLzYU3tNYMWzOMnRd2suOFHXi4eOTewYUQhVKhTFDLly8nPj4+zbb0ycnT05Pg4GAefvjh/AytUNLaeNbSqlVGUtq9+06dszN06WIkpR49wGTKmxgmb5nMrD2zeLPlm5KchBBAIU1QS5YsISEhIdN6d3d3NmzYwCOPPJKPURUuN27Axo3G1PB16+DixTt1JhMEBUHv3tC1K+T1jPyvd3/NxC0Teb7B87zb/t28PZkQotAodAnqzJkzhIWFZdtu165dNG9+b1Oci6Lbt2H7dmP2XXCwcT0pdaezbFnjZtpevYzJDvm1wMbKYysZunoo3R7oxuzus2UJIyFEikKXoH7++eds28TExDBmzBhiYmJ488038yGqgicsDP76y7iGtGUL7N0LqeePODsb15E6d4bAQKhXL+tVxPPKI+UeYWCDgXwe9DlODoXu6yiEyEOF7i/CwoUL00yQyExyknrooYfo3r17PkRmPxERsGeP0StKLhcupG3j6AjNmhlJqU0bo3h52SdegLPhZ6lYoiKVvCsxr+c8+wUihCiwClWCun79OocOHcqyjZubG1pr6taty5AhQ+jQoUM+RZf3tIazZ+HgwTvlwAE4dsyoS83LC5o0gUcfNZJSixbGc5cKgn9u/kOLuS3o91A/Puvymb3DEUIUUIUqQa1evRoXF5cMM/icnJxwdnamfPnyDBo0iKeffrpQ35SbmGgkopMnjZtjDx82ktGhQ8a1pPScnY1HpjdpAk2bGqVWLeMm2oLmcuRlOi3sREJSAq80fsXe4QghCrBClaAWLVqUsjq5g4MDJpMJd3d3Bg4cyHPPPcdDDz1k5whz7vZt436jc+eMZYJOnryTkE6fhswmKZYpA3Xrpi316oGra76Gf08i4iLosrgLYZFhbH52Mw+WvouHQQkhip1Ck6BiYmIICQnBZDLh4OBA3759efHFF2nRokWBm/kVGQmXLhnlwoU7iSh1uXkz62NUqgQ1axqldu07yahMmfx5D3lhwIoB/H35b83AADYAAAnZSURBVFY9uYpmFZvZOxwhRAFXaBJUYmIizz33HD179qRTp0759hRcrY2nwN64YZR9+3y4ft14Suzly3cSUeqS7uG9Vrm5QZUqxjJBlSvfSUY1a0KNGnl3Q6w9vdHyDZ54+Am61Oxi71CEEIVAoUlQXl5ezJkz56720RpiYoxZbrdv3ymZvb51604iSl3i0izK3SDb87q7G/cVlS0L5cunTUTJpVQp+0zrzm9aa3Ze2EmLSi1oWbklLWlp75CEEIWEzQlKKeUPfGw5Vjzwqtb6j3RtFDAZ6AeYgb3AYK11FFm4dg1mzjSSTHKJjk77OrNtkZFG0km3AtI9cXMDX1+47z5wcAinenUf7rsP/PzuJKLk4udnzKArDsknJ97Z9g5vBb/Fxmc20r56e3uHI4QoRGxKUEopH2AF0E1rvVMpFQCsVEpV01qnHuh6DggCGmitY5RS84H3gVezOv7Zs/D667ZEaEweKFHCSBpeXln/XKKEkYSSk1FySb2qQkjIfgICAmwLqpj4NexXpp+YzrP1n6Vttbb2DkcIUcgonf4GmrvZWan+wHCtdYtU2/YDb2mtV6Xathb4SWs92/K6AbBJa53lo22dnWvqMmU+wMEhHgeHOBwdY1N+Nl7HpfxsvI7HwSE2pa2jYzQODuasTnHXwsPD8fHxydVjFkXXSl3j8MOHue/GfTx08CEcdAGc8/7/7d1/bFXlHcfx95daoIorThSGUBREcE5j9GoWlygbBjJ+SAK2kLnsDxklabLEZTMxDJOyaeIfQ5dlW4SgBl2YE8IvV6Cxc3SDdTEVqyMo0y4OupEMZgp0uLb38t0fvTW1pfee2xv6nHvv55XwR597yv3kyXPPt89zzn1OjLS1tZFMJkkkEqGjFAR9DqOLY181Nze/7e5ZB3u+S3wzgfZBbe3p9kzHtQNfNLNKdz878EAzqwVqAcrLy7nuug2Rgrj3fX/ockulUkMeJy+f1zO+hw++/AEVn1Qw9fBUzqXOhY4Ue8lkEnfX2IpIn8PoCrmv8i1QRt81pYGSDH2U/ODj+kvJkD+r07OszQCJRMJbBz77IQYOHjyoJb4IXj36KhWnKlj282WhoxSEefPm0dnZSVtbW+goBUGfw+ji2FdRvxqU77pLBzD4eapV6fZMx1UBXUBhlnW5pI87P6blZAsAq76yisryy/ycDhEpavkWqD3AHWZ2O4CZ3QvMBd40s8NmNjt93CvAd82s/8tL3wN2ej4XwCRWTv/3NAteWUD19mr+l8y+ma+ISDZ5LfG5+1kzqwZeNDOnb+luEXAlMAPo/xP6ZeBm4C0zSwLHyHIHnxSO893nWbRtER3nOmj6ThPjrxgfOpKIFIG8vwfl7n8A7rnES9MGHJMCfpT+J0WkJ9XD8teW886pd9i9ajf3Tb8v+y+JiERQMDtJSDxtat1E09+beGnZSyy5ZUnoOCJSRFSgJC9199QxZ9IcFsxaEDqKiBQZfXtSRmTLkS10nOugbEyZipOIXBYqUJKzF468wJrX1/Bsy7Oho4hIEVOBkpzsPb6X2t/VsnDWQp558JnQcUSkiKlASWSHThxi5Y6VJKYm2FGzg7Flo/NMLhEpTSpQEom7s/7N9cyonEHDtxqYMHZC6EgiUuR0F59EYmbsXrWb893nmXTlpNBxRKQEaAYlGZ25cIbHDjzGp72fMnH8RKZXTg8dSURKhAqUDKurp4vF2xaz6e1NHDt9LHQcESkxWuKTS+pN9fLwaw/T+q9Wdq3cxd1T7w4dSURKjAqUDHHRL/Lo3kdpbG9ky9ItPDTnodCRRKQEaYlPhjhx9gT7P9zP0994mtV3rQ4dR0RKlGZQMsSNE2/kaN1RJl81OXQUESlhmkHJZ7a2baX+YD3uzpQJUyI/lllE5HJQgRIAGv7WwOq9qzl88jDJi8nQcUREVKAEWk62UL29mjun3MnOmp2Ul5WHjiQiogJV6o6dPsbibYuZ9oVp7HtkH1ePuzp0JBERQAWq5B3991EmjJ1A47cbuf6q60PHERH5jO7iK1HujplRc1sNS29ZSkV5RehIIiKfoxlUCbrQe4H5L89n1/u7AFScRCSWVKBKTG+ql5rtNTT/oxnHQ8cRERmWlvhKiLuz5vU1NHzYwPOLn2f5rctDRxIRGZZmUCXkiaYn2PruVjbM28DaxNrQcUREMsqrQJnZA2Z2xMzeM7NWM/vqMMdNNrNNZva+mb1lZn8ys9vzeW/JjbvTneqmLlHHk/c/GTqOiEhWI17iM7OJwE5gibu3mNk8YI+Z3eTuFwYdfhfQ6O5r07/7A2AjsGCk7y/RdSe7GXfFOJ5b+ByOawsjESkI+cygFgLH3b0FwN0PAqeA+YMPdPf97r5zQNMpdP1rVBz46ABzfjGH42eOY2aMMa3qikhhyFokzGws8MdLvLQfaB/U1g7MzPL/TQZ+DFzyOQ5mVgvUpn/sMrPj2TKOsknAmdAhcjX3+3NDvG1B9lVAk8xM/RWNxlZ0ceyrGVEOylqg3L0HGHJtyczWAalBzUkyzMrM7FpgH1Dv7s3DvN9mYHO2XKGYWau7J0LnKATqq9yov6JTX0VXyH2Vz3pPB1A1qK0q3T6EmX0J+D2w0d1/ncf7iohICcinQO0B7ui/G8/M7gXmAm+Y2bVmdtjMZqdfm0FfcfqJu2/LN7SIiBS/Ed+o4O5nzawaeNHMnL7lvUXu3mlm0+lbY6xMH74RmAw8bmaPp9u63f2BPLKHEtvlxxhSX+VG/RWd+iq6gu0rc9d2NyIiEj+651hERGJJBUpERGJJBUpERGJJBSoPZnabmX1iZvWhs8SVma01s3fTezW+Z2Z1oTPFTdQ9LUXjKVeFfo7SdkMjlN6L8JfAb0JniSszKwNmA19z9y4zuwH4yMz2uPs/A8eLhRz3tCxpGk+5KYZzlGZQI2BmY4CtwDrgdOA4seXuKXf/obt3pZv+A/QAZQFjxU3kPS1LncZTdMVyjtIMahgZ9iC8H6gH3nD3P5tZye/Inqmv0ltl9fsZ8Ft3PzE6yQrCTEawp6UAGk+ZPEURnKNUoIaRYQ/CFUCVu68b/VTxNFxfDWRmTwE3ACtGJVThMHLc01I0njIppnOUClTuvgncamZ/Sf88DfouRrp7dbhY8WVmPwVmASsGzaikb+/KBwe1VQE7AmQpCBpPWRXNOUo7SeSp/+4Yd68PmyR+0uvgvwKuAR5x92TgSLFjZpX0Lel93d3/mt7TshG4yd07w6aLF42nkSnkc5RmUHI5LQLWAq3AoQFP8l3v7k3BUsVIpj0tA0eLI42nEqMZlIiIxJIuxIqISCypQImISCypQImISCypQImISCypQImISCypQImISCypQImISCypQImISCz9H3KI8zqH1wxnAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "z = np.linspace(-5, 5, 200)\n",
    "\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([-5, 5], [1, 1], 'k--')\n",
    "plt.plot([0, 0], [-0.2, 1.2], 'k-')\n",
    "plt.plot([-5, 5], [-3/4, 7/4], 'g--')\n",
    "plt.plot(z, logit(z), \"b-\", linewidth=2)\n",
    "props = dict(facecolor='black', shrink=0.1)\n",
    "plt.annotate('수렴', xytext=(3.5, 0.7), xy=(5, 1), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.annotate('수렴', xytext=(-3.5, 0.3), xy=(-5, 0), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.annotate('선형', xytext=(2, 0.2), xy=(0, 0.5), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.grid(True)\n",
    "plt.title(\"로지스틱 활성화 함수\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.2, 1.2])\n",
    "\n",
    "save_fig(\"sigmoid_saturation_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Xavier와 He 초기화"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "he_init = tf.variance_scaling_initializer()\n",
    "hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu,\n",
    "                          kernel_initializer=he_init, name=\"hidden1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 수렴하지 않는 활성화 함수"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Leaky ReLU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def leaky_relu(z, alpha=0.01):\n",
    "    return np.maximum(alpha*z, z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xl8VPW5x/HPk4QtoARE0YsgqNQNrVa0tmqN1at1udcV0EJFbEWtS8UdKgU3lFK1xUoV96UVkIJYF6xdgr1FrbhgrRUQBVGLGyYsYcvkuX/8JhBDlsmQyTkz832/XufFZObMmSfHcb45v/PM75i7IyIiEjcFURcgIiJSHwWUiIjEkgJKRERiSQElIiKxpIASEZFYUkCJiEgsKaBEYsDMJpnZklrLqFqPPWVmY5u5vWY9x8zKzOz8Rh4fZmbltZZ/1Hqs1MxWN7H97mbmZvatVGsSUUBJi0l+yN2SoW0vaewDtBnbedDM1prZcjP7yMzeNbPbzKwkxeeXJj9o2zenTjO70czK6rn/CDNbAhxf56HhyW3t3EQ9d5vZoynW/m6y9pol5f3p7g+4e0mt5eBUn5t0KlAN/LuZz5M8VhR1ASIRmOXuZwCY2Q7Aw8Dv2DIkMs7d5wC9zWxH4GBgG2AF8H/uvipZY2Ob2BF4K7leEdApef+m/7fNbG9gb+BnfPWP0o1mdjrw54Y2bma9gDcbef2xwBuNFWhm+wHjgM+Bu8xsqLuvb+w5IqCAkjzn7p8mh8Lmmtm27r6ytWsws2OBPwALgC+AXkCJmX3T3RclV2ufPMpb4+4bk8/bBigFOiXD6XvJ7dR4KfnvDkC/RkqY29AD7v5BspYi4GhgV0LQ/MXdP0/WUZr8tyQ8xSuSPxcA5wMTgDuS/z4JzDOznwJ/cE1lI43QEJ+0GjM71cz+YWYfJIevJtcMrSWHzp41sw+Tw28vmtm+DWyni5m9YGYPmVlPM9tQ8yFZa51fmdm0FEsrTv5blXzufmY228zeT9bzjJntk+avnYo+wGvAQe5eSjiSqiYcHdW4GvgSOK3WfROBMmAtcLu7P+Xu5u4GPF2zkruXEY5gioGhwIXAAOBTdx/r7h83VpyZdQXmA+clt3EY8G8zO7rWah2T9X2SfM7FwAfAlcAP3P0ad/8COIJwxHo/sMzMjklh/0ieUkBJqzCzAcDZwP+4ey/CkFMx4UMWYL/k7V2BnYC/A7+oZzs9gb8Bc9x9qLsvA54Azqm1TgHhA/iBJmoyM9sLuBGY6u6VZtYHeBC42t37EI5mngGesSbG2rbC3cB4YJKZfQlcRgiiv5tZh+Q61yXDZ4qZFZvZ3cDhwLnAmUB/M/uDme3WwGuMAo4FDnP37YEfAOPM7Lha6xSZWXszK6zz3NOBanc/xd1/4e6XAPck66yxJllfzbm5V4BLgT3cfUbNSu5e5e4TgJ0JR1cvpLyXJO9oiE9ayw2Eoab5tT7ni0j+xe3uE81sW+BQwhHFfwG719lGP+CnwPXufk+t+38DPGVmFyWH6EoJRyB/bKCWk8xsOeGv/vbAaOC25GNXA18DnqtVpyXX7ZnC79nYkNUWj5nZb4GvJ3/sCJQAZxBCYS0ws57tPAlUAAe7+4rkdg4DriD87ovrec7ewF/d/SMAd3/NzN4i7NNnk+vckVxGAL+s9dwFQC8zO4IwbNgDOBJ4scFf1P2lhh5LPr4OeKqxdUQUUNJadgWGuHu9w25mdiuh02sW8B6wEPh2ndVOJTQQ9K19p7v/1cyWAoMIf9mfATzs7okGapnl7meYWRfCh/lSd99Qq86H3f3HDdS5a+O/JisJQVPXtoRQqevHQBtCoFYBle5eVec1DyUMn9U4paaBokbydx1f664XCPuxxhTggWTH4LuEYbqvU+vIE7jA3e+qW6C7zzGzHye3v0uylieB65KrlBOOamvqfZAwlJiq49x9djPWlzyhIT5pLR8BB9X3QPL80cWEczCXuvtE4J16Vr2e0AgwJHmOo7a7gGFm1oYQZI0O7wG4+5fAzcCtyaO3RutM0SKgvvNV+wJL66mhItlscDUhlGvOwW1agN9Tq8OwVndfgZldYmYvmdlnZlZtZivMbDYw392n13rODOA4wlDqH4AEsL+7L0zll3L33wLHAHsChxDOaRWZWSdC4J1ea/VzCKFbe9k/+Vi3eh57LpUaJP8ooKS1jAMuNLOTkx+shWZ2tJmdArQjDKNtD2BmBxCGBLfg7h8C/0s4f1L7Q/EhwofvxcDbtbrfmjIR2Fjr9W4D9jWzsZb8rpOZ7VNPIDbkDmCQmQ0ys5pzOhcRGh/uaeR51xOG2+pbXmNz+3htvyA0Ifwc2IswXHkA8BfgSTP7Stu8u89l8zm/XwEbzOwbZvaNFH+3hYQj2C8JR00VwKrk8kmt16lOnmvatBACESBR9zF18klDFFDS0i6qewRgZqclzxldSBgW+hRYRvhwXQY8D0wCXjCzTwhDSXc09ALuPo8whPSwmR2evK8CeIzwQd/k0VOtba0nnIO60MwOcPd/AkcB3yF0mS0ndJ19VuepS+r8ji8mtzeHcDTxk+Tv+TFwEvBdd3+rkVJuBd4H5tWz7E0IhLqOB37p7jPc/XN33+DuS93954TuvuMAzGy+ma0zs3XJ7UHosHsZuA84IaWdFZzk7oXuXpBcjHA+SqTFmf54kVxhZkcSzo3s6O5roq6nOczsLgB3T3l2BzO7HTiFEIZzgdWERpRTCUOXA939SQvflyokNGlUA2vrOc9VBkyp7xxUrXWWJ1+r7pDcYcnn1neUV/PcfsA/gS7uXl/YimxBTRKSS0qBx7ItnGr5oZkNaeCxf7j7d+vcdwXhvNZIQnNHV8KR1qvAqe7+DGw+Z9VCpjRwf7buc4kxHUFJTjCzrxHOvRzl7guirkdEtp6OoCTrmdm/CF/6vVThJJI7dAQlIiKxlJEjqG7dunnv3r0zsekWsWbNGjp2rO+7lNIY7bf0LFiwgEQiwd577x11KVlH77n0NbTvFi+G8nJo3x722gsKIujlfvXVVz9PTrnVqIwEVO/evZk3b17TK0akrKyM0tLSqMvIOtpv6SktLaW8vDzW/0/Eld5z6atv391wA/zsZ9C5M7zyCvTtW/9zMy0580uT9D0oEZE88NRTMGYMmMFjj0UXTs2hJgkRkRy3YAEMHgzucNNNcNxxTT8nDnQEJSKSw1auhJNPDv+edhqMHBl1RalTQImI5KjqajjrLHjnHdhnH3jwwTDEly0UUCIiOerGG2HWLCgpgSeegE4NTkYVTwooEZEcNHfudl9piti97uU/s0CzAip52YEVZjY2Q/WIiMhWeucdGDduLwDGjYPvfS/igtKUckCZWQlwJ+GSBiIiEkMVFaEpYs2aIgYMgKuvjrqi9KXUZm5mBYQLwo0iXFWzvnWGA8MBunfvTllZWQuV2PJWr14d6/riSvstPeXl5SQSCe27NOg91zzV1TB6dD8WLOhG794rGTZsPnPmJJp+Ykyl+j2oG4Hn3X2umdUbUO4+GZgM0L9/f4/zt7/17fT0aL+lp6SkhPLycu27NOg91zxjx8LcudClC9x449scd9zhUZe0VZoMKDM7Dejl7qNaoR4REUnDrFlw3XVhbr3HHoN27dZFXdJWS+UI6jhgLzN7KfnzzhAaJtx9QMYqExGRlLzzDvzgB+H2uHFw7LGQCyOjTQaUu/+o9s81HXzuPjYzJYmISKoqKuCkk2DVKhg4EK66KuqKWo6+ByUikqWqq2HIEFi4EPbdF+6/P7tmimhKsyeL1ZGTiEg8XHddmKW8S5cwU0SuXTpLR1AiIlnoiSfg+utDU8SUKbDrrlFX1PIUUCIiWebttzc3RdxyCxxT75d/sp8CSkQki5SXh5kiVq+GM86AK66IuqLMUUCJiGSJmqaIRYtgv/3g3ntzqymiLgWUiEiWGDMGnn4aunbNzaaIuhRQIiJZYMaMcH2nggKYOhX69Im6osxTQImIxNzbb8PQoeH2+PFw9NHR1tNaFFAiIjFWuynizDPh8sujrqj1KKBERGIqkYDvfz80Rey/f+43RdSlgBIRiamf/QyefRa22w5mzoTi4qgral0KKBGRGPr978PM5DVNEb17R11R61NAiYjEzFtvbW6KmDABjjoq2nqiooASEYmRL78MTRFr1sDgwTBiRNQVRUcBJSISEzVNEYsXh6aIyZPzqymiLgWUiEhMjB4Ns2fnb1NEXQooEZEYmD4dbr4ZCgth2rT8bIqoSwElIhKxt96Cs88OtydMgO9+N9JyYkMBJSISoRUr4KSTQlPEkCFw6aVRVxQfCigRkYgkEmH6ovfeg298Q00RdSmgREQi8tOfwh//CN26haaIDh2iriheFFAiIhGYNi3MTF5YCI8/Dr16RV1R/CigRERa2ZtvwrBh4fatt0JpaaTlxJYCSkSkFa1YEWaKqKyEs86CSy6JuqL4UkCJiLSSmqaI99+HAw+Eu+5SU0RjFFAiIq1k1KjQFLH99uES7mqKaJwCSkSkFUydCj//+eaZItQU0TQFlIhIhs2fD+ecE27ffruaIlKlgBIRyaAvvtjcFDF0KFx0UdQVZQ8FlIhIhlRVwRlnwJIl0L+/miKaSwElIpIhI0fCn/4EO+wQmiLat4+6ouyigBIRyYDHHoNf/AKKisJMET17Rl1R9lFAiYi0sDfegB/+MNy+/Xb4zneirSdbKaBERFrQF1/AKafA2rXhGk8XXhh1RdlLASUi0kKqqmDQoNAUcdBB8JvfqCliayigRERayDXXwJ//rKaIlqKAEhFpAb/7XZiZvKgIpk+HnXeOuqLsp4ASEdlKr7++uSniV7+Cww+Ptp5coYASEdkKn38emiLWrQvTGV1wQdQV5Q4FlIhImmqaIpYuhYMPhjvvVFNES1JAiYik6aqr4C9/ge7d1RSRCSkFlJndYGbzzewfZvaamekgVkTy2qOPhi/h1jRF9OgRdUW5pyjF9b4E+rv7RjPbHnjfzGa7+/sZrE1EJJZeew3OPTfcnjgRDjss2npyVUoB5e631fqxN7AaWFF7HTMbDgwH6N69O2VlZS1TYQasXr061vXFlfZbesrLy0kkEtp3aYjje668vA3nn38g69a15/jj/8Oeey4gZiUC8dx3zZXqERRm1hd4BtgOGOjuFbUfd/fJwGSA/v37e2mMr8hVVlZGnOuLK+239JSUlFBeXq59l4a4vec2boRjj4VPPoFDDoEZM3aiXbudoi6rXnHbd+lIuUnC3Re5e1/gKOBhM9snc2WJiMTPlVfCX/8KO+4Iv/89tGsXdUW5rdldfO7+OvAicGTLlyMiEk+PPBK+hNumTQin//qvqCvKfU0GlJnta2YDzUJ3v5n1AL4JzMt0cSIicfDqqzB8eLh9xx3w7W9HW0++SOUc1FLgfOBqM9sItAVGu/tLGa1MRCQGPv1080wR554L550XdUX5o8mAcveVgK5oIiJ5Z+NGGDgQli2Db30rHD1J69FMEiIiDbjiCpgzJzRFTJ+upojWpoASEanHww+HL+GqKSI6CigRkTrmzdvcFHHnnWqKiIoCSkSklk8+CU0R69eHhoiaKY2k9SmgRESSapoiPvwwHDVNnBh1RflNASUiknTZZfDCC+F80/Tp0LZt1BXlNwWUiAjw4IPw61+HUPr972GneE6xl1cUUCKS9155Bc4/P9y+884wEaxETwElInmtdlPE+efDj34UdUVSQwElInlrwwYYMAA++ggOPTRMBivxoYASkbx12WXwt7+pKSKuFFAikpfuvz+cb2rbFmbMCNMZSbwooEQk77z8MlxwQbj9m9/AN78ZbT1SPwWUiOSV5cvh1FPD+acf/xjOOSfqiqQhCigRyRsbNsDpp8PHH8Nhh8Htt0ddkTRGASUieePSS+Hvf4cePdQUkQ0UUCKSF+67L5xvatcuNEV07x51RdIUBZSI5LyXXgrnmyCE1MEHR1uPpEYBJSI5bflyOO20cP7pwgth2LCoK5JUKaBEJGfVboo4/HA1RWQbBZSI5Kyf/CQ0Rey8Mzz+eLh8u2QPBZSI5KR77oG77gpNETNnqikiGymgRCTnvPhiON8EcPfd0L9/tPVIehRQIpJTPv44NEVs3AgXXwxDh0ZdkaRLASUiOWP9+tAU8Z//wBFHwK23Rl2RbA0FlIjkjEsuCcN7PXvCtGlqish2CigRyQmTJ4elZqaIHXaIuiLZWgooEcl6c+fCRReF25MnqykiVyigRCSr1W6K+MlP4Kyzoq5IWooCSkSy1vr1IZyWL4fSUpgwIeqKpCUpoEQkK7mHYb2XXoJevdQUkYsUUCKSle6+G+69F9q3DzNFbL991BVJS1NAiUjW+fvfQ0s5hCmNvvGNaOuRzFBAiUhW+eijzU0Rl14KQ4ZEXZFkigJKRLJGTVPEJ5/AkUeqKSLXKaBEJCu4hwlgX345NEVMnQpFRVFXJZmkgBKRrHDXXXDffWqKyCcKKBGJvb/9bXNTxL33qikiXyigRCTWPvwwzFBeVQWXXQaDB0ddkbQWBZSIxNaGDQWceip8+ikcdRSMHx91RdKaUgooMzvPzOab2Twze9PMfpzpwkQkv7nD7bf35ZVXYJddYMoUNUXkmyb/c5tZIdAXONTdV5tZD+BdM5vl7h9lvEIRyUuTJsHs2TvRoQM88QR06xZ1RdLamgwod08AV9S66wtgA1BYez0zGw4MB+jevTtlZWUtV2ULW716dazriyvtt/SUl5eTSCS075ph/vzOXH7514ECLr/8bcrLP0W7r3ly4f9Xc/fmPcHsLqDA3Yc3tE7//v193rx5W1tbxpSVlVFaWhp1GVlH+y09paWllJeX88Ybb0RdSlZYtixcz+nTT2HgwGVMndoz6pKyUpz/fzWzV929yat2NatJwsxuBHoAF6VbmIhIQ9atCzNF1DRFDB/+XtQlSYRSDigz+wWwD3Cau2/IXEkiko/c4fzz4ZVXoHfvMFNEYWHzRngkt6TSJFEATAK6AAPcvSrjVYlI3vn1r+Ghh9jUFLHddlFXJFFL5QjqeOA8YFfg/8zspeRydGZLE5F8MWcOjBgRbt9/P3z969HWI/GQShffU4C1Qi0ikoc++AAGDIBEAq68Es44I+qKJC40k4SIRGbtWjj1VPjsM/jv/4abb466IokTBZSIRKKmKeLVV6FPnzBTRGFh08+T/KGAEpFI3HEHPPwwFBeHpoiuXaOuSOJGASUira6sLMxMDvDAA7DffpGWIzGlgBKRVlW7KeKqq2DgwKgrkrhSQIlIq1m7Fk45BT7/HI45BsaNi7oiiTMFlIi0CncYPhxeew123RUee0xNEdI4BZSItIpf/QoefRQ6dlRThKRGASUiGffXv8IVyYv2PPAA7LtvtPVIdlBAiUhGLV0aGiESCbjmmtAgIZIKBZSIZExl5eamiO99D268MeqKJJsooEQkI2qaIl5/HXbbDX73OzVFSPMooEQkI375S/jtbzc3RXTpEnVFkm0UUCLS4v7ylzAzOcCDD0K/fpGWI1lKASUiLWrJks1NEaNGwemnR12RZCsFlIi0mJqmiC++gOOOg+uvj7oiyWYKKBFpEe7wox/BG2/A7rurKUK2ngJKRFrEbbeF6Ys6dQpNESUlUVck2U4BJSJb7U9/CjOTAzz0EOyzT7T1SG5QQInIVnn/fRg0CKqr4ac/DZdwF2kJCigRSVtNU8SKFXD88XDddVFXJLlEASUiaXGHH/4Q5s+Hvn3Dl3LVFCEtSQElImm59VaYMkVNEZI5CigRabbnn4errw63H3kE9t472nokNymgRKRZ3ntvc1PE6NFw8slRVyS5SgElIilbsyYE0pdfwoknwtixUVckuUwBJSIpcYdzzoF//hO+9rVw+fYCfYJIBuntJSIpmTABpk2DbbYJTRGdO0ddkeQ6BZSINOmPf4SRI8PtRx6BvfaKth7JDwooEWnU4sVwxhmhKeJnP4OTToq6IskXCigRadCaNWGmiC+/hP/5HxgzJuqKJJ8ooESkXu4wbFhoithjjzC0p6YIaU16u4lIvcaPh8cfV1OEREcBJSJbmD07XK4dQjv5nntGW4/kJwWUiHzFu+/CmWeGIb6xY+F//zfqiiRfKaBEZJPVq8NMEeXloVtv9OioK5J8poASEWBzU8S//hWG9B5+WE0REi29/UQEgFtugenTYdttQ1PEtttGXZHkOwWUiPDss+Fy7RCaIvbYI9p6REABJZL3Fi2C738/DPFdd134Qq5IHKQUUGbWxsyuMLONZnZGposSkdaxatXmpoiTT4Zrr426IpHNUj2COhdw4KUM1iIircgdzj4b3n47TP6qpgiJm6JUVnL3SQBm1uDBv5kNB4YDdO/enbKyspaoLyNWr14d6/riSvstPeXl5SQSidjtu0cf7cWMGbvSsWMVI0e+yquvro26pC3oPZe+XNh3KQVUKtx9MjAZoH///l5aWtpSm25xZWVlxLm+uNJ+S09JSQnl5eWx2ndPPw333w9mMHVqESec8M2oS6qX3nPpy4V912IBJSLZYdEiGDw4DPHdcAOccELUFYnUTyPOInmkpimioiJcRqNmvj2ROFJAieSJ6moYOjQ0Rey9Nzz0kJoiJN709hTJE+PGwcyZ4bIZTzwRLqMhEmfNOgfl7qUZqkNEMuipp8Ll2s3gd7+Dvn2jrkikaWqSEMlxCxZsboq46SY4/vioKxJJjYb4RHLYypWhKWLlSjjtNBg5MuqKRFKngBLJUdXVcNZZ8M47sM8+8OCDYYhPJFsooERy1I03wqxZUFISmiI6dYq6IpHmUUCJ5KA//AHGjAlHTI89BrvvHnVFIs2ngBLJMQsWwJAh4fZNN8H3vhdtPSLpUkCJ5JDaTRGnnw7XXBN1RSLpU0CJ5IjqavjBD0JTRL9+8MADaoqQ7KaAyrCysjJ23HHHqMuQPHDDDfDkk2qKkNyhgGrAnnvuiZk1uDzxxBMAjB07tt7Hd9ppp4h/A8kns2bB2LFhbr0pU2C33aKuSGTrKaAa0a5dOzp27Fjv8q1vfQuAUaNGsWrVqq8sU6dOpX379hFXL/ninXfC0B6E+faOPTbaekRaiqY6asS9997LkJp2qAa0bduWtm3bfuW+qqoqttFMnNIKKirgpJPCZTQGDoSrroq6IpGWoyOoFvLFF1/w7rvvArBq1Sq6du0acUWS66qrQzv5woWw776br5ArkisUUA3o2LEj559/Pp06daJdu3YUFhbSqVOnTcsNN9zwlfUff/xxzj77bADeffddevTosemxTz75BDOjpKSkNX8FyXHXXRdmKe/SJTRFdOwYdUUiLUtDfHUkEgncnZdffnnTfZMmTeL+++9n3rx5X1m3qqqKwsJCrM6frf/617845JBDNv3cvXt3lixZssV6Iul64gm4/vrNTRG77hp1RSItTwFVx2677cbSpUvrfaxNmzZb3Ddz5kxOPvnkr9x35ZVXsnuduWXUNCEt5d//3twUcfPNcMwx0dYjkikKqDqWLFmS9nPdncrKSvbdd1+WL1/O/PnzddQkLaq8PDRFrF4NgwbBlVdGXZFI5iigGrBkyRJuuukmysrK+PjjjwHYZptt2GWXXRgwYACXXnop06dP58wzz/zK8zp37sy2227LDjvswM4778zw4cOjKF9yUE1TxKJFsN9+cN99aoqQ3KaAakBpaSnf/va3eeqpp+jbty8FBQVUVlby8ssvM2zYMFatWsWYMWM2De+ZGYWFhRQVfXWXlpWVRVC95KIxY+Dpp6FrVzVFSH5QQDWgqKiIDRs2sG7dOjZs2EDbtm1Zv349a9euJZFI0KZNGwoKCnRuSVrFjBnh+k4FBTB1KvTpE3VFIpmngGrAnDlzGD9+PEOGDGHZsmUkEgm6dOlCnz59uPrqq7nggguiLlHyxNtvw9Ch4fb48XD00dHWI9JaFFAN6NGjBxMnTtzq7ZSWlrJ8+fIWqEjyUXl5uHzG6tVw5plw+eVRVyTSevRFXZGYSiRg8ODQFPH1r8O996opQvJLXh1BuTvuHnUZIikZMwaeeSY0RcycCcXFUVck0rry5gjq9ddfZ4899mDgwIFRlyLSpBkzwuXa1RQh+SznA2rdunVcccUVHHrooSxatIhnn32WuXPnRl2WSIPeegvOOivcnjBBTRGSv3I6oObOncvXvvY1Jk2axNq1awGorKzklltuoaKiIuLqRLb05ZehKWLNmnD+acSIqCsSiU5OBtSaNWs477zzOProo1m2bNmmcKpRWVnJmDFjIqpOpH6JBHz/+7B4Mey/P0yerKYIyW851yTx5z//mcGDB1NRUcG6deu2eLy4uJjOnTtvujSGSFyMHg2zZ8N226kpQgRyKKAqKiq48MILmTlzJpWVlVs8bma0b9+eESNGcMQRR7D//vtHUKVI/aZPDzOTFxbCtGnQu3fUFYlELycC6sknn2TYsGGsWbOG9evXb/F4cXExvXv3ZurUqfTr10/z40msvPUW1BzQT5gA3/1upOWIxEZWn4P6/PPPOeWUUzjzzDNZsWLFFuFUUFBAhw4dGD16NPPnz6dfv34RVSpSv7pNEZdeGnVFIvGRlUdQ7s7UqVM577zzNk3mWlfHjh3ZY489mDJlCn379o2gSpHGJRJh+qLFi+GAA9QUIVJX1gXUxx9/zNChQ5k7d26955oKCwtp164dN998MxdeeCEFBVl9kCg57Npr4bnnoFs3NUWI1CdrAsrdue+++xgxYgTr1q2jqqpqi3U6duzIAQccwKOPPsouu+wSQZUiqZk2DW65ZXNThN6uIlvKioBasmQJgwcPZv78+axZs2aLx4uKimjfvj0TJ07k7LPP1mXWJdbefBOGDQu3b70Vjjwy2npE4iry8a/GJm+trq5m4sSJ7LPPPrz88sv1hlNxcTFHHXUUixYtYtiwYQonibUVK0JTRGVlmM7okkuirkgkviINqA0bNtCvXz+mTZu2xWMLFy6kf//+jBo1isrKShKJxFceb9u2LZ07d+bBBx9k9uzZ7Ljjjq1Vtkhaapoi3n8fDjwQ7rpLTREijYk0oG6//XYWL17Mueeey2effQZAVVUV48aNY//9929wSK+4uJgTTzyRxYsXM2DAgNYuWyQto0av/EFaAAAHLUlEQVTBH/8I228fZivv0CHqikTiLbJzUMuXL+f6669n/fr1VFdXc+6553LDDTcwaNAgli5dusX8eQDt2rWjU6dOPPTQQ5xwwgkRVC2SnvLyNvz855ubInr1iroikfhL6QjKzI4ws9fM7E0zm2dmh2ztC48YMYKNGzcCsHHjRp5//nkOPvhg/v3vf9fbPl5cXMygQYN47733FE6SNdzhk0/ggw9CD/ltt0FpabQ1iWQLa+oKs2ZWAiwGTnT3F82sFJgK9HH3LZME2GabbfzAAw9scJsrV65k/vz5VFdXN1lgQUEBRUVF7LnnnnTp0qXJ9VNRXl5OSUlJi2wrn2i/beYOVVVNL2vWQEXFGwDsuOP+7LFHxIVnGb3n0hfnfTdnzpxX3b1/U+ulMsR3LLDA3V8EcPcyM/sPcBTwh5qVzGw4MBygTZs2lJeX17sxd2fBggUphZOZ0aVLF3baaSfMrMFtNlcikWixbeWTXNpv7pBIWFpLdXVYmqNt2wTdu5eTI7uv1eTSe6615cK+SyWgdiUcQdW2OHn/Ju4+GZgM0L9/f583b169G3vggQe4+OKL653UtUZxcTHdunVj6tSpHHLIVo8mbqGsrIxSjbM0W5z228aNUFEB5eXpLfX03jRLQQGUlDS9dOkCkyaVUllZzhtvvNEyv3weidN7LtvEed+l+nWgVALKgESd+6pIowNw5cqVXHbZZfV25tWWSCR4/fXX6dq1a3NfQrLExo2pBUlDIbS1AVNYGAKkc+fUgqbu0qlT6i3i994bvvckIs2TSkB9CBxd575ewPTmvtjo0aPrvYhgXWbGyJEjufvuu5v7EtJKNmzYuiOYrf3ArgmYdJeOHfUdJJG4SyWgZgG3mdm+7v5PMzsY2BN4vjkvtHDhQu65556UAmrdunU88sgjnHXWWRx66KHNeRlJ0YYNqR+t1CwffXTQpiMfBYyIZFqTAeXuFWY2ALjfzJwwvHe8uzfr7Nvw4cMbPe8EYU69Dh06UFBQQGVlJRMmTFBANaC+gGnOUs/XzFLQcdOtwsJwfqWhAGlq6EwBIyJNSemLuu7+V+CgdF/kueeeY86cORQXF9OmTRuqq6tZu3Ytbdu2ZYcddqBnz5707duXvn370qtXL3r16kXPnj3p0aNHui8Ze+vXb90QWXoBs1lRUfOPWhYu/AfHHHMwJSXh0hAKGBHJpFaZSaJr165ce+219O7dm549e24KoI4dOzb95Jhav37rjmBSGOlsVDoBU3tJJ2A2bqwkh/9mEJGYaZWAOuiggzjooLQPwDIiDgHT2BBZU8NmOoIRkVyXFdeDqs+6dc0LlNrDaStWHE49V4lvlnQCpvbSoYMCRkSkMZEFVHMDpu7SRL9FEwpp0yYETLrfg1HAiIhkVkYCavlyuOaaTAYMmwImnXB5880XOOaY7yhgRERiLCMB9dFHMH584+tsTcCUlED79ukfwSxYUK1wEhGJuYwEVPfu8JOfZC5gREQk92UkoHbeGUaOzMSWRUQkX0R6yXcREZGGKKBERCSWFFAiIhJLCigREYklBZSIiMSSAkpERGJJASUiIrGkgBIRkVhSQImISCwpoEREJJbM3Vt+o2afAUtbfMMtpxvwedRFZCHtt/Rp36VH+y19cd53u7j79k2tlJGAijszm+fu/aOuI9tov6VP+y492m/py4V9pyE+ERGJJQWUiIjEUr4G1OSoC8hS2m/p075Lj/Zb+rJ+3+XlOSgREYm/fD2CEhGRmFNAiYhILCmgREQklvI+oMxsHzNbYWZjo64lG5jZeWY238zmmdmbZvbjqGuKMzM7wsxeS+6reWZ2SNQ1ZQO9z7ZeLny2FUVdQJTMrAS4E3gs6lqygZkVAn2BQ919tZn1AN41s1nu/lHE5cVO8v01AzjR3V80s1Jglpn1cffKaKuLL73Ptl6ufLbl7RGUmRUADwGjgM8iLicruHvC3a9w99XJu74ANgCFEZYVZ8cCC9z9RQB3LwP+AxwVZVFxp/fZ1smlz7acPoIys7bAC/U89B1gLPC8u881s2NatbCYa2y/ufuGWj//Epjq7h+0TmVZZ1dgcZ37Fifvl9TpfdY8N5Ijn205HVDJD9MtxvzN7DSgl7uPav2q4q+h/Vabmd0I9ABOa5WispMBiTr3VZHHIxfNpfdZ8+TaZ1tOB1QjjgP2MrOXkj/vDOGkorsPiK6s7GBmvwB2A06rc0QlX/UhcHSd+3oB0yOoJevofZaWnPps00wSQE2Xi7uPjbaSeEuObU8CugCD3b0q4pJizcw6E4b0jnT3f5rZwcBzQB93L4+2uvjS+6zlZPtnW74eQUl6jgfOA+YB/2dmNfdf6+5/iqyqmHL3CjMbANxvZk4Y3jte4dQkvc8E0BGUiIjElE7WiohILCmgREQklhRQIiISSwooERGJJQWUiIjEkgJKRERiSQElIiKxpIASEZFY+n/Excl4R1s4GwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(z, leaky_relu(z, 0.05), \"b-\", linewidth=2)\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([0, 0], [-0.5, 4.2], 'k-')\n",
    "plt.grid(True)\n",
    "props = dict(facecolor='black', shrink=0.1)\n",
    "plt.annotate('통과', xytext=(-3.5, 0.5), xy=(-5, -0.2), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.title(\"Leaky ReLU 활성화 함수\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.5, 4.2])\n",
    "\n",
    "save_fig(\"leaky_relu_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "텐서플로에서 Leaky ReLU 구현하기:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_텐서플로 1.4에서 tf.nn.leaky_\\__relu(z, alpha) 함수가 추가되었습니다._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def leaky_relu(z, name=None):\n",
    "    return tf.maximum(0.01 * z, z, name=name)\n",
    "\n",
    "hidden1 = tf.layers.dense(X, n_hidden1, activation=leaky_relu, name=\"hidden1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Leaky ReLU를 사용하여 신경망을 훈련시켜 보죠. 먼저 그래프를 정의합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=leaky_relu, name=\"hidden1\")\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=leaky_relu, name=\"hidden2\")\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "데이터를 로드합니다:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "주의: `tf.examples.tutorials.mnist`은 삭제될 예정이므로 대신 `tf.keras.datasets.mnist`를 사용하겠습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
    "X_train = X_train.astype(np.float32).reshape(-1, 28*28) / 255.0\n",
    "X_test = X_test.astype(np.float32).reshape(-1, 28*28) / 255.0\n",
    "y_train = y_train.astype(np.int32)\n",
    "y_test = y_test.astype(np.int32)\n",
    "X_valid, X_train = X_train[:5000], X_train[5000:]\n",
    "y_valid, y_train = y_train[:5000], y_train[5000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shuffle_batch(X, y, batch_size):\n",
    "    rnd_idx = np.random.permutation(len(X))\n",
    "    n_batches = len(X) // batch_size\n",
    "    for batch_idx in np.array_split(rnd_idx, n_batches):\n",
    "        X_batch, y_batch = X[batch_idx], y[batch_idx]\n",
    "        yield X_batch, y_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from tensorflow.examples.tutorials.mnist import input_data\n",
    "# mnist = input_data.read_data_sets(\"/tmp/data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 배치 데이터 정확도: 0.86 검증 세트 정확도: 0.9048\n",
      "5 배치 데이터 정확도: 0.94 검증 세트 정확도: 0.9498\n",
      "10 배치 데이터 정확도: 0.92 검증 세트 정확도: 0.9656\n",
      "15 배치 데이터 정확도: 0.94 검증 세트 정확도: 0.971\n",
      "20 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9762\n",
      "25 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9776\n",
      "30 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.978\n",
      "35 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9786\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 40\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        if epoch % 5 == 0:\n",
    "            acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "            acc_valid = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "            print(epoch, \"배치 데이터 정확도:\", acc_batch, \"검증 세트 정확도:\", acc_valid)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ELU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def elu(z, alpha=1):\n",
    "    return np.where(z < 0, alpha * (np.exp(z) - 1), z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmYFNW9xvHvj1UQFBRFwxJUBHFBbhxRUXGMC0vAqIAbmhijgxpz4SpqxCXGNXohoslFJSJG3HABCcimMS0qiI44CBgGISBrlK3BYXFg5tw/Tg+zMlv3TFX3vJ/nqWd66lR1/6Ys+rWqTp0y5xwiIiJhUy/oAkRERMqigBIRkVBSQImISCgpoEREJJQUUCIiEkoKKBERCSUFlIiIhJICSkREQkkBJSnFzMaY2aoi04gibdPM7P4qvl+amVX6bnYzi5jZjeW0NzazaIkprUj7KjMbWM76p5mZM7MjK/9XiCQnBZTUmNiXdY6Z/afENK9I+x/3s+6qsr7ozewhM4uUMf8cM1sF9C3RlBF7r7YV1Hpk7Iu/YyX+ruWxZQum/QZSSc65H5xzLUpMmZVdH7gU2Az8pwrrFNQ90sxuq+p68TCznmb2dzNbF9tW15Zob2NmX5hZq9qsS5JDg6ALkJT3F+fc72r6Q5xzHwAdzOwIoDvQHNgCfOSc+x7AzMp7iyOAPGBdbNlm+H8fzQoWMLPjgeOB+yj+P3d7Ykc9/yjvA8xsGnDWfpqXOee6V7D++cBQ4HtgpJnd7pzLL2+dIuteBKTjt01tagYsBl6MTcU459aZ2bPAC0C/2i1Nwk4BJSnDzHoBU4Fs/FFGe6CFmZ3mnPs6ttgBZtYC2OGc21Nk9YtiP3vggyYCnFLiIw4HTiynhLnl1eec6xer80TgVHzIfe6cyyqxaNOiNZpZI2AE8DvgVmAGMB3obmb3xMJ5v2Lr/wkYXjTQzOwU4HH837wOuA4f1EOdc2eW956V5ZybHqsVM3thP4uNA+42s17OuVmJ+FxJDTrFJ6nkKGABcKpzLh1/tJCP/9ItcCewFRhQMMPMugLDgOHAS2bWyTmX5pwzfJAA4JyLAI8ATYFfAr8BBgHfOefud86tr6hAM3sMmAy0jdX1mpk9U2KxvxXUaGYPA+tj9fZyzo1xzq3Eh+dHwDtm9rWZnVzOx/4sVvOUInWcCnwI/BPoCnwC/AG4G7i3jLpHxE7XljedXdHfX5bY/yiMA35bnfUlhTnnNGmqkQl/FJKDv15SdBpWpP2P+1l3FXBjGfMfAiL7WceAS4Dn8V/wjwDn4P9HrAkwDbi/xDpnARuA22K/XxWr8ddAfSDN/zPZt/z9wEKgTez3nwBRoE+Rv+kW4ACgfonPaoI/jXhKkXmdAAccXuTvHlikvXdsqrefv7k5/ujPyvnvMAF4ocS8OcDEIr/3jdX2z/28xyFAxwqmJhXsDznAtftpOwPIBZoFvd9qCs+kU3xS06p7Daq8nnOl2szsZaDgKOJAoAVwBTAQ2IU/aim5Tk/gLWCIc24SgHPuFTPLAu6KtZV0PP5LfF1s+QVmthh/6m9GbJk/x6b/AUbvK9q5XWb2DTDQzJbjj+4uB77DB2rpP9S5mfvfDOD89bW/l7cM0Bl4u8jffQRwNnBukWVy8UFe6ugp9jlb8Nf0asoyoCFwDP5/AEQUUBJa2/FBU9JBwLYy5t+M/4LLB/YCO51ze4suYGZnUiQInHNzzOzo2Jc8ReZ/BVwTW2cLMLFI82vA+FiPweX4I7CT8ddvCtzknCt52q5AH+Bh/JewAV8AF7jC62GfABtjnx3BHwFWVhfn3NIy5jcDdhRdLvbzsyLzOgPZzrmPynrjWHf9EWW1FdHHOfdhZYstISf286Bqri8pSAElYfU1cEIZ808Cviw50zm3DfZd4/ll7HVZ7zsS+KrIegU9/Lrgr7+cAbTBn95bgT86ubHI8pPM7D/A9cAT+NOI3ZxzKyrzRznnss3saqAxhUeCzsyax36/Hn/EB3AePsSK6ovvDVeqW3bJQC5iLcWvw7WIfVY+QOyz76b8ruvPAK+X0w6xHpDVVHBf1+o43kNSjAJKwurPwHQzm4U/1dYA/+XdnfIvpj8A/O9+2l6kSLfxArF7pD4BXgX6AyvxAdIVeBSYBZxWsLxzbq6Z7QR+BTwJNDazn1Thb7sfuIPCgDKKB9Eg4E3nXF4ZtebHathfGJXlY+CnRX7Pin3eXbFTo/+Lvw7X0cyOdYU9Hvep7im+WHf9gnvL6gHtzawbsMU5VzSMeuDDSQEl+6gXn9S0W/Z3o2457QOc7zo9EH/fz3f4nmw/B37qnFtczueNwgdMZhnT8fgODSWdhe9kcKNz7ivn3C7nXNQ5Nwffs6+7mR1qZgvNbLeZ7Y69H/gv1Pn4Xmg/q8J2meScqx+b6jnnzPleg99U4T0q6xXg9ILRJ5zvBXg3cBP+VOP3wPn4+5XK7SpfDWn405hf4DuJ/CH2+oESyw0AXnbOVXrUDkl9pv1BUklBl23nXKVHdzCzdsASfPfuMfjQaYQ/xfgwvmfZKbFTYfUpPD22q4zrXBHgtXKuQREbPaMLsVORJXwJ3Oqce3M/6/YDXnLOtajs3xdbbxz+qOX2qqxXG8zsWGAe0Nk5tznoeiQ8dAQlqejX5dyr837JhZ1za/CnmA7F3+i7EX/K66/4jgQXxJb7PnZktS32uiqn2Uq6CN9ho+TULo73LM+dQH8z61xD7x+PJ4H/UThJSTqCEqkjYiNYHO2cq6hbeq0xszb4XowvBF2LhI8CSkREQkmn+EREJJRqvZt5q1atXIcOHWr7Yytlx44dHHhgWfeGyv5om1VNdnY2eXl5HH/88UGXklSSbT/bvh2WLwfnoE0bOOKIitepCWHdbp9//vkm59xhFS1X6wHVoUMHMjOr8vib2hOJREhPTw+6jKSibVY16enpRKPR0P4bCKtk2s8yM+Hcc304DRsGf/oTlP+kl5oT1u0WG/KrQjrFJyKSIF9/DX37Qk4ODB4Mo0YFF06pQAElIpIA69fDhRfCxo3Qqxc8/zzU0zdsXLT5RETiFI1C796wahV07w5vvgmNGgVdVfJTQImIxGHXLrjoIli0CDp3hnfegWalRnyU6og7oMzswdgYZZ+a2QIzuykRhYmIhN3evXDllfDhh7633uzZ0KrUOPNSXYnoxbcVSHPO7TGzw4CVZjYzNiCliEhKcg5uvBGmTIGWLWHWLGjfPuiqUkvcAeWc+1ORXzvgHzxWbFh+M8sAMgBat25NJBKJ92NrRE5OTmhrCytts6qJRqPk5eVpm1VRGPez5547ipdf/jGNG+fxwAML2bhxOyErMZTbrSoSMtRRbDTi6fjBNi9zzr23v2XT0tJcWO8BCes9A2GmbVY1BfdBZWVlBV1KUgnbfvbUUzB0KNSv74+gflaVB63UorBttwJm9rlzLq2i5RLSScI597Vz7lj8E0BfNLOynoQqIpL0Xn3VhxPAuHHhDadUkNBefM65L/DPdTk3ke8rIhIGs2fDL2NP8Xr88cLXUjPiCigzO8nMLjPz90rHhs4/jcKnjYqIpIRPP4VLL4U9e+C22+D20D36MfXE20niG+BG4E4z24N/Cum9zrlP4q5MRCQksrP9EEY7dsA11/ijJ6l5cQWUc2478JsE1SIiEjrr1vkhjDZv9iE1bpyGMKot2swiIvuxdasfwmj1ajj9dHj9dWjYMOiq6g4FlIhIGXbuhP79YfFi6NIFpk2DED5aKaUpoERESti7Fy6/HD7+GNq29aNEHHpo0FXVPQooEZEinIOMDH/EdMghvmt5u3ZBV1U3KaBERIq46y4YPx6aNvUjk3fpEnRFdZcCSkQk5okn4LHHoEED/0yn008PuqK6TQElIgK89BLceqt/PX489OkTbD2igBIRYcYM+NWv/OtRo+Dqq4OtRzwFlIjUafPnw8CBvufeHXcUHkVJ8BRQIlJn/etffnSInTvh2mvhj38MuiIpSgElInXSmjXQqxds2QL9+sFf/wp+2GsJCwWUiNQ5W7b4IYzWrIEzz4SJE33PPQkXBZSI1Ck7d/ojpq++ghNOgKlT/T1PEj4KKBGpM/bsgUGDYN48aN/eD2HUsmXQVcn+KKBEpE7Iz4frr4fp0/24erNmQZs2QVcl5VFAiUidcOed8OKLfkTy6dPhuOOCrkgqooASkZQ3cqSfGjSASZOge/egK5LKUECJSEp78UW4/Xb/+m9/80/HleSggBKRlPXOO3Dddf716NFw1VXB1iNVo4ASkZQ0d67vsZeX5x+hMXRo0BVJVSmgRCTlLFni73XatQt+/Wt4+OGgK5LqUECJSEpZvdoPYbR1K/z85/DMMxrCKFkpoEQkZWza5MNp3To4+2x49VUNYZTMFFAikhJ27PCn9ZYuhZNOgr//HZo0CboqiYcCSkSS3p49/plO8+dDhw4wcya0aBF0VRIvBZSIJLX8fP803JkzoVUrP4TRj34UdFWSCAooEUlazsHw4fDyy9CsmX90e6dOQVcliaKAEpGk9fjj8MQT0LAhTJ4MaWlBVySJpIASkaQ0fjz87ne+C/mECXD++UFXJImmgBKRpDN1Ktxwg3/91FNw+eXB1iM1QwElIknlo4/gssv8EEb33AO33BJ0RVJTFFAikjQWLYL+/WH3bsjIgAceCLoiqUkKKBFJCt98A717QzQKl1wCY8ZoCKNUF3dAmdkQM1toZplm9qWZ3ZyIwkRECkSjDbnwQli/Hs45B155BerXD7oqqWlxjVJlZvWBY4EznXM5ZtYGWG5mU5xz6xJSoYjUaTk5cNddJ7FsGZx8MkyZAgccEHRVUhviCijnXB4wvMiszUAuoP+3EZG45ebCpZfC0qUHcdRR/kbcgw8OuiqpLYke53c0MNE5t7roTDPLADIAWrduTSQSSfDHJkZOTk5oawsrbbOqiUaj5OXlaZtVQn4+PPxwF95/vzUtWvzAgw9mkZ29i+zsoCtLHsn+7zNhAWVmDwFtgAEl25xzY4GxAGlpaS49PT1RH5tQkUiEsNYWVtpmVdOiRQui0ai2WQWcg2HD4P33oXlzeOyxRQwefFrQZSWdZP/3mZCAMrORwDHAAOdcbiLeU0Tqrkcf9TfgNmoEb78N9erlBF2SBCCuXnxmVs/MngHaAYMUTiISr+eeg7vv9l3IX34ZfvrToCuSoMTbzbwvMAQ4GvjIzD6JTRoVS0Sq7O23YcgQ//r//s8/40nqrnh78U0DdKuciMRtzhy44grfOeL3v4ebbgq6IgmaRpIQkcB9+SVcdBH88IM/gvr974OuSMJAASUigVq5Enr1gm3bYMAAf2pPQxgJKKBEJEDffQcXXgj/+Q+kp8NLL2kIIymkgBKRQHz/PfTtC8uXQ7duGsJISlNAiUit++EHPyL555/D0Uf7IYwOOijoqiRsFFAiUqvy8uAXv4B//ANat4bZs+GII4KuSsJIASUitcY5GDoUXn/dD2E0YwYcc0zQVUlYKaBEpNY89JDvpdeokb/m9F//FXRFEmYKKBGpFc8+C/fd57uQv/IKnHtu0BVJ2CmgRKTGTZoEN8eetf300/5+J5GKKKBEpEZFInDllX4Ioz/8oXCsPZGKKKBEpMZkZcHPf+6fjPub38C99wZdkSQTBZSI1IgVK6B3b9i+HQYNgief1BBGUjUKKBFJuG+/9ePrffstnHceTJigIYyk6hRQIpJQ27dDnz7+COonP4HJk6Fx46CrkmSkgBKRhNm9Gy6+GL74Ajp29DfiNm8edFWSrBRQIpIQeXlw9dXwz3/6oYtmz4bDDw+6KklmCigRiZtzcMst8NZbftDXmTPhqKOCrkqSnQJKROL2hz/AM8/4a01Tp8LJJwddkaQCBZSIxOXpp31A1asHr70GPXsGXZGkCgWUiFTbG2/4G3DBj7V38cXB1iOpRQElItXy/vu+U4RzfpTy668PuiJJNQooEamyBQsKhzD67W9hxIigK5JUpIASkSpZvtzfiJuTA1dcAaNHawgjqRkKKBGptA0b4MIL4bvv4IIL4G9/850jRGqCdi0RqZRt2/yR08qVkJbm73lq1CjoqiSVKaBEpEK7d/trTgsXQqdOMH26hjCSmqeAEpFy5eXBVVfBBx/Aj34Es2bBYYcFXZXUBQooEdkv5/yj2idPhhYt/BBGHToEXZXUFQooEdmv3/8exo6FAw7wQxiddFLQFUldooASkTL95S/w4IP+QYMTJ8JZZwVdkdQ1CigRKeX11+G//9u/HjsWLroo2HqkblJAiUgx771XOITRo4/CddcFXZHUVXEHlJk1NLPhZrbHzK5IRFEiEozMTLjkEtizB4YNgzvvDLoiqcsScQR1A+CATxLwXiISkK+/hr59/RBGgwfDqFEawkiC1SDeN3DOjQEws/7xlyMiQVi/3g9htHEj9OoFzz+vIYwkeHEHVGWYWQaQAdC6dWsikUhtfGyV5eTkhLa2sNI2q5poNEpeXl6otllOTgOGDu3GqlXNOO647QwdupC5c/OCLqsY7WfVk+zbrVYCyjk3FhgLkJaW5tLT02vjY6ssEokQ1trCStusalq0aEE0Gg3NNtu1yx8x/fvf0LkzfPjhQbRqdXbQZZWi/ax6kn276SBepI7auxeuvBI+/BDatIHZs6FVq6CrEimkgBKpg5yDG2+EKVOgZUs/vl779kFXJVKcAkqkDrrnHhg3Dpo0gWnT4IQTgq5IpLSEXYNyzqUn6r1EpOY89RQ88ogfwuiNN6BHj6ArEimbjqBE6pBXX4WhQ/3rcePgZz8Lth6R8iigROqI2bPhl7/0rx9/vPC1SFgpoETqgE8/hUsv9UMY3XYb3H570BWJVEwBJZLisrP9EEY7dsA11/ijJ5FkoIASSWHr1vkhjDZv9iE1bpyGMJLkoV1VJEVt3Qq9e8Pq1XD66f4ZTw0bBl2VSOUpoERS0M6d0L8/LF4MXbr4e50OPDDoqkSqRgElkmL27oXLL4ePP4a2bf0oEYceGnRVIlWngBJJIc5BRoY/YjrkEN+1vF27oKsSqR4FlEgKuesuGD8emjaFd97xp/dEkpUCSiRFPPEEPPYYNGgAb77pO0aIJDMFlEgKeOkluPVW/3r8eOjTJ9h6RBJBASWS5GbMgF/9yr8eNQquvjrYekQSRQElksTmz4eBA33PvTvuKDyKEkkFCiiRJPWvf/nRIXbu9AO//vGPQVckklgKKJEktGYN9OoFW7b4R2b89a9gFnRVIomlgBJJMlu2+CGM1qzxDxvUEEaSqhRQIklk507o1w+++so/pn3qVH/Pk0gqUkCJJIk9e2DQIJg3z48OMXOmHy1CJFUpoESSQH4+XH89TJ/ux9WbPduPsyeSyhRQIkngzjvhxRcLhzA67rigKxKpeQookZAbOdJPDRrApElw2mlBVyRSOxRQIiH24otw++3+9Qsv+K7lInWFAkokpN55B667zr9+4gkYPDjYekRqmwJKJITmzvU99vLy4He/g2HDgq5IpPYpoERCZskSf6/Trl3+COqRR4KuSCQYCiiREFm92l9n2roVLroInn1WQxhJ3aWAEgmJTZt8OK1bB2edBa+95nvuidRVCiiRENixw5/WW7oUTjwR/v53aNIk6KpEgqWAEgnYnj3+mU7z58OPfwyzZkHLlkFXJRI8BZRIgPLz/dNwZ86EVq38EEY/+lHQVYmEgwJKJCDOwfDh8PLLcOCBfpy9Tp2CrkokPBRQIgF5/HF/A27DhjB5Mpx6atAViYSLAkokAOPH+xtwzfxwRhdcEHRFIuETd0CZ2TlmtsDMvjSzTDM7PRGFiaSq7dsbcsMN/vWTT8IVVwRbj0hYxXWXhZm1ACYB/Zxz88wsHZhiZkc553YmokCRVBKNwqpVTXEO7r4bfvvboCsSCa94bwPsBWQ75+YBOOciZrYBOA+YWtYK2dnZpKenx/mxNSMajdKiRYugy0gq2maV9/33sHBhFgBHHpnORx9BSP8phI72s+pJ9u0Wb0AdDawoMW9FbP4+ZpYBZAA0bNiQaDQa58fWjLy8vNDWFlbaZpWze3d9li9vBkD9+vkcdlgUbbbK035WPcm+3eINKAPySszbS4lrW865scBYgLS0NJeZmRnnx9aMSCQS2qO7sNI2q9jKlX7oorw8OOSQdNq2je47kpLK0X5WPWHdblbJASbj7SSxFmhfYl772HyROm/FCn8ab/166NkTTjhBg7+KVFa8ATUF6GpmJwGYWXfgOODdeAsTSXbZ2XDOOX6E8tNP9+Pr1dONHSKVFtcpPufcNjMbBDxvZg5/eq+vcy55T3qKJMCSJXDeefDtt3D22f7puM2bB12VSHKJezB/59w/Ad0DLxKzYIF/bMamTT6kpkzxQxmJSNXohINIAs2c6U/rbdoEffvC1KkKJ5HqUkCJJMi4cf6ZTjk5cNVVMGmSnukkEg8FlEicnIP77oPrr/ddyUeMgAkToHHjoCsTSW56oLRIHHJy4Lrr4I03oH59GDMGMjKCrkokNSigRKrp66/h4ovhq698D73XXvPXnUQkMXSKT6Qapk2DtDQfTscdB599pnASSTQFlEgV5ObCHXdA//6wfTtceil8+il07hx0ZSKpR6f4RCopO9v3zluwwF9vevDBwocOikjiKaBEKuCc70I+dCjs3AkdOsArr8AZZwRdmUhq0yk+kXKsWgW9e8MNN/hwGjwYsrIUTiK1QQElUob8fPjzn+HEE2H2bGjZEl56yU8HHxx0dSJ1g07xiZTwxRfwm9/AvHn+90GDfFi1bh1sXSJ1jY6gRGI2bYIbb4RTTvHhdMQRfrii119XOIkEQQEldV5uLjz1FHTqBM8+63vo3XorLF0Kl1wSdHUidZdO8UmdlZfne+Pdd5/vDAFwwQXw5JPQpUugpYkIOoKSOig/HyZPhm7d4Be/8OF0/PHw9tswa5bCSSQsdAQldcbevTBxIjz6qH/iLUD79vDAA3D11f7UnoiEhwJKUl5ODrz4IowaBf/+t5/Xtq0fsigjQ4/FEAkrBZSkrFWr4C9/geeeg23b/LyOHf3wRNdcA40aBVqeiFRAASUpJTfXjzT+/PMwY4a/3gTQo4cfqujSS6GB9nqRpKB/qpISFi/2oTRhgr+fCaBhQ7jySh9Mp54abH0iUnUKKEla33wDb73lHxT42WeF8084AX79a9/x4bDDgqtPROKjgJKk8u9/+1B6443ioXTQQf5RGNdd5x8kqEdgiCQ/BZSE2t698MknMHMmTJ/ux8kr0LQp9OsHAwb4n02bBleniCSeAkpCZ+1aP4L4jBnw7ruFPfAAmjXzT7MdONA/BkOhJJK6FFASKOdg5Ur44AOYM8dPBfcqFejUyYdRnz5wzjnQpEkwtYpI7VJASa3KyfGPTP/sM/j0U/j4Y1i3rvgyBx0EPXv6QOrdG44+OphaRSRYCiipMZs3w6JFfsrK8qG0ZEnhvUkFDj0Uzj7bHx317Aknn6xhh0REASUJ8P33sGyZD6LFiwtDacOG0ss2aOADqHt3P512mh+ctZ6GLRaREhRQUik5ObB8OXz9tZ8KXi9Z0oOtW8te58AD/T1JJ50EXbv6m2W7ddM1JBGpHAWUkJvrrwOtXg1r1vip5OtodH9rN6JxYzjmmMIwKpiOOkpHRiJSfQqoFOQc7NoFW7bAt98Wn777rvS8TZv8OuVp3NgHzrHHFp82bZrHoEFnKIhEJOEUUCGUnw87dvhrOzk5xX9Go7B1qw+foj9LzsvNrfzn1asHbdr4ZyO1a+enkq9btSp7dIZI5AeFk4jUiLgCyswaAkOBR4FrnHOvJaSqEHHOPxp8zx7/pb9nT+Hr3bv9tGuXn6ryeudOHzolAygnx4dTvBo3hkMOgdat4fDD/c+ypsMP9+PVaYRvEQmbeL+WbgAc8EllV9iyxT88Li+vcMrPL/57deaXt+zevcWDpeTrgp85OadjVnq5ik5/1YQDD4Tmzf3ICQU/mzWDFi188LRsWfiz6OuCn+qIICLJzlwCvn3NLAI8U5kjKLPmDk4pMfcy4GZgJ9C3jLWujU2bgIFltN8EXA6sAa4po/02oD+QDQwpo/0e4HwgCxhWRs2P0KhRD+rXn0tu7gjM/Omu+vX96bEOHUbTqlU3dux4j5UrH6JevcK2evWgX79nadu2MytWTOWjj0btm1+/vp8eeWQCHTu24/33J/Lyy0+XugfozTffpFWrVrzwwgu88MILpeqbPn06TZs2ZcyYMbz++uul2iORCAAjR45k2rRpxdqaNGnCjBkzAHjwwQf5xz/+Uaz90EMP5a233gLgrrvuYt68ecXaGzZsyLvvvgvAsGHDyMrKKtbeqVMnxo4dC0BGRgbLli0r1t6tWzdGjx4NwNVXX83atWuLtZ9xxhk8+uijAAwYMIDNmzcXaz/vvPO49957AejTpw+7du0q1t6vXz+GDx8OQHp6OiVddtll3HzzzezcuZO+fUvve9deey3XXnstmzZtYuDA0vveTTfdxOWXX86aNWu45prS+95tt91G//79yc7OZsiQIWRlZbF3717S0tIAuOeeezj//PPJyspi2LDS+94jjzxCjx49mDt3LiNGjCjVPnr0aLp168Z7773HQw89VKr92WefpXPnzkydOpVRo0aVap8wYQLt2rVj4sSJPP3006Xaw7LvDR48mHUl7uhu27YtL730EqB9b3/7XvPmzTnyyCMZMqT0916Q+94HH3zwuXMurdRKJVR4BGVmjYA5ZTT1dM5V6kqHmWUAGQD16jXl4INzY/N9e5s2UY4+eh35+TuZN++HffPNfHh27LiJTp1WsmfPZt5/fzfgiiwDXbtu4LjjssnJ2cDMmTuLvbeZo0eP1ZxwwiK2bl3J5Mk5mBX0LvPvc/HFS+natRnLl2fxxhvbAUe9egXv4bjhhgWceGIuixcv5rnnSndnu+WWTDp2jPL55wuZMKF0+xlnzKd9+w2YLWLBgtLtOTnzWL9+BevXL+H770u3f/zxxxx88MEsXbqUaBnd6ebMmcMBBxzAsmXLymwv+JJYsWJFqfZdu3bta1+5cmWp9vz8/H3tq1evLtXesmXLfe1r164t1b5+/fp97evXry/Vvnbt2n3t3377ban21atX72vvORNgAAAFe0lEQVTfuHEj27dvL9a+cuXKfe1btmzhhx9+KNa+YsWKfe1lbZtly5YRiUTYvXt3me1Lly4lEomwbdu2MtuXLFlCJBLhu+++K7N90aJFNG/efN+227t3L865fcsuXLiQBg0asHz58jLXX7BgAbm5ft8rqz0zM5NoNMrChQvLbJ8/fz4bNmxg0aJFZbbPmzePFStWsGTJkjLbw7Lv5ebmlmpv2LCh9r0K9r2uXbsyf/78MtuD3vcqo9aPoNLS0lxmZmbcn1kTIpFImf+nI/unbVY16enpRKPRUv+3L+XTflY9Yd1uZlapIyj1vxIRkVBSQImISCgpoEREJJQScveLcy49Ee8jIiJSQEdQIiISSgooEREJJQWUiIiEkgJKRERCSQElIiKhpIASEZFQUkCJiEgoKaBERCSUFFAiIhJKCigREQklBZSIiISSAkpEREJJASUiIqGkgBIRkVBSQImISCgpoEREJJQUUCIiEkoKKBERCSUFlIiIhJICSkREQkkBJSIioaSAEhGRUFJAiYhIKCmgREQklBRQIiISSgooEREJJQWUiIiEkgJKRERCSQElIiKhpIASEZFQUkCJiEgoKaBERCSUFFAiIhJKCigREQmluALKzIaY2UIzyzSzL83s5kQVJiIidVuD6q5oZvWBY4EznXM5ZtYGWG5mU5xz6xJWoYiI1EnVDijnXB4wvMiszUAuUL/ksmaWAWQAtG7dmkgkUt2PrVE5OTmhrS2stM2qJhqNkpeXp21WRdrPqifZt5s558pfwKwRMKeMpp7Oudwiyz0D1HPOZZT3fmlpaS4zM7M6tda4SCRCenp60GUkFW2zqklPTycajZKVlRV0KUlF+1n1hHW7mdnnzrm0ipar8AgqFkKnV/BhDwFtgAGVrlBERKQc1T7FV8DMRgLHAAOKHlGJiIjEI55OEvWAMUBLYJBzbm/CqhIRkTovnm7mfYEhwNHAR2b2SWw6PzGliYhIXRZPL75pgCWwFhERkX00koSIiISSAkpEREKpwvugEv6BZhuBb2r1QyuvFbAp6CKSjLZZ1WmbVZ22WfWEdbv92Dl3WEUL1XpAhZmZZVbm5jEppG1WddpmVadtVj3Jvt10ik9EREJJASUiIqGkgCpubNAFJCFts6rTNqs6bbPqSertpmtQIiISSjqCEhGRUFJAiYhIKCmgREQklBRQ+2FmJ5jZFjO7P+haws7MhpjZQjPLNLMvzezmoGsKIzM7x8wWxLZRppmV+5w10b4Vj1T4Dov7eVCpyMxaAP8HvBp0LWFnZvWBY4EznXM5ZtYGWG5mU5xz6wIuLzRi+9QkoJ9zbp6ZpQNTzOwo59zOYKsLJ+1b1Zcq32E6gioh9pyrvwEjgI0BlxN6zrk859xw51xObNZmIBeoH2BZYdQLyHbOzQNwzkWADcB5QRYVZtq3qieVvsPq5BGUmTUC5pTR1BO4H3jXOTfXzC6s1cJCrLxtVuJJyqOBic651bVTWdI4GlhRYt6K2HypHO1blfMQKfIdVicDKvaFWur8v5kNANo750bUflXhtr9tVpSZPQS0AQbUSlHJxYC8EvP2orMYlaJ9q3JS7TusTgZUOfoAXczsk9jvbcFfbHTODQqurPAzs5HAMcCAEkdU4q0FSj5tuj3wZgC1JBXtW1WSUt9hGkmiHAW9X5xz9wdbSXjFznePAVoCg51zewMuKZTM7GD8Kb1znXOLzKw7MAs4yjkXDba6cNK+Fb9k/w7TEZTEqy8wBMgEPjKzgvn3OOfeC6yqkHHObTOzQcDzZubwp/f6KpzKpX2rjtMRlIiIhJIu0IqISCgpoEREJJQUUCIiEkoKKBERCSUFlIiIhJICSkREQkkBJSIioaSAEhGRUPp/cE/aMWhmtJsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(z, elu(z), \"b-\", linewidth=2)\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([-5, 5], [-1, -1], 'k--')\n",
    "plt.plot([0, 0], [-2.2, 3.2], 'k-')\n",
    "plt.grid(True)\n",
    "plt.title(r\"ELU 활성화 함수 ($\\alpha=1$)\", fontsize=14)\n",
    "plt.axis([-5, 5, -2.2, 3.2])\n",
    "\n",
    "save_fig(\"elu_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "텐서플로에서 ELU를 구현하는 것은 간단합니다. 층을 구성할 때 활성화 함수에 지정하기만 하면 됩니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.elu, name=\"hidden1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SELU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이 활성화 함수는 Günter Klambauer, Thomas Unterthiner, Andreas Mayr가 2017년에 쓴 [논문](https://arxiv.org/pdf/1706.02515.pdf)에서 소개되었습니다(나중에 책에 추가하겠습니다). 훈련할 때 SELU 활성화 함수를 사용한 완전 연결 신경망은 스스로 정규화를 합니다. 각 층의 출력은 훈련하는 동안 같은 평균과 분산을 유지하려는 경향이 있어 그래디언트 소실과 폭주 문제를 해결합니다. 이 활성화 함수는 심층 신경망에서 다른 활성화 함수보다 뛰어난 성능을 내므로 꼭 이 함수를 시도해봐야 합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def selu(z,\n",
    "         scale=1.0507009873554804934193349852946,\n",
    "         alpha=1.6732632423543772848170429916717):\n",
    "    return scale * elu(z, alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmYFOXV9/HvYXPhUSGi6AsouOBKNMmI2/PqqBjfKBjzsJm4YaIgRo2JuIDooxF3jZhEFETUqIk74gImLhmNAaKDAXfiLiAqio0OyNac94+725lphtm6Z6qq+/e5rrqmp6u6+8xNMb+p6tN3mbsjIiISN22iLkBERKQuCigREYklBZSIiMSSAkpERGJJASUiIrGkgBLJk5kdamYf1FherrFukJl90MTna9JjzGyYmc2uZ/1GZpbKWcpqrP/AzAY18BozzGxiY2sSKQQFlCSGmfUys3vN7CMz+8TM3jKzy2usH2Zm6cy63KVvZpsPzOy0DTz/JXX9ojeznczMzaxnHeteAqbk3P2dzOuc3cDPs3/meds14mcfm9k2u2wwkHK5+yp375SzVDb28WbWBfi/wOuNfYxIITT4H0MkDsysLfAU8CBwkruvMrNtgV/kbLrE3bdprbrcfR8z6wAcAGwLrAZecfe3M3XXd2SyDfCxu6/NbLs54Y/GTbMbmNmmwJHA28AJNR67LvPc79RXn5k9Dvz3Blb/x937NvD4jYA/AcuB0Wb2N3d/q77HiBSKAkqSYktgR+ARd18F4O6LgXFRFmVmGwOvAFsAbxHCZW8zG+3u12U2a2NmnYA17r68xsOPBrYwsz3d/TXgPcLPCfBh5msHYM96SlhRX33u3j9T557APoQAnOPuc3M23TRT43J3X5N5zN6EcCJTw3DgJTO7GrjJ3b+s77VF8qVTfJII7v4Z8CYw3sz6RF1PDR2BbsD/c/eDgX2Bu4Bda2zTA/gSeCh7h5n1A44B/heYZmZbu3sXdzdgcHY7d0+5+yXAy4QjqV8CpwDbA3909+kNFZgJlKlAd8JR271mdkvOZndmahxoZnuZ2QvAbOBZYF93X+Lul2dq6A98bGZ/aMwAiTSXjqAkSQYQ3u95xcz+CVzr7tNyttnKzD7Jue99d9+/JQpy9y/M7AfAIDObSjiaehS4J3N6zoAP3b1n9jFmdgwhEE519/szpy8rzexX7j419zXMbCfgAWAQ8DiwOXA3cBMwtHoz2xhY5+6razx2E2AU0Nfd52TuewCYb2YXZ4IfYLC7P5hZv3Hm9Ya4+8c5P+8/gP3MbC9gXTOHTaRRFFCSGO7+LnCwme0D/Ap4OBMKQ909ndksn/egGpqYstZ6MxsCXFzjrm0Jp/h2A34DLCGngcLMjgcuBwa4+/MA7n5N5ojlF2b2RB2v25vwHtDjHibPXGZm04Bf19imL/ANMA/Y+9uC3b8xsw8JAfoOIVSGAp8RjpjW/yHdVwI31jMOuPu8+taLFIICShLH3V8CjjezPwNPAEOAvxTgqb8inLLLtXnm67Kc+x8lnAJzIA2szPxy/5aZdSOEQbb2u83sEXevqrmdu88EZmYeswB4rMbqF4Aq4E4zexDoQjg1WPM03b/cfb8N/Fw/IoTiPMIR3b+Bw7PvNRFO5S3JvHY58PcNPE9d7nP3Y5uwvUijKaAkEcysbY2jJADcfbqZLSW8x1MIbwM9zaxjTjNDH2CZu6dyXn8lsNLMvg9Mz9RZ1/MuAGbUeFxVZtv9gfOB7xGOvtYB8wmn10bV2P6rzGv8AridEDCnu3vNENsgd5+fOXLbiOqjQDezzTLfn0I4+gJ4Dmhfx9PMBv4M/D7nfp3mkxajJglJij5mdreZbQ/hDRczO4lwSq3BRoFGmgEsAiaY2Xcyr1FGOFqpryHgFUKXW13LucBWuQ/IBE4F8BpwKLAZodniAuA4qrvnAHD3z939akLYPQa8aGa7ZZot6gqUXJcQTulll2WEI8avM8tPMq/j7r42d8k8x7o61imgpMXoCEqS4j3CqbJnM80H64A3gCMyLdpZdTVJABzt7i9mbl9tZpfUtd7MDgWuJLSMt8u85h+A8fXUVgbMoro1PNfCOu77IeFzSGNr3LcKmJFp954EYGajqG6lN0Lb+fWE976WEAL1IRrnYXdf73NZ1sSZLkRaiwJKEsHdvyI0Hvymnm3uAO5o4Hl6NrD+Y+CkJhcIqxp67hxPA5ea2f8SOvIWA5sAPwAuJHwoGUI4Ts7cXgesruN9rmGNfM32mfDLpTMpEksKKJHC2MjMqupZ/313/0/2G3evzBytnQucTPh8UpowM8T9wLWZ7VYRjqwK4Wg20LknEkemS76LiEgc6dBeRERiSQElIiKx1OrvQXXp0sV79uzZ2i/bKMuXL6djx7o+pykbojFrmvnz55NOp9l9992jLiVR4ryfvfMOLFsGG20Eu+4K7WL0zn5cx23OnDmfu/t6H7/I1epD2bNnTyorG30pmlZVUVFBeXl51GUkisasacrLy0mlUrH9PxBXcd3Pfv1rmDMHOneG2bOhd++oK6otruOWmX6rQTrFJyLSDDfdBOPHQ/v2MHVq/MKpGCigRESaaPp0OOuscHvyZDj44GjrKVYKKBGRJpg3D4YOhXXr4KKL4MQTo66oeCmgREQa6eOPoX9/qKqCn/0MLr006oqKW94BZWaXmdk8M3vRzF42s5GFKExEJE6WL4cBA2DhQjjwQLjtNqh78noplEJ08X0JlLn7GjPbCnjfzJ509/cL8NwiIpFLp8MR08svw447wiOPwMYbR11V8cs7oNz9dzW+7Um4sNrSfJ9XRCQuzj0XHn00tJM/8QR06RJ1RaWhIJ+DMrOdCdfk2RIY4u7LctYPB4YDdO3alYqKikK8bMFVVVXFtra40pg1TSqVIp1Oa8yaKMr97JFH/g833tibdu3WcfHF81i8eBmLF0dSSpMl/f9nQSeLNbPvES7Bfbi7v17XNmVlZR7XDynG9UNtcaYxa5rsB3Xnzp0bdSmJEtV+NmNGaIpYtw7uvDN5HXtx/f9pZnPcvayh7Qraxefu/yZcuO2QQj6viEhrmzcPhgxRO3mU8gooM+tjZkPMQi+LmXUD9gXieYgkItIINdvJf/pTtZNHJd/3oD4ETgPON7M1hMtRX+Tus/OuTEQkArnt5FOmqJ08KnkFVOYy3L8sUC0iIpFKp+G446rbyadOVTt5lDSThIhIxrnnwrRp1e3kWzV4QQhpSQooERHg5pvhhhvC7OQPPwy77BJ1RaKAEpGSN2MGnHFGuH3rrRDDzuySpIASkZL2yivV7eRjx8JJJ0VdkWQpoESkZC1eXN1Ofuyx8NvfRl2R1KSAEpGSlG0nX7AADjgAbr9d7eRxo4ASkZKTbSefMwd22EGzk8eVAkpESs5551W3k0+frnbyuFJAiUhJuflm+N3v1E6eBAooESkZTz4JZ54ZbqudPP4UUCJSErLt5Om02smTQgElIkUv207+9ddqJ08SBZSIFDW1kyeXAkpEipbayZNNASUiRSvbTt6pk2YnTyIFlIgUpVtuCe3k7dqFdvJdd426ImkqBZSIFJ0nn6w9O/khh0RbjzSPAkpEisqrr1a3k194IQwbFnVF0lwKKBEpGosXw1FHhXbyoUPVTp50CigRKQrLl8PRR1e3k99xB7TRb7hE0z+fiCReOg3HHw+VlWonLyYKKBFJvPPPD6GkdvLiooASkUS75Ra4/nq1kxcjBZSIJNZf/6p28mKmgBKRRHr1VRg8OLz/NGaM2smLkQJKRBKn5uzkQ4fCZZdFXZG0BAWUiCRKtp38o49g//3VTl7M9M8qIomRTsMJJ4R28l69wkSwaicvXgooEUmM88+HqVPVTl4qFFAikggTJ1a3kz/0EOy2W9QVSUtTQIlI7L34Ymd++ctwe9IkOPTQaOuR1qGAEpFYe/VVuPTSPb5tJz/55KgrktaigBKR2Prkk9BOvmJFO7WTlyAFlIjE0ooVMGBAaCffY49l3H672slLTd7/3GY2wszmmVmlmb1iZqcXojARKV3r1lXPTt6rF4wb9xqbbBJ1VdLa2uXzYDNrC+wMHOjuVWbWDXjHzKa5+6KCVCgiJSfbTr7FFqGd/NNP10RdkkQgryMod0+7+yh3r8rc9QWwGmibd2UiUpImToTrrquenVzt5KUrryOoOowH7nP3j2reaWbDgeEAXbt2paKiosAvWxhVVVWxrS2uNGZNk0qlSKfTGrMNePHFzowe/V3A+M1v3qJNm0+oqNB+1lxJHzdz98I8kdk4YC9goLuv3tB2ZWVlXllZWZDXLLSKigrKy8ujLiNRNGZNU15eTiqVYu7cuVGXEjuvvRYu1f711zB6NFxxRfU67WfNE9dxM7M57l7W0HYFOYIys+uAHWkgnERE6vLJJ3DUUSGchgyBceOirkjiIN8miTbABKAzMNjd1xakKhEpGStWVM9Ovt9+mp1cquW7GxwJjAB2AF4ws9mZpV/+pYlIsVu3LsxO/tJL1bOTq51csvI6gnL3xwErUC0iUmIuuCB06mXbybfeOuqKJE50IC0ikZg0Ca69Vu3ksmEKKBFpdX/7G5yemXNm4kTNTi51U0CJSKt67TUYPDhcHXf0aPj5z6OuSOJKASUirSbbTv7VVyGk1E4u9VFAiUiryG0nv/NOtZNL/bR7iEiLq9lO3rOn2smlcRRQItLiaraTT5+udnJpHAWUiLSomu3kDz2kdnJpPAWUiLSYmu3kt9wChx0WbT2SLAooEWkRNdvJL7gAfvGLqCuSpFFAiUjBffIJ9O9f3U5++eVRVyRJpIASkYLKtpN/+KHaySU/2m1EpGDWrYMTT1Q7uRSGAkpECmb06NCpp9nJpRAUUCJSELfeCtdcU91OvvvuUVckSaeAEpG8PfUUjBwZbqudXApFASUieXn9dRg0SO3kUngKKBFptk8/rT07udrJpZAUUCLSLDXbyffdV+3kUnjanUSkybLt5C++qHZyaTkKKBFpstx28q5do65IipECSkSapGY7+YMPqp1cWo4CSkQarWY7+c03Q79+0dYjxU0BJSKNUrOd/Pzz4ZRToq5Iip0CSkQaVLOdfNAguOKKqCuSUqCAEpF65baT/+lPaieX1qHdTEQ2SO3kEiUFlIhs0JgxoZ18883VTi6tTwElInW69Va4+mrNTi7RUUCJyHqeflrt5BI9BZSI1PL66zBwYGgnP+88tZNLdBRQIvKtmu3kAwfClVdGXZGUMgWUiADwzTfw4x+HdvK+feGuu9ROLtHS7ici37aT/+tfsP328OijaieX6OUdUGbW3sxGmdkaMzu2EEWJSOsaMyZM/Lr55jB9utrJJR4KcQR1KuDA7AI8l4i0ssmTQzt527aanVzipV2+T+DuEwDMbED+5YhIa8ptJz/88GjrEakp74BqDDMbDgwH6Nq1KxUVFa3xsk1WVVUV29riSmPWNKlUinQ6HYsx++CDTTnjjO+zdm07jj32I3be+T1iUFadtJ81T9LHrVUCyt0nAZMAysrKvLy8vDVetskqKiqIa21xpTFrmk6dOpFKpSIfs08/hZNPhuXLQzv5PfdsR5s220VaU320nzVP0sdNXXwiJSbbTv7BB6GdXLOTS1xptxQpIXW1k2+6adRVidRNASVSQi68sLqdXLOTS9wV7D0ody8v1HOJSOHddhtcdVV1O/kee0RdkUj9dAQlUgKeeQZOOy3cVju5JIUCSqTIvfFG6NRbuxbOPRdOPTXqikQaRwElUsSys5MvWxZC6qqroq5IpPEUUCJFSu3kknTaXUWK0Lp1cNJJoZ18u+1g2jS1k0vyKKBEitCFF8IDD1S3k2+zTdQViTSdAkqkyEyZUrudfM89o65IpHkUUCJF5JlnYMSIcHvCBLWTS7IpoESKRG47+fDhUVckkh8FlEgR+Oyz6nby//kftZNLcVBAiSRczXbyffaBu+5SO7kUB+3GIgmWbSefPTu0k2t2cikmCiiRBBs7Vu3kUrwUUCIJNWUKXHllaCd/4AG1k0vxUUCJJFBuO/kPfxhtPSItQQElkjBvvlndTj5qlNrJpXgpoEQSJLed/Oqro65IpOUooEQSIttO/v77aieX0qDdWyQB1q2DYcPUTi6lRQElkgAXXQT33692ciktCiiRmLv9drjiCrWTS+lRQInE2LPPVnfp3XST2smltCigRGLqzTdDp162nTz7uSeRUqGAEomhmu3kP/mJ2smlNCmgRGLmm2/gmGNCO3lZGdx9t9rJpTRptxeJkWw7+axZoZ38scfUTi6lSwElEiPZdvLNNlM7uYgCSiQm1E4uUpsCSiQGctvJjzgi2npE4kABJRKxt96qnp38nHPUTi6SpYASidCSJXDkkZBKhc49tZOLVFNAiURk5crq2cmz7eRt20ZdlUh8KKBEIlCznbxHjzA7eceOUVclEi8KKJEIXHwx3HdfdTv5tttGXZFI/OQdUGZ2sJm9bGavmFmlme1XiMJEitXSpR24/PLqdvI+faKuSCSe2uXzYDPrBDwM9Hf3WWZWDkwzs17uvqIQBYoUk5UrYeHCTQD44x/VTi5Sn7wCCjgCmO/uswDcvcLMFgOHAY/V9YD58+dTXl6e58u2jFQqRadOnaIuI1E0Zk0zZ85c3GGrrcq59164996oK0oG7WfNk/RxyzegdgDezbnv3cz93zKz4cBwgPbt25NKpfJ82ZaRTqdjW1tcacwab+nSDqxdG25vvfUyUimPtqAE0X7WPEkft3wDyoB0zn1ryXlvy90nAZMAysrKvLKyMs+XbRkVFRWxPbqLK41Z41RVwU47AZTTo8cKXnvtxahLShTtZ80T13Ezs0Ztl2+TxEJgu5z7tsvcLyIZN9wAn34auva+853VUZcjkgj5BtQ04Ltm1gfAzPoCuwJP5VuYSLFYsgSuuSbc3mGH+rcVkWp5neJz92VmNhiYYmZOOL13pLsn96SnSIFddlk4xXfUUeFrgt8SEGlVeX8Oyt3/7u77uHtfdz8g29EnIvDGGzBhQrgi7pVXRl2NSLJoJgmRFuIOZ58N6XSYoVwfyBVpGgWUSAt57DF46ino3Dmc5hORplFAibSAVavgN78Jty+9FLbcMtp6RJJIASXSAm64Ad59F3bfHU47LepqRJJJASVSYB9/DOPGhds33gjt20dbj0hSKaBECmzUKFi+PFyMsF+/qKsRSS4FlEgBPfEE/OUvsOmm4TSfiDSfAkqkQL7+GkaODLcvuwx69Yq2HpGkU0CJFMiYMbBgAZSVwVlnRV2NSPIpoEQKYOZMuOkmaNcOJk8OX0UkPwookTytWgWnnBJmjjjvPNhrr6grEikOCiiRPF16Kbz5JvTuDRddFHU1IsVDASWSh+efh6uuCpPBTpkCG28cdUUixUMBJdJMqRQcf3w4tTdmDBx4YNQViRQXBZRIM7iHlvIFC6BvX7j44qgrEik+CiiRZrjnHrj3XujYMdzWdEYihaeAEmmid9+F008Pt3//e9hpp2jrESlWCiiRJlixAgYODLNGDBwIJ58cdUUixUsBJdJI7uHKuPPmhaOmyZPBLOqqRIqXAkqkkSZMgLvvDhPBTp0KnTpFXZFIcVNAiTTCzJlw9tnh9m23wZ57RluPSClQQIk0YNEiGDQI1q4NIXXssVFXJFIaFFAi9fj6a+jfHxYvhoMOgmuuiboikdKhgBLZgLVrYehQmDsXdt4ZHn5Yn3cSaU0KKJE6uMMZZ8CMGdClC0yfDltuGXVVIqVFASVSh2uvhYkTYaON4NFH9WFckSgooERy3HYbnH9+uH333bD//tHWI1KqFFAiNfz5z3DqqeH2jTeG7j0RiYYCSiRj6lQ48cTw/tMVV8BZZ0VdkUhpU0CJEJohhg6FdBrGjoXRo6OuSEQUUFLypk2DY46BNWvg17+G3/426opEBBRQUuLuuSfMSr56NZx5Jlx/vSaAFYkLBZSUrFtugRNOCKf1LrwwNEUonETiQwElJccdrrwyXLLdHa6+GsaNUziJxE1eAWVm7c1slJmtMTNNoSmxt2ZNaCMfMyYE0s03w3nnRV2ViNSlXZ6PPxVwYHYBahFpUV9+GT7X9OyzsPHG4UO4AwdGXZWIbEheAeXuEwDMbEB925nZcGA4QNeuXamoqMjnZVtMVVVVbGuLq6SM2aJFGzNmTB8++qgjnTuv5vLLX2XLLb+mtUtPpVKk0+lEjFmcJGU/i5ukj1uDAWVmHYDn61h1kLuvbsyLuPskYBJAWVmZl5eXN6XGVlNRUUFca4urJIzZ44+HiV9TKejTBx5/vAPbbfeDSGrp1KkTqVQq9mMWN0nYz+Io6ePWYEBlQmi/VqhFpKDSabj44jArBMDRR8Ndd8Hmm0dbl4g0Tr7vQYnE0qefws9+Ft5vatMmhNS554bbIpIMCigpOo8+CqecAkuWQNeucO+9kOCzHCIlS39PStH4+usQTD/+cQinQw+Fl19WOIkkVUGOoNy9vBDPI9Jczz0HP/85vPdeuMjgVVeF2ch1Sk8kuXSKTxJt6dLw3tKUKeH7vfYKn2/ac89o6xKR/OnvS0kk93BxwV13DeHUoQNcein8618KJ5FioSMoSZw5c8JlMf7xj/D9wQfDxImwyy7R1iUihaUjKEmMjz+Gk0+GffYJ4dSlC0yeDH//u8JJpBjpCEpib+lSuO66cDmMFSugfXv41a/CJTI6dYq6OhFpKQooia2vvoLx48NFBL/6Ktx3zDFw7bWw007R1iYiLU8BJbGzZAn84Q/wxz+GGcgBDj8cLrsM9t032tpEpPUooCQ23n8ffvc7uO02+OabcN9//3e4mODBB0dbm4i0PgWURCqdhr/+FSZMgOnTQ/s4wFFHwQUXhIASkdKkgJJIfPZZ+PzSxInwwQfhvg4dYOjQ8MHbPn0iLU9EYkABJa1m1Sp48km45x6YNg1WZ64m1qsXnHZaaCHfaqtoaxSR+FBASYtaty58Zumee+DBB6ubHsxgwAAYORKOOEJz5onI+hRQUnCrV4dQevRRePhhWLiwet1ee4XrNP30p9CjR3Q1ikj8KaCkIL78EmbMCKE0Y0b155YAtt8+hNJxx8Eee0RXo4gkiwJKmmXlSpg1C6ZM6cXo0fDSS6EjL2uPPcIpvKOPDp9d0ik8EWkqBZQ0yooVUFkJ//xnuIz6Cy+EkILtAWjbFg45JATSgAGw446RlisiRUABJetxhw8/DJeumDkzLHPnwtq1tbf77nehd+8FDBvWg4MOgs02i6ZeESlOCqgSl07D22/Dv/8dLo+e/Zrttstq0wb23hv23z/M6nDIIbD11lBR8S7l5ep2EJHCU0CViDVr4J134I03wvLmm+Hr/PnZU3W1dekCZWVwwAFh6dtXR0gi0roUUEXkm2/CfHbvvVd7eeedcJSUe4ouq3t3+P734Xvfq/7avXv4rJKISFQUUAmxalW4YN+iRbWXhQvD1/feg8WL63+OXr1gt91g993DsttuYdlii9b5GUREmkIBFRF3qKoKl5bILp99Vvv7JUtC6CxaFG43pF076NkTdthh/aV3b+jYscV/LBGRglFANYN7mC1h+fIQMtll2TJIpepevvyy9vdLl4ajosZq2xa23Ra6dQtL9+7Vt7t1C0dH3buH7UREikHiAyqdDu+trFkTllWrwpv+K1fWvt2Y+95+e0ceeCB85icbPrlfs7c39H5OU2yySZgcdautQkdc9nbNZZttQgB17arwEZHS0uoB9f77MHhw7VCp63Zj12evH1QYjW+XbtcudLV17Aj/9V/ha6dOjV86d9YpNxGR+pgX9jd8wy9omzn8IOfeIcDpwArgyDoeNSyzfA4MWm9t27Yj2WijobRtu4CVK0+gTRtqLT16nEO3bgNYu3Y+r702ArPa6w86aCy77daPefMeY+7c62nTJhytZJczz7yCAw44gLfemsn114+hbdvaHW7jx49n77335umnn2bcuHHr1Tdx4kR22WUXHnvsMa6//vr11t9111306NGD++67j5tvvnm99Q8++CBdunThjjvu4I477lhv/fTp09l0002ZMGEC999//3rrKyoqALjuuut4/PHHa63bZJNNmDFjBgCXXXYZzzzzTK31W265JQ899BAAo0ePZtasWbXWt2/fnqeeegqAs88+m7lz59Za37t3byZNmgTA8OHD+c9//lNr/d5778348eMBOP7441lYc2ZZYP/99+fKK68EYODAgXzxxRe11h922GFcdNFFAPzoRz/im+yleDP69+/PqFGjACgvLyfXkCFDOP3001mxYgVHHrn+vjds2DCGDRvG559/zqBB6+97I0eOZOjQoSxYsIATTjhhvfXnnHMOAwYMYP78+YwYMYK5c+eydu1aysrKABg7diz9+vVj7ty5nH322es9/oorwr43c+ZMxowZs976Utn3jjvuOBYtWlRrfffu3bn77rsB7Xsb2vc222wztt12W0aMGLHe+ij3veeee26Ou5et96AcrX4EtfHG4U17s+qlvBwGDgyn6849t/Y6MxgyJMx+/dVXcMop668fOTJc6G7BAqjj34lzzgnT78yfD3X8O3HyydCvH0ye/GmdzQjZbrcvvwxHTiIi0vJa/QiqrKzMKysrW/U1G6uioqLOv3RkwzRmTVNeXk4qlVrvr32pn/az5onruJlZo46gNMe0iIjEkgJKRERiSQElIiKxpIASEZFYUkCJiEgsKaBERCSW8gooMxthZvPMrNLMXjGz0wtVmIiIlLZmf+zUzNoCOwMHunuVmXUD3jGzae6+qIGHi4iI1KvZAeXuaWBUjbu+AFYD601pambDgeEAXbt2/Xb6k7ipqqqKbW1xpTFrmlQqRTqd1pg1kfaz5kn6uDU4k4SZdQCer2PVQe6+usZ2twBt3H14fc+nmSSKi8asaTSTRPNoP2ueuI5bY2eSaPAIKhNC+zXwYuOAbsDARlcoIiJSj7ynPjWz64AdgYE1j6hERETykU+TRBtgAtAZGOzuBbiEn4iISJBPm/mRwAhgB+AFM5udWfoVpjQRESll+XTxPQ5YgxuKiIg0g2aSEBGRWFJAiYhILLX6FXXNbAnwYau+aON1AT6PuoiE0Zg1ncas6TRmzRPXcdve3bdqaKNj7SvPAAACUklEQVRWD6g4M7PKxnx4TKppzJpOY9Z0GrPmSfq46RSfiIjEkgJKRERiSQFV26SoC0ggjVnTacyaTmPWPIkeN70HJSIisaQjKBERiSUFlIiIxJICSkREYkkBtQFmtoeZLTWzS6KuJe7MbISZzTOzSjN7xcxOj7qmODKzg83s5cwYVZpZvddZE+1b+SiG32F5Xw+qGJlZJ+Am4C9R1xJ3ZtYW2Bk40N2rzKwb8I6ZTXP3RRGXFxuZfephoL+7zzKzcmCamfVy9xXRVhdP2rear1h+h+kIKkfmOld3AmOAJRGXE3vunnb3Ue5elbnrC2A10DbCsuLoCGC+u88CcPcKYDFwWJRFxZn2reYppt9hJXkEZWYdgOfrWHUQcAnwlLvPNLMftmphMVbfmOVcSXk8cJ+7f9Q6lSXGDsC7Ofe9m7lfGkf7VuOMo0h+h5VkQGV+oa53/t/MBgLbufuY1q8q3jY0ZjWZ2TigGzCwVYpKFgPSOfetRWcxGkX7VuMU2++wkgyoevwI2M3MZme+7w7hzUZ3HxxdWfFnZtcBOwIDc46oJFgI5F5tejvgwQhqSRTtW01SVL/DNJNEPbLdL+5+SbSVxFfmfPcEoDNwnLuvjbikWDKzLQin9A5x91fNrC/wV6CXu6eirS6etG/lL+m/w3QEJfk6EhgBVAIvmFn2/rHu/nRkVcWMuy8zs8HAFDNzwum9IxVO9dK+VeJ0BCUiIrGkN2hFRCSWFFAiIhJLCigREYklBZSIiMSSAkpERGJJASUiIrGkgBIRkVhSQImISCz9fzigoAwEQ/1SAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(z, selu(z), \"b-\", linewidth=2)\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([-5, 5], [-1.758, -1.758], 'k--')\n",
    "plt.plot([0, 0], [-2.2, 3.2], 'k-')\n",
    "plt.grid(True)\n",
    "plt.title(r\"SELU 활성화 함수\", fontsize=14)\n",
    "plt.axis([-5, 5, -2.2, 3.2])\n",
    "\n",
    "save_fig(\"selu_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "기본적으로 SELU 하이퍼파라미터(`scale`과 `alpha`)는 평균이 0, 표준 편차가 1에 가깝게 유지되도록 조정합니다(입력도 평균이 0, 표준 편차가 1로 표준화되었다고 가정합니다). 이 활성화 함수를 사용하면 100층으로 된 심층 신경망도 그래디언트 소실/폭주 문제없이 모든 층에서 대략 평균이 0이고 표준 편차가 1을 유지합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "층 0: -0.26 < 평균 < 0.27, 0.74 < 표준 편차 < 1.27\n",
      "층 10: -0.24 < 평균 < 0.27, 0.74 < 표준 편차 < 1.27\n",
      "층 20: -0.17 < 평균 < 0.18, 0.74 < 표준 편차 < 1.24\n",
      "층 30: -0.27 < 평균 < 0.24, 0.78 < 표준 편차 < 1.20\n",
      "층 40: -0.38 < 평균 < 0.39, 0.74 < 표준 편차 < 1.25\n",
      "층 50: -0.27 < 평균 < 0.31, 0.73 < 표준 편차 < 1.27\n",
      "층 60: -0.26 < 평균 < 0.43, 0.74 < 표준 편차 < 1.35\n",
      "층 70: -0.19 < 평균 < 0.21, 0.75 < 표준 편차 < 1.21\n",
      "층 80: -0.18 < 평균 < 0.16, 0.72 < 표준 편차 < 1.19\n",
      "층 90: -0.19 < 평균 < 0.16, 0.75 < 표준 편차 < 1.20\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "Z = np.random.normal(size=(500, 100))\n",
    "for layer in range(100):\n",
    "    W = np.random.normal(size=(100, 100), scale=np.sqrt(1/100))\n",
    "    Z = selu(np.dot(Z, W))\n",
    "    means = np.mean(Z, axis=1)\n",
    "    stds = np.std(Z, axis=1)\n",
    "    if layer % 10 == 0:\n",
    "        print(\"층 {}: {:.2f} < 평균 < {:.2f}, {:.2f} < 표준 편차 < {:.2f}\".format(\n",
    "            layer, means.min(), means.max(), stds.min(), stds.max()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "텐서플로 1.4 버전에 `tf.nn.selu()` 함수가 추가되었습니다. 이전 버전을 사용할 때는 다음 구현을 사용합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def selu(z,\n",
    "         scale=1.0507009873554804934193349852946,\n",
    "         alpha=1.6732632423543772848170429916717):\n",
    "    return scale * tf.where(z >= 0.0, z, alpha * tf.nn.elu(z))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "하지만 SELU 활성화 함수는 일반적인 드롭아웃과 함께 사용할 수 없습니다(드롭아웃은 SELU 활성화 함수의 자동 정규화 기능을 없애버립니다). 다행히 같은 논문에 실린 알파 드롭아웃(Alpha Dropout)을 사용할 수 있습니다. 텐서플로 1.4에 `tf.contrib.nn.alpha_dropout()`이 추가되었습니다(Linz 대학교 생물정보학 연구소(Institute of Bioinformatics)의 Johannes Kepler가 만든 [구현](https://github.com/bioinf-jku/SNNs/blob/master/selu.py)을 확인해 보세요)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SELU 활성화 함수를 사용한 신경망을 만들어 MNIST 문제를 풀어 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=selu, name=\"hidden1\")\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=selu, name=\"hidden2\")\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()\n",
    "n_epochs = 40\n",
    "batch_size = 50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 훈련할 차례입니다. 입력을 평균 0, 표준 편차 1로 스케일 조정해야 합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 배치 데이터 정확도: 0.88 검증 세트 정확도: 0.923\n",
      "5 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9574\n",
      "10 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9664\n",
      "15 배치 데이터 정확도: 0.96 검증 세트 정확도: 0.9684\n",
      "20 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9694\n",
      "25 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9688\n",
      "30 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9694\n",
      "35 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.97\n"
     ]
    }
   ],
   "source": [
    "means = X_train.mean(axis=0, keepdims=True)\n",
    "stds = X_train.std(axis=0, keepdims=True) + 1e-10\n",
    "X_val_scaled = (X_valid - means) / stds\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            X_batch_scaled = (X_batch - means) / stds\n",
    "            sess.run(training_op, feed_dict={X: X_batch_scaled, y: y_batch})\n",
    "        if epoch % 5 == 0:\n",
    "            acc_batch = accuracy.eval(feed_dict={X: X_batch_scaled, y: y_batch})\n",
    "            acc_valid = accuracy.eval(feed_dict={X: X_val_scaled, y: y_valid})\n",
    "            print(epoch, \"배치 데이터 정확도:\", acc_batch, \"검증 세트 정확도:\", acc_valid)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final_selu.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 배치 정규화"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "각 은닉층의 활성화 함수 전에 배치 정규화를 추가하기 위해 ELU 활성화 함수를 배치 정규화 층 이후에 수동으로 적용하겠습니다.\n",
    "\n",
    "노트: `tf.layers.dense()` 함수가 (책에서 사용하는) `tf.contrib.layers.arg_scope()`와 호환되지 않기 때문에 대신 파이썬의 `functools.partial()` 함수를 사용합니다. 이를 사용해 `tf.layers.dense()`에 필요한 매개변수가 자동으로 설정되도록 `my_dense_layer()`를 만듭니다(그렇지 않으면 `my_dense_layer()`를 호출할 때마다 덮어씌여질 것입니다). 다른 코드는 이전과 비슷합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "\n",
    "training = tf.placeholder_with_default(False, shape=(), name='training')\n",
    "\n",
    "hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\")\n",
    "bn1 = tf.layers.batch_normalization(hidden1, training=training, momentum=0.9)\n",
    "bn1_act = tf.nn.elu(bn1)\n",
    "\n",
    "hidden2 = tf.layers.dense(bn1_act, n_hidden2, name=\"hidden2\")\n",
    "bn2 = tf.layers.batch_normalization(hidden2, training=training, momentum=0.9)\n",
    "bn2_act = tf.nn.elu(bn2)\n",
    "\n",
    "logits_before_bn = tf.layers.dense(bn2_act, n_outputs, name=\"outputs\")\n",
    "logits = tf.layers.batch_normalization(logits_before_bn, training=training,\n",
    "                                       momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "training = tf.placeholder_with_default(False, shape=(), name='training')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "같은 매개변수를 계속 반복해서 쓰지 않도록 파이썬의 `partial()` 함수를 사용합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "my_batch_norm_layer = partial(tf.layers.batch_normalization,\n",
    "                              training=training, momentum=0.9)\n",
    "\n",
    "hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\")\n",
    "bn1 = my_batch_norm_layer(hidden1)\n",
    "bn1_act = tf.nn.elu(bn1)\n",
    "hidden2 = tf.layers.dense(bn1_act, n_hidden2, name=\"hidden2\")\n",
    "bn2 = my_batch_norm_layer(hidden2)\n",
    "bn2_act = tf.nn.elu(bn2)\n",
    "logits_before_bn = tf.layers.dense(bn2_act, n_outputs, name=\"outputs\")\n",
    "logits = my_batch_norm_layer(logits_before_bn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "각 층에 ELU 활성화 함수와 배치 정규화를 사용하여 MNIST를 위한 신경망을 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "batch_norm_momentum = 0.9\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n",
    "training = tf.placeholder_with_default(False, shape=(), name='training')\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    he_init = tf.variance_scaling_initializer()\n",
    "\n",
    "    my_batch_norm_layer = partial(\n",
    "            tf.layers.batch_normalization,\n",
    "            training=training,\n",
    "            momentum=batch_norm_momentum)\n",
    "\n",
    "    my_dense_layer = partial(\n",
    "            tf.layers.dense,\n",
    "            kernel_initializer=he_init)\n",
    "\n",
    "    hidden1 = my_dense_layer(X, n_hidden1, name=\"hidden1\")\n",
    "    bn1 = tf.nn.elu(my_batch_norm_layer(hidden1))\n",
    "    hidden2 = my_dense_layer(bn1, n_hidden2, name=\"hidden2\")\n",
    "    bn2 = tf.nn.elu(my_batch_norm_layer(hidden2))\n",
    "    logits_before_bn = my_dense_layer(bn2, n_outputs, name=\"outputs\")\n",
    "    logits = my_batch_norm_layer(logits_before_bn)\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "노트: 배치 정규화를 위해 별도의 업데이트 연산을 실행해 주어야 합니다(`sess.run([training_op, extra_update_ops],...`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 검증 세트 정확도: 0.8952\n",
      "1 검증 세트 정확도: 0.9202\n",
      "2 검증 세트 정확도: 0.9318\n",
      "3 검증 세트 정확도: 0.9422\n",
      "4 검증 세트 정확도: 0.9468\n",
      "5 검증 세트 정확도: 0.954\n",
      "6 검증 세트 정확도: 0.9568\n",
      "7 검증 세트 정확도: 0.96\n",
      "8 검증 세트 정확도: 0.962\n",
      "9 검증 세트 정확도: 0.9638\n",
      "10 검증 세트 정확도: 0.9662\n",
      "11 검증 세트 정확도: 0.9682\n",
      "12 검증 세트 정확도: 0.9672\n",
      "13 검증 세트 정확도: 0.9696\n",
      "14 검증 세트 정확도: 0.9706\n",
      "15 검증 세트 정확도: 0.9704\n",
      "16 검증 세트 정확도: 0.9718\n",
      "17 검증 세트 정확도: 0.9726\n",
      "18 검증 세트 정확도: 0.9738\n",
      "19 검증 세트 정확도: 0.9742\n"
     ]
    }
   ],
   "source": [
    "extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run([training_op, extra_update_ops],\n",
    "                     feed_dict={training: True, X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"검증 세트 정확도:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "어!? MNIST 정확도가 좋지 않네요. 물론 훈련을 더 오래하면 정확도가 높아지겠지만 이런 얕은 신경망에서는 배치 정규화와 ELU가 큰 효과를 내지 못합니다. 대부분 심층 신경망에서 빛을 발합니다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "업데이트 연산에 의존하는 훈련 연산을 만들 수도 있습니다:\n",
    "\n",
    "```python\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "    with tf.control_dependencies(extra_update_ops):\n",
    "        training_op = optimizer.minimize(loss)\n",
    "```\n",
    "\n",
    "이렇게 하면 훈련할 때 `training_op`만 평가하면 텐서플로가 업데이트 연산도 자동으로 실행할 것입니다:\n",
    "\n",
    "```python\n",
    "sess.run(training_op, feed_dict={training: True, X: X_batch, y: y_batch})\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "한가지 더, 훈련될 변수 개수가 전체 전역 변수 개수보다 적습니다. 이동 평균을 위한 변수는 훈련되는 변수가 아니기 때문입니다. 미리 학습한 신경망을 재사용할 경우(아래 참조) 이런 훈련되지 않는 변수를 놓쳐서는 안됩니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['hidden1/kernel:0',\n",
       " 'hidden1/bias:0',\n",
       " 'batch_normalization/gamma:0',\n",
       " 'batch_normalization/beta:0',\n",
       " 'hidden2/kernel:0',\n",
       " 'hidden2/bias:0',\n",
       " 'batch_normalization_1/gamma:0',\n",
       " 'batch_normalization_1/beta:0',\n",
       " 'outputs/kernel:0',\n",
       " 'outputs/bias:0',\n",
       " 'batch_normalization_2/gamma:0',\n",
       " 'batch_normalization_2/beta:0']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[v.name for v in tf.trainable_variables()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['hidden1/kernel:0',\n",
       " 'hidden1/bias:0',\n",
       " 'batch_normalization/gamma:0',\n",
       " 'batch_normalization/beta:0',\n",
       " 'batch_normalization/moving_mean:0',\n",
       " 'batch_normalization/moving_variance:0',\n",
       " 'hidden2/kernel:0',\n",
       " 'hidden2/bias:0',\n",
       " 'batch_normalization_1/gamma:0',\n",
       " 'batch_normalization_1/beta:0',\n",
       " 'batch_normalization_1/moving_mean:0',\n",
       " 'batch_normalization_1/moving_variance:0',\n",
       " 'outputs/kernel:0',\n",
       " 'outputs/bias:0',\n",
       " 'batch_normalization_2/gamma:0',\n",
       " 'batch_normalization_2/beta:0',\n",
       " 'batch_normalization_2/moving_mean:0',\n",
       " 'batch_normalization_2/moving_variance:0']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[v.name for v in tf.global_variables()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 그래디언트 클리핑"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MNIST를 위한 간단한 신경망을 만들고 그래디언트 클리핑을 적용해 보겠습니다. 시작 부분은 이전과 동일합니다(학습한 모델을 재사용하는 예를 만들기 위해 몇 개의 층을 더 추가했습니다. 아래 참조):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 50\n",
    "n_hidden3 = 50\n",
    "n_hidden4 = 50\n",
    "n_hidden5 = 50\n",
    "n_outputs = 10\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, name=\"hidden2\")\n",
    "    hidden3 = tf.layers.dense(hidden2, n_hidden3, activation=tf.nn.relu, name=\"hidden3\")\n",
    "    hidden4 = tf.layers.dense(hidden3, n_hidden4, activation=tf.nn.relu, name=\"hidden4\")\n",
    "    hidden5 = tf.layers.dense(hidden4, n_hidden5, activation=tf.nn.relu, name=\"hidden5\")\n",
    "    logits = tf.layers.dense(hidden5, n_outputs, name=\"outputs\")\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 그래디언트 클리핑을 적용합니다. 먼저 그래디언트를 구한 다음 `clip_by_value()` 함수를 사용해 클리핑하고 적용합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold = 1.0\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "grads_and_vars = optimizer.compute_gradients(loss)\n",
    "capped_gvs = [(tf.clip_by_value(grad, -threshold, threshold), var)\n",
    "              for grad, var in grads_and_vars]\n",
    "training_op = optimizer.apply_gradients(capped_gvs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "나머지는 이전과 동일합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 검증 세트 정확도: 0.288\n",
      "1 검증 세트 정확도: 0.794\n",
      "2 검증 세트 정확도: 0.8796\n",
      "3 검증 세트 정확도: 0.9058\n",
      "4 검증 세트 정확도: 0.9164\n",
      "5 검증 세트 정확도: 0.9218\n",
      "6 검증 세트 정확도: 0.9292\n",
      "7 검증 세트 정확도: 0.9356\n",
      "8 검증 세트 정확도: 0.9382\n",
      "9 검증 세트 정확도: 0.9414\n",
      "10 검증 세트 정확도: 0.9456\n",
      "11 검증 세트 정확도: 0.9472\n",
      "12 검증 세트 정확도: 0.9476\n",
      "13 검증 세트 정확도: 0.9532\n",
      "14 검증 세트 정확도: 0.9564\n",
      "15 검증 세트 정확도: 0.9566\n",
      "16 검증 세트 정확도: 0.9576\n",
      "17 검증 세트 정확도: 0.9592\n",
      "18 검증 세트 정확도: 0.9622\n",
      "19 검증 세트 정확도: 0.9614\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"검증 세트 정확도:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 학습된 모델 재사용하기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 텐서플로 모델 재사용하기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "먼저 그래프 구조를 로드해야 합니다. `import_meta_graph()` 함수가 그래프 연산들을 로드하여 기본 그래프에 적재하고 모델의 상태를 복원할 수 있도록 `Saver` 객체를 반환합니다. 기본적으로 `Saver` 객체는 `.meta` 확장자를 가진 파일에 그래프 구조를 저장하므로 이 파일을 로드해야 합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver = tf.train.import_meta_graph(\"./my_model_final.ckpt.meta\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "다음으로 훈련해야 할 모든 연산을 가져와야 합니다. 그래프 구조를 모를 때는 모든 연산을 출력해 볼 수 있습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X\n",
      "y\n",
      "hidden1/kernel/Initializer/random_uniform/shape\n",
      "hidden1/kernel/Initializer/random_uniform/min\n",
      "hidden1/kernel/Initializer/random_uniform/max\n",
      "hidden1/kernel/Initializer/random_uniform/RandomUniform\n",
      "hidden1/kernel/Initializer/random_uniform/sub\n",
      "hidden1/kernel/Initializer/random_uniform/mul\n",
      "hidden1/kernel/Initializer/random_uniform\n",
      "hidden1/kernel\n",
      "hidden1/kernel/Assign\n",
      "hidden1/kernel/read\n",
      "hidden1/bias/Initializer/zeros\n",
      "hidden1/bias\n",
      "hidden1/bias/Assign\n",
      "hidden1/bias/read\n",
      "dnn/hidden1/MatMul\n",
      "dnn/hidden1/BiasAdd\n",
      "dnn/hidden1/Relu\n",
      "hidden2/kernel/Initializer/random_uniform/shape\n",
      "hidden2/kernel/Initializer/random_uniform/min\n",
      "hidden2/kernel/Initializer/random_uniform/max\n",
      "hidden2/kernel/Initializer/random_uniform/RandomUniform\n",
      "hidden2/kernel/Initializer/random_uniform/sub\n",
      "hidden2/kernel/Initializer/random_uniform/mul\n",
      "hidden2/kernel/Initializer/random_uniform\n",
      "hidden2/kernel\n",
      "hidden2/kernel/Assign\n",
      "hidden2/kernel/read\n",
      "hidden2/bias/Initializer/zeros\n",
      "hidden2/bias\n",
      "hidden2/bias/Assign\n",
      "hidden2/bias/read\n",
      "dnn/hidden2/MatMul\n",
      "dnn/hidden2/BiasAdd\n",
      "dnn/hidden2/Relu\n",
      "hidden3/kernel/Initializer/random_uniform/shape\n",
      "hidden3/kernel/Initializer/random_uniform/min\n",
      "hidden3/kernel/Initializer/random_uniform/max\n",
      "hidden3/kernel/Initializer/random_uniform/RandomUniform\n",
      "hidden3/kernel/Initializer/random_uniform/sub\n",
      "hidden3/kernel/Initializer/random_uniform/mul\n",
      "hidden3/kernel/Initializer/random_uniform\n",
      "hidden3/kernel\n",
      "hidden3/kernel/Assign\n",
      "hidden3/kernel/read\n",
      "hidden3/bias/Initializer/zeros\n",
      "hidden3/bias\n",
      "hidden3/bias/Assign\n",
      "hidden3/bias/read\n",
      "dnn/hidden3/MatMul\n",
      "dnn/hidden3/BiasAdd\n",
      "dnn/hidden3/Relu\n",
      "hidden4/kernel/Initializer/random_uniform/shape\n",
      "hidden4/kernel/Initializer/random_uniform/min\n",
      "hidden4/kernel/Initializer/random_uniform/max\n",
      "hidden4/kernel/Initializer/random_uniform/RandomUniform\n",
      "hidden4/kernel/Initializer/random_uniform/sub\n",
      "hidden4/kernel/Initializer/random_uniform/mul\n",
      "hidden4/kernel/Initializer/random_uniform\n",
      "hidden4/kernel\n",
      "hidden4/kernel/Assign\n",
      "hidden4/kernel/read\n",
      "hidden4/bias/Initializer/zeros\n",
      "hidden4/bias\n",
      "hidden4/bias/Assign\n",
      "hidden4/bias/read\n",
      "dnn/hidden4/MatMul\n",
      "dnn/hidden4/BiasAdd\n",
      "dnn/hidden4/Relu\n",
      "hidden5/kernel/Initializer/random_uniform/shape\n",
      "hidden5/kernel/Initializer/random_uniform/min\n",
      "hidden5/kernel/Initializer/random_uniform/max\n",
      "hidden5/kernel/Initializer/random_uniform/RandomUniform\n",
      "hidden5/kernel/Initializer/random_uniform/sub\n",
      "hidden5/kernel/Initializer/random_uniform/mul\n",
      "hidden5/kernel/Initializer/random_uniform\n",
      "hidden5/kernel\n",
      "hidden5/kernel/Assign\n",
      "hidden5/kernel/read\n",
      "hidden5/bias/Initializer/zeros\n",
      "hidden5/bias\n",
      "hidden5/bias/Assign\n",
      "hidden5/bias/read\n",
      "dnn/hidden5/MatMul\n",
      "dnn/hidden5/BiasAdd\n",
      "dnn/hidden5/Relu\n",
      "outputs/kernel/Initializer/random_uniform/shape\n",
      "outputs/kernel/Initializer/random_uniform/min\n",
      "outputs/kernel/Initializer/random_uniform/max\n",
      "outputs/kernel/Initializer/random_uniform/RandomUniform\n",
      "outputs/kernel/Initializer/random_uniform/sub\n",
      "outputs/kernel/Initializer/random_uniform/mul\n",
      "outputs/kernel/Initializer/random_uniform\n",
      "outputs/kernel\n",
      "outputs/kernel/Assign\n",
      "outputs/kernel/read\n",
      "outputs/bias/Initializer/zeros\n",
      "outputs/bias\n",
      "outputs/bias/Assign\n",
      "outputs/bias/read\n",
      "dnn/outputs/MatMul\n",
      "dnn/outputs/BiasAdd\n",
      "loss/SparseSoftmaxCrossEntropyWithLogits/Shape\n",
      "loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits\n",
      "loss/Const\n",
      "loss/loss\n",
      "gradients/Shape\n",
      "gradients/grad_ys_0\n",
      "gradients/Fill\n",
      "gradients/loss/loss_grad/Reshape/shape\n",
      "gradients/loss/loss_grad/Reshape\n",
      "gradients/loss/loss_grad/Shape\n",
      "gradients/loss/loss_grad/Tile\n",
      "gradients/loss/loss_grad/Shape_1\n",
      "gradients/loss/loss_grad/Shape_2\n",
      "gradients/loss/loss_grad/Const\n",
      "gradients/loss/loss_grad/Prod\n",
      "gradients/loss/loss_grad/Const_1\n",
      "gradients/loss/loss_grad/Prod_1\n",
      "gradients/loss/loss_grad/Maximum/y\n",
      "gradients/loss/loss_grad/Maximum\n",
      "gradients/loss/loss_grad/floordiv\n",
      "gradients/loss/loss_grad/Cast\n",
      "gradients/loss/loss_grad/truediv\n",
      "gradients/zeros_like\n",
      "gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient\n",
      "gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim\n",
      "gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims\n",
      "gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul\n",
      "gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad\n",
      "gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps\n",
      "gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency\n",
      "gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1\n",
      "gradients/dnn/outputs/MatMul_grad/MatMul\n",
      "gradients/dnn/outputs/MatMul_grad/MatMul_1\n",
      "gradients/dnn/outputs/MatMul_grad/tuple/group_deps\n",
      "gradients/dnn/outputs/MatMul_grad/tuple/control_dependency\n",
      "gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden5/Relu_grad/ReluGrad\n",
      "gradients/dnn/hidden5/BiasAdd_grad/BiasAddGrad\n",
      "gradients/dnn/hidden5/BiasAdd_grad/tuple/group_deps\n",
      "gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden5/MatMul_grad/MatMul\n",
      "gradients/dnn/hidden5/MatMul_grad/MatMul_1\n",
      "gradients/dnn/hidden5/MatMul_grad/tuple/group_deps\n",
      "gradients/dnn/hidden5/MatMul_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden5/MatMul_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden4/Relu_grad/ReluGrad\n",
      "gradients/dnn/hidden4/BiasAdd_grad/BiasAddGrad\n",
      "gradients/dnn/hidden4/BiasAdd_grad/tuple/group_deps\n",
      "gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden4/MatMul_grad/MatMul\n",
      "gradients/dnn/hidden4/MatMul_grad/MatMul_1\n",
      "gradients/dnn/hidden4/MatMul_grad/tuple/group_deps\n",
      "gradients/dnn/hidden4/MatMul_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden4/MatMul_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden3/Relu_grad/ReluGrad\n",
      "gradients/dnn/hidden3/BiasAdd_grad/BiasAddGrad\n",
      "gradients/dnn/hidden3/BiasAdd_grad/tuple/group_deps\n",
      "gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden3/MatMul_grad/MatMul\n",
      "gradients/dnn/hidden3/MatMul_grad/MatMul_1\n",
      "gradients/dnn/hidden3/MatMul_grad/tuple/group_deps\n",
      "gradients/dnn/hidden3/MatMul_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden3/MatMul_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden2/Relu_grad/ReluGrad\n",
      "gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad\n",
      "gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps\n",
      "gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden2/MatMul_grad/MatMul\n",
      "gradients/dnn/hidden2/MatMul_grad/MatMul_1\n",
      "gradients/dnn/hidden2/MatMul_grad/tuple/group_deps\n",
      "gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden1/Relu_grad/ReluGrad\n",
      "gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad\n",
      "gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps\n",
      "gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden1/MatMul_grad/MatMul\n",
      "gradients/dnn/hidden1/MatMul_grad/MatMul_1\n",
      "gradients/dnn/hidden1/MatMul_grad/tuple/group_deps\n",
      "gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1\n",
      "clip_by_value/Minimum/y\n",
      "clip_by_value/Minimum\n",
      "clip_by_value/y\n",
      "clip_by_value\n",
      "clip_by_value_1/Minimum/y\n",
      "clip_by_value_1/Minimum\n",
      "clip_by_value_1/y\n",
      "clip_by_value_1\n",
      "clip_by_value_2/Minimum/y\n",
      "clip_by_value_2/Minimum\n",
      "clip_by_value_2/y\n",
      "clip_by_value_2\n",
      "clip_by_value_3/Minimum/y\n",
      "clip_by_value_3/Minimum\n",
      "clip_by_value_3/y\n",
      "clip_by_value_3\n",
      "clip_by_value_4/Minimum/y\n",
      "clip_by_value_4/Minimum\n",
      "clip_by_value_4/y\n",
      "clip_by_value_4\n",
      "clip_by_value_5/Minimum/y\n",
      "clip_by_value_5/Minimum\n",
      "clip_by_value_5/y\n",
      "clip_by_value_5\n",
      "clip_by_value_6/Minimum/y\n",
      "clip_by_value_6/Minimum\n",
      "clip_by_value_6/y\n",
      "clip_by_value_6\n",
      "clip_by_value_7/Minimum/y\n",
      "clip_by_value_7/Minimum\n",
      "clip_by_value_7/y\n",
      "clip_by_value_7\n",
      "clip_by_value_8/Minimum/y\n",
      "clip_by_value_8/Minimum\n",
      "clip_by_value_8/y\n",
      "clip_by_value_8\n",
      "clip_by_value_9/Minimum/y\n",
      "clip_by_value_9/Minimum\n",
      "clip_by_value_9/y\n",
      "clip_by_value_9\n",
      "clip_by_value_10/Minimum/y\n",
      "clip_by_value_10/Minimum\n",
      "clip_by_value_10/y\n",
      "clip_by_value_10\n",
      "clip_by_value_11/Minimum/y\n",
      "clip_by_value_11/Minimum\n",
      "clip_by_value_11/y\n",
      "clip_by_value_11\n",
      "GradientDescent/learning_rate\n",
      "GradientDescent/update_hidden1/kernel/ApplyGradientDescent\n",
      "GradientDescent/update_hidden1/bias/ApplyGradientDescent\n",
      "GradientDescent/update_hidden2/kernel/ApplyGradientDescent\n",
      "GradientDescent/update_hidden2/bias/ApplyGradientDescent\n",
      "GradientDescent/update_hidden3/kernel/ApplyGradientDescent\n",
      "GradientDescent/update_hidden3/bias/ApplyGradientDescent\n",
      "GradientDescent/update_hidden4/kernel/ApplyGradientDescent\n",
      "GradientDescent/update_hidden4/bias/ApplyGradientDescent\n",
      "GradientDescent/update_hidden5/kernel/ApplyGradientDescent\n",
      "GradientDescent/update_hidden5/bias/ApplyGradientDescent\n",
      "GradientDescent/update_outputs/kernel/ApplyGradientDescent\n",
      "GradientDescent/update_outputs/bias/ApplyGradientDescent\n",
      "GradientDescent\n",
      "eval/in_top_k/InTopKV2/k\n",
      "eval/in_top_k/InTopKV2\n",
      "eval/Cast\n",
      "eval/Const\n",
      "eval/accuracy\n",
      "init\n",
      "save/Const\n",
      "save/SaveV2/tensor_names\n",
      "save/SaveV2/shape_and_slices\n",
      "save/SaveV2\n",
      "save/control_dependency\n",
      "save/RestoreV2/tensor_names\n",
      "save/RestoreV2/shape_and_slices\n",
      "save/RestoreV2\n",
      "save/Assign\n",
      "save/Assign_1\n",
      "save/Assign_2\n",
      "save/Assign_3\n",
      "save/Assign_4\n",
      "save/Assign_5\n",
      "save/Assign_6\n",
      "save/Assign_7\n",
      "save/Assign_8\n",
      "save/Assign_9\n",
      "save/Assign_10\n",
      "save/Assign_11\n",
      "save/restore_all\n"
     ]
    }
   ],
   "source": [
    "for op in tf.get_default_graph().get_operations():\n",
    "    print(op.name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "웁스, 연산이 엄청 많네요! 텐서보드로 그래프를 시각화해보는 것이 더 좋을 것 같습니다. 다음 코드는 주피터에서 그래프를 그려줍니다(만약 브라우저에서 보이지 않는다면 `FileWriter`로 그래프를 저장한 다음 텐서보드에서 열어 보세요):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow_graph_in_jupyter import show_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script src=&quot;//cdnjs.cloudflare.com/ajax/libs/polymer/0.3.3/platform.js&quot;></script>\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.3745401188473625&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 784\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        unknown_rank: true\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 5\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\0002\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.13093073666095734\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.13093073666095734\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 22\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 50\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 50\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 50\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;2\\\\000\\\\000\\\\0002\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.24494896829128265\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.24494896829128265\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden3/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 39\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden3/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden3/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden3/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden3/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden3/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden3/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 50\\n        }\\n        dim {\\n          size: 50\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden3/kernel&quot;\\n  input: &quot;hidden3/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden3/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 50\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 50\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden3/bias&quot;\\n  input: &quot;hidden3/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden3/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden3/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;hidden3/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden3/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden3/MatMul&quot;\\n  input: &quot;hidden3/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden3/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden3/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;2\\\\000\\\\000\\\\0002\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.24494896829128265\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.24494896829128265\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden4/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 56\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden4/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden4/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden4/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden4/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden4/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden4/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 50\\n        }\\n        dim {\\n          size: 50\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden4/kernel&quot;\\n  input: &quot;hidden4/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden4/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 50\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 50\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden4/bias&quot;\\n  input: &quot;hidden4/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden4/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden4/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden3/Relu&quot;\\n  input: &quot;hidden4/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden4/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden4/MatMul&quot;\\n  input: &quot;hidden4/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden4/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden4/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;2\\\\000\\\\000\\\\0002\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.24494896829128265\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.24494896829128265\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden5/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 73\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden5/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden5/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden5/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden5/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden5/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden5/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 50\\n        }\\n        dim {\\n          size: 50\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden5/kernel&quot;\\n  input: &quot;hidden5/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden5/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 50\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 50\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden5/bias&quot;\\n  input: &quot;hidden5/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden5/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden5/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden4/Relu&quot;\\n  input: &quot;hidden5/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden5/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden5/MatMul&quot;\\n  input: &quot;hidden5/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden5/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden5/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;2\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.3162277638912201\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.3162277638912201\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 90\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 50\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden5/Relu&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/grad_ys_0&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;gradients/Shape&quot;\\n  input: &quot;gradients/grad_ys_0&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;gradients/Fill&quot;\\n  input: &quot;gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Truncate&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden5/Relu&quot;\\n  input: &quot;gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden5/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;gradients/dnn/hidden5/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden5/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/dnn/hidden5/Relu_grad/ReluGrad&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden5/Relu_grad/ReluGrad&quot;\\n  input: &quot;^gradients/dnn/hidden5/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden5/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden5/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/dnn/hidden5/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden5/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden5/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden4/Relu&quot;\\n  input: &quot;gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden5/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden5/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden5/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden5/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden5/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden5/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^gradients/dnn/hidden5/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden5/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;gradients/dnn/hidden5/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden4/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;gradients/dnn/hidden4/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden4/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/dnn/hidden4/Relu_grad/ReluGrad&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden4/Relu_grad/ReluGrad&quot;\\n  input: &quot;^gradients/dnn/hidden4/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden4/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden4/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/dnn/hidden4/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden4/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden4/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden3/Relu&quot;\\n  input: &quot;gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden4/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden4/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden4/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden4/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden4/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden4/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^gradients/dnn/hidden4/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden4/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;gradients/dnn/hidden4/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden3/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;gradients/dnn/hidden3/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden3/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/dnn/hidden3/Relu_grad/ReluGrad&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden3/Relu_grad/ReluGrad&quot;\\n  input: &quot;^gradients/dnn/hidden3/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden3/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden3/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/dnn/hidden3/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden3/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden3/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden3/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden3/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden3/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden3/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden3/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden3/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^gradients/dnn/hidden3/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden3/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;gradients/dnn/hidden3/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value/Minimum&quot;\\n  input: &quot;clip_by_value/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_1/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_1/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_1/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_1/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_1&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_1/Minimum&quot;\\n  input: &quot;clip_by_value_1/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_2/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_2/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_2/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_2/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_2&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_2/Minimum&quot;\\n  input: &quot;clip_by_value_2/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_3/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_3/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_3/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_3/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_3&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_3/Minimum&quot;\\n  input: &quot;clip_by_value_3/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_4/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_4/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden3/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_4/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_4/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_4&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_4/Minimum&quot;\\n  input: &quot;clip_by_value_4/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_5/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_5/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_5/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_5/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_5&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_5/Minimum&quot;\\n  input: &quot;clip_by_value_5/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_6/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_6/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden4/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_6/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_6/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_6&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_6/Minimum&quot;\\n  input: &quot;clip_by_value_6/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_7/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_7/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_7/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_7/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_7&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_7/Minimum&quot;\\n  input: &quot;clip_by_value_7/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_8/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_8/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden5/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_8/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_8/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_8&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_8/Minimum&quot;\\n  input: &quot;clip_by_value_8/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_9/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_9/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_9/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_9/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_9&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_9/Minimum&quot;\\n  input: &quot;clip_by_value_9/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_10/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_10/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_10/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_10/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_10&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_10/Minimum&quot;\\n  input: &quot;clip_by_value_10/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_11/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_11/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_11/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_11/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_11&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_11/Minimum&quot;\\n  input: &quot;clip_by_value_11/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden3/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden3/kernel&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden3/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden3/bias&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden4/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden4/kernel&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_6&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden4/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden4/bias&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_7&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden5/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden5/kernel&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_8&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden5/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden5/bias&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_9&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_10&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_11&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden3/bias/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden3/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden4/bias/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden4/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden5/bias/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden5/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2&quot;\\n  op: &quot;InTopKV2&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  input: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/in_top_k/InTopKV2&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n  attr {\\n    key: &quot;Truncate&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/accuracy&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^hidden1/bias/Assign&quot;\\n  input: &quot;^hidden1/kernel/Assign&quot;\\n  input: &quot;^hidden2/bias/Assign&quot;\\n  input: &quot;^hidden2/kernel/Assign&quot;\\n  input: &quot;^hidden3/bias/Assign&quot;\\n  input: &quot;^hidden3/kernel/Assign&quot;\\n  input: &quot;^hidden4/bias/Assign&quot;\\n  input: &quot;^hidden4/kernel/Assign&quot;\\n  input: &quot;^hidden5/bias/Assign&quot;\\n  input: &quot;^hidden5/kernel/Assign&quot;\\n  input: &quot;^outputs/bias/Assign&quot;\\n  input: &quot;^outputs/kernel/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 12\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;hidden3/bias&quot;\\n        string_val: &quot;hidden3/kernel&quot;\\n        string_val: &quot;hidden4/bias&quot;\\n        string_val: &quot;hidden4/kernel&quot;\\n        string_val: &quot;hidden5/bias&quot;\\n        string_val: &quot;hidden5/kernel&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 12\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;hidden3/bias&quot;\\n  input: &quot;hidden3/kernel&quot;\\n  input: &quot;hidden4/bias&quot;\\n  input: &quot;hidden4/kernel&quot;\\n  input: &quot;hidden5/bias&quot;\\n  input: &quot;hidden5/kernel&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 12\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;hidden3/bias&quot;\\n        string_val: &quot;hidden3/kernel&quot;\\n        string_val: &quot;hidden4/bias&quot;\\n        string_val: &quot;hidden4/kernel&quot;\\n        string_val: &quot;hidden5/bias&quot;\\n        string_val: &quot;hidden5/kernel&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 12\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;save/RestoreV2:2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2:3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden3/bias&quot;\\n  input: &quot;save/RestoreV2:4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden3/kernel&quot;\\n  input: &quot;save/RestoreV2:5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_6&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden4/bias&quot;\\n  input: &quot;save/RestoreV2:6&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_7&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden4/kernel&quot;\\n  input: &quot;save/RestoreV2:7&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_8&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden5/bias&quot;\\n  input: &quot;save/RestoreV2:8&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_9&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden5/kernel&quot;\\n  input: &quot;save/RestoreV2:9&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_10&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;save/RestoreV2:10&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_11&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;save/RestoreV2:11&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_10&quot;\\n  input: &quot;^save/Assign_11&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n  input: &quot;^save/Assign_6&quot;\\n  input: &quot;^save/Assign_7&quot;\\n  input: &quot;^save/Assign_8&quot;\\n  input: &quot;^save/Assign_9&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.3745401188473625&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "필요한 연산을 찾았다면 그래프의 `get_operation_by_name()`이나 `get_tensor_by_name()` 메서드를 사용하여 추출할 수 있습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tf.get_default_graph().get_tensor_by_name(\"X:0\")\n",
    "y = tf.get_default_graph().get_tensor_by_name(\"y:0\")\n",
    "\n",
    "accuracy = tf.get_default_graph().get_tensor_by_name(\"eval/accuracy:0\")\n",
    "\n",
    "training_op = tf.get_default_graph().get_operation_by_name(\"GradientDescent\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "원본 모델을 만들 때 다른 사람이 재사용하기 쉽게 연산에 명확한 이름을 부여하고 문서화를 하는 것이 좋습니다. 또 다른 방법은 처리해야 할 중요한 연산들을 모두 모아 놓은 컬렉션을 만드는 것입니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "for op in (X, y, accuracy, training_op):\n",
    "    tf.add_to_collection(\"my_important_ops\", op)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이렇게 하면 모델을 재사용할 때 다음과 같이 간단하게 쓸 수 있습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y, accuracy, training_op = tf.get_collection(\"my_important_ops\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 세션을 시작하고 모델을 복원하여 준비된 훈련 데이터로 훈련을 계속할 수 있습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_final.ckpt\")\n",
    "    # 모델 훈련 계속하기..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "실제로 테스트를 해보죠!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n",
      "0 검증 세트 정확도: 0.9632\n",
      "1 검증 세트 정확도: 0.963\n",
      "2 검증 세트 정확도: 0.9652\n",
      "3 검증 세트 정확도: 0.9652\n",
      "4 검증 세트 정확도: 0.9642\n",
      "5 검증 세트 정확도: 0.965\n",
      "6 검증 세트 정확도: 0.9684\n",
      "7 검증 세트 정확도: 0.9686\n",
      "8 검증 세트 정확도: 0.968\n",
      "9 검증 세트 정확도: 0.9688\n",
      "10 검증 세트 정확도: 0.97\n",
      "11 검증 세트 정확도: 0.9712\n",
      "12 검증 세트 정확도: 0.9672\n",
      "13 검증 세트 정확도: 0.97\n",
      "14 검증 세트 정확도: 0.971\n",
      "15 검증 세트 정확도: 0.972\n",
      "16 검증 세트 정확도: 0.972\n",
      "17 검증 세트 정확도: 0.9712\n",
      "18 검증 세트 정확도: 0.9712\n",
      "19 검증 세트 정확도: 0.971\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_final.ckpt\")\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"검증 세트 정확도:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_new_model_final.ckpt\")    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "또 다른 방법으로 원본 그래프를 만든 파이썬 코드에 접근할 수 있다면 `import_meta_graph()`를 대신 사용할 수 있습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 50\n",
    "n_hidden3 = 50\n",
    "n_hidden4 = 50\n",
    "n_outputs = 10\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, name=\"hidden2\")\n",
    "    hidden3 = tf.layers.dense(hidden2, n_hidden3, activation=tf.nn.relu, name=\"hidden3\")\n",
    "    hidden4 = tf.layers.dense(hidden3, n_hidden4, activation=tf.nn.relu, name=\"hidden4\")\n",
    "    hidden5 = tf.layers.dense(hidden4, n_hidden5, activation=tf.nn.relu, name=\"hidden5\")\n",
    "    logits = tf.layers.dense(hidden5, n_outputs, name=\"outputs\")\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n",
    "\n",
    "learning_rate = 0.01\n",
    "threshold = 1.0\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "grads_and_vars = optimizer.compute_gradients(loss)\n",
    "capped_gvs = [(tf.clip_by_value(grad, -threshold, threshold), var)\n",
    "              for grad, var in grads_and_vars]\n",
    "training_op = optimizer.apply_gradients(capped_gvs)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "그 다음 훈련을 계속할 수 있습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n",
      "0 검증 세트 정확도: 0.964\n",
      "1 검증 세트 정확도: 0.9628\n",
      "2 검증 세트 정확도: 0.9652\n",
      "3 검증 세트 정확도: 0.9652\n",
      "4 검증 세트 정확도: 0.964\n",
      "5 검증 세트 정확도: 0.965\n",
      "6 검증 세트 정확도: 0.9688\n",
      "7 검증 세트 정확도: 0.9686\n",
      "8 검증 세트 정확도: 0.9682\n",
      "9 검증 세트 정확도: 0.9686\n",
      "10 검증 세트 정확도: 0.9702\n",
      "11 검증 세트 정확도: 0.9714\n",
      "12 검증 세트 정확도: 0.9672\n",
      "13 검증 세트 정확도: 0.9696\n",
      "14 검증 세트 정확도: 0.971\n",
      "15 검증 세트 정확도: 0.9724\n",
      "16 검증 세트 정확도: 0.9718\n",
      "17 검증 세트 정확도: 0.9714\n",
      "18 검증 세트 정확도: 0.9714\n",
      "19 검증 세트 정확도: 0.9712\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_final.ckpt\")\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"검증 세트 정확도:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_new_model_final.ckpt\")    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "일반적으로 하위층만 재사용할 것입니다. `import_meta_graph()`를 사용하면 전체 그래프를 로드하지만 필요하지 않은 부분은 무시하면 됩니다. 이 예에서는 학습된 3번째 층 위에 4번째 은닉층을 새로 추가합니다(원래 4번째 층은 무시됩니다). 새로운 출력층도 추가하고 이 출력으로 손실을 계산하고 이를 최소화하기 위한 새로운 옵티마이저를 만듭니다. 전체 그래프(원본 그래프 전체와 새로운 연산)를 저장할 새로운 `Saver` 객체와 새로운 모든 변수를 초기화할 초기화 연산도 필요합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_hidden4 = 20  # 새 층\n",
    "n_outputs = 10  # 새 층\n",
    "\n",
    "saver = tf.train.import_meta_graph(\"./my_model_final.ckpt.meta\")\n",
    "\n",
    "X = tf.get_default_graph().get_tensor_by_name(\"X:0\")\n",
    "y = tf.get_default_graph().get_tensor_by_name(\"y:0\")\n",
    "\n",
    "hidden3 = tf.get_default_graph().get_tensor_by_name(\"dnn/hidden3/Relu:0\")\n",
    "\n",
    "new_hidden4 = tf.layers.dense(hidden3, n_hidden4, activation=tf.nn.relu, name=\"new_hidden4\")\n",
    "new_logits = tf.layers.dense(new_hidden4, n_outputs, name=\"new_outputs\")\n",
    "\n",
    "with tf.name_scope(\"new_loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=new_logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"new_eval\"):\n",
    "    correct = tf.nn.in_top_k(new_logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n",
    "\n",
    "with tf.name_scope(\"new_train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "new_saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "새로운 모델을 훈련시킵니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n",
      "0 검증 세트 정확도: 0.9118\n",
      "1 검증 세트 정확도: 0.9378\n",
      "2 검증 세트 정확도: 0.9462\n",
      "3 검증 세트 정확도: 0.9502\n",
      "4 검증 세트 정확도: 0.9526\n",
      "5 검증 세트 정확도: 0.9524\n",
      "6 검증 세트 정확도: 0.9568\n",
      "7 검증 세트 정확도: 0.96\n",
      "8 검증 세트 정확도: 0.9612\n",
      "9 검증 세트 정확도: 0.961\n",
      "10 검증 세트 정확도: 0.9632\n",
      "11 검증 세트 정확도: 0.9628\n",
      "12 검증 세트 정확도: 0.965\n",
      "13 검증 세트 정확도: 0.9654\n",
      "14 검증 세트 정확도: 0.9666\n",
      "15 검증 세트 정확도: 0.967\n",
      "16 검증 세트 정확도: 0.9674\n",
      "17 검증 세트 정확도: 0.9682\n",
      "18 검증 세트 정확도: 0.9686\n",
      "19 검증 세트 정확도: 0.9676\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    saver.restore(sess, \"./my_model_final.ckpt\")\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"검증 세트 정확도:\", accuracy_val)\n",
    "\n",
    "    save_path = new_saver.save(sess, \"./my_new_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "원본 모델을 만든 파이썬 코드에 접근할 수 있다면 필요한 부분만 재사용하고 나머지는 버릴 수 있습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300 # 재사용\n",
    "n_hidden2 = 50  # 재사용\n",
    "n_hidden3 = 50  # 재사용\n",
    "n_hidden4 = 20  # 새로 만듦!\n",
    "n_outputs = 10  # 새로 만듦!\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")       # 재사용\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, name=\"hidden2\") # 재사용\n",
    "    hidden3 = tf.layers.dense(hidden2, n_hidden3, activation=tf.nn.relu, name=\"hidden3\") # 재사용\n",
    "    hidden4 = tf.layers.dense(hidden3, n_hidden4, activation=tf.nn.relu, name=\"hidden4\") # 새로 만듦!\n",
    "    logits = tf.layers.dense(hidden4, n_outputs, name=\"outputs\")                         # 새로 만듦!\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "그러나 이전에 학습된 모델을 복원하기 위해 (복원할 변수 리스트를 전달합니다. 그렇지 않으면 그래프와 맞지 않는다고 에러를 낼 것입니다) `Saver` 객체를 하나 만들고 훈련이 끝난 후 새로운 모델을 저장하기 위해 또 다른 `Saver` 객체를 만들어야 합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n",
      "0 검증 세트 정확도: 0.9024\n",
      "1 검증 세트 정확도: 0.9336\n",
      "2 검증 세트 정확도: 0.9426\n",
      "3 검증 세트 정확도: 0.9474\n",
      "4 검증 세트 정확도: 0.952\n",
      "5 검증 세트 정확도: 0.9538\n",
      "6 검증 세트 정확도: 0.9558\n",
      "7 검증 세트 정확도: 0.959\n",
      "8 검증 세트 정확도: 0.9584\n",
      "9 검증 세트 정확도: 0.9604\n",
      "10 검증 세트 정확도: 0.9618\n",
      "11 검증 세트 정확도: 0.9624\n",
      "12 검증 세트 정확도: 0.964\n",
      "13 검증 세트 정확도: 0.9662\n",
      "14 검증 세트 정확도: 0.9664\n",
      "15 검증 세트 정확도: 0.9662\n",
      "16 검증 세트 정확도: 0.9672\n",
      "17 검증 세트 정확도: 0.9674\n",
      "18 검증 세트 정확도: 0.9684\n",
      "19 검증 세트 정확도: 0.9672\n"
     ]
    }
   ],
   "source": [
    "reuse_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,\n",
    "                               scope=\"hidden[123]\") # 정규표현식\n",
    "restore_saver = tf.train.Saver(reuse_vars) # 1-3층 복원\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    restore_saver.restore(sess, \"./my_model_final.ckpt\")\n",
    "\n",
    "    for epoch in range(n_epochs):                                        # 책에는 없음\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size): # 책에는 없음\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})    # 책에는 없음\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid}) # 책에는 없음\n",
    "        print(epoch, \"검증 세트 정확도:\", accuracy_val)                      # 책에는 없음\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_new_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 다른 프레임워크의 모델 재사용하기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이 예에서는 재사용하려는 각 변수에 대해 변수 초기화 할당 연산을 찾고, 초기화 될 값에 해당하는 두 번째 입력 핸들을 구합니다. 초기화가 실행될 때 여기에 `feed_dict` 매개변수를 사용하여 초깃값 대신 원하는 값을 주입합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 2\n",
    "n_hidden1 = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 61.  83. 105.]]\n"
     ]
    }
   ],
   "source": [
    "original_w = [[1., 2., 3.], [4., 5., 6.]] # 다른 프레임워크로부터 가중치를 로드\n",
    "original_b = [7., 8., 9.]                 # 다른 프레임워크로부터 편향을 로드\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n",
    "# [...] 모델의 나머지 부분을 구성\n",
    "\n",
    "# hidden1 변수의 할당 노드에 대한 핸들을 구합니다\n",
    "graph = tf.get_default_graph()\n",
    "assign_kernel = graph.get_operation_by_name(\"hidden1/kernel/Assign\")\n",
    "assign_bias = graph.get_operation_by_name(\"hidden1/bias/Assign\")\n",
    "init_kernel = assign_kernel.inputs[1]\n",
    "init_bias = assign_bias.inputs[1]\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init, feed_dict={init_kernel: original_w, init_bias: original_b})\n",
    "    # [...] 새 작업에 모델을 훈련시킵니다\n",
    "    print(hidden1.eval(feed_dict={X: [[10.0, 11.0]]}))  # 책에는 없음"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "또 다른 방법은 전용 할당 노드와 플레이스홀더를 만든는 것입니다. 이 방법은 더 번거롭고 효율적이지 않지만 하려는 방식이 잘 드러나는 방법입니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 61.  83. 105.]]\n"
     ]
    }
   ],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 2\n",
    "n_hidden1 = 3\n",
    "\n",
    "original_w = [[1., 2., 3.], [4., 5., 6.]] # 다른 프레임워크로부터 가중치를 로드\n",
    "original_b = [7., 8., 9.]                 # 다른 프레임워크로부터 편향을 로드\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n",
    "# [...] 모델의 나머지를 구성\n",
    "\n",
    "# hidden1 변수의 할당 노드에 대한 핸들을 구합니다\n",
    "with tf.variable_scope(\"\", default_name=\"\", reuse=True):  # 루트 범위\n",
    "    hidden1_weights = tf.get_variable(\"hidden1/kernel\")\n",
    "    hidden1_biases = tf.get_variable(\"hidden1/bias\")\n",
    "\n",
    "# 전용 플레이스홀더와 할당 노드를 만듭니다\n",
    "original_weights = tf.placeholder(tf.float32, shape=(n_inputs, n_hidden1))\n",
    "original_biases = tf.placeholder(tf.float32, shape=n_hidden1)\n",
    "assign_hidden1_weights = tf.assign(hidden1_weights, original_weights)\n",
    "assign_hidden1_biases = tf.assign(hidden1_biases, original_biases)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    sess.run(assign_hidden1_weights, feed_dict={original_weights: original_w})\n",
    "    sess.run(assign_hidden1_biases, feed_dict={original_biases: original_b})\n",
    "    # [...] 새 작업에 모델을 훈련시킵니다\n",
    "    print(hidden1.eval(feed_dict={X: [[10.0, 11.0]]}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`get_collection()`에 `scope`를 지정하여 변수의 핸들을 가져올 수도 있습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'hidden1/kernel:0' shape=(2, 3) dtype=float32_ref>,\n",
       " <tf.Variable 'hidden1/bias:0' shape=(3,) dtype=float32_ref>]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=\"hidden1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "또는 그래프의 `get_tensor_by_name()` 메서드를 사용할 수 있습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'hidden1/kernel:0' shape=(2, 3) dtype=float32_ref>"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.get_default_graph().get_tensor_by_name(\"hidden1/kernel:0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'hidden1/bias:0' shape=(3,) dtype=float32_ref>"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.get_default_graph().get_tensor_by_name(\"hidden1/bias:0\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 하위층 동결하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300 # 재사용\n",
    "n_hidden2 = 50  # 재사용\n",
    "n_hidden3 = 50  # 재사용\n",
    "n_hidden4 = 20  # 새로 만듦!\n",
    "n_outputs = 10  # 새로 만듦!\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")       # 재사용\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, name=\"hidden2\") # 재사용\n",
    "    hidden3 = tf.layers.dense(hidden2, n_hidden3, activation=tf.nn.relu, name=\"hidden3\") # 재사용\n",
    "    hidden4 = tf.layers.dense(hidden3, n_hidden4, activation=tf.nn.relu, name=\"hidden4\") # 새로 만듦!\n",
    "    logits = tf.layers.dense(hidden4, n_outputs, name=\"outputs\")                         # 새로 만듦!\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"train\"):                                         # 책에는 없음\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)     # 책에는 없음\n",
    "    train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\n",
    "                                   scope=\"hidden[34]|outputs\")\n",
    "    training_op = optimizer.minimize(loss, var_list=train_vars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "new_saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n",
      "0 검증 세트 정확도: 0.8962\n",
      "1 검증 세트 정확도: 0.9288\n",
      "2 검증 세트 정확도: 0.9402\n",
      "3 검증 세트 정확도: 0.944\n",
      "4 검증 세트 정확도: 0.9482\n",
      "5 검증 세트 정확도: 0.9506\n",
      "6 검증 세트 정확도: 0.9508\n",
      "7 검증 세트 정확도: 0.9538\n",
      "8 검증 세트 정확도: 0.9552\n",
      "9 검증 세트 정확도: 0.9564\n",
      "10 검증 세트 정확도: 0.9562\n",
      "11 검증 세트 정확도: 0.9568\n",
      "12 검증 세트 정확도: 0.9572\n",
      "13 검증 세트 정확도: 0.9578\n",
      "14 검증 세트 정확도: 0.9592\n",
      "15 검증 세트 정확도: 0.9578\n",
      "16 검증 세트 정확도: 0.9576\n",
      "17 검증 세트 정확도: 0.9604\n",
      "18 검증 세트 정확도: 0.9592\n",
      "19 검증 세트 정확도: 0.9604\n"
     ]
    }
   ],
   "source": [
    "reuse_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,\n",
    "                               scope=\"hidden[123]\") # 정규 표현식\n",
    "restore_saver = tf.train.Saver(reuse_vars) # 1-3층 복원\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    restore_saver.restore(sess, \"./my_model_final.ckpt\")\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"검증 세트 정확도:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_new_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300 # 재사용\n",
    "n_hidden2 = 50  # 재사용\n",
    "n_hidden3 = 50  # 재사용\n",
    "n_hidden4 = 20  # 새로 만듦!\n",
    "n_outputs = 10  # 새로 만듦!\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu,\n",
    "                              name=\"hidden1\") # 동결층 재사용\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu,\n",
    "                              name=\"hidden2\") # 동결층 재사용\n",
    "    hidden2_stop = tf.stop_gradient(hidden2)\n",
    "    hidden3 = tf.layers.dense(hidden2_stop, n_hidden3, activation=tf.nn.relu,\n",
    "                              name=\"hidden3\") # 동결하지 않고 재사용\n",
    "    hidden4 = tf.layers.dense(hidden3, n_hidden4, activation=tf.nn.relu,\n",
    "                              name=\"hidden4\") # 새로 만듦!\n",
    "    logits = tf.layers.dense(hidden4, n_outputs, name=\"outputs\") # 새로 만듦!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "훈련하는 코드는 이전과 완전히 동일합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n",
      "0 검증 세트 정확도: 0.902\n",
      "1 검증 세트 정확도: 0.931\n",
      "2 검증 세트 정확도: 0.9434\n",
      "3 검증 세트 정확도: 0.9476\n",
      "4 검증 세트 정확도: 0.9516\n",
      "5 검증 세트 정확도: 0.9524\n",
      "6 검증 세트 정확도: 0.9524\n",
      "7 검증 세트 정확도: 0.9558\n",
      "8 검증 세트 정확도: 0.9554\n",
      "9 검증 세트 정확도: 0.956\n",
      "10 검증 세트 정확도: 0.9566\n",
      "11 검증 세트 정확도: 0.9552\n",
      "12 검증 세트 정확도: 0.9574\n",
      "13 검증 세트 정확도: 0.9578\n",
      "14 검증 세트 정확도: 0.958\n",
      "15 검증 세트 정확도: 0.9572\n",
      "16 검증 세트 정확도: 0.9564\n",
      "17 검증 세트 정확도: 0.9578\n",
      "18 검증 세트 정확도: 0.9592\n",
      "19 검증 세트 정확도: 0.958\n"
     ]
    }
   ],
   "source": [
    "reuse_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,\n",
    "                               scope=\"hidden[123]\") # 정규 표현식\n",
    "restore_saver = tf.train.Saver(reuse_vars) # 1-3층 복원\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    restore_saver.restore(sess, \"./my_model_final.ckpt\")\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"검증 세트 정확도:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_new_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 동결층 캐싱하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300 # 재사용\n",
    "n_hidden2 = 50  # 재사용\n",
    "n_hidden3 = 50  # 재사용\n",
    "n_hidden4 = 20  # 새로 만듦!\n",
    "n_outputs = 10  # 새로 만듦!\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu,\n",
    "                              name=\"hidden1\") # 동결층 재사용\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu,\n",
    "                              name=\"hidden2\") # 동결층 재사용 & 캐싱\n",
    "    hidden2_stop = tf.stop_gradient(hidden2)\n",
    "    hidden3 = tf.layers.dense(hidden2_stop, n_hidden3, activation=tf.nn.relu,\n",
    "                              name=\"hidden3\") # 동결하지 않고 재사용\n",
    "    hidden4 = tf.layers.dense(hidden3, n_hidden4, activation=tf.nn.relu,\n",
    "                              name=\"hidden4\") # 새로 만듦!\n",
    "    logits = tf.layers.dense(hidden4, n_outputs, name=\"outputs\") # 새로 만듦!\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "reuse_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,\n",
    "                               scope=\"hidden[123]\") # 정규 표현식\n",
    "restore_saver = tf.train.Saver(reuse_vars) # 1-3층 복원\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n",
      "0 검증 세트 정확도: 0.902\n",
      "1 검증 세트 정확도: 0.931\n",
      "2 검증 세트 정확도: 0.9434\n",
      "3 검증 세트 정확도: 0.9476\n",
      "4 검증 세트 정확도: 0.9516\n",
      "5 검증 세트 정확도: 0.9524\n",
      "6 검증 세트 정확도: 0.9524\n",
      "7 검증 세트 정확도: 0.9558\n",
      "8 검증 세트 정확도: 0.9554\n",
      "9 검증 세트 정확도: 0.956\n",
      "10 검증 세트 정확도: 0.9566\n",
      "11 검증 세트 정확도: 0.9552\n",
      "12 검증 세트 정확도: 0.9574\n",
      "13 검증 세트 정확도: 0.9578\n",
      "14 검증 세트 정확도: 0.958\n",
      "15 검증 세트 정확도: 0.9572\n",
      "16 검증 세트 정확도: 0.9564\n",
      "17 검증 세트 정확도: 0.9578\n",
      "18 검증 세트 정확도: 0.9592\n",
      "19 검증 세트 정확도: 0.958\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "n_batches = len(X_train) // batch_size\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    restore_saver.restore(sess, \"./my_model_final.ckpt\")\n",
    "    \n",
    "    h2_cache = sess.run(hidden2, feed_dict={X: X_train})\n",
    "    h2_cache_valid = sess.run(hidden2, feed_dict={X: X_valid}) # 책에는 없음\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        shuffled_idx = np.random.permutation(len(X_train))\n",
    "        hidden2_batches = np.array_split(h2_cache[shuffled_idx], n_batches)\n",
    "        y_batches = np.array_split(y_train[shuffled_idx], n_batches)\n",
    "        for hidden2_batch, y_batch in zip(hidden2_batches, y_batches):\n",
    "            sess.run(training_op, feed_dict={hidden2:hidden2_batch, y:y_batch})\n",
    "\n",
    "        accuracy_val = accuracy.eval(feed_dict={hidden2: h2_cache_valid, # 책에는 없음\n",
    "                                                y: y_valid})             # 책에는 없음\n",
    "        print(epoch, \"검증 세트 정확도:\", accuracy_val)                      # 책에는 없음\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_new_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 고속 옵티마이저"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 모멘텀 옵티마이저"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,\n",
    "                                       momentum=0.9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 네스테로프 가속 경사"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,\n",
    "                                       momentum=0.9, use_nesterov=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AdaGrad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdagradOptimizer(learning_rate=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RMSProp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate,\n",
    "                                      momentum=0.9, decay=0.9, epsilon=1e-10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adam 최적화"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 학습률 스케줄링"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 50\n",
    "n_outputs = 10\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, name=\"hidden2\")\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"train\"):       # 책에는 없음\n",
    "    initial_learning_rate = 0.1\n",
    "    decay_steps = 10000\n",
    "    decay_rate = 1/10\n",
    "    global_step = tf.Variable(0, trainable=False, name=\"global_step\")\n",
    "    learning_rate = tf.train.exponential_decay(initial_learning_rate, global_step,\n",
    "                                               decay_steps, decay_rate)\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=0.9)\n",
    "    training_op = optimizer.minimize(loss, global_step=global_step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 검증 세트 정확도: 0.9654\n",
      "1 검증 세트 정확도: 0.9716\n",
      "2 검증 세트 정확도: 0.9726\n",
      "3 검증 세트 정확도: 0.9798\n",
      "4 검증 세트 정확도: 0.9798\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"검증 세트 정확도:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 규제로 과대적합 피하기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## $\\ell_1$과 $\\ell_2$ 규제"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$\\ell_1$ 규제를 직접 구현해 보죠. 먼저 평상시처럼 모델을 만듭니다(간단하게 하기 위해 은닉층을 하나만 두겠습니다):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_outputs = 10\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n",
    "    logits = tf.layers.dense(hidden1, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "그다음, 층의 가중치에 대한 핸들을 얻어 크로스 엔트로피 손실에 $\\ell_1$ 손실(즉, 가중치의 절댓값)을 더해 전체 손실을 계산합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "W1 = tf.get_default_graph().get_tensor_by_name(\"hidden1/kernel:0\")\n",
    "W2 = tf.get_default_graph().get_tensor_by_name(\"outputs/kernel:0\")\n",
    "\n",
    "scale = 0.001 # l1 규제 하이퍼파라미터\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,\n",
    "                                                              logits=logits)\n",
    "    base_loss = tf.reduce_mean(xentropy, name=\"avg_xentropy\")\n",
    "    reg_losses = tf.reduce_sum(tf.abs(W1)) + tf.reduce_sum(tf.abs(W2))\n",
    "    loss = tf.add(base_loss, scale * reg_losses, name=\"loss\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "나머지는 이전과 동일합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n",
    "\n",
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 검증 세트 정확도: 0.831\n",
      "1 검증 세트 정확도: 0.871\n",
      "2 검증 세트 정확도: 0.8838\n",
      "3 검증 세트 정확도: 0.8934\n",
      "4 검증 세트 정확도: 0.8966\n",
      "5 검증 세트 정확도: 0.8988\n",
      "6 검증 세트 정확도: 0.9016\n",
      "7 검증 세트 정확도: 0.9044\n",
      "8 검증 세트 정확도: 0.9058\n",
      "9 검증 세트 정확도: 0.906\n",
      "10 검증 세트 정확도: 0.9068\n",
      "11 검증 세트 정확도: 0.9054\n",
      "12 검증 세트 정확도: 0.907\n",
      "13 검증 세트 정확도: 0.9084\n",
      "14 검증 세트 정확도: 0.9088\n",
      "15 검증 세트 정확도: 0.9064\n",
      "16 검증 세트 정확도: 0.9066\n",
      "17 검증 세트 정확도: 0.9066\n",
      "18 검증 세트 정확도: 0.9066\n",
      "19 검증 세트 정확도: 0.9052\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 200\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"검증 세트 정확도:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "다른 방법으로는 `tf.layers.dense()` 함수에 규제 함수를 전달할 수 있습니다. 이 함수는 규제 손실을 계산하기 위한 연산을 만들고 규제 손실 컬렉션에 이 연산을 추가합니다. 모델 선언부는 이전과 동일합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 50\n",
    "n_outputs = 10\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "그다음, 동일한 매개변수를 매번 반복하지 않으려고 파이썬의 `partial()` 함수를 사용합니다. `kernel_regularizer` 매개변수를 지정해야 합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "scale = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_dense_layer = partial(\n",
    "    tf.layers.dense, activation=tf.nn.relu,\n",
    "    kernel_regularizer=tf.contrib.layers.l1_regularizer(scale))\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = my_dense_layer(X, n_hidden1, name=\"hidden1\")\n",
    "    hidden2 = my_dense_layer(hidden1, n_hidden2, name=\"hidden2\")\n",
    "    logits = my_dense_layer(hidden2, n_outputs, activation=None,\n",
    "                            name=\"outputs\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "기본 손실에 규제 손실을 추가합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):                                     # 책에는 없음\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(  # 책에는 없음\n",
    "        labels=y, logits=logits)                                # 책에는 없음\n",
    "    base_loss = tf.reduce_mean(xentropy, name=\"avg_xentropy\")   # 책에는 없음\n",
    "    reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
    "    loss = tf.add_n([base_loss] + reg_losses, name=\"loss\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "나머지는 평상시와 동일합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n",
    "\n",
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 검증 세트 정확도: 0.8274\n",
      "1 검증 세트 정확도: 0.8766\n",
      "2 검증 세트 정확도: 0.8952\n",
      "3 검증 세트 정확도: 0.9016\n",
      "4 검증 세트 정확도: 0.908\n",
      "5 검증 세트 정확도: 0.9096\n",
      "6 검증 세트 정확도: 0.9124\n",
      "7 검증 세트 정확도: 0.9154\n",
      "8 검증 세트 정확도: 0.9178\n",
      "9 검증 세트 정확도: 0.919\n",
      "10 검증 세트 정확도: 0.92\n",
      "11 검증 세트 정확도: 0.9224\n",
      "12 검증 세트 정확도: 0.9212\n",
      "13 검증 세트 정확도: 0.9228\n",
      "14 검증 세트 정확도: 0.9222\n",
      "15 검증 세트 정확도: 0.9218\n",
      "16 검증 세트 정확도: 0.9218\n",
      "17 검증 세트 정확도: 0.9228\n",
      "18 검증 세트 정확도: 0.9216\n",
      "19 검증 세트 정확도: 0.9214\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 200\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"검증 세트 정확도:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 드롭아웃"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "training = tf.placeholder_with_default(False, shape=(), name='training')\n",
    "\n",
    "dropout_rate = 0.5  # == 1 - keep_prob\n",
    "X_drop = tf.layers.dropout(X, dropout_rate, training=training)\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X_drop, n_hidden1, activation=tf.nn.relu,\n",
    "                              name=\"hidden1\")\n",
    "    hidden1_drop = tf.layers.dropout(hidden1, dropout_rate, training=training)\n",
    "    hidden2 = tf.layers.dense(hidden1_drop, n_hidden2, activation=tf.nn.relu,\n",
    "                              name=\"hidden2\")\n",
    "    hidden2_drop = tf.layers.dropout(hidden2, dropout_rate, training=training)\n",
    "    logits = tf.layers.dense(hidden2_drop, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=0.9)\n",
    "    training_op = optimizer.minimize(loss)    \n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 검증 세트 정확도: 0.9254\n",
      "1 검증 세트 정확도: 0.9452\n",
      "2 검증 세트 정확도: 0.9492\n",
      "3 검증 세트 정확도: 0.9566\n",
      "4 검증 세트 정확도: 0.9606\n",
      "5 검증 세트 정확도: 0.9594\n",
      "6 검증 세트 정확도: 0.9606\n",
      "7 검증 세트 정확도: 0.9672\n",
      "8 검증 세트 정확도: 0.969\n",
      "9 검증 세트 정확도: 0.97\n",
      "10 검증 세트 정확도: 0.9676\n",
      "11 검증 세트 정확도: 0.968\n",
      "12 검증 세트 정확도: 0.971\n",
      "13 검증 세트 정확도: 0.9704\n",
      "14 검증 세트 정확도: 0.972\n",
      "15 검증 세트 정확도: 0.9702\n",
      "16 검증 세트 정확도: 0.9728\n",
      "17 검증 세트 정확도: 0.971\n",
      "18 검증 세트 정확도: 0.9736\n",
      "19 검증 세트 정확도: 0.9742\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={training: True, X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"검증 세트 정확도:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 맥스 노름"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2개의 은닉층을 가진 간단한 MNIST 신경망을 만들어 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 50\n",
    "n_outputs = 10\n",
    "\n",
    "learning_rate = 0.01\n",
    "momentum = 0.9\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, name=\"hidden2\")\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
    "    training_op = optimizer.minimize(loss)    \n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "다음으로 첫 번째 은닉층의 가중치에 대한 핸들을 얻고 `clip_by_norm()` 함수를 사용해 가중치를 클리핑하는 연산을 만듭니다. 그런 다음 클리핑된 가중치를 가중치 변수에 할당하는 연산을 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold = 1.0\n",
    "weights = tf.get_default_graph().get_tensor_by_name(\"hidden1/kernel:0\")\n",
    "clipped_weights = tf.clip_by_norm(weights, clip_norm=threshold, axes=1)\n",
    "clip_weights = tf.assign(weights, clipped_weights)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "두 번째 층에 대해서도 동일하게 할 수 있습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights2 = tf.get_default_graph().get_tensor_by_name(\"hidden2/kernel:0\")\n",
    "clipped_weights2 = tf.clip_by_norm(weights2, clip_norm=threshold, axes=1)\n",
    "clip_weights2 = tf.assign(weights2, clipped_weights2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "초기와 연산과 `Saver` 객체를 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 모델을 훈련시킵니다. 이전과 매우 동일한데 `training_op`을 실행한 후에 `clip_weights`와 `clip_weights2` 연산을 실행하는 것만 다릅니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 검증 세트 정확도: 0.9568\n",
      "1 검증 세트 정확도: 0.9696\n",
      "2 검증 세트 정확도: 0.972\n",
      "3 검증 세트 정확도: 0.977\n",
      "4 검증 세트 정확도: 0.977\n",
      "5 검증 세트 정확도: 0.977\n",
      "6 검증 세트 정확도: 0.981\n",
      "7 검증 세트 정확도: 0.9814\n",
      "8 검증 세트 정확도: 0.9812\n",
      "9 검증 세트 정확도: 0.9822\n",
      "10 검증 세트 정확도: 0.9814\n",
      "11 검증 세트 정확도: 0.985\n",
      "12 검증 세트 정확도: 0.9822\n",
      "13 검증 세트 정확도: 0.9842\n",
      "14 검증 세트 정확도: 0.984\n",
      "15 검증 세트 정확도: 0.9848\n",
      "16 검증 세트 정확도: 0.984\n",
      "17 검증 세트 정확도: 0.9838\n",
      "18 검증 세트 정확도: 0.9842\n",
      "19 검증 세트 정확도: 0.9844\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:                                              # 책에는 없음\n",
    "    init.run()                                                          # 책에는 없음\n",
    "    for epoch in range(n_epochs):                                       # 책에는 없음\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):  # 책에는 없음\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "            clip_weights.eval()\n",
    "            clip_weights2.eval()                                        # 책에는 없음\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid}) # 책에는 없음\n",
    "        print(epoch, \"검증 세트 정확도:\", accuracy_val)                     # 책에는 없음\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")               # 책에는 없음"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "위 구현은 이해하기 쉽고 잘 작동하지만 조금 번거롭습니다. 더 나은 방법은 `max_norm_regularizer()` 함수를 만드는 것입니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "def max_norm_regularizer(threshold, axes=1, name=\"max_norm\",\n",
    "                         collection=\"max_norm\"):\n",
    "    def max_norm(weights):\n",
    "        clipped = tf.clip_by_norm(weights, clip_norm=threshold, axes=axes)\n",
    "        clip_weights = tf.assign(weights, clipped, name=name)\n",
    "        tf.add_to_collection(collection, clip_weights)\n",
    "        return None # 규제 손실을 위한 항이 없습니다\n",
    "    return max_norm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "그런 다음 (필요한 임계값을 지정해서) 맥스 노름 규제 매개변수에 넘길 함수를 만들기 위해 이 함수를 호출합니다. 은닉층을 만들 때 이 규제 함수를 `kernel_regularizer` 매개변수를 통해 전달할 수 있습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 50\n",
    "n_outputs = 10\n",
    "\n",
    "learning_rate = 0.01\n",
    "momentum = 0.9\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_norm_reg = max_norm_regularizer(threshold=1.0)\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu,\n",
    "                              kernel_regularizer=max_norm_reg, name=\"hidden1\")\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu,\n",
    "                              kernel_regularizer=max_norm_reg, name=\"hidden2\")\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
    "    training_op = optimizer.minimize(loss)    \n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "훈련 연산이 실행된 후에 가중치 클리핑 연산을 실행하는 것을 제외하면 이전과 동일합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 검증 세트 정확도: 0.9558\n",
      "1 검증 세트 정확도: 0.9704\n",
      "2 검증 세트 정확도: 0.9728\n",
      "3 검증 세트 정확도: 0.9758\n",
      "4 검증 세트 정확도: 0.9766\n",
      "5 검증 세트 정확도: 0.9786\n",
      "6 검증 세트 정확도: 0.9818\n",
      "7 검증 세트 정확도: 0.9808\n",
      "8 검증 세트 정확도: 0.9812\n",
      "9 검증 세트 정확도: 0.981\n",
      "10 검증 세트 정확도: 0.9816\n",
      "11 검증 세트 정확도: 0.9824\n",
      "12 검증 세트 정확도: 0.9804\n",
      "13 검증 세트 정확도: 0.9822\n",
      "14 검증 세트 정확도: 0.9822\n",
      "15 검증 세트 정확도: 0.982\n",
      "16 검증 세트 정확도: 0.9816\n",
      "17 검증 세트 정확도: 0.9824\n",
      "18 검증 세트 정확도: 0.9822\n",
      "19 검증 세트 정확도: 0.9826\n"
     ]
    }
   ],
   "source": [
    "clip_all_weights = tf.get_collection(\"max_norm\")\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "            sess.run(clip_all_weights)\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid}) # 책에는 없음\n",
    "        print(epoch, \"검증 세트 정확도:\", accuracy_val)                      # 책에는 없음\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")                # 책에는 없음"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 연습문제 해답"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "11장의 연습문제는 [11_deep_learning_exercise](11_deep_learning_exercises.ipynb) 노트북에 있습니다."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "nav_menu": {
   "height": "360px",
   "width": "416px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}