LTD_model_reg_LDPC.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyN7iDRVeQAfMF9Pc0UT5d6E",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/pollyjuice74/5G-Decoder/blob/main/LTD_model_reg_LDPC.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "5q1VAmIeUKIn"
      },
      "outputs": [],
      "source": [
        "!git clone https://github.com/pollyjuice74/5G-Decoder\n",
        "!pip install sionna"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "import tensorflow as tf\n",
        "import random\n",
        "import numpy as np\n",
        "import time\n",
        "from scipy.sparse import issparse, csr_matrix\n",
        "\n",
        "from sionna.fec.utils import generate_reg_ldpc, load_parity_check_examples, LinearEncoder, gm2pcm\n",
        "from sionna.utils.plotting import PlotBER\n",
        "from sionna.fec.ldpc import LDPCBPDecoder\n",
        "\n",
        "import os\n",
        "# os.chdir('../..')\n",
        "if os.path.exists('5G-Decoder'):\n",
        "  os.rename('5G-Decoder', '5G_Decoder')\n",
        "os.chdir('5G_Decoder/adv_nn')\n",
        "\n",
        "from dataset import *\n",
        "from attention import *\n",
        "from channel import *\n",
        "from args import *\n",
        "from model_functs import *\n",
        "from models import *"
      ],
      "metadata": {
        "id": "U5U5qUUVUeRm"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Loading LDPC code\")\n",
        "pcm, k, n, coderate = generate_reg_ldpc(v=3,\n",
        "                                        c=6,\n",
        "                                        n=100,\n",
        "                                        allow_flex_len=True,\n",
        "                                        verbose=True)\n",
        "\n",
        "encoder = LinearEncoder(pcm, is_pcm=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 435
        },
        "id": "6RF7dBDwWg0L",
        "outputId": "c15176dc-d214-44ee-a18e-d1b9b99d105f"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading LDPC code\n",
            "Setting n to:  100\n",
            "Number of edges (VN perspective):  300\n",
            "Number of edges (CN perspective):  300\n",
            "Generated regular (3,6) LDPC code of length n=100\n",
            "Code rate is r=0.500.\n",
            "Warning: The alias fec.utils.LinearEncoder will not be included in Sionna 1.0. Please use fec.linear.LinearEncoder instead.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAEoCAYAAAD4ypNZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAdi0lEQVR4nO3df2zcdR3H8Ve7rbfBdlda2JW5FqoSC44hdGw7IBqhukyi4IqBZOpAAkG7uR9GoCqoUewiifz+oURGjIzGGQFHIgspUkIsYysZMpQCYcmq424Q07sx2G1pP/5huHDrTe563/t8P9/v9/lIvgn73ve+9/78uOubz/d936szxhgBAABYUu93AAAAIFpIPgAAgFUkHwAAwCqSDwAAYBXJBwAAsIrkAwAAWEXyAQAArCL5AAAAVpF8AAAAq0g+AACAVc4mH/fcc49OPfVUzZw5U0uWLNELL7zgd0ih19fXp3PPPVdz5szR3Llzdemll2pkZKTomEOHDqmnp0fNzc2aPXu2uru7lclkfIo4OjZu3Ki6ujqtW7eusI+xsOvf//63vv71r6u5uVmzZs3SmWeeqZ07dxYeN8bo5ptv1sknn6xZs2apq6tLr7/+uo8Rh9P4+Lhuuukmtbe3a9asWfrEJz6hn/3sZ/rwL4UwFgFgHNTf328aGhrMgw8+aF555RVzzTXXmMbGRpPJZPwOLdSWLVtmNm3aZHbv3m127dplvvSlL5m2tjbz7rvvFo657rrrTGtrqxkYGDA7d+40S5cuNeedd56PUYffCy+8YE499VSzcOFCs3bt2sJ+xsKe//znP+aUU04xV155pdm+fbt58803zbZt28wbb7xROGbjxo0mkUiYxx57zLz00kvmK1/5imlvbzfvv/++j5GHzy233GKam5vNE088Yfbs2WO2bNliZs+ebe64447CMYyF+5xMPhYvXmx6enoK/x4fHzfz5s0zfX19PkYVPfv37zeSzODgoDHGmLGxMTNjxgyzZcuWwjH//Oc/jSQzNDTkV5ihduDAAXPaaaeZp556ynzuc58rJB+MhV033HCDueCCC475+MTEhGlpaTG33nprYd/Y2JiJxWLmkUcesRFiZFx88cXmW9/6VtG+FStWmJUrVxpjGIugcO6yy+HDhzU8PKyurq7Cvvr6enV1dWloaMjHyKInm81KkpqamiRJw8PDOnLkSNHYdHR0qK2tjbGpkZ6eHl188cVFfS4xFrb9+c9/1qJFi/S1r31Nc+fO1dlnn60HHnig8PiePXuUTqeLxiORSGjJkiWMh8fOO+88DQwM6LXXXpMkvfTSS3ruuee0fPlySYxFUEz3O4CjvfPOOxofH1cymSzan0wm9eqrr/oUVfRMTExo3bp1Ov/887VgwQJJUjqdVkNDgxobG4uOTSaTSqfTPkQZbv39/XrxxRe1Y8eOSY8xFna9+eabuu+++7Rhwwb94Ac/0I4dO/Td735XDQ0NWrVqVaHPS31uMR7euvHGG5XL5dTR0aFp06ZpfHxct9xyi1auXClJjEVAOJd8wA09PT3avXu3nnvuOb9DiaTR0VGtXbtWTz31lGbOnOl3OJE3MTGhRYsW6Re/+IUk6eyzz9bu3bt1//33a9WqVT5HFy1/+MMf9PDDD2vz5s369Kc/rV27dmndunWaN28eYxEgzl12OfHEEzVt2rRJVfuZTEYtLS0+RRUtq1ev1hNPPKG//vWvmj9/fmF/S0uLDh8+rLGxsaLjGRvvDQ8Pa//+/TrnnHM0ffp0TZ8+XYODg7rzzjs1ffp0JZNJxsKik08+WWeccUbRvtNPP1179+6VpEKf87lVe9///vd144036oorrtCZZ56pb3zjG1q/fr36+vokMRZB4Vzy0dDQoM7OTg0MDBT2TUxMaGBgQKlUysfIws8Yo9WrV+vRRx/V008/rfb29qLHOzs7NWPGjKKxGRkZ0d69exkbj1100UV6+eWXtWvXrsK2aNEirVy5svDfjIU9559//qSvnb/22ms65ZRTJEnt7e1qaWkpGo9cLqft27czHh577733VF9f/Kdr2rRpmpiYkMRYBIbfFa+l9Pf3m1gsZh566CHzj3/8w1x77bWmsbHRpNNpv0MLtW9/+9smkUiYZ555xrz11luF7b333iscc91115m2tjbz9NNPm507d5pUKmVSqZSPUUfHh7/tYgxjYdMLL7xgpk+fbm655Rbz+uuvm4cfftgcd9xx5ve//33hmI0bN5rGxkbz+OOPm7///e/mkksu4eudNbBq1SrzsY99rPBV2z/96U/mxBNPNNdff33hGMbCfU4mH8YYc9ddd5m2tjbT0NBgFi9ebJ5//nm/Qwo9SSW3TZs2FY55//33zXe+8x1zwgknmOOOO8589atfNW+99ZZ/QUfI0ckHY2HX1q1bzYIFC0wsFjMdHR3mN7/5TdHjExMT5qabbjLJZNLEYjFz0UUXmZGREZ+iDa9cLmfWrl1r2trazMyZM83HP/5x88Mf/tDk8/nCMYyF++qM+dBt4QAAAGrMuZoPAAAQbiQfAADAKpIPAABgFckHAACwiuQDAABYRfIBAACscjr5yOfz+slPfqJ8Pu93KJHHWLiDsXAHY+EWxiM4nL7PRy6XUyKRUDabVTwe9zucSGMs3MFYuIOxcAvjERxOr3wAAIDwIfkAAABWTa/Vie+55x7deuutSqfTOuuss3TXXXdp8eLFH/m8iYkJ7du3T3PmzNGBAwck/W8pDf76YAwYC/8xFu5gLNzCePjLGKMDBw5o3rx5k355uNTBnuvv7zcNDQ3mwQcfNK+88oq55pprTGNjo8lkMh/53NHR0WP+wBkbGxsbGxub29vo6OhH/q2vScHpkiVLdO655+ruu++W9L/VjNbWVq1Zs0Y33njj/31uNptVY2OjRkdHKRj6kEQiMWlfNpt15nxRE7T+C1q88A9zxR1BG4tcLqfW1laNjY2VjP3DPL/scvjwYQ0PD6u3t7ewr76+Xl1dXRoaGpp0fD6fL/pa1AeXWuLxOMnHR/C6f+jv6gSt/4IWL/zDXHFHEMairq7uI4/xvOD0nXfe0fj4uJLJZNH+ZDKpdDo96fi+vj4lEonC1tra6nVIAADAIb5/26W3t1fZbLawjY6O+h0SAACoIc8vu5x44omaNm2aMplM0f5MJqOWlpZJx8diMcViMc9ev9RyTw3KWqwr1YZq2lpNn7jexzbic6m95QhavLa4Ppf9EPX2+yVqc9HzlY+GhgZ1dnZqYGCgsG9iYkIDAwNKpVJevxwAAAiYmtznY8OGDVq1apUWLVqkxYsX6/bbb9fBgwd11VVX1eLlAABAgNQk+bj88sv19ttv6+abb1Y6ndZnPvMZPfnkk5OKUAEAQPQ498Ny1f4wUJSum/nVVtf72PX44A7mClwRhrlYyd/vmt1e3S9BG6xq+NVW1/vY9fjCIAwflFIwYy6HjfEJyxwotx21bm+55wpLv/v+VVsAABAtJB8AAMAqkg8AAGAVyQcAALAqdAWnLglDYZDrbXA9vrCij91mY3xcmgM27vbsSnttFA7beF1WPgAAgFUkHwAAwCqSDwAAYBXJBwAAsIqC0xpypUCpGq63gbs2wnVHzym/7mQZ5vPxHp06v/qOlQ8AAGAVyQcAALCK5AMAAFhF8gEAAKyKRMGpS0WELsWCyfwYC9fnhOvxlcuvdkz1NbyOLWrnKyUsczkMWPkAAABWkXwAAACrSD4AAIBVJB8AAMCqSBSculRQ5FIsQRPWYjHX2+B6fKW4dPdNTJ3XP/fu191lMRkrHwAAwCqSDwAAYBXJBwAAsIrkAwAAWBW6gtMoFQq51FYbsYR1HOE95t5kLn1elMuv+LwsTna9j8vl9fxh5QMAAFhF8gEAAKwi+QAAAFaRfAAAAKtCV3AaluKecrjUVpdiiRLXiwhdj69cYWgHP1lvh8vtLXd8bIwjKx8AAMAqkg8AAGAVyQcAALCK5AMAAFgVuoLTalAs5Y6wjEWt2+F6n7geX7njwx0vyxPmtgVNNZ89NsaRlQ8AAGAVyQcAALCq4uTj2Wef1Ze//GXNmzdPdXV1euyxx4oeN8bo5ptv1sknn6xZs2apq6tLr7/+ulfxAgCAgKs4+Th48KDOOuss3XPPPSUf/+Uvf6k777xT999/v7Zv367jjz9ey5Yt06FDh6oOFgAABF/FBafLly/X8uXLSz5mjNHtt9+uH/3oR7rkkkskSb/73e+UTCb12GOP6YorrphyoF4X7oWloNEPLvWdS7GUUutYXG+/62z0FeMBP7g+7zyt+dizZ4/S6bS6uroK+xKJhJYsWaKhoaGSz8nn88rlckUbAAAIL0+Tj3Q6LUlKJpNF+5PJZOGxo/X19SmRSBS21tZWL0MCAACO8f3bLr29vcpms4VtdHTU75AAAEANeZp8tLS0SJIymUzR/kwmU3jsaLFYTPF4vGgDAADh5Wny0d7erpaWFg0MDBT25XI5bd++XalUqqpzG2MmbS6dr9bq6uombX6x0XflvkbQxtFrfrXfpfkIt9mYK6Vew+U56nJstlT8bZd3331Xb7zxRuHfe/bs0a5du9TU1KS2tjatW7dOP//5z3Xaaaepvb1dN910k+bNm6dLL73Uy7gBAEBAVZx87Ny5U5///OcL/96wYYMkadWqVXrooYd0/fXX6+DBg7r22ms1NjamCy64QE8++aRmzpzpXdQAACCw6oxj69S5XE6JRELZbJb6jw/hfg5wCfMR5bIxV8q9bOHKHA3r+6eSv9++f9sFAABES8WXXeAP17PiajJ51/8vwPX4psrGT26Hte9c4nofcxfZyYIWby2w8gEAAKwi+QAAAFaRfAAAAKuo+QgZv67/lnqNcmOp5rlec/36uZeCeC0+aN9qsCFKbUV4sPIBAACsIvkAAABWkXwAAACrSD4AAIBVFJxOgddFiUefr5pzuVR85no7wlpcGtZ2SeFpB9wW5vfQ0fxqKysfAADAKpIPAABgFckHAACwiuQDAABYFeiCU5fu5unl+aJU7OSnsPZpWNtVLd5X5QlLP9n41WaXVXOHaRtY+QAAAFaRfAAAAKtIPgAAgFUkHwAAwKpAF5yWWyhT6zuSHstUXyMMxU4Ij7AUIAYxZj+EpZ/C0o6p8rr95f7dKxcrHwAAwCqSDwAAYBXJBwAAsIrkAwAAWOVswWkikSj6d6niGb/u4OZXoSswVVG/26Nkp/A8LH0FHK2cuZ3L5Sb97T4WVj4AAIBVJB8AAMAqkg8AAGAVyQcAALDK2YLTbDareDz+f4+xUdxFoR784HUxo0tF0tXcKdFGH/h1PiBKWPkAAABWkXwAAACrSD4AAIBVJB8AAMAqZwtOXRH1ojLu4ugPG0WeNu4GXIrrheKoPb8Km6u5Uza8xcoHAACwiuQDAABYRfIBAACsqij56Ovr07nnnqs5c+Zo7ty5uvTSSzUyMlJ0zKFDh9TT06Pm5mbNnj1b3d3dymQyngYNAACCq6LkY3BwUD09PXr++ef11FNP6ciRI/riF7+ogwcPFo5Zv369tm7dqi1btmhwcFD79u3TihUrPA8cdhhjJm3VqKurm7Sh9rweR9dV095y52it5zLvleqUOwfC8N4I4lypM1X09Ntvv625c+dqcHBQn/3sZ5XNZnXSSSdp8+bNuuyyyyRJr776qk4//XQNDQ1p6dKlH3nOXC6nRCJR1u3VETxUlsN1rnxLIszvlTC3zQ+u9Gclf7+rqvnIZrOSpKamJknS8PCwjhw5oq6ursIxHR0damtr09DQUMlz5PN55XK5og0AAITXlJOPiYkJrVu3Tueff74WLFggSUqn02poaFBjY2PRsclkUul0uuR5+vr6lEgkCltra+tUQwIAAAEw5eSjp6dHu3fvVn9/f1UB9Pb2KpvNFrbR0dGqzgcAANw2pTucrl69Wk888YSeffZZzZ8/v7C/paVFhw8f1tjYWNHqRyaTUUtLS8lzxWIxxWKxqYQRGkdfr6vmWp0r1/6AY3GlpuJYyn2NWscSxPetS3fSjZIg9mdFKx/GGK1evVqPPvqonn76abW3txc93tnZqRkzZmhgYKCwb2RkRHv37lUqlfImYgAAEGgVrXz09PRo8+bNevzxxzVnzpxCHUcikdCsWbOUSCR09dVXa8OGDWpqalI8HteaNWuUSqXK+qYLAAAIv4q+anus7w5v2rRJV155paT/3WTse9/7nh555BHl83ktW7ZM99577zEvuxwtil+1jdJlF9fjQ+25ftkFU8eYRVslf7+rus9HLZB8kHwg3Eg+wosxi7ZK/n5PqeA0Smy8mbw8n18/S12uMCdWtRaW9rtS0FkLYRmjqYpSW10SxHnHD8sBAACrSD4AAIBVJB8AAMAqkg8AAGAVBacfwfWinVpzqf02ilWDWLiF2ov6nTt5X5TmSr8EcSxY+QAAAFaRfAAAAKtIPgAAgFUkHwAAwCoKTlHgSvFUtapph8vtdTk2PwXtLsRBxJ2JSwtDO6oZn2P93ls5WPkAAABWkXwAAACrSD4AAIBVJB8AAMCqyBachrkIaqrC0n6/2lHNXVTLeR5K87qvvC7Ai/pYRr39rqtmfI5+bi6XUyKRKOu5rHwAAACrSD4AAIBVJB8AAMAqkg8AAGBVZAtO/SpSc6UgzZU4/OR1H4ThLqrlCvP8sTEHUJ4wz7OoY+UDAABYRfIBAACsIvkAAABWkXwAAACrIltw6rWgFRvaiMP1YjFX7oTqUp+Uy5W+k4LZf1Pldftd6k+XYkHtsfIBAACsIvkAAABWkXwAAACrSD4AAIBVFJwGRBCLsfgZ8tLC3LZaC0PfVTPfXWq/X3cIjrqwfF6y8gEAAKwi+QAAAFaRfAAAAKtIPgAAgFUUnAZEEAuKUFoY7nCKqXNpvG0UiJYqkPT6NaIkLIX8rHwAAACrSD4AAIBVJB8AAMCqipKP++67TwsXLlQ8Hlc8HlcqldJf/vKXwuOHDh1ST0+PmpubNXv2bHV3dyuTyXgeNAAACK6Kko/58+dr48aNGh4e1s6dO3XhhRfqkksu0SuvvCJJWr9+vbZu3aotW7ZocHBQ+/bt04oVK2oSeC3U1dVN2lB7xphJW5iV01bmon994PXrlnOuMI93qfd3uRumzvX+rDNVRtTU1KRbb71Vl112mU466SRt3rxZl112mSTp1Vdf1emnn66hoSEtXbq05PPz+bzy+Xzh37lcTq2trcpms4rH49WEVjHXq4MRHcxF//rA69ct59tNjDfCIJfLKZFIlPX3e8o1H+Pj4+rv79fBgweVSqU0PDysI0eOqKurq3BMR0eH2traNDQ0dMzz9PX1KZFIFLbW1taphgQAAAKg4uTj5Zdf1uzZsxWLxXTdddfp0Ucf1RlnnKF0Oq2GhgY1NjYWHZ9MJpVOp495vt7eXmWz2cI2OjpacSMAAEBwVHyTsU996lPatWuXstms/vjHP2rVqlUaHByccgCxWEyxWGzKzwcAAMFScfLR0NCgT37yk5Kkzs5O7dixQ3fccYcuv/xyHT58WGNjY0WrH5lMRi0tLRUHlkgkiv5t4/qnX9dYyyku4/pveLl0vd+lWMLyuuWcj/c3SnHp/ei1qu/zMTExoXw+r87OTs2YMUMDAwOFx0ZGRrR3716lUqlqXwYAAIRERSsfvb29Wr58udra2nTgwAFt3rxZzzzzjLZt26ZEIqGrr75aGzZsUFNTk+LxuNasWaNUKnXMb7oAAIDoqSj52L9/v775zW/qrbfeUiKR0MKFC7Vt2zZ94QtfkCTddtttqq+vV3d3t/L5vJYtW6Z77723JoEDAIBgqvo+H1774HvCR3MsTE9R8xFtLl3XdSkWIOqC9n6s5D4fFRec2uLHTcb84spkCtpEr5Yr7XWpj728mVa15wNKsTHPmMu1xw/LAQAAq0g+AACAVSQfAADAKpIPAABglbMFp7AvagVVFFd6q9z2h6Xvyv3Z+yC27WhhGbNyudI2V+KoBVY+AACAVSQfAADAKpIPAABgFckHAACwKnQFp9UURkWtqCpK/Bpb5tRkYWl/WNpRDpfa6lIsXoraZwUrHwAAwCqSDwAAYBXJBwAAsIrkAwAAWBW6gtNqCnTCUNzjUtGSS7F4/brlni8Mc6pcLo03ysOY1b4Pyj1/1PqdlQ8AAGAVyQcAALCK5AMAAFhF8gEAAKwKTMEphVHlcalPvI6FOeA2xiJ4ojZmXt8Bu5SjzxfmPi63T0ph5QMAAFhF8gEAAKwi+QAAAFaRfAAAAKsCU3CKyaJWgFlNYVgY+sWvdoW1P/1Cf/on6nfA9trRfZLL5ZRIJMp6LisfAADAKpIPAABgFckHAACwKjA1H6Wut0X9GrhfNRCutP9YXIrFS36Nd1j70y9R+owCjoWVDwAAYBXJBwAAsIrkAwAAWEXyAQAArHK24PToG5WUKpbyq4AqaIVbXsfrUvsprJss6u0PIq9/bZVflHYb/cnKBwAAsIzkAwAAWEXyAQAArKoq+di4caPq6uq0bt26wr5Dhw6pp6dHzc3Nmj17trq7u5XJZKqNEwAAhMSUk48dO3bo17/+tRYuXFi0f/369dq6dau2bNmiwcFB7du3TytWrKj4/NlsVsaYwlaNurq6SVvUhaVPPjxHvJgrpYSlr+Cucudxqblo4z1g4zVcN9XPAb/GzHVTSj7effddrVy5Ug888IBOOOGEwv5sNqvf/va3+tWvfqULL7xQnZ2d2rRpk/72t7/p+eef9yxoAAAQXFNKPnp6enTxxRerq6uraP/w8LCOHDlStL+jo0NtbW0aGhoqea58Pq9cLle0AQCA8Kr4Ph/9/f168cUXtWPHjkmPpdNpNTQ0qLGxsWh/MplUOp0ueb6+vj799Kc/rTQMAAAQUBWtfIyOjmrt2rV6+OGHNXPmTE8C6O3tVTabLWyjo6OenBcAALipopWP4eFh7d+/X+ecc05h3/j4uJ599lndfffd2rZtmw4fPqyxsbGi1Y9MJqOWlpaS54zFYorFYlOLvkxRLOb5MO6mV10fhLWvgjgvghizl2y0Nep9fCxT7YNSz6OPK0w+LrroIr388stF+6666ip1dHTohhtuUGtrq2bMmKGBgQF1d3dLkkZGRrR3716lUinvogYAAIFVUfIxZ84cLViwoGjf8ccfr+bm5sL+q6++Whs2bFBTU5Pi8bjWrFmjVCqlpUuXehc1AAAILM9/WO62225TfX29uru7lc/ntWzZMt17771evwwAAAioOuPYhaZcLqdEIqFsNqt4PO53OKHA9UX6oJQg9kkQYw4a+rj2wtrHlfz99nzlIyjCOvilhLVdlQhroV6U5rEU7ra5gj6uPa/72MbngNevwQ/LAQAAq0g+AACAVSQfAADAKpIPAABgVWQLTl0uqopaEaHrXB6PauJwpQ2S232M8PCjMNPGPA7ia7DyAQAArCL5AAAAVpF8AAAAq0g+AACAVZEtOHVZEAvtwlwwWG47wtJeP0S974J4h0q/XqMaLsXiB5fGh5UPAABgFckHAACwiuQDAABYRfIBAACsouAUnii3aMmlgid4q9TYluLSeLsyH4N4h0q/XqMUV8bRz9cth0uf06x8AAAAq0g+AACAVSQfAADAKpIPAABgFQWnH8GlQqYwiFrf+TF//JqzQRxbP2Iud3y8Pi7MotbeWrPRn6x8AAAAq0g+AACAVSQfAADAKpIPAABgFQWnH4FCJlQjaHfL9Kt4MUpFk+W2i+LScLTX6zaEoU8kVj4AAIBlJB8AAMAqkg8AAGAVyQcAALCKglOPuHw3wrAUKKH2wvIz5GGd82FoQyXC0F6vf8Y+LMWqrHwAAACrSD4AAIBVJB8AAMAqkg8AAGAVBacecaW4tBSvC55Qe4xF+eiryegTO7zsZxvj49IcYOUDAABYRfIBAACsIvkAAABWOVfz8cE1qVwu53MktRG0dgUt3jBjLMpHX01Gn9gR5X7+oO3l1JbUGZcqUCT961//Umtrq99hAACAKRgdHdX8+fP/7zHOJR8TExPat2+f5syZowMHDqi1tVWjo6OKx+N+hxZpuVyOsXAEY+EOxsItjIe/jDE6cOCA5s2bp/r6/1/V4dxll/r6+kLG9MHXmOLxOBPJEYyFOxgLdzAWbmE8/JNIJMo6joJTAABgFckHAACwyunkIxaL6cc//rFisZjfoUQeY+EOxsIdjIVbGI/gcK7gFAAAhJvTKx8AACB8SD4AAIBVJB8AAMAqkg8AAGAVyQcAALCK5AMAAFhF8gEAAKwi+QAAAFb9F2N6dS6qIQDLAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# for e2e model\n",
        "from sionna.utils import BinarySource, ebnodb2no\n",
        "from sionna.mapping import Mapper, Demapper\n",
        "from sionna.channel import AWGN\n",
        "# from sionna.fec.ldpc import LDPC5GDecoder, LDPC5GEncoder\n",
        "\n",
        "\n",
        "class Args():\n",
        "    def __init__(self, model_type, code_type='LDPC', n_look_up=121, k_look_up=80, n=400, k=200,\n",
        "                       n_rings=2, ls_active=True, split_diff=True, sigma=0.1,\n",
        "                       t_layers=1, d_model=128, heads=8, lr=5e-4,\n",
        "                       batch_size=160, batch_size_eval = 150,\n",
        "                       eval_train_iter=10, save_weights_iter=100,\n",
        "                       ebno_db_eval=2.5,\n",
        "                       ebno_db_min=0., ebno_db_max=4., ebno_db_stepsize=0.25,\n",
        "                       traindata_len=500, testdata_len=250, epochs=1000):\n",
        "        assert model_type in ['gen', 'dis'], \"Type must be: 'gen', Generator or 'dis', Discriminator.\"\n",
        "        assert code_type in ['POLAR', 'BCH', 'CCSDS', 'LDPC', 'MACKAY', 'LDPC5G', 'POLAR5G'], \"Invalid linear code type.\"\n",
        "\n",
        "\n",
        "        # model data\n",
        "        self.model_type = model_type\n",
        "\n",
        "        self.split_diff = split_diff\n",
        "        self.n_rings = n_rings # ring connectivity of mask\n",
        "        self.sigma = sigma\n",
        "        self.t_layers = t_layers\n",
        "        self.ls_active = ls_active\n",
        "\n",
        "        self.d_model = d_model\n",
        "        self.heads = heads\n",
        "\n",
        "        # training data\n",
        "        self.lr = lr\n",
        "        self.batch_size = batch_size\n",
        "        self.traindata_len = traindata_len\n",
        "        self.testdata_len = testdata_len\n",
        "        self.epochs = epochs\n",
        "\n",
        "        self.ebno_db_min = ebno_db_min\n",
        "        self.ebno_db_max = ebno_db_max\n",
        "        self.ebno_db_stepsize = ebno_db_stepsize\n",
        "\n",
        "        self.ebno_db_eval = ebno_db_eval\n",
        "        self.eval_train_iter = eval_train_iter\n",
        "        self.save_weights_iter = save_weights_iter\n",
        "        self.batch_size_eval = batch_size_eval\n",
        "\n",
        "        # code data\n",
        "        self.code_type = code_type\n",
        "        self.code = self.get_code(n_look_up, k_look_up) # n,k look up values in Get_Generator_and_Parity\n",
        "\n",
        "        # if self.code_type not in ['LDPC5G', 'POLAR5G']:\n",
        "        #     self.n, self.m, self.k = self.code.n, self.code.m, self.code.k\n",
        "        # else:\n",
        "        #     self.n, self.m, self.k = n, n-k, k\n",
        "\n",
        "        # self.n_steps = self.m + 5  # Number of diffusion steps\n",
        "\n",
        "    def get_code(self, n_look_up, k_look_up):\n",
        "        code = type('Code', (), {})() # class Code, no base class, no attributes/methods, () instantiate object\n",
        "        # code.n_look_up, code.k_look_up = n_look_up, k_look_up\n",
        "        # code.code_type = self.code_type\n",
        "\n",
        "        # if self.code_type not in ['LDPC5G', 'POLAR5G']:\n",
        "        #     G, H = Get_Generator_and_Parity(code)\n",
        "        #     code.G, code.H = tf.convert_to_tensor(G), csr_matrix( tf.convert_to_tensor(H) )\n",
        "\n",
        "        #     code.m, code.n = code.H.shape\n",
        "        #     code.k = code.n - code.m\n",
        "\n",
        "        return code\n",
        "\n",
        "\n",
        "class MHAttention(Layer):\n",
        "    def __init__(self, dims, heads, mask_length, linear=False, dropout=0.01):\n",
        "        super().__init__()\n",
        "        assert (dims % heads) == 0, 'dimension must be divisible by the number of heads'\n",
        "        self.linear = linear\n",
        "        self.dims = dims\n",
        "        self.heads = heads\n",
        "        self.dim_head = dims // heads\n",
        "\n",
        "        if linear:\n",
        "            self.k_proj = self.get_k_proj(mask_length) # n+m\n",
        "            self.proj_k = None\n",
        "            self.proj_v = None\n",
        "\n",
        "        self.to_q, self.to_k, self.to_v = [ Dense(self.dims, use_bias=False) for _ in range(3) ]\n",
        "        self.to_out = Dense(dims)\n",
        "        self.dropout = Dropout(dropout) # to d-dimentional embeddings\n",
        "\n",
        "    def build(self, input_shape):\n",
        "        # Creates shape (n,k_proj) proj matrices for key and\n",
        "        n_value = input_shape[1]\n",
        "        if self.linear:\n",
        "            self.proj_k = self.add_weight(\"proj_k\", shape=[n_value, self.k_proj], initializer=GlorotUniform())\n",
        "            self.proj_v = self.add_weight(\"proj_v\", shape=[n_value, self.k_proj], initializer=GlorotUniform())\n",
        "\n",
        "    def call(self, x, mask=None):\n",
        "        out_att = self.lin_attention(x, mask) if self.linear else self.attention(x, mask)\n",
        "        return out_att\n",
        "\n",
        "    def get_k_proj(self, mask_length):\n",
        "        # gets dimention for linear tranformer vector projection\n",
        "        for k_proj in range(mask_length // 2, 0, -1): # starts at half the mask length TO 0\n",
        "            if mask_length % k_proj == 0:\n",
        "                return tf.cast(k_proj, tf.int32)\n",
        "\n",
        "    def lin_attention(self, x, mask): # O(n)\n",
        "        shape = tf.shape(x) # (b, n, d)\n",
        "        b = tf.cast(shape[0], tf.int32)\n",
        "        n = tf.cast(shape[1], tf.int32)\n",
        "\n",
        "        assert x.shape[-1] is not None, \"The last dimension of x is undefined.\"\n",
        "\n",
        "        query, key, val = self.to_q(x), self.to_k(x), self.to_v(x)\n",
        "\n",
        "        # Project key and val into k-dimentional space\n",
        "        key = tf.einsum('bnd,nk->bkd', key, self.proj_k)\n",
        "        val = tf.einsum('bnd,nk->bkd', val, self.proj_v)\n",
        "\n",
        "        # Reshape splitting for heads\n",
        "        query = tf.reshape(query, (b, n, self.heads, self.dim_head))\n",
        "        key = tf.reshape(key, (b, self.k_proj, self.heads, self.dim_head))\n",
        "        val = tf.reshape(val, (b, self.k_proj, self.heads, self.dim_head))\n",
        "        query, key, val = [ tf.transpose(x, [0, 2, 1, 3]) for x in [query, key, val] ]\n",
        "\n",
        "        # Low-rank mask (n,k_proj)\n",
        "        mask = tf.expand_dims(mask, axis=-1)\n",
        "        mask = tf.image.resize(mask, [n, self.k_proj], method='nearest')\n",
        "        mask = tf.reshape(mask, (1, 1, n, self.k_proj))\n",
        "\n",
        "        # Main attn logic: sftmx( q@k / d**0.5 ) @ v\n",
        "        scores = tf.einsum('bhnd,bhkd->bhnk', query, key) / (tf.sqrt( tf.cast(self.dim_head, dtype=tf.float32) ))\n",
        "        scores += (mask * -1e9) if mask is not None else 0.\n",
        "        attn = tf.nn.softmax(scores, axis=-1) # (b,h,n,k_proj)\n",
        "        attn = self.dropout(attn)\n",
        "        out = tf.einsum('bhnk,bhkd->bhnd', attn, val)\n",
        "\n",
        "        # Reshape and pass through out layer\n",
        "        out = tf.transpose(out, [0, 2, 1, 3])\n",
        "        out = tf.reshape(out, (b, n, -1))\n",
        "        return self.to_out(out)\n",
        "\n",
        "    def attention(self, x, mask): # O(n^2)\n",
        "        shape = tf.shape(x)\n",
        "        b = shape[0]\n",
        "        n = shape[1]\n",
        "        x = x[:, :, tf.newaxis] # (b,n,1)\n",
        "\n",
        "        query, key, val = self.to_q(x), self.to_k(x), self.to_v(x) # (b, n, d)\n",
        "        query, key, val = [ tf.reshape(x, (b, n, self.heads, self.dim_head)) for x in [query, key, val] ]\n",
        "        query, key, val = [ tf.cast( tf.transpose(x, [0, 2, 1, 3]), tf.float32 )\n",
        "                                                                            for x in [query, key, val] ]\n",
        "\n",
        "        scores = tf.einsum('bhqd,bhkd->bhqk', query, key) / (tf.sqrt( tf.cast(self.dim_head, tf.float32) ))\n",
        "        scores += (mask * -1e9) if mask is not None else 0. # apply mask non-edge connections\n",
        "        attn = tf.nn.softmax(scores, axis=-1) #-1\n",
        "        attn = self.dropout(attn)\n",
        "        out = tf.einsum('bhqk,bhkd->bhqd', attn, val)\n",
        "\n",
        "        out = tf.transpose(out, [0, 2, 1, 3])\n",
        "        out = tf.reshape(out, (b, n, -1))\n",
        "        return self.to_out(out)\n",
        "\n",
        "\n",
        "\n",
        "class TransformerLayer(Layer):\n",
        "    def __init__(self, attn, ff, norm):\n",
        "        super().__init__()\n",
        "        self.attn, self.ff = attn, ff\n",
        "        self.norm1, self.norm2 = c(norm), c(norm)\n",
        "\n",
        "    def call(self, x, mask):\n",
        "        out = self.norm1( self.attn(x, mask) )\n",
        "        return self.norm2( self.ff(out) )\n",
        "\n",
        "\n",
        "class Transformer(Layer):\n",
        "    def __init__(self, d_model, heads, mask, N):\n",
        "        super().__init__()\n",
        "        self.transformer_layers = [ TransformerLayer( MHAttention(d_model, heads, mask_length=mask.shape[0]),\n",
        "                                                      FeedForward(d_model),\n",
        "                                                      PreNorm() ) for _ in range(N) ]\n",
        "        self.mask = mask\n",
        "\n",
        "    def call(self, x):\n",
        "        for transformer in self.transformer_layers:\n",
        "            x = transformer(x, self.mask)\n",
        "        return x\n",
        "\n",
        "\n",
        "class TransformerDiffusion( Layer ):\n",
        "    def __init__(self, args):\n",
        "        super().__init__()\n",
        "        self.model_type = args.model_type\n",
        "        self.n_steps = args.n_steps\n",
        "\n",
        "        code = args.code\n",
        "        # assert isinstance(code.H, tf.sparse.SparseTensor), \"Code's pcm must be sparse.\"\n",
        "        self.pcm = tf.cast(code.H, dtype=tf.int32)\n",
        "        # shapes\n",
        "        self.m, self.n = self.pcm.shape\n",
        "        self.k = self.n - self.m\n",
        "        self.dims = args.d_model\n",
        "        self.batch_size = args.batch_size\n",
        "\n",
        "        self.mask = self.create_mask(self.pcm)\n",
        "        # trans_call layers\n",
        "        self.src_embed = tf.Variable( tf.random.uniform([1, self.n + self.m, self.dims]), trainable=True ) # (b,n+m,d)\n",
        "        self.decoder = Transformer(args.d_model, args.heads, self.mask, args.t_layers) # (b,n)\n",
        "        self.time_embed = Embedding(self.m//2, args.d_model)\n",
        "        self.to_out = Dense(1)\n",
        "        self.to_n = Dense(self.n)\n",
        "        # diff layers\n",
        "        self.fc = Dense(1) ###\n",
        "\n",
        "        self.betas = tf.constant( tf.linspace(1e-3, 1e-2, args.n_steps)*0 + args.sigma )\n",
        "        self.betas_bar = tf.constant( tf.math.cumsum(self.betas, 0) )\n",
        "\n",
        "        self.split_diff = args.split_diff\n",
        "        self.ls_active = args.ls_active\n",
        "\n",
        "    def create_mask(self, H):\n",
        "        m,n = H.shape\n",
        "        mask = tf.eye(n+m, dtype=tf.float32) # (n+m, n+m)\n",
        "        indices = tf.where(H != 0)#H.indices\n",
        "        cn_con, vn_con = indices[:, 0], indices[:, 1]\n",
        "\n",
        "        for cn, vn_i in zip(cn_con, vn_con):\n",
        "            # cn to vn connections in the mask\n",
        "            mask = tf.tensor_scatter_nd_update(mask, [[n+cn, vn_i],[vn_i, n+cn]], [1.0,1.0])\n",
        "\n",
        "            # distance 2 vn neighbors of vn_i\n",
        "            related_vns = vn_con[cn_con==cn]\n",
        "            for vn_j in related_vns:\n",
        "                mask = tf.tensor_scatter_nd_update(mask, [[vn_i, vn_j],[vn_j, vn_i]], [1.0,1.0])\n",
        "\n",
        "        # -infinity where mask is not set\n",
        "        mask = tf.cast( tf.math.logical_not(mask > 0), dtype=tf.float32) # not(mask > 0) for setting non connections to -1e9\n",
        "        return mask\n",
        "\n",
        "    def get_sigma(self, t):\n",
        "        # make sure t is a positive int\n",
        "        t = tf.cast( tf.abs(t), tf.int32 )\n",
        "        # gather betas\n",
        "        betas_t = tf.gather(self.betas, t)\n",
        "        betas_bar_t = tf.gather(self.betas_bar, t)\n",
        "        sigma = betas_bar_t * betas_t / (betas_bar_t + betas_t)\n",
        "        return tf.cast(sigma, tf.float32)\n",
        "\n",
        "    def get_syndrome(self, r_t):\n",
        "        # Calculate syndrome (pcm @ r = 0) if r is correct in binary\n",
        "        r_t = tf.reshape(r_t, (self.n, -1)) # (n,b)\n",
        "        r_t_bin = tf.cast(llr_to_bin(r_t), dtype=tf.int32)\n",
        "        return (self.pcm @ r_t_bin) % 2 # (m,n)@(n,b)->(m,b)\n",
        "\n",
        "    def get_t_error(self, r_t):\n",
        "        t = tf.reduce_sum( self.get_syndrome(llr_to_bin(r_t)), axis=0 ) # (m,n)@(n,b)->(m,b)->(1,b)\n",
        "        t = tf.cast(tf.abs(t), dtype=tf.int32)\n",
        "        return t\n",
        "\n",
        "    # Extracts noise estimate z_hat from r\n",
        "    def tran_call(self, r_t):\n",
        "        # Compute synd and magn\n",
        "        syndrome = tf.reshape( self.get_syndrome(r_t), (self.batch_size, self.m) ) # (m,n)@(n,b)->(m,b) check nodes\n",
        "        magnitude = tf.reshape( tf.abs(r_t), (self.batch_size, self.n) ) #(n,b) variable nodes\n",
        "        # make sure their the same dtype\n",
        "        magnitude, syndrome = [ tf.cast(tensor, dtype=tf.float32) for tensor in [magnitude, syndrome] ]\n",
        "\n",
        "        # Concatenate synd and magn\n",
        "        nodes = tf.concat([magnitude, syndrome], axis=1)[:, :, tf.newaxis] # (b, n+m, 1)\n",
        "\n",
        "        # Embedding nodes w/ attn and 'time' (sum syn errs) dims\n",
        "        t = self.get_t_error(r_t)\n",
        "        nodes_emb = self.src_embed * nodes # (b, n+m, d)\n",
        "        time_emb = tf.reshape( self.time_embed(t), (self.batch_size, 1, self.dims) ) # (b,1,d)\n",
        "\n",
        "        # Applying embeds\n",
        "        emb_t = time_emb * nodes_emb # (b, n+m, d)\n",
        "        logits = self.decoder(emb_t) # (b, n+m, d)\n",
        "\n",
        "        # Converting to output shape\n",
        "        emb = tf.squeeze( self.to_out(logits), axis=-1 ) # (b,n+m,d)->(b, n+m)\n",
        "        z_hat = self.to_n(emb) # (b, n+m)->(b, n)\n",
        "        print(\"z_hat: \", z_hat.shape, z_hat.dtype, t.dtype)\n",
        "        return z_hat, t\n",
        "\n",
        "    # # optimal lambda l for theoretical and for error prediction\n",
        "    # def line_search(self, r_t, sigma, err_hat, lin_splits=20):\n",
        "    #     l_values =  tf.reshape( tf.linspace(1., 20., lin_splits), (1, 1, lin_splits) )\n",
        "    #     r_t, sigma, err_hat = [ tf.expand_dims(tensor, axis=-1) for tensor in [r_t, sigma, err_hat] ]# (n,b, 1)\n",
        "    #     # print(f\"sigma: {sigma}, err_hat: {err_hat}\")\n",
        "\n",
        "    #     # Compute theoretical step size w/ ls splits\n",
        "    #     z_hat_values = l_values*(sigma*err_hat) # (n,b, l), l is lin_splits\n",
        "    #     r_values = llr_to_bin(r_t - z_hat_values) # (n,b, l)\n",
        "    #     r_values = tf.reshape(r_values, [r_values.shape[0], -1]) # (n,b*l)\n",
        "    #     tf.print(\"r_values\", r_values.shape)\n",
        "\n",
        "    #     # sum of synds (m,n)@(n,b*l)->(m,b*l)->(b*l, 1)\n",
        "    #     sum_synds = tf.reduce_sum( tf.abs( (self.pcm @ r_values) % 2 ),\n",
        "    #                                axis=0 )\n",
        "    #     sum_synds = tf.reshape(sum_synds, (-1, lin_splits)) # (b, l)\n",
        "    #     tf.print(\"In linesearch Sum Syndromes: \", sum_synds)\n",
        "\n",
        "    #     # Pick optimal ls value\n",
        "    #     if self.model_type=='dis':\n",
        "    #          ixs = tf.math.argmin(sum_synds, axis=1, output_type=tf.int32)[:, tf.newaxis] # (b,1) w/ ixs of optimal line search for batch b\n",
        "    #     elif self.model_type=='gen':\n",
        "    #          ixs = tf.math.argmax(sum_synds, axis=1, output_type=tf.int32)[:, tf.newaxis] # (b,1)\n",
        "    #     # print(r_values.shape, z_hat_values.shape, ixs.shape)\n",
        "\n",
        "    #     r_values = r_t - z_hat_values\n",
        "    #     # (b, l, n) for indexing on l\n",
        "    #     r_values, z_hat_values = [ tf.reshape(tensor, [-1, lin_splits, r_values.shape[0]])\n",
        "    #                                         for tensor in [r_values, z_hat_values] ]\n",
        "\n",
        "    #     # concat range of batch ixs [0,...,n-1] and optimal line search ixs for gather_nd\n",
        "    #     indices = tf.concat([ tf.range(ixs.shape[0])[:, tf.newaxis], ixs ],\n",
        "    #                                                         axis=-1) # (b,2)\n",
        "\n",
        "    #     # print(r_values, z_hat_values, indices)\n",
        "    #     # ix on lin_splits w/ gather_nd st. ix,(b, l, n)->(n,b)\n",
        "    #     r_t1, z_hat = [ tf.reshape( tf.gather_nd(tensor, indices), (self.n, -1) )\n",
        "    #                                          for tensor in [r_values, z_hat_values] ]\n",
        "    #     # print(r_t1, z_hat_values)\n",
        "    #     return r_t1, z_hat # r at t-1\n",
        "\n",
        "    # def loss_fn(self, synd):\n",
        "    #     return tf.reduce_mean(tf.square(synd))\n",
        "\n",
        "    # def train_step(self, llr_ch):\n",
        "    #     with tf.GradientTape() as tape:\n",
        "    #         _, synd = self.tran_call(llr_ch,\n",
        "    #                                  tf.reduce_sum( self.get_syndrome(llr_ch) ))\n",
        "    #         loss = self.loss_fn(synd)\n",
        "    #     gradients = tape.gradient(loss, self.trainable_variables)\n",
        "    #     self.optimizer.apply_gradients(zip(gradients, self.trainable_variables))\n",
        "    #     return loss\n",
        "\n",
        "\n",
        "class Decoder( TransformerDiffusion ):\n",
        "    def __init__(self, args):\n",
        "        super().__init__(args)\n",
        "\n",
        "    # 'test' function\n",
        "    def call(self, r_t):\n",
        "        i = tf.constant(0)  # Initialize loop counter\n",
        "\n",
        "        def condition(r_t, i):\n",
        "            # Loop while i < self.m and syndrome sum is not zero\n",
        "            return tf.logical_and(i < 1, tf.reduce_sum(self.get_syndrome(r_t)) != 0) # CHANGE 5 TO SELF.M\n",
        "\n",
        "        def body(r_t, i):\n",
        "            # Perform reverse or split diffusion\n",
        "            r_t = tf.cond(\n",
        "                tf.logical_not(self.split_diff),\n",
        "                lambda: self.split_rdiff_call(r_t),\n",
        "                lambda: self.rev_diff_call(r_t),\n",
        "            )\n",
        "            return r_t, tf.add(i, 1)\n",
        "\n",
        "        # Run tf.while_loop with the loop variables\n",
        "        llr_hat, _ = tf.while_loop(\n",
        "            condition,\n",
        "            body,\n",
        "            loop_vars=[r_t, i],\n",
        "            maximum_iterations=self.m,\n",
        "            shape_invariants=[tf.TensorShape([self.batch_size, self.n]), i.get_shape()]\n",
        "        )\n",
        "\n",
        "        return llr_hat\n",
        "\n",
        "\n",
        "    # Refines recieved codeword r at time t\n",
        "    def rev_diff_call(self, r_t):\n",
        "        tf.print(\"Rev def call with line-search...\")\n",
        "\n",
        "        # Transformer error prediction\n",
        "        z_hat_crude, t = self.tran_call(r_t) # (b,n)\n",
        "        r_t1 = r_t - z_hat_crude*self.get_sigma(t)[:, tf.newaxis] # (b,n)\n",
        "        # tf.print(r_t1)\n",
        "\n",
        "        # # Refined estimate of the codeword for the ls diffusion step\n",
        "        # r_t1, z_hat = self.line_search(r_t, sigma, err_hat) if self.ls_active else 1.\n",
        "        # tf.print(\"After linesearch: \", r_t1)\n",
        "\n",
        "        print(\"r_t1\", r_t1.shape, r_t1.dtype)\n",
        "        return r_t1 # r at t-1, both (b,n)\n",
        "\n",
        "    def split_rdiff_call(self, r_t):\n",
        "        tf.print(\"Rev diff call with split diffusion...\")\n",
        "        # First half-step condition subproblem\n",
        "        z_hat_crude, t = self.tran_call(r_t)\n",
        "        # tf.print(\"fc input: \", (z_hat_crude * self.get_sigma(t)[:, tf.newaxis]))\n",
        "        r_t_half = r_t - 0.5 * self.fc( z_hat_crude * self.get_sigma(t)[:, tf.newaxis] )\n",
        "        # tf.print(\"r_t_half\", r_t_half)\n",
        "\n",
        "        # Full-step diffusion subproblem\n",
        "        r_t1 = r_t_half + tf.random.normal(r_t_half.shape) * tf.sqrt(self.get_sigma(t)[:, tf.newaxis])\n",
        "\n",
        "        # Second half-step condition subproblem\n",
        "        z_hat_crude_half, t = self.tran_call(r_t1)  # Reuse the second `tran_call`\n",
        "        r_t1 = r_t1 - 0.5 * self.fc(z_hat_crude_half * self.get_sigma(t)[:, tf.newaxis])\n",
        "        print(\"r_t1\", r_t1.shape, r_t1.dtype)\n",
        "        return r_t1  # r at t-1, both (b,n)\n",
        "\n",
        "\n",
        "class E2EModel(tf.keras.Model):\n",
        "    def __init__(self, encoder, decoder, k, n, return_infobits=False, es_no=False):\n",
        "        super().__init__()\n",
        "\n",
        "        self._n = n\n",
        "        self._k = k\n",
        "\n",
        "        self._binary_source = BinarySource()\n",
        "        self._num_bits_per_symbol = 2\n",
        "        self._mapper = Mapper(\"qam\", self._num_bits_per_symbol)\n",
        "        self._demapper = Demapper(\"app\", \"qam\", self._num_bits_per_symbol)\n",
        "        self._channel = AWGN()\n",
        "        self._decoder = decoder\n",
        "        self._encoder = encoder\n",
        "        self._return_infobits = return_infobits\n",
        "        self._es_no = es_no\n",
        "\n",
        "    @tf.function(jit_compile=False)\n",
        "    def call(self, batch_size, ebno_db):\n",
        "\n",
        "        # no rate-adjustment for uncoded transmission or es_no scenario\n",
        "        if self._decoder is not None and self._es_no==False:\n",
        "            no = ebnodb2no(ebno_db, self._num_bits_per_symbol, self._k/self._n)\n",
        "        else: #for uncoded transmissions the rate is 1\n",
        "            no = ebnodb2no(ebno_db, self._num_bits_per_symbol, 1)\n",
        "\n",
        "        b = self._binary_source([batch_size, self._k])\n",
        "        if self._encoder is not None:\n",
        "            c = self._encoder(b)\n",
        "        else:\n",
        "            c = b\n",
        "\n",
        "        # check that rate calculations are correct\n",
        "        assert self._n==c.shape[-1], \"Invalid value of n.\"\n",
        "\n",
        "        # zero padding to support odd codeword lengths\n",
        "        if self._n%2==1:\n",
        "            c_pad = tf.concat([c, tf.zeros([batch_size, 1])], axis=1)\n",
        "        else: # no padding\n",
        "            c_pad = c\n",
        "        x = self._mapper(c_pad)\n",
        "\n",
        "        y = self._channel([x, no])\n",
        "        llr = self._demapper([y, no])\n",
        "\n",
        "        # remove zero padded bit at the end\n",
        "        if self._n%2==1:\n",
        "            llr = llr[:,:-1]\n",
        "\n",
        "        # and run the decoder\n",
        "        if self._decoder is not None:\n",
        "            # tf.print('llr: ', llr)\n",
        "            ############################\n",
        "            llr_hat = self._decoder(llr)\n",
        "            ############################\n",
        "            # tf.print(\"llr_hat: \", llr_hat)\n",
        "\n",
        "        if self._return_infobits:\n",
        "            return b, llr_hat\n",
        "        else:\n",
        "            return c, llr_hat\n",
        "\n",
        "\n",
        "# args for decoder/discriminator\n",
        "args = Args(model_type='dis')\n",
        "args.code.H = pcm\n",
        "args.n, args.m = pcm.shape\n",
        "args.k = k\n",
        "args.n_steps = args.m + 5\n",
        "\n",
        "ltd_decoder = Decoder(args) # Linear Transformer Diffusion (LTD) Decoder\n",
        "\n",
        "e2e_ltd = E2EModel(encoder, ltd_decoder, k, n)"
      ],
      "metadata": {
        "id": "XOILyjSGXMdb"
      },
      "execution_count": 82,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def bin_to_llr(x):\n",
        "    \"\"\" Clip llrs to 20 for numerical stability \"\"\"\n",
        "    llr_vector = tf.where(x == 0, -20, 20)\n",
        "    return llr_vector\n",
        "\n",
        "\n",
        "def train_dec(model, args):\n",
        "    # loss\n",
        "    loss_fn = tf.keras.losses.MeanSquaredError()\n",
        "    # optimizer\n",
        "    scheduler = tf.keras.optimizers.schedules.CosineDecay(initial_learning_rate=args.lr, decay_steps=args.epochs) # 1000 is size of trainloader\n",
        "    optimizer = tf.keras.optimizers.Adam(learning_rate=scheduler)\n",
        "    # time start\n",
        "    time_start = time.time()\n",
        "\n",
        "    # SGD update iteration\n",
        "    @tf.function(jit_compile=False)\n",
        "    def train_step(batch_size):\n",
        "        # train for random SNRs within a pre-defined interval\n",
        "        ebno_db = tf.random.uniform([batch_size, 1],\n",
        "                                    minval=args.ebno_db_min,\n",
        "                                    maxval=args.ebno_db_max)\n",
        "\n",
        "        with tf.GradientTape() as tape:\n",
        "            c, llr_hat = model(batch_size, ebno_db)\n",
        "            # tf.print(c, llr_hat)\n",
        "\n",
        "            llr_y = bin_to_llr(c)\n",
        "            print(\"llr_hat\", llr_hat.shape, llr_hat.dtype)\n",
        "            loss_value = loss_fn(llr_y, llr_hat)\n",
        "\n",
        "        # and apply the SGD updates\n",
        "        weights = model.trainable_weights\n",
        "        grads = tape.gradient(loss_value, weights) # variables\n",
        "        optimizer.apply_gradients(zip(grads, weights))\n",
        "        return c, llr_hat\n",
        "\n",
        "    print(\"Training Linear Transformer Diffusion Model...\")\n",
        "    for epoch in range(1, args.epochs + 1):\n",
        "        train_step(args.batch_size)\n",
        "\n",
        "        # eval train iter\n",
        "        if True:#epoch % args.eval_train_iter == 0:\n",
        "            ebno_db = tf.random.uniform([args.batch_size, 1],\n",
        "                                          minval=args.ebno_db_eval,\n",
        "                                          maxval=args.ebno_db_eval)\n",
        "\n",
        "            c, llr_hat = model(args.batch_size, ebno_db)\n",
        "\n",
        "            # loss\n",
        "            llr_y = bin_to_llr(c)\n",
        "            loss_value = loss_fn(llr_y, llr_hat)\n",
        "\n",
        "            # ber\n",
        "            c_hat = llr_to_bin(llr_hat)\n",
        "            ber = compute_ber(c, c_hat).numpy()\n",
        "\n",
        "            # measure required time since last evaluation\n",
        "            duration = time.time() - time_start # in s\n",
        "            time_start = time.time() # reset counter\n",
        "\n",
        "            print(f'Training epoch {epoch}/{args.epochs}, LR={optimizer.learning_rate.numpy():.2e}, Loss={loss_value.numpy():.5e}, BER={ber}, duration: {duration:.2f}s')\n",
        "\n",
        "        # save weights iter\n",
        "        if epoch % args.save_weights_iter == 0:\n",
        "            pass\n",
        "\n",
        "        # heat-map visualization of the model's weights\n",
        "        # for var in self.trainable_variables:\n",
        "        #     var_name = var.name\n",
        "        #     var_value = var.numpy()\n",
        "\n",
        "        #     # Check if the variable is at least 2D (suitable for heatmap)\n",
        "        #     if len(var_value.shape) > 1:\n",
        "        #         plt.figure(figsize=(8, 6))\n",
        "        #         sns.heatmap(var_value, cmap='viridis')\n",
        "        #         plt.title(f'Heatmap of {var_name}')\n",
        "        #         plt.show()\n",
        "        #     else:\n",
        "        #         print(f\"{var_name} has shape {var_value.shape} which is not suitable for a heatmap.\")\n",
        "\n",
        "\n",
        "train_dec(e2e_ltd, args)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "NhZncoEEpgNv",
        "outputId": "25bec0a2-d301-4529-fdb6-790d22c67a1b"
      },
      "execution_count": 83,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Linear Transformer Diffusion Model...\n",
            "z_hat:  (160, 100) <dtype: 'float32'> <dtype: 'int32'>\n",
            "z_hat:  (160, 100) <dtype: 'float32'> <dtype: 'int32'>\n",
            "r_t1 (160, 100) <dtype: 'float32'>\n",
            "z_hat:  (160, 100) <dtype: 'float32'> <dtype: 'int32'>\n",
            "r_t1 (160, 100) <dtype: 'float32'>\n",
            "z_hat:  (160, 100) <dtype: 'float32'> <dtype: 'int32'>\n",
            "z_hat:  (160, 100) <dtype: 'float32'> <dtype: 'int32'>\n",
            "r_t1 (160, 100) <dtype: 'float32'>\n",
            "z_hat:  (160, 100) <dtype: 'float32'> <dtype: 'int32'>\n",
            "r_t1 (160, 100) <dtype: 'float32'>\n",
            "llr_hat (160, 100) <dtype: 'float32'>\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:You are casting an input of type complex64 to an incompatible dtype float32.  This will discard the imaginary part and may not be what you intended.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "llr_hat (160, 100) <dtype: 'float32'>\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 1/1000, LR=5.00e-04, Loss=2.77626e+02, BER=0.0915625, duration: 11.39s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 2/1000, LR=5.00e-04, Loss=2.77752e+02, BER=0.092875, duration: 0.10s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 3/1000, LR=5.00e-04, Loss=2.77665e+02, BER=0.0949375, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 4/1000, LR=5.00e-04, Loss=2.76734e+02, BER=0.0926875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 5/1000, LR=5.00e-04, Loss=2.76869e+02, BER=0.091875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 6/1000, LR=5.00e-04, Loss=2.77305e+02, BER=0.093375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 7/1000, LR=5.00e-04, Loss=2.77002e+02, BER=0.0896875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 8/1000, LR=5.00e-04, Loss=2.76627e+02, BER=0.090875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 9/1000, LR=5.00e-04, Loss=2.76838e+02, BER=0.093625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 10/1000, LR=5.00e-04, Loss=2.77510e+02, BER=0.094125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 11/1000, LR=5.00e-04, Loss=2.76885e+02, BER=0.0886875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 12/1000, LR=5.00e-04, Loss=2.77203e+02, BER=0.090625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 13/1000, LR=5.00e-04, Loss=2.76715e+02, BER=0.094, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 14/1000, LR=5.00e-04, Loss=2.76408e+02, BER=0.0878125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 15/1000, LR=5.00e-04, Loss=2.77092e+02, BER=0.0935, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 16/1000, LR=5.00e-04, Loss=2.76497e+02, BER=0.0878125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 17/1000, LR=5.00e-04, Loss=2.77712e+02, BER=0.094875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 18/1000, LR=5.00e-04, Loss=2.78090e+02, BER=0.0896875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 19/1000, LR=5.00e-04, Loss=2.77920e+02, BER=0.093875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 20/1000, LR=5.00e-04, Loss=2.78215e+02, BER=0.0935625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 21/1000, LR=5.00e-04, Loss=2.78831e+02, BER=0.0940625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 22/1000, LR=4.99e-04, Loss=2.76650e+02, BER=0.0874375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 23/1000, LR=4.99e-04, Loss=2.78367e+02, BER=0.093625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 24/1000, LR=4.99e-04, Loss=2.77463e+02, BER=0.0945, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 25/1000, LR=4.99e-04, Loss=2.76614e+02, BER=0.0894375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 26/1000, LR=4.99e-04, Loss=2.77159e+02, BER=0.0899375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 27/1000, LR=4.99e-04, Loss=2.78758e+02, BER=0.09525, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 28/1000, LR=4.99e-04, Loss=2.77248e+02, BER=0.0886875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 29/1000, LR=4.99e-04, Loss=2.78409e+02, BER=0.0926875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 30/1000, LR=4.99e-04, Loss=2.76842e+02, BER=0.0905625, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 31/1000, LR=4.99e-04, Loss=2.76679e+02, BER=0.0876875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 32/1000, LR=4.99e-04, Loss=2.76112e+02, BER=0.089875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 33/1000, LR=4.99e-04, Loss=2.78349e+02, BER=0.0948125, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 34/1000, LR=4.99e-04, Loss=2.77639e+02, BER=0.0893125, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 35/1000, LR=4.99e-04, Loss=2.77594e+02, BER=0.089, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 36/1000, LR=4.98e-04, Loss=2.76785e+02, BER=0.0926875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 37/1000, LR=4.98e-04, Loss=2.77997e+02, BER=0.0938125, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 38/1000, LR=4.98e-04, Loss=2.78205e+02, BER=0.0945625, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 39/1000, LR=4.98e-04, Loss=2.77843e+02, BER=0.0935625, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 40/1000, LR=4.98e-04, Loss=2.77300e+02, BER=0.091375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 41/1000, LR=4.98e-04, Loss=2.77254e+02, BER=0.0909375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 42/1000, LR=4.98e-04, Loss=2.77009e+02, BER=0.090625, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 43/1000, LR=4.98e-04, Loss=2.77559e+02, BER=0.0905625, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 44/1000, LR=4.98e-04, Loss=2.77226e+02, BER=0.0908125, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 45/1000, LR=4.98e-04, Loss=2.77430e+02, BER=0.0915625, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 46/1000, LR=4.98e-04, Loss=2.77294e+02, BER=0.0905, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 47/1000, LR=4.97e-04, Loss=2.78378e+02, BER=0.0936875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 48/1000, LR=4.97e-04, Loss=2.77781e+02, BER=0.093125, duration: 0.10s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 49/1000, LR=4.97e-04, Loss=2.77288e+02, BER=0.0906875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 50/1000, LR=4.97e-04, Loss=2.77269e+02, BER=0.0891875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 51/1000, LR=4.97e-04, Loss=2.77777e+02, BER=0.088625, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 52/1000, LR=4.97e-04, Loss=2.78768e+02, BER=0.0934375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 53/1000, LR=4.97e-04, Loss=2.76655e+02, BER=0.0899375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 54/1000, LR=4.97e-04, Loss=2.77961e+02, BER=0.09425, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 55/1000, LR=4.96e-04, Loss=2.76934e+02, BER=0.089875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 56/1000, LR=4.96e-04, Loss=2.77634e+02, BER=0.092875, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 57/1000, LR=4.96e-04, Loss=2.77000e+02, BER=0.0898125, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 58/1000, LR=4.96e-04, Loss=2.78051e+02, BER=0.0949375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 59/1000, LR=4.96e-04, Loss=2.77039e+02, BER=0.09125, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 60/1000, LR=4.96e-04, Loss=2.77316e+02, BER=0.0936875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 61/1000, LR=4.96e-04, Loss=2.76390e+02, BER=0.087875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 62/1000, LR=4.95e-04, Loss=2.77870e+02, BER=0.090375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 63/1000, LR=4.95e-04, Loss=2.77643e+02, BER=0.0935625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 64/1000, LR=4.95e-04, Loss=2.78469e+02, BER=0.09525, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 65/1000, LR=4.95e-04, Loss=2.76335e+02, BER=0.0865625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 66/1000, LR=4.95e-04, Loss=2.77919e+02, BER=0.094, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 67/1000, LR=4.95e-04, Loss=2.78095e+02, BER=0.0935, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 68/1000, LR=4.94e-04, Loss=2.77563e+02, BER=0.09225, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 69/1000, LR=4.94e-04, Loss=2.78014e+02, BER=0.090125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 70/1000, LR=4.94e-04, Loss=2.77532e+02, BER=0.0904375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 71/1000, LR=4.94e-04, Loss=2.75725e+02, BER=0.0874375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 72/1000, LR=4.94e-04, Loss=2.77291e+02, BER=0.0899375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 73/1000, LR=4.94e-04, Loss=2.77162e+02, BER=0.088875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 74/1000, LR=4.93e-04, Loss=2.77644e+02, BER=0.0940625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 75/1000, LR=4.93e-04, Loss=2.78194e+02, BER=0.0935625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 76/1000, LR=4.93e-04, Loss=2.78467e+02, BER=0.0910625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 77/1000, LR=4.93e-04, Loss=2.76633e+02, BER=0.086625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 78/1000, LR=4.93e-04, Loss=2.77014e+02, BER=0.0900625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 79/1000, LR=4.93e-04, Loss=2.78252e+02, BER=0.0933125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 80/1000, LR=4.92e-04, Loss=2.77022e+02, BER=0.090875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 81/1000, LR=4.92e-04, Loss=2.78817e+02, BER=0.0980625, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 82/1000, LR=4.92e-04, Loss=2.77263e+02, BER=0.08925, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 83/1000, LR=4.92e-04, Loss=2.76672e+02, BER=0.09025, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 84/1000, LR=4.92e-04, Loss=2.78274e+02, BER=0.0930625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 85/1000, LR=4.91e-04, Loss=2.77469e+02, BER=0.0915625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 86/1000, LR=4.91e-04, Loss=2.77570e+02, BER=0.091875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 87/1000, LR=4.91e-04, Loss=2.78568e+02, BER=0.0943125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 88/1000, LR=4.91e-04, Loss=2.77452e+02, BER=0.09025, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 89/1000, LR=4.91e-04, Loss=2.77681e+02, BER=0.0921875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 90/1000, LR=4.90e-04, Loss=2.77343e+02, BER=0.0873125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 91/1000, LR=4.90e-04, Loss=2.78671e+02, BER=0.0955625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 92/1000, LR=4.90e-04, Loss=2.77363e+02, BER=0.091375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 93/1000, LR=4.90e-04, Loss=2.77731e+02, BER=0.0879375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 94/1000, LR=4.89e-04, Loss=2.77273e+02, BER=0.08625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 95/1000, LR=4.89e-04, Loss=2.77883e+02, BER=0.0893125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 96/1000, LR=4.89e-04, Loss=2.77700e+02, BER=0.09225, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 97/1000, LR=4.89e-04, Loss=2.78217e+02, BER=0.0906875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 98/1000, LR=4.88e-04, Loss=2.76891e+02, BER=0.0906875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 99/1000, LR=4.88e-04, Loss=2.77697e+02, BER=0.0938125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 100/1000, LR=4.88e-04, Loss=2.77144e+02, BER=0.0915, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 101/1000, LR=4.88e-04, Loss=2.79465e+02, BER=0.093625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 102/1000, LR=4.88e-04, Loss=2.76881e+02, BER=0.08775, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 103/1000, LR=4.87e-04, Loss=2.77788e+02, BER=0.0926875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 104/1000, LR=4.87e-04, Loss=2.76623e+02, BER=0.09025, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 105/1000, LR=4.87e-04, Loss=2.76645e+02, BER=0.0863125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 106/1000, LR=4.87e-04, Loss=2.77041e+02, BER=0.0895625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 107/1000, LR=4.86e-04, Loss=2.78183e+02, BER=0.092625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 108/1000, LR=4.86e-04, Loss=2.77279e+02, BER=0.091125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 109/1000, LR=4.86e-04, Loss=2.78153e+02, BER=0.0933125, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 110/1000, LR=4.85e-04, Loss=2.77136e+02, BER=0.091375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 111/1000, LR=4.85e-04, Loss=2.77551e+02, BER=0.0925625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 112/1000, LR=4.85e-04, Loss=2.77980e+02, BER=0.0920625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 113/1000, LR=4.85e-04, Loss=2.76928e+02, BER=0.0905625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 114/1000, LR=4.84e-04, Loss=2.77359e+02, BER=0.090625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 115/1000, LR=4.84e-04, Loss=2.77148e+02, BER=0.092125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 116/1000, LR=4.84e-04, Loss=2.77694e+02, BER=0.09225, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 117/1000, LR=4.84e-04, Loss=2.77221e+02, BER=0.0904375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 118/1000, LR=4.83e-04, Loss=2.77336e+02, BER=0.089, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 119/1000, LR=4.83e-04, Loss=2.76651e+02, BER=0.086, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 120/1000, LR=4.83e-04, Loss=2.77374e+02, BER=0.092375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 121/1000, LR=4.82e-04, Loss=2.77569e+02, BER=0.0948125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 122/1000, LR=4.82e-04, Loss=2.77246e+02, BER=0.0906875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 123/1000, LR=4.82e-04, Loss=2.75967e+02, BER=0.0861875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 124/1000, LR=4.82e-04, Loss=2.78688e+02, BER=0.095125, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 125/1000, LR=4.81e-04, Loss=2.76880e+02, BER=0.0901875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 126/1000, LR=4.81e-04, Loss=2.76935e+02, BER=0.0901875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 127/1000, LR=4.81e-04, Loss=2.76647e+02, BER=0.087, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 128/1000, LR=4.80e-04, Loss=2.77327e+02, BER=0.09025, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 129/1000, LR=4.80e-04, Loss=2.77779e+02, BER=0.090875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 130/1000, LR=4.80e-04, Loss=2.78000e+02, BER=0.0910625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 131/1000, LR=4.79e-04, Loss=2.77941e+02, BER=0.09275, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 132/1000, LR=4.79e-04, Loss=2.77837e+02, BER=0.0918125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 133/1000, LR=4.79e-04, Loss=2.78576e+02, BER=0.0919375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 134/1000, LR=4.78e-04, Loss=2.77270e+02, BER=0.0871875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 135/1000, LR=4.78e-04, Loss=2.78776e+02, BER=0.094375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 136/1000, LR=4.78e-04, Loss=2.77164e+02, BER=0.089375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 137/1000, LR=4.78e-04, Loss=2.77459e+02, BER=0.091875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 138/1000, LR=4.77e-04, Loss=2.77100e+02, BER=0.0925, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 139/1000, LR=4.77e-04, Loss=2.77604e+02, BER=0.095625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 140/1000, LR=4.77e-04, Loss=2.78304e+02, BER=0.0939375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 141/1000, LR=4.76e-04, Loss=2.77704e+02, BER=0.0953125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 142/1000, LR=4.76e-04, Loss=2.77746e+02, BER=0.09, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 143/1000, LR=4.76e-04, Loss=2.77251e+02, BER=0.0911875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 144/1000, LR=4.75e-04, Loss=2.78168e+02, BER=0.0946875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 145/1000, LR=4.75e-04, Loss=2.76046e+02, BER=0.0911875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 146/1000, LR=4.75e-04, Loss=2.76566e+02, BER=0.0885, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 147/1000, LR=4.74e-04, Loss=2.78455e+02, BER=0.0895625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 148/1000, LR=4.74e-04, Loss=2.76837e+02, BER=0.0885, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 149/1000, LR=4.73e-04, Loss=2.78719e+02, BER=0.093625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 150/1000, LR=4.73e-04, Loss=2.77446e+02, BER=0.08875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 151/1000, LR=4.73e-04, Loss=2.78650e+02, BER=0.08925, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 152/1000, LR=4.72e-04, Loss=2.76988e+02, BER=0.0905, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 153/1000, LR=4.72e-04, Loss=2.76475e+02, BER=0.0905625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 154/1000, LR=4.72e-04, Loss=2.77658e+02, BER=0.08975, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 155/1000, LR=4.71e-04, Loss=2.77524e+02, BER=0.0881875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 156/1000, LR=4.71e-04, Loss=2.77390e+02, BER=0.09475, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 157/1000, LR=4.71e-04, Loss=2.77326e+02, BER=0.092, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 158/1000, LR=4.70e-04, Loss=2.77814e+02, BER=0.092875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 159/1000, LR=4.70e-04, Loss=2.78350e+02, BER=0.09325, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 160/1000, LR=4.69e-04, Loss=2.77310e+02, BER=0.0915625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 161/1000, LR=4.69e-04, Loss=2.77989e+02, BER=0.0915, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 162/1000, LR=4.69e-04, Loss=2.75634e+02, BER=0.0875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 163/1000, LR=4.68e-04, Loss=2.76599e+02, BER=0.087375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 164/1000, LR=4.68e-04, Loss=2.75696e+02, BER=0.089375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 165/1000, LR=4.68e-04, Loss=2.76947e+02, BER=0.089625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 166/1000, LR=4.67e-04, Loss=2.76544e+02, BER=0.0904375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 167/1000, LR=4.67e-04, Loss=2.77894e+02, BER=0.0921875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 168/1000, LR=4.66e-04, Loss=2.77980e+02, BER=0.0905, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 169/1000, LR=4.66e-04, Loss=2.77233e+02, BER=0.0891875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 170/1000, LR=4.66e-04, Loss=2.78538e+02, BER=0.0910625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 171/1000, LR=4.65e-04, Loss=2.78484e+02, BER=0.0951875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 172/1000, LR=4.65e-04, Loss=2.77846e+02, BER=0.091, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 173/1000, LR=4.64e-04, Loss=2.77056e+02, BER=0.0894375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 174/1000, LR=4.64e-04, Loss=2.76618e+02, BER=0.0905625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 175/1000, LR=4.64e-04, Loss=2.77595e+02, BER=0.0913125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 176/1000, LR=4.63e-04, Loss=2.77586e+02, BER=0.0890625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 177/1000, LR=4.63e-04, Loss=2.77621e+02, BER=0.0931875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 178/1000, LR=4.62e-04, Loss=2.77644e+02, BER=0.0929375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 179/1000, LR=4.62e-04, Loss=2.77366e+02, BER=0.090375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 180/1000, LR=4.62e-04, Loss=2.77420e+02, BER=0.0893125, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 181/1000, LR=4.61e-04, Loss=2.77191e+02, BER=0.092125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 182/1000, LR=4.61e-04, Loss=2.77336e+02, BER=0.09025, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 183/1000, LR=4.60e-04, Loss=2.76981e+02, BER=0.092375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 184/1000, LR=4.60e-04, Loss=2.76599e+02, BER=0.0878125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 185/1000, LR=4.59e-04, Loss=2.76696e+02, BER=0.086875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 186/1000, LR=4.59e-04, Loss=2.77537e+02, BER=0.089625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 187/1000, LR=4.59e-04, Loss=2.76797e+02, BER=0.094125, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 188/1000, LR=4.58e-04, Loss=2.77757e+02, BER=0.092, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 189/1000, LR=4.58e-04, Loss=2.77056e+02, BER=0.0925625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 190/1000, LR=4.57e-04, Loss=2.77677e+02, BER=0.0914375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 191/1000, LR=4.57e-04, Loss=2.78337e+02, BER=0.0924375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 192/1000, LR=4.56e-04, Loss=2.77838e+02, BER=0.094, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 193/1000, LR=4.56e-04, Loss=2.77075e+02, BER=0.092125, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 194/1000, LR=4.55e-04, Loss=2.77947e+02, BER=0.0925, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 195/1000, LR=4.55e-04, Loss=2.77047e+02, BER=0.0933125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 196/1000, LR=4.55e-04, Loss=2.77342e+02, BER=0.08925, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 197/1000, LR=4.54e-04, Loss=2.77899e+02, BER=0.092, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 198/1000, LR=4.54e-04, Loss=2.77738e+02, BER=0.089875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 199/1000, LR=4.53e-04, Loss=2.76852e+02, BER=0.08875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 200/1000, LR=4.53e-04, Loss=2.78706e+02, BER=0.09225, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 201/1000, LR=4.52e-04, Loss=2.77318e+02, BER=0.09175, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 202/1000, LR=4.52e-04, Loss=2.78049e+02, BER=0.0910625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 203/1000, LR=4.51e-04, Loss=2.77332e+02, BER=0.09275, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 204/1000, LR=4.51e-04, Loss=2.75867e+02, BER=0.086625, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 205/1000, LR=4.50e-04, Loss=2.77450e+02, BER=0.0943125, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 206/1000, LR=4.50e-04, Loss=2.77071e+02, BER=0.0910625, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 207/1000, LR=4.49e-04, Loss=2.78158e+02, BER=0.0915625, duration: 0.10s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 208/1000, LR=4.49e-04, Loss=2.78347e+02, BER=0.0931875, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 209/1000, LR=4.48e-04, Loss=2.77806e+02, BER=0.0899375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 210/1000, LR=4.48e-04, Loss=2.77862e+02, BER=0.0914375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 211/1000, LR=4.48e-04, Loss=2.77160e+02, BER=0.0889375, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 212/1000, LR=4.47e-04, Loss=2.77608e+02, BER=0.0913125, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 213/1000, LR=4.47e-04, Loss=2.78270e+02, BER=0.09425, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 214/1000, LR=4.46e-04, Loss=2.78260e+02, BER=0.0976875, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 215/1000, LR=4.46e-04, Loss=2.77628e+02, BER=0.0931875, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 216/1000, LR=4.45e-04, Loss=2.77576e+02, BER=0.0911875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 217/1000, LR=4.45e-04, Loss=2.78263e+02, BER=0.095, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 218/1000, LR=4.44e-04, Loss=2.77919e+02, BER=0.094125, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 219/1000, LR=4.44e-04, Loss=2.78510e+02, BER=0.0970625, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 220/1000, LR=4.43e-04, Loss=2.77996e+02, BER=0.092375, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 221/1000, LR=4.43e-04, Loss=2.77800e+02, BER=0.0955, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 222/1000, LR=4.42e-04, Loss=2.77254e+02, BER=0.0898125, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 223/1000, LR=4.42e-04, Loss=2.77171e+02, BER=0.0906875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 224/1000, LR=4.41e-04, Loss=2.78142e+02, BER=0.0920625, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 225/1000, LR=4.41e-04, Loss=2.78236e+02, BER=0.0935625, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 226/1000, LR=4.40e-04, Loss=2.77179e+02, BER=0.091, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 227/1000, LR=4.40e-04, Loss=2.78349e+02, BER=0.09225, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 228/1000, LR=4.39e-04, Loss=2.76510e+02, BER=0.090625, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 229/1000, LR=4.39e-04, Loss=2.77104e+02, BER=0.0888125, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 230/1000, LR=4.38e-04, Loss=2.77384e+02, BER=0.0939375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 231/1000, LR=4.38e-04, Loss=2.78010e+02, BER=0.093625, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 232/1000, LR=4.37e-04, Loss=2.75326e+02, BER=0.0889375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 233/1000, LR=4.36e-04, Loss=2.77199e+02, BER=0.0908125, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 234/1000, LR=4.36e-04, Loss=2.78347e+02, BER=0.0944375, duration: 0.09s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 235/1000, LR=4.35e-04, Loss=2.78406e+02, BER=0.0931875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 236/1000, LR=4.35e-04, Loss=2.76961e+02, BER=0.0921875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 237/1000, LR=4.34e-04, Loss=2.78007e+02, BER=0.092125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 238/1000, LR=4.34e-04, Loss=2.75244e+02, BER=0.0859375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 239/1000, LR=4.33e-04, Loss=2.79001e+02, BER=0.0934375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 240/1000, LR=4.33e-04, Loss=2.77611e+02, BER=0.0898125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 241/1000, LR=4.32e-04, Loss=2.77512e+02, BER=0.091125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 242/1000, LR=4.32e-04, Loss=2.76837e+02, BER=0.0885625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 243/1000, LR=4.31e-04, Loss=2.77933e+02, BER=0.091125, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 244/1000, LR=4.31e-04, Loss=2.77243e+02, BER=0.0883125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 245/1000, LR=4.30e-04, Loss=2.77643e+02, BER=0.09275, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 246/1000, LR=4.30e-04, Loss=2.78464e+02, BER=0.09375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 247/1000, LR=4.29e-04, Loss=2.78090e+02, BER=0.0945, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 248/1000, LR=4.28e-04, Loss=2.77394e+02, BER=0.090375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 249/1000, LR=4.28e-04, Loss=2.78815e+02, BER=0.09275, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 250/1000, LR=4.27e-04, Loss=2.75946e+02, BER=0.08725, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 251/1000, LR=4.27e-04, Loss=2.77192e+02, BER=0.0929375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 252/1000, LR=4.26e-04, Loss=2.76352e+02, BER=0.0849375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 253/1000, LR=4.26e-04, Loss=2.77119e+02, BER=0.08775, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 254/1000, LR=4.25e-04, Loss=2.77726e+02, BER=0.0913125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 255/1000, LR=4.25e-04, Loss=2.77890e+02, BER=0.0955, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 256/1000, LR=4.24e-04, Loss=2.77605e+02, BER=0.0945, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 257/1000, LR=4.23e-04, Loss=2.77611e+02, BER=0.0950625, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 258/1000, LR=4.23e-04, Loss=2.77535e+02, BER=0.0919375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 259/1000, LR=4.22e-04, Loss=2.77919e+02, BER=0.0920625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 260/1000, LR=4.22e-04, Loss=2.76931e+02, BER=0.090125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 261/1000, LR=4.21e-04, Loss=2.77622e+02, BER=0.0914375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 262/1000, LR=4.21e-04, Loss=2.76118e+02, BER=0.0874375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 263/1000, LR=4.20e-04, Loss=2.76749e+02, BER=0.0875625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 264/1000, LR=4.19e-04, Loss=2.76944e+02, BER=0.0919375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 265/1000, LR=4.19e-04, Loss=2.77227e+02, BER=0.0894375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 266/1000, LR=4.18e-04, Loss=2.76778e+02, BER=0.0899375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 267/1000, LR=4.18e-04, Loss=2.77079e+02, BER=0.0899375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 268/1000, LR=4.17e-04, Loss=2.77115e+02, BER=0.0908125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 269/1000, LR=4.17e-04, Loss=2.77948e+02, BER=0.090625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 270/1000, LR=4.16e-04, Loss=2.78318e+02, BER=0.0899375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 271/1000, LR=4.15e-04, Loss=2.78085e+02, BER=0.09175, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 272/1000, LR=4.15e-04, Loss=2.78940e+02, BER=0.0918125, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 273/1000, LR=4.14e-04, Loss=2.76706e+02, BER=0.0915625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 274/1000, LR=4.14e-04, Loss=2.77015e+02, BER=0.091875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 275/1000, LR=4.13e-04, Loss=2.78430e+02, BER=0.0946875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 276/1000, LR=4.12e-04, Loss=2.77859e+02, BER=0.0904375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 277/1000, LR=4.12e-04, Loss=2.77502e+02, BER=0.09325, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 278/1000, LR=4.11e-04, Loss=2.78241e+02, BER=0.093375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 279/1000, LR=4.11e-04, Loss=2.76768e+02, BER=0.0896875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 280/1000, LR=4.10e-04, Loss=2.78043e+02, BER=0.0895, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 281/1000, LR=4.09e-04, Loss=2.76109e+02, BER=0.0923125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 282/1000, LR=4.09e-04, Loss=2.76710e+02, BER=0.087875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 283/1000, LR=4.08e-04, Loss=2.78012e+02, BER=0.0920625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 284/1000, LR=4.08e-04, Loss=2.78472e+02, BER=0.093625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 285/1000, LR=4.07e-04, Loss=2.77685e+02, BER=0.092, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 286/1000, LR=4.06e-04, Loss=2.76255e+02, BER=0.085875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 287/1000, LR=4.06e-04, Loss=2.76095e+02, BER=0.090375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 288/1000, LR=4.05e-04, Loss=2.78236e+02, BER=0.0930625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 289/1000, LR=4.04e-04, Loss=2.78873e+02, BER=0.0958125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 290/1000, LR=4.04e-04, Loss=2.77986e+02, BER=0.0925, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 291/1000, LR=4.03e-04, Loss=2.78439e+02, BER=0.09275, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 292/1000, LR=4.03e-04, Loss=2.76680e+02, BER=0.0929375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 293/1000, LR=4.02e-04, Loss=2.76884e+02, BER=0.088625, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 294/1000, LR=4.01e-04, Loss=2.77803e+02, BER=0.0905625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 295/1000, LR=4.01e-04, Loss=2.77687e+02, BER=0.094125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 296/1000, LR=4.00e-04, Loss=2.76774e+02, BER=0.0894375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 297/1000, LR=3.99e-04, Loss=2.76510e+02, BER=0.09, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 298/1000, LR=3.99e-04, Loss=2.77301e+02, BER=0.090875, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 299/1000, LR=3.98e-04, Loss=2.76714e+02, BER=0.0880625, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 300/1000, LR=3.98e-04, Loss=2.77503e+02, BER=0.090375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 301/1000, LR=3.97e-04, Loss=2.78160e+02, BER=0.091625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 302/1000, LR=3.96e-04, Loss=2.76560e+02, BER=0.088625, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 303/1000, LR=3.96e-04, Loss=2.77731e+02, BER=0.089625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 304/1000, LR=3.95e-04, Loss=2.77570e+02, BER=0.089125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 305/1000, LR=3.94e-04, Loss=2.76641e+02, BER=0.0899375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 306/1000, LR=3.94e-04, Loss=2.76193e+02, BER=0.0904375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 307/1000, LR=3.93e-04, Loss=2.78303e+02, BER=0.093625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 308/1000, LR=3.92e-04, Loss=2.78773e+02, BER=0.0885625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 309/1000, LR=3.92e-04, Loss=2.78337e+02, BER=0.091375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 310/1000, LR=3.91e-04, Loss=2.76928e+02, BER=0.0896875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 311/1000, LR=3.91e-04, Loss=2.77258e+02, BER=0.090125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 312/1000, LR=3.90e-04, Loss=2.78238e+02, BER=0.091375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 313/1000, LR=3.89e-04, Loss=2.77140e+02, BER=0.0889375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 314/1000, LR=3.89e-04, Loss=2.79716e+02, BER=0.0979375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 315/1000, LR=3.88e-04, Loss=2.77712e+02, BER=0.091375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 316/1000, LR=3.87e-04, Loss=2.77863e+02, BER=0.092625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 317/1000, LR=3.87e-04, Loss=2.77943e+02, BER=0.0935625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 318/1000, LR=3.86e-04, Loss=2.77655e+02, BER=0.0915625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 319/1000, LR=3.85e-04, Loss=2.77885e+02, BER=0.08975, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 320/1000, LR=3.85e-04, Loss=2.77334e+02, BER=0.091375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 321/1000, LR=3.84e-04, Loss=2.77972e+02, BER=0.0879375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 322/1000, LR=3.83e-04, Loss=2.77276e+02, BER=0.0931875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 323/1000, LR=3.83e-04, Loss=2.77780e+02, BER=0.0905625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 324/1000, LR=3.82e-04, Loss=2.76915e+02, BER=0.09025, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 325/1000, LR=3.81e-04, Loss=2.77304e+02, BER=0.090375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 326/1000, LR=3.81e-04, Loss=2.78693e+02, BER=0.093, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 327/1000, LR=3.80e-04, Loss=2.77527e+02, BER=0.0909375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 328/1000, LR=3.79e-04, Loss=2.77216e+02, BER=0.0903125, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 329/1000, LR=3.79e-04, Loss=2.76993e+02, BER=0.0885625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 330/1000, LR=3.78e-04, Loss=2.77371e+02, BER=0.0895, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 331/1000, LR=3.77e-04, Loss=2.76004e+02, BER=0.0871875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 332/1000, LR=3.77e-04, Loss=2.77836e+02, BER=0.093125, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 333/1000, LR=3.76e-04, Loss=2.76334e+02, BER=0.086875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 334/1000, LR=3.75e-04, Loss=2.77063e+02, BER=0.0921875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 335/1000, LR=3.75e-04, Loss=2.77883e+02, BER=0.0918125, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 336/1000, LR=3.74e-04, Loss=2.78211e+02, BER=0.093125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 337/1000, LR=3.73e-04, Loss=2.77594e+02, BER=0.08975, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 338/1000, LR=3.72e-04, Loss=2.77691e+02, BER=0.09075, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 339/1000, LR=3.72e-04, Loss=2.77240e+02, BER=0.0894375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 340/1000, LR=3.71e-04, Loss=2.78071e+02, BER=0.0939375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 341/1000, LR=3.70e-04, Loss=2.77671e+02, BER=0.0890625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 342/1000, LR=3.70e-04, Loss=2.77487e+02, BER=0.0915625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 343/1000, LR=3.69e-04, Loss=2.78240e+02, BER=0.0969375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 344/1000, LR=3.68e-04, Loss=2.77187e+02, BER=0.0908125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 345/1000, LR=3.68e-04, Loss=2.77896e+02, BER=0.09375, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 346/1000, LR=3.67e-04, Loss=2.79041e+02, BER=0.0965625, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 347/1000, LR=3.66e-04, Loss=2.77120e+02, BER=0.0894375, duration: 0.08s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 348/1000, LR=3.66e-04, Loss=2.77819e+02, BER=0.095125, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n",
            "Training epoch 349/1000, LR=3.65e-04, Loss=2.75684e+02, BER=0.0876875, duration: 0.07s\n",
            "Rev def call with line-search...\n",
            "Rev def call with line-search...\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-83-6694b7089ab5>\u001b[0m in \u001b[0;36m<cell line: 83>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     81\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m \u001b[0mtrain_dec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me2e_ltd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-83-6694b7089ab5>\u001b[0m in \u001b[0;36mtrain_dec\u001b[0;34m(model, args)\u001b[0m\n\u001b[1;32m     46\u001b[0m                                           maxval=args.ebno_db_eval)\n\u001b[1;32m     47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 48\u001b[0;31m             \u001b[0mc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mllr_hat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mebno_db\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m             \u001b[0;31m# loss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m             \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/engine/training.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    588\u001b[0m             \u001b[0mlayout_map_lib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_map_subclass_model_variable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_layout_map\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    589\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 590\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    591\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    592\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mdoc_controls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdoc_in_current_and_subclasses\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m             \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1147\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_compute_dtype_object\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1148\u001b[0m                 ):\n\u001b[0;32m-> 1149\u001b[0;31m                     \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1150\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1151\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_activity_regularizer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     94\u001b[0m         \u001b[0mbound_signature\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 96\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     97\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"_keras_call_info_injected\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    830\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    831\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 832\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    833\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    834\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    866\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    867\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 868\u001b[0;31m       return tracing_compilation.call_function(\n\u001b[0m\u001b[1;32m    869\u001b[0m           \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_no_variable_creation_config\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    870\u001b[0m       )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/eager/polymorphic_function/tracing_compilation.py\u001b[0m in \u001b[0;36mcall_function\u001b[0;34m(args, kwargs, tracing_options)\u001b[0m\n\u001b[1;32m    137\u001b[0m   \u001b[0mbound_args\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction_type\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    138\u001b[0m   \u001b[0mflat_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction_type\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munpack_inputs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbound_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 139\u001b[0;31m   return function._call_flat(  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m    140\u001b[0m       \u001b[0mflat_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    141\u001b[0m   )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/eager/polymorphic_function/concrete_function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, tensor_inputs, captured_inputs)\u001b[0m\n\u001b[1;32m   1321\u001b[0m         and executing_eagerly):\n\u001b[1;32m   1322\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1323\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_inference_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall_preflattened\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1324\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1325\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/eager/polymorphic_function/atomic_function.py\u001b[0m in \u001b[0;36mcall_preflattened\u001b[0;34m(self, args)\u001b[0m\n\u001b[1;32m    214\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mcall_preflattened\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mSequence\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m     \u001b[0;34m\"\"\"Calls with flattened tensor inputs and returns the structured output.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 216\u001b[0;31m     \u001b[0mflat_outputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    217\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction_type\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpack_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mflat_outputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/eager/polymorphic_function/atomic_function.py\u001b[0m in \u001b[0;36mcall_flat\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m    249\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mrecord\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop_recording\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    250\u001b[0m           \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_bound_context\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 251\u001b[0;31m             outputs = self._bound_context.call_function(\n\u001b[0m\u001b[1;32m    252\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    253\u001b[0m                 \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/eager/context.py\u001b[0m in \u001b[0;36mcall_function\u001b[0;34m(self, name, tensor_inputs, num_outputs)\u001b[0m\n\u001b[1;32m   1484\u001b[0m     \u001b[0mcancellation_context\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcancellation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1485\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcancellation_context\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1486\u001b[0;31m       outputs = execute.execute(\n\u001b[0m\u001b[1;32m   1487\u001b[0m           \u001b[0mname\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"utf-8\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1488\u001b[0m           \u001b[0mnum_outputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnum_outputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     51\u001b[0m   \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 53\u001b[0;31m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[1;32m     54\u001b[0m                                         inputs, attrs, num_outputs)\n\u001b[1;32m     55\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "OuPmknHQuxFw"
      },
      "execution_count": 77,
      "outputs": []
    }
  ]
}