03_trt_20210317.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Latent representation inference\n",
    "\n",
    "This notebook uses TensorRT to infer latent represantation of S2 tensor set based on the CAE.\n",
    "\n",
    "The NVIDIA Docker container TensorFlow 21.02-tf2 was used to generate a computational environment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.python.compiler.tensorrt import trt_convert as trt\n",
    "import numpy as np\n",
    "from tensorflow import keras\n",
    "import datetime as dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datestring = dt.datetime.now().strftime(\"%Y%m%d\")\n",
    "print(datestring)\n",
    "suffix = '3'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "x_tensors = np.load('sentinel_gb_995_'+suffix+'_20210317.npz')['tensors']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "conversion_params = trt.DEFAULT_TRT_CONVERSION_PARAMS._replace(\n",
    "                    precision_mode='FP32',\n",
    "                    # Set this to a large enough number so it can cache all the engines.\n",
    "                    maximum_cached_engines=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf tensorrt/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_saved_model_dir=\"models/gb_postcode_encoder_\" + suffix\n",
    "output_saved_model_dir=\"tensorrt/gb_postcode_encoder_inter_\" + suffix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "converter = trt.TrtGraphConverterV2(\n",
    "            input_saved_model_dir=input_saved_model_dir,\n",
    "            conversion_params=conversion_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "converter.convert()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_input_fn():\n",
    "    num_runs = 10\n",
    "    for _ in range(num_runs):\n",
    "        inp1 = np.random.normal(size=(1000, 16, 16, 4)).astype(np.float32)\n",
    "        yield inp1,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "converter.build(my_input_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "converter.save(output_saved_model_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "input_data = tf.convert_to_tensor(x_tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "saved_model_loaded = tf.saved_model.load(output_saved_model_dir, tags=[trt.tag_constants.SERVING])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_func = saved_model_loaded.signatures[trt.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frozen_func = trt.convert_to_constants.convert_variables_to_constants_v2(graph_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# Initialise TensorRT Engine with random array\n",
    "_ = frozen_func(tf.convert_to_tensor(np.random.normal(size=(1000, 16, 16, 4)).astype(np.float32)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "batch_size = 1000\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "for i in range(0,len(x_tensors),batch_size):\n",
    "    end = i + len(input_data[i:i+batch_size])\n",
    "    output += [frozen_func(input_data[i:i+batch_size])[0].numpy()]\n",
    "    if end % 100000 == 0:\n",
    "        elapsed = time.time()-start\n",
    "        print(\"Records:\",end,round(elapsed,3),\"secs\",round(end/elapsed,3),\"recs/sec\")\n",
    "        \n",
    "print(\"Records:\",end,round(elapsed,3),\"secs\",round(end/elapsed,3),\"recs/sec\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "trt_output = np.concatenate(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "trt_results = trt_output.reshape(-1,64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = None\n",
    "model = keras.models.load_model(\"models/gb_postcode_encoder_\" + suffix)\n",
    "model.compile()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "keras_output = model(x_tensors[:1000])\n",
    "print(keras_output[0].numpy().reshape(-1,64))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras_results = keras_output.numpy().reshape(-1,64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inference with TensorRT\n",
    "trt_results[:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inference with TensorFlow/Keras\n",
    "keras_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.testing.assert_allclose(trt_results[:1000],keras_results,atol=1e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trt_error = abs(trt_results[:1000]-keras_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"TensorRT Absolute Error mean:\",trt_error.mean(),\"std dev:\",trt_error.std(),\"min:\",trt_error.min(),\"max:\",trt_error.max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trt_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "filename = \"sentinel_flattened_gb_postcode_trt_\" + suffix + \"_\" + datestring\n",
    "np.savez_compressed(filename,trt=trt_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}