dev/docker-volume/notebooks/02-ingest-raw-data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Raw data ingestion\n",
    "\n",
    "## Scope of notebook\n",
    "\n",
    "> Ingest raw data to local file system at `RAW_DATA_PATH`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loads required libraries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "from src.config import S3_SOURCE_BUCKET_NAME\n",
    "from src.IOController import create_pyspark_session, ingest_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Defines full path for S3 raw data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['s3n://ifood-data-architect-test-source/order.json.gz',\n",
       " 's3n://ifood-data-architect-test-source/status.json.gz',\n",
       " 's3n://ifood-data-architect-test-source/restaurant.csv.gz',\n",
       " 's3n://ifood-data-architect-test-source/consumer.csv.gz']"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files_to_ingest = ['order.json.gz', 'status.json.gz', 'restaurant.csv.gz', 'consumer.csv.gz']\n",
    "s3_paths = [f's3n://{S3_SOURCE_BUCKET_NAME}/{f}' for f in files_to_ingest]\n",
    "s3_paths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creates PySpark Session able to handle AWS S3 connection."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - in-memory</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://d9844b7e5261:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.0.0</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>my_app</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x7fa63f54b9a0>"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark = create_pyspark_session()\n",
    "spark"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ingests data to local file system."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ingestion started for `s3n://ifood-data-architect-test-source/order.json.gz`\n",
      "File successfully ingested into `/home/jovyan/data/raw/order`\n",
      "Ingestion started for `s3n://ifood-data-architect-test-source/status.json.gz`\n",
      "File successfully ingested into `/home/jovyan/data/raw/status`\n",
      "Ingestion started for `s3n://ifood-data-architect-test-source/restaurant.csv.gz`\n",
      "File successfully ingested into `/home/jovyan/data/raw/restaurant`\n",
      "Output already exists at `/home/jovyan/data/raw/consumer`. Ingestion skipped for this file.\n"
     ]
    }
   ],
   "source": [
    "ingest_data(s3_paths, spark)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ingestions successfully accomplished!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}