ModelGen.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = '/Users/shovitraj/Github/Python/MagNet_data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "dst = pd.read_csv(DATA_PATH +\"dst_labels.csv\")\n",
    "dst.timedelta = pd.to_timedelta(dst.timedelta)\n",
    "dst.set_index([\"period\", \"timedelta\"], inplace=True)\n",
    "\n",
    "sunspots = pd.read_csv(DATA_PATH + \"sunspots.csv\")\n",
    "sunspots.timedelta = pd.to_timedelta(sunspots.timedelta)\n",
    "sunspots.set_index([\"period\", \"timedelta\"], inplace=True)\n",
    "\n",
    "solar_wind = pd.read_csv(DATA_PATH + \"solar_wind.csv\")\n",
    "solar_wind.timedelta = pd.to_timedelta(solar_wind.timedelta)\n",
    "solar_wind.set_index([\"period\", \"timedelta\"], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>bx_gse</th>\n",
       "      <th>by_gse</th>\n",
       "      <th>bz_gse</th>\n",
       "      <th>theta_gse</th>\n",
       "      <th>phi_gse</th>\n",
       "      <th>bt</th>\n",
       "      <th>density</th>\n",
       "      <th>speed</th>\n",
       "      <th>temperature</th>\n",
       "      <th>dst</th>\n",
       "      <th>smoothed_ssn</th>\n",
       "      <th>gse_x_ace</th>\n",
       "      <th>gse_y_ace</th>\n",
       "      <th>gse_z_ace</th>\n",
       "      <th>by_gse_std</th>\n",
       "      <th>bz_gse_std</th>\n",
       "      <th>bt_std</th>\n",
       "      <th>density_std</th>\n",
       "      <th>speed_std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>timedelta</th>\n",
       "      <th>period</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0 days 00:00:00</th>\n",
       "      <th>train_a</th>\n",
       "      <td>-6.149500</td>\n",
       "      <td>1.6455</td>\n",
       "      <td>0.844667</td>\n",
       "      <td>7.103667</td>\n",
       "      <td>165.420333</td>\n",
       "      <td>7.140000</td>\n",
       "      <td>1.174167</td>\n",
       "      <td>355.278250</td>\n",
       "      <td>68880.341667</td>\n",
       "      <td>-7</td>\n",
       "      <td>65.400000</td>\n",
       "      <td>1.522377e+06</td>\n",
       "      <td>143704.600000</td>\n",
       "      <td>149496.700000</td>\n",
       "      <td>1.224754</td>\n",
       "      <td>0.580028</td>\n",
       "      <td>1.607043</td>\n",
       "      <td>0.479647</td>\n",
       "      <td>15.025364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 01:00:00</th>\n",
       "      <th>train_a</th>\n",
       "      <td>-6.693833</td>\n",
       "      <td>0.7665</td>\n",
       "      <td>1.265167</td>\n",
       "      <td>10.183333</td>\n",
       "      <td>174.267833</td>\n",
       "      <td>7.284833</td>\n",
       "      <td>0.815917</td>\n",
       "      <td>330.080583</td>\n",
       "      <td>52203.225000</td>\n",
       "      <td>-10</td>\n",
       "      <td>65.421154</td>\n",
       "      <td>1.522503e+06</td>\n",
       "      <td>143388.108333</td>\n",
       "      <td>149560.758333</td>\n",
       "      <td>0.907098</td>\n",
       "      <td>0.731278</td>\n",
       "      <td>0.325293</td>\n",
       "      <td>0.698365</td>\n",
       "      <td>17.203299</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           bx_gse  by_gse    bz_gse  theta_gse     phi_gse  \\\n",
       "timedelta       period                                                       \n",
       "0 days 00:00:00 train_a -6.149500  1.6455  0.844667   7.103667  165.420333   \n",
       "0 days 01:00:00 train_a -6.693833  0.7665  1.265167  10.183333  174.267833   \n",
       "\n",
       "                               bt   density       speed   temperature  dst  \\\n",
       "timedelta       period                                                       \n",
       "0 days 00:00:00 train_a  7.140000  1.174167  355.278250  68880.341667   -7   \n",
       "0 days 01:00:00 train_a  7.284833  0.815917  330.080583  52203.225000  -10   \n",
       "\n",
       "                         smoothed_ssn     gse_x_ace      gse_y_ace  \\\n",
       "timedelta       period                                               \n",
       "0 days 00:00:00 train_a     65.400000  1.522377e+06  143704.600000   \n",
       "0 days 01:00:00 train_a     65.421154  1.522503e+06  143388.108333   \n",
       "\n",
       "                             gse_z_ace  by_gse_std  bz_gse_std    bt_std  \\\n",
       "timedelta       period                                                     \n",
       "0 days 00:00:00 train_a  149496.700000    1.224754    0.580028  1.607043   \n",
       "0 days 01:00:00 train_a  149560.758333    0.907098    0.731278  0.325293   \n",
       "\n",
       "                         density_std  speed_std  \n",
       "timedelta       period                           \n",
       "0 days 00:00:00 train_a     0.479647  15.025364  \n",
       "0 days 01:00:00 train_a     0.698365  17.203299  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df=pd.read_csv('/Users/shovitraj/Github/Python/MagNet_data/hourly_df.csv',\n",
    "#                index_col=['timedelta','period'])\n",
    "# df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy.random import seed\n",
    "from tensorflow.random import set_seed\n",
    "\n",
    "seed(2020)\n",
    "set_seed(2021)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# subset of solar wind features to use for modeling\n",
    "SOLAR_WIND_FEATURES = [\n",
    "    \"bt\",\n",
    "    \"temperature\",\n",
    "    \"bx_gse\",\n",
    "    \"by_gse\",\n",
    "    \"bz_gse\",\n",
    "    \"speed\",\n",
    "    \"density\",\n",
    "]\n",
    "\n",
    "# all of the features we'll use, including sunspot numbers\n",
    "XCOLS = (\n",
    "    [col + \"_mean\" for col in SOLAR_WIND_FEATURES]\n",
    "    + [col + \"_std\" for col in SOLAR_WIND_FEATURES]\n",
    "    + [\"smoothed_ssn\"]\n",
    ")\n",
    "\n",
    "\n",
    "def impute_features(feature_df):\n",
    "    \"\"\"Imputes data using the following methods:\n",
    "    - `smoothed_ssn`: forward fill\n",
    "    - `solar_wind`: interpolation\n",
    "    \"\"\"\n",
    "    # forward fill sunspot data for the rest of the month\n",
    "    feature_df.smoothed_ssn = feature_df.smoothed_ssn.fillna(method=\"ffill\")\n",
    "    # interpolate between missing solar wind values\n",
    "    feature_df = feature_df.interpolate()\n",
    "    return feature_df\n",
    "\n",
    "\n",
    "def aggregate_hourly(feature_df, aggs=[\"mean\", \"std\"]):\n",
    "    \"\"\"Aggregates features to the floor of each hour using mean and standard deviation.\n",
    "    e.g. All values from \"11:00:00\" to \"11:59:00\" will be aggregated to \"11:00:00\".\n",
    "    \"\"\"\n",
    "    # group by the floor of each hour use timedelta index\n",
    "    agged = feature_df.groupby(\n",
    "        [\"period\", feature_df.index.get_level_values(1).floor(\"H\")]\n",
    "    ).agg(aggs)\n",
    "    # flatten hierachical column index\n",
    "    agged.columns = [\"_\".join(x) for x in agged.columns]\n",
    "    return agged\n",
    "\n",
    "\n",
    "def preprocess_features(solar_wind, sunspots, scaler=None, subset=None):\n",
    "    \"\"\"\n",
    "    Preprocessing steps:\n",
    "        - Subset the data\n",
    "        - Aggregate hourly\n",
    "        - Join solar wind and sunspot data\n",
    "        - Scale using standard scaler\n",
    "        - Impute missing values\n",
    "    \"\"\"\n",
    "    # select features we want to use\n",
    "    if subset:\n",
    "        solar_wind = solar_wind[subset]\n",
    "\n",
    "    # aggregate solar wind data and join with sunspots\n",
    "    hourly_features = aggregate_hourly(solar_wind).join(sunspots)\n",
    "\n",
    "    # subtract mean and divide by standard deviation\n",
    "    if scaler is None:\n",
    "        scaler = StandardScaler()\n",
    "        scaler.fit(hourly_features)\n",
    "\n",
    "    normalized = pd.DataFrame(\n",
    "        scaler.transform(hourly_features),\n",
    "        index=hourly_features.index,\n",
    "        columns=hourly_features.columns,\n",
    "    )\n",
    "\n",
    "    # impute missing values\n",
    "    imputed = impute_features(normalized)\n",
    "\n",
    "    # we want to return the scaler object as well to use later during prediction\n",
    "    return imputed, scaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(139872, 15)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>bt_mean</th>\n",
       "      <th>bt_std</th>\n",
       "      <th>temperature_mean</th>\n",
       "      <th>temperature_std</th>\n",
       "      <th>bx_gse_mean</th>\n",
       "      <th>bx_gse_std</th>\n",
       "      <th>by_gse_mean</th>\n",
       "      <th>by_gse_std</th>\n",
       "      <th>bz_gse_mean</th>\n",
       "      <th>bz_gse_std</th>\n",
       "      <th>speed_mean</th>\n",
       "      <th>speed_std</th>\n",
       "      <th>density_mean</th>\n",
       "      <th>density_std</th>\n",
       "      <th>smoothed_ssn</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">train_a</th>\n",
       "      <th>0 days 00:00:00</th>\n",
       "      <td>0.499705</td>\n",
       "      <td>2.443614</td>\n",
       "      <td>-0.375267</td>\n",
       "      <td>0.383941</td>\n",
       "      <td>-1.599207</td>\n",
       "      <td>-0.381502</td>\n",
       "      <td>0.419516</td>\n",
       "      <td>0.031658</td>\n",
       "      <td>0.300358</td>\n",
       "      <td>-0.651645</td>\n",
       "      <td>-0.738546</td>\n",
       "      <td>0.862524</td>\n",
       "      <td>-0.775827</td>\n",
       "      <td>-0.205724</td>\n",
       "      <td>0.139444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 01:00:00</th>\n",
       "      <td>0.547177</td>\n",
       "      <td>-0.224580</td>\n",
       "      <td>-0.479430</td>\n",
       "      <td>0.953178</td>\n",
       "      <td>-1.757995</td>\n",
       "      <td>-0.867747</td>\n",
       "      <td>0.179257</td>\n",
       "      <td>-0.272971</td>\n",
       "      <td>0.446103</td>\n",
       "      <td>-0.517913</td>\n",
       "      <td>-0.986904</td>\n",
       "      <td>0.995063</td>\n",
       "      <td>-0.861692</td>\n",
       "      <td>-0.058215</td>\n",
       "      <td>0.139444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 02:00:00</th>\n",
       "      <td>0.739905</td>\n",
       "      <td>-0.770240</td>\n",
       "      <td>-0.574831</td>\n",
       "      <td>-0.192518</td>\n",
       "      <td>-1.912116</td>\n",
       "      <td>-1.114317</td>\n",
       "      <td>0.183266</td>\n",
       "      <td>-0.822786</td>\n",
       "      <td>0.770174</td>\n",
       "      <td>-0.876490</td>\n",
       "      <td>-1.013548</td>\n",
       "      <td>0.554085</td>\n",
       "      <td>-0.846222</td>\n",
       "      <td>-0.220012</td>\n",
       "      <td>0.139444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 03:00:00</th>\n",
       "      <td>0.699098</td>\n",
       "      <td>-0.278783</td>\n",
       "      <td>-0.324709</td>\n",
       "      <td>0.325491</td>\n",
       "      <td>-1.809045</td>\n",
       "      <td>-0.783042</td>\n",
       "      <td>-0.378111</td>\n",
       "      <td>0.341156</td>\n",
       "      <td>0.621194</td>\n",
       "      <td>-0.290211</td>\n",
       "      <td>-0.826469</td>\n",
       "      <td>-0.211185</td>\n",
       "      <td>-0.404306</td>\n",
       "      <td>0.218373</td>\n",
       "      <td>0.139444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 04:00:00</th>\n",
       "      <td>0.223933</td>\n",
       "      <td>-0.225168</td>\n",
       "      <td>-0.313432</td>\n",
       "      <td>0.201600</td>\n",
       "      <td>-1.338802</td>\n",
       "      <td>-0.484910</td>\n",
       "      <td>0.072745</td>\n",
       "      <td>1.023019</td>\n",
       "      <td>0.467629</td>\n",
       "      <td>-0.478080</td>\n",
       "      <td>-0.601238</td>\n",
       "      <td>1.016033</td>\n",
       "      <td>-0.371487</td>\n",
       "      <td>0.097253</td>\n",
       "      <td>0.139444</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          bt_mean    bt_std  temperature_mean  \\\n",
       "period  timedelta                                               \n",
       "train_a 0 days 00:00:00  0.499705  2.443614         -0.375267   \n",
       "        0 days 01:00:00  0.547177 -0.224580         -0.479430   \n",
       "        0 days 02:00:00  0.739905 -0.770240         -0.574831   \n",
       "        0 days 03:00:00  0.699098 -0.278783         -0.324709   \n",
       "        0 days 04:00:00  0.223933 -0.225168         -0.313432   \n",
       "\n",
       "                         temperature_std  bx_gse_mean  bx_gse_std  \\\n",
       "period  timedelta                                                   \n",
       "train_a 0 days 00:00:00         0.383941    -1.599207   -0.381502   \n",
       "        0 days 01:00:00         0.953178    -1.757995   -0.867747   \n",
       "        0 days 02:00:00        -0.192518    -1.912116   -1.114317   \n",
       "        0 days 03:00:00         0.325491    -1.809045   -0.783042   \n",
       "        0 days 04:00:00         0.201600    -1.338802   -0.484910   \n",
       "\n",
       "                         by_gse_mean  by_gse_std  bz_gse_mean  bz_gse_std  \\\n",
       "period  timedelta                                                           \n",
       "train_a 0 days 00:00:00     0.419516    0.031658     0.300358   -0.651645   \n",
       "        0 days 01:00:00     0.179257   -0.272971     0.446103   -0.517913   \n",
       "        0 days 02:00:00     0.183266   -0.822786     0.770174   -0.876490   \n",
       "        0 days 03:00:00    -0.378111    0.341156     0.621194   -0.290211   \n",
       "        0 days 04:00:00     0.072745    1.023019     0.467629   -0.478080   \n",
       "\n",
       "                         speed_mean  speed_std  density_mean  density_std  \\\n",
       "period  timedelta                                                           \n",
       "train_a 0 days 00:00:00   -0.738546   0.862524     -0.775827    -0.205724   \n",
       "        0 days 01:00:00   -0.986904   0.995063     -0.861692    -0.058215   \n",
       "        0 days 02:00:00   -1.013548   0.554085     -0.846222    -0.220012   \n",
       "        0 days 03:00:00   -0.826469  -0.211185     -0.404306     0.218373   \n",
       "        0 days 04:00:00   -0.601238   1.016033     -0.371487     0.097253   \n",
       "\n",
       "                         smoothed_ssn  \n",
       "period  timedelta                      \n",
       "train_a 0 days 00:00:00      0.139444  \n",
       "        0 days 01:00:00      0.139444  \n",
       "        0 days 02:00:00      0.139444  \n",
       "        0 days 03:00:00      0.139444  \n",
       "        0 days 04:00:00      0.139444  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features, scaler = preprocess_features(solar_wind, sunspots, subset=SOLAR_WIND_FEATURES)\n",
    "print(features.shape)\n",
    "features.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check to make sure missing values are filled\n",
    "assert (features.isna().sum() == 0).all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>t0</th>\n",
       "      <th>t1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">train_a</th>\n",
       "      <th>0 days 00:00:00</th>\n",
       "      <td>-7</td>\n",
       "      <td>-10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 01:00:00</th>\n",
       "      <td>-10</td>\n",
       "      <td>-10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 02:00:00</th>\n",
       "      <td>-10</td>\n",
       "      <td>-6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 03:00:00</th>\n",
       "      <td>-6</td>\n",
       "      <td>-2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 04:00:00</th>\n",
       "      <td>-2</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         t0    t1\n",
       "period  timedelta                \n",
       "train_a 0 days 00:00:00  -7 -10.0\n",
       "        0 days 01:00:00 -10 -10.0\n",
       "        0 days 02:00:00 -10  -6.0\n",
       "        0 days 03:00:00  -6  -2.0\n",
       "        0 days 04:00:00  -2   3.0"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "YCOLS = [\"t0\", \"t1\"]\n",
    "\n",
    "\n",
    "def process_labels(dst):\n",
    "    y = dst.copy()\n",
    "    y[\"t1\"] = y.groupby(\"period\").dst.shift(-1)\n",
    "    y.columns = YCOLS\n",
    "    return y\n",
    "\n",
    "\n",
    "labels = process_labels(dst)\n",
    "labels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>t0</th>\n",
       "      <th>t1</th>\n",
       "      <th>bt_mean</th>\n",
       "      <th>bt_std</th>\n",
       "      <th>temperature_mean</th>\n",
       "      <th>temperature_std</th>\n",
       "      <th>bx_gse_mean</th>\n",
       "      <th>bx_gse_std</th>\n",
       "      <th>by_gse_mean</th>\n",
       "      <th>by_gse_std</th>\n",
       "      <th>bz_gse_mean</th>\n",
       "      <th>bz_gse_std</th>\n",
       "      <th>speed_mean</th>\n",
       "      <th>speed_std</th>\n",
       "      <th>density_mean</th>\n",
       "      <th>density_std</th>\n",
       "      <th>smoothed_ssn</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">train_a</th>\n",
       "      <th>0 days 00:00:00</th>\n",
       "      <td>-7</td>\n",
       "      <td>-10.0</td>\n",
       "      <td>0.499705</td>\n",
       "      <td>2.443614</td>\n",
       "      <td>-0.375267</td>\n",
       "      <td>0.383941</td>\n",
       "      <td>-1.599207</td>\n",
       "      <td>-0.381502</td>\n",
       "      <td>0.419516</td>\n",
       "      <td>0.031658</td>\n",
       "      <td>0.300358</td>\n",
       "      <td>-0.651645</td>\n",
       "      <td>-0.738546</td>\n",
       "      <td>0.862524</td>\n",
       "      <td>-0.775827</td>\n",
       "      <td>-0.205724</td>\n",
       "      <td>0.139444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 01:00:00</th>\n",
       "      <td>-10</td>\n",
       "      <td>-10.0</td>\n",
       "      <td>0.547177</td>\n",
       "      <td>-0.224580</td>\n",
       "      <td>-0.479430</td>\n",
       "      <td>0.953178</td>\n",
       "      <td>-1.757995</td>\n",
       "      <td>-0.867747</td>\n",
       "      <td>0.179257</td>\n",
       "      <td>-0.272971</td>\n",
       "      <td>0.446103</td>\n",
       "      <td>-0.517913</td>\n",
       "      <td>-0.986904</td>\n",
       "      <td>0.995063</td>\n",
       "      <td>-0.861692</td>\n",
       "      <td>-0.058215</td>\n",
       "      <td>0.139444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 02:00:00</th>\n",
       "      <td>-10</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>0.739905</td>\n",
       "      <td>-0.770240</td>\n",
       "      <td>-0.574831</td>\n",
       "      <td>-0.192518</td>\n",
       "      <td>-1.912116</td>\n",
       "      <td>-1.114317</td>\n",
       "      <td>0.183266</td>\n",
       "      <td>-0.822786</td>\n",
       "      <td>0.770174</td>\n",
       "      <td>-0.876490</td>\n",
       "      <td>-1.013548</td>\n",
       "      <td>0.554085</td>\n",
       "      <td>-0.846222</td>\n",
       "      <td>-0.220012</td>\n",
       "      <td>0.139444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 03:00:00</th>\n",
       "      <td>-6</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>0.699098</td>\n",
       "      <td>-0.278783</td>\n",
       "      <td>-0.324709</td>\n",
       "      <td>0.325491</td>\n",
       "      <td>-1.809045</td>\n",
       "      <td>-0.783042</td>\n",
       "      <td>-0.378111</td>\n",
       "      <td>0.341156</td>\n",
       "      <td>0.621194</td>\n",
       "      <td>-0.290211</td>\n",
       "      <td>-0.826469</td>\n",
       "      <td>-0.211185</td>\n",
       "      <td>-0.404306</td>\n",
       "      <td>0.218373</td>\n",
       "      <td>0.139444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 04:00:00</th>\n",
       "      <td>-2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.223933</td>\n",
       "      <td>-0.225168</td>\n",
       "      <td>-0.313432</td>\n",
       "      <td>0.201600</td>\n",
       "      <td>-1.338802</td>\n",
       "      <td>-0.484910</td>\n",
       "      <td>0.072745</td>\n",
       "      <td>1.023019</td>\n",
       "      <td>0.467629</td>\n",
       "      <td>-0.478080</td>\n",
       "      <td>-0.601238</td>\n",
       "      <td>1.016033</td>\n",
       "      <td>-0.371487</td>\n",
       "      <td>0.097253</td>\n",
       "      <td>0.139444</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         t0    t1   bt_mean    bt_std  temperature_mean  \\\n",
       "period  timedelta                                                         \n",
       "train_a 0 days 00:00:00  -7 -10.0  0.499705  2.443614         -0.375267   \n",
       "        0 days 01:00:00 -10 -10.0  0.547177 -0.224580         -0.479430   \n",
       "        0 days 02:00:00 -10  -6.0  0.739905 -0.770240         -0.574831   \n",
       "        0 days 03:00:00  -6  -2.0  0.699098 -0.278783         -0.324709   \n",
       "        0 days 04:00:00  -2   3.0  0.223933 -0.225168         -0.313432   \n",
       "\n",
       "                         temperature_std  bx_gse_mean  bx_gse_std  \\\n",
       "period  timedelta                                                   \n",
       "train_a 0 days 00:00:00         0.383941    -1.599207   -0.381502   \n",
       "        0 days 01:00:00         0.953178    -1.757995   -0.867747   \n",
       "        0 days 02:00:00        -0.192518    -1.912116   -1.114317   \n",
       "        0 days 03:00:00         0.325491    -1.809045   -0.783042   \n",
       "        0 days 04:00:00         0.201600    -1.338802   -0.484910   \n",
       "\n",
       "                         by_gse_mean  by_gse_std  bz_gse_mean  bz_gse_std  \\\n",
       "period  timedelta                                                           \n",
       "train_a 0 days 00:00:00     0.419516    0.031658     0.300358   -0.651645   \n",
       "        0 days 01:00:00     0.179257   -0.272971     0.446103   -0.517913   \n",
       "        0 days 02:00:00     0.183266   -0.822786     0.770174   -0.876490   \n",
       "        0 days 03:00:00    -0.378111    0.341156     0.621194   -0.290211   \n",
       "        0 days 04:00:00     0.072745    1.023019     0.467629   -0.478080   \n",
       "\n",
       "                         speed_mean  speed_std  density_mean  density_std  \\\n",
       "period  timedelta                                                           \n",
       "train_a 0 days 00:00:00   -0.738546   0.862524     -0.775827    -0.205724   \n",
       "        0 days 01:00:00   -0.986904   0.995063     -0.861692    -0.058215   \n",
       "        0 days 02:00:00   -1.013548   0.554085     -0.846222    -0.220012   \n",
       "        0 days 03:00:00   -0.826469  -0.211185     -0.404306     0.218373   \n",
       "        0 days 04:00:00   -0.601238   1.016033     -0.371487     0.097253   \n",
       "\n",
       "                         smoothed_ssn  \n",
       "period  timedelta                      \n",
       "train_a 0 days 00:00:00      0.139444  \n",
       "        0 days 01:00:00      0.139444  \n",
       "        0 days 02:00:00      0.139444  \n",
       "        0 days 03:00:00      0.139444  \n",
       "        0 days 04:00:00      0.139444  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = labels.join(features)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_train_test_val(data, test_per_period, val_per_period):\n",
    "    \"\"\"Splits data across periods into train, test, and validation\"\"\"\n",
    "    # assign the last `test_per_period` rows from each period to test\n",
    "    test = data.groupby(\"period\").tail(test_per_period)\n",
    "    interim = data[~data.index.isin(test.index)]\n",
    "    # assign the last `val_per_period` from the remaining rows to validation\n",
    "    val = interim.groupby(\"period\").tail(val_per_period)\n",
    "    # the remaining rows are assigned to train\n",
    "    train = interim[~interim.index.isin(val.index)]\n",
    "    return train, test, val\n",
    "\n",
    "\n",
    "train, test, val = get_train_test_val(data, test_per_period=6_000, val_per_period=3_000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7fdcb7e245e0>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ0AAAEYCAYAAACHoivJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAApMklEQVR4nO3de7hVVb3/8fenjYCKAiGeOGJuNBUvKCBqYSqm3bQUbwlmB6LyqJ3ILEvzV1nmOZ7ydLHU7qKmYWl511ITNU25KAqkeIl9EsMbysULHMDv748xNi4W+7LYrDW3e+3P63n2s+cac8w5x5hrzfWdY8yx5lREYGZmVoS3dXYBzMys+3DQMTOzwjjomJlZYRx0zMysMA46ZmZWGAcdMzMrjINOBSRFBX9NG7mNiXk9jRuxjvfkdXwt/9+njbz3SFogSRWue0ppHSU15m1MrGDZJklTKtlO2XJjJZ3WQvqYvO0xG7rODpRhvKTlkhZV8jmowvYaJZ0tafsK8/eS9AVJD+dyLpP0mKRLJe3Yge2v9zksf//y/j9bUrf5/sif/9L3+gVJd0v6UBW3UfEx1Rnrq5YenV2ALuI9Za//ADwMnF2StnIjt3FT3s6ijVjHWGAm8FPg68AngOnlmSQNAfYDzomO/1BrEam8T3Vw+UqMBQ4BvleW/mDe9t9quO3SMtwK/CfQqyT9IqAB+Pcqb68R+AbwF+DvFeT/DfAB4DvA/blMuwDHArsCT1ShTEcCy0pej8ll/DbwRhXW31W8AByep98BfBG4WdL7I+KOKqy/iGOq0znoVCAi7i99LWkl8GJ5elmeBkARsbrCbbxA+lBvjCOAyyPieUm3AuMknRYRq8ry/Rsg4LKObigiVpK+5AoXEcuK2LaknsCHgFMi4qGyecuAHm19Bmott4aOBE6NiB+WzLoF+F61WiLlda9Xknrlz3Vr/q/0/Zb0Z+AfwOeBDged3NuwSWceU0XqNs3jWsvN2HMlnSFpAfB/wDBJvSV9X9JcSa9IelbSDZKGli3fWrfGryWNk/SopFclzZT03ha2PxTYGbg2J10KbAV8uIXingDcGxFPSXqXpMtzV9vrkv4u6WJJ/dupb4tNd0mfz+Vekcu6fwvLDpT0U0mPS3pN0tOSrpS0TUmeKcAEYBuVdWG21L2m5AuS5kv6v9wd9mNJW5ZtOyR9W9LkXOflku6StFsL1TwY2JTUCm2XpK3yvntG0srczXViWZ535K6vf+Y8iyTdKGnrXJ87c9bbSuo9hpa9Pf9/tqWZEbG2FZK7hxZKGi1pRn5/miR9roJ6re1ek3Q2qZUDsEol3YqSekg6R9JTef0vSvpLS5/XsvW3+95JmifpmhaW3TeXYWxJ2p6Srpf0cv5M31v+OSzZH++RdJ+k10mtxYrlk5/HgXeVrPcoSffnz/USSb+T9M6ybTcf15MkPUb6rjisjWPqBKXu0+Z9ermkQWV5NpN0kaTFSt8z1wODW9hfe0u6Led7Tel4v2hD6r2x3NKpromkLpEvAa8C/yR1yWxB6opYRPqiOAW4X9LQiGjxC6PE/qRg8jVgBXAOcKOkxohYUpLvSODJiJiXX98AvEzqYru+OZOk0aSDpPkA+1dgIXBqzr898FXgZtbvVmyTpE8BPwCmAFfl7fyGVP9Sb891OZPUuvtXUlfFvXmfNNdzILA3b3ZptHUWem5e34Wkuu+a17GnpANLv4BJQXc+6Qy1J/Bd4Lq87dKW6VjgrrL93FrdtwTuJQWps4EFwAeBi5XOoH+Us14ObAecDjwN/AspuG1G6jb8bK7DZGBGXqa1bsTHSN1e50naBLgtIp5ro5hbkt6X/waeBMYBF0haHhFT2qtj9gvSl9mngPcCa0rmfQX4AnAWMDtvbxRvBsfWVPLeXQ58U1L/iHi5ZNkTgJdIn1ckjQTuAR4CPgO8BpwE3C5pdETMKlm2LzAVOJ/0mX+9wn1A3lYPYFvSe42kk4CLgUuAb5E+92cDd0naIyKWlyx+EDAc+CbwPNDUyjZOJHWXX5X30b+Sunr3lTQyIl7JWX8KHJfXNwN4P3Bl2br6AH8kdblPBJaTunNHb0i9N1pE+G8D/0gfkF+XpQUpyGzazrINpC+Y5cAXStIn5nU0lm3nZaB/SdqonO/4svU+AHy3LO0i0pd7v5K0n5AOrr6tlK8H6cskgBEl6VOAppLXjTnPxPz6baQv0VvL1ndczjelnX2ybc53ZNk2F7aQf0zOOya/bg5iU8rynZDzHV72Pj1B6s5oTjsmp48uSVN+Pz/bSpmnAX8ped18UrBjWb6fAy+SuuIAXgEmt7Evmut2SIWfxY+SAnfkv6eAHwNDy/JNyfPHlaXfBvwvqSu4rc/hlJLXZ+c8PcrWdSPw+w08lip67/LnYw3w7yV5Nsl1v6gk7Q7gUaBn2efrUeDaFvbHERWWcwrp5KxH/htM+qIP0glbH2Ap8Kuy5RpJLZlTy/bna8A7Wshbekw1AM8Bd5blaz4+J+fXO+d9c0ZZvovL1tf83bHHhrxH1f5z91p13RoR650tSfqYpAckLQFWk1pBfUgflvb8NdY9s5uT/69tsuem9t682bXW7FJSS+tjOV/z9HURsTSn9ZT01dwV9DqwinSmSIXlazY4//22LP0aUp3XIenk3GXwSp7/jw5ss9m7SfX8dVn61LzuA8vSb4t1r3Ott0/zOgcB11VYhg+RAv+C3M3UI58J/xEYQDp7h3QWerpSN+QwqbLRg62JiBtIX1ZHAT8ClpBa0g9JOqQs+xrS+1FqKqne27DxZgCHKnUzv1fpmlh7KnrvIuJp4C5Sy73Zh0hdyJcBSNo05/8d8EbJeyDgduCAsm2sJgXKSm1DOj5WkU6wjicN2LmA1CuwJXBF2fu/kNQiLd/2/dF+L8fOwNbAFaWJEfEX0olC8+d6X9JJX/mxN7Xs9ROkz8dPc5fdtu1svyYcdKprvZFnkj5Kaho/SvqQ7ksKEC8AvStY50ulL+LNC52ly47N6/trWd4HSB/4f8tJHwX6k4JRs/8inbn+GjgM2If0BVa+jfY09zGv070TqbtqcWlavo5wEemL4Ki8zXd3YJvNmrtv1tn/Jdsu7955qex1a/t0ZkQsrLAMW5O+WFaV/f0uzx+Q/x9H6u78MvAI8Iykr2sjLvpHxKsR8YeImBwRe5G6S9YA55VlfTnWH1TS/H5VI+j8J+l6z+GkE5fFki6RtFUby2zIe3cZsJ/S6EtIAejJePPi/ttJrYOvsf778B9A/7L9/HxElHYPtud50rE7ChhC6kE4J1L339Y5z+0tbHsYb77/zSoZpdrivsmeLZnf4rFX/jqfaB5EasFfBPxD6Vrz0RWUpWp8Tae6Whp+PI50YExsTsj97+31c2+IscD1se51i2aXAf+pNNLpE6QP65/KyndZRHy7pHx9OlCG5gPjX0oT89le+QE3DrgjIr5Ykm8IHdccRN4BNF/TKt324pYWascRpOsIlVpM+lL6fCvz5wNExPOk6zaflbQzabDEN0knDRd3oJzriYj7Jf2J1BIo1V/SJmWBp/n9eqYK211Ful7035LeAXyENNx9M1KwbcmGvHfXkK77nCDph6STqP8qmb+ENIT7QloZmVl2jGzozwVWRcTMVuY1l3MiJfUosbzsdSXbLt035d5B+nkErHvslQ6zX+dYBIiI2cDRef+OIl0n+q2kPSNibgVl2mhu6dTeZqzfvfQJ0hnZRssXsMewftdas8tJB+KppJFsV5Sd3W1GOhsr9ckOFGUhqcvhY2XpR7P+yU2l21xJujDfnvtz3nFl6cflbd9VwTrW0vojAStxKzAU+EdEzGzhr/xLh4iYHxFfJV232z0nN7e62q23pC1UNjovpzcAO7L+GXID6f0oNY7UtbkhQafdMkbEsxHxC9KZ/+6t5WMD3ru8D68jHT/Hklqml5fMf5XUwtoTeLCl96HC+nXEfaTA8q5W3v/5HVjnfFJrZZ19kwcDbceb++YB0jFefuyV79O1ImJ1biF+jRQHdulA+TrELZ3auxUYK+n7pP7jvUgjk5ZUaf2HkS5U3t7SzIhYqPR7gv8g9W1fWpblVmCCpDmkEU1H0YHRLBHxhqRvAr+QdAmpP/ldpDOpZWXZbwW+IumrpJE07yNdzC/3N+Dtkk4mndWtiIg55Zki4iVJ3wPOlPQqaSTTLqQRg3+hwiHPJcpHAlbi+6Qvynvyez0f2JwUiPaPiCMk9SW9T1eQuj1XkVpU/Xmz9fk46SRlkqSXSF/I81sKWqTAeKeky0kX0J8ndbV8mvRFf0pZ/uXAd3J31xPAeNKPbydGvtJcoebRdF+UdAuwJiJmSrqO9KPpB0mBdASptfXT1lbUgffuslzub5IGciwom38acDfwR0m/JAXerYCRQENEnLEB9axYRCyTdDpwoaSBpN9KLSV1Wx4ITIuIK9taRwvrXCPp66RrML8mdYFvQxrt9wRplBwRMV/SlcC3cvdh8+i1Q0vXJ+kjwImkk6kFpM/nZNLnYp2u+ZrqzFEMXfWP1kevfbuFvG8jHUD/JI1YuYt0MDax7oigibQ8aujXLawzgLPz9FXA79opb/NIoIdamLcVKUC8nP+uIPVbrx31kvNNoY3RayXpnydd5FxBChTvbaGum5K6kl4gfeBvJPWRr61Xzrc5acj1y3leU04fQ8notZwm0nDd+aQgvIjUzbJle+9TeV1oYSRgC/ttGiWj13Jaf1LwWZDL8DzpzPvUPL8X6Qt4HmkU2zLSF0T5SMR/J3WTrC6vZ1m+fqQL2Xfn+q7K++pO4JiyvFNIrdHReZsr8vs0uSzfRNofvdaQ9+3zpDPsyOlfJLVcFpNGSM4nXS/cpJ19WdF7V7LtRbmMJ7ayvl1In+nnSUF7Iek62qHl+2MDjvmK8pO+6O/M7+3rpBO5XwG7VnBcr/M5LDt+H851WUxq3Q0qy7MZ6Zh6KX+2rifddaT0c70z6ftiQX7/XyAF+X0r3Q/V+GseJmldUB4d9ALpF/NXtJff2pdHAj5Dap3c29nlqRalH3ceEhHr/WDQrEjuXuvCIuL/SD9wsyqJiEX4WqdZzfjgMjOzwrh7zczMCuOWjpmZFaZbX9PZaqutorGxsbOLYWbWpcyaNevFiBjYkWW7ddBpbGxk5sxa/l7MzKz+SPrfji7r7jUzMyuMg46ZmRXGQcfMzArjoGNmZoVx0DEzs8I46JiZWWEcdMzMrDAOOmZmVphu/ePQOc8spfGMDX2+l5l1Z029j9+o5YcNeWeVStJxcyas9yzEwrilY2ZmhXHQMTOzwjjomJlZYRx0zMysMA46ZmZWGAcdMzMrjIOOmZkVxkHHzMwK46BjZmaFcdAxM7PCOOiYmVlhHHTMzKwwDjpmZlYYBx0zMyuMg46ZmRXGQcfMzArjoGNmZoVx0DEzs8I46JiZWWEcdMzMrDAOOmZmVhgHHTMzK4yDjpmZFcZBx8zMClOzoCOpn6RTOrDczZL61aBIZmbWyWrZ0ukHrBd0JDW0tVBEHBoRS2pUJjMz60Q9arju84AdJM0GVgGvAIuA4cCukq4FtgV6Az+MiJ8BSGoCRgF9gFuAvwCjgWeAIyLi9ZY2JuldwE+AgcAa4NiIeKqFfCcCJwI0bDmwKhU1M7PK1LKlcwbwVEQMB04H9gHOiohd8/xJEbEXKcBMljSghXXsCFwYEbsBS4Cj29jeFTnvnqQgtailTBHxs4gYFRGjGjbr24FqmZlZR9WypVNuekQsKHk9WdKReXpbUoBZXLbMgoiYnadnAY0trVjSFsA2EfEHgIhYUa1Cm5lZ9RQZdF5tnpA0BjgEeE9EvCZpGqmbrdzKkuk1wKatrFvVKaKZmdVSLbvXlgNbtDKvL/ByDjhDgXdvzIYiYhmwUNJYAEm9JG22Mes0M7Pqq1nQiYjFwL2S5gLfLZt9K9BD0iPAOcD9VdjkJ0hddo8A9wHvqMI6zcysimravRYRx7eSvhL4cCvzGvPki8DuJennt7OtJ4D3daigZmZWCN+RwMzMClPkQIKqkHQhsF9Z8g8j4pLOKI+ZmVWuywWdiPhsZ5fBzMw6xt1rZmZWGAcdMzMrjIOOmZkVxkHHzMwK46BjZmaFcdAxM7PCOOiYmVlhHHTMzKwwDjpmZlYYBx0zMyuMg46ZmRXGQcfMzArjoGNmZoVx0DEzs8I46JiZWWEcdMzMrDAOOmZmVpgu9+TQahq2TV9mnndYZxfDzLqUpRu19JwqlaKrckvHzMwK46BjZmaFcdAxM7PCOOiYmVlhHHTMzKwwDjpmZlYYBx0zMyuMg46ZmRXGQcfMzArjoGNmZoVx0DEzs8I46JiZWWEcdMzMrDAOOmZmVhgHHTMzK4yDjpmZFcZBx8zMCtOtnxw655mlNJ5xU2cXw8w2QFPv4zdq+WFD3lmlknTcnAnd9/mhbumYmVlhHHTMzKwwDjpmZlYYBx0zMyuMg46ZmRXGQcfMzArjoGNmZoVx0DEzs8I46JiZWWEcdMzMrDAOOmZmVhgHHTMzK4yDjpmZFabNu0xLGtnW/Ih4sLrFMTOzetbeow3+J//vDYwCHgYE7AE8ALy3dkUzM7N602b3WkQcFBEHAf8LjIyIURGxFzACeLKIApqZWf2o9JrO0IhY+9ShiJgLDK9JiczMrG5V+uTQRyX9Avg1EMAJwKM1K5WZmdWlSoPOJ4GTgc/n13cDF9ekRGZmVrcqCjoRsULShcDtpJbO/IhYVdOSmZlZ3ako6EgaA1wKNJFGr20raUJE3F2zkpmZWd2ptHvtf4APRMR8AEk7Ab8B9qpVwczMrP5UOnptk+aAAxARjwOb1KZIZmZWrypt6cyU9Evg8vz648Cs2hTJzMzqVaUtnZOBecBk0gi2vwEntZZZUj9Jp2xoYSTdLKlfB5abJmnUhi5nZmbFqnT02krge/mvEv2AU4CLShMlNUTEmja2c2iF6zczsy6ovRt+/jYiPiZpDmmo9DoiYo9WFj0P2EHSbGAV8AqwiHQXg10lXQtsS7qn2w8j4md5e02ke7z1AW4B/gKMBp4BjoiI19so7gmSLgC2BCZFxPRW6nQicCJAw5YD21idmZlVW3stneYfg35kA9d7BrB7RAzPw61vyq8X5PmTIuIlSZsCMyRdExGLy9axIzA+Ij4j6bfA0aQ7IrRm84gYLekA4FfA7i1lygHuZwC9Bu24XiA1M7PaaTPoRMQiSQ3ALyPikI3YzvSSgAMwWdKReXpbUoApDzoLImJ2np4FNLazjd/kMt8taUtJ/SJiyUaU2czMqqzdgQT5GsxrkvpuxHZebZ7ILZ9DgPdExJ7AQ6RutnIrS6bX0H6rrLzV4laMmdlbTKVDplcAcyTdRkkAiYjJreRfDmzRyry+wMsR8ZqkocC7Ky1sO44D7pT0XmBpRCyt0nrNzKxKKg06N+W/ikTEYkn3SpoLvA48VzL7VuAkSY8A84H7K11vO16WdB95IEGV1mlmZlVU6ZDpS/NF/3eW3pmgnWWObyV9JfDhVuY15skXKRkIEBHnt7OtMZWUyczMOldFPw6V9FFgNqmVgqThkq6vYbnMzKwOVXpHgrOBfYAlAHlU2ZCalKgNki6UNLvs75NFl8PMzDqm0ms6qyNiqaTStMJHh0XEZ4veppmZVU+lQWeupOOBBkk7ku7Bdl/timVmZvWo0u61zwG7kX47cyWwFDi1RmUyM7M61d6913qT7ib9LmAO6Qedq4somJmZ1Z/2WjqXkm7AOYc0zLnNoctmZmZtae+azq4RMQwgP8StxTs3m5mZVaK9ls6q5gl3q5mZ2cZqr6Wzp6RleVrApvm1gIiILWtaOjMzqyvtPdqgoaiCmJlZ/at0yLSZmdlGc9AxM7PCOOiYmVlhHHTMzKwwDjpmZlYYBx0zMyuMg46ZmRXGQcfMzArjoGNmZoWp9CFudWnYNn2Zed5hnV0MM9sgSzdq6TlVKoV1jFs6ZmZWGAcdMzMrjIOOmZkVxkHHzMwK46BjZmaFcdAxM7PCOOiYmVlhHHTMzKwwDjpmZlYYBx0zMyuMg46ZmRXGQcfMzArjoGNmZoVx0DEzs8I46JiZWWEcdMzMrDAOOmZmVphu/eTQOc8spfGMmzq7GFahpt7Hb9Tyw4a8s0ol6bg5E/zcSuve3NIxM7PCOOiYmVlhHHTMzKwwDjpmZlYYBx0zMyuMg46ZmRXGQcfMzArjoGNmZoVx0DEzs8I46JiZWWG69W1wzKx+rFq1ioULF7JixYrOLkrd6N27N4MHD2aTTTap2joddMysLixcuJAtttiCxsZGJHV2cbq8iGDx4sUsXLiQIUOGVG297l4zs7qwYsUKBgwY4IBTJZIYMGBA1VuODjpmVjcccKqrFvvTQcfMzArjazpmVpeq/ayspvMOa3P+4sWLOfjggwF49tlnaWhoYODAgQBMnz6dnj17trrszJkzueyyy7jggguqV+C3KAcdM7MqGDBgALNnzwbg7LPPpk+fPnzpS19aO3/16tX06NHyV+6oUaMYNWpUEcXsdO5eMzOrkYkTJ3Laaadx0EEH8ZWvfIXp06czevRoRowYwejRo5k/fz4A06ZN4yMf+QiQAtakSZMYM2YM22+/fd21ftzSMTOroccff5zbb7+dhoYGli1bxt13302PHj24/fbb+epXv8o111yz3jKPPfYYd955J8uXL2fnnXfm5JNPrupvZTqTg46ZWQ0de+yxNDQ0ALB06VImTJjAE088gSRWrVrV4jKHHXYYvXr1olevXmy99dY899xzDB48uMhi14y718zMamjzzTdfO/21r32Ngw46iLlz53LDDTe0+huYXr16rZ1uaGhg9erVNS9nURx0zMwKsnTpUrbZZhsApkyZ0rmF6STuXjOzutTeEOfO8OUvf5kJEybwve99j/e9732dXZxOoYjo7DJ0ml6DdoxBE37Q2cWwCjX1Pn6jlh825J1VKknHzZkwp7OLULceffRRdtlll84uRt1pab9KmhURHRrjXbPuNUn9JJ3SgeVultSvBkUyM7NOVstrOv2A9YKOpIa2FoqIQyNiSY3KZGZmnaiWQec8YAdJsyXNkHSnpCuBOQCSrpU0S9I8SSc2LySpSdJWkholPSrp5znPnyRt2trGJH0mb+dhSddI2qyVfCdKmilp5prXlla7zmZm1oZaBp0zgKciYjhwOrAPcFZE7JrnT4qIvYBRwGRJA1pYx47AhRGxG7AEOLqN7f0+IvaOiD2BR4FPtZQpIn4WEaMiYlTDZn07Ui8zM+ugIkevTY+IBSWvJ0s6Mk9vSwowi8uWWRARs/P0LKCxjfXvLunbpG69PsAfN7bAZmZWXUUGnVebJySNAQ4B3hMRr0maBvRuYZmVJdNrgFa714ApwNiIeFjSRGDMRpXWzMyqrpZBZzmwRSvz+gIv54AzFHh3Fba3BbBI0ibAx4FnqrBOM+uqzq5y9/nZbV8DHjNmDGeeeSYf/OAH16b94Ac/4PHHH+eiiy5qMf/555/PqFGjOPTQQ7nyyivp16/fupts4W7V5a699lp22mkndt01Xbn4+te/zgEHHMAhhxyyAZUrTs2u6UTEYuBeSXOB75bNvhXoIekR4Bzg/ips8mvAA8BtwGNVWJ+ZWcXGjx/P1KlT10mbOnUq48ePb3fZm2++eb2AU6lrr72Wv/3tb2tff+tb33rLBhyo8W1wIuL4iNg9X+D/SEn6yoj4cETsERHHRsSYiJiW5zVGxIsR0RQRu5csc35EnN3Gti6OiCF5XZ+LiIk1rJqZ2TqOOeYYbrzxRlauTFcFmpqa+Oc//8mVV17JqFGj2G233fjGN77R4rKNjY28+OKLAJx77rnsvPPOHHLIIWsffQDw85//nL333ps999yTo48+mtdee4377ruP66+/ntNPP53hw4fz1FNPMXHiRK6++moA7rjjDkaMGMGwYcOYNGnS2rI1NjbyjW98g5EjRzJs2DAee6y483Tfe83MrAoGDBjAPvvsw6233gqkVs5xxx3Hueeey8yZM3nkkUe46667eOSRR1pdx6xZs5g6dSoPPfQQv//975kxY8baeUcddRQzZszg4YcfZpddduGXv/wlo0eP5vDDD+e73/0us2fPZocddlibf8WKFUycOJGrrrqKOXPmsHr1ai6++OK187faaisefPBBTj75ZM4///wa7JGWdbmgI+nC/Nuf0r9Pdna5zMxKu9iau9Z++9vfMnLkSEaMGMG8efPW6Qord88993DkkUey2WabseWWW3L44YevnTd37lz2339/hg0bxhVXXMG8efPaLMv8+fMZMmQIO+20EwATJkzg7rvvXjv/qKOOAmCvvfaiqampo1XeYF3uhp8R8dnOLoOZWUvGjh3LaaedxoMPPsjrr79O//79Of/885kxYwb9+/dn4sSJrT7OoJmkFtMnTpzItddey5577smUKVOYNm1am+tp776azY9PKPrRCV2upWNm9lbVp08fxowZw6RJkxg/fjzLli1j8803p2/fvjz33HPccsstbS5/wAEH8Ic//IHXX3+d5cuXc8MNN6ydt3z5cgYNGsSqVau44oor1qZvscUWLF++fL11DR06lKamJp588kkALr/8cg488MAq1bTjulxLx8ysIu0Mca6V8ePHc9RRRzF16lSGDh3KiBEj2G233dh+++3Zb7/92lx25MiRHHfccQwfPpztttuO/ffff+28c845h3333ZftttuOYcOGrQ0048aN4zOf+QwXXHDB2gEEAL179+aSSy7h2GOPZfXq1ey9996cdNJJtan0BvCjDfxogy7DjzawtvjRBrXRZR5tYGZmVs5Bx8zMCuOgY2ZmhXHQMTOzwjjomJlZYRx0zMysMP6djpnVpWGXDqvq+tob7r548WIOPvhgAJ599lkaGhoYOHAgANOnT6dnz55tLj9t2jR69uzJ6NGjq1PgtygHHTOzKhgwYACzZ88GKnsOTrlp06bRp0+fug867l4zM6uRWbNmceCBB7LXXnvxwQ9+kEWLFgFwwQUXsOuuu7LHHnswbtw4mpqa+MlPfsL3v/99hg8fzj333NPJJa8dt3TMzGogIvjc5z7Hddddx8CBA7nqqqs466yz+NWvfsV5553HggUL6NWrF0uWLKFfv36cdNJJG9w66oocdMzMamDlypXMnTuX97///QCsWbOGQYMGAbDHHnvw8Y9/nLFjxzJ27NhOLGXxHHTMzGogIthtt93461//ut68m266ibvvvpvrr7+ec845p91n49QTX9MxM6uBXr168cILL6wNOqtWrWLevHm88cYbPP300xx00EF85zvfYcmSJbzyyiutPqKg3rilY2Z1qbPv6P22t72Nq6++msmTJ7N06VJWr17Nqaeeyk477cQJJ5zA0qVLiQi+8IUv0K9fPz760Y9yzDHHcN111/GjH/1oncca1JNu/WiDUaNGxcyZMzu7GGZWBX60QW340QZmZtZlOeiYmVlhHHTMrG5058sFtVCL/emgY2Z1oXfv3ixevNiBp0oigsWLF9O7d++qrtej18ysLgwePJiFCxfywgsvdHZR6kbv3r0ZPHhwVdfpoGNmdWGTTTZhyJAhnV0Ma4e718zMrDAOOmZmVhgHHTMzK0y3viOBpOXA/M4uRw1sBbzY2YWokXqtW73WC1y3rqi9em0XEQM7suLuPpBgfkdv5fBWJmlmPdYL6rdu9VovcN26olrWy91rZmZWGAcdMzMrTHcPOj/r7ALUSL3WC+q3bvVaL3DduqKa1atbDyQwM7NidfeWjpmZFchBx8zMCtMtg46kD0maL+lJSWd0dnlaI+lXkp6XNLck7e2SbpP0RP7fv2TemblO8yV9sCR9L0lz8rwLJCmn95J0VU5/QFJjQfXaVtKdkh6VNE/S5+uhbpJ6S5ou6eFcr2/WQ71KytQg6SFJN9ZZvZpymWZLmllndesn6WpJj+Xj7T2dXreI6FZ/QAPwFLA90BN4GNi1s8vVSlkPAEYCc0vSvgOckafPAP47T++a69ILGJLr2JDnTQfeAwi4BfhwTj8F+EmeHgdcVVC9BgEj8/QWwOO5/F26brkMffL0JsADwLu7er1K6ncacCVwY718FvP2moCtytLqpW6XAp/O0z2Bfp1dt0Iq/lb6yzvujyWvzwTO7OxytVHeRtYNOvOBQXl6EOkHruvVA/hjrusg4LGS9PHAT0vz5OkepF8gqxPqeB3w/nqqG7AZ8CCwbz3UCxgM3AG8jzeDTpevV95eE+sHnS5fN2BLYEH5tjq7bt2xe20b4OmS1wtzWlfxLxGxCCD/3zqnt1avbfJ0efo6y0TEamApMKBmJW9Bbo6PILUKunzdchfUbOB54LaIqIt6AT8Avgy8UZJWD/UCCOBPkmZJOjGn1UPdtgdeAC7J3aK/kLQ5nVy37hh01EJaPYwbb61ebdW3U/eFpD7ANcCpEbGsrawtpL0l6xYRayJiOKllsI+k3dvI3iXqJekjwPMRMavSRVpIe8vVq8R+ETES+DDwWUkHtJG3K9WtB6l7/uKIGAG8SupOa00hdeuOQWchsG3J68HAPzupLB3xnKRBAPn/8zm9tXotzNPl6essI6kH0Bd4qWYlLyFpE1LAuSIifp+T66JuABGxBJgGfIiuX6/9gMMlNQFTgfdJ+jVdv14ARMQ/8//ngT8A+1AfdVsILMytbYCrSUGoU+vWHYPODGBHSUMk9SRd/Lq+k8u0Ia4HJuTpCaTrIc3p4/JokiHAjsD03HxeLundecTJv5Ut07yuY4A/R+6craVcjl8Cj0bE90pmdem6SRooqV+e3hQ4BHisq9crIs6MiMER0Ug6Xv4cESd09XoBSNpc0hbN08AHgLnUQd0i4lngaUk756SDgb/R2XWr9cWst+IfcChpxNRTwFmdXZ42yvkbYBGwinRG8SlSf+kdwBP5/9tL8p+V6zSfPLokp48iHUhPAT/mzTtR9AZ+BzxJGp2yfUH1ei+pCf4IMDv/HdrV6wbsATyU6zUX+HpO79L1KqvjGN4cSNDl60W67vFw/pvX/H1QD3XL2x4OzMyfyWuB/p1dN98Gx8zMCtMdu9fMzKyTOOiYmVlhHHTMzKwwDjpmZlYYBx0zMyuMg451K5JeKXs9UdKPa7X+NvINy3c1ni3pJUkL8vTtkg5XDe9+LmmspF1rtX6ztvTo7AKY1QNJDRGxptL8ETGH9BsKJE0h/fbl6pIstfzB8ljgRtIPBc0K5ZaOWSZpO0l3SHok/39nTp8i6ZiSfK/k/2OUngt0JTCnbF2XSzqi5PUVkg6vsBxrW1952xfn7fxd0oFKz1l6NAer5mU+IOmvkh6U9Lt8XzsknSfpb7lO50saDRwOfDe3rHbIf7fmG17eI2loybZ/ktMez/dgQ9JuSs8Nmp3Xu2NH9rd1T27pWHezqdJdoJu9nTdbFT8GLouISyVNAi4gtQrasg+we0QsKEv/BfAF4DpJfYHRvHm7kA3Vn/RIgcOBG0j3Qvs0MEPScNLdKv4fcEhEvCrpK8BpOXAdCQyNiJDULyKWSLqekpaVpDuAkyLiCUn7Ahfl7UF6tMaBwA7AnZLeBZwE/DAirsi3kmroYL2sG3LQse7m9Uh3gQZSq4J0iw9Izw45Kk9fTnrYVXumtxBwiIi7JF0oaeu8zmsi3fq9I27IQWMO8FzumkPSPFJQGEx6ANe96dZY9AT+CiwDVgC/kHQTqUttHblFNBr4XV4W0kO8mv02It4AnpD0d2BoXvdZkgYDv4+IJzpYL+uGHHTMWtd8j6jV5K7ofMPDniV5Xm1j+cuBj5NukjlpI8qxMv9/o2S6+XUPYA3p2T3jyxeUtA/pRo/jgP/gzRZMs7cBS0oDcZny+2RFRFwp6QHgMOCPkj4dEX/egPpYN+ZrOmZvuo/05QwpWPwlTzcBe+XpI0iPoq7EFOBUgIiYV40CtuJ+YL/c9YWkzSTtlFsxfSPi5lyO4Tn/ctJjwon0HKMFko7Ny0rSniXrPlbS2yTtQLo55nxJ2wN/j4gLSF2Te9SwblZnHHTM3jQZ+KSkR4BPAJ/P6T8HDpQ0nfT46bZaN2tFxHPAo8AlNShr6XZeACYCv8llv5/UDbYFcGNOu4t0jQnSM3FOV3qa5A6kAPspSc13Wj6iZPXz87K3kK77rACOA+bma2NDgctqWT+rL77LtFmNSNqMNKptZEQs7ezybKhWhnKbbRS3dMxqQFLzA9x+1BUDjlmtuKVjZmaFcUvHzMwK46BjZmaFcdAxM7PCOOiYmVlhHHTMzKww/x8gQmtC5xIu+gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ind = [0, 1, 2]\n",
    "names = [\"train_a\", \"train_b\", \"train_c\"]\n",
    "width = 0.75\n",
    "train_cnts = [len(df) for _, df in train.groupby(\"period\")]\n",
    "val_cnts = [len(df) for _, df in val.groupby(\"period\")]\n",
    "test_cnts = [len(df) for _, df in test.groupby(\"period\")]\n",
    "\n",
    "p1 = plt.barh(ind, train_cnts, width)\n",
    "p2 = plt.barh(ind, val_cnts, width, left=train_cnts)\n",
    "p3 = plt.barh(ind, test_cnts, width, left=np.add(val_cnts, train_cnts).tolist())\n",
    "\n",
    "plt.yticks(ind, names)\n",
    "plt.ylabel(\"Period\")\n",
    "plt.xlabel(\"Hourly Timesteps\")\n",
    "plt.title(\"Train/Validation/Test Splits over Periods\", fontsize=16)\n",
    "plt.legend([\"Train\", \"Validation\", \"Test\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import LSTM\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras import preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of train batches: 3523\n",
      "Number of val batches: 276\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "\n",
    "data_config = {\n",
    "    \"timesteps\": 32,\n",
    "    \"batch_size\": 32,\n",
    "}\n",
    "\n",
    "\n",
    "def timeseries_dataset_from_df(df, batch_size):\n",
    "    dataset = None\n",
    "    timesteps = data_config[\"timesteps\"]\n",
    "\n",
    "    # iterate through periods\n",
    "    for _, period_df in df.groupby(\"period\"):\n",
    "        # realign features and labels so that first sequence of 32 is aligned with the 33rd target\n",
    "        inputs = period_df[XCOLS][:-timesteps]\n",
    "        outputs = period_df[YCOLS][timesteps:]\n",
    "\n",
    "        period_ds = preprocessing.timeseries_dataset_from_array(\n",
    "            inputs,\n",
    "            outputs,\n",
    "            timesteps,\n",
    "            batch_size=batch_size,\n",
    "        )\n",
    "\n",
    "        if dataset is None:\n",
    "            dataset = period_ds\n",
    "        else:\n",
    "            dataset = dataset.concatenate(period_ds)\n",
    "\n",
    "    return dataset\n",
    "\n",
    "\n",
    "train_ds = timeseries_dataset_from_df(train, data_config[\"batch_size\"])\n",
    "val_ds = timeseries_dataset_from_df(val, data_config[\"batch_size\"])\n",
    "\n",
    "print(f\"Number of train batches: {len(train_ds)}\")\n",
    "print(f\"Number of val batches: {len(val_ds)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm_2 (LSTM)                (None, 512)               1081344   \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 2)                 1026      \n",
      "=================================================================\n",
      "Total params: 1,082,370\n",
      "Trainable params: 1,082,370\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# define our model\n",
    "model_config = {\"n_epochs\": 2, \"n_neurons\": 512, \"dropout\": 0.4, \"stateful\": False}\n",
    "\n",
    "model = Sequential()\n",
    "model.add(\n",
    "    LSTM(\n",
    "        model_config[\"n_neurons\"],\n",
    "        # usually set to (`batch_size`, `sequence_length`, `n_features`)\n",
    "        # setting the batch size to None allows for variable length batches\n",
    "        batch_input_shape=(None, data_config[\"timesteps\"], len(XCOLS)),\n",
    "        stateful=model_config[\"stateful\"],\n",
    "        dropout=model_config[\"dropout\"],\n",
    "    )\n",
    ")\n",
    "model.add(Dense(len(YCOLS)))\n",
    "model.compile(\n",
    "    loss=\"mean_squared_error\",\n",
    "    optimizer=\"adam\",\n",
    ")\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "3523/3523 [==============================] - 675s 191ms/step - loss: 426.8115 - val_loss: 469.7907\n",
      "Epoch 2/2\n",
      "3523/3523 [==============================] - 538s 153ms/step - loss: 355.4890 - val_loss: 363.1996\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(\n",
    "    train_ds,\n",
    "    batch_size=data_config[\"batch_size\"],\n",
    "    epochs=model_config[\"n_epochs\"],\n",
    "    verbose=1,\n",
    "    shuffle=False,\n",
    "    validation_data=val_ds,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAcwklEQVR4nO3deXCc9Z3n8fdXl2Vbki9Jlmwjy9iyfHAYMIRwxbkGLFNDsptkmQO2stSQqSWb2cmkILBVO2xNpXJMksnObjZZJpuZZGqAkIRkCJbxhCsBgiF2wmUs+cAHBtmWD7BlG9mSvvvH72l1S9bRklrd6kefV1WXpKefbv9+sfPVj8/ze75t7o6IiMRLQa4HICIimafiLiISQyruIiIxpOIuIhJDKu4iIjFUlOsBAFRWVnp9fX2uhyEikle2bNly2N2rBnpuQhT3+vp6Nm/enOthiIjkFTPbO9hzimVERGJIxV1EJIZU3EVEYkjFXUQkhlTcRURiSMVdRCSGVNxFRGIov4t7Zwc8dg/sfga6u3I9GhGRCWNC3MQ0agdehd9+DzZ9G0pnwtLrobEJlnwYppTnenQiIjmT38V94fvhzjdg15PQ2gzbH4NXfgSFJbDoulDoG5ugojbXIxURySqbCJ/EtHr1as9I+4HuLnjzhVDoW9bDsd3h+LxLQ5Ff1gTVK8Bs7H+WiEiOmdkWd1894HOxKu6p3KG9JSr0zfBW9P4zFyYLfd1VUJjf//EiIpPX5Czu/Z04EGKblmZ442no7kzJ6dfCko8opxeRvKLi3l9nR5TTbwgF//TRlJx+bZTTz8veeERERkHFfSiD5vSXQOO6UOznrlROLyITjop7utyhvRVa14dV/f7fhuMz60KhX9YEde+HwuLcjlNEBBX30TtxELZvODenb/iDUOgXfxhKK3I9ShGZpFTcM+HMyZDTtzT3zenrrw2FXjm9iGSZinum9XSHnL5lfcjqj74RjteugmXrQqFXTi8i40zFfTy5w+HtyUK/fzPgUU4fregXXqWcXkQyTsU9m04cDLFNa5TTd70HpTNCTt/YFPbTK6cXkQxQcc+VMydh11PJvjenjkBBMSy6NrmqnzE/16MUkTyVkeJuZoXAZuAtd7/RzO4F/gxoj065x92bo3PvBm4DuoHPufvGod47tsU9VSKnT7RDOLorHK9dlWyHMPcC5fQikrZMFffPA6uBipTi3uHuX+933grgAeAKYB7wOLDU3bsHe+9JUdxTJXL6RKHf/1vAYUZdtPNmLSy8Wjm9iAxpqOKeVtcsM1sArAO+BHx+mNNvAh50905gt5ntJBT659MfcsyZQVVjeFzzl9BxKNw01boBtvwTvPDdlJx+LSz5qHJ6ERmRdFsifgu4E+jfWeuzZnYrIa75K3c/BswHNqWcsz86JoMpq4bL/mN49Ob0G8INVK/+uF9OvxZmLMj1iEVkghv2Y/bM7EbgkLtv6ffUd4DFwCqgDfhG4iUDvM052Y+Z3W5mm81sc3t7+wAvmaRKpsPyG+Fj34Yv7IBPPwZX/jkc2wvNX4C/Wwn/9zp4+qvQ9kqIeERE+hk2czezLwO3AF1AKVABPOzuf5pyTj3wqLtfEF1Mxd2/HD23EbjX3QeNZSZd5j5a7duTfW/efJHenL5xbcjqldOLTCoZ2wppZmuAL0QXVGvdvS06/pfA+9z9ZjNbCdxP8oLqE0CDLqhmWMehlP70T4X99FNmQMNHQ6Ff8pGQ24tIbI35guogvmZmqwiRyx7gMwDuvtXMHgJeJ6z27xiqsMsolVXDpbeGx5lTocAn+t689pOQ09dfE7VDUE4vMtnoJqa46ekOWysT7RCO7AzHay5K9r2puVD76UViQHeoTmaHdyQLfW9Of17yE6fqr1FOL5KnVNwl6DgE2zeGQr/rKeg6nczpG9eGr8rpRfLGeGXukm/KquHSW8IjkdO3NkNrv5w+sZ9+5nm5HrGIjJJW7pLM6RPtEI7sCMd7c/q14Xvl9CITimIZGZnenH5DaHbWJ6dfCwuvgaKSXI9SZNJTcZfR62hP9qfvk9N/JNmffurMXI9SZFJS5i6jV1bVL6d/OrpL9jF47adQUBTl9OuU04tMIFq5y+j0dIePFGxd3y+nvzAU+mVNyulFxpliGRl/h3ckL8gmcvqKBSl9b5TTi2Sairtk18nDyb43u56McvqKkM8vW6ecXiRDlLlLdk2vhEv+NDzOnILdvwq7b7Y/BlsfDjn9wquT2yxn1uV6xCKxo5W7ZE9PN7y1JdkO4fD2cLzmwuQHhtderJxeJE2KZWRiOrwzukM2yum9Byrmp/S9uVY5vcgQVNxl4kvk9K0bYOcTfXP6xqbQ90Y5vUgfytxl4kvN6c+ejvbTR31vlNOLjJhW7jKx9fTAW5uT7RAOt4bjcy8MWywb10LtKuX0MikplpH4OLIrpe/NJuX0MqmpuEs8nTyc0p/+STh7CkrKo74368LXqbNyPUqRcaPMXeJpeiVc8ifhcfY0vPGrZN+brT+Lcvqrkn1vZi3M9YhFskYrd4mfnp6wnz7R96Y3p78gRDfLmpTTSywolpHJ7ciulL43UU5fPi/Z96b+WiiakutRioyYirtIwskjsGNjuCibmtMv+XDYZtnwUeX0kjeUuYskTJ8Dq/44PM6eht2/Tu6+ef3nYIVQf3WyHYJyeslTWrmLQEpOH7VDaG8Jx+dekNxmOe8S5fQyoSiWERmpI7vCar61GfY93zenb2yCRcrpJfdU3EXGIpHTtzbDzifh7Enl9DIhKHMXGYs+Of17oT99a3PfnH7hVcm+N7Pqcz1iEa3cRUatpwfe/l2yP30ip69eGfW9ifbTFxTkdJgSX4plRLJhwJy+Nsrp1ymnl4xTcRfJtlNHo74361Ny+rKQ0zdGOf202bkepeQ5Ze4i2TZtNqz6o/A4+17YT9+a2E//r8mcPtEOQTm9ZJhW7iLZ1NMDb/8+2femfVs4Xr0y2Q6h9hLl9JIWxTIiE9XRN8JqvqUZ9v0mmdMvvSHsvll0nXJ6GZSKu0g+OHUUdvxb2H2z84l+OX0TNPyBcnrpQ5m7SD6YNhsuvjk8enP65gFy+ugu2dmLcj1imcDSXrmbWSGwGXjL3W80s9nAj4B6YA/wKXc/Fp17N3Ab0A18zt03DvXeWrmLDCE1p2/dAIdeD8erVyQbnM1TTj8ZZSSWMbPPA6uBiqi4fw046u5fMbMvArPc/S4zWwE8AFwBzAMeB5a6e/dg763iLjICiZy+dQPs/Q14N5TVRBdk14X+9MWluR6lZMGYi7uZLQB+AHwJ+HxU3FuBNe7eZma1wNPu3hit2nH3L0ev3Qjc6+7PD/b+Ku4iozRYTr/4Q1HfG+X0cZaJzP1bwJ1Aecqxue7eBhAV+Oro+HxgU8p5+6NjIpJp/XP6Pc8k+9NveyTk9HXvT7ZDUE4/aQxb3M3sRuCQu28xszVpvOdADa/P+c8DM7sduB2grq4ujbcVkSEVl4Y7Xxs+Cuu+CW2/D1ssW5th4z3hUbU8KvTrlNPH3LCxjJl9GbgF6AJKgQrgYeByFMuI5Ieju5N9b/rk9DdEfW+uU06fhzK2zz1auX8hytz/FjiSckF1trvfaWYrgftJXlB9AmjQBVWRCeLUUdjxy6jvzRNwpgOKp8OSD4VCv/R65fR5Yrz2uX8FeMjMbgP2AZ8EcPetZvYQ8DphtX/HUIVdRLJs2my4+D+ER1cn7H4muc1y2y/ACkJOn+h7M/v8XI9YRkF3qIpI0NMTcvpEO4RDW8PxquXJbZbzLlVOP4Go/YCIjNzR3bD9sbD75pycvgkWfUA5fY6puIvI2PTm9M2w8/F+OX0TNFwfPo5Qskq9ZURkbAbM6ZsHzukb18Kcxbke8aSnlbuIjJ571PcmKvQHXwvHq5Yl+97Mv0w5/ThRLCMi2XFsT3I//Z7nopx+bt/+9MVTcz3K2FBxF5HsO30s5PQt61Ny+mkpfW+U04+VMncRyb6ps+CiT4VHV2fU9yaKb1oeDTn9eVcm+94op88ordxFJLvcoe2lZN+bRE5f2Zjse6OcPi2KZURk4jq2N8rp1ydz+unVyb43539AOf0gVNxFJD+cPgY7Hg+FfsfjcOZEMqdvbAoXZpXT91LmLiL5YeosuOiT4ZHI6ROfOpWa0yfaISinH5RW7iIy8SVy+kTfm4OvhuO9OX0TzF896XJ6xTIiEi+9OX0z7H0OerpScvomOH/NpMjpVdxFJL6Gzemvh+mVuR7luFDmLiLx1SenPxPl9P33078v6k8/eXJ6rdxFJJ7coe3lUOj75PRLk4U+z3N6xTIiIu/si1bz61Ny+qpk35s8zOlV3EVEUp1+J/S7SfS96TwORVOjvjeJ/fQTP6dX5i4ikmrqTLjwE+HRdQb2Ppvse9O6HrCQ0yfaIVQuyfWIR0wrdxGRBHc48EpU6NfDgdScfm0o9AtWQ0FhbscZUSwjIjIa7+yD1seivjfPpuT010d9b9ZAybScDU/FXURkrBI5fWtz6FOfmtM3rg05fVlVVoekzF1EZKwGyukT7RDOyemboLIhp8PVyl1EZCz65PTN4XuAOQ3JQr/g8nHJ6RXLiIhkyztvpnyO7DMhp59WmdKffk3GcnoVdxGRXHjv3ZDPn5PTfzDZn34MOb0ydxGRXCid0S+nfy7Z96a1GTC49Fb4w7/P+B+t4i4ikg1FJWHFvviDsPZrYQ99azPMWDA+f9y4vKuIiAzODGovCo9xkr/t0EREZFAq7iIiMaTiLiISQyruIiIxpOIuIhJDKu4iIjE0bHE3s1Ize9HMXjazrWb2P6Lj95rZW2b2UvRoSnnN3Wa208xazez68ZyAiIicK5197p3Ah9y9w8yKgWfNbEP03N+5+9dTTzazFcDNwEpgHvC4mS119+5MDlxERAY37Mrdg47ox+LoMVRDmpuAB9290913AzuBK8Y8UhERSVtambuZFZrZS8Ah4Jfu/kL01GfN7BUz+76ZzYqOzQfeTHn5/uhY//e83cw2m9nm9vb20c9ARETOkVZxd/dud18FLACuMLMLgO8Ai4FVQBvwjeh0G+gtBnjP+9x9tbuvrqrK7qeXiIjE3Yh2y7j7O8DTwA3ufjAq+j3AP5CMXvYD56W8bAHw9tiHKiIi6Upnt0yVmc2Mvp8KfARoMbPalNM+DrwWff8IcLOZTTGzRUAD8GJGRy0iIkNKZ7dMLfADMysk/DJ4yN0fNbN/NrNVhMhlD/AZAHffamYPAa8DXcAd2ikjIpJd+iQmEZE8NdQnMekOVRGRGFJxFxGJIRV3EZEYUnEXEYkhFXcRkRhScRcRiSEVdxGRGFJxFxGJIRV3EZEYUnEXEYkhFXcRkRhScRcRiSEVdxGRGFJxFxGJIRV3EZEYUnEXEYkhFXcRkRhScRcRiSEVdxGRGFJxFxGJIRV3EZEYUnEXEYkhFXcRkRhScRcRiSEVdxGRGFJxFxGJIRV3EZEYUnEXEYkhFXcRkRhScRcRiSEVdxGRGCrK9QDG4nBHJ//riR0sq61gWU05jTXlTCvJ6ymJiGREXlfCfUdP8ZMt+zl5phsAM6ifM51lNeUsq6lgWW05y2sqWDBrKgUFluPRiohkT14X90vrZvHqvdez/9hpth04TkvbCVoOHKflwAke23oA93De9JJCGmvKWVZbwfLoa2NNORWlxbmdgIjIODFPVMAcWr16tW/evDmj73nqTBfbD3bQ0haK/bbo67unz/aeM3/mVJbXJlf5y2oqqJ8zjaJCXYoQkYnPzLa4++qBnht25W5mpcCvgSnR+T9x9782s9nAj4B6YA/wKXc/Fr3mbuA2oBv4nLtvzMA8RmRaSRGrzpvJqvNm9h5zdw4cf4+WthN9VvpPtbbT3RN+yU0pKmDp3PIQ7aSs9GdPL8n2FERERm3YlbuZGTDd3TvMrBh4FvgL4N8BR939K2b2RWCWu99lZiuAB4ArgHnA48BSd+8e7M8Yj5X7SHR2dbPzUEefWGdb2wkOd3T2nlNdPiWl2IdV/uKqMkqKtMoXkdwY08rdQ/XviH4sjh4O3ASsiY7/AHgauCs6/qC7dwK7zWwnodA/P/opjK8pRYWsnDeDlfNm9DnefqKT1gOh4G+LCv8/PneEM909ABQVGEuqy3pX+ctqylleW0F1+RTC70QRkdxI64KqmRUCW4AlwLfd/QUzm+vubQDu3mZm1dHp84FNKS/fHx3r/563A7cD1NXVjX4G46iqfApV5VO4pqGy99jZ7h72HD7J61GG39J2nBd2H+XnL73de86sacV9dussqy1n6dxySosLczENEZmE0iruUaSyysxmAj8zswuGOH2gJes52Y+73wfcByGWSWccE0FxYQENc8tpmFvOTSnH3zl1prfYtxw4wbYDJ3jwxTc5fTakUQUG9ZXTWZ6IdqKiP3/mVK3yRSTjRrQV0t3fMbOngRuAg2ZWG63aa4FD0Wn7gfNSXrYAeJuYmzmthCvPn8OV58/pPdbd4+w7eoqWtuNsiwr/q/vfZf0rbb3nlE8p6s3wE18ba8opm5LXu1RFJMfSuaBaBZyNCvtU4N+ArwIfAI6kXFCd7e53mtlK4H6SF1SfABom8gXVbOvo7OrN8nsv4rad4ERnV+85dbOnnbNjp272NAp1M5aIRMZ0QRWoBX4Q5e4FwEPu/qiZPQ88ZGa3AfuATwK4+1Yzewh4HegC7hiqsE9GZVOKuGzhLC5bOKv3mLvz1june4t9YqX/+LaDRLs0mVpcyNKa8ijWSV7EnTlN2zRFpK/Y3sQUF++d7WbHwY4++/K3tR3n2KnkzVi1M0rP2bGzqHI6xboZSyTWxrpylxwqLS7kwgUzuHBBcpumu9N+orN3dZ+4A/fZnYc52x1+WZcUFoRtmik7dpbVVFBVPiVXUxGRLFJxz0NmRnVFKdUVpXxgaVXv8TNdPbxxuKPPHbjP7TzMw797q/ecyrKScPE2ZaW/pLpM2zRFYkbFPUZKigqiwl3Bx1JuLTh68kzyRqxopf/Pm/bS2RVuxiosMM6vnN5b7FfUhpV+TUWptmmK5CkV90lg9vQSrlpcyVWLkzdjdXX3sOfIqT47dn639xi/eDm5a3XG1OLeDD+x0l86t0w980XygP5fOkkVRZn8kuoybrwoefz4e2fDNs2Uvfk/3vymeuaL5BkVd+mjorSYy+tnc3n97N5jPT2unvkieUZbIWXU1DNfJLe0FVLGhXrmi0xcKu6SUWZG7Yyp1M6YygeXVfceH6hn/lOt7fx4y/7ec9QzXyRzVNwlKzLZM39FbbgZS9s0RQan4i45Ndqe+bOnl5yzY6dhrm7GEklQcZcJZyQ98x94cV+fnvmLopux1DNfJjsVd8kb6pkvkj7965a8VlhgLKqczqLK6ay9sLb3+EA983/++7c4sUk982VyUHGXWFLPfJnsdBOTTHrqmS/5SjcxiQxBPfMljlTcRQYwVM/8Xe0dvVn+tgMneHbH8D3zG+aWMaVI2zQle1TcRUagpKiA5bUVLK+tgEuSx490hJuxUlf6/XvmL66a3mdfvnrmy3hScRfJgDllU7hqyRSuWjJ0z/wte4/xiHrmSxboX5DIOBmsZ/67p8+y/aB65sv4UnEXybIZU9UzX8aftkKKTGAD9czf1nac4+8lb8YaqGf+osrpuhlrEtBWSJE8pZ75Mloq7iJ5ZrCe+e+d7Q7bNPv0zD80cM/8lB0751eqZ34cqbiLxERp8eA985P78sPXf9yV7JlfXGgsrirrs2NneU25eubnORV3kZgLPfOruLYheTPW2e4edh8+2fu5ty1tx9n0xhF+9vvkzVjqmZ/fVNxFJqHiwpDJL1XP/NhScReRXuqZHx/6X15EhqSe+flJxV1ERkU98yc23cQkIuOuf8/8cCFXPfPHSjcxiUhODdYz/9CJzj47dloOnOCZHYfp6jm3Z/6K2oreTL+yTD3zh6PiLiI5YWbMrShlbkUpaxqTN2Ol1zN/StRyIXkRd0m1euanUnEXkQllJD3zf/i8euYPZtjibmbnAT8EaoAe4D53/59mdi/wZ0B7dOo97t4cveZu4DagG/icu28ch7GLyCSinvkjM+wFVTOrBWrd/XdmVg5sAT4GfArocPev9zt/BfAAcAUwD3gcWOru3YP9GbqgKiKZNFDP/NYDJ2LXM39MF1TdvQ1oi74/YWbbgPlDvOQm4EF37wR2m9lOQqF/fsQjFxEZBfXMH2Hmbmb1hBTsBeBq4LNmdiuwGfgrdz9GKPybUl62nwF+GZjZ7cDtAHV1daMZu4hI2goKjLo506ibM43rV9b0Hh+oZ/6jL7/N/S/kd8/8tIu7mZUBPwX+q7sfN7PvAH8DePT1G8B/Agaa7TnZj7vfB9wHIZYZ+dBFRMZuuJ75r6ds1cynnvlpFXczKyYU9n9x94cB3P1gyvP/ADwa/bgfOC/l5QuAtxERyRND9czfeaijz778J1smZs/8dHbLGPD/gG3u/s2U47VRHg/wceC16PtHgPvN7JuEC6oNwIsZHbWISA6UFhdywfwZXDB/4vfMT2flfjVwC/Cqmb0UHbsH+CMzW0WIXPYAnwFw961m9hDwOtAF3DHUThkRkXw3lp75//7S+fy3dSsyPqZ0dss8y8A5evMQr/kS8KUxjEtEJK+l2zO/ZsbUcfnz472LX0RkghmoZ/54ULs1EZEYUnEXEYkhFXcRkRhScRcRiSEVdxGRGFJxFxGJIRV3EZEYUnEXEYmhYT+sIyuDMGsH9o7hLSqBwxkaTj6YbPMFzXmy0JxHZqG7Vw30xIQo7mNlZpsH+zSSOJps8wXNebLQnDNHsYyISAypuIuIxFBcivt9uR5Alk22+YLmPFlozhkSi8xdRET6isvKXUREUqi4i4jEUN4UdzO7wcxazWynmX1xgOfNzP4+ev4VM7s0F+PMpDTm/CfRXF8xs9+Y2cW5GGcmDTfnlPMuN7NuM/tENsc3HtKZs5mtMbOXzGyrmf0q22PMtDT+bc8ws1+Y2cvRnD+di3Fmipl938wOmdlrgzyf+frl7hP+ARQCu4DzgRLgZWBFv3OagA2EjwS8Engh1+POwpyvAmZF36+dDHNOOe9Jwkc9fiLX487C3/NMwmcS10U/V+d63FmY8z3AV6Pvq4CjQEmuxz6GOV8HXAq8NsjzGa9f+bJyvwLY6e5vuPsZ4EHo87GERD//0INNwEwzq832QDNo2Dm7+2/c/Vj04yZgQZbHmGnp/D0D/Bfgp8ChbA5unKQz5z8GHnb3fQDunu/zTmfODpSbmQFlhOLeld1hZo67/5owh8FkvH7lS3GfD7yZ8vP+6NhIz8knI53PbYTf/Pls2Dmb2Xzg48B3sziu8ZTO3/NSYJaZPW1mW8zs1qyNbnykM+f/DSwH3gZeBf7C3XuyM7ycyHj9ypcPyLYBjvXfw5nOOfkk7fmY2QcJxf2acR3R+Etnzt8C7nL37rCoy3vpzLkIuAz4MDAVeN7MNrn79vEe3DhJZ87XAy8BHwIWA780s2fc/fg4jy1XMl6/8qW47wfOS/l5AeE3+kjPySdpzcfMLgK+B6x19yNZGtt4SWfOq4EHo8JeCTSZWZe7/zwrI8y8dP9tH3b3k8BJM/s1cDGQr8U9nTl/GviKh0B6p5ntBpYBL2ZniFmX8fqVL7HMb4EGM1tkZiXAzcAj/c55BLg1uup8JfCuu7dle6AZNOyczawOeBi4JY9XcamGnbO7L3L3enevB34C/Oc8LuyQ3r/tfwWuNbMiM5sGvA/YluVxZlI6c95H+C8VzGwu0Ai8kdVRZlfG61derNzdvcvMPgtsJFxp/767bzWzP4+e/y5h50QTsBM4RfjNn7fSnPN/B+YA/ydayXZ5HnfUS3POsZLOnN19m5k9BrwC9ADfc/cBt9TlgzT/nv8G+Ccze5UQWdzl7nnbCtjMHgDWAJVmth/4a6AYxq9+qf2AiEgM5UssIyIiI6DiLiISQyruIiIxpOIuIhJDKu4iIjGk4i4iEkMq7iIiMfT/AbMK1oTCOpIWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for name, values in history.history.items():\n",
    "    plt.plot(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ds = timeseries_dataset_from_df(test, data_config[\"batch_size\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "558/558 [==============================] - 24s 43ms/step - loss: 331.0209\n",
      "Test RMSE: 18.19\n"
     ]
    }
   ],
   "source": [
    "mse = model.evaluate(test_ds)\n",
    "print(f\"Test RMSE: {mse**.5:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:Found untraced functions such as lstm_cell_2_layer_call_and_return_conditional_losses, lstm_cell_2_layer_call_fn, lstm_cell_2_layer_call_fn, lstm_cell_2_layer_call_and_return_conditional_losses, lstm_cell_2_layer_call_and_return_conditional_losses while saving (showing 5 of 5). These functions will not be directly callable after loading.\n",
      "WARNING:absl:Found untraced functions such as lstm_cell_2_layer_call_and_return_conditional_losses, lstm_cell_2_layer_call_fn, lstm_cell_2_layer_call_fn, lstm_cell_2_layer_call_and_return_conditional_losses, lstm_cell_2_layer_call_and_return_conditional_losses while saving (showing 5 of 5). These functions will not be directly callable after loading.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: model/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: model/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'timesteps': 32, 'batch_size': 32, 'solar_wind_subset': ['bt', 'temperature', 'bx_gse', 'by_gse', 'bz_gse', 'speed', 'density']}\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import pickle\n",
    "\n",
    "model.save(\"model\")\n",
    "\n",
    "with open(\"scaler.pck\", \"wb\") as f:\n",
    "    pickle.dump(scaler, f)\n",
    "\n",
    "data_config[\"solar_wind_subset\"] = SOLAR_WIND_FEATURES\n",
    "print(data_config)\n",
    "with open(\"config.json\", \"w\") as f:\n",
    "    json.dump(data_config, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import json\n",
    "# import pickle\n",
    "# from typing import Tuple\n",
    "# from tensorflow import keras\n",
    "# import numpy as np\n",
    "# import pandas as pd\n",
    "\n",
    "# # Load in serialized model, config, and scaler\n",
    "# model = keras.models.load_model(\"model\")\n",
    "\n",
    "# with open(\"config.json\", \"r\") as f:\n",
    "#     CONFIG = json.load(f)\n",
    "\n",
    "# with open(\"scaler.pck\", \"rb\") as f:\n",
    "#     scaler = pickle.load(f)\n",
    "\n",
    "# # Set global variables    \n",
    "# TIMESTEPS = CONFIG[\"timesteps\"]\n",
    "# SOLAR_WIND_FEATURES = [\n",
    "#     \"bt\",\n",
    "#     \"temperature\",\n",
    "#     \"bx_gse\",\n",
    "#     \"by_gse\",\n",
    "#     \"bz_gse\",\n",
    "#     \"speed\",\n",
    "#     \"density\",\n",
    "# ]\n",
    "# XCOLS = (\n",
    "#     [col + \"_mean\" for col in SOLAR_WIND_FEATURES]\n",
    "#     + [col + \"_std\" for col in SOLAR_WIND_FEATURES]\n",
    "#     + [\"smoothed_ssn\"]\n",
    "# )\n",
    "\n",
    "\n",
    "# # Define functions for preprocessing\n",
    "# def impute_features(feature_df):\n",
    "#     \"\"\"Imputes data using the following methods:\n",
    "#     - `smoothed_ssn`: forward fill\n",
    "#     - `solar_wind`: interpolation\n",
    "#     \"\"\"\n",
    "#     # forward fill sunspot data for the rest of the month\n",
    "#     feature_df.smoothed_ssn = feature_df.smoothed_ssn.fillna(method=\"ffill\")\n",
    "#     # interpolate between missing solar wind values\n",
    "#     feature_df = feature_df.interpolate()\n",
    "#     return feature_df\n",
    "\n",
    "\n",
    "# def aggregate_hourly(feature_df, aggs=[\"mean\", \"std\"]):\n",
    "#     \"\"\"Aggregates features to the floor of each hour using mean and standard deviation.\n",
    "#     e.g. All values from \"11:00:00\" to \"11:59:00\" will be aggregated to \"11:00:00\".\n",
    "#     \"\"\"\n",
    "#     # group by the floor of each hour use timedelta index\n",
    "#     agged = feature_df.groupby([feature_df.index.floor(\"H\")]).agg(aggs)\n",
    "\n",
    "#     # flatten hierachical column index\n",
    "#     agged.columns = [\"_\".join(x) for x in agged.columns]\n",
    "#     return agged\n",
    "\n",
    "\n",
    "# def preprocess_features(solar_wind, sunspots, scaler=None, subset=None):\n",
    "#     \"\"\"\n",
    "#     Preprocessing steps:\n",
    "#         - Subset the data\n",
    "#         - Aggregate hourly\n",
    "#         - Join solar wind and sunspot data\n",
    "#         - Scale using standard scaler\n",
    "#         - Impute missing values\n",
    "#     \"\"\"\n",
    "#     # select features we want to use\n",
    "#     if subset:\n",
    "#         solar_wind = solar_wind[subset]\n",
    "\n",
    "#     # aggregate solar wind data and join with sunspots\n",
    "#     hourly_features = aggregate_hourly(solar_wind).join(sunspots)\n",
    "\n",
    "#     # subtract mean and divide by standard deviation\n",
    "#     if scaler is None:\n",
    "#         scaler = StandardScaler()\n",
    "#         scaler.fit(hourly_features)\n",
    "\n",
    "#     normalized = pd.DataFrame(\n",
    "#         scaler.transform(hourly_features),\n",
    "#         index=hourly_features.index,\n",
    "#         columns=hourly_features.columns,\n",
    "#     )\n",
    "\n",
    "#     # impute missing values\n",
    "#     imputed = impute_features(normalized)\n",
    "\n",
    "#     # we want to return the scaler object as well to use later during prediction\n",
    "#     return imputed, scaler\n",
    "\n",
    "\n",
    "# # THIS MUST BE DEFINED FOR YOUR SUBMISSION TO RUN\n",
    "# def predict_dst(\n",
    "#     solar_wind_7d: pd.DataFrame,\n",
    "#     satellite_positions_7d: pd.DataFrame,\n",
    "#     latest_sunspot_number: float,\n",
    "# ) -> Tuple[float, float]:\n",
    "#     \"\"\"\n",
    "#     Take all of the data up until time t-1, and then make predictions for\n",
    "#     times t and t+1.\n",
    "#     Parameters\n",
    "#     ----------\n",
    "#     solar_wind_7d: pd.DataFrame\n",
    "#         The last 7 days of satellite data up until (t - 1) minutes [exclusive of t]\n",
    "#     satellite_positions_7d: pd.DataFrame\n",
    "#         The last 7 days of satellite position data up until the present time [inclusive of t]\n",
    "#     latest_sunspot_number: float\n",
    "#         The latest monthly sunspot number (SSN) to be available\n",
    "#     Returns\n",
    "#     -------\n",
    "#     predictions : Tuple[float, float]\n",
    "#         A tuple of two predictions, for (t and t + 1 hour) respectively; these should\n",
    "#         be between -2,000 and 500.\n",
    "#     \"\"\"\n",
    "#     # Re-format data to fit into our pipeline\n",
    "#     sunspots = pd.DataFrame(index=solar_wind_7d.index, columns=[\"smoothed_ssn\"])\n",
    "#     sunspots[\"smoothed_ssn\"] = latest_sunspot_number\n",
    "    \n",
    "#     # Process our features and grab last 32 (timesteps) hours\n",
    "#     features, s = preprocess_features(\n",
    "#         solar_wind_7d, sunspots, scaler=scaler, subset=SOLAR_WIND_FEATURES\n",
    "#     )\n",
    "#     model_input = features[-TIMESTEPS:][XCOLS].values.reshape(\n",
    "#         (1, TIMESTEPS, features.shape[1])\n",
    "#     )\n",
    "    \n",
    "#     # Make a prediction\n",
    "#     prediction_at_t0, prediction_at_t1 = model.predict(model_input)[0]\n",
    "\n",
    "#     # Optional check for unexpected values\n",
    "#     if not np.isfinite(prediction_at_t0):\n",
    "#         prediction_at_t0 = -12\n",
    "#     if not np.isfinite(prediction_at_t1):\n",
    "#         prediction_at_t1 = -12\n",
    "\n",
    "#     return prediction_at_t0, prediction_at_t1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}