01.03_Scrape_College_Stats.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pfr\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "combine_df = pd.read_csv(\"raw_data/pfr_combine_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 6218 entries, 0 to 6217\n",
      "Data columns (total 16 columns):\n",
      "Player                 6218 non-null object\n",
      "Pos                    6218 non-null object\n",
      "School                 6218 non-null object\n",
      "College                4841 non-null object\n",
      "Ht                     6218 non-null object\n",
      "Wt                     6218 non-null int64\n",
      "40yd                   6046 non-null float64\n",
      "Vertical               4796 non-null float64\n",
      "Bench                  4212 non-null float64\n",
      "Broad Jump             4754 non-null float64\n",
      "3Cone                  3993 non-null float64\n",
      "Shuttle                4063 non-null float64\n",
      "Drafted (tm/rnd/yr)    3892 non-null object\n",
      "Year                   6218 non-null int64\n",
      "Pfr_ID                 4895 non-null object\n",
      "Sref_Cfb_ID            4841 non-null object\n",
      "dtypes: float64(6), int64(2), object(8)\n",
      "memory usage: 777.3+ KB\n"
     ]
    }
   ],
   "source": [
    "combine_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Player</th>\n",
       "      <th>Pos</th>\n",
       "      <th>School</th>\n",
       "      <th>College</th>\n",
       "      <th>Ht</th>\n",
       "      <th>Wt</th>\n",
       "      <th>40yd</th>\n",
       "      <th>Vertical</th>\n",
       "      <th>Bench</th>\n",
       "      <th>Broad Jump</th>\n",
       "      <th>3Cone</th>\n",
       "      <th>Shuttle</th>\n",
       "      <th>Drafted (tm/rnd/yr)</th>\n",
       "      <th>Year</th>\n",
       "      <th>Pfr_ID</th>\n",
       "      <th>Sref_Cfb_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>John Abraham</td>\n",
       "      <td>OLB</td>\n",
       "      <td>South Carolina</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6-4</td>\n",
       "      <td>252</td>\n",
       "      <td>4.55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>New York Jets / 1st / 13th pick / 2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>AbraJo00</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Shaun Alexander</td>\n",
       "      <td>RB</td>\n",
       "      <td>Alabama</td>\n",
       "      <td>https://www.sports-reference.com/cfb/players/s...</td>\n",
       "      <td>6-0</td>\n",
       "      <td>218</td>\n",
       "      <td>4.58</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Seattle Seahawks / 1st / 19th pick / 2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>AlexSh00</td>\n",
       "      <td>shaun-alexander-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Darnell Alford</td>\n",
       "      <td>OT</td>\n",
       "      <td>Boston College</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6-4</td>\n",
       "      <td>334</td>\n",
       "      <td>5.56</td>\n",
       "      <td>25.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>8.48</td>\n",
       "      <td>4.98</td>\n",
       "      <td>Kansas City Chiefs / 6th / 188th pick / 2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>AlfoDa20</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Kyle Allamon</td>\n",
       "      <td>TE</td>\n",
       "      <td>Texas Tech</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6-2</td>\n",
       "      <td>253</td>\n",
       "      <td>4.97</td>\n",
       "      <td>29.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>104.0</td>\n",
       "      <td>7.29</td>\n",
       "      <td>4.49</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Rashard Anderson</td>\n",
       "      <td>CB</td>\n",
       "      <td>Jackson State</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6-2</td>\n",
       "      <td>206</td>\n",
       "      <td>4.55</td>\n",
       "      <td>34.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>123.0</td>\n",
       "      <td>7.18</td>\n",
       "      <td>4.15</td>\n",
       "      <td>Carolina Panthers / 1st / 23rd pick / 2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>AndeRa21</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Player  Pos          School  \\\n",
       "0      John Abraham  OLB  South Carolina   \n",
       "1   Shaun Alexander   RB         Alabama   \n",
       "2    Darnell Alford   OT  Boston College   \n",
       "3      Kyle Allamon   TE      Texas Tech   \n",
       "4  Rashard Anderson   CB   Jackson State   \n",
       "\n",
       "                                             College   Ht   Wt  40yd  \\\n",
       "0                                                NaN  6-4  252  4.55   \n",
       "1  https://www.sports-reference.com/cfb/players/s...  6-0  218  4.58   \n",
       "2                                                NaN  6-4  334  5.56   \n",
       "3                                                NaN  6-2  253  4.97   \n",
       "4                                                NaN  6-2  206  4.55   \n",
       "\n",
       "   Vertical  Bench  Broad Jump  3Cone  Shuttle  \\\n",
       "0       NaN    NaN         NaN    NaN      NaN   \n",
       "1       NaN    NaN         NaN    NaN      NaN   \n",
       "2      25.0   23.0        94.0   8.48     4.98   \n",
       "3      29.0    NaN       104.0   7.29     4.49   \n",
       "4      34.0    NaN       123.0   7.18     4.15   \n",
       "\n",
       "                            Drafted (tm/rnd/yr)  Year    Pfr_ID  \\\n",
       "0        New York Jets / 1st / 13th pick / 2000  2000  AbraJo00   \n",
       "1     Seattle Seahawks / 1st / 19th pick / 2000  2000  AlexSh00   \n",
       "2  Kansas City Chiefs / 6th / 188th pick / 2000  2000  AlfoDa20   \n",
       "3                                           NaN  2000       NaN   \n",
       "4    Carolina Panthers / 1st / 23rd pick / 2000  2000  AndeRa21   \n",
       "\n",
       "         Sref_Cfb_ID  \n",
       "0                NaN  \n",
       "1  shaun-alexander-1  \n",
       "2                NaN  \n",
       "3                NaN  \n",
       "4                NaN  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combine_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['OLB', 'RB', 'OT', 'TE', 'CB', 'K', 'P', 'FS', 'OG', 'ILB', 'DE',\n",
       "       'SS', 'DT', 'QB', 'WR', 'C', 'FB', 'LS', 'NT', 'EDGE', 'S', 'LB',\n",
       "       'DB', 'OL'], dtype=object)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combine_df.Pos.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we just want RBs with Sref IDs\n",
    "rb_df = combine_df.loc[(combine_df.Pos=='RB') & (combine_df.Sref_Cfb_ID.notnull())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Player</th>\n",
       "      <th>Pos</th>\n",
       "      <th>School</th>\n",
       "      <th>College</th>\n",
       "      <th>Ht</th>\n",
       "      <th>Wt</th>\n",
       "      <th>40yd</th>\n",
       "      <th>Vertical</th>\n",
       "      <th>Bench</th>\n",
       "      <th>Broad Jump</th>\n",
       "      <th>3Cone</th>\n",
       "      <th>Shuttle</th>\n",
       "      <th>Drafted (tm/rnd/yr)</th>\n",
       "      <th>Year</th>\n",
       "      <th>Pfr_ID</th>\n",
       "      <th>Sref_Cfb_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Shaun Alexander</td>\n",
       "      <td>RB</td>\n",
       "      <td>Alabama</td>\n",
       "      <td>https://www.sports-reference.com/cfb/players/s...</td>\n",
       "      <td>6-0</td>\n",
       "      <td>218</td>\n",
       "      <td>4.58</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Seattle Seahawks / 1st / 19th pick / 2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>AlexSh00</td>\n",
       "      <td>shaun-alexander-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Trung Canidate</td>\n",
       "      <td>RB</td>\n",
       "      <td>Arizona</td>\n",
       "      <td>https://www.sports-reference.com/cfb/players/t...</td>\n",
       "      <td>5-11</td>\n",
       "      <td>193</td>\n",
       "      <td>4.41</td>\n",
       "      <td>NaN</td>\n",
       "      <td>18.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>St. Louis Rams / 1st / 31st pick / 2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>CaniTr00</td>\n",
       "      <td>trung-canidate-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Doug Chapman</td>\n",
       "      <td>RB</td>\n",
       "      <td>Marshall</td>\n",
       "      <td>https://www.sports-reference.com/cfb/players/d...</td>\n",
       "      <td>5-10</td>\n",
       "      <td>215</td>\n",
       "      <td>4.56</td>\n",
       "      <td>38.5</td>\n",
       "      <td>16.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>6.84</td>\n",
       "      <td>4.20</td>\n",
       "      <td>Minnesota Vikings / 3rd / 88th pick / 2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>ChapDo00</td>\n",
       "      <td>doug-chapman-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>Ron Dayne</td>\n",
       "      <td>RB</td>\n",
       "      <td>Wisconsin</td>\n",
       "      <td>https://www.sports-reference.com/cfb/players/r...</td>\n",
       "      <td>5-11</td>\n",
       "      <td>259</td>\n",
       "      <td>4.65</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>New York Giants / 1st / 11th pick / 2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>DaynRo00</td>\n",
       "      <td>ron-dayne-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>Reuben Droughns</td>\n",
       "      <td>RB</td>\n",
       "      <td>Oregon</td>\n",
       "      <td>https://www.sports-reference.com/cfb/players/r...</td>\n",
       "      <td>5-11</td>\n",
       "      <td>215</td>\n",
       "      <td>4.66</td>\n",
       "      <td>30.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>7.34</td>\n",
       "      <td>4.41</td>\n",
       "      <td>Detroit Lions / 3rd / 81st pick / 2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>DrouRe00</td>\n",
       "      <td>reuben-droughns-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Player Pos     School  \\\n",
       "1   Shaun Alexander  RB    Alabama   \n",
       "41   Trung Canidate  RB    Arizona   \n",
       "48     Doug Chapman  RB   Marshall   \n",
       "76        Ron Dayne  RB  Wisconsin   \n",
       "80  Reuben Droughns  RB     Oregon   \n",
       "\n",
       "                                              College    Ht   Wt  40yd  \\\n",
       "1   https://www.sports-reference.com/cfb/players/s...   6-0  218  4.58   \n",
       "41  https://www.sports-reference.com/cfb/players/t...  5-11  193  4.41   \n",
       "48  https://www.sports-reference.com/cfb/players/d...  5-10  215  4.56   \n",
       "76  https://www.sports-reference.com/cfb/players/r...  5-11  259  4.65   \n",
       "80  https://www.sports-reference.com/cfb/players/r...  5-11  215  4.66   \n",
       "\n",
       "    Vertical  Bench  Broad Jump  3Cone  Shuttle  \\\n",
       "1        NaN    NaN         NaN    NaN      NaN   \n",
       "41       NaN   18.0         NaN    NaN      NaN   \n",
       "48      38.5   16.0       128.0   6.84     4.20   \n",
       "76       NaN    NaN         NaN    NaN      NaN   \n",
       "80      30.0   13.0       115.0   7.34     4.41   \n",
       "\n",
       "                           Drafted (tm/rnd/yr)  Year    Pfr_ID  \\\n",
       "1    Seattle Seahawks / 1st / 19th pick / 2000  2000  AlexSh00   \n",
       "41     St. Louis Rams / 1st / 31st pick / 2000  2000  CaniTr00   \n",
       "48  Minnesota Vikings / 3rd / 88th pick / 2000  2000  ChapDo00   \n",
       "76    New York Giants / 1st / 11th pick / 2000  2000  DaynRo00   \n",
       "80      Detroit Lions / 3rd / 81st pick / 2000  2000  DrouRe00   \n",
       "\n",
       "          Sref_Cfb_ID  \n",
       "1   shaun-alexander-1  \n",
       "41   trung-canidate-1  \n",
       "48     doug-chapman-1  \n",
       "76        ron-dayne-1  \n",
       "80  reuben-droughns-1  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rb_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 479 entries, 1 to 6205\n",
      "Data columns (total 16 columns):\n",
      "Player                 479 non-null object\n",
      "Pos                    479 non-null object\n",
      "School                 479 non-null object\n",
      "College                479 non-null object\n",
      "Ht                     479 non-null object\n",
      "Wt                     479 non-null int64\n",
      "40yd                   471 non-null float64\n",
      "Vertical               389 non-null float64\n",
      "Bench                  362 non-null float64\n",
      "Broad Jump             381 non-null float64\n",
      "3Cone                  272 non-null float64\n",
      "Shuttle                275 non-null float64\n",
      "Drafted (tm/rnd/yr)    302 non-null object\n",
      "Year                   479 non-null int64\n",
      "Pfr_ID                 406 non-null object\n",
      "Sref_Cfb_ID            479 non-null object\n",
      "dtypes: float64(6), int64(2), object(8)\n",
      "memory usage: 63.6+ KB\n"
     ]
    }
   ],
   "source": [
    "rb_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scrape the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# columns for scraped data\n",
    "basic_cols = [\"Year\", \"School\", \"Conf\", \"Class\", \"Pos\", \"G\"]\n",
    "rush_cols = [\"Rush_Att\", \"Rush_Yds\", \"Rush_Avg\", \"Rush_TD\"]\n",
    "rec_cols = [\"Rec\", \"Rec_Yds\", \"Rec_Avg\", \"Rec_TD\"]\n",
    "scrim_cols = [\"Plays\", \"Scrim_Yds\", \"Scrim_Avg\", \"Scrim_TD\"]\n",
    "col_dict = {\"rush_cols\": basic_cols + rush_cols + rec_cols + scrim_cols,\n",
    "            \"rec_cols\": basic_cols + rec_cols + rush_cols + scrim_cols}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set up data structures to store scraped data\n",
    "dfs = []\n",
    "errors = []\n",
    "url_template = \"https://www.sports-reference.com/cfb/players/{}.html\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sraped: shaun-alexander-1\n",
      "Sraped: trung-canidate-1\n",
      "Sraped: doug-chapman-1\n",
      "Sraped: ron-dayne-1\n",
      "Sraped: reuben-droughns-1\n",
      "Sraped: troy-hambrick-1\n",
      "Sraped: ronney-jenkins-1\n",
      "Sraped: thomas-jones-1\n",
      "Sraped: curtis-keaton-1\n",
      "Sraped: jamal-lewis-2\n",
      "Sraped: kevin-mcdougal-2\n",
      "Sraped: rondell-mealey-1\n",
      "Sraped: frank-moreau-1\n",
      "Sraped: chad-morton-1\n",
      "Sraped: travis-prentice-1\n",
      "Sraped: jr-redmond-1\n",
      "Sraped: shyrone-stith-1\n",
      "Sraped: michael-wiley-1\n",
      "Sraped: david-allen-1\n",
      "Sraped: kevan-barlow-1\n",
      "Sraped: michael-bennett-1\n",
      "Sraped: correll-buckhalter-1\n",
      "Sraped: jeff-chaney-1\n",
      "Sraped: derek-combs-1\n",
      "Sraped: travis-henry-1\n",
      "Sraped: derek-homer-1\n",
      "Sraped: james-jackson-1\n",
      "Sraped: rudi-johnson-1\n",
      "Sraped: lamont-jordan-1\n",
      "Sraped: deuce-mcallister-1\n",
      "Sraped: travis-minor-1\n",
      "Sraped: hodges-mitchell-1\n",
      "Sraped: anthony-thomas-2\n",
      "Sraped: ladainian-tomlinson-1\n",
      "Sraped: reggie-white-1\n",
      "Sraped: jonathan-adams-1\n",
      "Sraped: brian-allen-2\n",
      "Sraped: damien-anderson-1\n",
      "Sraped: ladell-betts-1\n",
      "Sraped: joe-burns-1\n",
      "Sraped: demontray-carter-2\n",
      "Sraped: rod-cartwright-1\n",
      "Sraped: eric-crouch-1\n",
      "Sraped: tj-duckett-1\n",
      "Sraped: tony-fisher-1\n",
      "Sraped: little-john-flowers-1\n",
      "Sraped: delvon-flowers-1\n",
      "Sraped: deshaun-foster-1\n",
      "Sraped: william-green-1\n",
      "Sraped: ennis-haywood-1\n",
      "Sraped: leonard-henry-1\n",
      "Sraped: dameon-hunter-1\n",
      "Sraped: eric-mccoo-1\n",
      "Sraped: dicenzo-miller-1\n",
      "Sraped: dave-minnich-1\n",
      "Sraped: maurice-morris-1\n",
      "Sraped: larry-ned-1\n",
      "Sraped: clinton-portis-1\n",
      "Sraped: tellis-redmon-1\n",
      "Sraped: josh-scobey-1\n",
      "Sraped: luke-staley-1\n",
      "Sraped: travis-stephens-1\n",
      "Sraped: chester-taylor-1\n",
      "Sraped: lew-thomas-1\n",
      "Sraped: jonathan-wells-1\n",
      "Sraped: levron-williams-1\n",
      "Sraped: ricky-williams-2\n",
      "Sraped: bj-askew-1\n",
      "Sraped: santonio-beard-1\n",
      "Sraped: chris-brown-1\n",
      "Sraped: kerry-carter-1\n",
      "Sraped: avon-cobourne-1\n",
      "Sraped: dahrran-diedrick-1\n",
      "Sraped: justin-fargas-1\n",
      "Sraped: ahmaad-galloway-1\n",
      "Sraped: earnest-graham-1\n",
      "Sraped: quentin-griffin-1\n",
      "Sraped: dwone-hicks-1\n",
      "Sraped: larry-johnson-1\n",
      "Sraped: malaefou-mackenzie-1\n",
      "Sraped: nick-maddox-1\n",
      "Sraped: sultan-mccullough-1\n",
      "Sraped: willis-mcgahee-1\n",
      "Sraped: artose-pinner-1\n",
      "Sraped: cecil-sapp-1\n",
      "Sraped: musa-smith-1\n",
      "Sraped: onterrio-smith-1\n",
      "Sraped: lee-suggs-1\n",
      "Sraped: labrandon-toefield-1\n",
      "Sraped: domanick-davis-1\n",
      "Sraped: rich-alexis-1\n",
      "Sraped: tatum-bell-1\n",
      "Sraped: ran-carthon-1\n",
      "Sraped: cedric-cobbs-1\n",
      "Sraped: josh-davis-2\n",
      "Sraped: adimchinobi-echemandu-1\n",
      "Sraped: clarence-farmer-1\n",
      "Sraped: steven-jackson-3\n",
      "Sraped: kevin-jones-3\n",
      "Sraped: julius-jones-1\n",
      "Sraped: greg-jones-1\n",
      "Sraped: derrick-knight-1\n",
      "Sraped: mewelde-moore-1\n",
      "Sraped: jarrett-payton-1\n",
      "Sraped: chris-perry-1\n",
      "Sraped: fred-russell-1\n",
      "Sraped: michael-turner-2\n",
      "Sraped: shaud-williams-1\n",
      "Sraped: quincy-wilson-1\n",
      "Sraped: renaldo-works-1\n",
      "Sraped: jason-wright-1\n",
      "Sraped: jj-arrington-1\n",
      "Sraped: marion-barber-iii-1\n",
      "Sraped: cedric-benson-1\n",
      "Sraped: decori-birmingham-1\n",
      "Sraped: ronnie-brown-1\n",
      "Sraped: deandra-cobb-1\n",
      "Sraped: anthony-davis-3\n",
      "Sraped: ciatrick-fason-1\n",
      "Sraped: lionel-gates-1\n",
      "Sraped: frank-gore-1\n",
      "Sraped: ryan-grant-1\n",
      "Sraped: kay-jay-harris-1\n",
      "Sraped: alex-haynes-1\n",
      "Sraped: noah-herron-1\n",
      "Sraped: cedric-houston-1\n",
      "Sraped: ray-hudson-2\n",
      "Sraped: ta-mclendon-1\n",
      "Sraped: ryan-moats-1\n",
      "Sraped: vernand-morency-1\n",
      "Sraped: alvin-pearman-1\n",
      "Sraped: bobby-purify-1\n",
      "Sraped: walter-reyes-1\n",
      "Sraped: eric-shelton-1\n",
      "Sraped: darren-sproles-1\n",
      "Sraped: carnell-williams-1\n",
      "Sraped: joseph-addai-1\n",
      "Sraped: mike-bell-1\n",
      "Sraped: reggie-bush-1\n",
      "Sraped: brian-calhoun-2\n",
      "Sraped: pj-daniels-1\n",
      "Sraped: quinton-ganther-1\n",
      "Sraped: ryan-gilbert-1\n",
      "Sraped: marques-hagans-1\n",
      "Sraped: andre-hall-1\n",
      "Sraped: jerome-harrison-1\n",
      "Sraped: taurean-henderson-1\n",
      "Sraped: cedric-humes-1\n",
      "Sraped: maurice-drew-1\n",
      "Sraped: wali-lundy-1\n",
      "Sraped: laurence-maroney-1\n",
      "Sraped: wendell-mathis-1\n",
      "Sraped: dontrell-moore-1\n",
      "Sraped: jerious-norwood-1\n",
      "Sraped: gerald-riggs-2\n",
      "Sraped: cory-ross-1\n",
      "Sraped: leon-washington-1\n",
      "Sraped: lendale-white-1\n",
      "Sraped: terrence-whitehead-1\n",
      "Sraped: deangelo-williams-1\n",
      "Sraped: lorenzo-booker-1\n",
      "Sraped: ahmad-bradshaw-1\n",
      "Sraped: michael-bush-1\n",
      "Sraped: thomas-clayton-1\n",
      "Sraped: jon-cornish-1\n",
      "Sraped: kenneth-darby-1\n",
      "Sraped: chris-henry-1\n",
      "Sraped: tony-hunt-1\n",
      "Sraped: nate-ilaoa-1\n",
      "Sraped: kenny-irons-1\n",
      "Sraped: brandon-jackson-2\n",
      "Sraped: brian-leonard-1\n",
      "Sraped: marshawn-lynch-1\n",
      "Sraped: ronnie-mcgill-1\n",
      "Sraped: tyrone-moss-1\n",
      "Sraped: adrian-peterson-1\n",
      "Sraped: antonio-pittman-1\n",
      "Sraped: gary-russell-1\n",
      "Sraped: kolby-smith-1\n",
      "Sraped: darius-walker-1\n",
      "Sraped: garrett-wolfe-1\n",
      "Sraped: dwayne-wright-1\n",
      "Sraped: deshawn-wynn-1\n",
      "Sraped: selvin-young-1\n",
      "Sraped: anthony-alridge-1\n",
      "Sraped: yvenson-bernard-1\n",
      "Sraped: cory-boyd-1\n",
      "Sraped: thomas-brown-1\n",
      "Sraped: jamaal-charles-1\n",
      "Sraped: tashard-choice-1\n",
      "Sraped: calvin-dawson-1\n",
      "Sraped: justin-forsett-1\n",
      "Sraped: matt-forte-1\n",
      "Sraped: benjarvus-green-ellis-1\n",
      "Sraped: michael-hart-1\n",
      "Sraped: chris-johnson-1\n",
      "Sraped: felix-jones-1\n",
      "Sraped: rafael-little-1\n",
      "Sraped: kregg-lumpkin-1\n",
      "Sraped: darren-mcfadden-1\n",
      "Sraped: kalvin-mcrae-1\n",
      "Sraped: rashard-mendenhall-1\n",
      "Sraped: jalen-parmele-1\n",
      "Sraped: allen-patrick-1\n",
      "Sraped: ray-rice-1\n",
      "Sraped: dantrell-savage-1\n",
      "Sraped: chad-simpson-1\n",
      "Sraped: steve-slaton-1\n",
      "Sraped: kevin-smith-1\n",
      "Sraped: jonathan-stewart-1\n",
      "Sraped: marcus-thomas-2\n",
      "Sraped: ryan-torain-1\n",
      "Sraped: chauncey-washington-1\n",
      "Sraped: kahlil-bell-1\n",
      "Sraped: donald-brown-1\n",
      "Sraped: andre-brown-1\n",
      "Sraped: glen-coffee-1\n",
      "Sraped: james-davis-1\n",
      "Sraped: arian-foster-1\n",
      "Sraped: mike-goodson-1\n",
      "Sraped: shonn-greene-1\n",
      "Sraped: pj-hill-1\n",
      "Sraped: jeremiah-johnson-1\n",
      "Sraped: ian-johnson-1\n",
      "Sraped: gartrell-johnson-1\n",
      "Sraped: anthony-kimble-1\n",
      "Sraped: marlon-lucky-1\n",
      "Sraped: lesean-mccoy-1\n",
      "Sraped: knowshon-moreno-1\n",
      "Sraped: chris-ogbonnaya-1\n",
      "Sraped: cedric-peerman-1\n",
      "Sraped: javon-ringer-1\n",
      "Sraped: kory-sheets-1\n",
      "Sraped: tyrell-sutton-1\n",
      "Sraped: chris-wells-1\n",
      "Sraped: jahvid-best-1\n",
      "Sraped: legarrette-blount-1\n",
      "Sraped: chris-brown-4\n",
      "Sraped: anthony-dixon-1\n",
      "Sraped: andre-dixon-1\n",
      "Sraped: jonathan-dwyer-1\n",
      "Sraped: toby-gerhart-1\n",
      "Sraped: montario-hardesty-1\n",
      "Sraped: javarris-james-1\n",
      "Sraped: brandon-james-1\n",
      "Sraped: stafon-johnson-1\n",
      "Sraped: darius-marshall-1\n",
      "Sraped: ryan-mathews-1\n",
      "Sraped: joe-mcknight-1\n",
      "Sraped: shawnbrey-mcneal-1\n",
      "Sraped: lonyae-miller-1\n",
      "Sraped: brandon-minor-1\n",
      "Sraped: charles-scott-1\n",
      "Sraped: cj-spiller-1\n",
      "Sraped: james-starks-1\n",
      "Sraped: ben-tate-1\n",
      "Sraped: keith-toston-1\n",
      "Sraped: keiland-williams-1\n",
      "Sraped: armando-allen-1\n",
      "Sraped: anthony-allen-1\n",
      "Sraped: matt-asiata-1\n",
      "Sraped: damien-berry-1\n",
      "Sraped: allen-bradford-1\n",
      "Sraped: delone-carter-1\n",
      "Sraped: john-clay-1\n",
      "Sraped: graig-cooper-1\n",
      "Sraped: noel-devine-1\n",
      "Sraped: shaun-draughn-1\n",
      "Sraped: darren-evans-1\n",
      "Sraped: mario-fannin-1\n",
      "Sraped: alex-green-2\n",
      "Sraped: jamie-harper-1\n",
      "Sraped: roy-helu-1\n",
      "Sraped: kendall-hunter-1\n",
      "Sraped: mark-ingram-1\n",
      "Sraped: mikel-leshoure-1\n",
      "Sraped: dion-lewis-1\n",
      "Sraped: derrick-locke-1\n",
      "Sraped: demarco-murray-1\n",
      "Sraped: bilal-powell-1\n",
      "Sraped: stevan-ridley-1\n",
      "Sraped: jacquizz-rodgers-1\n",
      "Sraped: evan-royster-1\n",
      "Sraped: brandon-saine-1\n",
      "Sraped: darel-scott-1\n",
      "Sraped: vai-taua-1\n",
      "Sraped: daniel-thomas-1\n",
      "Sraped: jordan-todman-1\n",
      "Sraped: shane-vereen-1\n",
      "Sraped: johnny-white-1\n",
      "Sraped: ryan-williams-2\n",
      "Sraped: edwin-baker-1\n",
      "Sraped: mike-ball-1\n",
      "Sraped: vick-ballard-1\n",
      "Sraped: brandon-bolden-1\n",
      "Sraped: lennon-creer-1\n",
      "Sraped: terrance-ganaway-1\n",
      "Sraped: jonas-gray-1\n",
      "Sraped: cyrus-gray-1\n",
      "Sraped: dan-herron-1\n",
      "Sraped: ronnie-hillman-1\n",
      "Sraped: lamichael-james-1\n",
      "Sraped: doug-martin-1\n",
      "Sraped: davin-meggett-1\n",
      "Sraped: lamar-miller-1\n",
      "Sraped: alfred-morris-1\n",
      "Sraped: isaiah-pead-1\n",
      "Sraped: bernard-pierce-1\n",
      "Sraped: chris-polk-1\n",
      "Sraped: tauren-poole-1\n",
      "Sraped: chris-rainey-1\n",
      "Sraped: trent-richardson-1\n",
      "Sraped: darrell-scott-1\n",
      "Sraped: robert-turbin-1\n",
      "Sraped: marc-tyler-1\n",
      "Sraped: fozzy-whittaker-1\n",
      "Sraped: david-wilson-1\n",
      "Sraped: cj-anderson-1\n",
      "Sraped: montee-ball-1\n",
      "Sraped: kenjon-barner-1\n",
      "Sraped: leveon-bell-1\n",
      "Sraped: giovani-bernard-1\n",
      "Sraped: rex-burkhead-1\n",
      "Sraped: knile-davis-1\n",
      "Sraped: andre-ellington-1\n",
      "Sraped: michael-ford-2\n",
      "Sraped: johnathan-franklin-1\n",
      "Sraped: mike-gillislee-1\n",
      "Sraped: ray-graham-1\n",
      "Sraped: dj-harper-1\n",
      "Sraped: montel-harris-1\n",
      "Sraped: mike-james-1\n",
      "Sraped: jawan-jamison-1\n",
      "Sraped: stefphon-jefferson-1\n",
      "Sraped: eddie-lacy-1\n",
      "Sraped: marcus-lattimore-1\n",
      "Sraped: onterio-mccalebb-1\n",
      "Sraped: christine-michael-1\n",
      "Sraped: joseph-randle-1\n",
      "Sraped: theo-riddick-1\n",
      "Sraped: robbie-rouse-1\n",
      "Sraped: zac-stacy-1\n",
      "Sraped: stepfan-taylor-1\n",
      "Sraped: chris-thompson-1\n",
      "Sraped: matthew-tucker-1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sraped: spencer-ware-1\n",
      "Sraped: kerwynn-williams-1\n",
      "Sraped: george-winn-1\n",
      "Sraped: cierre-wood-1\n",
      "Sraped: antonio-andrews-1\n",
      "Sraped: kapri-bibbs-1\n",
      "Sraped: alfred-blue-1\n",
      "Sraped: kadeem-carey-1\n",
      "Sraped: tim-cornett-1\n",
      "Sraped: david-fluellen-1\n",
      "Sraped: devonta-freeman-1\n",
      "Sraped: tyler-gaffney-1\n",
      "Sraped: marion-grice-1\n",
      "Sraped: jeremy-hill-2\n",
      "Sraped: carlos-hyde-1\n",
      "Sraped: storm-johnson-1\n",
      "Sraped: henry-josey-1\n",
      "Sraped: james-wilder-jr-1\n",
      "Sraped: tre-mason-1\n",
      "Sraped: adam-muema-1\n",
      "Sraped: ladarius-perkins-1\n",
      "Sraped: silas-redd-1\n",
      "Sraped: bishop-sankey-1\n",
      "Sraped: lache-seastrunk-1\n",
      "Sraped: charles-sims-2\n",
      "Sraped: jerome-smith-1\n",
      "Sraped: deanthony-thomas-1\n",
      "Sraped: james-white-2\n",
      "Sraped: andre-williams-2\n",
      "Sraped: damien-williams-1\n",
      "Sraped: ameer-abdullah-1\n",
      "Sraped: jay-ajayi-1\n",
      "Sraped: javorius-allen-1\n",
      "Sraped: cameron-artis-payne-1\n",
      "Sraped: malcolm-brown-2\n",
      "Sraped: dominique-brown-1\n",
      "Sraped: bj-catalon-1\n",
      "Sraped: david-cobb-1\n",
      "Sraped: tevin-coleman-1\n",
      "Sraped: mike-davis-9\n",
      "Sraped: michael-dyer-1\n",
      "Sraped: jahwan-edwards-1\n",
      "Sraped: melvin-gordon-1\n",
      "Sraped: todd-gurley-1\n",
      "Sraped: dee-hart-1\n",
      "Sraped: braylon-heard-1\n",
      "Sraped: kenny-hilliard-1\n",
      "Sraped: duke-johnson-1\n",
      "Sraped: matt-jones-3\n",
      "Sraped: jeremy-langford-1\n",
      "Sraped: terrence-magee-1\n",
      "Sraped: marcus-murphy-1\n",
      "Sraped: thomas-rawls-1\n",
      "Sraped: josh-robinson-2\n",
      "Sraped: karlos-williams-1\n",
      "Sraped: trey-williams-2\n",
      "Sraped: tj-yeldon-1\n",
      "Sraped: peyton-barber-1\n",
      "Sraped: devontae-booker-1\n",
      "Sraped: tra-carson-1\n",
      "Sraped: alex-collins-1\n",
      "Sraped: kenneth-dixon-1\n",
      "Sraped: kenyan-drake-1\n",
      "Sraped: ezekiel-elliott-1\n",
      "Sraped: tyler-ervin-1\n",
      "Sraped: josh-ferguson-3\n",
      "Sraped: derrick-henry-2\n",
      "Sraped: jordan-howard-1\n",
      "Sraped: devon-johnson-1\n",
      "Sraped: daniel-lasco-1\n",
      "Sraped: tre-madden-1\n",
      "Sraped: keith-marshall-1\n",
      "Sraped: paul-perkins-1\n",
      "Sraped: cj-prosise-1\n",
      "Sraped: wendell-smallwood-1\n",
      "Sraped: kelvin-taylor-1\n",
      "Sraped: shadrach-thornton-1\n",
      "Sraped: deandre-washington-1\n",
      "Sraped: brandon-wilds-1\n",
      "Sraped: jonathan-williams-2\n",
      "Sraped: chris-carson-1\n",
      "Sraped: corey-clement-1\n",
      "Sraped: james-conner-1\n",
      "Sraped: dalvin-cook-1\n",
      "Sraped: justin-davis-2\n",
      "Sraped: matt-dayes-1\n",
      "Sraped: donta-foreman-1\n",
      "Sraped: leonard-fournette-1\n",
      "Sraped: wayne-gallman-1\n",
      "Sraped: brian-hill-4\n",
      "Sraped: elijah-hood-1\n",
      "Sraped: kareem-hunt-1\n",
      "Sraped: rushel-shell-3\n",
      "Sraped: aaron-jones-6\n",
      "Sraped: alvin-kamara-1\n",
      "Sraped: tj-logan-1\n",
      "Sraped: marlon-mack-1\n",
      "Sraped: christian-mccaffrey-1\n",
      "Sraped: elijah-mcguire-1\n",
      "Sraped: jeremy-mcnichols-1\n",
      "Sraped: dare-ogunbowale-1\n",
      "Sraped: samaje-perine-1\n",
      "Sraped: donnel-pumphrey-1\n",
      "Sraped: devine-redding-1\n",
      "Sraped: deveon-smith-1\n",
      "Sraped: jahad-thomas-1\n",
      "Sraped: stanley-boom-williams-1\n",
      "Sraped: joe-williams-7\n",
      "Sraped: jamaal-williams-1\n",
      "Sraped: josh-adams-2\n",
      "Sraped: kalen-ballage-1\n",
      "Sraped: saquon-barkley-1\n",
      "Sraped: nick-chubb-1\n",
      "Sraped: lavon-coleman-1\n",
      "Sraped: justin-crawford-1\n",
      "Sraped: dmitri-flowers-1\n",
      "Sraped: jarvion-franklin-1\n",
      "Sraped: royce-freeman-1\n",
      "Sraped: derrius-guice-1\n",
      "Sraped: kyle-hicks-1\n",
      "Sraped: nyheim-hines-1\n",
      "Sraped: justin-jackson-5\n",
      "Sraped: kerryon-johnson-1\n",
      "Sraped: ronald-jones-4\n",
      "Sraped: john-kelly-6\n",
      "Sraped: sony-michel-1\n",
      "Sraped: ryan-nail-1\n",
      "Sraped: rashaad-penny-1\n",
      "Sraped: kamryn-pettway-1\n",
      "Sraped: bo-scarbrough-1\n",
      "Sraped: akrum-wadley-1\n",
      "Sraped: mark-walton-1\n",
      "Sraped: jordan-wilkins-1\n",
      "Sraped: darrel-williams-1\n"
     ]
    }
   ],
   "source": [
    "for player_id in rb_df.Sref_Cfb_ID:\n",
    "    url = url_template.format(player_id)\n",
    "    pq_obj = pfr.create_pq(url)\n",
    "    try:\n",
    "        # scrape the rushing data\n",
    "        rush_data = pfr.get_row_data(pq_obj,\"#rushing > tbody > tr\")\n",
    "        rush_df = pd.DataFrame(rush_data, columns=col_dict[\"rush_cols\"])\n",
    "\n",
    "        # if df isn't empty then there is a rushing table on the page\n",
    "        # so lets add the player id and append\n",
    "        if rush_df.shape[0] > 0: \n",
    "            rush_df[\"Sref_Cfb_ID\"] = player_id\n",
    "            dfs.append(rush_df)\n",
    "\n",
    "        # otherwis, when rush_df is empty scrape data using #receiving\n",
    "        # selector\n",
    "        else:\n",
    "            rec_data = pfr.get_row_data(pq_obj, \"#receiving > tbody > tr\")\n",
    "            rec_df = pd.DataFrame(rec_data, columns=col_dict[\"rec_cols\"])\n",
    "            rec_df[\"Sref_Cfb_ID\"] = player_id\n",
    "            dfs.append(rec_df)\n",
    "        print(\"Sraped:\", player_id)\n",
    "    except Exception as e:\n",
    "        errors.append([player_id, url, e])\n",
    "        print(\"ERROR:\", player_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "479"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(errors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set up the rushing and receiving data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "rush_rec_df = pd.concat(dfs, ignore_index=True, sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1671 entries, 0 to 1670\n",
      "Data columns (total 19 columns):\n",
      "Year           1671 non-null object\n",
      "School         1671 non-null object\n",
      "Conf           1671 non-null object\n",
      "Class          1671 non-null object\n",
      "Pos            1671 non-null object\n",
      "G              1671 non-null object\n",
      "Rush_Att       1671 non-null object\n",
      "Rush_Yds       1671 non-null object\n",
      "Rush_Avg       1671 non-null object\n",
      "Rush_TD        1671 non-null object\n",
      "Rec            1671 non-null object\n",
      "Rec_Yds        1671 non-null object\n",
      "Rec_Avg        1671 non-null object\n",
      "Rec_TD         1671 non-null object\n",
      "Plays          1671 non-null object\n",
      "Scrim_Yds      1671 non-null object\n",
      "Scrim_Avg      1671 non-null object\n",
      "Scrim_TD       1671 non-null object\n",
      "Sref_Cfb_ID    1671 non-null object\n",
      "dtypes: object(19)\n",
      "memory usage: 248.1+ KB\n"
     ]
    }
   ],
   "source": [
    "rush_rec_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "rush_rec_df.to_csv(\"raw_data/rb_yearly_college_rush_rec_stats.csv\",\n",
    "                   index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# some seasons contain bowl game stats, should have a column indicating that\n",
    "rush_rec_df[\"Stats_Incl_Bowl_Gm\"] = rush_rec_df.Year.str.contains(\"\\*\").astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# now get rid of * symbol\n",
    "rush_rec_df.loc[:, \"Year\"] = rush_rec_df.Year.str.strip(\"\\*\").astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1671 entries, 0 to 1670\n",
      "Data columns (total 20 columns):\n",
      "Year                  1671 non-null int64\n",
      "School                1671 non-null object\n",
      "Conf                  1671 non-null object\n",
      "Class                 1671 non-null object\n",
      "Pos                   1671 non-null object\n",
      "G                     1671 non-null object\n",
      "Rush_Att              1671 non-null object\n",
      "Rush_Yds              1671 non-null object\n",
      "Rush_Avg              1671 non-null object\n",
      "Rush_TD               1671 non-null object\n",
      "Rec                   1671 non-null object\n",
      "Rec_Yds               1671 non-null object\n",
      "Rec_Avg               1671 non-null object\n",
      "Rec_TD                1671 non-null object\n",
      "Plays                 1671 non-null object\n",
      "Scrim_Yds             1671 non-null object\n",
      "Scrim_Avg             1671 non-null object\n",
      "Scrim_TD              1671 non-null object\n",
      "Sref_Cfb_ID           1671 non-null object\n",
      "Stats_Incl_Bowl_Gm    1671 non-null int64\n",
      "dtypes: int64(2), object(18)\n",
      "memory usage: 261.2+ KB\n"
     ]
    }
   ],
   "source": [
    "rush_rec_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>School</th>\n",
       "      <th>Conf</th>\n",
       "      <th>Class</th>\n",
       "      <th>Pos</th>\n",
       "      <th>G</th>\n",
       "      <th>Rush_Att</th>\n",
       "      <th>Rush_Yds</th>\n",
       "      <th>Rush_Avg</th>\n",
       "      <th>Rush_TD</th>\n",
       "      <th>Rec</th>\n",
       "      <th>Rec_Yds</th>\n",
       "      <th>Rec_Avg</th>\n",
       "      <th>Rec_TD</th>\n",
       "      <th>Plays</th>\n",
       "      <th>Scrim_Yds</th>\n",
       "      <th>Scrim_Avg</th>\n",
       "      <th>Scrim_TD</th>\n",
       "      <th>Sref_Cfb_ID</th>\n",
       "      <th>Stats_Incl_Bowl_Gm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1996</td>\n",
       "      <td>Alabama</td>\n",
       "      <td>SEC</td>\n",
       "      <td></td>\n",
       "      <td>RB</td>\n",
       "      <td>11</td>\n",
       "      <td>77</td>\n",
       "      <td>589</td>\n",
       "      <td>7.6</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>53</td>\n",
       "      <td>7.6</td>\n",
       "      <td>0</td>\n",
       "      <td>84</td>\n",
       "      <td>642</td>\n",
       "      <td>7.6</td>\n",
       "      <td>6</td>\n",
       "      <td>shaun-alexander-1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1997</td>\n",
       "      <td>Alabama</td>\n",
       "      <td>SEC</td>\n",
       "      <td></td>\n",
       "      <td>RB</td>\n",
       "      <td>9</td>\n",
       "      <td>90</td>\n",
       "      <td>415</td>\n",
       "      <td>4.6</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>37</td>\n",
       "      <td>9.3</td>\n",
       "      <td>0</td>\n",
       "      <td>94</td>\n",
       "      <td>452</td>\n",
       "      <td>4.8</td>\n",
       "      <td>3</td>\n",
       "      <td>shaun-alexander-1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1998</td>\n",
       "      <td>Alabama</td>\n",
       "      <td>SEC</td>\n",
       "      <td></td>\n",
       "      <td>RB</td>\n",
       "      <td>11</td>\n",
       "      <td>258</td>\n",
       "      <td>1178</td>\n",
       "      <td>4.6</td>\n",
       "      <td>13</td>\n",
       "      <td>26</td>\n",
       "      <td>385</td>\n",
       "      <td>14.8</td>\n",
       "      <td>4</td>\n",
       "      <td>284</td>\n",
       "      <td>1563</td>\n",
       "      <td>5.5</td>\n",
       "      <td>17</td>\n",
       "      <td>shaun-alexander-1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1999</td>\n",
       "      <td>Alabama</td>\n",
       "      <td>SEC</td>\n",
       "      <td></td>\n",
       "      <td>RB</td>\n",
       "      <td>11</td>\n",
       "      <td>302</td>\n",
       "      <td>1383</td>\n",
       "      <td>4.6</td>\n",
       "      <td>19</td>\n",
       "      <td>25</td>\n",
       "      <td>323</td>\n",
       "      <td>12.9</td>\n",
       "      <td>4</td>\n",
       "      <td>327</td>\n",
       "      <td>1706</td>\n",
       "      <td>5.2</td>\n",
       "      <td>23</td>\n",
       "      <td>shaun-alexander-1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1996</td>\n",
       "      <td>Arizona</td>\n",
       "      <td>Pac-10</td>\n",
       "      <td></td>\n",
       "      <td>WR</td>\n",
       "      <td>11</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>trung-canidate-1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year   School    Conf Class Pos   G Rush_Att Rush_Yds Rush_Avg Rush_TD Rec  \\\n",
       "0  1996  Alabama     SEC        RB  11       77      589      7.6       6   7   \n",
       "1  1997  Alabama     SEC        RB   9       90      415      4.6       3   4   \n",
       "2  1998  Alabama     SEC        RB  11      258     1178      4.6      13  26   \n",
       "3  1999  Alabama     SEC        RB  11      302     1383      4.6      19  25   \n",
       "4  1996  Arizona  Pac-10        WR  11                                          \n",
       "\n",
       "  Rec_Yds Rec_Avg Rec_TD Plays Scrim_Yds Scrim_Avg Scrim_TD  \\\n",
       "0      53     7.6      0    84       642       7.6        6   \n",
       "1      37     9.3      0    94       452       4.8        3   \n",
       "2     385    14.8      4   284      1563       5.5       17   \n",
       "3     323    12.9      4   327      1706       5.2       23   \n",
       "4                            0         0                  0   \n",
       "\n",
       "         Sref_Cfb_ID  Stats_Incl_Bowl_Gm  \n",
       "0  shaun-alexander-1                   0  \n",
       "1  shaun-alexander-1                   0  \n",
       "2  shaun-alexander-1                   0  \n",
       "3  shaun-alexander-1                   0  \n",
       "4   trung-canidate-1                   0  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rush_rec_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "rush_rec_df.to_csv(\"processed_data/rb_yearly_college_rush_rec_stats.csv\",\n",
    "                   index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}