preprocess recollected tweets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evacuation Tweet Preview"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```sh\n",
    "search_tweets.py --credential-file=creds.yml --query='(\"evacuate\" OR \"evacuating\" OR \"leaving\" OR \"leave\" OR \"escape\" OR \"escaping\") -people -residents lang:en -is:retweet -has:links -is:nullcast' --start-time=2017-09-04T00:00 --end-time=2017-09-17T00:00 --results-per-call=500 --expansions=\"geo.place_id,author_id\" --tweet-fields=\"id,text,public_metrics,created_at,entities,geo,in_reply_to_user_id,lang,referenced_tweets\" --user-fields=\"id,created_at,description,location,public_metrics,verified,username,name\" --place-fields=\"geo,name,full_name\" --filename-prefix=\"evacuate\"\n",
    "```\n",
    "code: https://github.com/twitterdev/search-tweets-python/tree/v2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from preprocess import load_tweets, aggregate_tweet_info, geolocate, convert_located_tweets_to_dataframe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`evacuate.json`: ~974k tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "974002it [00:20, 48542.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "969796 tweets\n",
      "676655 users\n",
      "9052 places\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/xli/anaconda3/lib/python3.7/site-packages/carmen/resolvers/profile.py:42: UserWarning: Duplicate location name \"amman  jordan\"\n",
      "  warnings.warn('Duplicate location name \"%s\"' % alias)\n",
      "/home/xli/anaconda3/lib/python3.7/site-packages/carmen/resolvers/profile.py:42: UserWarning: Duplicate location name \"amman jordan\"\n",
      "  warnings.warn('Duplicate location name \"%s\"' % alias)\n",
      "/home/xli/anaconda3/lib/python3.7/site-packages/carmen/resolvers/profile.py:42: UserWarning: Duplicate location name \"st joseph county united states\"\n",
      "  warnings.warn('Duplicate location name \"%s\"' % alias)\n",
      "/home/xli/anaconda3/lib/python3.7/site-packages/carmen/resolvers/profile.py:42: UserWarning: Duplicate location name \"st tammany parish united states\"\n",
      "  warnings.warn('Duplicate location name \"%s\"' % alias)\n",
      "100%|██████████| 969796/969796 [2:12:17<00:00, 122.18it/s]  \n"
     ]
    }
   ],
   "source": [
    "evac_data = load_tweets('evacuate.json')\n",
    "for k, v in evac_data.items():\n",
    "    print(len(v), k)\n",
    "evac_tweets = aggregate_tweet_info(evac_data)\n",
    "geolocate(evac_tweets) # in-place\n",
    "df_vac_tweets = convert_located_tweets_to_dataframe(evac_tweets)\n",
    "df_vac_tweets.to_csv('evacuation_reformatted.csv.gz', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>username</th>\n",
       "      <th>user_location</th>\n",
       "      <th>tweet_time</th>\n",
       "      <th>text</th>\n",
       "      <th>tweet_place_type</th>\n",
       "      <th>tweet_place_name</th>\n",
       "      <th>tweet_lat</th>\n",
       "      <th>tweet_lon</th>\n",
       "      <th>loc_lat</th>\n",
       "      <th>loc_lon</th>\n",
       "      <th>loc_country</th>\n",
       "      <th>loc_state</th>\n",
       "      <th>loc_county</th>\n",
       "      <th>loc_city</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>909205251361665024</td>\n",
       "      <td>50491692</td>\n",
       "      <td>SparKLeShiNes</td>\n",
       "      <td>Bay Area</td>\n",
       "      <td>2017-09-16 23:59:58+00:00</td>\n",
       "      <td>Kevin she's not leaving you...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>37.878695</td>\n",
       "      <td>-122.370941</td>\n",
       "      <td>United States</td>\n",
       "      <td>California</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>909205250975961088</td>\n",
       "      <td>361525569</td>\n",
       "      <td>__sxmiraa</td>\n",
       "      <td>Kennington, London</td>\n",
       "      <td>2017-09-16 23:59:58+00:00</td>\n",
       "      <td>Leave out in 2017 is childishhhh pls pls</td>\n",
       "      <td>None</td>\n",
       "      <td>Canterbury, England</td>\n",
       "      <td>51.276756</td>\n",
       "      <td>1.090136</td>\n",
       "      <td>51.275970</td>\n",
       "      <td>1.075610</td>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>England</td>\n",
       "      <td>Kent</td>\n",
       "      <td>Canterbury</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>909205248681631744</td>\n",
       "      <td>788780861881782272</td>\n",
       "      <td>sofiabertox1</td>\n",
       "      <td>Glasgow, Scotland</td>\n",
       "      <td>2017-09-16 23:59:58+00:00</td>\n",
       "      <td>Am a needy bastard take it or leave it😘 but in...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>55.857809</td>\n",
       "      <td>-4.242511</td>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>Scotland</td>\n",
       "      <td>Lanarkshire</td>\n",
       "      <td>Glasgow</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id             user_id       username       user_location  \\\n",
       "0  909205251361665024            50491692  SparKLeShiNes            Bay Area   \n",
       "1  909205250975961088           361525569      __sxmiraa  Kennington, London   \n",
       "2  909205248681631744  788780861881782272   sofiabertox1   Glasgow, Scotland   \n",
       "\n",
       "                 tweet_time  \\\n",
       "0 2017-09-16 23:59:58+00:00   \n",
       "1 2017-09-16 23:59:58+00:00   \n",
       "2 2017-09-16 23:59:58+00:00   \n",
       "\n",
       "                                                text tweet_place_type  \\\n",
       "0                     Kevin she's not leaving you...             None   \n",
       "1           Leave out in 2017 is childishhhh pls pls             None   \n",
       "2  Am a needy bastard take it or leave it😘 but in...             None   \n",
       "\n",
       "      tweet_place_name  tweet_lat  tweet_lon    loc_lat     loc_lon  \\\n",
       "0                 None        NaN        NaN  37.878695 -122.370941   \n",
       "1  Canterbury, England  51.276756   1.090136  51.275970    1.075610   \n",
       "2                 None        NaN        NaN  55.857809   -4.242511   \n",
       "\n",
       "      loc_country   loc_state   loc_county    loc_city  \n",
       "0   United States  California                           \n",
       "1  United Kingdom     England         Kent  Canterbury  \n",
       "2  United Kingdom    Scotland  Lanarkshire     Glasgow  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_vac_tweets.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from preprocess import load_tweets_csv, tweet_summary, is_none"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "evac_tweets = load_tweets_csv('evacuation_reformatted.csv.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "969796 tweets\n",
      "    51128 with original geo info\n",
      "    416736 geolocated\n",
      "    273718 from US\n",
      "676655 users\n",
      "    486719 have location in profile\n",
      "    303672 geolocated\n",
      "202149 US users\n",
      "    9950 without state info\n",
      "    70400 without county info\n",
      "    71105 without city info\n"
     ]
    }
   ],
   "source": [
    "tweet_summary(evac_tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_fl_users(tweets):\n",
    "    users = tweets.groupby('username').first()\n",
    "    return users[(users.loc_country == 'United States') & (users.loc_state == 'Florida')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "22340 FL users\n",
    "8529 without county info\n",
    "8596 without city info\n",
    "Miami                 2529\n",
    "Tampa                 1631\n",
    "Orlando               1598\n",
    "Jacksonville           586\n",
    "Tallahassee            446\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25188 FL users\n",
      "9234 without county info\n",
      "9270 without city info\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Miami                 3576\n",
       "Tampa                 2097\n",
       "Orlando               2096\n",
       "Jacksonville           745\n",
       "Tallahassee            466\n",
       "Fort Lauderdale        384\n",
       "St Petersburg          322\n",
       "Gainesville            241\n",
       "Sarasota               220\n",
       "West Palm Beach        196\n",
       "Fort Myers             191\n",
       "Naples                 161\n",
       "Pensacola              159\n",
       "Boca Raton             153\n",
       "Jacksonville Beach     149\n",
       "Clearwater             137\n",
       "Miami Beach            136\n",
       "Hollywood              125\n",
       "Coral Gables           125\n",
       "Lakeland               123\n",
       "Edgewood               121\n",
       "Cape Coral             120\n",
       "Daytona Beach          117\n",
       "Jupiter                113\n",
       "Pembroke Pines         102\n",
       "Bradenton               99\n",
       "Port St Lucie           95\n",
       "Melbourne               93\n",
       "Ocala                   92\n",
       "Oviedo                  84\n",
       "Kissimmee               82\n",
       "Venus                   82\n",
       "St Augustine            78\n",
       "Delray Beach            71\n",
       "Homestead               68\n",
       "Boynton Beach           67\n",
       "Largo                   66\n",
       "Sweetwater              65\n",
       "Wesley Chapel           64\n",
       "Palm Beach              63\n",
       "Key West                62\n",
       "Riverview               61\n",
       "Panama City             57\n",
       "Coral Springs           57\n",
       "Palm Harbor             56\n",
       "New Port Richey         55\n",
       "Hialeah Gardens         53\n",
       "Spring Hill             51\n",
       "North Miami             51\n",
       "Pompano Beach           50\n",
       "Name: loc_city, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fl_users = get_fl_users(evac_tweets)\n",
    "print(f\"\"\"{len(fl_users)} FL users\n",
    "{sum(is_none(county) for county in fl_users.loc_county)} without county info\n",
    "{sum(is_none(city) for city in fl_users.loc_city)} without city info\"\"\")\n",
    "fl_users.loc_city.value_counts().iloc[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Miami-Dade County\n",
      "Hillsborough County\n",
      "Orange County\n",
      "Duval County\n",
      "Leon County\n"
     ]
    }
   ],
   "source": [
    "print(f\"\"\"{fl_users[fl_users.loc_city == 'Miami'].loc_county.iloc[0]}\n",
    "{fl_users[fl_users.loc_city == 'Tampa'].loc_county.iloc[0]}\n",
    "{fl_users[fl_users.loc_city == 'Orlando'].loc_county.iloc[0]}\n",
    "{fl_users[fl_users.loc_city == 'Jacksonville'].loc_county.iloc[0]}\n",
    "{fl_users[fl_users.loc_city == 'Tallahassee'].loc_county.iloc[0]}\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import re\n",
    "from preprocess import load_tweets, aggregate_tweet_info\n",
    "\n",
    "def to_dict(list_of_raw_tweets):\n",
    "    return {tweet['id']: tweet for tweet in list_of_raw_tweets}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "974002it [00:18, 52800.00it/s]\n"
     ]
    }
   ],
   "source": [
    "raw_tweets = to_dict(aggregate_tweet_info(load_tweets('evacuate.json')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "969796it [02:03, 7861.87it/s]\n"
     ]
    }
   ],
   "source": [
    "fl_cities = {\n",
    "    'Miami': 'Miami-Dade County',\n",
    "    'Tampa': 'Hillsborough County',\n",
    "    'Orlando': 'Orange County',\n",
    "    'Jacksonville': 'Duval County',\n",
    "    'Tallahassee': 'Leon County',\n",
    "    'Florida': None\n",
    "}\n",
    "\n",
    "for idx, tweet in tqdm(evac_tweets.iterrows()):\n",
    "    if not is_none(tweet.loc_city):\n",
    "        continue\n",
    "    usr = raw_tweets[tweet.tweet_id]['user']\n",
    "    uloc = (usr['location'] if 'location' in usr else '') + ' ' + usr['description']\n",
    "    for city in fl_cities:\n",
    "        if  re.search(fr'(\\b{city.lower()})|({city.lower()}\\b)', uloc.lower()) or re.search(city, uloc):\n",
    "            evac_tweets.at[idx, 'loc_country'] = 'United States'\n",
    "            evac_tweets.at[idx, 'loc_state'] = 'Florida'\n",
    "            evac_tweets.at[idx, 'loc_county'] = fl_cities[city]\n",
    "            evac_tweets.at[idx, 'loc_city'] = city if city != 'Florida' else None\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38792"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fl_tweets = evac_tweets[evac_tweets.loc_state == 'Florida']\n",
    "len(fl_tweets) # 33982"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extract info of positive tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "labeled_tweets = pd.read_csv('labeled_5000.csv', dtype={'tweet_id': str, 'user_id': str})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "fl_tweets = fl_tweets.join(labeled_tweets.set_index('tweet_id')[['label']], on='tweet_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "neutral     2005\n",
       "positive    1727\n",
       "negative    1267\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fl_tweets.label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnt = 0\n",
    "for idx, tweet in fl_users.iterrows():\n",
    "    if tweet.loc_city != 'Miami' or not is_none(tweet.user_location) and 'miami' in tweet.user_location.lower():\n",
    "        continue\n",
    "    cnt += 1\n",
    "    print('%03d' % cnt, tweet.tweet_id, raw_tweets[tweet.tweet_id]['user']['description'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from classification import tokenize, train, tweets_before_landfall\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "labeled_mask = np.array([not is_none(label) for label in fl_tweets.label])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4139it [00:01, 2895.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({'positive': 1579, 'neutral': 1434, 'negative': 1126})\n",
      "Counter({'positive': 1682, 'neutral': 1244, 'negative': 1213})\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative       0.80      0.87      0.83      1126\n",
      "     neutral       0.79      0.68      0.73      1434\n",
      "    positive       0.81      0.86      0.83      1579\n",
      "\n",
      "    accuracy                           0.80      4139\n",
      "   macro avg       0.80      0.80      0.80      4139\n",
      "weighted avg       0.80      0.80      0.80      4139\n",
      "\n",
      "auc=0.927 (positive vs others)\n",
      "auc=0.950 (negative vs others)\n",
      "auc=0.913 (weighted ovr)\n"
     ]
    }
   ],
   "source": [
    "X,y,vec,metas,clf,preds,probas = train(tweets_before_landfall(fl_tweets[labeled_mask]), False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "33793it [01:15, 447.28it/s]\n"
     ]
    }
   ],
   "source": [
    "# predictions on unlabeled data\n",
    "idxs = []\n",
    "u_preds = clf.predict(vec.transform(tokenize(txt) for idx, txt in tqdm(fl_tweets[~labeled_mask].iterrows()) if not idxs.append(idx)))\n",
    "pred_pos_idxs = np.array(idxs)[u_preds == 'positive']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_idxs = fl_tweets.index[fl_tweets.label == 'positive'].to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    2318\n",
       "True     1727\n",
       "Name: annotated, dtype: int64"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_tweet_metas = fl_tweets.loc[np.append(pos_idxs, pred_pos_idxs)][['tweet_time', 'loc_city', 'loc_county', 'username']]\n",
    "pos_tweet_metas = pos_tweet_metas.rename(columns={'tweet_time': 'time', 'loc_city': 'city', 'loc_county': 'county'})\n",
    "pos_tweet_metas['annotated'] = False\n",
    "pos_tweet_metas.iloc[:len(pos_idxs), -1] = True\n",
    "pos_tweet_metas['annotated'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fl_tweets[labeled_mask].to_csv('evacuation_FL_labeled_5000.csv', index=False)\n",
    "pos_tweet_metas.sort_values(by=['time']).to_csv('irma_positive_tweet_metadata.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1727 labeled & 2318 predicted positive (=4045) from 3518 users\n",
    "\n",
    "1533 unknown cities, 1529 unknown counties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1533"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(is_none(city) for city in pos_tweet_metas.city)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}