extract_english.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import the required librarie\n",
    "import pandas as pd\n",
    "import re, string, unicodedata\n",
    "import nltk\n",
    "#import contractions\n",
    "#import inflect\n",
    "from nltk import word_tokenize, sent_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import LancasterStemmer, WordNetLemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "no_plastic_file_url = 'https://raw.githubusercontent.com/Vancouver-Datajam/project_5/master/hashtags/noplastic.csv'\n",
    "zero_waste_file_url = 'https://raw.githubusercontent.com/Vancouver-Datajam/project_5/master/hashtags/zerowaste.csv'\n",
    "sustain_file_url = 'https://raw.githubusercontent.com/Vancouver-Datajam/project_5/master/hashtags/sustainability.csv'\n",
    "plastic_pollute_file_url = 'https://raw.githubusercontent.com/Vancouver-Datajam/project_5/master/hashtags/plasticpollutes.csv'\n",
    "plastic_poll1_file_url = 'https://raw.githubusercontent.com/Vancouver-Datajam/project_5/master/hashtags/plasticpollution1.csv'\n",
    "plastic_poll2_file_url = 'https://raw.githubusercontent.com/Vancouver-Datajam/project_5/master/hashtags/plasticpollution2.csv'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load the datasets using .pd.read_csv().\n",
    "no_plastic_data = pd.read_csv(no_plastic_file_url)\n",
    "zero_waste_data = pd.read_csv(zero_waste_file_url)\n",
    "sustainab_data = pd.read_csv(sustain_file_url)\n",
    "plastic_pollute_data = pd.read_csv(plastic_pollute_file_url)\n",
    "plastic_poll1_data = pd.read_csv(plastic_poll1_file_url)\n",
    "plastic_poll2_data = pd.read_csv(plastic_poll2_file_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#no_plastic_data.columns #(325, 31)\n",
    "#zero_waste_data.shape #(302, 31)\n",
    "#sustainab_data.shape #(5000, 31)\n",
    "#plastic_pollute_data.shape #(64, 31)\n",
    "#plastic_poll2_data.shape #(1407, 31) \n",
    "#plastic_poll1_data.shape #(2378, 72) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rem_url(tweet_text):\n",
    "     return re.sub(r\"http\\S+\", \"\", tweet_text)      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#no_plastic_data\n",
    "no_plastic_en = no_plastic_data.loc[no_plastic_data['lang'] == 'en', \n",
    "                                    ['full_text', 'entities', 'retweet_count', 'favorite_count']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#zero_waste_data\n",
    "zero_waste_en = zero_waste_data.loc[zero_waste_data['lang'] == 'en', \n",
    "                                    ['full_text', 'entities', 'retweet_count', 'favorite_count']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sustainab_data\n",
    "sustainab_en = sustainab_data.loc[sustainab_data['lang'] == 'en', \n",
    "                                    ['full_text', 'entities', 'retweet_count', 'favorite_count']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plastic_pollute_data\n",
    "plastic_pollute_en = plastic_pollute_data.loc[plastic_pollute_data['lang'] == 'en', \n",
    "                                    ['full_text', 'entities', 'retweet_count', 'favorite_count']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plastic_poll2\n",
    "plastic_poll2_en = plastic_poll2_data.loc[plastic_poll2_data['lang'] == 'en', \n",
    "                                    ['full_text', 'entities', 'retweet_count', 'favorite_count']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plastic_poll1_data.columns\n",
    "plastic_poll1_en = plastic_poll1_data.loc[plastic_poll1_data['lang'] == 'en', ['full_text', 'hashtags', 'retweet_count', 'favorite_count']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "no_plastic_en['full_text'] = no_plastic_en['full_text'].apply(rem_url)\n",
    "zero_waste_en['full_text'] = zero_waste_en['full_text'].apply(rem_url)\n",
    "sustainab_en['full_text'] = sustainab_en['full_text'].apply(rem_url)\n",
    "plastic_pollute_en['full_text'] = plastic_pollute_en['full_text'].apply(rem_url)\n",
    "plastic_poll2_en['full_text'] = plastic_poll2_en['full_text'].apply(rem_url)\n",
    "plastic_poll1_en['full_text'] = plastic_poll1_en['full_text'].apply(rem_url)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                             full_text  \\\n",
      "0    Great feeling when you protect&amp;let it go i...   \n",
      "1    We all need to live more sustainably or we wil...   \n",
      "2    @PemaKhanduBJP @PMOIndia @HMOIndia @DefenceMin...   \n",
      "3    One of the most refreshing drinks you can have...   \n",
      "4    Today's anniversary #sale items! Offer valid u...   \n",
      "..                                                 ...   \n",
      "315  This is just horrible.\\n\\n#noplastic #plasticf...   \n",
      "320  \"We have the opportunity to decide whether we ...   \n",
      "321  New recipes with new colours. Love being creat...   \n",
      "323  Dinner's ready  #sustainableliving #noplastic ...   \n",
      "324  @HughFW no plastic in @clipperteas teabags.. #...   \n",
      "\n",
      "                                              entities  retweet_count  \\\n",
      "0    {'hashtags': [{'text': 'ROBIN', 'indices': [55...              1   \n",
      "1    {'hashtags': [{'text': 'SupportYourLocalFarmer...              0   \n",
      "2    {'hashtags': [{'text': 'NoPlastic', 'indices':...              0   \n",
      "3    {'hashtags': [{'text': 'sweetmango', 'indices'...              0   \n",
      "4    {'hashtags': [{'text': 'sale', 'indices': [20,...              1   \n",
      "..                                                 ...            ...   \n",
      "315  {'hashtags': [{'text': 'noplastic', 'indices':...              5   \n",
      "320  {'hashtags': [{'text': 'Sustainability', 'indi...              2   \n",
      "321  {'hashtags': [{'text': 'soapmaking', 'indices'...              3   \n",
      "323  {'hashtags': [{'text': 'sustainableliving', 'i...              0   \n",
      "324  {'hashtags': [{'text': 'organic', 'indices': [...              1   \n",
      "\n",
      "     favorite_count  \n",
      "0                 4  \n",
      "1                 1  \n",
      "2                 0  \n",
      "3                 2  \n",
      "4                 2  \n",
      "..              ...  \n",
      "315               7  \n",
      "320               3  \n",
      "321              11  \n",
      "323               1  \n",
      "324               2  \n",
      "\n",
      "[175 rows x 4 columns]\n"
     ]
    }
   ],
   "source": [
    "print(no_plastic_en)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}