YScrawler.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import scrapy\n",
    "import json\n",
    "from scrapy.crawler import CrawlerProcess\n",
    "from scrapy.spiders import CrawlSpider\n",
    "from scrapy.utils.project import get_project_settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from scrapy import optional_features\n",
    "optional_features.remove('boto')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "companies = []\n",
    "with open('companies.csv','r') as f:\n",
    "    for line in f:\n",
    "        companies.append(line[:-1])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(companies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "companies = map(lambda x:x.lower(),companies)\n",
    "companies_l = map(lambda x:x.split(),companies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "title = ''\n",
    "urls = ''\n",
    "post_text = []\n",
    "max_pages = 2\n",
    "paras = []\n",
    "class YsItem(scrapy.Item):\n",
    "    # define the fields for your item here like:\n",
    "    cname = scrapy.Field()\n",
    "    title = scrapy.Field()\n",
    "    post = scrapy.Field()\n",
    "    \n",
    "class YsSpider(CrawlSpider):\n",
    "    global max_pages\n",
    "    name = \"ysposts\"\n",
    "    allowed_domains = [\"yourstory.com\"]\n",
    "    start_urls = []\n",
    "    for c in companies[0:36]:\n",
    "        for page in range(1,20):\n",
    "            start_urls.append('http://yourstory.com/?s='+c+'&paged='+str(page))\n",
    "    \n",
    "    def process_title(self,t):\n",
    "        exclude = set(string.punctuation)\n",
    "        t = ''.join(ch if ch not in exclude else ' ' for ch in list(t))\n",
    "        t = t.strip()\n",
    "        t = t.lower()\n",
    "        t = t.split()\n",
    "        return t\n",
    "    \n",
    "    def get_cname(self,title):\n",
    "        cname = []\n",
    "        for word in title:\n",
    "            for comp in companies_l:\n",
    "                if word in comp:\n",
    "                    cname.append(comp)\n",
    "        return cname\n",
    "        \n",
    "    \n",
    "    def getURLS(self,title):\n",
    "        url_vec = []\n",
    "        for t in title:\n",
    "            t = self.process_title(t)\n",
    "            uv = 0\n",
    "            for word in t:\n",
    "                for comp in companies_l:\n",
    "                    if word in comp:\n",
    "                        uv=1\n",
    "                        #print(t)\n",
    "            url_vec.append(uv)\n",
    "        return url_vec\n",
    "\n",
    "    def parse(self, response):\n",
    "            global title\n",
    "            global urls\n",
    "            title = response.xpath('''//div[@class=\"title-small \n",
    "                                    bentonCondensed bold \n",
    "                                    color-black-2 truncate-2\"] \n",
    "                                   /text()''').extract()\n",
    "            title = [x.strip().encode('ascii','replace') for x in title]\n",
    "            if len(title[0])!=0:                \n",
    "                urls = response.xpath('//a[@class=\"block\"]/@href')\n",
    "                url_vec = self.getURLS(title)\n",
    "                for i in range(len(url_vec)):\n",
    "                    if url_vec[i]==1:\n",
    "                        url = urls[i].extract()\n",
    "                        yield scrapy.Request(url, callback=self.parse_dir_contents)\n",
    " \n",
    "\n",
    "\n",
    "    def parse_dir_contents(self, response):\n",
    "        global paras\n",
    "        global post_text\n",
    "        item = YsItem()\n",
    "        t = response.xpath('//h3[@class=\"title color-ys\"]/text()').extract()\n",
    "        #print(t[0])\n",
    "        it = self.process_title(t[0].encode('ascii','replace'))\n",
    "        #print(it)\n",
    "        item['title'] = it\n",
    "        item['cname'] = self.get_cname(it)\n",
    "        paras = response.xpath('//div[@class=\"ys_post_content text\"]//p')\n",
    "        post_text = ' '\n",
    "        for para in paras:\n",
    "            post_text += \" \".join(para.xpath('text()').extract())\n",
    "            post_text += \" \"\n",
    "        post_text = self.process_title(post_text.encode('ascii','replace'))\n",
    "        item['post'] = post_text\n",
    "        if len(item['cname'])!=0:\n",
    "            with open('ysnote_new.json','a') as f:\n",
    "                json.dump({'cname':item.get('cname'),\n",
    "                           'title':item.get('title'),\n",
    "                           'post':item.get('post')},f)\n",
    "        \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:scrapy.utils.log:Scrapy 1.0.5 started (bot: scrapybot)\n",
      "2016-03-25 01:24:33 [scrapy] INFO: Scrapy 1.0.5 started (bot: scrapybot)\n",
      "INFO:scrapy.utils.log:Optional features available: ssl, http11\n",
      "2016-03-25 01:24:33 [scrapy] INFO: Optional features available: ssl, http11\n",
      "INFO:scrapy.utils.log:Overridden settings: {}\n",
      "2016-03-25 01:24:33 [scrapy] INFO: Overridden settings: {}\n",
      "INFO:scrapy.middleware:Enabled extensions: CloseSpider, TelnetConsole, LogStats, CoreStats, SpiderState\n",
      "2016-03-25 01:24:33 [scrapy] INFO: Enabled extensions: CloseSpider, TelnetConsole, LogStats, CoreStats, SpiderState\n",
      "INFO:scrapy.middleware:Enabled downloader middlewares: HttpAuthMiddleware, DownloadTimeoutMiddleware, UserAgentMiddleware, RetryMiddleware, DefaultHeadersMiddleware, MetaRefreshMiddleware, HttpCompressionMiddleware, RedirectMiddleware, CookiesMiddleware, ChunkedTransferMiddleware, DownloaderStats\n",
      "2016-03-25 01:24:33 [scrapy] INFO: Enabled downloader middlewares: HttpAuthMiddleware, DownloadTimeoutMiddleware, UserAgentMiddleware, RetryMiddleware, DefaultHeadersMiddleware, MetaRefreshMiddleware, HttpCompressionMiddleware, RedirectMiddleware, CookiesMiddleware, ChunkedTransferMiddleware, DownloaderStats\n",
      "INFO:scrapy.middleware:Enabled spider middlewares: HttpErrorMiddleware, OffsiteMiddleware, RefererMiddleware, UrlLengthMiddleware, DepthMiddleware\n",
      "2016-03-25 01:24:33 [scrapy] INFO: Enabled spider middlewares: HttpErrorMiddleware, OffsiteMiddleware, RefererMiddleware, UrlLengthMiddleware, DepthMiddleware\n",
      "INFO:scrapy.middleware:Enabled item pipelines: \n",
      "2016-03-25 01:24:33 [scrapy] INFO: Enabled item pipelines: \n",
      "INFO:scrapy.core.engine:Spider opened\n",
      "2016-03-25 01:24:33 [scrapy] INFO: Spider opened\n",
      "INFO:scrapy.extensions.logstats:Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n",
      "2016-03-25 01:24:33 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n",
      "DEBUG:scrapy.telnet:Telnet console listening on 127.0.0.1:6023\n",
      "2016-03-25 01:24:33 [scrapy] DEBUG: Telnet console listening on 127.0.0.1:6023\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=1> (referer: None)\n",
      "2016-03-25 01:24:35 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=1> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=4> (referer: None)\n",
      "2016-03-25 01:24:35 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=4> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=7> (referer: None)\n",
      "2016-03-25 01:24:35 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=7> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=2> (referer: None)\n",
      "2016-03-25 01:24:35 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=2> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=6> (referer: None)\n",
      "2016-03-25 01:24:35 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=6> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=3> (referer: None)\n",
      "2016-03-25 01:24:35 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=3> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=8> (referer: None)\n",
      "2016-03-25 01:24:35 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=8> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=5> (referer: None)\n",
      "2016-03-25 01:24:35 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=5> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=9> (referer: None)\n",
      "2016-03-25 01:24:36 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=9> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=12> (referer: None)\n",
      "2016-03-25 01:24:36 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=12> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2016/02/startup-journeys-health-quotes/> (referer: http://yourstory.com/?s=gapoon&paged=1)\n",
      "2016-03-25 01:24:36 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2016/02/startup-journeys-health-quotes/> (referer: http://yourstory.com/?s=gapoon&paged=1)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2015/10/gapoon/> (referer: http://yourstory.com/?s=gapoon&paged=1)\n",
      "2016-03-25 01:24:36 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2015/10/gapoon/> (referer: http://yourstory.com/?s=gapoon&paged=1)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=14> (referer: None)\n",
      "2016-03-25 01:24:36 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=14> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=10> (referer: None)\n",
      "2016-03-25 01:24:36 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=10> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=13> (referer: None)\n",
      "2016-03-25 01:24:36 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=13> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=16> (referer: None)\n",
      "2016-03-25 01:24:36 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=16> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=15> (referer: None)\n",
      "2016-03-25 01:24:37 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=15> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=17> (referer: None)\n",
      "2016-03-25 01:24:37 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=17> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=11> (referer: None)\n",
      "2016-03-25 01:24:37 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=11> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=19> (referer: None)\n",
      "2016-03-25 01:24:37 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=19> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=1> (referer: None)\n",
      "2016-03-25 01:24:37 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=1> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=18> (referer: None)\n",
      "2016-03-25 01:24:38 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=gapoon&paged=18> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=4> (referer: None)\n",
      "2016-03-25 01:24:38 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=4> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=3> (referer: None)\n",
      "2016-03-25 01:24:38 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=3> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=2> (referer: None)\n",
      "2016-03-25 01:24:38 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=2> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=5> (referer: None)\n",
      "2016-03-25 01:24:38 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=5> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=6> (referer: None)\n",
      "2016-03-25 01:24:38 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=6> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2015/12/juicifix/> (referer: http://yourstory.com/?s=gapoon&paged=3)\n",
      "2016-03-25 01:24:39 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2015/12/juicifix/> (referer: http://yourstory.com/?s=gapoon&paged=3)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=8> (referer: None)\n",
      "2016-03-25 01:24:39 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=8> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=7> (referer: None)\n",
      "2016-03-25 01:24:39 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=7> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=9> (referer: None)\n",
      "2016-03-25 01:24:39 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=9> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=10> (referer: None)\n",
      "2016-03-25 01:24:39 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=10> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2015/10/parcelled-fund-raise-delhivery/> (referer: http://yourstory.com/?s=parcelled&paged=1)\n",
      "2016-03-25 01:24:39 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2015/10/parcelled-fund-raise-delhivery/> (referer: http://yourstory.com/?s=parcelled&paged=1)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=11> (referer: None)\n",
      "2016-03-25 01:24:39 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=11> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=12> (referer: None)\n",
      "2016-03-25 01:24:40 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=12> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2009/03/public-health-and-mainstream-media-2/> (referer: http://yourstory.com/?s=gapoon&paged=16)\n",
      "2016-03-25 01:24:40 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2009/03/public-health-and-mainstream-media-2/> (referer: http://yourstory.com/?s=gapoon&paged=16)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=14> (referer: None)\n",
      "2016-03-25 01:24:40 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=14> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=13> (referer: None)\n",
      "2016-03-25 01:24:40 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=13> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2011/12/karisma-kapur-falls-in-love-with-baby-products-online-store-babyoye-com-turns-investor/> (referer: http://yourstory.com/?s=gapoon&paged=15)\n",
      "2016-03-25 01:24:40 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2011/12/karisma-kapur-falls-in-love-with-baby-products-online-store-babyoye-com-turns-investor/> (referer: http://yourstory.com/?s=gapoon&paged=15)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=15> (referer: None)\n",
      "2016-03-25 01:24:40 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=15> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=16> (referer: None)\n",
      "2016-03-25 01:24:40 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=16> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=17> (referer: None)\n",
      "2016-03-25 01:24:40 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=17> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2016/02/cardekho-funding/> (referer: http://yourstory.com/?s=parcelled&paged=9)\n",
      "2016-03-25 01:24:41 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2016/02/cardekho-funding/> (referer: http://yourstory.com/?s=parcelled&paged=9)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=18> (referer: None)\n",
      "2016-03-25 01:24:41 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=18> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2015/11/kunal-shah-freecharge/> (referer: http://yourstory.com/?s=parcelled&paged=12)\n",
      "2016-03-25 01:24:41 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2015/11/kunal-shah-freecharge/> (referer: http://yourstory.com/?s=parcelled&paged=12)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2016/03/hod-life/> (referer: http://yourstory.com/?s=parcelled&paged=12)\n",
      "2016-03-25 01:24:41 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2016/03/hod-life/> (referer: http://yourstory.com/?s=parcelled&paged=12)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2016/02/closeconnexions/> (referer: http://yourstory.com/?s=parcelled&paged=12)\n",
      "2016-03-25 01:24:41 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2016/02/closeconnexions/> (referer: http://yourstory.com/?s=parcelled&paged=12)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=19> (referer: None)\n",
      "2016-03-25 01:24:41 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=parcelled&paged=19> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2016/02/snackible/> (referer: http://yourstory.com/?s=parcelled&paged=11)\n",
      "2016-03-25 01:24:41 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2016/02/snackible/> (referer: http://yourstory.com/?s=parcelled&paged=11)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=1> (referer: None)\n",
      "2016-03-25 01:24:41 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=1> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=5> (referer: None)\n",
      "2016-03-25 01:24:41 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=5> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=4> (referer: None)\n",
      "2016-03-25 01:24:42 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=4> (referer: None)\n",
      "DEBUG:scrapy.dupefilters:Filtered duplicate request: <GET http://yourstory.com/2015/12/funding-roundup-dec-2015-3/> - no more duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)\n",
      "2016-03-25 01:24:42 [scrapy] DEBUG: Filtered duplicate request: <GET http://yourstory.com/2015/12/funding-roundup-dec-2015-3/> - no more duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=2> (referer: None)\n",
      "2016-03-25 01:24:42 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=2> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=3> (referer: None)\n",
      "2016-03-25 01:24:42 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=3> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=10> (referer: None)\n",
      "2016-03-25 01:24:42 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=10> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=11> (referer: None)\n",
      "2016-03-25 01:24:42 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=11> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=7> (referer: None)\n",
      "2016-03-25 01:24:42 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=7> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=6> (referer: None)\n",
      "2016-03-25 01:24:42 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=6> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=9> (referer: None)\n",
      "2016-03-25 01:24:42 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=9> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=8> (referer: None)\n",
      "2016-03-25 01:24:43 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=8> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=12> (referer: None)\n",
      "2016-03-25 01:24:43 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=12> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2015/12/funding-roundup-dec-2015-3/> (referer: http://yourstory.com/?s=parcelled&paged=11)\n",
      "2016-03-25 01:24:43 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2015/12/funding-roundup-dec-2015-3/> (referer: http://yourstory.com/?s=parcelled&paged=11)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=13> (referer: None)\n",
      "2016-03-25 01:24:43 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=13> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=15> (referer: None)\n",
      "2016-03-25 01:24:43 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=15> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=14> (referer: None)\n",
      "2016-03-25 01:24:43 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=14> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=16> (referer: None)\n",
      "2016-03-25 01:24:43 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=16> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=17> (referer: None)\n",
      "2016-03-25 01:24:43 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=17> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=18> (referer: None)\n",
      "2016-03-25 01:24:43 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=18> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2015/10/opinio-funding/> (referer: http://yourstory.com/?s=parcelled&paged=18)\n",
      "2016-03-25 01:24:43 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2015/10/opinio-funding/> (referer: http://yourstory.com/?s=parcelled&paged=18)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=19> (referer: None)\n",
      "2016-03-25 01:24:44 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=rivigo&paged=19> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=1> (referer: None)\n",
      "2016-03-25 01:24:44 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=1> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=2> (referer: None)\n",
      "2016-03-25 01:24:44 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=2> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=3> (referer: None)\n",
      "2016-03-25 01:24:44 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=3> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2014/03/wello/> (referer: http://yourstory.com/?s=rivigo&paged=5)\n",
      "2016-03-25 01:24:45 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2014/03/wello/> (referer: http://yourstory.com/?s=rivigo&paged=5)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=4> (referer: None)\n",
      "2016-03-25 01:24:45 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=4> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2015/12/saif-partners-30-million-into-rivigo/> (referer: http://yourstory.com/?s=rivigo&paged=1)\n",
      "2016-03-25 01:24:45 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2015/12/saif-partners-30-million-into-rivigo/> (referer: http://yourstory.com/?s=rivigo&paged=1)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2014/01/jobs-roundup-3/> (referer: http://yourstory.com/?s=rivigo&paged=5)\n",
      "2016-03-25 01:24:45 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2014/01/jobs-roundup-3/> (referer: http://yourstory.com/?s=rivigo&paged=5)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=5> (referer: None)\n",
      "2016-03-25 01:24:45 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=5> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=6> (referer: None)\n",
      "2016-03-25 01:24:45 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=6> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=8> (referer: None)\n",
      "2016-03-25 01:24:46 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=8> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=7> (referer: None)\n",
      "2016-03-25 01:24:46 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=7> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2013/05/jobs-roundup-zipdial-shopzone-ixigo-and-many-more-are-looking-to-hire-you/> (referer: http://yourstory.com/?s=rivigo&paged=8)\n",
      "2016-03-25 01:24:46 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2013/05/jobs-roundup-zipdial-shopzone-ixigo-and-many-more-are-looking-to-hire-you/> (referer: http://yourstory.com/?s=rivigo&paged=8)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2015/09/opinio/> (referer: http://yourstory.com/?s=opinio&paged=1)\n",
      "2016-03-25 01:24:46 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2015/09/opinio/> (referer: http://yourstory.com/?s=opinio&paged=1)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=10> (referer: None)\n",
      "2016-03-25 01:24:46 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=10> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2015/12/amit-jain-cardekho/> (referer: http://yourstory.com/?s=opinio&paged=3)\n",
      "2016-03-25 01:24:46 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2015/12/amit-jain-cardekho/> (referer: http://yourstory.com/?s=opinio&paged=3)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=9> (referer: None)\n",
      "2016-03-25 01:24:46 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=9> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=12> (referer: None)\n",
      "2016-03-25 01:24:46 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=12> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=11> (referer: None)\n",
      "2016-03-25 01:24:46 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=11> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2016/02/health-tech-india/> (referer: http://yourstory.com/?s=opinio&paged=5)\n",
      "2016-03-25 01:24:46 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2016/02/health-tech-india/> (referer: http://yourstory.com/?s=opinio&paged=5)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=13> (referer: None)\n",
      "2016-03-25 01:24:47 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=13> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=16> (referer: None)\n",
      "2016-03-25 01:24:47 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=16> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=15> (referer: None)\n",
      "2016-03-25 01:24:47 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=15> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=14> (referer: None)\n",
      "2016-03-25 01:24:47 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=14> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=17> (referer: None)\n",
      "2016-03-25 01:24:47 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=17> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2015/11/carwale-cartrade-merger/> (referer: http://yourstory.com/?s=opinio&paged=7)\n",
      "2016-03-25 01:24:47 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2015/11/carwale-cartrade-merger/> (referer: http://yourstory.com/?s=opinio&paged=7)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=18> (referer: None)\n",
      "2016-03-25 01:24:47 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=18> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=opinio&paged=19> (referer: None)\n",
      "2016-03-25 01:24:47 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=opinio&paged=19> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=&paged=1> (referer: None)\n",
      "2016-03-25 01:24:47 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=&paged=1> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=2> (referer: None)\n",
      "2016-03-25 01:24:48 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=2> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2015/11/tinyowl-perspective-founder/> (referer: http://yourstory.com/?s=opinio&paged=9)\n",
      "2016-03-25 01:24:48 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2015/11/tinyowl-perspective-founder/> (referer: http://yourstory.com/?s=opinio&paged=9)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=2>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:48 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=2>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=4> (referer: None)\n",
      "2016-03-25 01:24:48 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=4> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=3> (referer: None)\n",
      "2016-03-25 01:24:48 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=3> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=4>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:48 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=4>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=3>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:48 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=3>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=5> (referer: None)\n",
      "2016-03-25 01:24:48 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=5> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=5>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:48 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=5>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=8> (referer: None)\n",
      "2016-03-25 01:24:48 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=8> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=6> (referer: None)\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=6> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=7> (referer: None)\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=7> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=8>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=8>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=6>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=6>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=10> (referer: None)\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=10> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=9> (referer: None)\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=9> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=7>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=7>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=10>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=10>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=9>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=9>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=11> (referer: None)\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=11> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=11>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=11>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=12> (referer: None)\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=12> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=12>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=12>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=13> (referer: None)\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=13> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=13>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=13>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=15> (referer: None)\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=15> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=15>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=15>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=16> (referer: None)\n",
      "2016-03-25 01:24:49 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=16> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/?s=&paged=1> (referer: None)\n",
      "2016-03-25 01:24:50 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/?s=&paged=1> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=16>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:50 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=16>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=14> (referer: None)\n",
      "2016-03-25 01:24:50 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=14> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=14>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:50 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=14>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=17> (referer: None)\n",
      "2016-03-25 01:24:50 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=17> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=17>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:50 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=17>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=18> (referer: None)\n",
      "2016-03-25 01:24:50 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=18> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=18>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:50 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=18>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2015/04/tie-bangalore-health-retail-startups-flipkart-mdhil/> (referer: http://yourstory.com/?s=opinio&paged=19)\n",
      "2016-03-25 01:24:50 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2015/04/tie-bangalore-health-retail-startups-flipkart-mdhil/> (referer: http://yourstory.com/?s=opinio&paged=19)\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=19> (referer: None)\n",
      "2016-03-25 01:24:50 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=19> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=19>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:50 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=19>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=2> (referer: None)\n",
      "2016-03-25 01:24:50 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=2> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=3> (referer: None)\n",
      "2016-03-25 01:24:50 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=3> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2015/06/google-analytics/> (referer: http://yourstory.com/?s=opinio&paged=17)\n",
      "2016-03-25 01:24:50 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2015/06/google-analytics/> (referer: http://yourstory.com/?s=opinio&paged=17)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=2>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:51 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=2>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=3>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:51 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=3>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (200) <GET http://yourstory.com/2015/06/techie-tuesdays-how-lohith-vrushabendrappa/> (referer: http://yourstory.com/?s=opinio&paged=17)\n",
      "2016-03-25 01:24:51 [scrapy] DEBUG: Crawled (200) <GET http://yourstory.com/2015/06/techie-tuesdays-how-lohith-vrushabendrappa/> (referer: http://yourstory.com/?s=opinio&paged=17)\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=5> (referer: None)\n",
      "2016-03-25 01:24:51 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=5> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=4> (referer: None)\n",
      "2016-03-25 01:24:51 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=4> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=5>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:51 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=5>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=4>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:51 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=4>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=6> (referer: None)\n",
      "2016-03-25 01:24:51 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=6> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=6>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:51 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=6>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=9> (referer: None)\n",
      "2016-03-25 01:24:51 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=9> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=9>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:51 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=9>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=7> (referer: None)\n",
      "2016-03-25 01:24:51 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=7> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=7>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:52 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=7>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=11> (referer: None)\n",
      "2016-03-25 01:24:52 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=11> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=8> (referer: None)\n",
      "2016-03-25 01:24:52 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=8> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=11>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:52 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=11>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=8>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:52 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=8>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=12> (referer: None)\n",
      "2016-03-25 01:24:52 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=12> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=10> (referer: None)\n",
      "2016-03-25 01:24:52 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=10> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=13> (referer: None)\n",
      "2016-03-25 01:24:52 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=13> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=12>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:52 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=12>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=10>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:52 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=10>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=13>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:52 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=13>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=15> (referer: None)\n",
      "2016-03-25 01:24:52 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=15> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=15>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:52 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=15>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=14> (referer: None)\n",
      "2016-03-25 01:24:52 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=14> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=16> (referer: None)\n",
      "2016-03-25 01:24:52 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=16> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=14>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:52 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=14>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=16>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:52 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=16>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=17> (referer: None)\n",
      "2016-03-25 01:24:53 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=17> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=19> (referer: None)\n",
      "2016-03-25 01:24:53 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=19> (referer: None)\n",
      "DEBUG:scrapy.core.engine:Crawled (404) <GET http://yourstory.com/?s=&paged=18> (referer: None)\n",
      "2016-03-25 01:24:53 [scrapy] DEBUG: Crawled (404) <GET http://yourstory.com/?s=&paged=18> (referer: None)\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=17>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:53 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=17>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=19>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:53 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=19>: HTTP status code is not handled or not allowed\n",
      "DEBUG:scrapy.spidermiddlewares.httperror:Ignoring response <404 http://yourstory.com/?s=&paged=18>: HTTP status code is not handled or not allowed\n",
      "2016-03-25 01:24:53 [scrapy] DEBUG: Ignoring response <404 http://yourstory.com/?s=&paged=18>: HTTP status code is not handled or not allowed\n",
      "INFO:scrapy.core.engine:Closing spider (finished)\n",
      "2016-03-25 01:24:53 [scrapy] INFO: Closing spider (finished)\n",
      "INFO:scrapy.statscollectors:Dumping Scrapy stats:\n",
      "{'downloader/request_bytes': 38834,\n",
      " 'downloader/request_count': 139,\n",
      " 'downloader/request_method_count/GET': 139,\n",
      " 'downloader/response_bytes': 1937054,\n",
      " 'downloader/response_count': 139,\n",
      " 'downloader/response_status_count/200': 103,\n",
      " 'downloader/response_status_count/404': 36,\n",
      " 'dupefilter/filtered': 2,\n",
      " 'finish_reason': 'finished',\n",
      " 'finish_time': datetime.datetime(2016, 3, 24, 19, 54, 53, 289000),\n",
      " 'log_count/DEBUG': 177,\n",
      " 'log_count/INFO': 7,\n",
      " 'request_depth_max': 1,\n",
      " 'response_received_count': 139,\n",
      " 'scheduler/dequeued': 139,\n",
      " 'scheduler/dequeued/memory': 139,\n",
      " 'scheduler/enqueued': 139,\n",
      " 'scheduler/enqueued/memory': 139,\n",
      " 'start_time': datetime.datetime(2016, 3, 24, 19, 54, 33, 790000)}\n",
      "2016-03-25 01:24:53 [scrapy] INFO: Dumping Scrapy stats:\n",
      "{'downloader/request_bytes': 38834,\n",
      " 'downloader/request_count': 139,\n",
      " 'downloader/request_method_count/GET': 139,\n",
      " 'downloader/response_bytes': 1937054,\n",
      " 'downloader/response_count': 139,\n",
      " 'downloader/response_status_count/200': 103,\n",
      " 'downloader/response_status_count/404': 36,\n",
      " 'dupefilter/filtered': 2,\n",
      " 'finish_reason': 'finished',\n",
      " 'finish_time': datetime.datetime(2016, 3, 24, 19, 54, 53, 289000),\n",
      " 'log_count/DEBUG': 177,\n",
      " 'log_count/INFO': 7,\n",
      " 'request_depth_max': 1,\n",
      " 'response_received_count': 139,\n",
      " 'scheduler/dequeued': 139,\n",
      " 'scheduler/dequeued/memory': 139,\n",
      " 'scheduler/enqueued': 139,\n",
      " 'scheduler/enqueued/memory': 139,\n",
      " 'start_time': datetime.datetime(2016, 3, 24, 19, 54, 33, 790000)}\n",
      "INFO:scrapy.core.engine:Spider closed (finished)\n",
      "2016-03-25 01:24:53 [scrapy] INFO: Spider closed (finished)\n"
     ]
    }
   ],
   "source": [
    "process = CrawlerProcess(get_project_settings())\n",
    "process.crawl(YsSpider)\n",
    "process.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}