counter_and_leak.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gzip\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "import csv\n",
    "import time\n",
    "import sys\n",
    "import math\n",
    "import random\n",
    "import seaborn as sns\n",
    "from collections import defaultdict\n",
    "import collections\n",
    "import json\n",
    "import shutil\n",
    "import bisect\n",
    "\n",
    "from IPython.display import display, clear_output\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "#some numbers that takes a while to compute, and I dont want to run the computation again\n",
    "nrows_train = 37670294\n",
    "nrows_test = 2528244\n",
    "nuser_ids = 1198787\n",
    "nbookings_train = 3000694\n",
    "\n",
    "base_path = os.getcwd()\n",
    "data_path = os.path.join(base_path, 'data/')\n",
    "\n",
    "\n",
    "with open(os.path.join(data_path, 'train_augmented.csv')) as inf:\n",
    "    legge = csv.reader(inf)\n",
    "    columns = next(legge)\n",
    "\n",
    "TRAIN_FEAT_DICT = dict(list(zip(columns, range(len(columns)))))\n",
    "\n",
    "with open(os.path.join(data_path, 'train_augmented.csv')) as inf:\n",
    "    legge = csv.reader(inf)\n",
    "    columns = next(legge)\n",
    "    \n",
    "TEST_FEAT_DICT = dict(list(zip(columns, range(len(columns)))))\n",
    "\n",
    "\n",
    "def count_lines(filename, iszip=False):\n",
    "    \"\"\"Counts the number of lines in a file\"\"\"\n",
    "    if iszip:\n",
    "        with gzip.open(filename) as file:\n",
    "            for i, line in enumerate(file):\n",
    "                if i % 2**20 == 0:\n",
    "                    print('Got to line {:.2f}M'.format(i/(10**6)))\n",
    "    else:\n",
    "        with open(filename) as file:\n",
    "            for i, line in enumerate(file):\n",
    "                if i % 2**20 == 0:\n",
    "                    print('Got to line {:.2f}M'.format(i/(10**6)))\n",
    "\n",
    "    return i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_magnitude(number):\n",
    "    \"\"\"\n",
    "    A function that returns the order of magnitude of a number.\n",
    "    unit is the label of the order of magnitude, etodivide the exponent\n",
    "    to divide the number by in order to obtain the appropriate division.\n",
    "    \"\"\"\n",
    "    magnitude = math.log(number, 10)\n",
    "    n_unit = int(magnitude / 3)\n",
    "    if n_unit == 0:\n",
    "        unit = ''\n",
    "        etodivide = 0\n",
    "    elif n_unit == 1:\n",
    "        unit = 'K'\n",
    "        etodivide = 3\n",
    "    elif n_unit == 2:\n",
    "        unit = 'M'\n",
    "        etodivide = 6\n",
    "    elif n_unit == 3:\n",
    "        unit = 'G'\n",
    "        etodivide = 9\n",
    "    else:\n",
    "        unit = 'E' + str(magnitude)\n",
    "        etodivide = magnitude\n",
    "        \n",
    "    return unit, etodivide\n",
    "\n",
    "\n",
    "def print_processing_status(current_line, tot_line, s_time, frequency=None, pre_message=None):\n",
    "    \"\"\"\n",
    "    A function that prints to screen the processing status of another function.\n",
    "    Current_line is the line being processed right now, \n",
    "    tot_line the total number of lines to be processed,\n",
    "    s_time the time the computation started,\n",
    "    frequency how often the print message should be printed (if None, 1/100 of tot_line),\n",
    "    pre_message the string that should be printed pefore the status message.\n",
    "    \"\"\"\n",
    "    if frequency is None:\n",
    "        frequency = int(tot_line / 100)\n",
    "    message = 'Processing line {0:.1f}{4} (of {1:.1f}{4}). \\t Elapsed time: {2:.0f} secs, \\t ETA: {3:.0f} secs.' \n",
    "    \n",
    "        \n",
    "    if current_line == 1:\n",
    "        print('Processing the first line...')\n",
    "    elif current_line > 1: \n",
    "        if current_line % frequency == 0:\n",
    "            unit, etodivide = get_magnitude(tot_line)\n",
    "            loc_time = time.time() - s_time\n",
    "\n",
    "            clear_output(wait=True)\n",
    "            if pre_message is not None:\n",
    "                print(pre_message)\n",
    "            print(message.format(current_line / (10 ** etodivide), tot_line / (10 ** etodivide), \n",
    "                  loc_time, (tot_line / current_line - 1) * loc_time, unit))\n",
    "            sys.stdout.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def maybestrint(el):\n",
    "    if el == '':\n",
    "        return 'NO'\n",
    "    else:\n",
    "        try:\n",
    "            return str(int(float(el)))\n",
    "        except ValueError:\n",
    "            return str(el)\n",
    "\n",
    "def encode_key(values):\n",
    "    \"\"\"produces a string to be ued as key in dictionaries.\n",
    "    when a tuple or a list is given, it joins the elements with a +\"\"\"\n",
    "    try: \n",
    "        ret = '+'.join([maybestrint(v) for v in values])\n",
    "    except ValueError:\n",
    "        ret = maybestrint(values)\n",
    "        \n",
    "    return ret\n",
    "\n",
    "def decode_key(key):\n",
    "    \"\"\"from a string of values separated with +, it returns the correspondig tuple of values\"\"\"\n",
    "    helper = tuple(key.split('+'))\n",
    "    ret = ()\n",
    "    for k in helper:\n",
    "        try: \n",
    "            ret += (int(float(k)),)\n",
    "        except ValueError:\n",
    "            if k == 'NO':\n",
    "                ret += ('',)\n",
    "            else:\n",
    "                try:\n",
    "                    ret += (float(k),)\n",
    "                except ValueError:\n",
    "                    ret += (k,)\n",
    "    return ret\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def convert_dict_to_df(ds, db):\n",
    "    \"\"\"takes two dictionaries of dictionaries, produces dataframes out of them\n",
    "    and return their concatenation\"\"\"\n",
    "    df_seen = pd.DataFrame.from_dict(ds)\n",
    "    df_book = pd.DataFrame.from_dict(db)\n",
    "\n",
    "    return pd.concat((df_seen, df_book), keys=[0.0, 1.0]).fillna(0)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class Simple_Cluster_Counter:\n",
    "    \"\"\"\n",
    "    an object that counts how many lines with given hotel_cluster and is_booking there\n",
    "    are associated to the possible values of the features feats.\n",
    "    feats is a tuple of columns of the dataframe.\n",
    "    counter is a DataFrame, with indices the hotel_cluster and is_booking keys, and\n",
    "    columns the possible combinations of values of feats.\n",
    "    \"\"\"\n",
    "    def __init__(self, feats):\n",
    "        self.features = feats\n",
    "        first = [0.0] * 100 + [1.0] * 100\n",
    "        second = list(range(100)) + list(range(100))\n",
    "        \n",
    "        paired = list(zip(first, second))\n",
    "        indici = pd.MultiIndex.from_tuples(paired, names=['is_booking', 'cluster_id'])\n",
    "        self.counter = pd.DataFrame(index=indici)\n",
    "        \n",
    "        self.features_position = [TRAIN_FEAT_DICT[f] for f in feats]\n",
    "        \n",
    "        self.best_table = pd.DataFrame(index=list(range(100)))\n",
    "                 \n",
    "            \n",
    "    def process_line(self, line, ds_counter, db_counter):\n",
    "        \"\"\"\n",
    "        process one line to a dictionary of dictionary\n",
    "        \"\"\"\n",
    "        f_vals = (line[p] for p in self.features_position)\n",
    "        f_key = encode_key(f_vals)\n",
    "        hc_val = float(line[TRAIN_FEAT_DICT['hotel_cluster']])\n",
    "        isb_val = float(line[TRAIN_FEAT_DICT['is_booking']])        \n",
    "                \n",
    "        if isb_val == 0:\n",
    "            ds_counter[f_key][hc_val] += 1\n",
    "        else:\n",
    "            db_counter[f_key][hc_val] += 1\n",
    "        \n",
    "    \n",
    "    def count_file_by_lines(self, filename, nlines):\n",
    "        \n",
    "        ds_counter = defaultdict(lambda: defaultdict(lambda: 0))\n",
    "        db_counter = defaultdict(lambda: defaultdict(lambda: 0))      \n",
    "        \n",
    "        with open(filename) as infile:\n",
    "            legge = csv.reader(infile)\n",
    "            \n",
    "            start_time = time.time()\n",
    "            columns = next(legge)\n",
    "            \n",
    "            for i, line in enumerate(legge):\n",
    "                print_processing_status(i, nlines, start_time, frequency=10000, \n",
    "                                       pre_message='Creating counters...')\n",
    "                self.process_line(line, ds_counter, db_counter)\n",
    "                \n",
    "        self.counter = convert_dict_to_df(ds_counter, db_counter)\n",
    "                \n",
    "    \n",
    "    def get_top_10(self, values):\n",
    "        \"\"\"\n",
    "        gets the top 10 hotel_clusters that with the given values.\n",
    "        it takes into account only those values that have a score of at least WARNING_LIMIT,\n",
    "        while COEFF_ISBOOKING is the weight to give to bookings versus only seen\n",
    "        \"\"\"\n",
    "        WARNING_LIMIT = 200\n",
    "        COEFF_ISBOOKING = 8\n",
    "        \n",
    "        key = values\n",
    "        warn = False\n",
    "        if key in self.counter.columns:\n",
    "            loc_points = self.counter.ix[0.0][key] + COEFF_ISBOOKING * self.counter.ix[1.0][key]\n",
    "            \n",
    "            if loc_points.sum() < WARNING_LIMIT:\n",
    "                warn = True\n",
    "                ret = [-1 for i in range(10)]\n",
    "            else:\n",
    "                ret = loc_points.sort_values(ascending=False).index[:20]\n",
    "                \n",
    "        else:\n",
    "            warn = True\n",
    "            ret = [-1 for i in range(20)]\n",
    "            \n",
    "        return ret, warn\n",
    "    \n",
    "    \n",
    "    def compute_best_table(self, premess=''):\n",
    "        \"\"\"\n",
    "        computes the best_table, which encode the most recurrent hotel_clusters\n",
    "        with a given key\n",
    "        \"\"\"\n",
    "        \n",
    "        start_time = time.time()\n",
    "        ncol = len(self.counter.columns)\n",
    "        for i, col in enumerate(self.counter.columns):\n",
    "            if i>2:\n",
    "                print_processing_status(i, ncol, start_time, frequency=max(int(ncol/200), 1), pre_message=premess)\n",
    "            \n",
    "            new_col = pd.Series(index=list(range(100)))\n",
    "            \n",
    "            points, warn = self.get_top_10(col)\n",
    "            \n",
    "            if warn:\n",
    "                new_col = new_col.fillna(0)\n",
    "            else: \n",
    "                for pos, cl in enumerate(points):\n",
    "                    new_col.ix[cl] = 1 / (pos+1)\n",
    "                new_col = new_col.fillna(0)\n",
    "                \n",
    "            self.best_table[col] = new_col\n",
    "            \n",
    "    \n",
    "    def save_counter_to_file(self, dir_path):\n",
    "        filename = os.path.join(dir_path, 'counter.csv')\n",
    "        self.counter.to_csv(filename)\n",
    "            \n",
    "    def load_counter_from_file(self, dir_path):\n",
    "        filename = os.path.join(dir_path, 'counter.csv')\n",
    "        self.counter = pd.read_csv(filename, index_col=[0, 1])\n",
    "        \n",
    "    def save_best_table_to_file(self, dir_path):\n",
    "        filename = os.path.join(dir_path, 'best_table.csv')\n",
    "        self.best_table.to_csv(filename)\n",
    "        \n",
    "    def load_best_table_from_file(self, dir_path):\n",
    "        filename = os.path.join(dir_path, 'best_table.csv')\n",
    "        self.best_table = pd.read_csv(filename, index_col=[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class Multi_Cluster_Counter:\n",
    "    \"\"\"\n",
    "    a class that puts together many Simple_Cluster_Counter s and computes them\n",
    "    at the same time\n",
    "    \"\"\"\n",
    "    def __init__(self, feats_list, name):\n",
    "        self.name = name\n",
    "        self.path = os.path.join('data/objs', name)\n",
    "        self.features_list = feats_list\n",
    "        self.one= dict()\n",
    "        for feats in feats_list:\n",
    "            f_key = encode_key(feats)\n",
    "            self.one[f_key] = Simple_Cluster_Counter(feats)\n",
    "        \n",
    "        self.coeff_index = []\n",
    "        for feats in self.features_list:\n",
    "            self.coeff_index += [encode_key(feats)]\n",
    "        \n",
    "        self.coeff = pd.Series(index=self.coeff_index)\n",
    "        self.coeff = self.coeff.fillna(0)\n",
    "        \n",
    "        self.user_coeff = defaultdict(self.create_zero_serie)\n",
    "        \n",
    "    \n",
    "    def create_zero_serie(self):\n",
    "        return pd.Series(index=self.coeff_index).fillna(0)\n",
    "    \n",
    "    \n",
    "    def count_file_by_lines(self, filename, nlines):\n",
    "        ds_counter = dict()\n",
    "        db_counter = dict()\n",
    "        \n",
    "        for feats in self.features_list:\n",
    "            ds_counter[encode_key(feats)] = defaultdict(lambda: defaultdict(lambda: 0))\n",
    "            db_counter[encode_key(feats)] = defaultdict(lambda: defaultdict(lambda: 0))\n",
    "        \n",
    "        with open(filename) as infile:\n",
    "            legge = csv.reader(infile)\n",
    "            \n",
    "            start_time = time.time()\n",
    "            columns = next(legge)\n",
    "            \n",
    "            for i, line in enumerate(legge):\n",
    "                print_processing_status(i, nlines, start_time, frequency=10000)\n",
    "                \n",
    "                for feats in self.features_list:\n",
    "                    f_key = encode_key(feats)\n",
    "                    self.one[f_key].process_line(line, ds_counter[f_key], db_counter[f_key])\n",
    "        \n",
    "        print('Converting dicts to dataframes:')\n",
    "        for i, feats in enumerate(self.features_list):\n",
    "            print(\"converting n. {} of {}\".format(i, len(self.features_list)))\n",
    "            f_key = encode_key(feats)\n",
    "            self.one[f_key].counter = convert_dict_to_df(ds_counter[f_key], db_counter[f_key])\n",
    "    \n",
    "    \n",
    "    def compute_best_tables(self):\n",
    "        done_message = ''\n",
    "        for feats in self.features_list:\n",
    "            premessa = done_message + 'Computing for the features ' + str(feats) + '...'\n",
    "            self.one[encode_key(feats)].compute_best_table(premess=premessa)\n",
    "            done_message += 'Feat ' + str(feats) + 'computed! \\n'\n",
    "    \n",
    "    \n",
    "    def train_coefficients_on_file(self, filename, nlines):\n",
    "        with open(filename) as infile:\n",
    "            legge = csv.reader(infile)\n",
    "            start_time = time.time()\n",
    "            next(legge)\n",
    "            \n",
    "            for i, line in enumerate(legge):\n",
    "                print_processing_status(i, nlines, start_time, frequency=10000)\n",
    "                \n",
    "\n",
    "                line_cluster = float(line[TRAIN_FEAT_DICT['hotel_cluster']])\n",
    "                user_id = int(float(line[TRAIN_FEAT_DICT['user_id']]))\n",
    "\n",
    "                for feats in self.features_list:\n",
    "                    feat_key = encode_key(feats)\n",
    "                    val_key = encode_key((line[TRAIN_FEAT_DICT[f]] for f in feats))\n",
    "                    \n",
    "                    line_value = self.one[feat_key].best_table.ix[line_cluster][val_key]\n",
    "                    \n",
    "                    self.coeff[feat_key] += line_value\n",
    "                    self.user_coeff[user_id][feat_key] += line_value\n",
    "                             \n",
    "                        \n",
    "    def train_coefficients_randomly(self, filename, nsamples, nlines):\n",
    "        NBLOCKS = 1000\n",
    "        sample_indices = []\n",
    "        \n",
    "        start_time = time.time()\n",
    "        for n in range(NBLOCKS):\n",
    "            print_processing_status(n, NBLOCKS, start_time, frequency=1, \n",
    "                                   pre_message='Preparing indices...')\n",
    "            indices = list(range(n*int(nlines/NBLOCKS), (n+1)*int(nlines/NBLOCKS), 1))\n",
    "            random.shuffle(indices)\n",
    "            indices = indices[:int(nsamples/NBLOCKS)]\n",
    "            sample_indices += sorted(indices)\n",
    "        \n",
    "        with open(filename) as infile:\n",
    "            legge = csv.reader(infile)\n",
    "            start_time = time.time()\n",
    "            next(legge)\n",
    "            \n",
    "            samples_seen = 0\n",
    "            for i, line in enumerate(legge):\n",
    "                try:\n",
    "                    next_index = sample_indices[samples_seen]\n",
    "                except IndexError:\n",
    "                    next_index = -1\n",
    "                \n",
    "                if i != next_index:\n",
    "                    pass\n",
    "                else:\n",
    "                    samples_seen += 1\n",
    "                    print_processing_status(samples_seen, nsamples, start_time, frequency=10000)\n",
    "\n",
    "\n",
    "                    line_cluster = float(line[TRAIN_FEAT_DICT['hotel_cluster']])\n",
    "                    user_id = int(float(line[TRAIN_FEAT_DICT['user_id']]))\n",
    "\n",
    "                    for feats in self.features_list:\n",
    "                        feat_key = encode_key(feats)\n",
    "                        val_key = encode_key((line[TRAIN_FEAT_DICT[f]] for f in feats))\n",
    "\n",
    "                        line_value = self.one[feat_key].best_table.ix[line_cluster][val_key]\n",
    "\n",
    "                        self.coeff[feat_key] += line_value\n",
    "        #                self.user_coeff[user_id][feat_key] += line_value\n",
    "                        \n",
    "                        \n",
    "    \n",
    "    def save_counter_to_file(self, overwrite=False):\n",
    "        print('Saving...')\n",
    "        if not os.path.exists(self.path):\n",
    "            os.makedirs(self.path)\n",
    "        elif overwrite:\n",
    "            shutil.rmtree(self.path)\n",
    "            os.makedirs(self.path)\n",
    "            \n",
    "        with open(os.path.join(self.path, 'features_list.pickle'), 'wb') as outfile:\n",
    "            pickle.dump(self.features_list, outfile)\n",
    "        \n",
    "        for feats in self.features_list:\n",
    "            name = encode_key(feats)\n",
    "            if not os.path.exists(os.path.join(self.path, name)):\n",
    "                os.makedirs(os.path.join(self.path, name))\n",
    "            self.one[name].save_counter_to_file(os.path.join(self.path, name))\n",
    "            print('Save of {} succesfull!'.format(name))\n",
    "            \n",
    "            \n",
    "    def load_counter_from_file(self, path):\n",
    "        print('Loading counters...')\n",
    "        if os.path.exists(path):\n",
    "            \n",
    "            for feats in self.features_list:\n",
    "                folder = os.path.join(path, encode_key(feats))\n",
    "                try:\n",
    "                    self.one[encode_key(feats)].load_counter_from_file(folder)\n",
    "                    print('Counter {} loaded succesfully'.format(feats))\n",
    "                except FileNotFoundError:\n",
    "                    print('Directory {} not found, load error!'.format(folder))\n",
    "            \n",
    "        else:\n",
    "            raise FileNotFoundError(\"Directory {} not fount\".format(path))\n",
    "\n",
    "            \n",
    "    def save_best_tables(self):\n",
    "        if not os.path.exists(self.path):\n",
    "            os.makedirs(self.path)\n",
    "        \n",
    "        for feats in self.features_list:\n",
    "            name = encode_key(feats)\n",
    "            if not os.path.exists(os.path.join(self.path, name)):\n",
    "                os.makedirs(os.path.join(self.path, name))\n",
    "            self.one[name].save_best_table_to_file(os.path.join(self.path, name))\n",
    "            print('Saving of best table for {} succesful!'.format(name))\n",
    "            \n",
    "\n",
    "    def load_best_tables_from_file(self, path):\n",
    "        print('Loading best tables...')\n",
    "        if os.path.exists(path):\n",
    "            \n",
    "            for feats in self.features_list:\n",
    "                folder = os.path.join(path, encode_key(feats))\n",
    "                try:\n",
    "                    self.one[encode_key(feats)].load_best_table_from_file(folder)\n",
    "                    print('Best table for {} loaded succesfully'.format(feats))\n",
    "                except FileNotFoundError:\n",
    "                    print('Directory {} not found, load error!'.format(folder))\n",
    "            \n",
    "        else:\n",
    "            raise FileNotFoundError(\"Directory {} not fount\".format(path))\n",
    "            \n",
    "    \n",
    "    def save_coefficients_to_file(self):\n",
    "        filename = os.path.join(self.path, 'global_coefficients.csv')\n",
    "        self.coeff.to_csv(filename)\n",
    "        \n",
    "        users_filename = os.path.join(self.path, 'users_coefficients.pickle')\n",
    "        with open(users_filename, 'wb') as outf:\n",
    "            pickle.dump(self.user_coeff, outf)\n",
    "        \n",
    "        \n",
    "    def load_coefficients_from_file(self):\n",
    "        filename = os.path.join(self.path, 'global_coefficients.csv')\n",
    "        self.coeff = pd.read_csv(filename, index_col=[0], header=-1)[1]\n",
    "        self.coeff = self.coeff.rename('Coefficients')\n",
    "        \n",
    "        users_filename = os.path.join(self.path, 'users_coefficients.pickle')\n",
    "        with open(users_filename, 'rb') as inf:\n",
    "            self.user_coeff = pickle.load(inf)\n",
    "            \n",
    "        print('Load coefficients succesfull!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "some_feat = [('user_location_city',), ('srch_destination_id',), \n",
    "               ('hotel_country', 'user_location_country'), ('hotel_market', 'stay_span'),\n",
    "               ('srch_destination_type_id', 'stay_span'), ('ci_month', 'hotel_country'),\n",
    "               ('ci_month', 'user_location_country'), ('user_location_region', 'stay_span'),\n",
    "               ('dayofweek', 'hotel_country'), ('dayofweek', 'user_location_region')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "cnt = Multi_Cluster_Counter(cnt_feat, name='first')\n",
    "cnt.count_file_by_lines('data/train_auto_augmented.csv', nrows_train)\n",
    "cnt.save_counter_to_file()\n",
    "\n",
    "cnt.compute_best_tables()\n",
    "cnt.save_best_tables()\n",
    "\n",
    "cnt.train_coefficients_randomly('data/train_auto_augmented.csv', 3000000, nrows_train)\n",
    "cnt.save_coefficients_to_file()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#cnt = Multi_Cluster_Counter(some_feat, name='first')\n",
    "#cnt.load_counter_from_file('data/objs/first')\n",
    "#cnt.load_best_tables_from_file('data/objs/first/')\n",
    "#cnt.load_coefficients_from_file()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def generate_submission(coeff, best_tables):\n",
    "    \"\"\"\n",
    "    generates a submission file according to the given\n",
    "    best_tables and the coefficients\n",
    "    \"\"\"\n",
    "    with open('data/test_augmented.csv') as infile, \\\n",
    "    open('data/submissions/third_good_try.csv', 'w') as outfile:\n",
    "        leggo = csv.reader(infile)\n",
    "        outfile.write(\"id,hotel_cluster \\n\")\n",
    "        \n",
    "        next(leggo)\n",
    "                \n",
    "        start_time = time.time()\n",
    "        nogo = 0\n",
    "\n",
    "        for i, line in enumerate(leggo):\n",
    "            points = pd.Series(index=list(range(100))).fillna(0)\n",
    "            \n",
    "            if i>2:\n",
    "                print_processing_status(i, nrows_test, start_time, frequency=2000,\n",
    "                                       pre_message='Writing submission... (nogo = {})'.format(nogo))\n",
    "\n",
    "            for f_key in best_tables.keys():\n",
    "                feats = decode_key(f_key)\n",
    "                try:\n",
    "                    vals = [int(float(line[TEST_FEAT_DICT[f]])) for f in feats]\n",
    "                    v_key = encode_key(vals)\n",
    "                    try:\n",
    "                        points += coeff[f_key] * best_tables[f_key][v_key]\n",
    "                    except KeyError:\n",
    "                        nogo += 1\n",
    "                \n",
    "                except ValueError:\n",
    "                    nogo += 1\n",
    "                    pass  \n",
    "\n",
    "            top5 = points.sort_values(ascending=False)[:5]\n",
    "            top5_clusters = list(top5.index)\n",
    "\n",
    "            towrite = str(i) + \", \" + \" \".join((str(a) for a in top5_clusters)) + '\\n' \n",
    "\n",
    "            outfile.write(towrite)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "b_tables = dict()\n",
    "for feats in cnt_feat:\n",
    "    b_tables[encode_key(feats)] = cnt.one[encode_key(feats)].best_table\n",
    "\n",
    "generate_submission(cnt.coeff, b_tables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "LEAK_FEATS = ('id', 'user_location_city', 'srch_destination_id', 'orig_destination_distance')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generate_keys_for_leak():\n",
    "    \"\"\"\n",
    "    scan the test file to find the keys that on needs to generate the leak solution\n",
    "    \"\"\"\n",
    "    test_keys = []\n",
    "    \n",
    "    \n",
    "    with open(os.path.join(data_path, 'test_augmented.csv')) as infile:\n",
    "        legge = csv.reader(infile)\n",
    "        start_time = time.time()\n",
    "        \n",
    "        for i, line in enumerate(legge):\n",
    "            if i>2: print_processing_status(i, nrows_test, start_time, frequency=10000)\n",
    "            \n",
    "            vals = (line[TEST_FEAT_DICT[f]] for f in LEAK_FEATS)\n",
    "            key = encode_key(vals)\n",
    "            \n",
    "            test_keys += [key] \n",
    "                \n",
    "    return test_keys\n",
    "\n",
    "\n",
    "def try_float(e):\n",
    "    try:\n",
    "        ret = float(e)\n",
    "    except ValueError:\n",
    "        ret = e\n",
    "    return ret\n",
    "\n",
    "\n",
    "def find_leaked_guess():\n",
    "    \"\"\"\n",
    "    scans the train file to look for lines that match the leak keys\n",
    "    \"\"\"\n",
    "    \n",
    "    with open(os.path.join(data_path, 'data/leak_keys_no_nan.pickle'), 'rb') as infile:\n",
    "        leak_vals = pickle.load(infile)\n",
    "    \n",
    "    #sorts the leak_keys for quicker searches\n",
    "    helper = [10**16 * el[1] + 10**8 * el[2] + el[3] for el in leak_vals]\n",
    "    helper = sorted(helper)\n",
    "    \n",
    "    clusters = defaultdict(lambda: [])\n",
    "    guesses = defaultdict(lambda: [])\n",
    "    \n",
    "    with open('data/train_augmented.csv') as train_file:\n",
    "        leggo = csv.reader(train_file)\n",
    "        premessa = 'Found: {}'\n",
    "        \n",
    "        next(leggo)\n",
    "        \n",
    "        count = 0\n",
    "        start_time = time.time()\n",
    "        for i, line in enumerate(leggo):\n",
    "            if i>2: print_processing_status(i, nrows_train, start_time, \n",
    "                                            frequency=10000, pre_message=premessa.format(count))\n",
    "        \n",
    "            now_key = tuple(try_float(line[TRAIN_FEAT_DICT[f]]) for f in LEAK_FEATS[1:])\n",
    "            now_helper = 10**16 * now_key[0] + 10**8 * now_key[1] + now_key[2]\n",
    "            \n",
    "            pos = bisect.bisect_left(helper, now_helper)\n",
    "\n",
    "            try:\n",
    "                if now_helper == helper[pos] or now_helper == helper[pos+1]:\n",
    "                    count += 1\n",
    "                    found = False\n",
    "                    if len(clusters[now_key]) > 0:\n",
    "                        for p in clusters[now_key]:\n",
    "                            if line[TRAIN_FEAT_DICT['hotel_cluster']] == p[0]:\n",
    "                                p[1] += 1\n",
    "                                found = True                        \n",
    "                    if not found:\n",
    "                        clusters[now_key].insert(0, [line[TRAIN_FEAT_DICT['hotel_cluster']], 1])\n",
    "            except IndexError:\n",
    "                try:\n",
    "                    if now_helper == helper[pos]:\n",
    "                        count += 1\n",
    "                        found = False\n",
    "                        if len(clusters[now_key]) > 0:\n",
    "                            for p in clusters[now_key]:\n",
    "                                if line[TRAIN_FEAT_DICT['hotel_cluster']] == p[0]:\n",
    "                                    p[1] += 1\n",
    "                                    found = True                        \n",
    "                        if not found:\n",
    "                            clusters[now_key].insert(0, [line[TRAIN_FEAT_DICT['hotel_cluster']], 1])\n",
    "                except IndexError:\n",
    "                    pass\n",
    "            \n",
    "    return clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def remove_duplicates(seq):\n",
    "    seen = set()\n",
    "    seen_add = seen.add\n",
    "    return [x for x in seq if not (x in seen or seen_add(x))]\n",
    "\n",
    "\n",
    "def update_submission_with_leak(submission_filename):\n",
    "    \"\"\"\n",
    "    takes a submission file and adds the leaked solutions to the submission,\n",
    "    giving them priority\n",
    "    \"\"\"\n",
    "    \n",
    "    \n",
    "    with open('data/leak/final_leak_sol.pickle', 'rb') as inf:\n",
    "        leak_guess = pickle.load(inf)\n",
    "        \n",
    "    new_sub_name = submission_filename[:-4] + '_leaked.csv'\n",
    "    \n",
    "    with open(submission_filename, 'r') as infile, open(new_sub_name, 'w') as outfile:\n",
    "        \n",
    "        legge = csv.reader(infile)\n",
    "        start_time = time.time()\n",
    "        next(legge)\n",
    "        \n",
    "        outfile.write('id,hotel_cluster\\n')\n",
    "        \n",
    "        for i, line in enumerate(legge):\n",
    "            \n",
    "            if i > 2: \n",
    "                print_processing_status(i, nrows_test, start_time, frequency=10000)\n",
    "                \n",
    "            line_id = float(line[0])\n",
    "            string_old_guesses = line[1]\n",
    "            old_guesses = [int(e) for e in string_old_guesses.split(' ')[1:]]\n",
    "            \n",
    "            new_guesses = leak_guess[line_id] + old_guesses\n",
    "            try:\n",
    "                new_guesses = [int(float(g)) for g in new_guesses]\n",
    "            except ValueError:\n",
    "                print(new_guesses)\n",
    "                raise ValueError\n",
    "            new_guesses = remove_duplicates(new_guesses)[:5]\n",
    "            \n",
    "            towrite = line[0] + \",\" + \" \".join((str(a) for a in new_guesses)) + '\\n'\n",
    "            \n",
    "            outfile.write(towrite)\n",
    "            \n",
    "        print('DONE!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing line 2.5M (of 2.5M). \t Elapsed time: 70 secs, \t ETA: 0 secs.\n",
      "DONE!\n"
     ]
    }
   ],
   "source": [
    "update_submission_with_leak(os.path.join(data_path, 'submissions/third_good_try_bis.csv'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}