Visualize_F1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "import altair as alt\n",
    "from altair import datum\n",
    "import numpy as np\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get Data\n",
    "## Extract Citation Numbers from Bibtex file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "p2bbl_file = r'main.bbl' # path to .bbl-file\n",
    "with open(p2bbl_file, 'r') as fp:\n",
    "    fContent = fp.read()\n",
    "    \n",
    "pattern = r'\\\\bibitem\\[.*?\\]\\{(.+?)\\}' # use raw string format 'r' -> see https://docs.python.org/3/howto/regex.html#the-backslash-plague\n",
    "re.compile(pattern)\n",
    "matchObj_list = re.findall(pattern, fContent, flags=re.DOTALL) # re.DOTALL is important because of newlines within brackets \n",
    "df_cit = pd.DataFrame({'Reference: Abbreviation': matchObj_list, 'Reference: Number': range(1,len(matchObj_list)+1)})\n",
    "df_cit['Reference: Number'] = '[' + df_cit['Reference: Number'].astype(str) + ']'\n",
    "df_cit['Reference: Abbreviation'] = df_cit['Reference: Abbreviation'].astype(str) \n",
    "# df_cit.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Read Performance Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = r'DNN-NILM_low-freq_Performance.xlsx'\n",
    "df = pd.read_excel(path)\n",
    "unnamed_columns = []\n",
    "for column in df.columns:\n",
    "    if column.find('Unnamed:') != -1:\n",
    "        unnamed_columns.append(column)\n",
    "df = df.drop(columns=unnamed_columns)\n",
    "df['Reference: Abbreviation'] = df['Reference: Abbreviation'].astype(str) \n",
    "# df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sanitize data\n",
    "# replace '?' with -1 values in case of 'Input: Window Size'\n",
    "df.loc[ df['Input: Window Size'] == '?', 'Input: Window Size'] = -1\n",
    "df.loc[ df['Input: Window Size'] == 'nan', 'Input: Window Size' ] = -1\n",
    "df.loc[ df['Input: Window Size'] == 'not appl', 'Input: Window Size' ] = -1\n",
    "df.loc[ df['Input: Window Size'] == np.nan, 'Input: Window Size'] = -1\n",
    "df['Input: Window Size'] = df['Input: Window Size'].astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Join dataframes with citation keys\n",
    "df = df.merge(df_cit, on='Reference: Abbreviation', how='left')\n",
    "df = df.rename(columns={'Reference: Number_y': 'Reference: Number'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation on Metrics for noised, unseen\n",
    "(Done directly in Excel):\n",
    "* classificaton\n",
    "    * F1: 12 publications\n",
    "* regression\n",
    "    * MAE: 20 publications\n",
    "    * SAE:  12 publications\n",
    "    * EstAcc: 7 publications"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Investigate: F1 for noised, unseen case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_f1 = df[ (df['Performance Metric: Type'] == 'F1') & \n",
    "            (df['Training: Type of Data'] == 'noised') & \n",
    "            (df['Evaluation: Scenario'] == 'unseen')].copy()\n",
    "df_f1 = df_f1[['Reference: Abbreviation', 'Reference: Number', \n",
    "                'Appliance', 'Training: Dataset', 'Model: Basic Type', 'Model: Denomination', \n",
    "                'Input: Window Size', 'Input: Sampling Rate', 'Performance Metric: Value']].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['kelly2015b' 'bonfigli2018' 'barsim2018' 'nascimento2016' 'murray2019'\n",
      " 'krystalakos2018' 'sudoso2019' 'linh2019' 'cavdar2019' 'yue2020'\n",
      " 'massidda2020' 'rafiq2020' 'kukunuri2020']\n",
      "13\n"
     ]
    }
   ],
   "source": [
    "print(df_f1['Reference: Abbreviation'].unique())\n",
    "print(len(df_f1['Reference: Abbreviation'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove Baseline Model \n",
    "df_f1 = df_f1[ ~(df_f1['Model: Denomination'] == 'BL =0') ]\n",
    "# Note to self: Ke15, Zh2016 are baseline/reference models -> remove these Models from other authors\n",
    "df_f1 = df_f1[ ~(df_f1['Model: Denomination'].str.contains('Ke15') | \n",
    "                 df_f1['Model: Denomination'].str.contains('Zh2016') | \n",
    "                 df_f1['Model: Denomination'].str.contains('He2016')) ] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['CNN dAE' 'CNN-RNN' 'LSTM-bi' 'dAE' 'CNN RctAng' 'GRU-bi' 'CNN s2p'\n",
      " 'STL Iterative Pruning (30%)' 'STL Rank 4 Tensor D.' 'RNN' 'TP-NILM'\n",
      " 'RNN-GRU' 'CNN' 'RecCNN' 'MFS-LSTM' 'S2SwA' 'BERT']\n",
      "['dishwasher' 'fridge' 'kettle' 'microwave' 'washing machine'\n",
      " 'tumble dryer']\n",
      "['UK-DALE' 'REDD' 'REFIT']\n"
     ]
    }
   ],
   "source": [
    "# Group Results\n",
    "grouped = df_f1.groupby(['Reference: Abbreviation', 'Appliance', 'Training: Dataset'])\n",
    "list_of_maxs = []\n",
    "for group in grouped:\n",
    "    group = group[1]\n",
    "    min_row = group[ group['Performance Metric: Value'] == group['Performance Metric: Value'].max()]\n",
    "    list_of_maxs.append(min_row)\n",
    "df_f1 = pd.concat(list_of_maxs)\n",
    "print(df_f1['Model: Denomination'].unique())\n",
    "print(df_f1['Appliance'].unique())\n",
    "print(df_f1['Training: Dataset'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create column that combines 'Reference' and 'Number' to check that numbers are correct\n",
    "df_f1['No-Ref'] = df_f1['Reference: Abbreviation'] + '_' + df_f1['Reference: Number']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "UK-DALE :  39\n",
      "REDD :  24\n",
      "REFIT :  5\n"
     ]
    }
   ],
   "source": [
    "datasets = df_f1['Training: Dataset'].unique()\n",
    "for dataset in datasets:\n",
    "    print(dataset, ': ', np.sum(df_f1['Training: Dataset']==dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dishwasher :  17\n",
      "fridge :  17\n",
      "kettle :  8\n",
      "microwave :  14\n",
      "washing machine :  11\n",
      "tumble dryer :  1\n"
     ]
    }
   ],
   "source": [
    "appliances = df_f1['Appliance'].unique()\n",
    "for app in appliances:\n",
    "    print(app, ': ', np.sum(df_f1['Appliance'] == app))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove appliances with low number of results\n",
    "df_f1 = df_f1[~(df_f1['Appliance'] == 'tumble dryer')]\n",
    "# # df_f1 = df_f1[~((df_f1['Appliance'] == 'lighting'))]\n",
    "# df_f1 = df_f1[~((df_f1['Appliance'] == 'cooker'))]\n",
    "# df_f1 = df_f1[~((df_f1['Appliance'] == 'shower'))]\n",
    "# df_f1 = df_f1[~((df_f1['Appliance'] == 'stove/oven'))]\n",
    "\n",
    "# remove Morgan2017 -> works on proprietary dataset & only fridge\n",
    "# df_f1 = df_f1[~(df_f1['Reference: Abbreviation'] == 'Morgan2017') ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.35\n"
     ]
    }
   ],
   "source": [
    "# Average number of results per appliance\n",
    "appliances = df_f1['Appliance'].unique()\n",
    "n_apps = 0\n",
    "for app in appliances:\n",
    "    n_apps = n_apps + np.sum(df_f1['Appliance'] == app)\n",
    "print(n_apps/len(appliances)/4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([], dtype=object)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find 'na' Reference: Numbers\n",
    "# df['Reference: Number'].isna()\n",
    "df['Reference: Abbreviation'][df['Reference: Number'].isna()].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overview "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-c2f4a68672b946c7b9177f482b13d8f9\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-c2f4a68672b946c7b9177f482b13d8f9\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-c2f4a68672b946c7b9177f482b13d8f9\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.8.1?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function loadScript(lib) {\n",
       "      return new Promise(function(resolve, reject) {\n",
       "        var s = document.createElement('script');\n",
       "        s.src = paths[lib];\n",
       "        s.async = true;\n",
       "        s.onload = () => resolve(paths[lib]);\n",
       "        s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else if (typeof vegaEmbed === \"function\") {\n",
       "      displayChart(vegaEmbed);\n",
       "    } else {\n",
       "      loadScript(\"vega\")\n",
       "        .then(() => loadScript(\"vega-lite\"))\n",
       "        .then(() => loadScript(\"vega-embed\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}, \"facet\": {\"spacing\": 7}, \"header\": {\"labelFontSize\": 14, \"titleFontSize\": 14}}, \"data\": {\"name\": \"data-11d10523e82d1c034478e4bd0267b0d3\"}, \"facet\": {\"column\": {\"type\": \"nominal\", \"field\": \"Appliance\", \"title\": null}}, \"spec\": {\"layer\": [{\"mark\": {\"type\": \"point\", \"clip\": true, \"filled\": true, \"size\": 90}, \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"Reference: Number\", \"legend\": {\"title\": \"Publication\"}, \"scale\": {\"scheme\": \"category20\"}}, \"x\": {\"type\": \"nominal\", \"field\": \"Training: Dataset\", \"title\": \"Dataset\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"Performance Metric: Value\", \"scale\": {\"domain\": [0, 1]}, \"title\": \"F1-score\"}}, \"transform\": [{\"filter\": {\"field\": \"Training: Dataset\", \"oneOf\": [\"REDD\", \"UK-DALE\", \"REFIT\"]}}]}], \"width\": 90}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.8.1.json\", \"datasets\": {\"data-11d10523e82d1c034478e4bd0267b0d3\": [{\"Reference: Abbreviation\": \"barsim2018\", \"Reference: Number\": \"[63]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 10800.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.804, \"No-Ref\": \"barsim2018_[63]\"}, {\"Reference: Abbreviation\": \"barsim2018\", \"Reference: Number\": \"[63]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 10800.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.927, \"No-Ref\": \"barsim2018_[63]\"}, {\"Reference: Abbreviation\": \"barsim2018\", \"Reference: Number\": \"[63]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 10800.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.819, \"No-Ref\": \"barsim2018_[63]\"}, {\"Reference: Abbreviation\": \"barsim2018\", \"Reference: Number\": \"[63]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 10800.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.366, \"No-Ref\": \"barsim2018_[63]\"}, {\"Reference: Abbreviation\": \"barsim2018\", \"Reference: Number\": \"[63]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 10800.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.41, \"No-Ref\": \"barsim2018_[63]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 6912.0, \"Input: Sampling Rate\": 3, \"Performance Metric: Value\": 0.49200000000000005, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 9216.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.509, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 1488.0, \"Input: Sampling Rate\": 3, \"Performance Metric: Value\": 0.7170000000000001, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 3072.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.828, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 768.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.956, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 288.0, \"Input: Sampling Rate\": 3, \"Performance Metric: Value\": 0.168, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 1728.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.45399999999999996, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 6144.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.675, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"cavdar2019\", \"Reference: Number\": \"[144]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"CNN-RNN\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.698, \"No-Ref\": \"cavdar2019_[144]\"}, {\"Reference: Abbreviation\": \"cavdar2019\", \"Reference: Number\": \"[144]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"CNN-RNN\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.9501999999999999, \"No-Ref\": \"cavdar2019_[144]\"}, {\"Reference: Abbreviation\": \"cavdar2019\", \"Reference: Number\": \"[144]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"CNN-RNN\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.9578, \"No-Ref\": \"cavdar2019_[144]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"LSTM-bi\", \"Input: Window Size\": 9216.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.74, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"dAE\", \"Input: Window Size\": 3072.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.87, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"dAE\", \"Input: Window Size\": 768.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.93, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN RctAng\", \"Input: Window Size\": 768.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.93, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"dAE\", \"Input: Window Size\": 1728.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.26, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"LSTM-bi\", \"Input: Window Size\": 6144.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.27, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"krystalakos2018\", \"Reference: Number\": \"[167]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"GRU-bi\", \"Input: Window Size\": 300.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.5, \"No-Ref\": \"krystalakos2018_[167]\"}, {\"Reference: Abbreviation\": \"krystalakos2018\", \"Reference: Number\": \"[167]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"GRU-bi\", \"Input: Window Size\": 300.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.57, \"No-Ref\": \"krystalakos2018_[167]\"}, {\"Reference: Abbreviation\": \"krystalakos2018\", \"Reference: Number\": \"[167]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN s2p\", \"Input: Window Size\": 300.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.88, \"No-Ref\": \"krystalakos2018_[167]\"}, {\"Reference: Abbreviation\": \"krystalakos2018\", \"Reference: Number\": \"[167]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"GRU-bi\", \"Input: Window Size\": 300.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.04, \"No-Ref\": \"krystalakos2018_[167]\"}, {\"Reference: Abbreviation\": \"krystalakos2018\", \"Reference: Number\": \"[167]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN s2p\", \"Input: Window Size\": 600.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.35, \"No-Ref\": \"krystalakos2018_[167]\"}, {\"Reference: Abbreviation\": \"kukunuri2020\", \"Reference: Number\": \"[139]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"STL Iterative Pruning (30%)\", \"Input: Window Size\": 5940.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.26, \"No-Ref\": \"kukunuri2020_[139]\"}, {\"Reference: Abbreviation\": \"kukunuri2020\", \"Reference: Number\": \"[139]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"STL Rank 4 Tensor D.\", \"Input: Window Size\": 5940.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.77, \"No-Ref\": \"kukunuri2020_[139]\"}, {\"Reference: Abbreviation\": \"kukunuri2020\", \"Reference: Number\": \"[139]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"STL Iterative Pruning (30%)\", \"Input: Window Size\": 5940.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.27, \"No-Ref\": \"kukunuri2020_[139]\"}, {\"Reference: Abbreviation\": \"linh2019\", \"Reference: Number\": \"[150]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 3, \"Performance Metric: Value\": 0.01, \"No-Ref\": \"linh2019_[150]\"}, {\"Reference: Abbreviation\": \"linh2019\", \"Reference: Number\": \"[150]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 3, \"Performance Metric: Value\": 0.75, \"No-Ref\": \"linh2019_[150]\"}, {\"Reference: Abbreviation\": \"massidda2020\", \"Reference: Number\": \"[134]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"TP-NILM\", \"Input: Window Size\": 30600.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.809, \"No-Ref\": \"massidda2020_[134]\"}, {\"Reference: Abbreviation\": \"massidda2020\", \"Reference: Number\": \"[134]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"TP-NILM\", \"Input: Window Size\": 30600.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.871, \"No-Ref\": \"massidda2020_[134]\"}, {\"Reference: Abbreviation\": \"massidda2020\", \"Reference: Number\": \"[134]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"TP-NILM\", \"Input: Window Size\": 30600.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.863, \"No-Ref\": \"massidda2020_[134]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 2400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.74, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REFIT\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN\", \"Input: Window Size\": 2400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.82, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REFIT\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 2400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.82, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN\", \"Input: Window Size\": 6400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.67, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 6400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.67, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REFIT\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN\", \"Input: Window Size\": 6400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.93, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN\", \"Input: Window Size\": 720.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.95, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 720.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.95, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REFIT\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 720.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.87, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"REFIT\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 2400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.86, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"nascimento2016\", \"Reference: Number\": \"[121]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"GRU-bi\", \"Input: Window Size\": 2352.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.8242, \"No-Ref\": \"nascimento2016_[121]\"}, {\"Reference: Abbreviation\": \"nascimento2016\", \"Reference: Number\": \"[121]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RecCNN\", \"Input: Window Size\": 9604.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.9497, \"No-Ref\": \"nascimento2016_[121]\"}, {\"Reference: Abbreviation\": \"nascimento2016\", \"Reference: Number\": \"[121]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RecCNN\", \"Input: Window Size\": 764.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.9537, \"No-Ref\": \"nascimento2016_[121]\"}, {\"Reference: Abbreviation\": \"rafiq2020\", \"Reference: Number\": \"[64]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"MFS-LSTM\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.809, \"No-Ref\": \"rafiq2020_[64]\"}, {\"Reference: Abbreviation\": \"rafiq2020\", \"Reference: Number\": \"[64]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"MFS-LSTM\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.872, \"No-Ref\": \"rafiq2020_[64]\"}, {\"Reference: Abbreviation\": \"rafiq2020\", \"Reference: Number\": \"[64]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"MFS-LSTM\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.965, \"No-Ref\": \"rafiq2020_[64]\"}, {\"Reference: Abbreviation\": \"rafiq2020\", \"Reference: Number\": \"[64]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"MFS-LSTM\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.317, \"No-Ref\": \"rafiq2020_[64]\"}, {\"Reference: Abbreviation\": \"rafiq2020\", \"Reference: Number\": \"[64]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"MFS-LSTM\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.765, \"No-Ref\": \"rafiq2020_[64]\"}, {\"Reference: Abbreviation\": \"sudoso2019\", \"Reference: Number\": \"[109]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"S2SwA\", \"Input: Window Size\": 9216.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.608, \"No-Ref\": \"sudoso2019_[109]\"}, {\"Reference: Abbreviation\": \"sudoso2019\", \"Reference: Number\": \"[109]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"S2SwA\", \"Input: Window Size\": 3072.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.824, \"No-Ref\": \"sudoso2019_[109]\"}, {\"Reference: Abbreviation\": \"sudoso2019\", \"Reference: Number\": \"[109]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"S2SwA\", \"Input: Window Size\": 768.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.967, \"No-Ref\": \"sudoso2019_[109]\"}, {\"Reference: Abbreviation\": \"sudoso2019\", \"Reference: Number\": \"[109]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"S2SwA\", \"Input: Window Size\": 1728.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.585, \"No-Ref\": \"sudoso2019_[109]\"}, {\"Reference: Abbreviation\": \"sudoso2019\", \"Reference: Number\": \"[109]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"S2SwA\", \"Input: Window Size\": 6144.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.701, \"No-Ref\": \"sudoso2019_[109]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.523, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.667, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.756, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.766, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.907, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.476, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.014, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.559, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.325, \"No-Ref\": \"yue2020_[106]\"}]}}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.FacetChart(...)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Visualization ,\n",
    "upper_bound = 1 #14\n",
    "sortByDataset = alt.Chart().mark_point(clip=True, size=90, filled=True).encode(\n",
    "    x=alt.X('Training: Dataset:N', title='Dataset'),\n",
    "    y=alt.Y('Performance Metric: Value:Q', title='F1-score', scale=alt.Scale(domain=(0, upper_bound))),\n",
    "#    color=alt.Color('Reference: Abbreviation:N', legend=alt.Legend(title='Publication'), scale=alt.Scale(scheme = 'category20')),\n",
    "    color=alt.Color('Reference: Number:N', legend=alt.Legend(title='Publication'), scale=alt.Scale(scheme = 'category20')),\n",
    "#    color=alt.Color('Input: Sampling Rate:Q', legend=alt.Legend(title='Sampling Rate'), scale=alt.Scale(domain=(0,10), type='symlog')),\n",
    "#    color=alt.Color('Input: Window Size:Q', legend=alt.Legend(title='Window Size'), scale=alt.Scale(type='log')),\n",
    ").transform_filter(\n",
    "    alt.FieldOneOfPredicate(field='Training: Dataset', oneOf=['REDD', 'UK-DALE', 'REFIT'])\n",
    ")\n",
    "# text = alt.Chart().mark_text(\n",
    "#     clip=True, align='left', dy=-5, dx=-24,\n",
    "#     fontSize=13\n",
    "# ).encode(\n",
    "#     x=alt.X('Training: Dataset:N'),\n",
    "#     y=alt.Y('Performance Metric: Value:Q', title='MAE [W]', scale=alt.Scale(domain=(0, upper_bound))),\n",
    "#     text = 'Reference: Abbreviation:N'\n",
    "# )\n",
    "\n",
    "alt.layer(\n",
    "    sortByDataset,  \n",
    "    data=df_f1, width=90#, height=500\n",
    ").facet(\n",
    "    column=alt.Column('Appliance:N', title=None)\n",
    ").configure_facet(\n",
    "    spacing=7\n",
    ").configure_header(\n",
    "    titleFontSize=14,\n",
    "    labelFontSize=14\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variant ZOOM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Individualized focus views\n",
    "b = 20 \n",
    "h = 250\n",
    "params = {\n",
    "    'Kettle':          { 'app' : 'kettle',          'width': b*1, 'height': h, 'upper_bound': 1, 'lower_bound': 0.85 }, \n",
    "    'Microwave':       { 'app' : 'microwave',       'width': b*3, 'height': h, 'upper_bound': 1, 'lower_bound': 0.85 }, \n",
    "    'Dishwasher':      { 'app' : 'dishwasher',      'width': b*3, 'height': h, 'upper_bound': 1, 'lower_bound': 0.7 },\n",
    "    'Washing machine': { 'app' : 'washing machine', 'width': b*3, 'height': h, 'upper_bound': 1, 'lower_bound': 0.7 },\n",
    "    'Fridge':          { 'app' : 'fridge',          'width': b*3, 'height': h, 'upper_bound': 1, 'lower_bound': 0.7 }, \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-e13f0b354d024f2db7132fffa53835e4\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-e13f0b354d024f2db7132fffa53835e4\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-e13f0b354d024f2db7132fffa53835e4\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.8.1?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function loadScript(lib) {\n",
       "      return new Promise(function(resolve, reject) {\n",
       "        var s = document.createElement('script');\n",
       "        s.src = paths[lib];\n",
       "        s.async = true;\n",
       "        s.onload = () => resolve(paths[lib]);\n",
       "        s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else if (typeof vegaEmbed === \"function\") {\n",
       "      displayChart(vegaEmbed);\n",
       "    } else {\n",
       "      loadScript(\"vega\")\n",
       "        .then(() => loadScript(\"vega-lite\"))\n",
       "        .then(() => loadScript(\"vega-embed\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}}, \"hconcat\": [{\"layer\": [{\"mark\": {\"type\": \"point\", \"clip\": true, \"filled\": true, \"size\": 90}, \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"Reference: Number\", \"legend\": {\"title\": \"Publication\"}, \"scale\": {\"scheme\": \"category20\"}}, \"x\": {\"type\": \"nominal\", \"field\": \"Training: Dataset\", \"title\": \"Dataset\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"Performance Metric: Value\", \"scale\": {\"domain\": [0.85, 1]}, \"title\": \"F\\u2081\"}}, \"title\": \"Kettle\", \"transform\": [{\"filter\": {\"field\": \"Appliance\", \"oneOf\": [\"kettle\"]}}]}], \"height\": 250, \"width\": 20}, {\"layer\": [{\"mark\": {\"type\": \"point\", \"clip\": true, \"filled\": true, \"size\": 90}, \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"Reference: Number\", \"legend\": {\"title\": \"Publication\"}, \"scale\": {\"scheme\": \"category20\"}}, \"x\": {\"type\": \"nominal\", \"field\": \"Training: Dataset\", \"title\": \"Dataset\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"Performance Metric: Value\", \"scale\": {\"domain\": [0.85, 1]}, \"title\": \"F\\u2081\"}}, \"title\": \"Microwave\", \"transform\": [{\"filter\": {\"field\": \"Appliance\", \"oneOf\": [\"microwave\"]}}]}], \"height\": 250, \"width\": 60}, {\"layer\": [{\"mark\": {\"type\": \"point\", \"clip\": true, \"filled\": true, \"size\": 90}, \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"Reference: Number\", \"legend\": {\"title\": \"Publication\"}, \"scale\": {\"scheme\": \"category20\"}}, \"x\": {\"type\": \"nominal\", \"field\": \"Training: Dataset\", \"title\": \"Dataset\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"Performance Metric: Value\", \"scale\": {\"domain\": [0.7, 1]}, \"title\": \"F\\u2081\"}}, \"title\": \"Dishwasher\", \"transform\": [{\"filter\": {\"field\": \"Appliance\", \"oneOf\": [\"dishwasher\"]}}]}], \"height\": 250, \"width\": 60}, {\"layer\": [{\"mark\": {\"type\": \"point\", \"clip\": true, \"filled\": true, \"size\": 90}, \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"Reference: Number\", \"legend\": {\"title\": \"Publication\"}, \"scale\": {\"scheme\": \"category20\"}}, \"x\": {\"type\": \"nominal\", \"field\": \"Training: Dataset\", \"title\": \"Dataset\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"Performance Metric: Value\", \"scale\": {\"domain\": [0.7, 1]}, \"title\": \"F\\u2081\"}}, \"title\": \"Washing machine\", \"transform\": [{\"filter\": {\"field\": \"Appliance\", \"oneOf\": [\"washing machine\"]}}]}], \"height\": 250, \"width\": 60}, {\"layer\": [{\"mark\": {\"type\": \"point\", \"clip\": true, \"filled\": true, \"size\": 90}, \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"Reference: Number\", \"legend\": {\"title\": \"Publication\"}, \"scale\": {\"scheme\": \"category20\"}}, \"x\": {\"type\": \"nominal\", \"field\": \"Training: Dataset\", \"title\": \"Dataset\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"Performance Metric: Value\", \"scale\": {\"domain\": [0.7, 1]}, \"title\": \"F\\u2081\"}}, \"title\": \"Fridge\", \"transform\": [{\"filter\": {\"field\": \"Appliance\", \"oneOf\": [\"fridge\"]}}]}], \"height\": 250, \"width\": 60}], \"bounds\": \"flush\", \"data\": {\"name\": \"data-11d10523e82d1c034478e4bd0267b0d3\"}, \"spacing\": 50, \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.8.1.json\", \"datasets\": {\"data-11d10523e82d1c034478e4bd0267b0d3\": [{\"Reference: Abbreviation\": \"barsim2018\", \"Reference: Number\": \"[63]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 10800.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.804, \"No-Ref\": \"barsim2018_[63]\"}, {\"Reference: Abbreviation\": \"barsim2018\", \"Reference: Number\": \"[63]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 10800.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.927, \"No-Ref\": \"barsim2018_[63]\"}, {\"Reference: Abbreviation\": \"barsim2018\", \"Reference: Number\": \"[63]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 10800.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.819, \"No-Ref\": \"barsim2018_[63]\"}, {\"Reference: Abbreviation\": \"barsim2018\", \"Reference: Number\": \"[63]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 10800.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.366, \"No-Ref\": \"barsim2018_[63]\"}, {\"Reference: Abbreviation\": \"barsim2018\", \"Reference: Number\": \"[63]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 10800.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.41, \"No-Ref\": \"barsim2018_[63]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 6912.0, \"Input: Sampling Rate\": 3, \"Performance Metric: Value\": 0.49200000000000005, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 9216.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.509, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 1488.0, \"Input: Sampling Rate\": 3, \"Performance Metric: Value\": 0.7170000000000001, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 3072.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.828, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 768.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.956, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 288.0, \"Input: Sampling Rate\": 3, \"Performance Metric: Value\": 0.168, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 1728.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.45399999999999996, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 6144.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.675, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"cavdar2019\", \"Reference: Number\": \"[144]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"CNN-RNN\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.698, \"No-Ref\": \"cavdar2019_[144]\"}, {\"Reference: Abbreviation\": \"cavdar2019\", \"Reference: Number\": \"[144]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"CNN-RNN\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.9501999999999999, \"No-Ref\": \"cavdar2019_[144]\"}, {\"Reference: Abbreviation\": \"cavdar2019\", \"Reference: Number\": \"[144]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"CNN-RNN\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.9578, \"No-Ref\": \"cavdar2019_[144]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"LSTM-bi\", \"Input: Window Size\": 9216.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.74, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"dAE\", \"Input: Window Size\": 3072.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.87, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"dAE\", \"Input: Window Size\": 768.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.93, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN RctAng\", \"Input: Window Size\": 768.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.93, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"dAE\", \"Input: Window Size\": 1728.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.26, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"LSTM-bi\", \"Input: Window Size\": 6144.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.27, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"krystalakos2018\", \"Reference: Number\": \"[167]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"GRU-bi\", \"Input: Window Size\": 300.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.5, \"No-Ref\": \"krystalakos2018_[167]\"}, {\"Reference: Abbreviation\": \"krystalakos2018\", \"Reference: Number\": \"[167]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"GRU-bi\", \"Input: Window Size\": 300.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.57, \"No-Ref\": \"krystalakos2018_[167]\"}, {\"Reference: Abbreviation\": \"krystalakos2018\", \"Reference: Number\": \"[167]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN s2p\", \"Input: Window Size\": 300.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.88, \"No-Ref\": \"krystalakos2018_[167]\"}, {\"Reference: Abbreviation\": \"krystalakos2018\", \"Reference: Number\": \"[167]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"GRU-bi\", \"Input: Window Size\": 300.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.04, \"No-Ref\": \"krystalakos2018_[167]\"}, {\"Reference: Abbreviation\": \"krystalakos2018\", \"Reference: Number\": \"[167]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN s2p\", \"Input: Window Size\": 600.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.35, \"No-Ref\": \"krystalakos2018_[167]\"}, {\"Reference: Abbreviation\": \"kukunuri2020\", \"Reference: Number\": \"[139]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"STL Iterative Pruning (30%)\", \"Input: Window Size\": 5940.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.26, \"No-Ref\": \"kukunuri2020_[139]\"}, {\"Reference: Abbreviation\": \"kukunuri2020\", \"Reference: Number\": \"[139]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"STL Rank 4 Tensor D.\", \"Input: Window Size\": 5940.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.77, \"No-Ref\": \"kukunuri2020_[139]\"}, {\"Reference: Abbreviation\": \"kukunuri2020\", \"Reference: Number\": \"[139]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"STL Iterative Pruning (30%)\", \"Input: Window Size\": 5940.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.27, \"No-Ref\": \"kukunuri2020_[139]\"}, {\"Reference: Abbreviation\": \"linh2019\", \"Reference: Number\": \"[150]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 3, \"Performance Metric: Value\": 0.01, \"No-Ref\": \"linh2019_[150]\"}, {\"Reference: Abbreviation\": \"linh2019\", \"Reference: Number\": \"[150]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 3, \"Performance Metric: Value\": 0.75, \"No-Ref\": \"linh2019_[150]\"}, {\"Reference: Abbreviation\": \"massidda2020\", \"Reference: Number\": \"[134]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"TP-NILM\", \"Input: Window Size\": 30600.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.809, \"No-Ref\": \"massidda2020_[134]\"}, {\"Reference: Abbreviation\": \"massidda2020\", \"Reference: Number\": \"[134]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"TP-NILM\", \"Input: Window Size\": 30600.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.871, \"No-Ref\": \"massidda2020_[134]\"}, {\"Reference: Abbreviation\": \"massidda2020\", \"Reference: Number\": \"[134]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"TP-NILM\", \"Input: Window Size\": 30600.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.863, \"No-Ref\": \"massidda2020_[134]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 2400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.74, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REFIT\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN\", \"Input: Window Size\": 2400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.82, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REFIT\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 2400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.82, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN\", \"Input: Window Size\": 6400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.67, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 6400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.67, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REFIT\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN\", \"Input: Window Size\": 6400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.93, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN\", \"Input: Window Size\": 720.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.95, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 720.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.95, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REFIT\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 720.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.87, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"REFIT\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 2400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.86, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"nascimento2016\", \"Reference: Number\": \"[121]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"GRU-bi\", \"Input: Window Size\": 2352.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.8242, \"No-Ref\": \"nascimento2016_[121]\"}, {\"Reference: Abbreviation\": \"nascimento2016\", \"Reference: Number\": \"[121]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RecCNN\", \"Input: Window Size\": 9604.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.9497, \"No-Ref\": \"nascimento2016_[121]\"}, {\"Reference: Abbreviation\": \"nascimento2016\", \"Reference: Number\": \"[121]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RecCNN\", \"Input: Window Size\": 764.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.9537, \"No-Ref\": \"nascimento2016_[121]\"}, {\"Reference: Abbreviation\": \"rafiq2020\", \"Reference: Number\": \"[64]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"MFS-LSTM\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.809, \"No-Ref\": \"rafiq2020_[64]\"}, {\"Reference: Abbreviation\": \"rafiq2020\", \"Reference: Number\": \"[64]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"MFS-LSTM\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.872, \"No-Ref\": \"rafiq2020_[64]\"}, {\"Reference: Abbreviation\": \"rafiq2020\", \"Reference: Number\": \"[64]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"MFS-LSTM\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.965, \"No-Ref\": \"rafiq2020_[64]\"}, {\"Reference: Abbreviation\": \"rafiq2020\", \"Reference: Number\": \"[64]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"MFS-LSTM\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.317, \"No-Ref\": \"rafiq2020_[64]\"}, {\"Reference: Abbreviation\": \"rafiq2020\", \"Reference: Number\": \"[64]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"MFS-LSTM\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.765, \"No-Ref\": \"rafiq2020_[64]\"}, {\"Reference: Abbreviation\": \"sudoso2019\", \"Reference: Number\": \"[109]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"S2SwA\", \"Input: Window Size\": 9216.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.608, \"No-Ref\": \"sudoso2019_[109]\"}, {\"Reference: Abbreviation\": \"sudoso2019\", \"Reference: Number\": \"[109]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"S2SwA\", \"Input: Window Size\": 3072.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.824, \"No-Ref\": \"sudoso2019_[109]\"}, {\"Reference: Abbreviation\": \"sudoso2019\", \"Reference: Number\": \"[109]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"S2SwA\", \"Input: Window Size\": 768.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.967, \"No-Ref\": \"sudoso2019_[109]\"}, {\"Reference: Abbreviation\": \"sudoso2019\", \"Reference: Number\": \"[109]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"S2SwA\", \"Input: Window Size\": 1728.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.585, \"No-Ref\": \"sudoso2019_[109]\"}, {\"Reference: Abbreviation\": \"sudoso2019\", \"Reference: Number\": \"[109]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"S2SwA\", \"Input: Window Size\": 6144.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.701, \"No-Ref\": \"sudoso2019_[109]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.523, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.667, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.756, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.766, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.907, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.476, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.014, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.559, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.325, \"No-Ref\": \"yue2020_[106]\"}]}}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.HConcatChart(...)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Visualization Dishwasher,\n",
    "layers = []\n",
    "for key, par in params.items():\n",
    "    upper_bound = par['upper_bound']\n",
    "    lower_bound = par['lower_bound']\n",
    "    width = par['width']\n",
    "    height = par['height']\n",
    "    app = par['app']\n",
    "    sortByDataset = alt.Chart().mark_point(clip=True, size=90, filled=True).encode(\n",
    "        x=alt.X('Training: Dataset:N', title='Dataset'),\n",
    "        y=alt.Y('Performance Metric: Value:Q', title='F₁', scale=alt.Scale(domain=(lower_bound, upper_bound))),\n",
    "        color=alt.Color('Reference: Number:N', legend=alt.Legend(title='Publication'), scale=alt.Scale(scheme = 'category20')), \n",
    "#       color=alt.Color('No-Ref:N', legend=alt.Legend(title='Publication'), scale=alt.Scale(scheme = 'category20')), \n",
    "#        color=alt.Color('Reference: Abbreviation:N', legend=alt.Legend(title='Publication'), scale=alt.Scale(scheme = 'category20')), \n",
    "#        shape=alt.Shape('Model: Basic Type', sort=['recurrent', 'feedforward'], legend=alt.Legend(title='Network Type'))\n",
    "#        shape=alt.Shape('Model: Basic Type', sort=['recurrent', 'feedforward'], legend=alt.Legend(title='Network Type'))\n",
    "    ).transform_filter(\n",
    "        alt.FieldOneOfPredicate(field='Appliance', oneOf=[app])\n",
    "    ).properties(\n",
    "        title=key\n",
    "    )\n",
    "#    text = alt.Chart().mark_text(\n",
    "#        clip=True, align='left', dy=-7, dx=-15,\n",
    "#        fontSize=13\n",
    "#    ).encode(\n",
    "#        x=alt.X('Training: Dataset:N'),\n",
    "#        y=alt.Y('Performance Metric: Value:Q', title='MAE [W]', scale=alt.Scale(domain=(lower_bound, upper_bound))),\n",
    "#    #    text = 'Reference: Abbreviation:N'\n",
    "#        text = 'Reference: Number:N'\n",
    "#    ).transform_filter(\n",
    "#        alt.FieldOneOfPredicate(field='Appliance', oneOf=[key])\n",
    "#    )\n",
    "    layers.append(alt.layer(\n",
    "        sortByDataset, \n",
    "        data=df_f1, width=width, height=height\n",
    "    ))\n",
    "alt.hconcat(*layers, spacing=50, bounds='flush')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note to self: I double checked the values for the top result in the corresponding publications. *They are correct.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-b90f211deb17420398ab63def25b7866\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-b90f211deb17420398ab63def25b7866\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-b90f211deb17420398ab63def25b7866\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.8.1?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function loadScript(lib) {\n",
       "      return new Promise(function(resolve, reject) {\n",
       "        var s = document.createElement('script');\n",
       "        s.src = paths[lib];\n",
       "        s.async = true;\n",
       "        s.onload = () => resolve(paths[lib]);\n",
       "        s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else if (typeof vegaEmbed === \"function\") {\n",
       "      displayChart(vegaEmbed);\n",
       "    } else {\n",
       "      loadScript(\"vega\")\n",
       "        .then(() => loadScript(\"vega-lite\"))\n",
       "        .then(() => loadScript(\"vega-embed\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}, \"facet\": {\"spacing\": 7}, \"header\": {\"labelFontSize\": 14, \"titleFontSize\": 14}}, \"data\": {\"name\": \"data-11d10523e82d1c034478e4bd0267b0d3\"}, \"facet\": {\"column\": {\"type\": \"nominal\", \"field\": \"Appliance\", \"title\": null}}, \"spec\": {\"layer\": [{\"mark\": {\"type\": \"point\", \"clip\": true, \"filled\": true, \"size\": 90}, \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"Reference: Number\", \"legend\": {\"title\": \"Publication\"}, \"scale\": {\"scheme\": \"category20\"}}, \"x\": {\"type\": \"nominal\", \"field\": \"Training: Dataset\", \"title\": \"Dataset\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"Performance Metric: Value\", \"scale\": {\"domain\": [0.7, 1]}, \"title\": \"F1-score\"}}}], \"width\": 90}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.8.1.json\", \"datasets\": {\"data-11d10523e82d1c034478e4bd0267b0d3\": [{\"Reference: Abbreviation\": \"barsim2018\", \"Reference: Number\": \"[63]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 10800.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.804, \"No-Ref\": \"barsim2018_[63]\"}, {\"Reference: Abbreviation\": \"barsim2018\", \"Reference: Number\": \"[63]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 10800.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.927, \"No-Ref\": \"barsim2018_[63]\"}, {\"Reference: Abbreviation\": \"barsim2018\", \"Reference: Number\": \"[63]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 10800.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.819, \"No-Ref\": \"barsim2018_[63]\"}, {\"Reference: Abbreviation\": \"barsim2018\", \"Reference: Number\": \"[63]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 10800.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.366, \"No-Ref\": \"barsim2018_[63]\"}, {\"Reference: Abbreviation\": \"barsim2018\", \"Reference: Number\": \"[63]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 10800.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.41, \"No-Ref\": \"barsim2018_[63]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 6912.0, \"Input: Sampling Rate\": 3, \"Performance Metric: Value\": 0.49200000000000005, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 9216.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.509, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 1488.0, \"Input: Sampling Rate\": 3, \"Performance Metric: Value\": 0.7170000000000001, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 3072.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.828, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 768.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.956, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 288.0, \"Input: Sampling Rate\": 3, \"Performance Metric: Value\": 0.168, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 1728.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.45399999999999996, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"bonfigli2018\", \"Reference: Number\": \"[21]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN dAE\", \"Input: Window Size\": 6144.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.675, \"No-Ref\": \"bonfigli2018_[21]\"}, {\"Reference: Abbreviation\": \"cavdar2019\", \"Reference: Number\": \"[144]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"CNN-RNN\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.698, \"No-Ref\": \"cavdar2019_[144]\"}, {\"Reference: Abbreviation\": \"cavdar2019\", \"Reference: Number\": \"[144]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"CNN-RNN\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.9501999999999999, \"No-Ref\": \"cavdar2019_[144]\"}, {\"Reference: Abbreviation\": \"cavdar2019\", \"Reference: Number\": \"[144]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"CNN-RNN\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.9578, \"No-Ref\": \"cavdar2019_[144]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"LSTM-bi\", \"Input: Window Size\": 9216.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.74, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"dAE\", \"Input: Window Size\": 3072.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.87, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"dAE\", \"Input: Window Size\": 768.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.93, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN RctAng\", \"Input: Window Size\": 768.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.93, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"dAE\", \"Input: Window Size\": 1728.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.26, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"kelly2015b\", \"Reference: Number\": \"[14]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"LSTM-bi\", \"Input: Window Size\": 6144.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.27, \"No-Ref\": \"kelly2015b_[14]\"}, {\"Reference: Abbreviation\": \"krystalakos2018\", \"Reference: Number\": \"[167]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"GRU-bi\", \"Input: Window Size\": 300.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.5, \"No-Ref\": \"krystalakos2018_[167]\"}, {\"Reference: Abbreviation\": \"krystalakos2018\", \"Reference: Number\": \"[167]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"GRU-bi\", \"Input: Window Size\": 300.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.57, \"No-Ref\": \"krystalakos2018_[167]\"}, {\"Reference: Abbreviation\": \"krystalakos2018\", \"Reference: Number\": \"[167]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN s2p\", \"Input: Window Size\": 300.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.88, \"No-Ref\": \"krystalakos2018_[167]\"}, {\"Reference: Abbreviation\": \"krystalakos2018\", \"Reference: Number\": \"[167]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"GRU-bi\", \"Input: Window Size\": 300.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.04, \"No-Ref\": \"krystalakos2018_[167]\"}, {\"Reference: Abbreviation\": \"krystalakos2018\", \"Reference: Number\": \"[167]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN s2p\", \"Input: Window Size\": 600.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.35, \"No-Ref\": \"krystalakos2018_[167]\"}, {\"Reference: Abbreviation\": \"kukunuri2020\", \"Reference: Number\": \"[139]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"STL Iterative Pruning (30%)\", \"Input: Window Size\": 5940.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.26, \"No-Ref\": \"kukunuri2020_[139]\"}, {\"Reference: Abbreviation\": \"kukunuri2020\", \"Reference: Number\": \"[139]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"STL Rank 4 Tensor D.\", \"Input: Window Size\": 5940.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.77, \"No-Ref\": \"kukunuri2020_[139]\"}, {\"Reference: Abbreviation\": \"kukunuri2020\", \"Reference: Number\": \"[139]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"STL Iterative Pruning (30%)\", \"Input: Window Size\": 5940.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.27, \"No-Ref\": \"kukunuri2020_[139]\"}, {\"Reference: Abbreviation\": \"linh2019\", \"Reference: Number\": \"[150]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 3, \"Performance Metric: Value\": 0.01, \"No-Ref\": \"linh2019_[150]\"}, {\"Reference: Abbreviation\": \"linh2019\", \"Reference: Number\": \"[150]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 3, \"Performance Metric: Value\": 0.75, \"No-Ref\": \"linh2019_[150]\"}, {\"Reference: Abbreviation\": \"massidda2020\", \"Reference: Number\": \"[134]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"TP-NILM\", \"Input: Window Size\": 30600.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.809, \"No-Ref\": \"massidda2020_[134]\"}, {\"Reference: Abbreviation\": \"massidda2020\", \"Reference: Number\": \"[134]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"TP-NILM\", \"Input: Window Size\": 30600.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.871, \"No-Ref\": \"massidda2020_[134]\"}, {\"Reference: Abbreviation\": \"massidda2020\", \"Reference: Number\": \"[134]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"TP-NILM\", \"Input: Window Size\": 30600.0, \"Input: Sampling Rate\": 60, \"Performance Metric: Value\": 0.863, \"No-Ref\": \"massidda2020_[134]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 2400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.74, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REFIT\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN\", \"Input: Window Size\": 2400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.82, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REFIT\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 2400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.82, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN\", \"Input: Window Size\": 6400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.67, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 6400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.67, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REFIT\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN\", \"Input: Window Size\": 6400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.93, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"CNN\", \"Input: Window Size\": 720.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.95, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 720.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.95, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REFIT\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 720.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.87, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"murray2019\", \"Reference: Number\": \"[56]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"REFIT\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RNN-GRU\", \"Input: Window Size\": 2400.0, \"Input: Sampling Rate\": 8, \"Performance Metric: Value\": 0.86, \"No-Ref\": \"murray2019_[56]\"}, {\"Reference: Abbreviation\": \"nascimento2016\", \"Reference: Number\": \"[121]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"GRU-bi\", \"Input: Window Size\": 2352.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.8242, \"No-Ref\": \"nascimento2016_[121]\"}, {\"Reference: Abbreviation\": \"nascimento2016\", \"Reference: Number\": \"[121]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RecCNN\", \"Input: Window Size\": 9604.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.9497, \"No-Ref\": \"nascimento2016_[121]\"}, {\"Reference: Abbreviation\": \"nascimento2016\", \"Reference: Number\": \"[121]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"recurrent\", \"Model: Denomination\": \"RecCNN\", \"Input: Window Size\": 764.0, \"Input: Sampling Rate\": 4, \"Performance Metric: Value\": 0.9537, \"No-Ref\": \"nascimento2016_[121]\"}, {\"Reference: Abbreviation\": \"rafiq2020\", \"Reference: Number\": \"[64]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"MFS-LSTM\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.809, \"No-Ref\": \"rafiq2020_[64]\"}, {\"Reference: Abbreviation\": \"rafiq2020\", \"Reference: Number\": \"[64]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"MFS-LSTM\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.872, \"No-Ref\": \"rafiq2020_[64]\"}, {\"Reference: Abbreviation\": \"rafiq2020\", \"Reference: Number\": \"[64]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"MFS-LSTM\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.965, \"No-Ref\": \"rafiq2020_[64]\"}, {\"Reference: Abbreviation\": \"rafiq2020\", \"Reference: Number\": \"[64]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"MFS-LSTM\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.317, \"No-Ref\": \"rafiq2020_[64]\"}, {\"Reference: Abbreviation\": \"rafiq2020\", \"Reference: Number\": \"[64]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"MFS-LSTM\", \"Input: Window Size\": -1.0, \"Input: Sampling Rate\": 1, \"Performance Metric: Value\": 0.765, \"No-Ref\": \"rafiq2020_[64]\"}, {\"Reference: Abbreviation\": \"sudoso2019\", \"Reference: Number\": \"[109]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"S2SwA\", \"Input: Window Size\": 9216.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.608, \"No-Ref\": \"sudoso2019_[109]\"}, {\"Reference: Abbreviation\": \"sudoso2019\", \"Reference: Number\": \"[109]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"S2SwA\", \"Input: Window Size\": 3072.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.824, \"No-Ref\": \"sudoso2019_[109]\"}, {\"Reference: Abbreviation\": \"sudoso2019\", \"Reference: Number\": \"[109]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"S2SwA\", \"Input: Window Size\": 768.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.967, \"No-Ref\": \"sudoso2019_[109]\"}, {\"Reference: Abbreviation\": \"sudoso2019\", \"Reference: Number\": \"[109]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"S2SwA\", \"Input: Window Size\": 1728.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.585, \"No-Ref\": \"sudoso2019_[109]\"}, {\"Reference: Abbreviation\": \"sudoso2019\", \"Reference: Number\": \"[109]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"combined\", \"Model: Denomination\": \"S2SwA\", \"Input: Window Size\": 6144.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.701, \"No-Ref\": \"sudoso2019_[109]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.523, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"dishwasher\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.667, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.756, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"fridge\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.766, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"kettle\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.907, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.476, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"microwave\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.014, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"REDD\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.559, \"No-Ref\": \"yue2020_[106]\"}, {\"Reference: Abbreviation\": \"yue2020\", \"Reference: Number\": \"[106]\", \"Appliance\": \"washing machine\", \"Training: Dataset\": \"UK-DALE\", \"Model: Basic Type\": \"feedforward\", \"Model: Denomination\": \"BERT\", \"Input: Window Size\": 2880.0, \"Input: Sampling Rate\": 6, \"Performance Metric: Value\": 0.325, \"No-Ref\": \"yue2020_[106]\"}]}}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.FacetChart(...)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# VERSION FOR PUBLICATION\n",
    "# Visualization ,\n",
    "upper_bound = 1 #14\n",
    "lower_bound = 0.7\n",
    "sortByDataset = alt.Chart().mark_point(clip=True, size=90, filled=True).encode(\n",
    "    x=alt.X('Training: Dataset:N', title='Dataset'),\n",
    "    y=alt.Y('Performance Metric: Value:Q', title='F1-score', scale=alt.Scale(domain=(lower_bound, upper_bound))),\n",
    "#    color=alt.Color('Reference: Abbreviation:N', legend=alt.Legend(title='Publication'), scale=alt.Scale(scheme = 'category20')),\n",
    "    color=alt.Color('Reference: Number:N', legend=alt.Legend(title='Publication'), scale=alt.Scale(scheme = 'category20')),\n",
    ")\n",
    "\n",
    "alt.layer(\n",
    "    sortByDataset,  \n",
    "    data=df_f1, width=90#, height=500\n",
    ").facet(\n",
    "    column=alt.Column('Appliance:N', title=None)\n",
    ").configure_facet(\n",
    "    spacing=7\n",
    ").configure_header(\n",
    "    titleFontSize=14,\n",
    "    labelFontSize=14\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kettle\n",
      "2\n",
      "     Reference: Abbreviation Reference: Number Appliance Training: Dataset  \\\n",
      "1497               rafiq2020              [64]    kettle           UK-DALE   \n",
      "1048              sudoso2019             [109]    kettle           UK-DALE   \n",
      "\n",
      "     Model: Basic Type Model: Denomination  Input: Window Size  \\\n",
      "1497          combined            MFS-LSTM                -1.0   \n",
      "1048          combined               S2SwA               768.0   \n",
      "\n",
      "     Input: Sampling Rate  Performance Metric: Value            No-Ref  \n",
      "1497                    1                      0.965    rafiq2020_[64]  \n",
      "1048                    6                      0.967  sudoso2019_[109]  \n",
      "\n",
      "microwave\n",
      "4\n",
      "     Reference: Abbreviation Reference: Number  Appliance Training: Dataset  \\\n",
      "584               murray2019              [56]  microwave              REDD   \n",
      "587               murray2019              [56]  microwave              REDD   \n",
      "498           nascimento2016             [121]  microwave              REDD   \n",
      "1199              cavdar2019             [144]  microwave              REDD   \n",
      "\n",
      "     Model: Basic Type Model: Denomination  Input: Window Size  \\\n",
      "584        feedforward                 CNN               720.0   \n",
      "587          recurrent             RNN-GRU               720.0   \n",
      "498          recurrent              RecCNN               764.0   \n",
      "1199          combined             CNN-RNN                -1.0   \n",
      "\n",
      "     Input: Sampling Rate  Performance Metric: Value                No-Ref  \n",
      "584                     8                     0.9500       murray2019_[56]  \n",
      "587                     8                     0.9500       murray2019_[56]  \n",
      "498                     4                     0.9537  nascimento2016_[121]  \n",
      "1199                    4                     0.9578      cavdar2019_[144]  \n",
      "\n",
      "dishwasher\n",
      "4\n",
      "     Reference: Abbreviation Reference: Number   Appliance Training: Dataset  \\\n",
      "1392            massidda2020             [134]  dishwasher           UK-DALE   \n",
      "603               murray2019              [56]  dishwasher             REFIT   \n",
      "649               murray2019              [56]  dishwasher             REFIT   \n",
      "515           nascimento2016             [121]  dishwasher              REDD   \n",
      "\n",
      "     Model: Basic Type Model: Denomination  Input: Window Size  \\\n",
      "1392       feedforward             TP-NILM             30600.0   \n",
      "603        feedforward                 CNN              2400.0   \n",
      "649          recurrent             RNN-GRU              2400.0   \n",
      "515          recurrent              GRU-bi              2352.0   \n",
      "\n",
      "     Input: Sampling Rate  Performance Metric: Value                No-Ref  \n",
      "1392                   60                     0.8090    massidda2020_[134]  \n",
      "603                     8                     0.8200       murray2019_[56]  \n",
      "649                     8                     0.8200       murray2019_[56]  \n",
      "515                     4                     0.8242  nascimento2016_[121]  \n",
      "\n",
      "washing machine\n",
      "3\n",
      "     Reference: Abbreviation Reference: Number        Appliance  \\\n",
      "1501               rafiq2020              [64]  washing machine   \n",
      "651               murray2019              [56]  washing machine   \n",
      "1393            massidda2020             [134]  washing machine   \n",
      "\n",
      "     Training: Dataset Model: Basic Type Model: Denomination  \\\n",
      "1501           UK-DALE          combined            MFS-LSTM   \n",
      "651              REFIT         recurrent             RNN-GRU   \n",
      "1393           UK-DALE       feedforward             TP-NILM   \n",
      "\n",
      "      Input: Window Size Input: Sampling Rate  Performance Metric: Value  \\\n",
      "1501                -1.0                    1                      0.765   \n",
      "651               2400.0                    8                      0.860   \n",
      "1393             30600.0                   60                      0.863   \n",
      "\n",
      "                  No-Ref  \n",
      "1501      rafiq2020_[64]  \n",
      "651      murray2019_[56]  \n",
      "1393  massidda2020_[134]  \n",
      "\n",
      "fridge\n",
      "4\n",
      "     Reference: Abbreviation Reference: Number Appliance Training: Dataset  \\\n",
      "151               barsim2018              [63]    fridge           UK-DALE   \n",
      "604               murray2019              [56]    fridge             REFIT   \n",
      "543           nascimento2016             [121]    fridge              REDD   \n",
      "1217              cavdar2019             [144]    fridge              REDD   \n",
      "\n",
      "     Model: Basic Type Model: Denomination  Input: Window Size  \\\n",
      "151        feedforward             CNN dAE             10800.0   \n",
      "604        feedforward                 CNN              6400.0   \n",
      "543          recurrent              RecCNN              9604.0   \n",
      "1217          combined             CNN-RNN                -1.0   \n",
      "\n",
      "     Input: Sampling Rate  Performance Metric: Value                No-Ref  \n",
      "151                     1                     0.9270       barsim2018_[63]  \n",
      "604                     8                     0.9300       murray2019_[56]  \n",
      "543                     4                     0.9497  nascimento2016_[121]  \n",
      "1217                    4                     0.9502      cavdar2019_[144]  \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Evaluate x best performing Models\n",
    "apps = ['kettle', 'microwave', 'dishwasher', 'washing machine', 'fridge']\n",
    "authors = []\n",
    "for app in apps:\n",
    "    df_app = df_f1[ df_f1['Appliance'] == app]\n",
    "    df_app = df_app.sort_values('Performance Metric: Value')\n",
    "    first_i_values = round(len(df_app)/4)\n",
    "    df_app = df_app.iloc[-first_i_values:,:]\n",
    "    print(app)\n",
    "    print(first_i_values)\n",
    "    authors.extend(df_app['Reference: Abbreviation'].values)\n",
    "    print(df_app)\n",
    "#    print('publications: ', df_app['Reference: Abbreviation'].values)\n",
    "#    print('values', df_app['Performance Metric: Value'].values)\n",
    "#    print('datasets: ', df_app['Training: Dataset'].unique())\n",
    "#    print('no of ff: ', np.sum(df_app['Model: Basic Type'] == 'feedforward'), \n",
    "#          'no of rnn: ',  np.sum(df_app['Model: Basic Type'] == 'recurrent'), \n",
    "#          'no of combined', np.sum(df_app['Model: Basic Type'] == 'combined') )\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>authors</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>murray2019</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>nascimento2016</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>rafiq2020</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cavdar2019</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>massidda2020</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sudoso2019</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>barsim2018</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          authors  counts\n",
       "2      murray2019       6\n",
       "3  nascimento2016       3\n",
       "0       rafiq2020       2\n",
       "4      cavdar2019       2\n",
       "5    massidda2020       2\n",
       "1      sudoso2019       1\n",
       "6      barsim2018       1"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "authors = np.asarray(authors)\n",
    "counts = []\n",
    "for author in pd.Series(authors).unique():\n",
    "    count = np.sum(authors == author)\n",
    "    counts.append(count)\n",
    "counts = pd.DataFrame({'authors': pd.Series(authors).unique(), 'counts': counts})\n",
    "counts.sort_values('counts', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernel_info": {
   "name": "genpurp_p36"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "nteract": {
   "version": "0.15.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}