summary-so-far.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "57e66c22-0fb4-42cd-a6c7-abd3d5eed6e4",
   "metadata": {
    "tags": []
   },
   "source": [
    "# SATRE feature questionnaire (intermediate summary)\n",
    "\n",
    "This is an initial intermediate overview of the feature survey results.\n",
    "It may contain sensitive information, and should not be made public without further review.\n",
    "\n",
    "This notebook can be executed from the command line and exported as HTML:\n",
    "```\n",
    "python -mnbconvert --to html --execute summary-so-far.ipynb\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47c51d61-0eaf-422e-8ba3-b7ac26f0664b",
   "metadata": {
    "tags": []
   },
   "source": [
    "Results were exported from JISC online surveys with these options:\n",
    "- Include date ~~and time~~ of response submission\n",
    "- Include section headings\n",
    "- Microsoft Excel 2007 and later (.xlsx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f29dae6f-fcba-476e-91a0-f0ad81b49067",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime \n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "# Requires openpyxl for read_excel\n",
    "from pathlib import Path\n",
    "import re\n",
    "from IPython.display import Markdown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9ce8ee2-bde8-4d8e-81e3-9f3644a771f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import spreadsheet cleaning loader\n",
    "from load_survey_data import load_data, headers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1959ba3a-bde7-4a17-93b9-a14c1dd3fc39",
   "metadata": {},
   "source": [
    "Choose a spreadsheet to load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41b91b53-36b3-45f6-98b6-be4bd9c2a8eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "sheet = \"summarise-all-results-2023-05-16-1359.xlsx\"\n",
    "format = \"%Y-%m-%d %H:%M:%S %Z\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79c822f9-054a-4fa2-a657-7665a3a1ab1e",
   "metadata": {},
   "source": [
    "Each column in the spreadsheet should be either:\n",
    "- Section heading (no data)\n",
    "- Free text\n",
    "- Category/ranking question\n",
    "- Timestamp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f281001-a733-4d3b-89a9-30db0e7ead9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = load_data(sheet, format)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a77495c-3334-4ed4-b27c-529b215b35cb",
   "metadata": {},
   "source": [
    "## Quick overview of completions per day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "160476b7-cf97-4da0-8099-6cf1a99fda95",
   "metadata": {},
   "outputs": [],
   "source": [
    "completion_dt = df['CompletionDate'].dt.date\n",
    "start_date = completion_dt.min()\n",
    "days_since_start = pd.Series((completion_dt - start_date).dt.days)\n",
    "ax = days_since_start.hist(bins=(days_since_start.max()))\n",
    "ax.set_xlabel(\"Days since start of survey (\" + start_date.strftime(\"%a %d %b %Y\") + \")\")\n",
    "ax.set_ylabel(\"Number of responses\")\n",
    "ax.set_title(f\"Total: {len(df)} responses\");"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fa8b197-86a8-4432-8b45-838fc0ae3d80",
   "metadata": {},
   "source": [
    "## Quickly show all the data\n",
    "\n",
    "- Rankings are shown as bar plots, all use the same X and Y axes\n",
    "- Free text answers are in alphabetical order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4b24389-0480-4fa5-b18d-4875040801d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def truncate_with_ellipsis(s, n):\n",
    "    if len(s) > n:\n",
    "        s = s[:(n - 1)]\n",
    "        return f\"{s}…\"\n",
    "    return s\n",
    "\n",
    "\n",
    "def show_text_answer(column):\n",
    "    values = sorted(v.strip() for v in column.dropna().values if v.strip())\n",
    "    display(Markdown(f\"### {h} ({len(values)} answers)\"))\n",
    "    display(Markdown(\"\\n\".join(f\"- {v}\" for v in values)))\n",
    "\n",
    "\n",
    "def show_ranking_answer(column, ax, y_max):\n",
    "    counts = column.value_counts().sort_index()\n",
    "    counts.plot(kind='bar', ax=ax)\n",
    "    ax.set_ylim(0, y_max + 1)\n",
    "    ax.grid(axis='y')\n",
    "    # ax.tick_params(axis=\"x\", labelrotation='auto')\n",
    "    ax.set_ylabel(truncate_with_ellipsis(h, 55))\n",
    "    return ax\n",
    "\n",
    "\n",
    "# Maximum number for any answer\n",
    "y_max = 0\n",
    "for (h, t) in headers.items():\n",
    "    if t == \"RANKING\":\n",
    "        y_max = max(y_max, df[h].value_counts().max())\n",
    "\n",
    "max_plots_in_one_section = 5\n",
    "\n",
    "matplotlib.rcParams.update({'figure.max_open_warning': 0})\n",
    "\n",
    "fig = None\n",
    "axs = []\n",
    "ax_n = 0\n",
    "for (h, t) in headers.items():\n",
    "    if t != \"RANKING\":\n",
    "        # Delete unused plot spaces\n",
    "        for ax in axs[ax_n:]:\n",
    "            fig.delaxes(ax)\n",
    "        # Show plot NOW, so that we can interleave plots and markdown\n",
    "        if ax_n:\n",
    "            plt.show()\n",
    "        fig = None\n",
    "        axs = []\n",
    "        ax_n = 0\n",
    "\n",
    "    if t == \"SECTION\":\n",
    "        display(Markdown(f\"## {h}\"))\n",
    "    if t == \"TEXT\":\n",
    "        show_text_answer(df[h])\n",
    "    if t == \"RANKING\":\n",
    "        if not fig:\n",
    "            fig, axs = plt.subplots(1, max_plots_in_one_section, figsize=(10, 5))\n",
    "            fig.subplots_adjust(wspace=1)\n",
    "        ax = axs[ax_n]\n",
    "        show_ranking_answer(df[h], ax, y_max)\n",
    "        ax_n += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70257b6e-f3b0-47ac-abab-6d9bf54fee9b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}