Feature: analyze Survey Results per week and report a summary of them

.github/workflows/create-summary.yml

@@ -0,0 +1,137 @@
+name: Create Summary
+
+on:
+  schedule:
+  - cron: '0 7 * * 1'
+  workflow_dispatch:
+    inputs:
+      year:
+        description: 'Year to create the summary of'
+        required: true
+      week:
+        description: 'ISO week to create the summary of'
+        required: true
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+
+    name: Create Summary
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Install rclone
+      shell: bash
+      run: |
+        curl -sL https://rclone.org/install.sh | sudo bash
+
+        rclone config create --no-obscure openttd s3 \
+          provider Cloudflare \
+          access_key_id ${{ secrets.R2_SURVEY_ACCESS_KEY_ID }} \
+          secret_access_key ${{ secrets.R2_SURVEY_SECRET_ACCESS_KEY }} \
+          endpoint ${{ secrets.R2_SURVEY_ENDPOINT }} \
+          acl private \
+          no_check_bucket true
+
+    - name: Calculate dates
+      shell: bash
+      id: dates
+      run: |
+        year="${{ inputs.year }}"
+        week="${{ inputs.week }}"
+
+        # If no year/week is given, detect the week before today.
+        # This assumes this script is started on a Monday.
+        if [ -z "${year}" ] || [ -z "${week}"]; then
+          year=$(date -d "yesterday" +%G)
+          week=$(date -d "yesterday" +%V)
+        fi
+
+        # Calculate the start and end of the week.
+        day_in_week=$(date -d "${year}-01-01" +%u)
+        if [ "${day_in_week}" -lt 5 ]; then
+            first_monday=$(date -d "${year}-01-01 -${day_in_week} days +1 day" +%Y-%m-%d)
+        else
+            first_monday=$(date -d "${year}-01-01 -${day_in_week} days +8 day" +%Y-%m-%d)
+        fi
+        start_date=$(date -d "${first_monday} +${week} weeks -1 week" +%Y-%m-%d)
+        end_date=$(date -d "${start_date} +6 days" +%Y-%m-%d)
+
+        # Ensure the start date is actually in the week we want.
+        start_check=$(date -d "${start_date}" +%G-%V)
+        if [ "${start_check}" != "${year}-${week}" ]; then
+          echo "Start date ${start_date} is not in week ${week} of ${year}, but in ${start_check}"
+          exit 1
+        fi
+
+        # Ensure the end date is actually in the week we want.
+        end_check=$(date -d "${end_date}" +%G-%V)
+        if [ "${end_check}" != "${year}-${week}" ]; then
+          echo "End date ${end_date} is not in week ${week} of ${year}, but in ${end_check}"
+          exit 1
+        fi
+
+        if [ "${week}" -lt 10 ]; then
+          week="0${week}"
+        fi
+
+        echo "Week: ${week}"
+        echo "Year: ${year}"
+        echo "Start date: ${start_date}"
+        echo "End date: ${end_date}"
+
+        echo "week=${week}" >> "$GITHUB_OUTPUT"
+        echo "year=${year}" >> "$GITHUB_OUTPUT"
+        echo "start_date=${start_date}" >> "$GITHUB_OUTPUT"
+        echo "end_date=${end_date}" >> "$GITHUB_OUTPUT"
+
+    - name: Download packed results
+      shell: bash
+      run: |
+        echo "Downloading packs for week ${{ steps.dates.outputs.week }} in ${{ steps.dates.outputs.year }}: [${{ steps.dates.outputs.start_date }} .. ${{ steps.dates.outputs.end_date }}]"
+
+        mkdir -p packed
+        for i in $(seq 0 6); do
+          date=$(date -d "${start_date} +${i} days" +%Y-%m-%d)
+          date_year=$(date -d "${start_date} +${i} days" +%Y)
+          date_month=$(date -d "${start_date} +${i} days" +%m)
+
+          echo "Downloading ${date}"
+          rclone copy -v openttd:survey-packed-prod/${date_year}/${date_month}/openttd-survey-pack.${date}.tar.xz packed
+        done
+
+    - name: Run analysis
+      shell: bash
+      run: |
+        mkdir -p _data/summaries/${{ steps.dates.outputs.year }}
+        python -m analysis packed/* > _data/summaries/${{ steps.dates.outputs.year }}/wk${{ steps.dates.outputs.week }}.json
+
+    - name: Create summary entry
+      shell: bash
+      run: |
+        mkdir -p _summaries/${{ steps.dates.outputs.year }}
+        summary="_summaries/${{ steps.dates.outputs.year }}/wk${{ steps.dates.outputs.week }}.md"
+
+        echo "---" > ${summary}
+        echo "name: ${{ steps.dates.outputs.year }} - Week ${{ steps.dates.outputs.week }}" >> ${summary}
+        echo "title: Survey Result Summary - ${{ steps.dates.outputs.year }} - Week ${{ steps.dates.outputs.week }}"
+        echo "active_nav: summaries" >> ${summary}
+        echo "year: \"${{ steps.dates.outputs.year }}\"" >> ${summary}
+        echo "week: wk${{ steps.dates.outputs.week }}" >> ${summary}
+        echo "start_date: \"${{ steps.dates.outputs.start_date }}\"" >> ${summary}
+        echo "end_date: \"${{ steps.dates.outputs.end_date }}\"" >> ${summary}
+        echo "---" >> ${summary}
+
+    - name: Commit and push
+      shell: bash
+      run: |
+        git config --global user.name "OpenTTD Survey"
+        git config --global user.email "[email protected]"
+
+        git add _data/summaries/${{ steps.dates.outputs.year }}/wk${{ steps.dates.outputs.week }}.json
+        git add _summaries/${{ steps.dates.outputs.year }}/wk${{ steps.dates.outputs.week }}.md
+
+        git commit -m "Add: summary for week ${{ steps.dates.outputs.week }} of ${{ steps.dates.outputs.year }}"
+        git push

README.md

@@ -8,6 +8,12 @@ This is a [Jekyll](https://jekyllrb.com/) website, and is served by nginx as a s
 
 ## Development
 
+### Survey results
+
+To summarize survey results, the Python application `analysis` processes a bundle of JSONs and outputs another JSON with the summary.
+
+To run it, simply execute `python3 -m analysis <tar-xz bundle files>`
+
 ### Running a local server
 
 If you do not want to run a server, but just build the current site, replace `serve` with `build` in the examples below.

_config.yml

@@ -23,10 +23,10 @@ defaults:
       layout: "default"
   - scope:
       path: ""
-      type: "analysis"
+      type: "summaries"
     values:
-      layout: "analysis"
+      layout: "summaries"
 
 collections:
-  analysis:
+  summaries:
     output: true

_layouts/default.html

@@ -25,8 +25,8 @@
                 <li class="{% if page.active_nav == 'participate' %}selected{% endif %}">
                     <a href="{{ site.baseurl }}/participate.html">Participate</a>
                 </li>
-                <li class="{% if page.active_nav == 'results' %}selected{% endif %}">
-                    <a href="{{ site.baseurl }}/results.html">Results</a>
+                <li class="{% if page.active_nav == 'summaries' %}selected{% endif %}">
+                    <a href="{{ site.baseurl }}/summaries.html">Summaries</a>
                 </li>
             </ul>
         </nav>

_layouts/summaries.html

@@ -0,0 +1,83 @@
+---
+layout: default
+---
+
+<div id="section-full">
+    <div class="section-header">
+        <h3>Survey Result Summary - {{ page.name }}</h3>
+    </div>
+    <div class="section-item">
+        <div class="content">
+            <p>
+                Below is the summary of the survey results from {{ page.start_date }} to {{ page.end_date }}.
+            </p>
+
+            <ul>
+                {% for version in site.data.summaries[page.year][page.week] %}
+                    <li>
+                        <a href="#{{ version[0] }}">{{ version[0] }}</a>
+                    </li>
+                {% endfor %}
+            </ul>
+
+            <p>
+                The following statistics are based on "seconds played".
+                That is to say, if a setting was "true" for 100 seconds of play-time and "false" for 200 seconds of play-time, it would be 33% true and 66% false.
+                This is to avoid biasing the results towards shorter games.
+                <br />
+                Worded differently, the following statistics are not counting how often something was used, but how long it was used for.
+            </p>
+
+            <p>
+                Be mindful that these numbers are only based on those people willing to send us survey results.
+                It might not be representative of the entire OpenTTD player base.
+            </p>
+        </div>
+    </div>
+
+    {% for version in site.data.summaries[page.year][page.week] %}
+        <div class="section-header" id="{{ version[0] }}">
+            <h3>{{ version[0] }}</h3>
+        </div>
+        <div class="section-item">
+            <div class="content">
+                {% if version[1] == nil %}
+                    <p>
+                        Due to low number of games played, no summary is available.
+                    </p>
+                    {% continue %}
+                {% endif %}
+
+                {% assign seconds = version[1].summary.seconds | times: 1.0 %}
+
+                <p>
+                    We received surveys for a total of {{ version[1].summary.seconds | divided_by: 3600 | round: 2 }} hours of games played, over a total of {{ version[1].summary.ids }} games.
+                    This is an average of {{ seconds | divided_by: version[1].summary.ids | divided_by: 3600 | round: 2 }} hours per game.
+                </p>
+
+                <table class="summary-table">
+                {% for summary in version[1] %}
+                    {% if summary[0] == "summary" %}{% continue %}{% endif %}
+                    <tr id="{{ version[0] }}-{{ summary[0] }}" class="setting">
+                        <th colspan="3">
+                            <a href="#{{ version[0] }}-{{ summary[0] }}">{{ summary[0] }}</a>
+                        </th>
+                    </tr>
+                    {% for line in summary[1] %}
+                        {% assign percentage = line[1] | divided_by: seconds | percentage %}
+                    <tr>
+                        <td style="width: 40px;"></td>
+                        <td><pre>{{ line[0] }}</pre></td>
+                        {% if percentage == "0.0" %}
+                        <td style="text-align: right;">&lt;0.1%</td>
+                        {% else %}
+                        <td style="text-align: right;">{{ percentage }}%</td>
+                        {% endif %}
+                    </tr>
+                    {% endfor %}
+                {% endfor %}
+                </table>
+            </div>
+        </div>
+    {% endfor %}
+</div>

_plugins/openttd-filters.rb

@@ -0,0 +1,10 @@
+module Jekyll
+    module OpenTTDFilters
+
+        def percentage(value)
+            return sprintf("%.1f", value * 100)
+        end
+    end
+end
+
+Liquid::Template.register_filter(Jekyll::OpenTTDFilters)