Feature: ability to generate quarterly reports

OpenTTD · Oct 4, 2024 · a263d9e · a263d9e
1 parent 7291136
commit a263d9e
Show file tree

Hide file tree

Showing 182 changed files with 437 additions and 204 deletions.
diff --git a/.github/workflows/create-summary-quarter.yml b/.github/workflows/create-summary-quarter.yml
@@ -0,0 +1,134 @@
+name: Create Summary
+
+on:
+  schedule:
+  - cron: '0 8 1 * *'
+  workflow_dispatch:
+    inputs:
+      year:
+        description: 'Year to create the summary of'
+        required: true
+      quarter:
+        description: 'Quarter to create the summary of'
+        required: true
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+
+    name: Create Summary
+
+    steps:
+    - name: Generate access token
+      id: generate_token
+      uses: tibdex/github-app-token@v2
+      with:
+        app_id: ${{ secrets.SURVEY_SUMMARY_APP_ID }}
+        private_key: ${{ secrets.SURVEY_SUMMARY_APP_PRIVATE_KEY }}
+        installation_retrieval_mode: "repository"
+        installation_retrieval_payload: "OpenTTD/survey-web"
+
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        token: ${{ steps.generate_token.outputs.token }}
+
+    - name: Install rclone
+      shell: bash
+      run: |
+        curl -sL https://rclone.org/install.sh | sudo bash
+
+        rclone config create --no-obscure openttd s3 \
+          provider Cloudflare \
+          access_key_id ${{ secrets.R2_SURVEY_ACCESS_KEY_ID }} \
+          secret_access_key ${{ secrets.R2_SURVEY_SECRET_ACCESS_KEY }} \
+          endpoint ${{ secrets.R2_SURVEY_ENDPOINT }} \
+          acl private \
+          no_check_bucket true
+
+    - name: Calculate dates
+      shell: bash
+      id: dates
+      run: |
+        year="$1"
+        quarter="$2"
+
+        # If no year/quarter is given, detect the quarter before today.
+        # This assumes this script is started on the first day of the next quarter.
+        if [ -z "${year}" ] || [ -z "${quarter}" ]; then
+            year=$(date -d "yesterday" +%Y)
+            quarter=$(date -d "yesterday" +%q)
+        fi
+
+        # Calculate the start and end of the quarter.
+        start_month=$(( (${quarter} - 1) * 3 + 1 ))
+        end_month=$(( ${quarter} * 3 ))
+        start_date=$(date -d "${year}-${start_month}-01" +%Y-%m-%d)
+        end_date=$(date -d "${year}-${end_month}-01 +1 month -1 day" +%Y-%m-%d)
+
+        # Ensure the start date is actually in the quarter we want.
+        start_check=$(date -d "${start_date}" +%Y-%q)
+        if [ "${start_check}" != "${year}-${quarter}" ]; then
+            echo "Start date ${start_date} is not in ${year}-${quarter}, but in ${start_check}"
+            exit 1
+        fi
+
+        # Ensure the end date is actually in the quarter we want.
+        end_check=$(date -d "${end_date}" +%Y-%q)
+        if [ "${end_check}" != "${year}-${quarter}" ]; then
+            echo "End date ${end_date} is not in ${year}-${quarter}, but in ${end_check}"
+            exit 1
+        fi
+
+        echo "Quarter: ${quarter}"
+        echo "Year: ${year}"
+        echo "Start date: ${start_date}"
+        echo "End date: ${end_date}"
+
+        echo "quarter=${quarter}" >> "$GITHUB_OUTPUT"
+        echo "year=${year}" >> "$GITHUB_OUTPUT"
+        echo "start_date=${start_date}" >> "$GITHUB_OUTPUT"
+        echo "end_date=${end_date}" >> "$GITHUB_OUTPUT"
+
+    - name: Download packed results
+      shell: bash
+      run: |
+        echo "Downloading packs for quarter ${{ steps.dates.outputs.quarter }} in ${{ steps.dates.outputs.year }}: [${{ steps.dates.outputs.start_date }} .. ${{ steps.dates.outputs.end_date }}]"
+
+        mkdir -p packed
+        for i in $(seq 0 93); do
+          date=$(date -d "${{ steps.dates.outputs.start_date }} +${i} days" +%Y-%m-%d)
+          date_year=$(date -d "${{ steps.dates.outputs.start_date }} +${i} days" +%Y)
+          date_month=$(date -d "${{ steps.dates.outputs.start_date }} +${i} days" +%m)
+
+          echo "Downloading ${date}"
+          rclone copy -v openttd:survey-packed-prod/${date_year}/${date_month}/openttd-survey-pack.${date}.tar.xz packed
+
+          if [ "${date}" = "${{ steps.dates.outputs.end_date }}" ]; then
+            break
+          fi
+        done
+
+    - name: Run analysis
+      shell: bash
+      run: |
+        mkdir -p _data/summaries/${{ steps.dates.outputs.year }}
+        python -m analysis q packed/* > _data/summaries/${{ steps.dates.outputs.year }}/q${{ steps.dates.outputs.quarter }}.json
+
+    - name: Create summary entry
+      shell: bash
+      run: |
+        python -m create_markdown q ${{ steps.dates.outputs.year }} ${{ steps.dates.outputs.quarter }} ${{ steps.dates.outputs.start_date }} ${{ steps.dates.outputs.end_date }}
+
+    - name: Commit and push
+      shell: bash
+      run: |
+        git config --global user.name "OpenTTD Survey"
+        git config --global user.email "[email protected]"
+
+        git add _data/summaries/${{ steps.dates.outputs.year }}/q${{ steps.dates.outputs.quarter }}.json
+        git add _summaries/${{ steps.dates.outputs.year }}/q${{ steps.dates.outputs.quarter }}.md
+        git add _summaries/${{ steps.dates.outputs.year }}/q${{ steps.dates.outputs.quarter }}
+
+        git commit -m "Add: summary for Q${{ steps.dates.outputs.quarter }} of ${{ steps.dates.outputs.year }}"
+        git push
diff --git a/.github/workflows/create-summary.yml → .github/workflows/create-summary-week.yml b/.github/workflows/create-summary.yml → .github/workflows/create-summary-week.yml
@@ -117,12 +117,12 @@ jobs:
       shell: bash
       run: |
         mkdir -p _data/summaries/${{ steps.dates.outputs.year }}
-        python -m analysis packed/* > _data/summaries/${{ steps.dates.outputs.year }}/wk${{ steps.dates.outputs.week }}.json
+        python -m analysis wk packed/* > _data/summaries/${{ steps.dates.outputs.year }}/wk${{ steps.dates.outputs.week }}.json
 
     - name: Create summary entry
       shell: bash
       run: |
-        python -m create_markdown ${{ steps.dates.outputs.year }} ${{ steps.dates.outputs.week }} ${{ steps.dates.outputs.start_date }} ${{ steps.dates.outputs.end_date }}
+        python -m create_markdown wk ${{ steps.dates.outputs.year }} ${{ steps.dates.outputs.week }} ${{ steps.dates.outputs.start_date }} ${{ steps.dates.outputs.end_date }}
 
     - name: Commit and push
       shell: bash

diff --git a/_layouts/summaries.html b/_layouts/summaries.html
@@ -14,13 +14,14 @@ <h3>Survey Result Summary - {{ page.title }}</h3>
 
             <ul>
                 {% for summary in site.summaries %}
-                    {% if summary.year != page.year or summary.week != page.week or summary.version == nil %}{% continue %}{% endif %}
+                    {% if summary.year != page.year or summary.filename != page.filename or summary.version == nil %}{% continue %}{% endif %}
                     <li>
                         <a href="{{ summary.url }}">{{ summary.version }}</a>
                     </li>
                 {% endfor %}
             </ul>
 
+            {% if page.type == "week" %}
             <p>
                 <small>
                     Note: some versions are not shown, as they do not have a summary created for them.
@@ -30,6 +31,7 @@ <h3>Survey Result Summary - {{ page.title }}</h3>
                     For more details about these thresholds and validations, please check <a href="https://github.com/OpenTTD/survey-web/blob/main/analysis/__main__.py">the source code</a>.
                 </small>
             </p>
+            {% endif %}
         </div>
     </div>
 </div>
diff --git a/_layouts/summary.html b/_layouts/summary.html
@@ -29,7 +29,7 @@ <h3>Survey Result Summary - {{ page.title }}</h3>
                 Entries listed as "(other)" are the collection of all values lower than 0.1% of the total.
             </p>
 
-            {% assign versions = site.data.summaries[page.year][page.week] %}
+            {% assign versions = site.data.summaries[page.year][page.filename] %}
             {% assign version = versions[page.version] %}
             {% assign seconds = version.summary.seconds | times: 1.0 %}
 

diff --git a/_summaries/2024/wk01.md b/_summaries/2024/wk01.md
@@ -2,7 +2,8 @@
 title: 2024 - Week 01
 active_nav: summaries
 year: "2024"
-week: "wk01"
+type: week
+filename: "wk01"
 start_date: "2024-01-01"
 end_date: "2024-01-07"
 ---
diff --git a/_summaries/2024/wk01/jgrpp-0.56.2.md b/_summaries/2024/wk01/jgrpp-0.56.2.md
@@ -2,7 +2,7 @@
 title: 2024 - Week 01 - jgrpp-0.56.2
 active_nav: summaries
 year: "2024"
-week: "wk01"
+filename: "wk01"
 version: "jgrpp-0.56.2"
 start_date: "2024-01-01"
 end_date: "2024-01-07"

diff --git a/_summaries/2024/wk02.md b/_summaries/2024/wk02.md
@@ -2,7 +2,8 @@
 title: 2024 - Week 02
 active_nav: summaries
 year: "2024"
-week: "wk02"
+type: week
+filename: "wk02"
 start_date: "2024-01-08"
 end_date: "2024-01-14"
 ---
diff --git a/_summaries/2024/wk02/jgrpp-0.56.2.md b/_summaries/2024/wk02/jgrpp-0.56.2.md
@@ -2,7 +2,7 @@
 title: 2024 - Week 02 - jgrpp-0.56.2
 active_nav: summaries
 year: "2024"
-week: "wk02"
+filename: "wk02"
 version: "jgrpp-0.56.2"
 start_date: "2024-01-08"
 end_date: "2024-01-14"

diff --git a/_summaries/2024/wk03.md b/_summaries/2024/wk03.md
@@ -2,7 +2,8 @@
 title: 2024 - Week 03
 active_nav: summaries
 year: "2024"
-week: "wk03"
+type: week
+filename: "wk03"
 start_date: "2024-01-15"
 end_date: "2024-01-21"
 ---
diff --git a/_summaries/2024/wk03/jgrpp-0.56.2.md b/_summaries/2024/wk03/jgrpp-0.56.2.md
@@ -2,7 +2,7 @@
 title: 2024 - Week 03 - jgrpp-0.56.2
 active_nav: summaries
 year: "2024"
-week: "wk03"
+filename: "wk03"
 version: "jgrpp-0.56.2"
 start_date: "2024-01-15"
 end_date: "2024-01-21"

diff --git a/_summaries/2024/wk03/vanilla-master.md b/_summaries/2024/wk03/vanilla-master.md
@@ -2,7 +2,7 @@
 title: 2024 - Week 03 - vanilla-master
 active_nav: summaries
 year: "2024"
-week: "wk03"
+filename: "wk03"
 version: "vanilla-master"
 start_date: "2024-01-15"
 end_date: "2024-01-21"

diff --git a/_summaries/2024/wk04.md b/_summaries/2024/wk04.md
@@ -2,7 +2,8 @@
 title: 2024 - Week 04
 active_nav: summaries
 year: "2024"
-week: "wk04"
+type: week
+filename: "wk04"
 start_date: "2024-01-22"
 end_date: "2024-01-28"
 ---
diff --git a/_summaries/2024/wk04/jgrpp-0.56.2.md b/_summaries/2024/wk04/jgrpp-0.56.2.md
@@ -2,7 +2,7 @@
 title: 2024 - Week 04 - jgrpp-0.56.2
 active_nav: summaries
 year: "2024"
-week: "wk04"
+filename: "wk04"
 version: "jgrpp-0.56.2"
 start_date: "2024-01-22"
 end_date: "2024-01-28"

diff --git a/_summaries/2024/wk04/vanilla-master.md b/_summaries/2024/wk04/vanilla-master.md
@@ -2,7 +2,7 @@
 title: 2024 - Week 04 - vanilla-master
 active_nav: summaries
 year: "2024"
-week: "wk04"
+filename: "wk04"
 version: "vanilla-master"
 start_date: "2024-01-22"
 end_date: "2024-01-28"

diff --git a/_summaries/2024/wk05.md b/_summaries/2024/wk05.md
@@ -2,7 +2,8 @@
 title: 2024 - Week 05
 active_nav: summaries
 year: "2024"
-week: "wk05"
+type: week
+filename: "wk05"
 start_date: "2024-01-29"
 end_date: "2024-02-04"
 ---
diff --git a/_summaries/2024/wk05/14.0-beta1.md b/_summaries/2024/wk05/14.0-beta1.md
@@ -2,7 +2,7 @@
 title: 2024 - Week 05 - 14.0-beta1
 active_nav: summaries
 year: "2024"
-week: "wk05"
+filename: "wk05"
 version: "14.0-beta1"
 start_date: "2024-01-29"
 end_date: "2024-02-04"

diff --git a/_summaries/2024/wk05/jgrpp-0.56.2.md b/_summaries/2024/wk05/jgrpp-0.56.2.md
@@ -2,7 +2,7 @@
 title: 2024 - Week 05 - jgrpp-0.56.2
 active_nav: summaries
 year: "2024"
-week: "wk05"
+filename: "wk05"
 version: "jgrpp-0.56.2"
 start_date: "2024-01-29"
 end_date: "2024-02-04"

diff --git a/_summaries/2024/wk05/vanilla-master.md b/_summaries/2024/wk05/vanilla-master.md
@@ -2,7 +2,7 @@
 title: 2024 - Week 05 - vanilla-master
 active_nav: summaries
 year: "2024"
-week: "wk05"
+filename: "wk05"
 version: "vanilla-master"
 start_date: "2024-01-29"
 end_date: "2024-02-04"

diff --git a/_summaries/2024/wk06.md b/_summaries/2024/wk06.md
@@ -2,7 +2,8 @@
 title: 2024 - Week 06
 active_nav: summaries
 year: "2024"
-week: "wk06"
+type: week
+filename: "wk06"
 start_date: "2024-02-05"
 end_date: "2024-02-11"
 ---
diff --git a/_summaries/2024/wk06/14.0-beta2.md b/_summaries/2024/wk06/14.0-beta2.md
@@ -2,7 +2,7 @@
 title: 2024 - Week 06 - 14.0-beta2
 active_nav: summaries
 year: "2024"
-week: "wk06"
+filename: "wk06"
 version: "14.0-beta2"
 start_date: "2024-02-05"
 end_date: "2024-02-11"

diff --git a/_summaries/2024/wk06/14.0-beta3.md b/_summaries/2024/wk06/14.0-beta3.md
@@ -2,7 +2,7 @@
 title: 2024 - Week 06 - 14.0-beta3
 active_nav: summaries
 year: "2024"
-week: "wk06"
+filename: "wk06"
 version: "14.0-beta3"
 start_date: "2024-02-05"
 end_date: "2024-02-11"

diff --git a/_summaries/2024/wk06/jgrpp-0.56.2.md b/_summaries/2024/wk06/jgrpp-0.56.2.md
@@ -2,7 +2,7 @@
 title: 2024 - Week 06 - jgrpp-0.56.2
 active_nav: summaries
 year: "2024"
-week: "wk06"
+filename: "wk06"
 version: "jgrpp-0.56.2"
 start_date: "2024-02-05"
 end_date: "2024-02-11"

diff --git a/_summaries/2024/wk06/jgrpp-0.57.1.md b/_summaries/2024/wk06/jgrpp-0.57.1.md
@@ -2,7 +2,7 @@
 title: 2024 - Week 06 - jgrpp-0.57.1
 active_nav: summaries
 year: "2024"
-week: "wk06"
+filename: "wk06"
 version: "jgrpp-0.57.1"
 start_date: "2024-02-05"
 end_date: "2024-02-11"

diff --git a/_summaries/2024/wk06/vanilla-master.md b/_summaries/2024/wk06/vanilla-master.md
@@ -2,7 +2,7 @@
 title: 2024 - Week 06 - vanilla-master
 active_nav: summaries
 year: "2024"
-week: "wk06"
+filename: "wk06"
 version: "vanilla-master"
 start_date: "2024-02-05"
 end_date: "2024-02-11"

diff --git a/_summaries/2024/wk07.md b/_summaries/2024/wk07.md
@@ -2,7 +2,8 @@
 title: 2024 - Week 07
 active_nav: summaries
 year: "2024"
-week: "wk07"
+type: week
+filename: "wk07"
 start_date: "2024-02-12"
 end_date: "2024-02-18"
 ---
diff --git a/_summaries/2024/wk07/14.0-beta3.md b/_summaries/2024/wk07/14.0-beta3.md
@@ -2,7 +2,7 @@
 title: 2024 - Week 07 - 14.0-beta3
 active_nav: summaries
 year: "2024"
-week: "wk07"
+filename: "wk07"
 version: "14.0-beta3"
 start_date: "2024-02-12"
 end_date: "2024-02-18"

diff --git a/_summaries/2024/wk07/jgrpp-0.56.2.md b/_summaries/2024/wk07/jgrpp-0.56.2.md
@@ -2,7 +2,7 @@
 title: 2024 - Week 07 - jgrpp-0.56.2
 active_nav: summaries
 year: "2024"
-week: "wk07"
+filename: "wk07"
 version: "jgrpp-0.56.2"
 start_date: "2024-02-12"
 end_date: "2024-02-18"