From ece28a7ccf982d7da072ee397980f76858a27c03 Mon Sep 17 00:00:00 2001
From: Andy Lulham <a.lulham@gmail.com>
Date: Sat, 1 Jun 2024 23:35:13 +0100
Subject: [PATCH] feat: script for fetching election candidates

---
 .github/workflows/fetch_candidates.yml | 50 ++++++++++++++++++++++++++
 x_notes/fetch_candidates.py            | 14 ++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 .github/workflows/fetch_candidates.yml
 create mode 100644 x_notes/fetch_candidates.py

diff --git a/.github/workflows/fetch_candidates.yml b/.github/workflows/fetch_candidates.yml
new file mode 100644
index 00000000..096f3050
--- /dev/null
+++ b/.github/workflows/fetch_candidates.yml
@@ -0,0 +1,50 @@
+name: Fetch candidates
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 6 * * *"
+
+permissions:
+  contents: write
+
+concurrency:
+  group: all_workflows
+
+jobs:
+  fetch:
+    name: Fetch candidates
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: Set up python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: pyproject.toml
+          cache: poetry
+
+      - name: Install dependencies
+        run: poetry install --only main
+
+      - name: Checkout previous version
+        uses: actions/checkout@v4
+        with:
+          ref: gh-pages
+          path: previous
+
+      - name: Copy files into place
+        run: cp -r previous/* output
+
+      - name: Fetch candidates
+        run: poetry run python -m x_notes.fetch_candidates
+
+      - name: Deploy 🚀
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: output
+          single-commit: true
diff --git a/x_notes/fetch_candidates.py b/x_notes/fetch_candidates.py
new file mode 100644
index 00000000..a3625503
--- /dev/null
+++ b/x_notes/fetch_candidates.py
@@ -0,0 +1,14 @@
+import csv
+import json
+from io import StringIO
+
+import requests
+
+dc_csv = "https://candidates.democracyclub.org.uk/data/export_csv/?election_date=&ballot_paper_id=&election_id=parl.2024-07-04&party_id=&cancelled=&has_twitter_username=yes&extra_fields=twitter_username&format=csv"
+
+r = requests.get(dc_csv, stream=True)
+data = list(csv.DictReader(StringIO(r.text)))
+handles = [row["twitter_username"].replace('"', "") for row in data]
+
+with open("output/data/ge2024-candidates.json", "w") as fh:
+    json.dump(handles, fh)