-
Notifications
You must be signed in to change notification settings - Fork 0
121 lines (100 loc) · 3.42 KB
/
nightly.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
name: Nightly case scrape
on:
workflow_dispatch:
pull_request:
schedule:
- cron: '15 4 * * *'
jobs:
scrape:
runs-on: ubuntu-latest
steps:
- name: Set current date as env variable
run: echo "BEGIN_COURTS_RUN=$(date +'%s')" >> $GITHUB_ENV
- uses: actions/checkout@v3
- name: upgrade sqlite3
run: |
sudo apt-get update
sudo apt-get install sqlite3
- name: Install requirements
run: |
pip install -U pyopenssl cryptography
pip install -r requirements.txt
- name: Download latest database zip
uses: robinraju/[email protected]
with:
latest: true
tag: "nightly"
fileName: "*.db.zip"
- name: Decrypt database
run: |
unzip -P '${{ secrets.CASE_DB_PW }}' cases.db.zip
- name: Setup tmate session
uses: mxschmitt/action-tmate@v3
- name: Run scrape
env:
SCRAPY_SETTINGS_MODULE: "courtscraper.settings"
run: |
echo $BEGIN_COURTS_RUN
echo PYTHONPATH="courtscraper" >> $GITHUB_ENV
make get_new_records
- name: Setup database for upload
run: |
zip -p '${{ secrets.CASE_DB_PW }}' cases.db.zip cases.db
- name: Upload new release
uses: WebFreak001/[email protected]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: https://uploads.github.com/repos/datamade/court-scrapers/releases/131985702/assets{?name,label}
release_id: 131985702
asset_path: ./cases.db.zip
asset_name: cases.db.zip
asset_content_type: application/zip # required by GitHub API
max_releases: 7
- name: keepalive
uses: gautamkrishnar/keepalive-workflow@v1
deploy:
name: Deploy to Heroku
needs: scrape
runs-on: ubuntu-latest
env:
HEROKU_ORGANIZATION: ${{ secrets.HEROKU_ORG }}
steps:
- uses: actions/checkout@v3
- name: Install requirements
run: pip install -r requirements.txt
- name: Download latest database zip
uses: robinraju/[email protected]
with:
latest: true
tag: "nightly"
fileName: "*.db.zip"
- name: Decrypt database
run: |
unzip -P '${{ secrets.CASE_DB_PW }}' cases.db.zip
- name: Install heroku-builds plugin
run: |
heroku plugins:install heroku-builds
- name: Login to Heroku CLI
uses: akhileshns/[email protected]
with:
heroku_api_key: ${{ secrets.HEROKU_API_KEY }}
heroku_app_name: ""
heroku_email: ${{ secrets.HEROKU_EMAIL }}
justlogin: true
- name: Install Datasette plugins
run: |
datasette install datasette-auth-passwords datasette-auth-tokens
- name: Get hashed Datasette password
run: |
# Store hash as an environment variable
hash=$(echo '${{ secrets.DATASETTE_INSTANCE_PW }}' \
| datasette hash-password --no-confirm); \
echo "hash=$hash" >> $GITHUB_ENV
- name: Deploy Datasette instance to Heroku
run: |
datasette publish heroku cases.db \
-n court-scraper \
-m metadata.json \
--install datasette-auth-passwords \
--plugin-secret datasette-auth-passwords root_password_hash '${{ env.hash }}'