Archive #1760

	name: Archive

	on:
	schedule:
	- cron: "7 2,11 * * *"
	workflow_dispatch:

	env:
	CI: true
	PIPENV_VENV_IN_PROJECT: true
	SCRAPY_SETTINGS_MODULE: city_scrapers.settings.archive
	AUTOTHROTTLE_MAX_DELAY: 30.0
	AUTOTHROTTLE_START_DELAY: 1.5
	AUTOTHROTTLE_TARGET_CONCURRENCY: 3.0
	OPENVPN_USER: ${{ secrets.OPENVPN_USER }}
	OPENVPN_PASS: ${{ secrets.OPENVPN_PASS }}
	OPENVPN_CONFIG: ${{ secrets.OPENVPN_CONFIG }}

	jobs:
	crawl:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v1

	- name: Connect to OpenVPN
	run: \|
	sudo apt-get install -y openvpn
	echo "$OPENVPN_USER" \| sudo tee -a /etc/openvpn/client/auth
	echo "$OPENVPN_PASS" \| sudo tee -a /etc/openvpn/client/auth
	echo "$OPENVPN_CONFIG" \| sudo tee -a /etc/openvpn/ovpn.conf
	sudo openvpn --config /etc/openvpn/ovpn.conf --daemon
	sleep 120

	- name: Set up Python 3.11
	uses: actions/setup-python@v1
	with:
	python-version: 3.11

	- name: Install Pipenv
	uses: dschep/install-pipenv-action@v1

	- name: Cache Python dependencies
	uses: actions/cache@v1
	with:
	path: .venv
	key: pip-3.11-${{ hashFiles('**/Pipfile.lock') }}
	restore-keys: \|
	pip-3.11-
	pip-

	- name: Install dependencies
	run: pipenv sync
	env:
	PIPENV_DEFAULT_PYTHON_VERSION: 3.11

	- name: Run scrapers
	run: \|
	export PYTHONPATH=$(pwd):$PYTHONPATH
	./.deploy.sh

	- name: Notify slack on job failure
	id: slack
	uses: slackapi/[email protected]
	with:
	channel-id: 'C01A6HC2FU6'
	slack-message: 'Archive Cronjob Failed'
	env:
	SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
	if: ${{ failure() }}

Provide feedback