main.yml

name: Daily Scrape

# Run the workflow daily
on:
  schedule:
    - cron: "0 0 * * *" 

permissions:
  contents: write 

jobs:
  scrape_data:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout repository
      uses: actions/checkout@v2

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.11' 

    - name: Install dependencies
      run: |
        python -m venv .venv
        source .venv/bin/activate
        pip install -r src/scripts/requirements.txt  # Assumes you have a requirements.txt file

    - name: Run scraping script
      env:
        GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} 
      run: |
        source .venv/bin/activate
        python src/scripts/snapshot_crawler.py

    - name: Pull latest changes
      run: |
        git config --local user.name "github-actions[bot]"
        git config --local user.email "github-actions[bot]@users.noreply.github.com"
        git pull origin main

    - name: Commit and push changes
      run: |
        git add .
        git commit -m "Add generated YAML files"
        git push origin main
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}