Scheduled Scraper Job #462
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scheduled Scraper Job | |
| on: | |
| schedule: | |
| # - cron: '*/1 * * * *' # Runs every ~minute for testing purposes | |
| - cron: '0 */5 * * *' # Runs every 5 hours | |
| workflow_dispatch: # Allows manual triggering | |
| jobs: | |
| scrape: | |
| runs-on: ubuntu-latest # Changed to ubuntu for better database compatibility | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 # Updated to v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 # Updated to v4 | |
| with: | |
| python-version: '3.8' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r scraper/requirements.txt | |
| - name: Run parser and save to database | |
| env: | |
| DB_HOST: ${{ secrets.DB_HOST }} | |
| DB_PORT: ${{ secrets.DB_PORT }} | |
| DB_USER: ${{ secrets.DB_USER }} | |
| DB_PASSWORD: ${{ secrets.DB_PASSWORD }} | |
| DB_NAME: ${{ secrets.DB_NAME }} | |
| EMAIL: ${{ secrets.EMAIL }} | |
| PASSWORD: ${{ secrets.PASSWORD }} | |
| run: python scraper/scraper.py |