Scrape Routescanner and MSC daily (v2) #396
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflows starts scraping Routescanner and MSC each day at 05:00 UTC, | |
| # using the v2 scrapers, and opens a PR to merge the gathered data. | |
| name: Scrape Routescanner and MSC daily (v2) | |
| on: | |
| schedule: | |
| - cron: '0 5 * * *' | |
| push: | |
| paths: | |
| - ".github/workflows/daily_run_v2.yml" | |
| - "webscrapers/routescanner_automated_v2.py" | |
| - "webscrapers/msc_automated_v2.py" | |
| workflow_dispatch: | |
| jobs: | |
| routescanner-v2: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| - run: pip install -U pandas requests fake_useragent | |
| - run: python routescanner_automated_v2.py | |
| working-directory: ./webscrapers | |
| - uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| branch: "data_staging" | |
| commit_options: '--signoff' | |
| commit_message: "Add Routescanner v2 data" | |
| msc-v2: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| - run: pip install -U pandas requests | |
| - run: python msc_automated_v2.py | |
| working-directory: ./webscrapers | |
| - uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| branch: "data_staging" | |
| commit_options: '--signoff' | |
| commit_message: "Add MSC v2 data" |