scrape-au-auto.yml

# This is a basic workflow that is automatically triggered to run at a specific UTC time each day.

name: scrape-au-auto

# Controls when the action will run
on:
  schedule:
  - cron: 15 01 * * *           # runs once a day at 19:15 UTC (05:15 AEST)

concurrency:
  group: scrape-and-schedule
  cancel-in-progress: false

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  # This workflow contains a single job called "scrape"
  scrape:
    # The type of runner that the job will run on
    runs-on: ubuntu-20.04

    # Steps represent a sequence of tasks that will be executed as part of the job
    steps:
    - name: checkout repo content
      uses: actions/checkout@v4       # checkout the repository content to GitHub runner.
    - name: setup python
      uses: actions/setup-python@v5
      with:
        python-version: 3.10.14
    - name: setup dependencies
      working-directory: ./mcd-scr-au
      run: |
        sudo apt install -y xvfb
        pip install -r requirements.txt
        playwright install firefox
    - name: execute py script       # run mcd-pwp-au.py to get the latest data
      working-directory: ./mcd-scr-au
      run: |
        xvfb-run -- python3 mcd-pwp-au.py
#        scrapy crawl mcd-scr-au

    - name: Commit and push if it changed
      run: |
        git config user.name "${GITHUB_ACTOR}"
        git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
        git add -A
        timestamp=$(date -u)
        git pull
        git commit -m "Scraped: [${timestamp}] mcd-pwp-au (auto)" 
        git push "https://${GITHUB_ACTOR}:${TOKEN}@github.com/${GITHUB_REPOSITORY}.git" HEAD || exit 0