Hourly Pipeline #4241
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Hourly Pipeline | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "10 * * * *" | |
| jobs: | |
| generate-hourly-fgb: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Build docker image | |
| run: docker build -t gradient-pipeline ./pipeline | |
| - name: set date | |
| id: set_date | |
| run: | | |
| DATE_TWO_DAYS_AGO=$(date -d "2 days ago" +%Y-%m-%d) | |
| echo "DATE_TWO_DAYS_AGO=$DATE_TWO_DAYS_AGO" >> $GITHUB_OUTPUT | |
| - name: set hour | |
| id: set_hour | |
| run: | | |
| HOUR_TWO_DAYS_AGO=$(date -d "2 days ago" +%H) | |
| echo "HOUR_TWO_DAYS_AGO=$HOUR_TWO_DAYS_AGO" >> $GITHUB_OUTPUT | |
| - name: Check if the date and hour are set | |
| run: cat $GITHUB_OUTPUT | |
| - name: process changesets for the hour ${{ steps.set_hour.outputs.HOUR_TWO_DAYS_AGO }}:00 on ${{ steps.set_date.outputs.DATE_TWO_DAYS_AGO }} | |
| run: | | |
| docker run -v ./data:/tmp gradient-pipeline sh -c "node cli.js process-hour $DATE_TWO_DAYS_AGO $HOUR_TWO_DAYS_AGO" | |
| env: | |
| DATE_TWO_DAYS_AGO: ${{ steps.set_date.outputs.DATE_TWO_DAYS_AGO }} | |
| HOUR_TWO_DAYS_AGO: ${{ steps.set_hour.outputs.HOUR_TWO_DAYS_AGO }} | |
| - name: convert to fgb for the hour ${{ steps.set_hour.outputs.HOUR_TWO_DAYS_AGO }}:00 on ${{ steps.set_date.outputs.DATE_TWO_DAYS_AGO }} | |
| run: | | |
| docker run -v ./data:/tmp gradient-pipeline sh -c "ogr2ogr -f FlatGeobuf /tmp/${DATE_TWO_DAYS_AGO}T${HOUR_TWO_DAYS_AGO}:00.fgb /tmp/${DATE_TWO_DAYS_AGO}T${HOUR_TWO_DAYS_AGO}/${DATE_TWO_DAYS_AGO}T${HOUR_TWO_DAYS_AGO}:00.geojson -skipfailures" | |
| env: | |
| DATE_TWO_DAYS_AGO: ${{ steps.set_date.outputs.DATE_TWO_DAYS_AGO }} | |
| HOUR_TWO_DAYS_AGO: ${{ steps.set_hour.outputs.HOUR_TWO_DAYS_AGO }} | |
| - name: Check if the pipeline ran successfully | |
| run: ls -lh ./data/*.fgb | |
| - name: Authenticate with GCP | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GCS_SERVICE_ACCOUNT_KEY }} | |
| - name: Upload to GCS | |
| uses: google-github-actions/upload-cloud-storage@v2 | |
| with: | |
| path: data | |
| destination: osm-tardis | |
| parent: false | |
| glob: '*.fgb' | |
| gzip: false | |
| headers: |- | |
| content-type: application/octet-stream |