Basemap vector: sync data sources (all) #30
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Basemap vector: sync data sources" | |
| run-name: "Basemap vector: sync data sources (${{ inputs.source }})" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| source: | |
| description: 'Data source' | |
| required: true | |
| default: all | |
| type: choice | |
| options: | |
| - all | |
| - grpk | |
| - stvk | |
| - address-registry | |
| permissions: { } | |
| jobs: | |
| basemap-vector-datasource-address-registry: | |
| name: Address registry sync and transform | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| container: ghcr.io/osgeo/gdal:ubuntu-full-3.9.0 | |
| permissions: | |
| contents: read | |
| concurrency: basemap-vector-datasource-address-registry | |
| if: ${{ inputs.source == 'all' || inputs.source == 'address-registry' }} | |
| environment: | |
| name: basemap-vector-datasource-address-registry | |
| url: https://cdn.startupgov.lt/tiles/vector/sources/address-registry/houses-espg-4326.gpkg.zip | |
| steps: | |
| - name: Install required packages | |
| run: apt-get update && apt-get install -y git csvkit | |
| # Needed only for GitHub action for uploading to S3 | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| sparse-checkout: | | |
| .github | |
| scripts | |
| - name: Generate houses GPKG in EPSG:4326 | |
| run: sh scripts/create-houses-geopackage.sh | |
| - name: Create output directory | |
| run: mkdir output | |
| - name: Copy files to output | |
| run: | | |
| cp houses-espg-4326.gpkg.zip output/houses-espg-4326.gpkg.zip | |
| cp data-sources/data-source-checksums.txt output/data-source-checksums.txt | |
| - name: Generate checksum | |
| working-directory: output | |
| run: md5sum houses-espg-4326.gpkg.zip >> checksums.txt | |
| - name: Upload to S3 | |
| uses: ./.github/actions/rclone-sync | |
| with: | |
| s3-endpoint: ${{ vars.S3_ENDPOINT }} | |
| s3-access-key-id: ${{ secrets.S3_ACCESS_KEY_ID }} | |
| s3-secret-access-key: ${{ secrets.S3_SECRET_ACCESS_KEY }} | |
| source-path: output | |
| destination-path: /tiles/vector/sources/address-registry | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: address-registry-house-numbers | |
| path: output | |
| if-no-files-found: error | |
| compression-level: 0 | |
| basemap-vector-datasource-grpk: | |
| name: GRPK sync and transform | |
| runs-on: ubuntu-latest | |
| # GeoPortal download is really slow. Downloading less than 3 GB takes more than 3 hours :( | |
| timeout-minutes: 360 | |
| container: ghcr.io/osgeo/gdal:ubuntu-full-3.9.0 | |
| permissions: { } | |
| concurrency: basemap-vector-datasource-grpk | |
| if: ${{ inputs.source == 'all' || inputs.source == 'grpk' }} | |
| environment: | |
| name: basemap-vector-datasource-grpk | |
| url: https://cdn.startupgov.lt/tiles/vector/sources/grpk/grpk-espg-4326.shp.zip | |
| steps: | |
| # Needed only for GitHub action for uploading to S3 | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| sparse-checkout: .github | |
| - name: Create output directory | |
| run: mkdir output | |
| - name: Download GRPK from GeoPortal | |
| run: wget -nv https://www.geoportal.lt/download/opendata/GRPK/GRPK_Open_SHP.zip | |
| - name: Calculate GRPK_Open_SHP.zip checksum | |
| run: md5sum GRPK_Open_SHP.zip >> output/data-source-checksums.txt | |
| - name: Unzip GRPK data source | |
| run: unzip GRPK_Open_SHP.zip -x "GRPK_Open_SHP/INFO_/*" && rm GRPK_Open_SHP.zip | |
| - name: Transform to EPSG:4326 | |
| run: ogr2ogr -f "ESRI Shapefile" -lco ENCODING=UTF-8 -t_srs EPSG:4326 output/grpk-espg-4326.shp.zip GRPK_Open_SHP | |
| - name: Generate checksum | |
| working-directory: output | |
| run: md5sum grpk-espg-4326.shp.zip >> checksums.txt | |
| - name: Upload to S3 | |
| uses: ./.github/actions/rclone-sync | |
| with: | |
| s3-endpoint: ${{ vars.S3_ENDPOINT }} | |
| s3-access-key-id: ${{ secrets.S3_ACCESS_KEY_ID }} | |
| s3-secret-access-key: ${{ secrets.S3_SECRET_ACCESS_KEY }} | |
| source-path: output | |
| destination-path: /tiles/vector/sources/grpk | |
| basemap-vector-datasource-stvk: | |
| name: STVK sync and transform | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| container: ghcr.io/osgeo/gdal:ubuntu-full-3.9.0 | |
| permissions: { } | |
| concurrency: basemap-vector-datasource-stvk | |
| if: ${{ inputs.source == 'all' || inputs.source == 'stvk' }} | |
| environment: | |
| name: basemap-vector-datasource-stvk | |
| url: https://cdn.startupgov.lt/tiles/vector/sources/stvk/stvk-4326.gpkg.zip | |
| steps: | |
| # Needed only for GitHub action for uploading to S3 | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| sparse-checkout: .github | |
| - name: Create output directory | |
| run: mkdir output | |
| - name: Generate STVK GPKG in EPSG:4326 | |
| working-directory: output | |
| run: | | |
| ogr2ogr -t_srs EPSG:4326 -dsco DATETIME_FORMAT=UTC -nlt CONVERT_TO_LINEAR -dsco VERSION=1.4 stvk-4326.gpkg.zip \ | |
| WFS:"https://services.stvk.lt/wfs/stvk-services?SERVICE=WFS&REQUEST=GetCapabilities&VERSION=2.0.0" \ | |
| nac_parkai reg_parkai valstybiniai_rezervatai valstybiniai_draustiniai | |
| - name: Generate checksum | |
| working-directory: output | |
| run: md5sum stvk-4326.gpkg.zip >> checksums.txt | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: stvk | |
| path: output | |
| if-no-files-found: error | |
| compression-level: 0 | |
| - name: Upload to S3 | |
| uses: ./.github/actions/rclone-sync | |
| with: | |
| s3-endpoint: ${{ vars.S3_ENDPOINT }} | |
| s3-access-key-id: ${{ secrets.S3_ACCESS_KEY_ID }} | |
| s3-secret-access-key: ${{ secrets.S3_SECRET_ACCESS_KEY }} | |
| source-path: output | |
| destination-path: /tiles/vector/sources/stvk | |