Merge pull request #51 from govlt/upload-pmtiles

Upload PMTiles to S3
govlt · Apr 15, 2024 · 9914acd · 9914acd
2 parents eac7f60 + c153c54
commit 9914acd
Show file tree

Hide file tree

Showing 4 changed files with 79 additions and 18 deletions.
diff --git a/.github/actions/rclone-sync/action.yml b/.github/actions/rclone-sync/action.yml
@@ -0,0 +1,36 @@
+name: Rclone sync
+description: Sync files so S3 using Rclone
+
+
+inputs:
+  s3-endpoint:
+    description: Required input set S3 endpoint
+    required: true
+  s3-access-key-id:
+    description: Required input set S3 access key id
+    required: true
+  s3-secret-access-key:
+    description: Required input set S3 secret access key
+    required: true
+  destination-path:
+    description: Required input to set destination path
+    required: true
+  source-path:
+    description: Optional input to set source path
+    required: false
+    default: '.'
+
+
+runs:
+  using: composite
+  steps:
+    - name: Sync to S3
+      uses: docker://docker.io/rclone/rclone:1.66
+      with:
+        args: "sync ${{ inputs.source-path }} s3:${{ inputs.destination-path }} --s3-acl=public-read --fast-list --checksum --update --use-server-modtime --metadata --inplace --delete-during"
+      env:
+        RCLONE_CONFIG_S3_TYPE: s3
+        RCLONE_CONFIG_S3_PROVIDER: Other
+        RCLONE_CONFIG_S3_ENDPOINT: ${{ inputs.s3-endpoint }}
+        RCLONE_CONFIG_S3_ACCESS_KEY_ID: ${{ inputs.s3-access-key-id }}
+        RCLONE_CONFIG_S3_SECRET_ACCESS_KEY: ${{ inputs.s3-secret-access-key }}
diff --git a/.github/workflows/basemap-vector-data-source.yml b/.github/workflows/basemap-vector-data-source.yml
@@ -23,18 +23,19 @@ jobs:
       - name: Unzip GRPK data source
         run: unzip GRPK_Open_SHP.zip -x "GRPK_Open_SHP/INFO_/*" && rm GRPK_Open_SHP.zip
 
+      - name: Create output directory
+        run: mkdir output
+
       - name: Transform to EPSG:4326
-        run: ogr2ogr -f "ESRI Shapefile" -lco ENCODING=UTF-8 -t_srs EPSG:4326 grpk-espg-4326.shp.zip GRPK_Open_SHP
+        run: ogr2ogr -f "ESRI Shapefile" -lco ENCODING=UTF-8 -t_srs EPSG:4326 output/grpk-espg-4326.shp.zip GRPK_Open_SHP
 
-      - name: Sync to S3
-        uses: docker://docker.io/rclone/rclone:1.66
+      - name: Upload to S3
+        uses: ./.github/actions/rclone-sync
         with:
-          args: "sync . s3:/tiles/sources/grpk --include grpk-espg-4326.shp.zip --s3-acl=public-read --fast-list --checksum --update --use-server-modtime --metadata --inplace --delete-during"
-        env:
-          RCLONE_CONFIG_S3_TYPE: s3
-          RCLONE_CONFIG_S3_PROVIDER: Other
-          RCLONE_CONFIG_S3_ENDPOINT: ${{ secrets.S3_BIIP_ENDPOINT }}
-          RCLONE_CONFIG_S3_ACCESS_KEY_ID: ${{ secrets.S3_BIIP_ACCESS_KEY_ID }}
-          RCLONE_CONFIG_S3_SECRET_ACCESS_KEY: ${{ secrets.S3_BIIP_SECRET_ACCESS_KEY }}
+          s3-endpoint: ${{ vars.S3_ENDPOINT }}
+          s3-access-key-id: ${{ secrets.S3_ACCESS_KEY_ID }}
+          s3-secret-access-key: ${{ secrets.S3_SECRET_ACCESS_KEY }}
+          source-path: output
+          destination-path: /tiles/vector/sources/grpk
 
 
diff --git a/.github/workflows/basemap-vector-publish.yml b/.github/workflows/basemap-vector-publish.yml
@@ -26,9 +26,12 @@ jobs:
       style-source-url: https://vector.startupgov.lt/vector
       style-glyphs-url: https://basemap.startupgov.lt/vector/font/{fontstack}/{range}.pbf
       style-sprites-base-url: https://basemap.startupgov.lt/vector/sprite
+      s3-endpoint: ${{ vars.S3_ENDPOINT }}
     secrets:
-      cloudflare-api-token: ${{  secrets.CLOUDFLARE_PAGES_API_TOKEN }}
-      cloudflare-account-id: ${{  secrets.CLOUDFLARE_PAGES_ACCOUNT_ID }}
+      cloudflare-api-token: ${{ secrets.CLOUDFLARE_PAGES_API_TOKEN }}
+      cloudflare-account-id: ${{ secrets.CLOUDFLARE_PAGES_ACCOUNT_ID }}
+      s3-access-key-id: ${{ secrets.S3_ACCESS_KEY_ID }}
+      s3-secret-access-key: ${{ secrets.S3_SECRET_ACCESS_KEY }}ß
 
   publish-vector-basemap-preview:
     name: Publish vector basemap preview
@@ -41,6 +44,10 @@ jobs:
       style-source-url: https://dev-vector.startupgov.lt/vector
       style-glyphs-url: https://basemap.startupgov.lt/vector/font/{fontstack}/{range}.pbf
       style-sprites-base-url: https://basemap.startupgov.lt/vector/sprite
+      s3-endpoint: ${{ vars.S3_DEVELOPMENT_ENDPOINT }}
     secrets:
-      cloudflare-api-token: ${{  secrets.CLOUDFLARE_PAGES_API_TOKEN }}
-      cloudflare-account-id: ${{  secrets.CLOUDFLARE_PAGES_ACCOUNT_ID }}
+      cloudflare-api-token: ${{ secrets.CLOUDFLARE_PAGES_API_TOKEN }}
+      cloudflare-account-id: ${{ secrets.CLOUDFLARE_PAGES_ACCOUNT_ID }}
+      s3-access-key-id: ${{ secrets.S3_DEVELOPMENT_ACCESS_KEY_ID }}
+      s3-secret-access-key: ${{ secrets.S3_DEVELOPMENT_SECRET_ACCESS_KEY }}
+
diff --git a/.github/workflows/reusable-workflow-publish-basemap-vector.yml b/.github/workflows/reusable-workflow-publish-basemap-vector.yml
@@ -15,6 +15,10 @@ on:
         description: Required input to set style sprites base url
         type: string
         required: true
+      s3-endpoint:
+        description: Required input to set S3 endpoint
+        type: string
+        required: true
       archive-version:
         description: Optional input to set PMTiles archive version
         type: string
@@ -31,14 +35,18 @@ on:
       cloudflare-account-id:
         description: Cloudflare Account ID
         required: true
-
+      s3-access-key-id:
+        description: S3 access key id
+        required: true
+      s3-secret-access-key:
+        description: S3 secret access key
+        required: true
 
 jobs:
   generate-basemap-vector-pmtiles:
-    name: Generate and upload PMTiles
+    name: PMTiles to S3
     runs-on: ubuntu-latest
     timeout-minutes: 45
-    if: false
     permissions:
       contents: read
     steps:
@@ -57,6 +65,15 @@ jobs:
         with:
           cache-data-sources: true
 
+      - name: Upload PMTiles to S3
+        uses: ./.github/actions/rclone-sync
+        with:
+          s3-endpoint: ${{ inputs.s3-endpoint }}
+          s3-access-key-id: ${{ secrets.s3-access-key-id }}
+          s3-secret-access-key: ${{ secrets.s3-secret-access-key }}
+          source-path: vector/data/output
+          destination-path: /tiles/vector/pmtiles
+
   generate-basemap-demo-site:
     name: Build and upload demo site
     runs-on: ubuntu-latest
@@ -125,7 +142,7 @@ jobs:
       - name: Prepare styles output
         run: |
           mkdir -p output/font output/sprite output/styles
-            
+          
           styles=("positron" "bright" "openmaptiles")
           for style in "${styles[@]}"; do
             mkdir -p output/styles/"$style"