Upload PMTiles to S3

govlt · Apr 15, 2024 · fa9f251 · fa9f251
1 parent 96a3cac
commit fa9f251
Show file tree

Hide file tree

Showing 3 changed files with 68 additions and 8 deletions.
diff --git a/.github/actions/rclone-sync/action.yml b/.github/actions/rclone-sync/action.yml
@@ -0,0 +1,36 @@
+name: Rclone sync
+description: Sync files so S3 using Rclone
+
+
+inputs:
+  s3-endpoint:
+    description: Required input set S3 endpoint
+    required: true
+  s3-access-key-id:
+    description: Required input set S3 access key id
+    required: true
+  s3-secret-access-key:
+    description: Required input set S3 secret access key
+    required: true
+  destination-path:
+    description: Required input to set destination path
+    required: true
+  source-path:
+    description: Optional input to set source path
+    required: false
+    default: '.'
+
+
+runs:
+  using: composite
+  steps:
+    - name: Sync to S3
+      uses: docker://docker.io/rclone/rclone:1.66
+      with:
+        args: "sync ${{ inputs.source-path }} s3:${{ inputs.destination-path }}--s3-acl=public-read --fast-list --checksum --update --use-server-modtime --metadata --inplace --delete-during"
+      env:
+        RCLONE_CONFIG_S3_TYPE: s3
+        RCLONE_CONFIG_S3_PROVIDER: Other
+        RCLONE_CONFIG_S3_ENDPOINT: ${{ inputs.s3-endpoint }}
+        RCLONE_CONFIG_S3_ACCESS_KEY_ID: ${{ inputs.s3-access-key-id }}
+        RCLONE_CONFIG_S3_SECRET_ACCESS_KEY: ${{ inputs.s3-secret-access-key }}
diff --git a/.github/workflows/basemap-vector-publish.yml b/.github/workflows/basemap-vector-publish.yml
@@ -26,9 +26,12 @@ jobs:
       style-source-url: https://vector.startupgov.lt/vector
       style-glyphs-url: https://basemap.startupgov.lt/vector/font/{fontstack}/{range}.pbf
       style-sprites-base-url: https://basemap.startupgov.lt/vector/sprite
+      s3-endpoint: ${{ vars.S3_ENDPOINT }}
     secrets:
-      cloudflare-api-token: ${{  secrets.CLOUDFLARE_PAGES_API_TOKEN }}
-      cloudflare-account-id: ${{  secrets.CLOUDFLARE_PAGES_ACCOUNT_ID }}
+      cloudflare-api-token: ${{ secrets.CLOUDFLARE_PAGES_API_TOKEN }}
+      cloudflare-account-id: ${{ secrets.CLOUDFLARE_PAGES_ACCOUNT_ID }}
+      s3-access-key-id: ${{ secrets.S3_ACCESS_KEY_ID }}
+      s3-secret-access-key: ${{ secrets.S3_SECRET_ACCESS_KEY }}ß
 
   publish-vector-basemap-preview:
     name: Publish vector basemap preview
@@ -41,6 +44,10 @@ jobs:
       style-source-url: https://dev-vector.startupgov.lt/vector
       style-glyphs-url: https://basemap.startupgov.lt/vector/font/{fontstack}/{range}.pbf
       style-sprites-base-url: https://basemap.startupgov.lt/vector/sprite
+      s3-endpoint: ${{ vars.S3_DEVELOPMENT_ENDPOINT }}
     secrets:
-      cloudflare-api-token: ${{  secrets.CLOUDFLARE_PAGES_API_TOKEN }}
-      cloudflare-account-id: ${{  secrets.CLOUDFLARE_PAGES_ACCOUNT_ID }}
+      cloudflare-api-token: ${{ secrets.CLOUDFLARE_PAGES_API_TOKEN }}
+      cloudflare-account-id: ${{ secrets.CLOUDFLARE_PAGES_ACCOUNT_ID }}
+      s3-access-key-id: ${{ secrets.S3_DEVELOPMENT_ACCESS_KEY_ID }}
+      s3-secret-access-key: ${{ secrets.S3_DEVELOPMENT_SECRET_ACCESS_KEY }}
+
diff --git a/.github/workflows/reusable-workflow-publish-basemap-vector.yml b/.github/workflows/reusable-workflow-publish-basemap-vector.yml
@@ -15,6 +15,10 @@ on:
         description: Required input to set style sprites base url
         type: string
         required: true
+      s3-endpoint:
+        description: Required input to set S3 endpoint
+        type: string
+        required: true
       archive-version:
         description: Optional input to set PMTiles archive version
         type: string
@@ -31,14 +35,18 @@ on:
       cloudflare-account-id:
         description: Cloudflare Account ID
         required: true
-
+      s3-access-key-id:
+        description: S3 access key id
+        required: true
+      s3-secret-access-key:
+        description: S3 secret access key
+        required: true
 
 jobs:
   generate-basemap-vector-pmtiles:
-    name: Generate and upload PMTiles
+    name: PMTiles to S3
     runs-on: ubuntu-latest
     timeout-minutes: 45
-    if: false
     permissions:
       contents: read
     steps:
@@ -57,6 +65,15 @@ jobs:
         with:
           cache-data-sources: true
 
+      - name: Upload PMTiles to S3
+        uses: ./.github/actions/rclone-sync
+        with:
+          s3-endpoint: ${{ inputs.s3-endpoint }}
+          s3-access-key-id: ${{ secrets.s3-access-key-id }}
+          s3-secret-access-key: ${{ secrets.s3-secret-access-key }}
+          source-path: vector/data/output
+          destination-path: /tiles/vector/lithuania.pmtiles
+
   generate-basemap-demo-site:
     name: Build and upload demo site
     runs-on: ubuntu-latest
@@ -125,7 +142,7 @@ jobs:
       - name: Prepare styles output
         run: |
           mkdir -p output/font output/sprite output/styles
-            
+          
           styles=("positron" "bright" "openmaptiles")
           for style in "${styles[@]}"; do
             mkdir -p output/styles/"$style"