From fa9f25161e9bb3c30eb9ab2fb65d136a7447bc56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Karolis=20Vy=C4=8Dius?= <k.vycius@gmail.com>
Date: Mon, 15 Apr 2024 13:16:54 +0300
Subject: [PATCH] Upload PMTiles to S3

---
 .github/actions/rclone-sync/action.yml        | 36 +++++++++++++++++++
 .github/workflows/basemap-vector-publish.yml  | 15 +++++---
 ...usable-workflow-publish-basemap-vector.yml | 25 ++++++++++---
 3 files changed, 68 insertions(+), 8 deletions(-)
 create mode 100644 .github/actions/rclone-sync/action.yml

diff --git a/.github/actions/rclone-sync/action.yml b/.github/actions/rclone-sync/action.yml
new file mode 100644
index 00000000..df25c8d9
--- /dev/null
+++ b/.github/actions/rclone-sync/action.yml
@@ -0,0 +1,36 @@
+name: Rclone sync
+description: Sync files so S3 using Rclone
+
+
+inputs:
+  s3-endpoint:
+    description: Required input set S3 endpoint
+    required: true
+  s3-access-key-id:
+    description: Required input set S3 access key id
+    required: true
+  s3-secret-access-key:
+    description: Required input set S3 secret access key
+    required: true
+  destination-path:
+    description: Required input to set destination path
+    required: true
+  source-path:
+    description: Optional input to set source path
+    required: false
+    default: '.'
+
+
+runs:
+  using: composite
+  steps:
+    - name: Sync to S3
+      uses: docker://docker.io/rclone/rclone:1.66
+      with:
+        args: "sync ${{ inputs.source-path }} s3:${{ inputs.destination-path }}--s3-acl=public-read --fast-list --checksum --update --use-server-modtime --metadata --inplace --delete-during"
+      env:
+        RCLONE_CONFIG_S3_TYPE: s3
+        RCLONE_CONFIG_S3_PROVIDER: Other
+        RCLONE_CONFIG_S3_ENDPOINT: ${{ inputs.s3-endpoint }}
+        RCLONE_CONFIG_S3_ACCESS_KEY_ID: ${{ inputs.s3-access-key-id }}
+        RCLONE_CONFIG_S3_SECRET_ACCESS_KEY: ${{ inputs.s3-secret-access-key }}
diff --git a/.github/workflows/basemap-vector-publish.yml b/.github/workflows/basemap-vector-publish.yml
index 8efaa8d2..40697540 100644
--- a/.github/workflows/basemap-vector-publish.yml
+++ b/.github/workflows/basemap-vector-publish.yml
@@ -26,9 +26,12 @@ jobs:
       style-source-url: https://vector.startupgov.lt/vector
       style-glyphs-url: https://basemap.startupgov.lt/vector/font/{fontstack}/{range}.pbf
       style-sprites-base-url: https://basemap.startupgov.lt/vector/sprite
+      s3-endpoint: ${{ vars.S3_ENDPOINT }}
     secrets:
-      cloudflare-api-token: ${{  secrets.CLOUDFLARE_PAGES_API_TOKEN }}
-      cloudflare-account-id: ${{  secrets.CLOUDFLARE_PAGES_ACCOUNT_ID }}
+      cloudflare-api-token: ${{ secrets.CLOUDFLARE_PAGES_API_TOKEN }}
+      cloudflare-account-id: ${{ secrets.CLOUDFLARE_PAGES_ACCOUNT_ID }}
+      s3-access-key-id: ${{ secrets.S3_ACCESS_KEY_ID }}
+      s3-secret-access-key: ${{ secrets.S3_SECRET_ACCESS_KEY }}ß
 
   publish-vector-basemap-preview:
     name: Publish vector basemap preview
@@ -41,6 +44,10 @@ jobs:
       style-source-url: https://dev-vector.startupgov.lt/vector
       style-glyphs-url: https://basemap.startupgov.lt/vector/font/{fontstack}/{range}.pbf
       style-sprites-base-url: https://basemap.startupgov.lt/vector/sprite
+      s3-endpoint: ${{ vars.S3_DEVELOPMENT_ENDPOINT }}
     secrets:
-      cloudflare-api-token: ${{  secrets.CLOUDFLARE_PAGES_API_TOKEN }}
-      cloudflare-account-id: ${{  secrets.CLOUDFLARE_PAGES_ACCOUNT_ID }}
+      cloudflare-api-token: ${{ secrets.CLOUDFLARE_PAGES_API_TOKEN }}
+      cloudflare-account-id: ${{ secrets.CLOUDFLARE_PAGES_ACCOUNT_ID }}
+      s3-access-key-id: ${{ secrets.S3_DEVELOPMENT_ACCESS_KEY_ID }}
+      s3-secret-access-key: ${{ secrets.S3_DEVELOPMENT_SECRET_ACCESS_KEY }}
+
diff --git a/.github/workflows/reusable-workflow-publish-basemap-vector.yml b/.github/workflows/reusable-workflow-publish-basemap-vector.yml
index b5cd6197..963be753 100644
--- a/.github/workflows/reusable-workflow-publish-basemap-vector.yml
+++ b/.github/workflows/reusable-workflow-publish-basemap-vector.yml
@@ -15,6 +15,10 @@ on:
         description: Required input to set style sprites base url
         type: string
         required: true
+      s3-endpoint:
+        description: Required input to set S3 endpoint
+        type: string
+        required: true
       archive-version:
         description: Optional input to set PMTiles archive version
         type: string
@@ -31,14 +35,18 @@ on:
       cloudflare-account-id:
         description: Cloudflare Account ID
         required: true
-
+      s3-access-key-id:
+        description: S3 access key id
+        required: true
+      s3-secret-access-key:
+        description: S3 secret access key
+        required: true
 
 jobs:
   generate-basemap-vector-pmtiles:
-    name: Generate and upload PMTiles
+    name: PMTiles to S3
     runs-on: ubuntu-latest
     timeout-minutes: 45
-    if: false
     permissions:
       contents: read
     steps:
@@ -57,6 +65,15 @@ jobs:
         with:
           cache-data-sources: true
 
+      - name: Upload PMTiles to S3
+        uses: ./.github/actions/rclone-sync
+        with:
+          s3-endpoint: ${{ inputs.s3-endpoint }}
+          s3-access-key-id: ${{ secrets.s3-access-key-id }}
+          s3-secret-access-key: ${{ secrets.s3-secret-access-key }}
+          source-path: vector/data/output
+          destination-path: /tiles/vector/lithuania.pmtiles
+
   generate-basemap-demo-site:
     name: Build and upload demo site
     runs-on: ubuntu-latest
@@ -125,7 +142,7 @@ jobs:
       - name: Prepare styles output
         run: |
           mkdir -p output/font output/sprite output/styles
-            
+          
           styles=("positron" "bright" "openmaptiles")
           for style in "${styles[@]}"; do
             mkdir -p output/styles/"$style"