From ec30cbc20f25cb33a9f45902468aebee703ddd2f Mon Sep 17 00:00:00 2001
From: Nicolas Oudard <nicolas@oudard.org>
Date: Wed, 3 Apr 2024 14:42:12 +0200
Subject: [PATCH] CI/CD to deploy dags to s3

---
 .github/workflows/push_dags_to_preprod.yml | 22 ++++++++++++++++++++
 .github/workflows/push_dags_to_prod.yml    | 24 ++++++++++++++++++++++
 2 files changed, 46 insertions(+)
 create mode 100644 .github/workflows/push_dags_to_preprod.yml
 create mode 100644 .github/workflows/push_dags_to_prod.yml

diff --git a/.github/workflows/push_dags_to_preprod.yml b/.github/workflows/push_dags_to_preprod.yml
new file mode 100644
index 0000000..3549258
--- /dev/null
+++ b/.github/workflows/push_dags_to_preprod.yml
@@ -0,0 +1,22 @@
+name: Push DAGs to S3
+
+on:
+  - pull_request
+      
+env:
+  S3_HOST: https://cellar-c2.services.clever-cloud.com
+  FOLDER_SOURCE: development/
+  S3_BUCKET_DESTINATION: s3://qfdmo-airflow-dags/preprod/
+  AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }}
+
+jobs:
+  backup-production:
+    name: Copy local folder to Preprod s3 bucket
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install AWS CLI dependencies
+        run: |
+          pip install awscli
+      - name: Backup
+        run: aws --endpoint-url ${{ env.S3_HOST }} s3 sync --quiet ${{ env.FOLDER_SOURCE }} ${{ env.S3_BUCKET_DESTINATION }}
diff --git a/.github/workflows/push_dags_to_prod.yml b/.github/workflows/push_dags_to_prod.yml
new file mode 100644
index 0000000..9343c58
--- /dev/null
+++ b/.github/workflows/push_dags_to_prod.yml
@@ -0,0 +1,24 @@
+name: Push DAGs to S3
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+
+env:
+  S3_HOST: https://cellar-c2.services.clever-cloud.com
+  FOLDER_SOURCE: development/
+  S3_BUCKET_DESTINATION: s3://qfdmo-airflow-dags/production/
+  AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }}
+
+jobs:
+  backup-production:
+    name: Copy local folder to Production s3 bucket
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install AWS CLI dependencies
+        run: |
+          pip install awscli
+      - name: Backup
+        run: aws --endpoint-url ${{ env.S3_HOST }} s3 sync --quiet ${{ env.FOLDER_SOURCE }} ${{ env.S3_BUCKET_DESTINATION }}