CI/CD to deploy dags to s3

incubateur-ademe · Apr 3, 2024 · ec30cbc · ec30cbc
1 parent 05cafe2
commit ec30cbc
Show file tree

Hide file tree

Showing 2 changed files with 46 additions and 0 deletions.
diff --git a/.github/workflows/push_dags_to_preprod.yml b/.github/workflows/push_dags_to_preprod.yml
@@ -0,0 +1,22 @@
+name: Push DAGs to S3
+
+on:
+  - pull_request
+
+env:
+  S3_HOST: https://cellar-c2.services.clever-cloud.com
+  FOLDER_SOURCE: development/
+  S3_BUCKET_DESTINATION: s3://qfdmo-airflow-dags/preprod/
+  AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }}
+
+jobs:
+  backup-production:
+    name: Copy local folder to Preprod s3 bucket
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install AWS CLI dependencies
+        run: |
+          pip install awscli
+      - name: Backup
+        run: aws --endpoint-url ${{ env.S3_HOST }} s3 sync --quiet ${{ env.FOLDER_SOURCE }} ${{ env.S3_BUCKET_DESTINATION }}
diff --git a/.github/workflows/push_dags_to_prod.yml b/.github/workflows/push_dags_to_prod.yml
@@ -0,0 +1,24 @@
+name: Push DAGs to S3
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+
+env:
+  S3_HOST: https://cellar-c2.services.clever-cloud.com
+  FOLDER_SOURCE: development/
+  S3_BUCKET_DESTINATION: s3://qfdmo-airflow-dags/production/
+  AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }}
+
+jobs:
+  backup-production:
+    name: Copy local folder to Production s3 bucket
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install AWS CLI dependencies
+        run: |
+          pip install awscli
+      - name: Backup
+        run: aws --endpoint-url ${{ env.S3_HOST }} s3 sync --quiet ${{ env.FOLDER_SOURCE }} ${{ env.S3_BUCKET_DESTINATION }}