diff --git a/.github/workflows/push_dags_to_preprod.yml b/.github/workflows/push_dags_to_preprod.yml new file mode 100644 index 0000000..3549258 --- /dev/null +++ b/.github/workflows/push_dags_to_preprod.yml @@ -0,0 +1,22 @@ +name: Push DAGs to S3 + +on: + - pull_request + +env: + S3_HOST: https://cellar-c2.services.clever-cloud.com + FOLDER_SOURCE: development/ + S3_BUCKET_DESTINATION: s3://qfdmo-airflow-dags/preprod/ + AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }} + +jobs: + backup-production: + name: Copy local folder to Preprod s3 bucket + runs-on: ubuntu-latest + steps: + - name: Install AWS CLI dependencies + run: | + pip install awscli + - name: Backup + run: aws --endpoint-url ${{ env.S3_HOST }} s3 sync --quiet ${{ env.FOLDER_SOURCE }} ${{ env.S3_BUCKET_DESTINATION }} diff --git a/.github/workflows/push_dags_to_prod.yml b/.github/workflows/push_dags_to_prod.yml new file mode 100644 index 0000000..9343c58 --- /dev/null +++ b/.github/workflows/push_dags_to_prod.yml @@ -0,0 +1,24 @@ +name: Push DAGs to S3 + +on: + push: + tags: + - "v*.*.*" + +env: + S3_HOST: https://cellar-c2.services.clever-cloud.com + FOLDER_SOURCE: development/ + S3_BUCKET_DESTINATION: s3://qfdmo-airflow-dags/production/ + AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_KEY }} + +jobs: + backup-production: + name: Copy local folder to Production s3 bucket + runs-on: ubuntu-latest + steps: + - name: Install AWS CLI dependencies + run: | + pip install awscli + - name: Backup + run: aws --endpoint-url ${{ env.S3_HOST }} s3 sync --quiet ${{ env.FOLDER_SOURCE }} ${{ env.S3_BUCKET_DESTINATION }}