From 232759714265c01fa921a19dcc7d448c184f379c Mon Sep 17 00:00:00 2001 From: Nitai Bezerra da Silva Date: Mon, 13 Mar 2023 15:18:02 -0300 Subject: [PATCH 1/9] build(dockerfile): creates its own container Decouple from the airflow container shared with other projects. Fixes #48. --- Dockerfile | 5 +++++ docker-compose.yml | 3 ++- requirements.txt | 9 +++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 Dockerfile create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e78f9e3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM apache/airflow:2.5.1 + +RUN pip install --no-cache-dir --user 'apache-airflow[microsoft.mssql,google_auth]' +COPY requirements.txt / +RUN pip install --no-cache-dir -r /requirements.txt diff --git a/docker-compose.yml b/docker-compose.yml index 557e5e0..c611299 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,7 +39,8 @@ version: '3' x-airflow-common: &airflow-common - image: ghcr.io/economiagovbr/airflow2-docker:latest + build: + context: . environment: &airflow-common-env AIRFLOW__CORE__EXECUTOR: LocalExecutor diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ed35915 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +s3fs==2023.1.0 +ipdb==0.13.11 +pytest==7.2.1 +mock==5.0.1 +pytest-mock==3.10.0 +unidecode==1.2.0 +xlrd==1.2.0 +ijson==3.0.4 +openpyxl==3.0.7 \ No newline at end of file From 28175a441d75100a5b73c7b5c3f8754357d7be36 Mon Sep 17 00:00:00 2001 From: Nitai Bezerra da Silva Date: Fri, 17 Mar 2023 06:04:23 -0300 Subject: [PATCH 2/9] build(dockerfile): bundle src code into image --- Dockerfile | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Dockerfile b/Dockerfile index e78f9e3..293d07c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,33 @@ FROM apache/airflow:2.5.1 +USER root + +# Copy Ro-dou core files from the host Docker context +COPY src /opt/airflow/dags/ro_dou + +# Install Git +RUN apt-get update && \ + apt-get install -y git && \ + rm -rf /var/lib/apt/lists/* + +# Clone the FastETL GitHub repository and copy specified files +RUN git clone https://github.com/economiagovbr/FastETL.git /tmp/repo-FastETL && \ + mkdir -p /opt/airflow/plugins/FastETL/hooks && \ + mkdir -p /opt/airflow/plugins/FastETL/custom_functions/utils && \ + cp /tmp/repo-FastETL/hooks/dou_hook.py /opt/airflow/plugins/FastETL/hooks/dou_hook.py && \ + cp -r /tmp/repo-FastETL/custom_functions/utils/* /opt/airflow/plugins/FastETL/custom_functions/utils/ + +# Remove Git and clean up package cache +RUN apt-get remove -y git && \ + apt-get autoremove -y && \ + rm -rf /tmp/repo-ro-dou && \ + rm -rf /tmp/repo-FastETL + +USER airflow + +# Install additional Airflow dependencies RUN pip install --no-cache-dir --user 'apache-airflow[microsoft.mssql,google_auth]' + +# Copy and install requirements.txt COPY requirements.txt / RUN pip install --no-cache-dir -r /requirements.txt From ecae26669de3c320f66f4d2240898e49395aef1b Mon Sep 17 00:00:00 2001 From: Nitai Bezerra da Silva Date: Fri, 17 Mar 2023 06:08:29 -0300 Subject: [PATCH 3/9] build(docker-compose): remove FastETL mapping From now on it's bundled into the docker image. --- docker-compose.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c611299..83691b2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,9 +74,6 @@ - ./src:/opt/airflow/dags/ro_dou - ./dag_confs:/opt/airflow/dags/ro_dou/dag_confs - ./tests:/opt/airflow/tests - - - ./FastETL/hooks/dou_hook.py:/opt/airflow/plugins/FastETL/hooks/dou_hook.py - - ./FastETL/custom_functions/utils/:/opt/airflow/plugins/FastETL/custom_functions/utils/ depends_on: postgres: condition: service_healthy From 01468d700895a0fe1b512320a9188ad26becb060 Mon Sep 17 00:00:00 2001 From: Nitai Bezerra da Silva Date: Fri, 17 Mar 2023 06:31:20 -0300 Subject: [PATCH 4/9] chore(docker-compose): add comment Th src code and the test folders don't need to be mapped into container in production. Src code is bundled into the docker image. --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 83691b2..6a15d57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,9 +71,9 @@ AIRFLOW_CONN_EXAMPLE_DATABASE_CONN: postgresql://airflow:airflow@postgres:5432/airflow volumes: - - ./src:/opt/airflow/dags/ro_dou + - ./src:/opt/airflow/dags/ro_dou # for development purpose - ./dag_confs:/opt/airflow/dags/ro_dou/dag_confs - - ./tests:/opt/airflow/tests + - ./tests:/opt/airflow/tests # for test purpose depends_on: postgres: condition: service_healthy From e78d69a9717a34c74c93d2e821c7538a549f62d2 Mon Sep 17 00:00:00 2001 From: Nitai Bezerra da Silva Date: Mon, 20 Mar 2023 11:40:59 -0300 Subject: [PATCH 5/9] docs(readme): remove old instruction --- README.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/README.md b/README.md index 829cdab..7f14a05 100644 --- a/README.md +++ b/README.md @@ -26,15 +26,10 @@ isso é necessário ter o **Docker Compose** na versão 1.29 ou maior clonar o repositório no seu computador, acesse o diretório pela linha de comando e execute os comandos a seguir: -```bash -make install-deps -``` -Este comando baixa as dependências necessárias. - ```bash make run ``` -Este comando baixa as imagens docker e sobe todos os contêineres do Ro-dou. +Este comando baixa as imagens docker necessárias, faz build da contêiner do Ro-dou e sobe tudo. O Airflow pode demorar alguns minutos para se configurar a primeira vez. Após isso ele estará disponível em http://localhost:8080/. Para se autenticar From 4fb882b0b01a2dc4ac7fd3a11768d181299ff80c Mon Sep 17 00:00:00 2001 From: Nitai Bezerra da Silva Date: Thu, 16 Mar 2023 19:37:30 -0300 Subject: [PATCH 6/9] cicd(build_publish): creates docker image --- .../workflows/docker_build_and_publish.yml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/docker_build_and_publish.yml diff --git a/.github/workflows/docker_build_and_publish.yml b/.github/workflows/docker_build_and_publish.yml new file mode 100644 index 0000000..d6b3340 --- /dev/null +++ b/.github/workflows/docker_build_and_publish.yml @@ -0,0 +1,31 @@ +name: Build and Publish Docker Image + +on: + push: + branches: + - main + - build_publish_image + +jobs: + build_and_push: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: ghcr.io/${{ github.repository_owner }}/${{ github.repository }}:latest From c71b3539dbf96fcfad726da898452837a7437e83 Mon Sep 17 00:00:00 2001 From: Nitai Bezerra da Silva Date: Thu, 16 Mar 2023 19:40:22 -0300 Subject: [PATCH 7/9] ci(build_publish): fix image name Repository name must be lowercase. --- .github/workflows/docker_build_and_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker_build_and_publish.yml b/.github/workflows/docker_build_and_publish.yml index d6b3340..3e7aea5 100644 --- a/.github/workflows/docker_build_and_publish.yml +++ b/.github/workflows/docker_build_and_publish.yml @@ -28,4 +28,4 @@ jobs: with: context: . push: true - tags: ghcr.io/${{ github.repository_owner }}/${{ github.repository }}:latest + tags: ghcr.io/${{ github.repository_owner }}/ro-dou:latest From 3c00839688edea65f08dcaeffe6aafbc354433c0 Mon Sep 17 00:00:00 2001 From: Nitai Bezerra da Silva Date: Thu, 16 Mar 2023 20:00:43 -0300 Subject: [PATCH 8/9] ci(build_publish): publish also on tag Publishes another image under a tagged url. --- .../workflows/docker_build_and_publish.yml | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker_build_and_publish.yml b/.github/workflows/docker_build_and_publish.yml index 3e7aea5..9e08b68 100644 --- a/.github/workflows/docker_build_and_publish.yml +++ b/.github/workflows/docker_build_and_publish.yml @@ -2,9 +2,8 @@ name: Build and Publish Docker Image on: push: - branches: - - main - - build_publish_image + tags: + - '*' jobs: build_and_push: @@ -23,9 +22,20 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push Docker image + - name: Extract tag name + shell: bash + run: echo "TAG_NAME=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_ENV + + - name: Build and push Docker image with tag + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: ghcr.io/${{ github.repository }}:${{ env.TAG_NAME }} + + - name: Build and push Docker image with latest tag uses: docker/build-push-action@v2 with: context: . push: true - tags: ghcr.io/${{ github.repository_owner }}/ro-dou:latest + tags: ghcr.io/${{ github.repository }}:latest From 1f334b034b103374394145fa19c788fd60ff6246 Mon Sep 17 00:00:00 2001 From: Nitai Bezerra da Silva Date: Thu, 16 Mar 2023 20:05:30 -0300 Subject: [PATCH 9/9] ci(build_publish): fix image name Repository name must be lowercase. --- .github/workflows/docker_build_and_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker_build_and_publish.yml b/.github/workflows/docker_build_and_publish.yml index 9e08b68..4d201c6 100644 --- a/.github/workflows/docker_build_and_publish.yml +++ b/.github/workflows/docker_build_and_publish.yml @@ -31,11 +31,11 @@ jobs: with: context: . push: true - tags: ghcr.io/${{ github.repository }}:${{ env.TAG_NAME }} + tags: ghcr.io/${{ github.repository_owner }}/ro-dou:${{ env.TAG_NAME }} - name: Build and push Docker image with latest tag uses: docker/build-push-action@v2 with: context: . push: true - tags: ghcr.io/${{ github.repository }}:latest + tags: ghcr.io/${{ github.repository_owner }}/ro-dou:latest