From 54a16cbf0bbde4e1127fb09cdf7dfe844f8338d5 Mon Sep 17 00:00:00 2001 From: Nitai Bezerra da Silva Date: Fri, 17 Mar 2023 06:04:23 -0300 Subject: [PATCH 1/4] build(dockerfile): bundle src code into image --- Dockerfile | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Dockerfile b/Dockerfile index e78f9e3..293d07c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,33 @@ FROM apache/airflow:2.5.1 +USER root + +# Copy Ro-dou core files from the host Docker context +COPY src /opt/airflow/dags/ro_dou + +# Install Git +RUN apt-get update && \ + apt-get install -y git && \ + rm -rf /var/lib/apt/lists/* + +# Clone the FastETL GitHub repository and copy specified files +RUN git clone https://github.com/economiagovbr/FastETL.git /tmp/repo-FastETL && \ + mkdir -p /opt/airflow/plugins/FastETL/hooks && \ + mkdir -p /opt/airflow/plugins/FastETL/custom_functions/utils && \ + cp /tmp/repo-FastETL/hooks/dou_hook.py /opt/airflow/plugins/FastETL/hooks/dou_hook.py && \ + cp -r /tmp/repo-FastETL/custom_functions/utils/* /opt/airflow/plugins/FastETL/custom_functions/utils/ + +# Remove Git and clean up package cache +RUN apt-get remove -y git && \ + apt-get autoremove -y && \ + rm -rf /tmp/repo-ro-dou && \ + rm -rf /tmp/repo-FastETL + +USER airflow + +# Install additional Airflow dependencies RUN pip install --no-cache-dir --user 'apache-airflow[microsoft.mssql,google_auth]' + +# Copy and install requirements.txt COPY requirements.txt / RUN pip install --no-cache-dir -r /requirements.txt From 19b6e4007c7aa2a4d51c5dc898cc4a4884ad86d5 Mon Sep 17 00:00:00 2001 From: Nitai Bezerra da Silva Date: Fri, 17 Mar 2023 06:08:29 -0300 Subject: [PATCH 2/4] build(docker-compose): remove FastETL mapping From now on it's bundled into the docker image. --- docker-compose.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c611299..83691b2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,9 +74,6 @@ - ./src:/opt/airflow/dags/ro_dou - ./dag_confs:/opt/airflow/dags/ro_dou/dag_confs - ./tests:/opt/airflow/tests - - - ./FastETL/hooks/dou_hook.py:/opt/airflow/plugins/FastETL/hooks/dou_hook.py - - ./FastETL/custom_functions/utils/:/opt/airflow/plugins/FastETL/custom_functions/utils/ depends_on: postgres: condition: service_healthy From 9a48c2ca3d3c2ceaa578aa9259dfe19c2395f4f3 Mon Sep 17 00:00:00 2001 From: Nitai Bezerra da Silva Date: Fri, 17 Mar 2023 06:31:20 -0300 Subject: [PATCH 3/4] chore(docker-compose): add comment Th src code and the test folders don't need to be mapped into container in production. Src code is bundled into the docker image. --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 83691b2..6a15d57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,9 +71,9 @@ AIRFLOW_CONN_EXAMPLE_DATABASE_CONN: postgresql://airflow:airflow@postgres:5432/airflow volumes: - - ./src:/opt/airflow/dags/ro_dou + - ./src:/opt/airflow/dags/ro_dou # for development purpose - ./dag_confs:/opt/airflow/dags/ro_dou/dag_confs - - ./tests:/opt/airflow/tests + - ./tests:/opt/airflow/tests # for test purpose depends_on: postgres: condition: service_healthy From 7798d7a147baefd3344523f079ba7f9a36101214 Mon Sep 17 00:00:00 2001 From: Nitai Bezerra da Silva Date: Mon, 20 Mar 2023 11:40:59 -0300 Subject: [PATCH 4/4] docs(readme): remove old instruction --- README.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/README.md b/README.md index 829cdab..7f14a05 100644 --- a/README.md +++ b/README.md @@ -26,15 +26,10 @@ isso é necessário ter o **Docker Compose** na versão 1.29 ou maior clonar o repositório no seu computador, acesse o diretório pela linha de comando e execute os comandos a seguir: -```bash -make install-deps -``` -Este comando baixa as dependências necessárias. - ```bash make run ``` -Este comando baixa as imagens docker e sobe todos os contêineres do Ro-dou. +Este comando baixa as imagens docker necessárias, faz build da contêiner do Ro-dou e sobe tudo. O Airflow pode demorar alguns minutos para se configurar a primeira vez. Após isso ele estará disponível em http://localhost:8080/. Para se autenticar