-
Notifications
You must be signed in to change notification settings - Fork 0
/
Dockerfile
52 lines (38 loc) · 1.1 KB
/
Dockerfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
FROM python:3.9.5
# Install supervisord and copy the configuration file
RUN apt-get update && apt-get install -y supervisor
COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
# Copy the code into the container
COPY ./requirements_data-pipeline.txt /
# Install the required packages
RUN pip install --no-cache-dir -r requirements_data-pipeline.txt
# Airflow setup
ENV AIRFLOW_HOME=/app/airflow
WORKDIR $AIRFLOW_HOME/
# Copy the DAG file to the DAGs directory
RUN mkdir -p data/ scripts/ web_api/
COPY ./dags/data_pipeline.py dags/
COPY ./scripts scripts/
COPY ./data data/
COPY ./web_api /
COPY ./logs logs/
ENV PYTHONPATH=$PYTHONPATH:scripts/
WORKDIR $AIRFLOW_HOME/data/raw
#unzip data
RUN apt-get update -y \
&& apt-get install -y unzip \
&& unzip archive.zip
WORKDIR $AIRFLOW_HOME
# Initialize the Airflow database
RUN airflow db init
# # Disable DAG examples
COPY ./airflow.cfg airflow.cfg
RUN airflow users create \
--username admin \
--firstname Admin \
--lastname User \
--role Admin \
--email admin@example.com \
--password admin
# Expose the Airflow webserver port
EXPOSE 8080