From 2dbbe655b73ee3947890890e45d3bfbb3e742d07 Mon Sep 17 00:00:00 2001 From: Marcelo Castro Date: Mon, 12 Dec 2022 11:04:41 -0300 Subject: [PATCH] Airbyte-Proxy: Added a variable to control timeout (#19493) * Added a default proxy timeout of 5min * Changed proxy timeout to be a .env variable * Update docs/operator-guides/configuring-airbyte.md Co-authored-by: Evan Tahler * Changed file permissions Co-authored-by: Evan Tahler Co-authored-by: Nataly Merezhuk <65251165+natalyjazzviolin@users.noreply.github.com> Co-authored-by: Marcos Marx --- .env | 1 + airbyte-proxy/nginx-auth.conf.template | 10 ++++++++++ airbyte-proxy/nginx-no-auth.conf.template | 10 ++++++++++ airbyte-proxy/run.sh | 2 +- airbyte-proxy/test.sh | 7 ++++--- docker-compose.yaml | 1 + docs/operator-guides/configuring-airbyte.md | 1 + 7 files changed, 28 insertions(+), 4 deletions(-) diff --git a/.env b/.env index e8d526eb1580..f6cc4a7f16c9 100644 --- a/.env +++ b/.env @@ -36,6 +36,7 @@ HACK_LOCAL_ROOT_PARENT=/tmp # Set to empty values, e.g. "" to disable basic auth BASIC_AUTH_USERNAME=airbyte BASIC_AUTH_PASSWORD=password +BASIC_AUTH_PROXY_TIMEOUT=600 ### DATABASE ### # Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db diff --git a/airbyte-proxy/nginx-auth.conf.template b/airbyte-proxy/nginx-auth.conf.template index 2a6f279329be..7dfc53b42638 100644 --- a/airbyte-proxy/nginx-auth.conf.template +++ b/airbyte-proxy/nginx-auth.conf.template @@ -14,6 +14,11 @@ http { proxy_pass "${PROXY_PASS_WEB}"; + proxy_connect_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; + proxy_send_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; + proxy_read_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; + send_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; + error_page 401 /etc/nginx/401.html; location ~ (401.html)$ { alias /etc/nginx/$1; @@ -35,6 +40,11 @@ http { proxy_pass "${PROXY_PASS_API}"; + proxy_connect_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; + proxy_send_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; + proxy_read_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; + send_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; + error_page 401 /etc/nginx/401.html; location ~ (401.html)$ { alias /etc/nginx/$1; diff --git a/airbyte-proxy/nginx-no-auth.conf.template b/airbyte-proxy/nginx-no-auth.conf.template index ab3a2e8d9e67..b09ee488f91e 100644 --- a/airbyte-proxy/nginx-no-auth.conf.template +++ b/airbyte-proxy/nginx-no-auth.conf.template @@ -10,6 +10,11 @@ http { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_pass "${PROXY_PASS_WEB}"; + + proxy_connect_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; + proxy_send_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; + proxy_read_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; + send_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; } } @@ -22,6 +27,11 @@ http { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_pass "${PROXY_PASS_API}"; + + proxy_connect_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; + proxy_send_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; + proxy_read_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; + send_timeout ${BASIC_AUTH_PROXY_TIMEOUT}; } } diff --git a/airbyte-proxy/run.sh b/airbyte-proxy/run.sh index 032c60e4996c..264026393984 100644 --- a/airbyte-proxy/run.sh +++ b/airbyte-proxy/run.sh @@ -16,7 +16,7 @@ else TEMPLATE_PATH="/etc/nginx/templates/nginx-auth.conf.template" fi -envsubst '${PROXY_PASS_WEB} ${PROXY_PASS_API} ${CONNECTOR_BUILDER_SERVER_API} ${PROXY_PASS_RESOLVER}' < $TEMPLATE_PATH > /etc/nginx/nginx.conf +envsubst '${PROXY_PASS_WEB} ${PROXY_PASS_API} ${CONNECTOR_BUILDER_SERVER_API} ${PROXY_PASS_RESOLVER} ${BASIC_AUTH_PROXY_TIMEOUT}' < $TEMPLATE_PATH > /etc/nginx/nginx.conf echo "starting nginx..." nginx -v diff --git a/airbyte-proxy/test.sh b/airbyte-proxy/test.sh index 4abec191f4f8..3b97c60df974 100755 --- a/airbyte-proxy/test.sh +++ b/airbyte-proxy/test.sh @@ -5,13 +5,14 @@ PORT=18000 BASIC_AUTH_USERNAME=airbyte BASIC_AUTH_PASSWORD=password BASIC_AUTH_UPDATED_PASSWORD=pa55w0rd +BASIC_AUTH_PROXY_TIMEOUT=120 TEST_HOST=localhost VERSION="${VERSION:-dev}" # defaults to "dev", otherwise it is set by environment's $VERSION echo "testing with proxy container airbyte/proxy:$VERSION" function start_container () { - CMD="docker run -d -p $PORT:8000 --env BASIC_AUTH_USERNAME=$1 --env BASIC_AUTH_PASSWORD=$2 --env PROXY_PASS_WEB=http://localhost --env PROXY_PASS_API=http://localhost --env CONNECTOR_BUILDER_SERVER_API=http://localhost --name $NAME airbyte/proxy:$VERSION" + CMD="docker run -d -p $PORT:8000 --env BASIC_AUTH_USERNAME=$1 --env BASIC_AUTH_PASSWORD=$2 --env BASIC_AUTH_PROXY_TIMEOUT=$3 --env PROXY_PASS_WEB=http://localhost --env PROXY_PASS_API=http://localhost --env CONNECTOR_BUILDER_SERVER_API=http://localhost --name $NAME airbyte/proxy:$VERSION" echo $CMD eval $CMD wait_for_docker; @@ -42,7 +43,7 @@ echo "Testing airbyte proxy..." stop_container; # just in case there was a failure of a previous test run echo "Starting $NAME" -start_container $BASIC_AUTH_USERNAME $BASIC_AUTH_PASSWORD +start_container $BASIC_AUTH_USERNAME $BASIC_AUTH_PASSWORD $BASIC_AUTH_PROXY_TIMEOUT echo "Testing access without auth" RESPONSE=`curl "http://$TEST_HOST:$PORT" -i --silent` @@ -67,7 +68,7 @@ fi stop_container; echo "Starting $NAME with updated password" -start_container $BASIC_AUTH_USERNAME $BASIC_AUTH_UPDATED_PASSWORD +start_container $BASIC_AUTH_USERNAME $BASIC_AUTH_UPDATED_PASSWORD $BASIC_AUTH_PROXY_TIMEOUT echo "Testing access with orignial paassword" RESPONSE=`curl "http://$BASIC_AUTH_USERNAME:$BASIC_AUTH_PASSWORD@$TEST_HOST:$PORT" -i --silent` diff --git a/docker-compose.yaml b/docker-compose.yaml index 88f141c5aaaf..46fbab29492c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -237,6 +237,7 @@ services: environment: - BASIC_AUTH_USERNAME=${BASIC_AUTH_USERNAME} - BASIC_AUTH_PASSWORD=${BASIC_AUTH_PASSWORD} + - BASIC_AUTH_PROXY_TIMEOUT=${BASIC_AUTH_PROXY_TIMEOUT} networks: - airbyte_internal - airbyte_public diff --git a/docs/operator-guides/configuring-airbyte.md b/docs/operator-guides/configuring-airbyte.md index c3b406a34052..6be58be99e03 100644 --- a/docs/operator-guides/configuring-airbyte.md +++ b/docs/operator-guides/configuring-airbyte.md @@ -50,6 +50,7 @@ Set to empty values, e.g. "" to disable basic auth. **Be sure to change these va 1. BASIC_AUTH_USERNAME=airbyte 2. BASIC_AUTH_PASSWORD=password +3. BASIC_AUTH_PROXY_TIMEOUT=600 - Defines the proxy timeout time for requests to Airbyte Server. Main use should be for dynamic discover when creating a connection (S3, JDBC, etc) that takes a long time. #### Secrets