Airbyte-Proxy: Added a variable to control timeout (#19493)

* Added a default proxy timeout of 5min * Changed proxy timeout to be a .env variable * Update docs/operator-guides/configuring-airbyte.md Co-authored-by: Evan Tahler <evantahler@gmail.com> * Changed file permissions Co-authored-by: Evan Tahler <evantahler@gmail.com> Co-authored-by: Nataly Merezhuk <65251165+natalyjazzviolin@users.noreply.github.com> Co-authored-by: Marcos Marx <marcosmarxm@users.noreply.github.com>
airbytehq · Dec 12, 2022 · 2dbbe65 · 2dbbe65
1 parent fbba4ec
commit 2dbbe65
Show file tree

Hide file tree

Showing 7 changed files with 28 additions and 4 deletions.
diff --git a/.env b/.env
@@ -36,6 +36,7 @@ HACK_LOCAL_ROOT_PARENT=/tmp
 # Set to empty values, e.g. "" to disable basic auth
 BASIC_AUTH_USERNAME=airbyte
 BASIC_AUTH_PASSWORD=password
+BASIC_AUTH_PROXY_TIMEOUT=600
 
 ### DATABASE ###
 # Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db

diff --git a/airbyte-proxy/nginx-auth.conf.template b/airbyte-proxy/nginx-auth.conf.template
@@ -14,6 +14,11 @@ http {
 
       proxy_pass "${PROXY_PASS_WEB}";
 
+      proxy_connect_timeout       ${BASIC_AUTH_PROXY_TIMEOUT};
+      proxy_send_timeout          ${BASIC_AUTH_PROXY_TIMEOUT};
+      proxy_read_timeout          ${BASIC_AUTH_PROXY_TIMEOUT};
+      send_timeout                ${BASIC_AUTH_PROXY_TIMEOUT};
+
       error_page 401 /etc/nginx/401.html;
       location ~ (401.html)$ {
         alias /etc/nginx/$1;
@@ -35,6 +40,11 @@ http {
 
       proxy_pass "${PROXY_PASS_API}";
 
+      proxy_connect_timeout       ${BASIC_AUTH_PROXY_TIMEOUT};
+      proxy_send_timeout          ${BASIC_AUTH_PROXY_TIMEOUT};
+      proxy_read_timeout          ${BASIC_AUTH_PROXY_TIMEOUT};
+      send_timeout                ${BASIC_AUTH_PROXY_TIMEOUT};
+
       error_page 401 /etc/nginx/401.html;
       location ~ (401.html)$ {
         alias /etc/nginx/$1;

diff --git a/airbyte-proxy/nginx-no-auth.conf.template b/airbyte-proxy/nginx-no-auth.conf.template
@@ -10,6 +10,11 @@ http {
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 
       proxy_pass "${PROXY_PASS_WEB}";
+
+      proxy_connect_timeout       ${BASIC_AUTH_PROXY_TIMEOUT};
+      proxy_send_timeout          ${BASIC_AUTH_PROXY_TIMEOUT};
+      proxy_read_timeout          ${BASIC_AUTH_PROXY_TIMEOUT};
+      send_timeout                ${BASIC_AUTH_PROXY_TIMEOUT};
     }
   }
 
@@ -22,6 +27,11 @@ http {
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 
       proxy_pass "${PROXY_PASS_API}";
+
+      proxy_connect_timeout       ${BASIC_AUTH_PROXY_TIMEOUT};
+      proxy_send_timeout          ${BASIC_AUTH_PROXY_TIMEOUT};
+      proxy_read_timeout          ${BASIC_AUTH_PROXY_TIMEOUT};
+      send_timeout                ${BASIC_AUTH_PROXY_TIMEOUT};
     }
   }
 

diff --git a/airbyte-proxy/run.sh b/airbyte-proxy/run.sh
@@ -16,7 +16,7 @@ else
   TEMPLATE_PATH="/etc/nginx/templates/nginx-auth.conf.template"
 fi
 
-envsubst '${PROXY_PASS_WEB} ${PROXY_PASS_API} ${CONNECTOR_BUILDER_SERVER_API} ${PROXY_PASS_RESOLVER}' < $TEMPLATE_PATH > /etc/nginx/nginx.conf
+envsubst '${PROXY_PASS_WEB} ${PROXY_PASS_API} ${CONNECTOR_BUILDER_SERVER_API} ${PROXY_PASS_RESOLVER} ${BASIC_AUTH_PROXY_TIMEOUT}' < $TEMPLATE_PATH > /etc/nginx/nginx.conf
 
 echo "starting nginx..."
 nginx -v

diff --git a/airbyte-proxy/test.sh b/airbyte-proxy/test.sh
@@ -5,13 +5,14 @@ PORT=18000
 BASIC_AUTH_USERNAME=airbyte
 BASIC_AUTH_PASSWORD=password
 BASIC_AUTH_UPDATED_PASSWORD=pa55w0rd
+BASIC_AUTH_PROXY_TIMEOUT=120
 TEST_HOST=localhost
 VERSION="${VERSION:-dev}" # defaults to "dev", otherwise it is set by environment's $VERSION
 
 echo "testing with proxy container airbyte/proxy:$VERSION"
 
 function start_container () {
-  CMD="docker run -d -p $PORT:8000 --env BASIC_AUTH_USERNAME=$1 --env BASIC_AUTH_PASSWORD=$2 --env PROXY_PASS_WEB=http://localhost --env PROXY_PASS_API=http://localhost --env CONNECTOR_BUILDER_SERVER_API=http://localhost --name $NAME airbyte/proxy:$VERSION"
+  CMD="docker run -d -p $PORT:8000 --env BASIC_AUTH_USERNAME=$1 --env BASIC_AUTH_PASSWORD=$2 --env BASIC_AUTH_PROXY_TIMEOUT=$3 --env PROXY_PASS_WEB=http://localhost --env PROXY_PASS_API=http://localhost --env CONNECTOR_BUILDER_SERVER_API=http://localhost --name $NAME airbyte/proxy:$VERSION"
   echo $CMD
   eval $CMD
   wait_for_docker;
@@ -42,7 +43,7 @@ echo "Testing airbyte proxy..."
 stop_container; # just in case there was a failure of a previous test run
 
 echo "Starting $NAME"
-start_container $BASIC_AUTH_USERNAME $BASIC_AUTH_PASSWORD
+start_container $BASIC_AUTH_USERNAME $BASIC_AUTH_PASSWORD $BASIC_AUTH_PROXY_TIMEOUT
 
 echo "Testing access without auth"
 RESPONSE=`curl "http://$TEST_HOST:$PORT" -i --silent`
@@ -67,7 +68,7 @@ fi
 stop_container;
 
 echo "Starting $NAME with updated password"
-start_container $BASIC_AUTH_USERNAME $BASIC_AUTH_UPDATED_PASSWORD
+start_container $BASIC_AUTH_USERNAME $BASIC_AUTH_UPDATED_PASSWORD $BASIC_AUTH_PROXY_TIMEOUT
 
 echo "Testing access with orignial paassword"
 RESPONSE=`curl "http://$BASIC_AUTH_USERNAME:$BASIC_AUTH_PASSWORD@$TEST_HOST:$PORT" -i --silent`

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -237,6 +237,7 @@ services:
     environment:
       - BASIC_AUTH_USERNAME=${BASIC_AUTH_USERNAME}
       - BASIC_AUTH_PASSWORD=${BASIC_AUTH_PASSWORD}
+      - BASIC_AUTH_PROXY_TIMEOUT=${BASIC_AUTH_PROXY_TIMEOUT}
     networks:
       - airbyte_internal
       - airbyte_public

diff --git a/docs/operator-guides/configuring-airbyte.md b/docs/operator-guides/configuring-airbyte.md
@@ -50,6 +50,7 @@ Set to empty values, e.g. "" to disable basic auth. **Be sure to change these va
 
 1. BASIC_AUTH_USERNAME=airbyte
 2. BASIC_AUTH_PASSWORD=password
+3. BASIC_AUTH_PROXY_TIMEOUT=600 - Defines the proxy timeout time for requests to Airbyte Server. Main use should be for dynamic discover when creating a connection (S3, JDBC, etc) that takes a long time.
 
 #### Secrets