Add proxy from env

command/run.go

@@ -30,6 +30,7 @@ func (r Run) Execute(args Args, config *config.CouperFile, logEntry *logrus.Entr
 	set.StringVar(&config.Settings.HealthPath, "health-path", config.Settings.HealthPath, "-health-path /healthz")
 	set.IntVar(&config.Settings.DefaultPort, "p", config.Settings.DefaultPort, "-p 8080")
 	set.BoolVar(&config.Settings.XForwardedHost, "xfh", config.Settings.XForwardedHost, "-xfh")
+	set.BoolVar(&config.Settings.NoProxyFromEnv, "no-proxy-from-env", config.Settings.NoProxyFromEnv, "-no-proxy-from-env")
 	set.StringVar(&config.Settings.RequestIDFormat, "request-id-format", config.Settings.RequestIDFormat, "-request-id-format uuid4")
 	if err := set.Parse(args.Filter(set)); err != nil {
 		return err

config/runtime/server.go

@@ -152,7 +152,7 @@ func NewServerConfiguration(conf *config.CouperFile, log *logrus.Entry) (ServerC
 					return nil, diags
 				}
 
-				backend, err := newProxy(confCtx, &backendConf, srvConf.API.CORS, log, serverOptions)
+				backend, err := newProxy(confCtx, &backendConf, srvConf.API.CORS, log, serverOptions, conf.Settings.NoProxyFromEnv)
 				if err != nil {
 					return nil, err
 				}
@@ -169,17 +169,20 @@ func NewServerConfiguration(conf *config.CouperFile, log *logrus.Entry) (ServerC
 	return serverConfiguration, nil
 }
 
-func newProxy(ctx *hcl.EvalContext, beConf *config.Backend, corsOpts *config.CORS, log *logrus.Entry, srvOpts *server.Options) (http.Handler, error) {
+func newProxy(
+	ctx *hcl.EvalContext, beConf *config.Backend, corsOpts *config.CORS,
+	log *logrus.Entry, srvOpts *server.Options,
+	noProxyFromEnv bool,
+) (http.Handler, error) {
 	if err := validateOrigin(ctx, beConf); err != nil {
 		return nil, err
 	}
-
 	corsOptions, err := handler.NewCORSOptions(corsOpts)
 	if err != nil {
 		return nil, err
 	}
 
-	proxyOptions, err := handler.NewProxyOptions(beConf, corsOptions)
+	proxyOptions, err := handler.NewProxyOptions(beConf, corsOptions, noProxyFromEnv)
 	if err != nil {
 		return nil, err
 	}

config/settings.go

@@ -11,6 +11,7 @@ var DefaultSettings = Settings{
 type Settings struct {
 	DefaultPort     int    `hcl:"default_port,optional"`
 	HealthPath      string `hcl:"health_path,optional"`
+	NoProxyFromEnv  bool   `hcl:"no_proxy_from_env,optional"`
 	LogFormat       string `hcl:"log_format,optional"`
 	RequestIDFormat string `hcl:"request_id_format,optional"`
 	XForwardedHost  bool   `hcl:"xfh,optional"`

errors/code.go

@@ -22,6 +22,7 @@ const (
 	APIError Code = 4000 + iota
 	APIRouteNotFound
 	APIConnect
+	APIProxyConnect
 	APIReqBodySizeExceeded
 )
 
@@ -54,6 +55,7 @@ var codes = map[Code]string{
 	APIError:               "API failed",
 	APIRouteNotFound:       "API route not found",
 	APIConnect:             "API upstream connection error",
+	APIProxyConnect:        "upstream connection error via configured proxy",
 	APIReqBodySizeExceeded: "Request body size exceeded",
 	// 5xxx
 	AuthorizationRequired: "Authorization required",

errors/http.go

@@ -9,7 +9,7 @@ func httpStatus(code Code) int {
 	switch code {
 	case APIRouteNotFound, FilesRouteNotFound, RouteNotFound, SPARouteNotFound:
 		return http.StatusNotFound
-	case APIConnect, UpstreamResponseValidationFailed:
+	case APIConnect, APIProxyConnect, UpstreamResponseValidationFailed:
 		return http.StatusBadGateway
 	case APIReqBodySizeExceeded:
 		return http.StatusRequestEntityTooLarge

handler/openapi_validator_test.go

@@ -61,7 +61,7 @@ func TestOpenAPIValidator_ValidateRequest(t *testing.T) {
 		RequestBodyLimit: "64MiB",
 	}
 
-	proxyOpts, err := handler.NewProxyOptions(beConf, &handler.CORSOptions{})
+	proxyOpts, err := handler.NewProxyOptions(beConf, &handler.CORSOptions{}, config.DefaultSettings.NoProxyFromEnv)
 	helper.Must(err)
 
 	backend, err := handler.NewProxy(proxyOpts, log.WithContext(context.Background()), &server.Options{APIErrTpl: errors.DefaultJSON}, eval.NewENVContext(nil))

handler/proxy.go

@@ -117,6 +117,7 @@ func (c *CORSOptions) AllowsOrigin(origin string) bool {
 func NewProxy(options *ProxyOptions, log *logrus.Entry, srvOpts *server.Options, evalCtx *hcl.EvalContext) (http.Handler, error) {
 	logConf := *logging.DefaultConfig
 	logConf.TypeFieldKey = "couper_backend"
+	logConf.NoProxyFromEnv = options.NoProxyFromEnv
 	env.DecodeWithPrefix(&logConf, "BACKEND_")
 
 	var apiValidation eval.BufferOption
@@ -150,6 +151,11 @@ func (p *Proxy) getTransport(scheme, origin, hostname string) *http.Transport {
 
 		d := &net.Dialer{Timeout: p.options.ConnectTimeout}
 
+		var proxyFunc func(req *http.Request) (*url.URL, error)
+		if !p.options.NoProxyFromEnv {
+			proxyFunc = http.ProxyFromEnvironment
+		}
+
 		transport = &http.Transport{
 			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
 				conn, err := d.DialContext(ctx, network, addr)
@@ -160,6 +166,7 @@ func (p *Proxy) getTransport(scheme, origin, hostname string) *http.Transport {
 			},
 			DisableCompression:    true,
 			MaxConnsPerHost:       p.options.MaxConnections,
+			Proxy:                 proxyFunc,
 			ResponseHeaderTimeout: p.options.TTFBTimeout,
 			TLSClientConfig:       tlsConf,
 		}
@@ -252,7 +259,11 @@ func (p *Proxy) roundtrip(rw http.ResponseWriter, req *http.Request) {
 	roundtripInfo.BeReq, roundtripInfo.BeResp = outreq, res
 	if err != nil {
 		roundtripInfo.Err = err
-		p.srvOptions.APIErrTpl.ServeError(couperErr.APIConnect).ServeHTTP(rw, req)
+		errCode := couperErr.APIConnect
+		if strings.HasPrefix(err.Error(), "proxyconnect") {
+			errCode = couperErr.APIProxyConnect
+		}
+		p.srvOptions.APIErrTpl.ServeError(errCode).ServeHTTP(rw, req)
 		return
 	}
 

handler/proxy_options.go

@@ -16,13 +16,14 @@ type ProxyOptions struct {
 	Context                              hcl.Body
 	BackendName                          string
 	CORS                                 *CORSOptions
+	NoProxyFromEnv                       bool
 	DisableCertValidation                bool
 	MaxConnections                       int
 	OpenAPI                              *OpenAPIValidatorOptions
 	RequestBodyLimit                     int64
 }
 
-func NewProxyOptions(conf *config.Backend, corsOpts *CORSOptions) (*ProxyOptions, error) {
+func NewProxyOptions(conf *config.Backend, corsOpts *CORSOptions, noProxyFromEnv bool) (*ProxyOptions, error) {
 	var timeout, connTimeout, ttfbTimeout time.Duration
 	if err := parseDuration(conf.Timeout, &timeout); err != nil {
 		return nil, err
@@ -56,6 +57,7 @@ func NewProxyOptions(conf *config.Backend, corsOpts *CORSOptions) (*ProxyOptions
 		ConnectTimeout:        connTimeout,
 		DisableCertValidation: conf.DisableCertValidation,
 		MaxConnections:        conf.MaxConnections,
+		NoProxyFromEnv:        noProxyFromEnv,
 		OpenAPI:               openAPIValidatorOptions,
 		RequestBodyLimit:      bodyLimit,
 		TTFBTimeout:           ttfbTimeout,

handler/proxy_test.go

@@ -712,7 +712,7 @@ func TestProxy_SetGetBody_LimitBody_Roundtrip(t *testing.T) {
 			proxyOpts, err := handler.NewProxyOptions(&config.Backend{
 				Remain:           helper.NewProxyContext("set_request_headers = { x = req.post }"), // ensure buffering is enabled
 				RequestBodyLimit: testcase.limit,
-			}, nil)
+			}, nil, config.DefaultSettings.NoProxyFromEnv)
 			if err != nil {
 				subT.Error(err)
 				return

logging/access_log.go

@@ -105,47 +105,47 @@ func (log *AccessLog) ServeHTTP(rw http.ResponseWriter, req *http.Request, nextH
 	nextHandler.ServeHTTP(rw, req)
 	serveDone := time.Now()
 
+	fields := Fields{}
+
 	reqCtx := req
 	if isUpstreamRequest && roundtripInfo != nil && roundtripInfo.BeReq != nil {
 		reqCtx = roundtripInfo.BeReq
 		if roundtripInfo.BeResp != nil {
 			reqCtx.TLS = roundtripInfo.BeResp.TLS
 		}
+
+		backendName, _ := reqCtx.Context().Value(request.BackendName).(string)
+		if backendName == "" {
+			endpointName, _ := reqCtx.Context().Value(request.Endpoint).(string)
+			fields["endpoint"] = endpointName
+		}
+
+		if !log.conf.NoProxyFromEnv {
+			u, err := http.ProxyFromEnvironment(roundtripInfo.BeReq)
+			if err == nil && u != nil {
+				fields["proxy"] = u.Host
+			}
+		}
 	}
 
-	uniqueID := reqCtx.Context().Value(request.UID)
+	fields["method"] = reqCtx.Method
+	fields["proto"] = reqCtx.Proto
+	fields["server"] = reqCtx.Context().Value(request.ServerName)
+	fields["uid"] = reqCtx.Context().Value(request.UID)
 
 	requestFields := Fields{
 		"headers": filterHeader(log.conf.RequestHeaders, reqCtx.Header),
 	}
+	fields["request"] = requestFields
 
 	if req.ContentLength > 0 {
 		requestFields["bytes"] = reqCtx.ContentLength
 	}
 
-	serverName, _ := reqCtx.Context().Value(request.ServerName).(string)
-
-	fields := Fields{
-		"method":  reqCtx.Method,
-		"proto":   reqCtx.Proto,
-		"request": requestFields,
-		"server":  serverName,
-		"uid":     uniqueID,
-	}
-
 	if h, ok := nextHandler.(fmt.Stringer); ok {
 		fields["handler"] = h.String()
 	}
 
-	if isUpstreamRequest {
-		backendName, _ := reqCtx.Context().Value(request.BackendName).(string)
-		if backendName == "" {
-			endpointName, _ := reqCtx.Context().Value(request.Endpoint).(string)
-			backendName = serverName + ":" + endpointName
-		}
-		fields["backend"] = backendName
-	}
-
 	if log.conf.TypeFieldKey != "" {
 		fields["type"] = log.conf.TypeFieldKey
 	}

logging/config.go

@@ -1,10 +1,11 @@
 package logging
 
 type Config struct {
+	NoProxyFromEnv  bool
 	ParentFieldKey  string   `env:"log_parent_field"`
-	TypeFieldKey    string   `env:"log_type_value"`
 	RequestHeaders  []string `env:"log_request_headers"`
 	ResponseHeaders []string `env:"log_response_headers"`
+	TypeFieldKey    string   `env:"log_type_value"`
 }
 
 var DefaultConfig = &Config{

server/http_integration_test.go

@@ -10,6 +10,7 @@ import (
 	"io/ioutil"
 	"net"
 	"net/http"
+	"net/http/httptest"
 	"net/url"
 	"os"
 	"path"
@@ -30,6 +31,7 @@ import (
 var (
 	testBackend    *test.Backend
 	testWorkingDir string
+	testProxyAddr  = "http://127.0.0.1:9999"
 )
 
 func TestMain(m *testing.M) {
@@ -47,6 +49,11 @@ func setup() {
 		panic(err)
 	}
 
+	err = os.Setenv("HTTP_PROXY", testProxyAddr)
+	if err != nil {
+		panic(err)
+	}
+
 	wd, err := os.Getwd()
 	if err != nil {
 		panic(err)
@@ -56,14 +63,16 @@ func setup() {
 
 func teardown() {
 	println("INTEGRATION: close test backend...")
-	if err := os.Unsetenv("COUPER_TEST_BACKEND_ADDR"); err != nil {
-		panic(err)
+	for _, key := range []string{"COUPER_TEST_BACKEND_ADDR", "HTTP_PROXY"} {
+		if err := os.Unsetenv(key); err != nil {
+			panic(err)
+		}
 	}
 	testBackend.Close()
 }
 
 func newCouper(file string, helper *test.Helper) (func(), *logrustest.Hook) {
-	gatewayConf, err := configload.LoadFile(filepath.Join(testWorkingDir, file))
+	couperFile, err := configload.LoadFile(filepath.Join(testWorkingDir, file))
 	helper.Must(err)
 
 	log, hook := logrustest.NewNullLogger()
@@ -80,7 +89,7 @@ func newCouper(file string, helper *test.Helper) (func(), *logrustest.Hook) {
 	//log.Out = os.Stdout
 
 	go func() {
-		if err := command.NewRun(ctx).Execute([]string{file}, gatewayConf, log.WithContext(ctx)); err != nil {
+		if err := command.NewRun(ctx).Execute([]string{file}, couperFile, log.WithContext(ctx)); err != nil {
 			shutdownFn()
 			panic(err)
 		}
@@ -214,8 +223,8 @@ func TestHTTPServer_ServeHTTP(t *testing.T) {
 				expectation{http.StatusNotFound, []byte(`{"code": 4001}`), http.Header{"Content-Type": {"application/json"}}, ""},
 			},
 			{
-				testRequest{http.MethodGet, "http://anyserver:8080/v1/connect-error/"},
-				expectation{http.StatusBadGateway, []byte(`{"code": 4002}`), http.Header{"Content-Type": {"application/json"}}, "api"},
+				testRequest{http.MethodGet, "http://anyserver:8080/v1/connect-error/"}, // in this case proxyconnect fails
+				expectation{http.StatusBadGateway, []byte(`{"code": 4003}`), http.Header{"Content-Type": {"application/json"}}, "api"},
 			},
 			{
 				testRequest{http.MethodGet, "http://anyserver:8080/v1x"},
@@ -450,6 +459,40 @@ func TestHTTPServer_XFHHeader(t *testing.T) {
 	shutdown()
 }
 
+func TestHTTPServer_ProxyFromEnv(t *testing.T) {
+	helper := test.New(t)
+
+	seen := make(chan struct{})
+	origin := httptest.NewUnstartedServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		rw.WriteHeader(http.StatusNoContent)
+		go func() {
+			seen <- struct{}{}
+		}()
+	}))
+	ln, err := net.Listen("tcp4", testProxyAddr[7:])
+	helper.Must(err)
+	origin.Listener = ln
+	origin.Start()
+	defer origin.Close()
+
+	confPath := path.Join("testdata/integration", "api/01_couper.hcl")
+	shutdown, _ := newCouper(confPath, test.New(t))
+	defer shutdown()
+
+	req, err := http.NewRequest(http.MethodGet, "http://anyserver:8080/v1/proxy", nil)
+	helper.Must(err)
+
+	_, err = newClient().Do(req)
+	helper.Must(err)
+
+	timer := time.NewTimer(time.Second)
+	select {
+	case <-timer.C:
+		t.Error("Missing proxy call")
+	case <-seen:
+	}
+}
+
 func TestHTTPServer_Gzip(t *testing.T) {
 	client := newClient()
 

server/testdata/integration/api/01_couper.hcl

@@ -10,6 +10,12 @@ server "api" {
       backend = "anything"
     }
 
+    endpoint "/proxy" {
+      backend {
+        origin = "http://example.com"
+      }
+    }
+
     endpoint "/connect-error" {
       backend {
         connect_timeout = "2s"