Support Proxy protocol (go-gitea#12527)

This PR adds functionality to allow Gitea to sit behind an
HAProxy and HAProxy protocolled connections directly.

Fix go-gitea#7508

Signed-off-by: Andrew Thornton <art27@cantab.net>

cmd/web.go

@@ -76,7 +76,7 @@ func runHTTPRedirector() {
 		http.Redirect(w, r, target, http.StatusTemporaryRedirect)
 	})
 
-	err := runHTTP("tcp", source, "HTTP Redirector", handler)
+	err := runHTTP("tcp", source, "HTTP Redirector", handler, setting.RedirectorUseProxyProtocol)
 	if err != nil {
 		log.Fatal("Failed to start port redirection: %v", err)
 	}
@@ -231,40 +231,38 @@ func listen(m http.Handler, handleRedirector bool) error {
 		if handleRedirector {
 			NoHTTPRedirector()
 		}
-		err = runHTTP("tcp", listenAddr, "Web", m)
+		err = runHTTP("tcp", listenAddr, "Web", m, setting.UseProxyProtocol)
 	case setting.HTTPS:
 		if setting.EnableAcme {
 			err = runACME(listenAddr, m)
 			break
-		} else {
-			if handleRedirector {
-				if setting.RedirectOtherPort {
-					go runHTTPRedirector()
-				} else {
-					NoHTTPRedirector()
-				}
+		}
+		if handleRedirector {
+			if setting.RedirectOtherPort {
+				go runHTTPRedirector()
+			} else {
+				NoHTTPRedirector()
 			}
-			err = runHTTPS("tcp", listenAddr, "Web", setting.CertFile, setting.KeyFile, m)
 		}
+		err = runHTTPS("tcp", listenAddr, "Web", setting.CertFile, setting.KeyFile, m, setting.UseProxyProtocol, setting.ProxyProtocolTLSBridging)
 	case setting.FCGI:
 		if handleRedirector {
 			NoHTTPRedirector()
 		}
-		err = runFCGI("tcp", listenAddr, "FCGI Web", m)
+		err = runFCGI("tcp", listenAddr, "FCGI Web", m, setting.UseProxyProtocol)
 	case setting.HTTPUnix:
 		if handleRedirector {
 			NoHTTPRedirector()
 		}
-		err = runHTTP("unix", listenAddr, "Web", m)
+		err = runHTTP("unix", listenAddr, "Web", m, setting.UseProxyProtocol)
 	case setting.FCGIUnix:
 		if handleRedirector {
 			NoHTTPRedirector()
 		}
-		err = runFCGI("unix", listenAddr, "Web", m)
+		err = runFCGI("unix", listenAddr, "Web", m, setting.UseProxyProtocol)
 	default:
 		log.Fatal("Invalid protocol: %s", setting.Protocol)
 	}
-
 	if err != nil {
 		log.Critical("Failed to start server: %v", err)
 	}

cmd/web_acme.go

@@ -113,14 +113,14 @@ func runACME(listenAddr string, m http.Handler) error {
 
 			log.Info("Running Let's Encrypt handler on %s", setting.HTTPAddr+":"+setting.PortToRedirect)
 			// all traffic coming into HTTP will be redirect to HTTPS automatically (LE HTTP-01 validation happens here)
-			err := runHTTP("tcp", setting.HTTPAddr+":"+setting.PortToRedirect, "Let's Encrypt HTTP Challenge", myACME.HTTPChallengeHandler(http.HandlerFunc(runLetsEncryptFallbackHandler)))
+			err := runHTTP("tcp", setting.HTTPAddr+":"+setting.PortToRedirect, "Let's Encrypt HTTP Challenge", myACME.HTTPChallengeHandler(http.HandlerFunc(runLetsEncryptFallbackHandler)), setting.RedirectorUseProxyProtocol)
 			if err != nil {
 				log.Fatal("Failed to start the Let's Encrypt handler on port %s: %v", setting.PortToRedirect, err)
 			}
 		}()
 	}
 
-	return runHTTPSWithTLSConfig("tcp", listenAddr, "Web", tlsConfig, m)
+	return runHTTPSWithTLSConfig("tcp", listenAddr, "Web", tlsConfig, m, setting.UseProxyProtocol, setting.ProxyProtocolTLSBridging)
 }
 
 func runLetsEncryptFallbackHandler(w http.ResponseWriter, r *http.Request) {

cmd/web_graceful.go

@@ -15,8 +15,8 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 )
 
-func runHTTP(network, listenAddr, name string, m http.Handler) error {
-	return graceful.HTTPListenAndServe(network, listenAddr, name, m)
+func runHTTP(network, listenAddr, name string, m http.Handler, useProxyProtocol bool) error {
+	return graceful.HTTPListenAndServe(network, listenAddr, name, m, useProxyProtocol)
 }
 
 // NoHTTPRedirector tells our cleanup routine that we will not be using a fallback http redirector
@@ -36,7 +36,7 @@ func NoInstallListener() {
 	graceful.GetManager().InformCleanup()
 }
 
-func runFCGI(network, listenAddr, name string, m http.Handler) error {
+func runFCGI(network, listenAddr, name string, m http.Handler, useProxyProtocol bool) error {
 	// This needs to handle stdin as fcgi point
 	fcgiServer := graceful.NewServer(network, listenAddr, name)
 
@@ -47,7 +47,7 @@ func runFCGI(network, listenAddr, name string, m http.Handler) error {
 			}
 			m.ServeHTTP(resp, req)
 		}))
-	})
+	}, useProxyProtocol)
 	if err != nil {
 		log.Fatal("Failed to start FCGI main server: %v", err)
 	}

cmd/web_https.go

@@ -129,14 +129,14 @@ var (
 	defaultCiphersChaChaFirst = append(defaultCiphersChaCha, defaultCiphersAES...)
 )
 
-// runHTTPs listens on the provided network address and then calls
+// runHTTPS listens on the provided network address and then calls
 // Serve to handle requests on incoming TLS connections.
 //
 // Filenames containing a certificate and matching private key for the server must
 // be provided. If the certificate is signed by a certificate authority, the
 // certFile should be the concatenation of the server's certificate followed by the
 // CA's certificate.
-func runHTTPS(network, listenAddr, name, certFile, keyFile string, m http.Handler) error {
+func runHTTPS(network, listenAddr, name, certFile, keyFile string, m http.Handler, useProxyProtocol, proxyProtocolTLSBridging bool) error {
 	tlsConfig := &tls.Config{}
 	if tlsConfig.NextProtos == nil {
 		tlsConfig.NextProtos = []string{"h2", "http/1.1"}
@@ -184,9 +184,9 @@ func runHTTPS(network, listenAddr, name, certFile, keyFile string, m http.Handle
 		return err
 	}
 
-	return graceful.HTTPListenAndServeTLSConfig(network, listenAddr, name, tlsConfig, m)
+	return graceful.HTTPListenAndServeTLSConfig(network, listenAddr, name, tlsConfig, m, useProxyProtocol, proxyProtocolTLSBridging)
 }
 
-func runHTTPSWithTLSConfig(network, listenAddr, name string, tlsConfig *tls.Config, m http.Handler) error {
-	return graceful.HTTPListenAndServeTLSConfig(network, listenAddr, name, tlsConfig, m)
+func runHTTPSWithTLSConfig(network, listenAddr, name string, tlsConfig *tls.Config, m http.Handler, useProxyProtocol, proxyProtocolTLSBridging bool) error {
+	return graceful.HTTPListenAndServeTLSConfig(network, listenAddr, name, tlsConfig, m, useProxyProtocol, proxyProtocolTLSBridging)
 }

custom/conf/app.example.ini

@@ -29,6 +29,18 @@ RUN_MODE = ; prod
 ;; The protocol the server listens on. One of 'http', 'https', 'unix' or 'fcgi'. Defaults to 'http'
 ;PROTOCOL = http
 ;;
+;; Expect PROXY protocol headers on connections
+;USE_PROXY_PROTOCOL = false
+;;
+;; Use PROXY protocol in TLS Bridging mode
+;PROXY_PROTOCOL_TLS_BRIDGING = false
+;;
+; Timeout to wait for PROXY protocol header (set to 0 to have no timeout)
+;PROXY_PROTOCOL_HEADER_TIMEOUT=5s
+;;
+; Accept PROXY protocol headers with UNKNOWN type
+;PROXY_PROTOCOL_ACCEPT_UNKNOWN=false
+;;
 ;; Set the domain for the server
 ;DOMAIN = localhost
 ;;
@@ -51,6 +63,8 @@ RUN_MODE = ; prod
 ;REDIRECT_OTHER_PORT = false
 ;PORT_TO_REDIRECT = 80
 ;;
+;; expect PROXY protocol header on connections to https redirector.
+;REDIRECTOR_USE_PROXY_PROTOCOL = %(USE_PROXY_PROTOCOL)
 ;; Minimum and maximum supported TLS versions
 ;SSL_MIN_VERSION=TLSv1.2
 ;SSL_MAX_VERSION=
@@ -76,13 +90,19 @@ RUN_MODE = ; prod
 ;; Do not set this variable if PROTOCOL is set to 'unix'.
 ;LOCAL_ROOT_URL = %(PROTOCOL)s://%(HTTP_ADDR)s:%(HTTP_PORT)s/
 ;;
+;; When making local connections pass the PROXY protocol header.
+;LOCAL_USE_PROXY_PROTOCOL = %(USE_PROXY_PROTOCOL)
+;;
 ;; Disable SSH feature when not available
 ;DISABLE_SSH = false
 ;;
 ;; Whether to use the builtin SSH server or not.
 ;START_SSH_SERVER = false
 ;;
-;; Username to use for the builtin SSH server.
+;; Expect PROXY protocol header on connections to the built-in SSH server
+;SSH_SERVER_USE_PROXY_PROTOCOL = false
+;;
+;; Username to use for the builtin SSH server. If blank, then it is the value of RUN_USER.
 ;BUILTIN_SSH_SERVER_USER = %(RUN_USER)s
 ;;
 ;; Domain name to be exposed in clone URL

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -238,6 +238,10 @@ The following configuration set `Content-Type: application/vnd.android.package-a
 ## Server (`server`)
 
 - `PROTOCOL`: **http**: \[http, https, fcgi, http+unix, fcgi+unix\]
+- `USE_PROXY_PROTOCOL`: **false**: Expect PROXY protocol headers on connections
+- `PROXY_PROTOCOL_TLS_BRIDGING`: **false**: When protocol is https, expect PROXY protocol headers after TLS negotiation.
+- `PROXY_PROTOCOL_HEADER_TIMEOUT`: **5s**: Timeout to wait for PROXY protocol header (set to 0 to have no timeout)
+- `PROXY_PROTOCOL_ACCEPT_UNKNOWN`: **false**: Accept PROXY protocol headers with Unknown type.
 - `DOMAIN`: **localhost**: Domain name of this server.
 - `ROOT_URL`: **%(PROTOCOL)s://%(DOMAIN)s:%(HTTP\_PORT)s/**:
    Overwrite the automatically generated public URL.
@@ -262,12 +266,15 @@ The following configuration set `Content-Type: application/vnd.android.package-a
    most cases you do not need to change the default value. Alter it only if
    your SSH server node is not the same as HTTP node. Do not set this variable
    if `PROTOCOL` is set to `http+unix`.
+- `LOCAL_USE_PROXY_PROTOCOL`: **%(USE_PROXY_PROTOCOL)**: When making local connections pass the PROXY protocol header.
+   This should be set to false if the local connection will go through the proxy.
 - `PER_WRITE_TIMEOUT`: **30s**: Timeout for any write to the connection. (Set to -1 to
    disable all timeouts.)
 - `PER_WRITE_PER_KB_TIMEOUT`: **10s**: Timeout per Kb written to connections.
 
 - `DISABLE_SSH`: **false**: Disable SSH feature when it's not available.
 - `START_SSH_SERVER`: **false**: When enabled, use the built-in SSH server.
+- `SSH_SERVER_USE_PROXY_PROTOCOL`: **false**: Expect PROXY protocol header on connections to the built-in SSH Server.
 - `BUILTIN_SSH_SERVER_USER`: **%(RUN_USER)s**: Username to use for the built-in SSH Server.
 - `SSH_USER`: **%(BUILTIN_SSH_SERVER_USER)**: SSH username displayed in clone URLs. This is only for people who configure the SSH server themselves; in most cases, you want to leave this blank and modify the `BUILTIN_SSH_SERVER_USER`.
 - `SSH_DOMAIN`: **%(DOMAIN)s**: Domain name of this server, used for displayed clone URL.
@@ -313,6 +320,7 @@ The following configuration set `Content-Type: application/vnd.android.package-a
 - `LFS_LOCKS_PAGING_NUM`: **50**: Maximum number of LFS Locks returned per page.
 
 - `REDIRECT_OTHER_PORT`: **false**: If true and `PROTOCOL` is https, allows redirecting http requests on `PORT_TO_REDIRECT` to the https port Gitea listens on.
+- `REDIRECTOR_USE_PROXY_PROTOCOL`: **%(USE_PROXY_PROTOCOL)**: expect PROXY protocol header on connections to https redirector.
 - `PORT_TO_REDIRECT`: **80**: Port for the http redirection service to listen on. Used when `REDIRECT_OTHER_PORT` is true.
 - `SSL_MIN_VERSION`: **TLSv1.2**: Set the minimum version of ssl support.
 - `SSL_MAX_VERSION`: **\<empty\>**: Set the maximum version of ssl support.

modules/graceful/server.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/proxyprotocol"
 	"code.gitea.io/gitea/modules/setting"
 )
 
@@ -79,16 +80,27 @@ func NewServer(network, address, name string) *Server {
 
 // ListenAndServe listens on the provided network address and then calls Serve
 // to handle requests on incoming connections.
-func (srv *Server) ListenAndServe(serve ServeFunction) error {
+func (srv *Server) ListenAndServe(serve ServeFunction, useProxyProtocol bool) error {
 	go srv.awaitShutdown()
 
-	l, err := GetListener(srv.network, srv.address)
+	listener, err := GetListener(srv.network, srv.address)
 	if err != nil {
 		log.Error("Unable to GetListener: %v", err)
 		return err
 	}
 
-	srv.listener = newWrappedListener(l, srv)
+	// we need to wrap the listener to take account of our lifecycle
+	listener = newWrappedListener(listener, srv)
+
+	// Now we need to take account of ProxyProtocol settings...
+	if useProxyProtocol {
+		listener = &proxyprotocol.Listener{
+			Listener:           listener,
+			ProxyHeaderTimeout: setting.ProxyProtocolHeaderTimeout,
+			AcceptUnknown:      setting.ProxyProtocolAcceptUnknown,
+		}
+	}
+	srv.listener = listener
 
 	srv.BeforeBegin(srv.network, srv.address)
 
@@ -97,22 +109,44 @@ func (srv *Server) ListenAndServe(serve ServeFunction) error {
 
 // ListenAndServeTLSConfig listens on the provided network address and then calls
 // Serve to handle requests on incoming TLS connections.
-func (srv *Server) ListenAndServeTLSConfig(tlsConfig *tls.Config, serve ServeFunction) error {
+func (srv *Server) ListenAndServeTLSConfig(tlsConfig *tls.Config, serve ServeFunction, useProxyProtocol, proxyProtocolTLSBridging bool) error {
 	go srv.awaitShutdown()
 
 	if tlsConfig.MinVersion == 0 {
 		tlsConfig.MinVersion = tls.VersionTLS12
 	}
 
-	l, err := GetListener(srv.network, srv.address)
+	listener, err := GetListener(srv.network, srv.address)
 	if err != nil {
 		log.Error("Unable to get Listener: %v", err)
 		return err
 	}
 
-	wl := newWrappedListener(l, srv)
-	srv.listener = tls.NewListener(wl, tlsConfig)
+	// we need to wrap the listener to take account of our lifecycle
+	listener = newWrappedListener(listener, srv)
+
+	// Now we need to take account of ProxyProtocol settings... If we're not bridging then we expect that the proxy will forward the connection to us
+	if useProxyProtocol && !proxyProtocolTLSBridging {
+		listener = &proxyprotocol.Listener{
+			Listener:           listener,
+			ProxyHeaderTimeout: setting.ProxyProtocolHeaderTimeout,
+			AcceptUnknown:      setting.ProxyProtocolAcceptUnknown,
+		}
+	}
+
+	// Now handle the tls protocol
+	listener = tls.NewListener(listener, tlsConfig)
+
+	// Now if we're bridging then we need the proxy to tell us who we're bridging for...
+	if useProxyProtocol && proxyProtocolTLSBridging {
+		listener = &proxyprotocol.Listener{
+			Listener:           listener,
+			ProxyHeaderTimeout: setting.ProxyProtocolHeaderTimeout,
+			AcceptUnknown:      setting.ProxyProtocolAcceptUnknown,
+		}
+	}
 
+	srv.listener = listener
 	srv.BeforeBegin(srv.network, srv.address)
 
 	return srv.Serve(serve)

modules/graceful/server_http.go

@@ -28,14 +28,14 @@ func newHTTPServer(network, address, name string, handler http.Handler) (*Server
 
 // HTTPListenAndServe listens on the provided network address and then calls Serve
 // to handle requests on incoming connections.
-func HTTPListenAndServe(network, address, name string, handler http.Handler) error {
+func HTTPListenAndServe(network, address, name string, handler http.Handler, useProxyProtocol bool) error {
 	server, lHandler := newHTTPServer(network, address, name, handler)
-	return server.ListenAndServe(lHandler)
+	return server.ListenAndServe(lHandler, useProxyProtocol)
 }
 
 // HTTPListenAndServeTLSConfig listens on the provided network address and then calls Serve
 // to handle requests on incoming connections.
-func HTTPListenAndServeTLSConfig(network, address, name string, tlsConfig *tls.Config, handler http.Handler) error {
+func HTTPListenAndServeTLSConfig(network, address, name string, tlsConfig *tls.Config, handler http.Handler, useProxyProtocol, proxyProtocolTLSBridging bool) error {
 	server, lHandler := newHTTPServer(network, address, name, handler)
-	return server.ListenAndServeTLSConfig(tlsConfig, lHandler)
+	return server.ListenAndServeTLSConfig(tlsConfig, lHandler, useProxyProtocol, proxyProtocolTLSBridging)
 }

modules/private/internal.go

@@ -14,6 +14,7 @@ import (
 	"code.gitea.io/gitea/modules/httplib"
 	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/proxyprotocol"
 	"code.gitea.io/gitea/modules/setting"
 )
 
@@ -50,7 +51,32 @@ func newInternalRequest(ctx context.Context, url, method string) *httplib.Reques
 		req.SetTransport(&http.Transport{
 			DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
 				var d net.Dialer
-				return d.DialContext(ctx, "unix", setting.HTTPAddr)
+				conn, err := d.DialContext(ctx, "unix", setting.HTTPAddr)
+				if err != nil {
+					return conn, err
+				}
+				if setting.LocalUseProxyProtocol {
+					if err = proxyprotocol.WriteLocalHeader(conn); err != nil {
+						_ = conn.Close()
+						return nil, err
+					}
+				}
+				return conn, err
+			},
+		})
+	} else if setting.LocalUseProxyProtocol {
+		req.SetTransport(&http.Transport{
+			DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
+				var d net.Dialer
+				conn, err := d.DialContext(ctx, network, address)
+				if err != nil {
+					return conn, err
+				}
+				if err = proxyprotocol.WriteLocalHeader(conn); err != nil {
+					_ = conn.Close()
+					return nil, err
+				}
+				return conn, err
 			},
 		})
 	}