TUN-7553: Add flag to enable management diagnostic services

With the new flag --management-diagnostics (an opt-in flag)
cloudflared's will be able to report additional diagnostic information
over the management.argotunnel.com request path.
Additions include the /metrics prometheus endpoint; which is already
bound to a local port via --metrics.
/debug/pprof/(goroutine|heap) are also provided to allow for remotely
retrieving heap information from a running cloudflared connector.

CHANGES.md

@@ -1,3 +1,7 @@
+## 2023.6.2
+### New Features
+- You can now enable additional diagnostics over the management.argotunnel.com service for your active cloudflared connectors via a new runtime flag `--management-diagnostics` (or env `TUNNEL_MANAGEMENT_DIAGNOSTICS`). This feature is provided as opt-out and requires the flag to enable. Endpoints such as /metrics provides your prometheus metrics endpoint another mechanism to be reached. Additionally /debug/pprof/(goroutine|heap) are also introduced to allow for remotely retrieving active pprof information from a running cloudflared connector.
+
 ## 2023.4.1
 ### New Features
 - You can now stream your logs from your remote cloudflared to your local terminal with `cloudflared tail <TUNNEL-ID>`. This new feature requires the remote cloudflared to be version 2023.4.1 or higher.

cmd/cloudflared/tunnel/cmd.go

@@ -413,6 +413,7 @@ func StartServer(
 
 		mgmt := management.New(
 			c.String("management-hostname"),
+			c.Bool("management-diagnostics"),
 			serviceIP,
 			clientID,
 			c.String(connectorLabelFlag),
@@ -764,6 +765,12 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_POST_QUANTUM"},
 			Hidden:  FipsEnabled,
 		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    "management-diagnostics",
+			Usage:   "Enables the in-depth diagnostic routes to be made available over the management service (/debug/pprof, /metrics, etc.)",
+			EnvVars: []string{"TUNNEL_MANAGEMENT_DIAGNOSTICS"},
+			Value:   false,
+		}),
 		selectProtocolFlag,
 		overwriteDNSFlag,
 	}...)

component-tests/test_management.py

@@ -55,7 +55,7 @@ def test_get_metrics(self, tmp_path, component_tests_config):
         config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
         LOGGER.debug(config)
         config_path = write_config(tmp_path, config.full_config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True):
+        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--management-diagnostics"], new_process=True):
             wait_tunnel_ready(require_min_connections=1)
             cfd_cli = CloudflaredCli(config, config_path, LOGGER)
             url = cfd_cli.get_management_url("metrics", config, config_path)
@@ -76,7 +76,7 @@ def test_get_pprof_heap(self, tmp_path, component_tests_config):
         config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
         LOGGER.debug(config)
         config_path = write_config(tmp_path, config.full_config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True):
+        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--management-diagnostics"], new_process=True):
             wait_tunnel_ready(require_min_connections=1)
             cfd_cli = CloudflaredCli(config, config_path, LOGGER)
             url = cfd_cli.get_management_url("debug/pprof/heap", config, config_path)
@@ -85,6 +85,26 @@ def test_get_pprof_heap(self, tmp_path, component_tests_config):
             # Assert response.
             assert resp.status_code == 200, "Expected cloudflared to return 200 for /debug/pprof/heap"
             assert resp.headers["Content-Type"] == "application/octet-stream", "Expected /debug/pprof/heap to have return a binary response"
+
+    """
+        test_get_metrics_when_disabled will verify that diagnostic endpoints (such as /metrics) return 404 and are unmounted. 
+    """
+    def test_get_metrics_when_disabled(self, tmp_path, component_tests_config):
+        # TUN-7377 : wait_tunnel_ready does not work properly in windows.
+        # Skipping this test for windows for now and will address it as part of tun-7377
+        if platform.system() == "Windows":
+            return
+        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
+        LOGGER.debug(config)
+        config_path = write_config(tmp_path, config.full_config)
+        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True):
+            wait_tunnel_ready(require_min_connections=1)
+            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
+            url = cfd_cli.get_management_url("metrics", config, config_path)
+            resp = send_request(url)
+
+            # Assert response.
+            assert resp.status_code == 404, "Expected cloudflared to return 404 for /metrics"
 
 
 @retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)

management/service.go

@@ -31,6 +31,17 @@ const (
 	reasonIdleLimitExceeded                      = "session was idle for too long"
 )
 
+var (
+	// CORS middleware required to allow dash to access management.argotunnel.com requests
+	corsHandler = cors.Handler(cors.Options{
+		// Allows for any subdomain of cloudflare.com
+		AllowedOrigins: []string{"https://*.cloudflare.com"},
+		// Required to present cookies or other authentication across origin boundries
+		AllowCredentials: true,
+		MaxAge:           300, // Maximum value not ignored by any of major browsers
+	})
+)
+
 type ManagementService struct {
 	// The management tunnel hostname
 	Hostname string
@@ -54,6 +65,7 @@ type ManagementService struct {
 }
 
 func New(managementHostname string,
+	enableDiagServices bool,
 	serviceIP string,
 	clientID uuid.UUID,
 	label string,
@@ -71,25 +83,21 @@ func New(managementHostname string,
 	}
 	r := chi.NewRouter()
 	r.Use(ValidateAccessTokenQueryMiddleware)
-	r.Get("/ping", ping)
-	r.Head("/ping", ping)
+
+	// Default management services
+	r.With(corsHandler).Get("/ping", ping)
+	r.With(corsHandler).Head("/ping", ping)
 	r.Get("/logs", s.logs)
-	r.Get("/metrics", s.metricsHandler.ServeHTTP)
+	r.With(corsHandler).Get("/host_details", s.getHostDetails)
 
-	// Supports only heap and goroutine
-	r.Get("/debug/pprof/{profile:heap|goroutine}", pprof.Index)
+	// Diagnostic management services
+	if enableDiagServices {
+		// Prometheus endpoint
+		r.With(corsHandler).Get("/metrics", s.metricsHandler.ServeHTTP)
+		// Supports only heap and goroutine
+		r.With(corsHandler).Get("/debug/pprof/{profile:heap|goroutine}", pprof.Index)
+	}
 
-	r.Route("/host_details", func(r chi.Router) {
-		// CORS middleware required to allow dash to access management.argotunnel.com requests
-		r.Use(cors.Handler(cors.Options{
-			// Allows for any subdomain of cloudflare.com
-			AllowedOrigins: []string{"https://*.cloudflare.com"},
-			// Required to present cookies or other authentication across origin boundries
-			AllowCredentials: true,
-			MaxAge:           300, // Maximum value not ignored by any of major browsers
-		}))
-		r.Get("/", s.getHostDetails)
-	})
 	s.router = r
 	return s
 }

management/service_test.go

@@ -3,9 +3,13 @@ package management
 import (
 	"context"
 	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -15,9 +19,23 @@ import (
 )
 
 var (
-	noopLogger = zerolog.New(io.Discard)
+	noopLogger         = zerolog.New(io.Discard)
+	managementHostname = "https://management.argotunnel.com"
 )
 
+func TestDisableDiagnosticRoutes(t *testing.T) {
+	mgmt := New("management.argotunnel.com", false, "1.1.1.1:80", uuid.Nil, "", &noopLogger, nil)
+	for _, path := range []string{"/metrics", "/debug/pprof/goroutine", "/debug/pprof/heap"} {
+		t.Run(strings.Replace(path, "/", "_", -1), func(t *testing.T) {
+			req := httptest.NewRequest("GET", managementHostname+path+"?access_token="+validToken, nil)
+			recorder := httptest.NewRecorder()
+			mgmt.ServeHTTP(recorder, req)
+			resp := recorder.Result()
+			require.Equal(t, http.StatusNotFound, resp.StatusCode)
+		})
+	}
+}
+
 func TestReadEventsLoop(t *testing.T) {
 	sentEvent := EventStartStreaming{
 		ClientEvent: ClientEvent{Type: StartStreaming},

orchestration/orchestrator_test.go

@@ -51,7 +51,7 @@ func TestUpdateConfiguration(t *testing.T) {
 	initConfig := &Config{
 		Ingress: &ingress.Ingress{},
 	}
-	orchestrator, err := NewOrchestrator(context.Background(), initConfig, testTags, []ingress.Rule{ingress.NewManagementRule(management.New("management.argotunnel.com", "1.1.1.1:80", uuid.Nil, "", &testLogger, nil))}, &testLogger)
+	orchestrator, err := NewOrchestrator(context.Background(), initConfig, testTags, []ingress.Rule{ingress.NewManagementRule(management.New("management.argotunnel.com", false, "1.1.1.1:80", uuid.Nil, "", &testLogger, nil))}, &testLogger)
 	require.NoError(t, err)
 	initOriginProxy, err := orchestrator.GetOriginProxy()
 	require.NoError(t, err)