[OSS] feat: access logs for listeners and listener filters

.changelog/15864.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+connect: adds support for Envoy access logging. Access logging can be enabled using the [`proxy-defaults`](https://developer.hashicorp.com/consul/docs/connect/config-entries/proxy-defaults#accesslogs) config entry.
+```

agent/configentry/merge_service_config.go

@@ -112,6 +112,9 @@ func MergeServiceConfig(defaults *structs.ServiceConfigResponse, service *struct
 	if err := mergo.Merge(&ns.Proxy.Expose, defaults.Expose); err != nil {
 		return nil, err
 	}
+	if err := mergo.Merge(&ns.Proxy.AccessLogs, defaults.AccessLogs); err != nil {
+		return nil, err
+	}
 
 	// defaults.EnvoyExtensions contains the extensions from the proxy defaults config entry followed by extensions from
 	// the service defaults config entry. This adds the extensions to structs.NodeService.Proxy which in turn is copied

agent/configentry/merge_service_config_test.go

@@ -21,14 +21,35 @@ func Test_MergeServiceConfig_TransparentProxy(t *testing.T) {
 		want *structs.NodeService
 	}{
 		{
-			name: "inherit transparent proxy settings",
+			name: "inherit transparent proxy settings + kitchen sink",
 			args: args{
 				defaults: &structs.ServiceConfigResponse{
 					Mode: structs.ProxyModeTransparent,
 					TransparentProxy: structs.TransparentProxyConfig{
 						OutboundListenerPort: 10101,
 						DialedDirectly:       true,
 					},
+					ProxyConfig: map[string]interface{}{
+						"foo": "bar",
+					},
+					Expose: structs.ExposeConfig{
+						Checks: true,
+						Paths: []structs.ExposePath{
+							{
+								ListenerPort: 8080,
+								Path:         "/",
+								Protocol:     "http",
+							},
+						},
+					},
+					MeshGateway: structs.MeshGatewayConfig{Mode: structs.MeshGatewayModeRemote},
+					AccessLogs: structs.AccessLogsConfig{
+						Enabled:             true,
+						DisableListenerLogs: true,
+						Type:                structs.FileLogSinkType,
+						Path:                "/tmp/accesslog.txt",
+						JSONFormat:          "{ \"custom_start_time\": \"%START_TIME%\" }",
+					},
 				},
 				service: &structs.NodeService{
 					ID:      "foo-proxy",
@@ -52,6 +73,27 @@ func Test_MergeServiceConfig_TransparentProxy(t *testing.T) {
 						OutboundListenerPort: 10101,
 						DialedDirectly:       true,
 					},
+					Config: map[string]interface{}{
+						"foo": "bar",
+					},
+					Expose: structs.ExposeConfig{
+						Checks: true,
+						Paths: []structs.ExposePath{
+							{
+								ListenerPort: 8080,
+								Path:         "/",
+								Protocol:     "http",
+							},
+						},
+					},
+					MeshGateway: structs.MeshGatewayConfig{Mode: structs.MeshGatewayModeRemote},
+					AccessLogs: structs.AccessLogsConfig{
+						Enabled:             true,
+						DisableListenerLogs: true,
+						Type:                structs.FileLogSinkType,
+						Path:                "/tmp/accesslog.txt",
+						JSONFormat:          "{ \"custom_start_time\": \"%START_TIME%\" }",
+					},
 				},
 			},
 		},

agent/configentry/resolve.go

@@ -38,6 +38,7 @@ func ComputeResolvedServiceConfig(
 		thisReply.MeshGateway = proxyConf.MeshGateway
 		thisReply.Expose = proxyConf.Expose
 		thisReply.EnvoyExtensions = proxyConf.EnvoyExtensions
+		thisReply.AccessLogs = proxyConf.AccessLogs
 
 		// Extract the global protocol from proxyConf for upstream configs.
 		rawProtocol := proxyConf.Config["protocol"]

agent/configentry/resolve_test.go

@@ -129,6 +129,62 @@ func Test_ComputeResolvedServiceConfig(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "proxy inherits kitchen sink from proxy-defaults",
+			args: args{
+				scReq: &structs.ServiceConfigRequest{
+					Name: "sid",
+				},
+				entries: &ResolvedServiceConfigSet{
+					ProxyDefaults: map[string]*structs.ProxyConfigEntry{
+						acl.DefaultEnterpriseMeta().PartitionOrDefault(): {
+							Config: map[string]interface{}{
+								"foo": "bar",
+							},
+							Expose: structs.ExposeConfig{
+								Checks: true,
+								Paths:  []structs.ExposePath{},
+							},
+							Mode:        structs.ProxyModeTransparent,
+							MeshGateway: remoteMeshGW,
+							TransparentProxy: structs.TransparentProxyConfig{
+								OutboundListenerPort: 6666,
+								DialedDirectly:       true,
+							},
+							AccessLogs: structs.AccessLogsConfig{
+								Enabled:             true,
+								DisableListenerLogs: true,
+								Type:                structs.FileLogSinkType,
+								Path:                "/tmp/accesslog.txt",
+								JSONFormat:          "{ \"custom_start_time\": \"%START_TIME%\" }",
+							},
+						},
+					},
+				},
+			},
+			want: &structs.ServiceConfigResponse{
+				ProxyConfig: map[string]interface{}{
+					"foo": "bar",
+				},
+				Expose: structs.ExposeConfig{
+					Checks: true,
+					Paths:  []structs.ExposePath{},
+				},
+				Mode:        structs.ProxyModeTransparent,
+				MeshGateway: remoteMeshGW,
+				TransparentProxy: structs.TransparentProxyConfig{
+					OutboundListenerPort: 6666,
+					DialedDirectly:       true,
+				},
+				AccessLogs: structs.AccessLogsConfig{
+					Enabled:             true,
+					DisableListenerLogs: true,
+					Type:                structs.FileLogSinkType,
+					Path:                "/tmp/accesslog.txt",
+					JSONFormat:          "{ \"custom_start_time\": \"%START_TIME%\" }",
+				},
+			},
+		},
 		{
 			name: "proxy upstream mesh-gateway inherits service-defaults",
 			args: args{

agent/grpc-internal/services/subscribe/subscribe_test.go

@@ -164,6 +164,7 @@ func TestServer_Subscribe_IntegrationWithBackend(t *testing.T) {
 									MeshGateway:      &pbservice.MeshGatewayConfig{},
 									Expose:           &pbservice.ExposeConfig{},
 									TransparentProxy: &pbservice.TransparentProxyConfig{},
+									AccessLogs:       &pbservice.AccessLogsConfig{},
 								},
 								Connect:        &pbservice.ServiceConnect{},
 								RaftIndex:      raftIndex(ids, "reg2", "reg2"),
@@ -197,6 +198,7 @@ func TestServer_Subscribe_IntegrationWithBackend(t *testing.T) {
 									MeshGateway:      &pbservice.MeshGatewayConfig{},
 									Expose:           &pbservice.ExposeConfig{},
 									TransparentProxy: &pbservice.TransparentProxyConfig{},
+									AccessLogs:       &pbservice.AccessLogsConfig{},
 								},
 								Connect:        &pbservice.ServiceConnect{},
 								RaftIndex:      raftIndex(ids, "reg3", "reg3"),
@@ -249,6 +251,7 @@ func TestServer_Subscribe_IntegrationWithBackend(t *testing.T) {
 								MeshGateway:      &pbservice.MeshGatewayConfig{},
 								Expose:           &pbservice.ExposeConfig{},
 								TransparentProxy: &pbservice.TransparentProxyConfig{},
+								AccessLogs:       &pbservice.AccessLogsConfig{},
 							},
 							Connect:        &pbservice.ServiceConnect{},
 							RaftIndex:      raftIndex(ids, "reg3", "reg3"),
@@ -547,6 +550,7 @@ func TestServer_Subscribe_IntegrationWithBackend_ForwardToDC(t *testing.T) {
 									MeshGateway:      &pbservice.MeshGatewayConfig{},
 									Expose:           &pbservice.ExposeConfig{},
 									TransparentProxy: &pbservice.TransparentProxyConfig{},
+									AccessLogs:       &pbservice.AccessLogsConfig{},
 								},
 								Connect:        &pbservice.ServiceConnect{},
 								EnterpriseMeta: pbcommon.DefaultEnterpriseMeta,
@@ -580,6 +584,7 @@ func TestServer_Subscribe_IntegrationWithBackend_ForwardToDC(t *testing.T) {
 									MeshGateway:      &pbservice.MeshGatewayConfig{},
 									Expose:           &pbservice.ExposeConfig{},
 									TransparentProxy: &pbservice.TransparentProxyConfig{},
+									AccessLogs:       &pbservice.AccessLogsConfig{},
 								},
 								Connect:        &pbservice.ServiceConnect{},
 								EnterpriseMeta: pbcommon.DefaultEnterpriseMeta,
@@ -633,6 +638,7 @@ func TestServer_Subscribe_IntegrationWithBackend_ForwardToDC(t *testing.T) {
 								MeshGateway:      &pbservice.MeshGatewayConfig{},
 								Expose:           &pbservice.ExposeConfig{},
 								TransparentProxy: &pbservice.TransparentProxyConfig{},
+								AccessLogs:       &pbservice.AccessLogsConfig{},
 							},
 							Connect:        &pbservice.ServiceConnect{},
 							EnterpriseMeta: pbcommon.DefaultEnterpriseMeta,
@@ -1046,6 +1052,7 @@ func TestNewEventFromSteamEvent(t *testing.T) {
 													MeshGateway:      &pbservice.MeshGatewayConfig{},
 													Expose:           &pbservice.ExposeConfig{},
 													TransparentProxy: &pbservice.TransparentProxyConfig{},
+													AccessLogs:       &pbservice.AccessLogsConfig{},
 												},
 												Connect:        &pbservice.ServiceConnect{},
 												EnterpriseMeta: &pbcommon.EnterpriseMeta{},
@@ -1068,6 +1075,7 @@ func TestNewEventFromSteamEvent(t *testing.T) {
 													MeshGateway:      &pbservice.MeshGatewayConfig{},
 													Expose:           &pbservice.ExposeConfig{},
 													TransparentProxy: &pbservice.TransparentProxyConfig{},
+													AccessLogs:       &pbservice.AccessLogsConfig{},
 												},
 												Connect:        &pbservice.ServiceConnect{},
 												EnterpriseMeta: &pbcommon.EnterpriseMeta{},
@@ -1107,6 +1115,7 @@ func TestNewEventFromSteamEvent(t *testing.T) {
 									MeshGateway:      &pbservice.MeshGatewayConfig{},
 									Expose:           &pbservice.ExposeConfig{},
 									TransparentProxy: &pbservice.TransparentProxyConfig{},
+									AccessLogs:       &pbservice.AccessLogsConfig{},
 								},
 								Connect:        &pbservice.ServiceConnect{},
 								EnterpriseMeta: &pbcommon.EnterpriseMeta{},

agent/structs/config_entry.go

@@ -1,7 +1,6 @@
 package structs
 
 import (
-	"encoding/json"
 	"errors"
 	"fmt"
 	"net"
@@ -453,30 +452,8 @@ func (e *ProxyConfigEntry) Validate() error {
 		return fmt.Errorf("invalid name (%q), only %q is supported", e.Name, ProxyConfigGlobal)
 	}
 
-	switch e.AccessLogs.Type {
-	case "", StdErrLogSinkType, StdOutLogSinkType:
-		// OK
-	case FileLogSinkType:
-		if e.AccessLogs.Path == "" {
-			return errors.New("path must be specified when using file type access logs")
-		}
-	default:
-		return fmt.Errorf("invalid access log type: %s", e.AccessLogs.Type)
-	}
-
-	if e.AccessLogs.JSONFormat != "" && e.AccessLogs.TextFormat != "" {
-		return errors.New("cannot specify both access log JSONFormat and TextFormat")
-	}
-
-	if e.AccessLogs.Type != FileLogSinkType && e.AccessLogs.Path != "" {
-		return errors.New("path is only valid for file type access logs")
-	}
-
-	if e.AccessLogs.JSONFormat != "" {
-		msg := json.RawMessage{}
-		if err := json.Unmarshal([]byte(e.AccessLogs.JSONFormat), &msg); err != nil {
-			return fmt.Errorf("invalid access log json for JSON format: %w", err)
-		}
+	if err := e.AccessLogs.Validate(); err != nil {
+		return err
 	}
 
 	if err := validateConfigEntryMeta(e.Meta); err != nil {
@@ -1186,6 +1163,7 @@ type ServiceConfigResponse struct {
 	TransparentProxy  TransparentProxyConfig `json:",omitempty"`
 	Mode              ProxyMode              `json:",omitempty"`
 	Destination       DestinationConfig      `json:",omitempty"`
+	AccessLogs        AccessLogsConfig       `json:",omitempty"`
 	Meta              map[string]string      `json:",omitempty"`
 	EnvoyExtensions   []EnvoyExtension       `json:",omitempty"`
 	QueryMeta