Skip to content

Commit

Permalink
O11Y: Added support for custom tags (#5565)
Browse files Browse the repository at this point in the history
* O11Y: Added support for custom tags
  • Loading branch information
zasweq authored Aug 11, 2022
1 parent 7981af4 commit c7fe135
Show file tree
Hide file tree
Showing 10 changed files with 236 additions and 647 deletions.
87 changes: 74 additions & 13 deletions gcp/observability/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,6 @@ import (

gcplogging "cloud.google.com/go/logging"
"golang.org/x/oauth2/google"
configpb "google.golang.org/grpc/gcp/observability/internal/config"
"google.golang.org/protobuf/encoding/protojson"
)

const (
Expand All @@ -41,6 +39,69 @@ const (

var logFilterPatternRegexp = regexp.MustCompile(logFilterPatternRegexpStr)

// logFilter represents a method logging configuration.
type logFilter struct {
// Pattern is a string which can select a group of method names. By
// default, the Pattern is an empty string, matching no methods.
//
// Only "*" Wildcard is accepted for Pattern. A Pattern is in the form
// of <service>/<method> or just a character "*" .
//
// If the Pattern is "*", it specifies the defaults for all the
// services; If the Pattern is <service>/*, it specifies the defaults
// for all methods in the specified service <service>; If the Pattern is
// */<method>, this is not supported.
//
// Examples:
// - "Foo/Bar" selects only the method "Bar" from service "Foo"
// - "Foo/*" selects all methods from service "Foo"
// - "*" selects all methods from all services.
Pattern string `json:"pattern,omitempty"`
// HeaderBytes is the number of bytes of each header to log. If the size of
// the header is greater than the defined limit, content past the limit will
// be truncated. The default value is 0.
HeaderBytes int32 `json:"header_bytes,omitempty"`
// MessageBytes is the number of bytes of each message to log. If the size
// of the message is greater than the defined limit, content pass the limit
// will be truncated. The default value is 0.
MessageBytes int32 `json:"message_bytes,omitempty"`
}

// config is configuration for observability behaviors. By default, no
// configuration is required for tracing/metrics/logging to function. This
// config captures the most common knobs for gRPC users. It's always possible to
// override with explicit config in code.
type config struct {
// EnableCloudTrace represents whether the tracing data upload to
// CloudTrace should be enabled or not.
EnableCloudTrace bool `json:"enable_cloud_trace,omitempty"`
// EnableCloudMonitoring represents whether the metrics data upload to
// CloudMonitoring should be enabled or not.
EnableCloudMonitoring bool `json:"enable_cloud_monitoring,omitempty"`
// EnableCloudLogging represents Whether the logging data upload to
// CloudLogging should be enabled or not.
EnableCloudLogging bool `json:"enable_cloud_logging,omitempty"`
// DestinationProjectID is the destination GCP project identifier for the
// uploading log entries. If empty, the gRPC Observability plugin will
// attempt to fetch the project_id from the GCP environment variables, or
// from the default credentials.
DestinationProjectID string `json:"destination_project_id,omitempty"`
// LogFilters is a list of method config. The order matters here - the first
// Pattern which matches the current method will apply the associated config
// options in the logFilter. Any other logFilter that also matches that
// comes later will be ignored. So a logFilter of "*/*" should appear last
// in this list.
LogFilters []logFilter `json:"log_filters,omitempty"`
// GlobalTraceSamplingRate is the global setting that controls the
// probability of a RPC being traced. For example, 0.05 means there is a 5%
// chance for a RPC to be traced, 1.0 means trace every call, 0 means don’t
// start new traces.
GlobalTraceSamplingRate float64 `json:"global_trace_sampling_rate,omitempty"`
// CustomTags a list of custom tags that will be attached to every log
// entry.
CustomTags map[string]string `json:"custom_tags,omitempty"`
}

// fetchDefaultProjectID fetches the default GCP project id from environment.
func fetchDefaultProjectID(ctx context.Context) string {
// Step 1: Check ENV var
Expand All @@ -62,25 +123,25 @@ func fetchDefaultProjectID(ctx context.Context) string {
return credentials.ProjectID
}

func validateFilters(config *configpb.ObservabilityConfig) error {
for _, filter := range config.GetLogFilters() {
func validateFilters(config *config) error {
for _, filter := range config.LogFilters {
if filter.Pattern == "*" {
continue
}
match := logFilterPatternRegexp.FindStringSubmatch(filter.Pattern)
if match == nil {
return fmt.Errorf("invalid log filter pattern: %v", filter.Pattern)
return fmt.Errorf("invalid log filter Pattern: %v", filter.Pattern)
}
}
return nil
}

// unmarshalAndVerifyConfig unmarshals a json string representing an
// observability config into its protobuf format, and also verifies the
// observability config into its internal go format, and also verifies the
// configuration's fields for validity.
func unmarshalAndVerifyConfig(rawJSON json.RawMessage) (*configpb.ObservabilityConfig, error) {
var config configpb.ObservabilityConfig
if err := protojson.Unmarshal(rawJSON, &config); err != nil {
func unmarshalAndVerifyConfig(rawJSON json.RawMessage) (*config, error) {
var config config
if err := json.Unmarshal(rawJSON, &config); err != nil {
return nil, fmt.Errorf("error parsing observability config: %v", err)
}
if err := validateFilters(&config); err != nil {
Expand All @@ -93,7 +154,7 @@ func unmarshalAndVerifyConfig(rawJSON json.RawMessage) (*configpb.ObservabilityC
return &config, nil
}

func parseObservabilityConfig() (*configpb.ObservabilityConfig, error) {
func parseObservabilityConfig() (*config, error) {
if fileSystemPath := os.Getenv(envObservabilityConfigJSON); fileSystemPath != "" {
content, err := ioutil.ReadFile(fileSystemPath) // TODO: Switch to os.ReadFile once dropped support for go 1.15
if err != nil {
Expand All @@ -107,14 +168,14 @@ func parseObservabilityConfig() (*configpb.ObservabilityConfig, error) {
return nil, nil
}

func ensureProjectIDInObservabilityConfig(ctx context.Context, config *configpb.ObservabilityConfig) error {
if config.GetDestinationProjectId() == "" {
func ensureProjectIDInObservabilityConfig(ctx context.Context, config *config) error {
if config.DestinationProjectID == "" {
// Try to fetch the GCP project id
projectID := fetchDefaultProjectID(ctx)
if projectID == "" {
return fmt.Errorf("empty destination project ID")
}
config.DestinationProjectId = projectID
config.DestinationProjectID = projectID
}
return nil
}
14 changes: 6 additions & 8 deletions gcp/observability/exporting.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"context"
"encoding/json"
"fmt"
"os"

gcplogging "cloud.google.com/go/logging"
grpclogrecordpb "google.golang.org/grpc/gcp/observability/internal/logging"
Expand All @@ -45,20 +44,19 @@ type cloudLoggingExporter struct {
logger *gcplogging.Logger
}

func newCloudLoggingExporter(ctx context.Context, projectID string) (*cloudLoggingExporter, error) {
c, err := gcplogging.NewClient(ctx, fmt.Sprintf("projects/%v", projectID))
func newCloudLoggingExporter(ctx context.Context, config *config) (*cloudLoggingExporter, error) {
c, err := gcplogging.NewClient(ctx, fmt.Sprintf("projects/%v", config.DestinationProjectID))
if err != nil {
return nil, fmt.Errorf("failed to create cloudLoggingExporter: %v", err)
}
defer logger.Infof("Successfully created cloudLoggingExporter")
customTags := getCustomTags(os.Environ())
if len(customTags) != 0 {
logger.Infof("Adding custom tags: %+v", customTags)
if len(config.CustomTags) != 0 {
logger.Infof("Adding custom tags: %+v", config.CustomTags)
}
return &cloudLoggingExporter{
projectID: projectID,
projectID: config.DestinationProjectID,
client: c,
logger: c.Logger("microservices.googleapis.com/observability/grpc", gcplogging.CommonLabels(customTags)),
logger: c.Logger("microservices.googleapis.com/observability/grpc", gcplogging.CommonLabels(config.CustomTags)),
}, nil
}

Expand Down
Loading

0 comments on commit c7fe135

Please sign in to comment.