Skip to content

Commit

Permalink
add properties.discovery.yaml support and fix root path regex
Browse files Browse the repository at this point in the history
  • Loading branch information
@rmfitzpatrick
rmfitzpatrick committed Mar 20, 2023
1 parent acdcf14 commit efc86c3
Show file tree
Hide file tree
Showing 9 changed files with 265 additions and 55 deletions.
48 changes: 47 additions & 1 deletion internal/confmapprovider/discovery/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,4 +101,50 @@ successfully started observers.
1. Stop all temporary components before continuing on to the actual Collector service (or exiting early with `--dry-run`).


By default, the Discovery mode is provided with pre-made discovery config components in [`bundle.d`](./bundle/README.md).
By default, the Discovery mode is provided with pre-made discovery config components in [`bundle.d`](./bundle/README.md).


### Discovery properties

Configuring discovery components is performed by merging discovery properties with the config.d receivers
and extensions `*.discovery.yaml` files. Discovery properties are of the form:

```yaml
splunk.discovery.receivers.<receiver-type(/name)>.config.<field>(<::subfield>)*: <value>
splunk.discovery.extensions.<observer-type(/name)>.config.<field>(<::subfield>)*: <value>
splunk.discovery.receivers.<receiver-type(/name)>.enabled.: <true or false>
splunk.discovery.extensions.<observer-type(/name)>.enabled: <true or false>

# Examples
splunk.discovery.receivers.prometheus_simple.config.labels::my_label: my_label_value
splunk.discovery.receivers.prometheus_simple.enabled: true

splunk.discovery.extensions.docker_observer.config.endpoint: tcp://localhost:8080
splunk.discovery.extensions.k8s_observer.enabled: false
```
These properties can be in `config.d/properties.discovery.yaml` or specified at run time with `--set` command line options.

Each discovery property also has an equivalent environment variable form using `_x<hex pair>_` encoded delimiters for
non-word characters `[^a-zA-Z0-9_]`:

```bash
SPLUNK_DISCOVERY_RECEIVERS_receiver_x2d_type_x2f_receiver_x2d_name_CONFIG_field_x3a__x3a_subfield=value
SPLUNK_DISCOVERY_EXTENSIONS_observer_x2d_type_x2f_observer_x2d_name_CONFIG_field_x3a__x3a_subfield=value
SPLUNK_DISCOVERY_RECEIVERS_receiver_x2d_type_x2f_receiver_x2d_name_ENABLED=<true or false>
SPLUNK_DISCOVERY_EXTENSIONS_observer_x2d_type_x2f_observer_x2d_name_ENABLED=<true or false>
# Examples
SPLUNK_DISCOVERY_RECEIVERS_prometheus_simple_CONFIG_labels_x3a__x3a_my_label="my_username"
SPLUNK_DISCOVERY_RECEIVERS_prometheus_simple_ENABLED=true
SPLUNK_DISCOVERY_EXTENSIONS_docker_observer_CONFIG_endpoint="tcp://localhost:8080"
SPLUNK_DISCOVERY_EXTENSIONS_k8s_observer_ENABLED=false
```

The priority order for discovery config content from lowest to highest is:

1. `config.d/<receivers or extensions>/*.discovery.yaml` file content (lowest).
2. `config.d/properties.discovery.yaml` file content.
3. `SPLUNK_DISCOVERY_<xyz>` environment variables available to the collector process.
4. `--set splunk.discovery.<xyz>` commandline options (highest).
117 changes: 79 additions & 38 deletions internal/confmapprovider/discovery/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,47 +32,49 @@ import (
)

const (
typeService = "service"
typeReceiver = "receiver"
typeExporter = "exporter"
typeExtension = "extension"
typeProcessor = "processor"
typeDiscoveryObserver = "discovery.extension"
typeReceiverToDiscover = "discovery.receiver"
typeService = "service"
typeReceiver = "receiver"
typeExporter = "exporter"
typeExtension = "extension"
typeProcessor = "processor"
typeDiscoveryObserver = "discovery.extension"
typeReceiverToDiscover = "discovery.receiver"
typeDiscoveryProperties = "discovery.properties"
)

var (
defaultType = component.NewID("default")

discoveryDirRegex = fmt.Sprintf("[^%s]*", compilablePathSeparator)
serviceEntryRegex = regexp.MustCompile(fmt.Sprintf("%s%s*service\\.(yaml|yml)$", discoveryDirRegex, compilablePathSeparator))
configDirRootRegex = fmt.Sprintf("^[^%s]*", pathSeparatorForCharacterRange)
serviceEntryRegex = regexp.MustCompile(fmt.Sprintf("%s[%s]?service\\.(yaml|yml)$", configDirRootRegex, pathSeparatorForCharacterRange))
discoveryPropertiesEntryRegex = regexp.MustCompile(fmt.Sprintf("%s[%s]?properties\\.discovery\\.(yaml|yml)$", configDirRootRegex, pathSeparatorForCharacterRange))

_, exporterEntryRegex = dirAndEntryRegex("exporters")
extensionsDirRegex, extensionEntryRegex = dirAndEntryRegex("extensions")
discoveryObserverEntryRegex = regexp.MustCompile(fmt.Sprintf("%s%s[^%s]*\\.discovery\\.(yaml|yml)$", extensionsDirRegex, compilablePathSeparator, compilablePathSeparator))
discoveryObserverEntryRegex = regexp.MustCompile(fmt.Sprintf("%s[%s][^%s]*\\.discovery\\.(yaml|yml)$", extensionsDirRegex, pathSeparatorForCharacterRange, pathSeparatorForCharacterRange))

_, processorEntryRegex = dirAndEntryRegex("processors")
receiversDirRegex, receiverEntryRegex = dirAndEntryRegex("receivers")
receiverToDiscoverEntryRegex = regexp.MustCompile(fmt.Sprintf("%s%s[^%s]*\\.discovery\\.(yaml|yml)$", receiversDirRegex, compilablePathSeparator, compilablePathSeparator))
receiverToDiscoverEntryRegex = regexp.MustCompile(fmt.Sprintf("%s[%s][^%s]*\\.discovery\\.(yaml|yml)$", receiversDirRegex, pathSeparatorForCharacterRange, pathSeparatorForCharacterRange))
)

// Config is a model for stitching together the final Collector configuration with additional discovery component
// fields for use w/ discovery mode (not yet implemented). It allows individual yaml files to be added to a config.d
// directory and be sourced in the final config such that small changes don't require a central configuration file,
// and possible eliminates the need for one overall (still in design).
// fields for use w/ discovery mode. It allows individual yaml files to be added to a config.d directory and
// be sourced in the final config such that small changes don't apply to a central configuration file,
// and possibly eliminates the need for one overall (still in design and dependent on aliasing and array insertion operators).
type Config struct {
logger *zap.Logger
// Service is for pipelines and final settings
// Service is for pipelines and final settings.
// It must be in the root config directory and named "service.yaml"
Service ServiceEntry
// Exporters is a map of exporters to use in final config.
// They must be in `config.d/exporters` directory.
Exporters map[component.ID]ExporterEntry
// Extensions is a map of extensions to use in final config.
// They must be in `config.d/extensions` directory.
Extensions map[component.ID]ExtensionEntry
// DiscoveryObservers is a map of observer extensions to use in discovery,
// overriding the default settings. They must be in `config.d/extensions` directory
// and end with ".discovery.yaml".
// DiscoveryObservers is a map of observer extensions to use in discovery.
// They must be in `config.d/extensions` directory and end with ".discovery.yaml".
DiscoveryObservers map[component.ID]ExtensionEntry
// Processors is a map of extensions to use in final config.
// They must be in `config.d/processors` directory.
Expand All @@ -84,6 +86,10 @@ type Config struct {
// underlying discovery receiver. They must be in `config.d/receivers` directory and
// end with ".discovery.yaml".
ReceiversToDiscover map[component.ID]ReceiverToDiscoverEntry
// DiscoveryProperties is a mapping of discovery properties to their values for
// configuring discovery mode components.
// It must be in the root config directory and named "properties.discovery.yaml".
DiscoveryProperties PropertiesEntry
}

func NewConfig(logger *zap.Logger) *Config {
Expand All @@ -96,12 +102,13 @@ func NewConfig(logger *zap.Logger) *Config {
Processors: map[component.ID]ProcessorEntry{},
Receivers: map[component.ID]ReceiverEntry{},
ReceiversToDiscover: map[component.ID]ReceiverToDiscoverEntry{},
DiscoveryProperties: PropertiesEntry{Entry{}},
}
}

func dirAndEntryRegex(dirName string) (*regexp.Regexp, *regexp.Regexp) {
dirRegex := regexp.MustCompile(fmt.Sprintf("%s%s*%s", discoveryDirRegex, compilablePathSeparator, dirName))
entryRegex := regexp.MustCompile(fmt.Sprintf("%s%s[^%s]*\\.(yaml|yml)$", dirRegex, compilablePathSeparator, compilablePathSeparator))
dirRegex := regexp.MustCompile(fmt.Sprintf("%s[%s]*%s", configDirRootRegex, pathSeparatorForCharacterRange, dirName))
entryRegex := regexp.MustCompile(fmt.Sprintf("%s[%s][^%s]*\\.(yaml|yml)$", dirRegex, pathSeparatorForCharacterRange, pathSeparatorForCharacterRange))
return dirRegex, entryRegex
}

Expand Down Expand Up @@ -210,6 +217,16 @@ func (ReceiverToDiscoverEntry) ErrorF(path string, err error) error {
return errorF(typeReceiverToDiscover, path, err)
}

var _ entryType = (*PropertiesEntry)(nil)

type PropertiesEntry struct {
Entry `yaml:",inline"`
}

func (PropertiesEntry) ErrorF(path string, err error) error {
return errorF(typeDiscoveryProperties, path, err)
}

// Load will walk the file tree from the configDPath root, loading the component
// files as they are discovered, determined by their parent directory and filename.
func (c *Config) Load(configDPath string) error {
Expand Down Expand Up @@ -237,6 +254,11 @@ func (c *Config) LoadFS(dirfs fs.FS) error {
// and unmarshal to the underlying ServiceEntry
tmpSEMap := map[string]ServiceEntry{typeService: c.Service}
return loadEntry(typeService, dirfs, path, tmpSEMap)
case isDiscoveryPropertiesEntryPath(path):
// c.DiscoveryProperties is not a map[string]PropertiesEntry, so we form a tmp
// and unmarshal to the underlying PropertiesEntry
tmpDPMap := map[string]PropertiesEntry{typeDiscoveryProperties: c.DiscoveryProperties}
return loadEntry(typeDiscoveryProperties, dirfs, path, tmpDPMap)
case isExporterEntryPath(path):
return loadEntry(typeExporter, dirfs, path, c.Exporters)
case isExtensionEntryPath(path):
Expand Down Expand Up @@ -265,6 +287,7 @@ func (c *Config) LoadFS(dirfs fs.FS) error {
c.Exporters = nil
c.Processors = nil
c.Extensions = nil
c.DiscoveryProperties = PropertiesEntry{nil}
}
return err
}
Expand Down Expand Up @@ -335,6 +358,10 @@ func isReceiverToDiscoverEntryPath(path string) bool {
return receiverToDiscoverEntryRegex.MatchString(path)
}

func isDiscoveryPropertiesEntryPath(path string) bool {
return discoveryPropertiesEntryRegex.MatchString(path)
}

func loadEntry[K keyType, V entryType](componentType string, fs fs.FS, path string, target map[K]V) error {
tmpDest := map[K]V{}

Expand All @@ -351,16 +378,17 @@ func loadEntry[K keyType, V entryType](componentType string, fs fs.FS, path stri
return nil
}

if componentType == typeService {
// Shallow entry case where resulting entry is not a map[component.ID]Entry
if componentType == typeService || componentType == typeDiscoveryProperties {
// set directly on target and exit
typeServiceK, err := stringToKeyType(componentType, componentID)
typeShallowK, err := stringToKeyType(componentType, componentID)
if err != nil {
return err
}
serviceEntry := target[typeServiceK].Self()
tmpDstSM := tmpDest[typeServiceK].ToStringMap()
shallowEntry := target[typeShallowK].Self()
tmpDstSM := tmpDest[typeShallowK].ToStringMap()
for k, v := range tmpDstSM {
serviceEntry[keyTypeToString(k)] = v
shallowEntry[keyTypeToString(k)] = v
}
return nil
}
Expand All @@ -381,11 +409,14 @@ func unmarshalEntry[K keyType, V entryType](componentType string, fs fs.FS, path

var unmarshalDst any = dst

// service is map[string]ServiceEntry{typeService: ServiceEntry} and we want dst to be &ServiceEntry
if componentType == typeService {
var s any = typeService
// service key is always string so this type assertion is safe
se := (*dst)[s.(K)]
// shallow cases where dst is map[string]Entry{<typeEntry>: <Entry>} but we want it to be &<Entry>
if componentType == typeService || componentType == typeDiscoveryProperties {
var shallowType any = typeService
if componentType == typeDiscoveryProperties {
shallowType = typeDiscoveryProperties
}
// key is always string so this type assertion is safe
se := (*dst)[shallowType.(K)]
unmarshalDst = &se
}

Expand All @@ -394,12 +425,20 @@ func unmarshalEntry[K keyType, V entryType](componentType string, fs fs.FS, path
return
}

if componentType == typeService {
// reset map[string]ServiceEntry dst w/ unmarshalled ServiceEntry and return
var tService any = typeService
var serviceEntry any = *(unmarshalDst.(*ServiceEntry))
(*dst)[tService.(K)] = serviceEntry.(V)
return tService.(K), nil
if componentType == typeService || componentType == typeDiscoveryProperties {
var shallowType any
var entry any
// reset map[string]<EntryType> dst w/ unmarshalled Entry and return
switch componentType {
case typeService:
shallowType = typeService
entry = *(unmarshalDst.(*ServiceEntry))
case typeDiscoveryProperties:
shallowType = typeDiscoveryProperties
entry = *(unmarshalDst.(*PropertiesEntry))
}
(*dst)[shallowType.(K)] = entry.(V)
return shallowType.(K), nil
}

entry := *(unmarshalDst.(*map[K]V))
Expand Down Expand Up @@ -490,11 +529,13 @@ func keyTypeToString[K keyType](key K) string {
return ret
}

var compilablePathSeparator = func() string {
// pathSeparatorForCharacterRange will return the platform specific path separator for use in [%s] or [^%s]
// range template string.
var pathSeparatorForCharacterRange = func() string {
if os.PathSeparator == '\\' {
// fs.Stat doesn't use os.PathSeparator so accept '/' as well.
// TODO: determine if we even need anything but "/"
return "(\\\\|/)"
return "\\\\/"
}
return string(os.PathSeparator)
}()
Expand Down
Loading

0 comments on commit efc86c3

Please sign in to comment.