Skip to content

Commit

Permalink
Cherry-pick #17894 to 7.x: Agent push ECS meta to fleet (#17921)
Browse files Browse the repository at this point in the history
* [Elastic-Agent] Agent push ECS meta to fleet (#17894)

[Elastic-Agent] Agent push ECS meta to fleet (#17894)

* [Elastic-Agent] ECS compliant Elastic agent metadata sent to fleet  (#18006)

* use meta object

* changelog

* agent.* => elastic.agent.*
  • Loading branch information
michalpristas authored May 4, 2020
1 parent 6478f45 commit a1ee6fd
Show file tree
Hide file tree
Showing 11 changed files with 334 additions and 159 deletions.
1 change: 1 addition & 0 deletions x-pack/elastic-agent/CHANGELOG.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
- Display the stability of the agent at enroll and start. {pull}17336[17336]
- Expose stream.* variables in events {pull}17468[17468]
- Monitoring configuration reloadable {pull}17855[17855]
- Pack ECS metadata to request payload send to fleet {pull}17894[17894]
- Allow CLI overrides of paths {pull}17781[17781]
- Enable Filebeat input: S3, Azureeventhub, cloudfoundry, httpjson, netflow, o365audit. {pull}17909[17909]
- Use data subfolder as default for process logs {pull}17960[17960]
2 changes: 1 addition & 1 deletion x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ func (c *EnrollCmd) Execute() error {

metadata, err := metadata()
if err != nil {
return errors.New(err, "acquiring hostname")
return errors.New(err, "acquiring metadata failed")
}

r := &fleetapi.EnrollRequest{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ package filters

import (
"fmt"
"runtime"

"github.com/Masterminds/semver"

Expand All @@ -15,8 +14,6 @@ import (
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/boolexp"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
"github.com/elastic/go-sysinfo"
)

const (
Expand All @@ -25,24 +22,6 @@ const (
validateVersionFuncName = "validate_version"
)

// List of variables available to be used in constraint definitions.
const (
// `agent.id` is a generated (in standalone) or assigned (in fleet) agent identifier.
agentIDKey = "agent.id"
// `agent.version` specifies current version of an agent.
agentVersionKey = "agent.version"
// `host.architecture` defines architecture of a host (e.g. x86_64, arm, ppc, mips).
hostArchKey = "host.architecture"
// `os.family` defines a family of underlying operating system (e.g. redhat, debian, freebsd, windows).
osFamilyKey = "os.family"
// `os.kernel` specifies current version of a kernel in a semver format.
osKernelKey = "os.kernel"
// `os.platform` specifies platform agent is running on (e.g. centos, ubuntu, windows).
osPlatformKey = "os.platform"
// `os.version` specifies version of underlying operating system (e.g. 10.12.6).
osVersionKey = "os.version"
)

var (
boolexpVarStore *constraintVarStore
boolexpMethodsRegs *boolexp.MethodsReg
Expand Down Expand Up @@ -245,30 +224,20 @@ func newVarStore() (*constraintVarStore, error) {
}

func initVarStore(store *constraintVarStore) error {
sysInfo, err := sysinfo.Host()
agentInfo, err := info.NewAgentInfo()
if err != nil {
return err
}

agentInfo, err := info.NewAgentInfo()
meta, err := agentInfo.ECSMetadataFlatMap()
if err != nil {
return err
return errors.New(err, "failed to gather host metadata")
}

info := sysInfo.Info()

// Agent
store.vars[agentIDKey] = agentInfo.AgentID()
store.vars[agentVersionKey] = release.Version()

// Host
store.vars[hostArchKey] = info.Architecture

// Operating system
store.vars[osFamilyKey] = runtime.GOOS
store.vars[osKernelKey] = info.KernelVersion
store.vars[osPlatformKey] = info.OS.Family
store.vars[osVersionKey] = info.OS.Version
// keep existing, overwrite gathered
for k, v := range meta {
store.vars[k] = v
}

return nil
}
8 changes: 4 additions & 4 deletions x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,16 +179,16 @@ func (f *fleetGateway) execute(ctx context.Context) (*fleetapi.CheckinResponse,
// get events
ee, ack := f.reporter.Events()

var metaData map[string]interface{}
if m, err := metadata(); err == nil {
metaData = m
ecsMeta, err := metadata()
if err != nil {
f.log.Error(errors.New("failed to load metadata", err))
}

// checkin
cmd := fleetapi.NewCheckinCmd(f.agentInfo, f.client)
req := &fleetapi.CheckinRequest{
Events: ee,
Metadata: metaData,
Metadata: ecsMeta,
}

resp, err := cmd.Execute(ctx, req)
Expand Down
202 changes: 202 additions & 0 deletions x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,202 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

package info

import (
"fmt"
"os"
"runtime"
"strings"

"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
"github.com/elastic/go-sysinfo"
"github.com/elastic/go-sysinfo/types"
)

// ECSMeta is a collection of agent related metadata in ECS compliant object form.
type ECSMeta struct {
Agent *AgentECSMeta `json:"elastic.agent"`
Host *HostECSMeta `json:"host"`
OS *SystemECSMeta `json:"os"`
}

// AgentECSMeta is a collection of agent metadata in ECS compliant object form.
type AgentECSMeta struct {
// ID is a generated (in standalone) or assigned (in fleet) agent identifier.
ID string `json:"id"`
// Version specifies current version of an agent.
Version string `json:"version"`
}

// SystemECSMeta is a collection of operating system metadata in ECS compliant object form.
type SystemECSMeta struct {
// Family defines a family of underlying operating system (e.g. redhat, debian, freebsd, windows).
Family string `json:"family"`
// Kernel specifies current version of a kernel in a semver format.
Kernel string `json:"kernel"`
// Platform specifies platform agent is running on (e.g. centos, ubuntu, windows).
Platform string `json:"platform"`
// Version specifies version of underlying operating system (e.g. 10.12.6).
Version string `json:"version"`
// Name is a operating system name.
// Currently we just normalize the name (i.e. macOS, Windows, Linux). See https://www.elastic.co/guide/en/ecs/current/ecs-html
Name string `json:"name"`
// Full is an operating system name, including the version or code name.
FullName string `json:"full"`
}

// HostECSMeta is a collection of host metadata in ECS compliant object form.
type HostECSMeta struct {
// Arch defines architecture of a host (e.g. x86_64, arm, ppc, mips).
Arch string `json:"architecture"`
// Hostname specifies hostname of the host.
Hostname string `json:"hostname"`
// Name specifies hostname of the host.
Name string `json:"name"`
// ID is a Unique host id.
// As hostname is not always unique, use values that are meaningful in your environment.
ID string `json:"id"`
// IP is Host ip addresses.
// Note: this field should contain an array of values.
IP []string `json:"ip"`
// Mac is Host mac addresses.
// Note: this field should contain an array of values.
MAC []string `json:"mac"`
}

// List of variables available to be used in constraint definitions.
const (
// `agent.id` is a generated (in standalone) or assigned (in fleet) agent identifier.
agentIDKey = "agent.id"
// `agent.version` specifies current version of an agent.
agentVersionKey = "agent.version"

// `os.family` defines a family of underlying operating system (e.g. redhat, debian, freebsd, windows).
osFamilyKey = "os.family"
// `os.kernel` specifies current version of a kernel in a semver format.
osKernelKey = "os.kernel"
// `os.platform` specifies platform agent is running on (e.g. centos, ubuntu, windows).
osPlatformKey = "os.platform"
// `os.version` specifies version of underlying operating system (e.g. 10.12.6).
osVersionKey = "os.version"
// `os.name` is a operating system name.
// Currently we just normalize the name (i.e. macOS, Windows, Linux). See https://www.elastic.co/guide/en/ecs/current/ecs-os.html
osNameKey = "os.name"
// `os.full` is an operating system name, including the version or code name.
osFullKey = "os.full"

// `host.architecture` defines architecture of a host (e.g. x86_64, arm, ppc, mips).
hostArchKey = "host.architecture"
// `host.hostname` specifies hostname of the host.
hostHostnameKey = "host.hostname"
// `host.name` specifies hostname of the host.
hostNameKey = "host.name"
// `host.id` is a Unique host id.
// As hostname is not always unique, use values that are meaningful in your environment.
hostIDKey = "host.id"
// `host.ip` is Host ip addresses.
// Note: this field should contain an array of values.
hostIPKey = "host.ip"
// `host.mac` is Host mac addresses.
// Note: this field should contain an array of values.
hostMACKey = "host.mac"
)

// ECSMetadata returns an agent ECS compliant metadata.
func (i *AgentInfo) ECSMetadata() (*ECSMeta, error) {
hostname, err := os.Hostname()
if err != nil {
return nil, err
}

sysInfo, err := sysinfo.Host()
if err != nil {
return nil, err
}

info := sysInfo.Info()

return &ECSMeta{
Agent: &AgentECSMeta{
ID: i.agentID,
Version: release.Version(),
},
Host: &HostECSMeta{
Arch: info.Architecture,
Hostname: hostname,
Name: hostname,
ID: info.UniqueID,
IP: info.IPs,
MAC: info.MACs,
},

// Operating system
OS: &SystemECSMeta{
Family: runtime.GOOS,
Kernel: info.KernelVersion,
Platform: info.OS.Family,
Version: info.OS.Version,
Name: info.OS.Name,
FullName: getFullOSName(info),
},
}, nil
}

// ECSMetadataFlatMap returns an agent ECS compliant metadata in a form of flattened map.
func (i *AgentInfo) ECSMetadataFlatMap() (map[string]interface{}, error) {
hostname, err := os.Hostname()
if err != nil {
return nil, err
}

// TODO: remove these values when kibana migrates to ECS
meta := make(map[string]interface{})

sysInfo, err := sysinfo.Host()
if err != nil {
return nil, err
}

info := sysInfo.Info()

// Agent
meta[agentIDKey] = i.agentID
meta[agentVersionKey] = release.Version()

// Host
meta[hostArchKey] = info.Architecture
meta[hostHostnameKey] = hostname
meta[hostNameKey] = hostname
meta[hostIDKey] = info.UniqueID
meta[hostIPKey] = fmt.Sprintf("[%s]", strings.Join(info.IPs, ","))
meta[hostMACKey] = fmt.Sprintf("[%s]", strings.Join(info.MACs, ","))

// Operating system
meta[osFamilyKey] = runtime.GOOS
meta[osKernelKey] = info.KernelVersion
meta[osPlatformKey] = info.OS.Family
meta[osVersionKey] = info.OS.Version
meta[osNameKey] = info.OS.Name
meta[osFullKey] = getFullOSName(info)

return meta, nil
}

func getFullOSName(info types.HostInfo) string {
var sb strings.Builder
sb.WriteString(info.OS.Name)
if codeName := info.OS.Codename; codeName != "" {
sb.WriteString(" ")
sb.WriteString(codeName)
}

if version := info.OS.Version; version != "" {
sb.WriteString("(")
sb.WriteString(version)
sb.WriteString(")")
}

return sb.String()
}
21 changes: 10 additions & 11 deletions x-pack/elastic-agent/pkg/agent/application/local_meta.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,20 @@
package application

import (
"os"
"runtime"

"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
)

func metadata() (map[string]interface{}, error) {
hostname, err := os.Hostname()
func metadata() (*info.ECSMeta, error) {
agentInfo, err := info.NewAgentInfo()
if err != nil {
return nil, err
}

return map[string]interface{}{
"platform": runtime.GOOS,
"version": release.Version(),
"host": hostname,
}, nil
meta, err := agentInfo.ECSMetadata()
if err != nil {
return nil, errors.New(err, "failed to gather host metadata")
}

return meta, nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,12 @@ func (b *Monitor) Reload(rawConfig *config.Config) error {
return err
}

b.config = cfg.MonitoringConfig
if cfg == nil || cfg.MonitoringConfig == nil {
b.config = &monitoringConfig.MonitoringConfig{}
} else {
b.config = cfg.MonitoringConfig
}

return nil
}

Expand Down
5 changes: 3 additions & 2 deletions x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,16 @@ import (
"net/http"
"time"

"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
)

const checkingPath = "/api/ingest_manager/fleet/agents/%s/checkin"

// CheckinRequest consists of multiple events reported to fleet ui.
type CheckinRequest struct {
Events []SerializableEvent `json:"events"`
Metadata map[string]interface{} `json:"local_metadata,omitempty"`
Events []SerializableEvent `json:"events"`
Metadata *info.ECSMeta `json:"local_metadata,omitempty"`
}

// SerializableEvent is a representation of the event to be send to the Fleet API via the checkin
Expand Down
Loading

0 comments on commit a1ee6fd

Please sign in to comment.