Add V1 controller and spec support to the shipper

cmd/cmd.go

@@ -28,6 +28,7 @@ func NewCommand() *cobra.Command {
 	cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("v"))
 	cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("e"))
 	cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("d"))
+	cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("E"))
 	cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("environment"))
 
 	run := runCmd()

config/config.go

@@ -7,8 +7,6 @@ package config
 import (
 	"flag"
 	"fmt"
-	"io/ioutil"
-	"path/filepath"
 
 	"github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/logp"
@@ -23,6 +21,7 @@ const (
 var (
 	configPath     string
 	configFilePath string
+	overwrites     = config.SettingFlag(nil, "E", "Configuration overwrite")
 )
 
 // A lot of the code here is the same as what's in elastic-agent, but it lives in an internal/ library
@@ -34,46 +33,57 @@ func init() {
 
 //ShipperConfig defines the options present in the config file
 type ShipperConfig struct {
-	Log     logp.Config       `config:"logging"`
 	TLS     bool              `config:"tls"`
 	Cert    string            `config:"cert"`       //TLS cert file, if TLS is enabled
 	Key     string            `config:"key"`        //TLS Keyfile, if specified
 	Port    int               `config:"port"`       //Port to listen on
 	Monitor monitoring.Config `config:"monitoring"` //Queue monitoring settings
 }
 
-// ReadConfig returns the populated config from the specified path
-func ReadConfig() (ShipperConfig, error) {
-	path := configFile()
-
-	contents, err := ioutil.ReadFile(path)
+func ReadConfigFromString(in string) (ShipperConfig, error) {
+	raw, err := config.NewConfigWithYAML([]byte(in), "")
 	if err != nil {
-		return ShipperConfig{}, fmt.Errorf("error reading input file %s: %w", path, err)
+		return ShipperConfig{}, fmt.Errorf("error reading config from yaml: %w", err)
 	}
 
-	raw, err := config.NewConfigWithYAML(contents, "")
+	//merge the -E flags with the supplied config
+	merged, err := config.MergeConfigs(raw, overwrites)
 	if err != nil {
-		return ShipperConfig{}, fmt.Errorf("error reading config from yaml: %w", err)
+		return ShipperConfig{}, fmt.Errorf("error merging -E flags with existing config: %w", err)
 	}
-	// systemd environment will send us to stdout environment, which we want
-	config := ShipperConfig{
+
+	baseConfig := ShipperConfig{
 		Port:    defaultPort,
-		Log:     logp.DefaultConfig(logp.SystemdEnvironment),
 		Monitor: monitoring.DefaultConfig(),
 	}
-	err = raw.Unpack(&config)
+
+	// Doing this to wrap the current config used by the elastic-agent, not really sure what the final config will be for the shipper.
+	wrapper := struct {
+		Shipper ShipperConfig `struct:"shipper"`
+	}{
+		Shipper: baseConfig,
+	}
+
+	err = merged.Unpack(&wrapper)
 	if err != nil {
-		return config, fmt.Errorf("error unpacking shipper config: %w", err)
+		return baseConfig, fmt.Errorf("error unpacking shipper config: %w", err)
 	}
-	return config, nil
+	return wrapper.Shipper, nil
 }
 
-func configFile() string {
-	if configFilePath == "" || configFilePath == defaultConfigName {
-		return filepath.Join(configPath, defaultConfigName)
+// GetLoggingConfig returns the logging config we get from the CLI
+func GetLoggingConfig() (logp.Config, error) {
+	wrapper := struct {
+		Logging logp.Config `struct:"logging"`
+	}{
+		Logging: logp.DefaultConfig(logp.DefaultEnvironment),
 	}
-	if filepath.IsAbs(configFilePath) {
-		return configFilePath
+
+	// I'm sort of assuming that the agent will always configure logging over the CLI, since that's what it looks like from testing.
+	err := overwrites.Unpack(&wrapper)
+	if err != nil {
+		return logp.Config{}, fmt.Errorf("error unpacking logging config: %w", err)
 	}
-	return filepath.Join(configPath, configFilePath)
+
+	return wrapper.Logging, nil
 }

config/config_test.go

@@ -0,0 +1,56 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConfigIngest(t *testing.T) {
+	in := `
+shipper:
+    port: 50052
+    tls: false
+    #cert: # path to TLS cert
+    #key: # path to TLS keyfile
+
+    #log level
+    logging.level: debug
+    logging.selectors: ["*"]
+    logging.to_stderr: true
+
+    queue:
+      test: #There is no actual "test" queue type, remove this later.
+        events: 512
+
+    monitoring:
+      enabled: true
+      interval: 5s
+      log: true
+      http:
+        enabled: true
+        host: "not_localhost"
+        port: 8484
+        name: "queue"
+
+    outputs:
+      default:
+        type: elasticsearch
+        hosts: [127.0.0.1:9200]
+        api-key: "example-key"
+        # username: "elastic"
+        # password: "changeme"
+`
+
+	out, err := ReadConfigFromString(in)
+	require.NoError(t, err)
+	t.Logf("Got config: %#v", out)
+	assert.Equal(t, out.Port, 50052)
+	assert.Equal(t, out.Monitor.ExpvarOutput.Host, "not_localhost")
+	assert.Equal(t, out.Monitor.ExpvarOutput.Port, 8484)
+}

dev-tools/set_docs_version

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+import argparse
+from subprocess import check_call
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Used to set the current docs version. Doesn't commit changes.")
+    parser.add_argument("version",
+                        help="The new docs version")
+    args = parser.parse_args()
+    version = args.version
+
+    # make sure we have no dirty files in this branch (might throw off `make update`)
+    check_call("git clean -dfx", shell=True)
+
+    # edit the file
+    with open("docs/version.asciidoc", "r") as f:
+        lines = f.readlines()
+    for i, line in enumerate(lines):
+        if line.startswith(":stack-version:"):
+            lines[i] = ":stack-version: {}\n".format(version)
+    with open("docs/version.asciidoc", "w") as f:
+        f.writelines(lines)
+
+    #check_call("make update", shell=True)
+
+
+if __name__ == "__main__":
+    main()

dev-tools/set_version

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import re
+import sys
+from subprocess import check_call
+
+vendored_elastic_agent = os.path.normpath(
+    "vendor/github.com/elastic/elastic-agent-shipper")
+
+
+goversion_var_elastic_agent = "defaultAgentVersion"		# version in agent
+goversion_var = "appVersion"						# version for custom beats
+
+
+def get_rootfolder():
+    vendored_elastic_agent = os.path.normpath(
+        "vendor/github.com/elastic/elastic-agent-shipper")
+    script_directory = os.path.abspath(
+        os.path.dirname(os.path.realpath(__file__)))
+    index = script_directory.find(vendored_elastic_agent)
+    if index > 0:
+        # Community beat detected, version files are stored at the root folder of the project
+        return os.path.abspath(script_directory[:index])
+
+    # Libbeat detected
+    return os.path.dirname(script_directory)
+
+
+def replace_in_file(filename, varname, version):
+    new_lines = []
+    with open(filename, 'r') as f:
+        for line in f:
+            if line.startswith("const " + varname):
+                new_lines.append('const {} = "{}"\n'.format(varname, version))
+            else:
+                new_lines.append(line)
+
+    with open(filename, 'w') as f:
+        for line in new_lines:
+            f.write(line)
+    print("Set version {} in file {}".format(version, filename))
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Used to set the current version. Doesn't commit changes.")
+    parser.add_argument("version",
+                        help="The new version")
+    args = parser.parse_args()
+    version = args.version
+
+    is_libbeat = vendored_elastic_agent not in os.path.realpath(__file__)
+    if is_libbeat:
+        goversion_filepath = os.path.join(
+            get_rootfolder(), "version", "version.go")
+        go_var = goversion_var_elastic_agent
+    else:
+        goversion_filepath = os.path.join(get_rootfolder(), "version.go")
+        go_var = goversion_var
+
+    # Create version.go and version.yml files
+    replace_in_file(goversion_filepath, go_var, version)
+
+
+if __name__ == "__main__":
+    main()

docs/version.asciidoc

@@ -0,0 +1,8 @@
+:stack-version: 8.4.0
+:doc-branch: main
+:go-version: 1.17.10
+:release-state: unreleased
+:python: 3.7
+:docker: 1.12
+:docker-compose: 1.11
+:libpcap: 0.8

elastic-agent-shipper.yml

@@ -2,18 +2,14 @@
 
 # This file is an example configuration file highlighting only the most common
 # options.
+# Since we're now running under the agent controller, the shipper won't actually read this file, but it serves as an example for what config the shipper will currently expect
 
 # The gRPC port that the shipper will listen on
 port: 50052
 tls: false
 #cert: # path to TLS cert
 #key: # path to TLS keyfile
 
-#log level
-logging.level: debug
-logging.selectors: ["*"]
-logging.to_stderr: true
-
 queue:
   test: #There is no actual "test" queue type, remove this later.
     events: 512
@@ -25,5 +21,5 @@ monitoring:
   http:
     enabled: true
     host: "localhost"
-    port: 8282
+    port: 8383
     name: "queue"

go.mod

@@ -5,36 +5,32 @@ go 1.17
 require (
 	github.com/elastic/elastic-agent-libs v0.2.5
 	github.com/spf13/cobra v1.3.0
-	google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8
+	google.golang.org/genproto v0.0.0-20220615141314-f1464d18c36b
 	google.golang.org/grpc v1.47.0
 	google.golang.org/protobuf v1.28.0
 )
 
 require (
-	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20220603155004-ac7e079a9403
+	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20220615180530-c614236230bf
 	github.com/elastic/elastic-agent v0.0.0-20220330154707-da786a47a0c5
+	github.com/elastic/elastic-agent-client/v7 v7.0.0-20220607160924-1a71765a8bbe
 	github.com/magefile/mage v1.13.0
 	github.com/stretchr/testify v1.7.0
 	go.elastic.co/go-licence-detector v0.5.0
-	golang.org/x/net v0.0.0-20220531201128-c960675eff93
+	golang.org/x/net v0.0.0-20220615171555-694bf12d69de
 )
 
 require (
 	github.com/akavel/rsrc v0.10.2 // indirect
 	github.com/armon/go-radix v1.0.0 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dlclark/regexp2 v1.4.0 // indirect
-	github.com/dop251/goja v0.0.0-20220516123900-4418d4575a41 // indirect
-	github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d // indirect
-	github.com/elastic/elastic-agent-client/v7 v7.0.0-20220524131921-43bacbeec516 // indirect
 	github.com/elastic/go-licenser v0.4.0 // indirect
 	github.com/elastic/go-structform v0.0.9 // indirect
-	github.com/elastic/go-sysinfo v1.7.1 // indirect
+	github.com/elastic/go-sysinfo v1.8.0 // indirect
 	github.com/elastic/go-ucfg v0.8.5 // indirect
 	github.com/elastic/go-windows v1.0.1 // indirect
 	github.com/fatih/color v1.13.0 // indirect
-	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/gobuffalo/here v0.6.0 // indirect
 	github.com/gofrs/uuid v4.2.0+incompatible // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
@@ -54,7 +50,6 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect
 	github.com/sergi/go-diff v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
@@ -66,12 +61,12 @@ require (
 	go.uber.org/multierr v1.8.0 // indirect
 	go.uber.org/zap v1.21.0 // indirect
 	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
-	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
+	golang.org/x/sys v0.0.0-20220615212526-ff8c426e75fd // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.1.10 // indirect
-	golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect
+	golang.org/x/tools v0.1.11 // indirect
+	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
@@ -81,4 +76,5 @@ require (
 replace (
 	github.com/dop251/goja => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20
 	github.com/dop251/goja_nodejs => github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6
+	github.com/elastic/elastic-agent => ../elastic-agent
 )