[Ingest Manager] Agent includes pgp file

x-pack/elastic-agent/magefile.go

@@ -10,10 +10,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
@@ -40,6 +44,7 @@ const (
 	buildDir       = "build"
 	metaDir        = "_meta"
 	snapshotEnv    = "SNAPSHOT"
+	devEnv         = "DEV"
 	configFile     = "elastic-agent.yml"
 	agentDropPath  = "AGENT_DROP_PATH"
 )
@@ -620,6 +625,23 @@ func buildVars() map[string]string {
 	isSnapshot, _ := os.LookupEnv(snapshotEnv)
 	vars["github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release.snapshot"] = isSnapshot
 
+	fetchPgp := true
+	if isDevFlag, devFound := os.LookupEnv(devEnv); devFound {
+		if isDev, err := strconv.ParseBool(isDevFlag); err == nil && isDev {
+			vars["github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release.allowEmptyPgp"] = "true"
+			fetchPgp = false
+		}
+	}
+	fmt.Println("fetching pgp", fetchPgp)
+
+	if fetchPgp {
+		pgp, err := loadPGPFromWeb()
+		if err != nil {
+			panic(err)
+		}
+		vars["github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release.escPgp"] = string(pgp)
+	}
+
 	return vars
 }
 
@@ -628,3 +650,23 @@ func injectBuildVars(m map[string]string) {
 		m[k] = v
 	}
 }
+
+func loadPGPFromWeb() (string, error) {
+	const publicKeyURI = "https://artifacts.elastic.co/GPG-KEY-elasticsearch"
+
+	resp, err := http.Get(publicKeyURI)
+	if err != nil {
+		return "", fmt.Errorf("failed loading public key: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return "", fmt.Errorf("call to '%s' returned unsuccessful status code: %d", publicKeyURI, resp.StatusCode)
+	}
+
+	rawPgp, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	return url.PathEscape(string(rawPgp)), nil
+}

x-pack/elastic-agent/pkg/agent/application/stream.go

@@ -19,6 +19,7 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/monitoring"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/server"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/state"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
 )
 
 type operatorStream struct {
@@ -56,7 +57,8 @@ func streamFactory(ctx context.Context, cfg *configuration.SettingsConfig, srv *
 
 func newOperator(ctx context.Context, log *logger.Logger, id routingKey, config *configuration.SettingsConfig, srv *server.Server, r state.Reporter, m monitoring.Monitor) (*operation.Operator, error) {
 	fetcher := downloader.NewDownloader(log, config.DownloadConfig)
-	verifier, err := downloader.NewVerifier(log, config.DownloadConfig)
+	allowEmptyPgp, pgp := release.PGP()
+	verifier, err := downloader.NewVerifier(log, config.DownloadConfig, allowEmptyPgp, pgp)
 	if err != nil {
 		return nil, errors.New(err, "initiating verifier")
 	}

x-pack/elastic-agent/pkg/artifact/config.go

@@ -30,10 +30,6 @@ type Config struct {
 	// Timeout: timeout for downloading package
 	Timeout time.Duration `json:"timeout" config:"timeout"`
 
-	// PgpFile: filepath to a public key used for verifying downloaded artifacts
-	// if not file is present elastic-agent will try to load public key from elastic.co website.
-	PgpFile string `json:"pgpfile" config:"pgpfile"`
-
 	// InstallPath: path to the directory containing installed packages
 	InstallPath string `yaml:"installPath" config:"install_path"`
 
@@ -52,7 +48,6 @@ func DefaultConfig() *Config {
 		SourceURI:       "https://artifacts.elastic.co/downloads/",
 		TargetDirectory: filepath.Join(dataPath, "downloads"),
 		Timeout:         30 * time.Second,
-		PgpFile:         filepath.Join(dataPath, "elastic.pgp"),
 		InstallPath:     filepath.Join(dataPath, "install"),
 	}
 }

x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go

@@ -14,7 +14,6 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"sync"
 
 	"golang.org/x/crypto/openpgp"
 
@@ -29,15 +28,22 @@ const (
 // Verifier verifies a downloaded package by comparing with public ASC
 // file from elastic.co website.
 type Verifier struct {
-	config   *artifact.Config
-	pgpBytes []byte
+	config        *artifact.Config
+	pgpBytes      []byte
+	allowEmptyPgp bool
 }
 
 // NewVerifier create a verifier checking downloaded package on preconfigured
 // location agains a key stored on elastic.co website.
-func NewVerifier(config *artifact.Config) (*Verifier, error) {
+func NewVerifier(config *artifact.Config, allowEmptyPgp bool, pgp []byte) (*Verifier, error) {
+	if len(pgp) == 0 && !allowEmptyPgp {
+		return nil, errors.New("expecting PGP but retrieved none", errors.TypeSecurity)
+	}
+
 	v := &Verifier{
-		config: config,
+		config:        config,
+		allowEmptyPgp: allowEmptyPgp,
+		pgpBytes:      pgp,
 	}
 
 	return v, nil
@@ -58,9 +64,10 @@ func (v *Verifier) Verify(programName, version string) (bool, error) {
 		// remove bits so they can be redownloaded
 		os.Remove(fullPath)
 		os.Remove(fullPath + ".sha512")
+		return isMatch, err
 	}
 
-	return isMatch, err
+	return v.verifyAsc(filename, fullPath)
 }
 
 func (v *Verifier) verifyHash(filename, fullPath string) (bool, error) {
@@ -106,15 +113,9 @@ func (v *Verifier) verifyHash(filename, fullPath string) (bool, error) {
 }
 
 func (v *Verifier) verifyAsc(filename, fullPath string) (bool, error) {
-	var err error
-	var pgpBytesLoader sync.Once
-
-	pgpBytesLoader.Do(func() {
-		err = v.loadPGP(v.config.PgpFile)
-	})
-
-	if err != nil {
-		return false, errors.New(err, "loading PGP")
+	if len(v.pgpBytes) == 0 {
+		// no pgp available skip verification process
+		return true, nil
 	}
 
 	ascBytes, err := v.getPublicAsc(filename)
@@ -153,18 +154,3 @@ func (v *Verifier) getPublicAsc(filename string) ([]byte, error) {
 
 	return b, nil
 }
-
-func (v *Verifier) loadPGP(file string) error {
-	var err error
-
-	if file == "" {
-		return errors.New("pgp file not specified for verifier", errors.TypeConfig)
-	}
-
-	v.pgpBytes, err = ioutil.ReadFile(file)
-	if err != nil {
-		return errors.New(err, errors.TypeFilesystem)
-	}
-
-	return nil
-}

x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go

@@ -59,7 +59,7 @@ func TestFetchVerify(t *testing.T) {
 	assert.NoError(t, err)
 
 	downloader := NewDownloader(config)
-	verifier, err := NewVerifier(config)
+	verifier, err := NewVerifier(config, true, nil)
 	assert.NoError(t, err)
 
 	// first download verify should fail:
@@ -157,7 +157,7 @@ func TestVerify(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	testVerifier, err := NewVerifier(config)
+	testVerifier, err := NewVerifier(config, true, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go

@@ -105,7 +105,7 @@ func TestVerify(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			testVerifier, err := NewVerifier(config)
+			testVerifier, err := NewVerifier(config, true, nil)
 			if err != nil {
 				t.Fatal(err)
 			}

x-pack/elastic-agent/pkg/artifact/download/http/verifier.go

@@ -16,7 +16,6 @@ import (
 	"os"
 	"path"
 	"strings"
-	"sync"
 
 	"golang.org/x/crypto/openpgp"
 
@@ -32,20 +31,27 @@ const (
 // Verifier verifies a downloaded package by comparing with public ASC
 // file from elastic.co website.
 type Verifier struct {
-	config   *artifact.Config
-	client   http.Client
-	pgpBytes []byte
+	config        *artifact.Config
+	client        http.Client
+	pgpBytes      []byte
+	allowEmptyPgp bool
 }
 
 // NewVerifier create a verifier checking downloaded package on preconfigured
 // location agains a key stored on elastic.co website.
-func NewVerifier(config *artifact.Config) (*Verifier, error) {
+func NewVerifier(config *artifact.Config, allowEmptyPgp bool, pgp []byte) (*Verifier, error) {
+	if len(pgp) == 0 && !allowEmptyPgp {
+		return nil, errors.New("expecting PGP but retrieved none", errors.TypeSecurity)
+	}
+
 	client := http.Client{Timeout: config.Timeout}
 	rtt := withHeaders(client.Transport, headers)
 	client.Transport = rtt
 	v := &Verifier{
-		config: config,
-		client: client,
+		config:        config,
+		client:        client,
+		allowEmptyPgp: allowEmptyPgp,
+		pgpBytes:      pgp,
 	}
 
 	return v, nil
@@ -71,9 +77,10 @@ func (v *Verifier) Verify(programName, version string) (bool, error) {
 		// remove bits so they can be redownloaded
 		os.Remove(fullPath)
 		os.Remove(fullPath + ".sha512")
+		return isMatch, err
 	}
 
-	return isMatch, err
+	return v.verifyAsc(programName, version)
 }
 
 func (v *Verifier) verifyHash(filename, fullPath string) (bool, error) {
@@ -120,15 +127,9 @@ func (v *Verifier) verifyHash(filename, fullPath string) (bool, error) {
 }
 
 func (v *Verifier) verifyAsc(programName, version string) (bool, error) {
-	var err error
-	var pgpBytesLoader sync.Once
-
-	pgpBytesLoader.Do(func() {
-		err = v.loadPGP(v.config.PgpFile)
-	})
-
-	if err != nil {
-		return false, errors.New(err, "loading PGP")
+	if len(v.pgpBytes) == 0 {
+		// no pgp available skip verification process
+		return true, nil
 	}
 
 	filename, err := artifact.GetArtifactName(programName, version, v.config.OS(), v.config.Arch())
@@ -202,33 +203,3 @@ func (v *Verifier) getPublicAsc(sourceURI string) ([]byte, error) {
 
 	return ioutil.ReadAll(resp.Body)
 }
-
-func (v *Verifier) loadPGP(file string) error {
-	var err error
-
-	if file == "" {
-		v.pgpBytes, err = v.loadPGPFromWeb()
-		return err
-	}
-
-	v.pgpBytes, err = ioutil.ReadFile(file)
-	if err != nil {
-		return errors.New(err, errors.TypeFilesystem, errors.M(errors.MetaKeyPath, file))
-	}
-
-	return nil
-}
-
-func (v *Verifier) loadPGPFromWeb() ([]byte, error) {
-	resp, err := v.client.Get(publicKeyURI)
-	if err != nil {
-		return nil, errors.New(err, "failed loading public key", errors.TypeNetwork, errors.M(errors.MetaKeyURI, publicKeyURI))
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		return nil, errors.New(fmt.Sprintf("call to '%s' returned unsuccessful status code: %d", publicKeyURI, resp.StatusCode), errors.TypeNetwork, errors.M(errors.MetaKeyURI, publicKeyURI))
-	}
-
-	return ioutil.ReadAll(resp.Body)
-}

x-pack/elastic-agent/pkg/artifact/download/localremote/verifier.go

@@ -17,26 +17,26 @@ import (
 
 // NewVerifier creates a downloader which first checks local directory
 // and then fallbacks to remote if configured.
-func NewVerifier(log *logger.Logger, config *artifact.Config) (download.Verifier, error) {
+func NewVerifier(log *logger.Logger, config *artifact.Config, allowEmptyPgp bool, pgp []byte) (download.Verifier, error) {
 	verifiers := make([]download.Verifier, 0, 3)
 
-	fsVer, err := fs.NewVerifier(config)
+	fsVer, err := fs.NewVerifier(config, allowEmptyPgp, pgp)
 	if err != nil {
 		return nil, err
 	}
 	verifiers = append(verifiers, fsVer)
 
 	// try snapshot repo before official
 	if release.Snapshot() {
-		snapshotVerifier, err := snapshot.NewVerifier(config)
+		snapshotVerifier, err := snapshot.NewVerifier(config, allowEmptyPgp, pgp)
 		if err != nil {
 			log.Error(err)
 		} else {
 			verifiers = append(verifiers, snapshotVerifier)
 		}
 	}
 
-	remoteVer, err := http.NewVerifier(config)
+	remoteVer, err := http.NewVerifier(config, allowEmptyPgp, pgp)
 	if err != nil {
 		return nil, err
 	}

x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go

@@ -38,7 +38,6 @@ func snapshotConfig(config *artifact.Config) (*artifact.Config, error) {
 		SourceURI:       snapshotURI,
 		TargetDirectory: config.TargetDirectory,
 		Timeout:         config.Timeout,
-		PgpFile:         config.PgpFile,
 		InstallPath:     config.InstallPath,
 		DropPath:        config.DropPath,
 	}, nil

x-pack/elastic-agent/pkg/artifact/download/snapshot/verifier.go

@@ -12,10 +12,10 @@ import (
 
 // NewVerifier creates a downloader which first checks local directory
 // and then fallbacks to remote if configured.
-func NewVerifier(config *artifact.Config, downloaders ...download.Downloader) (download.Verifier, error) {
+func NewVerifier(config *artifact.Config, allowEmptyPgp bool, pgp []byte) (download.Verifier, error) {
 	cfg, err := snapshotConfig(config)
 	if err != nil {
 		return nil, err
 	}
-	return http.NewVerifier(cfg)
+	return http.NewVerifier(cfg, allowEmptyPgp, pgp)
 }