Fix Elastic Agent non-fleet broken upgrade between 8.3.x releases (#701)

* Fix Elastic Agent non-fleet broken upgrade between 8.3.x releases

* Migrates vault directory on linux and windows to the top directory of the
  agent, so it can be shared without needing the upgrade handler call,
  like for example with side-by-side install/upgrade from .rpm/.deb
* Extended vault to allow read-only open, useful when the vault at particular location needs to be only read not created.

* Correct the typo in the log messages

* Update lint flagged function comment with 'unused', was flagged with 'deadcode' on the previous run

* Address code review feedback

* Add missing import for linux utz

* Change vault path from Top() to Config(), this a better location, next to fleet.enc based on the install/upgrade testing with .rpm/.deb installs

* Fix the missing state migration for .rpm/.deb upgrade. The post install script now performs the migration and creates the symlink after that.

* Fix typo in the postinstall script

* Update the vault migration code, add the agent configuration match check with the agent secret

(cherry picked from commit 8ef98f1)

dev-tools/packaging/packages.yml

@@ -20,11 +20,8 @@ shared:
   # Deb/RPM spec for community beats.
   - &deb_rpm_agent_spec
     <<: *common
-    post_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/systemd-daemon-reload.sh'
+    post_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/linux/postinstall.sh.tmpl'
     files:
-      /usr/share/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
-        source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
-        mode: 0755
       /usr/share/{{.BeatName}}/LICENSE.txt:
         source: '{{ repo.RootDir }}/LICENSE.txt'
         mode: 0644
@@ -1083,11 +1080,6 @@ specs:
       spec:
         <<: *deb_rpm_agent_spec
         <<: *elastic_license_for_deb_rpm
-        files:
-          /usr/share/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
-            source: /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}
-            symlink: true
-            mode: 0755
 
     - os: linux
       arch: amd64

dev-tools/packaging/templates/linux/postinstall.sh.tmpl

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -e
+
+symlink="/usr/share/elastic-agent/bin/elastic-agent"
+old_agent_dir="$( dirname "$(readlink -f -- "$symlink")" )"
+
+commit_hash="{{ commit_short }}"
+
+yml_path="$old_agent_dir/state.yml"
+enc_path="$old_agent_dir/state.enc"
+
+new_agent_dir="$( dirname "$old_agent_dir")/elastic-agent-$commit_hash"
+
+if ! [[ "$old_agent_dir" -ef "$new_agent_dir" ]]; then
+    echo "migrate state from $old_agent_dir to $new_agent_dir"
+
+    if test -f "$yml_path"; then
+        echo "found "$yml_path", copy to "$new_agent_dir"."
+        cp "$yml_path" "$new_agent_dir"
+    fi
+
+    if test -f "$enc_path"; then
+        echo "found "$enc_path", copy to "$new_agent_dir"."
+        cp "$enc_path" "$new_agent_dir"
+    fi
+
+    if test -f "$symlink"; then
+        echo "found symlink $symlink, unlink"
+        unlink "$symlink"
+    fi
+
+    echo "create symlink "$symlink" to "$new_agent_dir/elastic-agent""
+    ln -s "$new_agent_dir/elastic-agent" "$symlink"
+fi
+
+systemctl daemon-reload 2> /dev/null
+exit 0

internal/pkg/agent/application/paths/paths_linux.go

@@ -14,5 +14,5 @@ const defaultAgentVaultPath = "vault"
 
 // AgentVaultPath is the directory that contains all the files for the value
 func AgentVaultPath() string {
-	return filepath.Join(Home(), defaultAgentVaultPath)
+	return filepath.Join(Config(), defaultAgentVaultPath)
 }

internal/pkg/agent/application/paths/paths_windows.go

@@ -42,5 +42,5 @@ func ArePathsEqual(expected, actual string) bool {
 
 // AgentVaultPath is the directory that contains all the files for the value
 func AgentVaultPath() string {
-	return filepath.Join(Home(), defaultAgentVaultPath)
+	return filepath.Join(Config(), defaultAgentVaultPath)
 }

internal/pkg/agent/application/secret/secret.go

@@ -6,6 +6,7 @@ package secret
 
 import (
 	"encoding/json"
+	"fmt"
 	"runtime"
 	"sync"
 	"time"
@@ -51,7 +52,7 @@ func Create(key string, opts ...OptionFunc) error {
 	options := applyOptions(opts...)
 	v, err := vault.New(options.vaultPath)
 	if err != nil {
-		return err
+		return fmt.Errorf("could not create new vault: %w", err)
 	}
 	defer v.Close()
 
@@ -79,23 +80,25 @@ func Create(key string, opts ...OptionFunc) error {
 		CreatedOn: time.Now().UTC(),
 	}
 
-	b, err := json.Marshal(secret)
-	if err != nil {
-		return err
-	}
-
-	return v.Set(key, b)
+	return set(v, key, secret)
 }
 
 // GetAgentSecret read the agent secret from the vault
 func GetAgentSecret(opts ...OptionFunc) (secret Secret, err error) {
 	return Get(agentSecretKey, opts...)
 }
 
+// SetAgentSecret saves the agent secret from the vault
+// This is needed for migration from 8.3.0-8.3.2 to higher versions
+func SetAgentSecret(secret Secret, opts ...OptionFunc) error {
+	return Set(agentSecretKey, secret, opts...)
+}
+
 // Get reads the secret key from the vault
 func Get(key string, opts ...OptionFunc) (secret Secret, err error) {
 	options := applyOptions(opts...)
-	v, err := vault.New(options.vaultPath)
+	// open vault readonly, will not create the vault directory or the seed it was not created before
+	v, err := vault.New(options.vaultPath, vault.WithReadonly(true))
 	if err != nil {
 		return secret, err
 	}
@@ -110,12 +113,32 @@ func Get(key string, opts ...OptionFunc) (secret Secret, err error) {
 	return secret, err
 }
 
+// Set saves the secret key to the vault
+func Set(key string, secret Secret, opts ...OptionFunc) error {
+	options := applyOptions(opts...)
+	v, err := vault.New(options.vaultPath)
+	if err != nil {
+		return fmt.Errorf("could not create new vault: %w", err)
+	}
+	defer v.Close()
+	return set(v, key, secret)
+}
+
+func set(v *vault.Vault, key string, secret Secret) error {
+	b, err := json.Marshal(secret)
+	if err != nil {
+		return fmt.Errorf("could not marshal secret: %w", err)
+	}
+
+	return v.Set(key, b)
+}
+
 // Remove removes the secret key from the vault
 func Remove(key string, opts ...OptionFunc) error {
 	options := applyOptions(opts...)
 	v, err := vault.New(options.vaultPath)
 	if err != nil {
-		return err
+		return fmt.Errorf("could not create new vault: %w", err)
 	}
 	defer v.Close()
 

internal/pkg/agent/application/upgrade/upgrade.go

@@ -10,7 +10,6 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"runtime"
 	"strings"
 
 	"github.com/otiai10/copy"
@@ -33,7 +32,6 @@ const (
 	agentName       = "elastic-agent"
 	hashLen         = 6
 	agentCommitFile = ".elastic-agent.active.commit"
-	darwin          = "darwin"
 )
 
 var (
@@ -161,11 +159,6 @@ func (u *Upgrader) Upgrade(ctx context.Context, a Action, reexecNow bool) (_ ree
 		return nil, nil
 	}
 
-	// Copy vault directory for linux/windows only
-	if err := copyVault(newHash); err != nil {
-		return nil, errors.New(err, "failed to copy vault")
-	}
-
 	if err := copyActionStore(newHash); err != nil {
 		return nil, errors.New(err, "failed to copy action store")
 	}
@@ -300,36 +293,6 @@ func copyActionStore(newHash string) error {
 	return nil
 }
 
-func getVaultPath(newHash string) string {
-	vaultPath := paths.AgentVaultPath()
-	if runtime.GOOS == darwin {
-		return vaultPath
-	}
-	newHome := filepath.Join(filepath.Dir(paths.Home()), fmt.Sprintf("%s-%s", agentName, newHash))
-	return filepath.Join(newHome, filepath.Base(vaultPath))
-}
-
-// Copies the vault files for windows and linux
-func copyVault(newHash string) error {
-	// No vault files to copy on darwin
-	if runtime.GOOS == darwin {
-		return nil
-	}
-
-	vaultPath := paths.AgentVaultPath()
-	newVaultPath := getVaultPath(newHash)
-
-	err := copyDir(vaultPath, newVaultPath)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-
-	return nil
-}
-
 // shutdownCallback returns a callback function to be executing during shutdown once all processes are closed.
 // this goes through runtime directory of agent and copies all the state files created by processes to new versioned
 // home directory with updated process name to match new version.

internal/pkg/agent/cmd/run.go

@@ -37,6 +37,7 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/agent/configuration"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/control/server"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/errors"
+	"github.com/elastic/elastic-agent/internal/pkg/agent/migration"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/storage"
 	"github.com/elastic/elastic-agent/internal/pkg/cli"
 	"github.com/elastic/elastic-agent/internal/pkg/config"
@@ -121,6 +122,22 @@ func run(override cfgOverrider) error {
 		createAgentID = false
 	}
 
+	// This is specific for the agent upgrade from 8.3.0 - 8.3.2 to 8.x and above on Linux and Windows platforms.
+	// Addresses the issue: https://github.com/elastic/elastic-agent/issues/682
+	// The vault directory was located in the hash versioned "Home" directory of the agent.
+	// This moves the vault directory two levels up into  the "Config" directory next to fleet.enc file
+	// in order to be able to "upgrade" the agent from deb/rpm that is not invoking the upgrade handle and
+	// doesn't perform the migration of the state or vault.
+	// If the agent secret doesn't exist, then search for the newest agent secret in the agent data directories
+	// and migrate it into the new vault location.
+	err = migration.MigrateAgentSecret(logger)
+	logger.Debug("migration of agent secret completed, err: %v", err)
+	if err != nil {
+		err = errors.New(err, "failed to perfrom the agent secret migration")
+		logger.Error(err)
+		return err
+	}
+
 	// Ensure we have the agent secret created.
 	// The secret is not created here if it exists already from the previous enrollment.
 	// This is needed for compatibility with agent running in standalone mode,