Skip to content

Commit

Permalink
Updated migration query to match items with deprecated field present (#…
Browse files Browse the repository at this point in the history 
…1959) (#1960)

Co-authored-by: Anderson Queiroz <anderson.queiroz@elastic.co>
(cherry picked from commit b2f3e20)

Co-authored-by: Michal Pristas <michal.pristas@gmail.com>
  • Loading branch information
@mergify @michalpristas
mergify[bot] and michalpristas authored Oct 6, 2022
1 parent ce4fe62 commit dbd7891
Show file tree
Hide file tree
Showing 2 changed files with 88 additions and 24 deletions.
15 changes: 8 additions & 7 deletions internal/pkg/dl/migration.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,13 +217,14 @@ func migrateToV8_5(ctx context.Context, bulker bulk.Bulk) error {
// this change fixes.
func migrateAgentOutputs() (string, string, []byte, error) {
const (
migrationName = "AgentOutputs"
fieldOutputs = "outputs"
fieldRetiredAt = "retiredAt"
migrationName = "AgentOutputs"
fieldOutputs = "outputs"
fieldDefaultAPIKeyID = "default_api_key_id" // nolint:gosec,G101 // this is not a credential
fieldRetiredAt = "retiredAt"
)

query := dsl.NewRoot()
query.Query().Bool().MustNot().Exists(fieldOutputs)
query.Query().Bool().Must().Exists(fieldDefaultAPIKeyID)

fields := map[string]interface{}{fieldRetiredAt: timeNow().UTC().Format(time.RFC3339)}
painless := `
Expand Down Expand Up @@ -253,9 +254,9 @@ ctx._source['` + fieldOutputs + `']['default'].permissions_hash=ctx._source.poli
// Erase deprecated fields
ctx._source.default_api_key_history=null;
ctx._source.default_api_key="";
ctx._source.default_api_key_id="";
ctx._source.policy_output_permissions_hash="";
ctx._source.default_api_key=null;
ctx._source.default_api_key_id=null;
ctx._source.policy_output_permissions_hash=null;
`
query.Param("script", map[string]interface{}{
"lang": "painless",
Expand Down
97 changes: 80 additions & 17 deletions internal/pkg/dl/migration_integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,12 +155,29 @@ func TestMigrateOutputs_withDefaultAPIKeyHistory(t *testing.T) {
for i, id := range agentIDs {
wantOutputType := "elasticsearch" //nolint:goconst // test cases have some duplication

got, err := FindAgent(
context.Background(), bulker, QueryAgentByID, FieldID, id, WithIndexName(index))
if err != nil {
assert.NoError(t, err, "failed to find agent ID %q", id) // we want to continue even if a single agent fails
continue
}
res, err := SearchWithOneParam(context.Background(), bulker, QueryAgentByID, index, FieldID, id)
require.NoError(t, err)
require.Len(t, res.Hits, 1)

var got model.Agent
err = res.Hits[0].Unmarshal(&got)
require.NoError(t, err, "could not unmarshal ES document into model.Agent")

gotDeprecatedFields := struct {
// Deprecated. Use Outputs instead. API key the Elastic Agent uses to authenticate with elasticsearch
DefaultAPIKey *string `json:"default_api_key,omitempty"`

// Deprecated. Use Outputs instead. Default API Key History
DefaultAPIKeyHistory []model.ToRetireAPIKeyIdsItems `json:"default_api_key_history,omitempty"`

// Deprecated. Use Outputs instead. ID of the API key the Elastic Agent uses to authenticate with elasticsearch
DefaultAPIKeyID *string `json:"default_api_key_id,omitempty"`

// Deprecated. Use Outputs instead. The policy output permissions hash
PolicyOutputPermissionsHash *string `json:"policy_output_permissions_hash,omitempty"`
}{}
err = res.Hits[0].Unmarshal(&gotDeprecatedFields)
require.NoError(t, err, "could not unmarshal ES document into gotDeprecatedFields")

wantToRetireAPIKeyIds := []model.ToRetireAPIKeyIdsItems{
{
Expand Down Expand Up @@ -196,11 +213,36 @@ func TestMigrateOutputs_withDefaultAPIKeyHistory(t *testing.T) {
}

// Assert deprecated fields
assert.Empty(t, got.DefaultAPIKey)
assert.Empty(t, got.DefaultAPIKey)
assert.Empty(t, got.PolicyOutputPermissionsHash)
assert.Nil(t, got.DefaultAPIKeyHistory)
assert.Nil(t, gotDeprecatedFields.DefaultAPIKey)
assert.Nil(t, gotDeprecatedFields.DefaultAPIKeyID)
assert.Nil(t, gotDeprecatedFields.PolicyOutputPermissionsHash)
assert.Nil(t, gotDeprecatedFields.DefaultAPIKeyHistory)
}
}

func TestMigrateOutputs_dontMigrateTwice(t *testing.T) {
now, err := time.Parse(time.RFC3339, nowStr)
require.NoError(t, err, "could not parse time "+nowStr)
timeNow = func() time.Time {
return now
}

index, bulker := ftesting.SetupCleanIndex(context.Background(), t, FleetAgents)
apiKey := bulk.APIKey{
ID: "testAgent_",
Key: "testAgent_key_",
}

agentIDs := createSomeAgents(t, 25, apiKey, index, bulker)

migratedAgents, err := migrate(context.Background(), bulker, migrateAgentOutputs)
require.NoError(t, err)
assert.Equal(t, len(agentIDs), migratedAgents)

migratedAgents2, err := migrate(context.Background(), bulker, migrateAgentOutputs)
require.NoError(t, err)

assert.Equal(t, 0, migratedAgents2)
}

func TestMigrateOutputs_nil_DefaultAPIKeyHistory(t *testing.T) {
Expand Down Expand Up @@ -249,9 +291,30 @@ func TestMigrateOutputs_nil_DefaultAPIKeyHistory(t *testing.T) {
migratedAgents, err := migrate(context.Background(), bulker, migrateAgentOutputs)
require.NoError(t, err)

got, err := FindAgent(
context.Background(), bulker, QueryAgentByID, FieldID, agentID, WithIndexName(index))
require.NoError(t, err, "failed to find agent ID %q", agentID) // we want to continue even if a single agent fails
res, err := SearchWithOneParam(
context.Background(), bulker, QueryAgentByID, index, FieldID, agentID)
require.NoError(t, err, "failed to find agent ID %q", agentID)
require.Len(t, res.Hits, 1)

var got model.Agent
err = res.Hits[0].Unmarshal(&got)
require.NoError(t, err, "could not unmarshal ES document into model.Agent")

gotDeprecatedFields := struct {
// Deprecated. Use Outputs instead. API key the Elastic Agent uses to authenticate with elasticsearch
DefaultAPIKey *string `json:"default_api_key,omitempty"`

// Deprecated. Use Outputs instead. Default API Key History
DefaultAPIKeyHistory []model.ToRetireAPIKeyIdsItems `json:"default_api_key_history,omitempty"`

// Deprecated. Use Outputs instead. ID of the API key the Elastic Agent uses to authenticate with elasticsearch
DefaultAPIKeyID *string `json:"default_api_key_id,omitempty"`

// Deprecated. Use Outputs instead. The policy output permissions hash
PolicyOutputPermissionsHash *string `json:"policy_output_permissions_hash,omitempty"`
}{}
err = res.Hits[0].Unmarshal(&gotDeprecatedFields)
require.NoError(t, err, "could not unmarshal ES document into gotDeprecatedFields")

assert.Equal(t, 1, migratedAgents)

Expand All @@ -273,10 +336,10 @@ func TestMigrateOutputs_nil_DefaultAPIKeyHistory(t *testing.T) {
}

// Assert deprecated fields
assert.Empty(t, got.DefaultAPIKey)
assert.Empty(t, got.DefaultAPIKey)
assert.Empty(t, got.PolicyOutputPermissionsHash)
assert.Nil(t, got.DefaultAPIKeyHistory)
assert.Nil(t, gotDeprecatedFields.DefaultAPIKey)
assert.Nil(t, gotDeprecatedFields.DefaultAPIKey)
assert.Nil(t, gotDeprecatedFields.PolicyOutputPermissionsHash)
assert.Nil(t, gotDeprecatedFields.DefaultAPIKeyHistory)
}

func TestMigrateOutputs_no_agent_document(t *testing.T) {
Expand Down

0 comments on commit dbd7891

Please sign in to comment.