From 55be1af6dbbed4e45e90bc84207207076dbb4cca Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 21 Dec 2022 07:49:34 +0100 Subject: [PATCH 01/33] Refactor to remove global shell type for shellinit (#1081) --- cmd/stack.go | 7 ++----- internal/stack/errors.go | 2 +- internal/stack/shellinit.go | 19 ++++--------------- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/cmd/stack.go b/cmd/stack.go index 043c1e9e9..c96193111 100644 --- a/cmd/stack.go +++ b/cmd/stack.go @@ -202,12 +202,9 @@ func setupStackCommand() *cobraext.Command { if err != nil { return cobraext.FlagParsingError(err, cobraext.ShellInitShellFlagName) } - if shellName == cobraext.ShellInitShellDetect { - shellName = stack.AutodetectedShell() + shellName = stack.AutodetectShell() fmt.Fprintf(cmd.OutOrStderr(), "Detected shell: %s\n", shellName) - } else { - stack.SelectShell(shellName) } profile, err := profile.LoadProfile(profileName) @@ -215,7 +212,7 @@ func setupStackCommand() *cobraext.Command { return errors.Wrap(err, "error loading profile") } - shellCode, err := stack.ShellInit(profile) + shellCode, err := stack.ShellInit(profile, shellName) if err != nil { return errors.Wrap(err, "shellinit failed") } diff --git a/internal/stack/errors.go b/internal/stack/errors.go index a7840c79a..91a8bad85 100644 --- a/internal/stack/errors.go +++ b/internal/stack/errors.go @@ -9,5 +9,5 @@ import "fmt" // UndefinedEnvError formats an error reported for undefined variable. func UndefinedEnvError(envName string) error { return fmt.Errorf("undefined environment variable: %s. If you have started the Elastic stack using the elastic-package tool, "+ - `please load stack environment variables using '%s' or set their values manually`, envName, helpText(shellType)) + `please load stack environment variables using '%s' or set their values manually`, envName, helpText(AutodetectShell())) } diff --git a/internal/stack/shellinit.go b/internal/stack/shellinit.go index 4da610617..55f459b60 100644 --- a/internal/stack/shellinit.go +++ b/internal/stack/shellinit.go @@ -27,24 +27,13 @@ var ( CACertificateEnv = environment.WithElasticPackagePrefix("CA_CERT") ) -var shellType string - -func init() { - shellType = detectShell() -} - -// SelectShell selects the shell to use. -func SelectShell(shell string) { - shellType = shell -} - -// AutodetectedShell returns an error if shell could not be detected. -func AutodetectedShell() string { - return shellType +// AutodetectShell returns the detected shell used. +func AutodetectShell() string { + return detectShell() } // ShellInit method exposes environment variables that can be used for testing purposes. -func ShellInit(elasticStackProfile *profile.Profile) (string, error) { +func ShellInit(elasticStackProfile *profile.Profile, shellType string) (string, error) { config, err := StackInitConfig(elasticStackProfile) if err != nil { return "", nil From f19ffaf61b118ebef59552a404e889e788de1992 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 29 Dec 2022 07:50:18 -0500 Subject: [PATCH 02/33] Update add-to-ecosystem-project.yml --- .github/workflows/add-to-ecosystem-project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/add-to-ecosystem-project.yml b/.github/workflows/add-to-ecosystem-project.yml index 5cd4cbf25..eb05219f3 100644 --- a/.github/workflows/add-to-ecosystem-project.yml +++ b/.github/workflows/add-to-ecosystem-project.yml @@ -22,7 +22,7 @@ jobs: projectid: ${{ env.PROJECT_ID }} contentid: ${{ github.event.issue.node_id }} env: - PROJECT_ID: "PN_kwDOAGc3Zs4ABFUO" + PROJECT_ID: "PVT_kwDOAGc3Zs4AEzn4" GITHUB_TOKEN: ${{ secrets.ECOSYSTEM_USER_TOKEN }} - uses: octokit/graphql-action@v2.x id: label_team From 9db943f071b1fe341c2d5a868b696f51305cc2b9 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 29 Dec 2022 07:54:08 -0500 Subject: [PATCH 03/33] Fix syntax in project action --- .github/workflows/add-issues-to-ingest-board.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/add-issues-to-ingest-board.yml b/.github/workflows/add-issues-to-ingest-board.yml index e3c937cda..78760830f 100644 --- a/.github/workflows/add-issues-to-ingest-board.yml +++ b/.github/workflows/add-issues-to-ingest-board.yml @@ -36,7 +36,7 @@ jobs: if: github.event.label.name == env.ECOSYSTEM_LABEL with: query: | - mutation updateIngestArea($item_id: ID!, $project_id: ID!, $areaField_id: ID!, $area_id: String) { + mutation updateIngestArea($item_id: ID!, $project_id: ID!, $area_field_id: ID!, $area_id: String) { updateProjectV2ItemFieldValue( input: { itemId: $item_id, projectId: $project_id, fieldId: $area_field_id, value: { singleSelectOptionId: $area_id } }) { clientMutationId From 04c4283720239b070ca692d9a213e9aada92cd69 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 29 Dec 2022 08:08:04 -0500 Subject: [PATCH 04/33] Delete add-to-ecosystem-project.yml Ecosystem project no longer exists, so this just action is stuck in a failure loop. --- .../workflows/add-to-ecosystem-project.yml | 39 ------------------- 1 file changed, 39 deletions(-) delete mode 100644 .github/workflows/add-to-ecosystem-project.yml diff --git a/.github/workflows/add-to-ecosystem-project.yml b/.github/workflows/add-to-ecosystem-project.yml deleted file mode 100644 index eb05219f3..000000000 --- a/.github/workflows/add-to-ecosystem-project.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Add to Project -on: - issues: - types: - - opened -jobs: - add_to_project: - runs-on: ubuntu-latest - steps: - - uses: octokit/graphql-action@v2.x - id: add_to_project - with: - headers: '{"GraphQL-Features": "projects_next_graphql"}' - query: | - mutation add_to_project($projectid:ID!,$contentid:ID!) { - addProjectNextItem(input:{projectId:$projectid contentId:$contentid}) { - projectNextItem { - id - } - } - } - projectid: ${{ env.PROJECT_ID }} - contentid: ${{ github.event.issue.node_id }} - env: - PROJECT_ID: "PVT_kwDOAGc3Zs4AEzn4" - GITHUB_TOKEN: ${{ secrets.ECOSYSTEM_USER_TOKEN }} - - uses: octokit/graphql-action@v2.x - id: label_team - with: - query: | - mutation add_label($issueid:ID!, $labelids:[ID!]!) { - addLabelsToLabelable(input: {labelableId: $issueid, labelIds: $labelids}) { - clientMutationId - } - } - issueid: ${{ github.event.issue.node_id }} - labelids: "LA_kwDOEBH20c7PrEba" - env: - GITHUB_TOKEN: ${{ secrets.ECOSYSTEM_USER_TOKEN }} From f6872fecc9cb62f5f71fba04510733349dbb1af0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Dec 2022 17:38:43 +0000 Subject: [PATCH 05/33] Chore(deps): bump github.com/go-git/go-billy/v5 from 5.3.1 to 5.4.0 (#1085) Bumps [github.com/go-git/go-billy/v5](https://github.com/go-git/go-billy) from 5.3.1 to 5.4.0. - [Release notes](https://github.com/go-git/go-billy/releases) - [Commits](https://github.com/go-git/go-billy/compare/v5.3.1...v5.4.0) --- updated-dependencies: - dependency-name: github.com/go-git/go-billy/v5 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 8074fce14..ce838fe79 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/elastic/go-ucfg v0.8.6 github.com/elastic/package-spec/v2 v2.2.0 github.com/fatih/color v1.13.0 - github.com/go-git/go-billy/v5 v5.3.1 + github.com/go-git/go-billy/v5 v5.4.0 github.com/go-git/go-git/v5 v5.5.1 github.com/google/go-cmp v0.5.9 github.com/google/go-github/v32 v32.1.0 diff --git a/go.sum b/go.sum index 72f2b5e6b..1cd6073c4 100644 --- a/go.sum +++ b/go.sum @@ -125,8 +125,9 @@ github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxI github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= -github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-billy/v5 v5.4.0 h1:Vaw7LaSTRJOUric7pe4vnzBSgyuf2KrLsu2Y4ZpQBDE= +github.com/go-git/go-billy/v5 v5.4.0/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ= github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= github.com/go-git/go-git/v5 v5.5.1 h1:5vtv2TB5PM/gPM+EvsHJ16hJh4uAkdGcKilcwY7FYwo= From 6a5fb51e5d4ec89035638f936a0b2dcdaee972c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 17:38:10 +0000 Subject: [PATCH 06/33] Chore(deps): bump github.com/shirou/gopsutil/v3 from 3.22.11 to 3.22.12 (#1086) Bumps [github.com/shirou/gopsutil/v3](https://github.com/shirou/gopsutil) from 3.22.11 to 3.22.12. - [Release notes](https://github.com/shirou/gopsutil/releases) - [Commits](https://github.com/shirou/gopsutil/commits/v3.22.12) --- updated-dependencies: - dependency-name: github.com/shirou/gopsutil/v3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ce838fe79..150b6c0e8 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 - github.com/shirou/gopsutil/v3 v3.22.11 + github.com/shirou/gopsutil/v3 v3.22.12 github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.1 golang.org/x/oauth2 v0.3.0 diff --git a/go.sum b/go.sum index 1cd6073c4..5157020fa 100644 --- a/go.sum +++ b/go.sum @@ -334,8 +334,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v3 v3.22.11 h1:kxsPKS+Eeo+VnEQ2XCaGJepeP6KY53QoRTETx3+1ndM= -github.com/shirou/gopsutil/v3 v3.22.11/go.mod h1:xl0EeL4vXJ+hQMAGN8B9VFpxukEMA0XdevQOe5MZ1oY= +github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs= +github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= From a24627b06dbbfe99262ae42792dbfc016dc72415 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Jan 2023 17:39:30 +0000 Subject: [PATCH 07/33] Chore(deps): bump golang.org/x/oauth2 from 0.3.0 to 0.4.0 (#1088) Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.3.0 to 0.4.0. - [Release notes](https://github.com/golang/oauth2/releases) - [Commits](https://github.com/golang/oauth2/compare/v0.3.0...v0.4.0) --- updated-dependencies: - dependency-name: golang.org/x/oauth2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 19 ++++++++++--------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 150b6c0e8..271e3fef6 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/shirou/gopsutil/v3 v3.22.12 github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.1 - golang.org/x/oauth2 v0.3.0 + golang.org/x/oauth2 v0.4.0 golang.org/x/tools v0.4.0 gopkg.in/yaml.v3 v3.0.1 gotest.tools/gotestsum v1.8.2 @@ -135,11 +135,11 @@ require ( golang.org/x/crypto v0.4.0 // indirect golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e // indirect golang.org/x/mod v0.7.0 // indirect - golang.org/x/net v0.4.0 // indirect + golang.org/x/net v0.5.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.3.0 // indirect - golang.org/x/term v0.3.0 // indirect - golang.org/x/text v0.5.0 // indirect + golang.org/x/sys v0.4.0 // indirect + golang.org/x/term v0.4.0 // indirect + golang.org/x/text v0.6.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 // indirect diff --git a/go.sum b/go.sum index 5157020fa..90c5317a2 100644 --- a/go.sum +++ b/go.sum @@ -453,12 +453,12 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= -golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= +golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= +golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -507,16 +507,17 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -524,8 +525,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 3c04a470b001d12d45f0dfe7db6def5311219cda Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Jan 2023 17:33:43 +0000 Subject: [PATCH 08/33] Chore(deps): bump github.com/go-git/go-git/v5 from 5.5.1 to 5.5.2 (#1089) Bumps [github.com/go-git/go-git/v5](https://github.com/go-git/go-git) from 5.5.1 to 5.5.2. - [Release notes](https://github.com/go-git/go-git/releases) - [Commits](https://github.com/go-git/go-git/compare/v5.5.1...v5.5.2) --- updated-dependencies: - dependency-name: github.com/go-git/go-git/v5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 271e3fef6..a18e33b9b 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/elastic/package-spec/v2 v2.2.0 github.com/fatih/color v1.13.0 github.com/go-git/go-billy/v5 v5.4.0 - github.com/go-git/go-git/v5 v5.5.1 + github.com/go-git/go-git/v5 v5.5.2 github.com/google/go-cmp v0.5.9 github.com/google/go-github/v32 v32.1.0 github.com/google/go-querystring v1.1.0 diff --git a/go.sum b/go.sum index 90c5317a2..6630e3f72 100644 --- a/go.sum +++ b/go.sum @@ -130,8 +130,8 @@ github.com/go-git/go-billy/v5 v5.4.0 h1:Vaw7LaSTRJOUric7pe4vnzBSgyuf2KrLsu2Y4ZpQ github.com/go-git/go-billy/v5 v5.4.0/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ= github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= -github.com/go-git/go-git/v5 v5.5.1 h1:5vtv2TB5PM/gPM+EvsHJ16hJh4uAkdGcKilcwY7FYwo= -github.com/go-git/go-git/v5 v5.5.1/go.mod h1:uz5PQ3d0gz7mSgzZhSJToM6ALPaKCdSnl58/Xb5hzr8= +github.com/go-git/go-git/v5 v5.5.2 h1:v8lgZa5k9ylUw+OR/roJHTxR4QItsNFI5nKtAXFuynw= +github.com/go-git/go-git/v5 v5.5.2/go.mod h1:BE5hUJ5yaV2YMxhmaP4l6RBQ08kMxKSPD4BlxtH7OjI= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= From e5831f4e2ac5e6cc3b0d1eb8737e65e3c945d181 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 9 Jan 2023 14:33:41 +0100 Subject: [PATCH 09/33] Add CA fingerprint to agents configuration in 8.x versions (#1082) --- internal/certs/certs.go | 21 ++++++++++++++++++- .../profile/_static/docker-compose-stack.yml | 2 ++ internal/profile/_static/kibana_config_80.yml | 8 +++++++ internal/profile/_static/kibana_config_8x.yml | 8 +++++++ internal/profile/certs.go | 8 +++++++ 5 files changed, 46 insertions(+), 1 deletion(-) diff --git a/internal/certs/certs.go b/internal/certs/certs.go index 4c808a4a8..bf037f991 100644 --- a/internal/certs/certs.go +++ b/internal/certs/certs.go @@ -10,8 +10,10 @@ import ( "crypto/elliptic" "crypto/rand" "crypto/rsa" + "crypto/sha256" "crypto/tls" "crypto/x509" + "encoding/hex" "encoding/pem" "fmt" "io" @@ -19,6 +21,7 @@ import ( "net" "os" "path/filepath" + "strings" "time" "github.com/elastic/elastic-package/internal/common" @@ -255,7 +258,23 @@ func (c *Certificate) WriteCert(w io.Writer) error { return nil } -// WriteCertFile writes the PEM-encoded certificate in the given file. +// WriteEnv writes the environment variables about the certificate in the given writer. +func (c *Certificate) WriteEnv(w io.Writer) error { + fingerprint := c.Fingerprint() + _, err := fmt.Fprintf(w, "%s=%s\n", + "ELASTIC_PACKAGE_CA_TRUSTED_FINGERPRINT", + strings.ToUpper(hex.EncodeToString(fingerprint))) + return err +} + +// Fingerprint returns the fingerprint of the certificate. The fingerprint +// of a CA can be used to verify certificates. +func (c *Certificate) Fingerprint() []byte { + f := sha256.Sum256(c.cert.Raw) + return f[:] +} + +// WriteCertFile writes the PEM-encoded certifiacte in the given file. func (c *Certificate) WriteCertFile(path string) error { err := os.MkdirAll(filepath.Dir(path), 0755) if err != nil { diff --git a/internal/profile/_static/docker-compose-stack.yml b/internal/profile/_static/docker-compose-stack.yml index cc31b1718..827e1e62c 100644 --- a/internal/profile/_static/docker-compose-stack.yml +++ b/internal/profile/_static/docker-compose-stack.yml @@ -34,6 +34,8 @@ services: test: "sh /usr/share/kibana/healthcheck.sh" retries: 600 interval: 1s + env_file: + - "../certs/ca.env" environment: # Is there a better way to add certificates to Kibana/Fleet? - "NODE_EXTRA_CA_CERTS=/usr/share/kibana/config/certs/ca-cert.pem" diff --git a/internal/profile/_static/kibana_config_80.yml b/internal/profile/_static/kibana_config_80.yml index 7a69e6fd9..4e3c72ec0 100644 --- a/internal/profile/_static/kibana_config_80.yml +++ b/internal/profile/_static/kibana_config_80.yml @@ -49,3 +49,11 @@ xpack.fleet.agentPolicies: id: default-fleet-server package: name: fleet_server +xpack.fleet.outputs: + - id: fleet-default-output + name: default + type: elasticsearch + hosts: [ https://elasticsearch:9200 ] + ca_trusted_fingerprint: "${ELASTIC_PACKAGE_CA_TRUSTED_FINGERPRINT}" + is_default: true + is_default_monitoring: true diff --git a/internal/profile/_static/kibana_config_8x.yml b/internal/profile/_static/kibana_config_8x.yml index f8a145b76..6d9564c00 100644 --- a/internal/profile/_static/kibana_config_8x.yml +++ b/internal/profile/_static/kibana_config_8x.yml @@ -51,3 +51,11 @@ xpack.fleet.agentPolicies: id: default-fleet-server package: name: fleet_server +xpack.fleet.outputs: + - id: fleet-default-output + name: default + type: elasticsearch + hosts: [ https://elasticsearch:9200 ] + ca_trusted_fingerprint: "${ELASTIC_PACKAGE_CA_TRUSTED_FINGERPRINT}" + is_default: true + is_default_monitoring: true diff --git a/internal/profile/certs.go b/internal/profile/certs.go index 820d6a2df..49f6b48f2 100644 --- a/internal/profile/certs.go +++ b/internal/profile/certs.go @@ -32,6 +32,9 @@ var ( // CAKeyFile is the path to the CA key file inside a profile. CAKeyFile = configFile(filepath.Join(CertificatesDirectory, "ca-key.pem")) + + // CAEnvFile is the path to the file with environment variables about the CA. + CAEnvFile = configFile(filepath.Join(CertificatesDirectory, "ca.env")) ) // initTLSCertificates initializes all the certificates needed to run the services @@ -41,6 +44,7 @@ func initTLSCertificates(profilePath string, configMap map[configFile]*simpleFil certsDir := filepath.Join(profilePath, CertificatesDirectory) caCertFile := filepath.Join(profilePath, string(CACertificateFile)) caKeyFile := filepath.Join(profilePath, string(CAKeyFile)) + envFile := filepath.Join(profilePath, string(CAEnvFile)) ca, err := initCA(caCertFile, caKeyFile) if err != nil { @@ -54,6 +58,10 @@ func initTLSCertificates(profilePath string, configMap map[configFile]*simpleFil if err != nil { return err } + err = certWriterToSimpleFile(configMap, profilePath, envFile, ca.WriteEnv) + if err != nil { + return err + } for _, service := range tlsServices { certsDir := filepath.Join(certsDir, service) From de57284a2c93bffbef7c65be4e55094529a42dc4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Jan 2023 10:10:45 +0000 Subject: [PATCH 10/33] Chore(deps): bump golang.org/x/tools from 0.4.0 to 0.5.0 (#1087) Bumps [golang.org/x/tools](https://github.com/golang/tools) from 0.4.0 to 0.5.0. - [Release notes](https://github.com/golang/tools/releases) - [Commits](https://github.com/golang/tools/compare/v0.4.0...v0.5.0) --- updated-dependencies: - dependency-name: golang.org/x/tools dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a18e33b9b..935fb361e 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.1 golang.org/x/oauth2 v0.4.0 - golang.org/x/tools v0.4.0 + golang.org/x/tools v0.5.0 gopkg.in/yaml.v3 v3.0.1 gotest.tools/gotestsum v1.8.2 helm.sh/helm/v3 v3.10.3 diff --git a/go.sum b/go.sum index 6630e3f72..6bbce114d 100644 --- a/go.sum +++ b/go.sum @@ -545,8 +545,8 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4= -golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.5.0 h1:+bSpV5HIeWkuvgaMfI3UmKRThoTA5ODJTUd8T17NO+4= +golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 69f25ac9345ade590c51e3127e45a22cbb1d160f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Jan 2023 12:21:54 +0000 Subject: [PATCH 11/33] Chore(deps): bump github.com/elastic/package-spec/v2 from 2.2.0 to 2.3.0 (#1092) Bumps [github.com/elastic/package-spec/v2](https://github.com/elastic/package-spec) from 2.2.0 to 2.3.0. - [Release notes](https://github.com/elastic/package-spec/releases) - [Commits](https://github.com/elastic/package-spec/compare/v2.2.0...v2.3.0) --- updated-dependencies: - dependency-name: github.com/elastic/package-spec/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 935fb361e..e82cc90ae 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.17.7 github.com/elastic/go-licenser v0.4.1 github.com/elastic/go-ucfg v0.8.6 - github.com/elastic/package-spec/v2 v2.2.0 + github.com/elastic/package-spec/v2 v2.3.0 github.com/fatih/color v1.13.0 github.com/go-git/go-billy/v5 v5.4.0 github.com/go-git/go-git/v5 v5.5.2 diff --git a/go.sum b/go.sum index 6bbce114d..8d0f74c6b 100644 --- a/go.sum +++ b/go.sum @@ -91,8 +91,8 @@ github.com/elastic/go-ucfg v0.8.6 h1:stUeyh2goTgGX+/wb9gzKvTv0YB0231LTpKUgCKj4U0 github.com/elastic/go-ucfg v0.8.6/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul8OsQc= github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg= -github.com/elastic/package-spec/v2 v2.2.0 h1:hNTtw1PmM9BcehF5g7E4NNNIXNTxravcbrwATucYOAU= -github.com/elastic/package-spec/v2 v2.2.0/go.mod h1:bGpJHojIN9FYyNbqNySJ7D3mDVqOe1sL2l4m7BCSjWo= +github.com/elastic/package-spec/v2 v2.3.0 h1:7pWSZzyUVyHYpmHG7GYz9xKk/dKMPy8k71lNKkurABM= +github.com/elastic/package-spec/v2 v2.3.0/go.mod h1:bGpJHojIN9FYyNbqNySJ7D3mDVqOe1sL2l4m7BCSjWo= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= From d46148dea5d33683fe5aeb18d41de75474f08708 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 13 Jan 2023 13:34:08 +0100 Subject: [PATCH 12/33] Add common dynamic mappings from ECS automatically (#1073) Add dynamic mappings defined in a JSON file as part of the "elasticsearch" key in the manifest file of the built package. All the mappings added dynamically have been prefixed with a common string (_embedded_ecs) to allow distinguish them from the ones defined by the developers. To avoid issues in previous Elastic stack versions, the prefix is separated by a dash instead of a dot. In versions < 8.6.0 the dot in the names caused that packages were not installed. It was interpreted that the string after the dot was an inner element of the map, causing validation errors. --- internal/builder/_static/ecs_mappings.json | 889 ++++++++++++++++++ internal/builder/dynamic_mappings.go | 246 +++++ internal/builder/dynamic_mappings_test.go | 59 ++ internal/builder/packages.go | 8 + internal/builder/testdata/ecs.template.yml | 9 + internal/builder/testdata/empty.yml | 1 + internal/builder/testdata/existing.yml | 13 + .../testdata/expected.empty.mappings.yml | 9 + .../testdata/expected.existing.mappings.yml | 16 + .../packages/buildmanifest/build_manifest.go | 11 +- 10 files changed, 1259 insertions(+), 2 deletions(-) create mode 100644 internal/builder/_static/ecs_mappings.json create mode 100644 internal/builder/dynamic_mappings.go create mode 100644 internal/builder/dynamic_mappings_test.go create mode 100644 internal/builder/testdata/ecs.template.yml create mode 100644 internal/builder/testdata/empty.yml create mode 100644 internal/builder/testdata/existing.yml create mode 100644 internal/builder/testdata/expected.empty.mappings.yml create mode 100644 internal/builder/testdata/expected.existing.mappings.yml diff --git a/internal/builder/_static/ecs_mappings.json b/internal/builder/_static/ecs_mappings.json new file mode 100644 index 000000000..f2f6f7660 --- /dev/null +++ b/internal/builder/_static/ecs_mappings.json @@ -0,0 +1,889 @@ +{ + "mappings": { + "dynamic_templates": [ + { + "ecs_timestamp": { + "path_match": "@timestamp", + "mapping": { + "type": "date", + "ignore_malformed": false + } + } + }, + { + "data_stream_to_constant": { + "path_match": "data_stream.*", + "mapping": { + "type": "constant_keyword" + } + } + }, + { + "resolved_ip_to_ip": { + "match": "resolved_ip", + "mapping": { + "type": "ip" + } + } + }, + { + "forwarded_ip_to_ip": { + "match_mapping_type": "string", + "match": "forwarded_ip", + "mapping": { + "type": "ip" + } + } + }, + { + "ip_to_ip": { + "match_mapping_type": "string", + "match": "ip", + "mapping": { + "type": "ip" + } + } + }, + { + "port_to_long": { + "match": "port", + "mapping": { + "type": "long" + } + } + }, + { + "thread_id_to_long": { + "path_match": "*.thread.id", + "mapping": { + "type": "long" + } + } + }, + { + "status_code_to_long": { + "match": "status_code", + "mapping": { + "type": "long" + } + } + }, + { + "line_to_long": { + "path_match": "*.file.line", + "mapping": { + "type": "long" + } + } + }, + { + "priority_to_long": { + "path_match": "log.syslog.priority", + "mapping": { + "type": "long" + } + } + }, + { + "code_to_long": { + "path_match": "*.facility.code", + "mapping": { + "type": "long" + } + } + }, + { + "code_to_long": { + "path_match": "*.severity.code", + "mapping": { + "type": "long" + } + } + }, + { + "bytes_to_long": { + "match": "bytes", + "path_unmatch": "*.data.bytes", + "mapping": { + "type": "long" + } + } + }, + { + "packets_to_long": { + "match": "packets", + "mapping": { + "type": "long" + } + } + }, + { + "public_key_exponent_to_long": { + "match": "public_key_exponent", + "mapping": { + "type": "long" + } + } + }, + { + "severity_to_long": { + "path_match": "event.severity", + "mapping": { + "type": "long" + } + } + }, + { + "duration_to_long": { + "path_match": "event.duration", + "mapping": { + "type": "long" + } + } + }, + { + "pid_to_long": { + "match": "pid", + "mapping": { + "type": "long" + } + } + }, + { + "uptime_to_long": { + "match": "uptime", + "mapping": { + "type": "long" + } + } + }, + { + "sequence_to_long": { + "match": "sequence", + "mapping": { + "type": "long" + } + } + }, + { + "entropy_to_long": { + "match": "*entropy", + "mapping": { + "type": "long" + } + } + }, + { + "size_to_long": { + "match": "*size", + "mapping": { + "type": "long" + } + } + }, + { + "entrypoint_to_long": { + "match": "entrypoint", + "mapping": { + "type": "long" + } + } + }, + { + "ttl_to_long": { + "match": "ttl", + "mapping": { + "type": "long" + } + } + }, + { + "major_to_long": { + "match": "major", + "mapping": { + "type": "long" + } + } + }, + { + "minor_to_long": { + "match": "minor", + "mapping": { + "type": "long" + } + } + }, + { + "as_number_to_long": { + "path_match": "*.as.number", + "mapping": { + "type": "long" + } + } + }, + { + "pgid_to_long": { + "match": "pgid", + "mapping": { + "type": "long" + } + } + }, + { + "exit_code_to_long": { + "match": "exit_code", + "mapping": { + "type": "long" + } + } + }, + { + "chi_to_long": { + "match": "chi2", + "mapping": { + "type": "long" + } + } + }, + { + "args_count_to_long": { + "match": "args_count", + "mapping": { + "type": "long" + } + } + }, + { + "virtual_address_to_long": { + "match": "virtual_address", + "mapping": { + "type": "long" + } + } + }, + { + "io_text_to_wildcard": { + "path_match": "*.io.text", + "mapping": { + "type": "wildcard" + } + } + }, + { + "strings_to_wildcard": { + "path_match": "registry.data.strings", + "mapping": { + "type": "wildcard" + } + } + }, + { + "path_to_wildcard": { + "path_match": "*url.path", + "mapping": { + "type": "wildcard" + } + } + }, + { + "message_id_to_wildcard": { + "match": "message_id", + "mapping": { + "type": "wildcard" + } + } + }, + { + "command_line_to_multifield": { + "match": "command_line", + "mapping": { + "type": "wildcard", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "error_stack_trace_to_multifield": { + "match": "stack_trace", + "mapping": { + "type": "wildcard", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "http_content_to_multifield": { + "path_match": "*.body.content", + "mapping": { + "type": "wildcard", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "url_full_to_multifield": { + "path_match": "*.url.full", + "mapping": { + "type": "wildcard", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "url_original_to_multifield": { + "path_match": "*.url.original", + "mapping": { + "type": "wildcard", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "user_agent_original_to_multifield": { + "path_match": "user_agent.original", + "mapping": { + "type": "wildcard", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "error_message_to_match_only": { + "path_match": "error.message", + "mapping": { + "type": "match_only_text" + } + } + }, + { + "message_match_only_text": { + "path_match": "message", + "mapping": { + "type": "match_only_text" + } + } + }, + { + "agent_name_to_keyword": { + "path_match": "agent.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "service_name_to_keyword": { + "path_match": "*.service.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "sections_name_to_keyword": { + "path_match": "*.sections.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "resource_name_to_keyword": { + "path_match": "*.resource.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "observer_name_to_keyword": { + "path_match": "observer.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "question_name_to_keyword": { + "path_match": "*.question.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "group_name_to_keyword": { + "path_match": "*.group.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "geo_name_to_keyword": { + "path_match": "*.geo.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "host_name_to_keyword": { + "path_match": "host.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "severity_name_to_keyword": { + "path_match": "*.severity.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "title_to_multifield": { + "match": "title", + "mapping": { + "type": "keyword", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "executable_to_multifield": { + "match": "executable", + "mapping": { + "type": "keyword", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "file_path_to_multifield": { + "path_match": "*.file.path", + "mapping": { + "type": "keyword", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "file_target_path_to_multifield": { + "path_match": "*.file.target_path", + "mapping": { + "type": "keyword", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "name_to_multifield": { + "match": "name", + "mapping": { + "type": "keyword", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "full_name_to_multifield": { + "match": "full_name", + "mapping": { + "type": "keyword", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "os_full_to_multifield": { + "path_match": "*.os.full", + "mapping": { + "type": "keyword", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "working_directory_to_multifield": { + "match": "working_directory", + "mapping": { + "type": "keyword", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "timestamp_to_date": { + "match": "timestamp", + "mapping": { + "type": "date" + } + } + }, + { + "delivery_timestamp_to_date": { + "match": "delivery_timestamp", + "mapping": { + "type": "date" + } + } + }, + { + "not_after_to_date": { + "match": "not_after", + "mapping": { + "type": "date" + } + } + }, + { + "not_before_to_date": { + "match": "not_before", + "mapping": { + "type": "date" + } + } + }, + { + "accessed_to_date": { + "match": "accessed", + "mapping": { + "type": "date" + } + } + }, + { + "origination_timestamp_to_date": { + "match": "origination_timestamp", + "mapping": { + "type": "date" + } + } + }, + { + "created_to_date": { + "match": "created", + "mapping": { + "type": "date" + } + } + }, + { + "installed_to_date": { + "match": "installed", + "mapping": { + "type": "date" + } + } + }, + { + "creation_date_to_date": { + "match": "creation_date", + "mapping": { + "type": "date" + } + } + }, + { + "ctime_to_date": { + "match": "ctime", + "mapping": { + "type": "date" + } + } + }, + { + "mtime_to_date": { + "match": "mtime", + "mapping": { + "type": "date" + } + } + }, + { + "ingested_to_date": { + "match": "ingested", + "mapping": { + "type": "date" + } + } + }, + { + "start_to_date": { + "match": "start", + "mapping": { + "type": "date" + } + } + }, + { + "end_to_date": { + "match": "end", + "mapping": { + "type": "date" + } + } + }, + { + "score_base_to_float": { + "path_match": "*.score.base", + "mapping": { + "type": "float" + } + } + }, + { + "score_temporal_to_float": { + "path_match": "*.score.temporal", + "mapping": { + "type": "float" + } + } + }, + { + "score_to_float": { + "match": "*_score", + "mapping": { + "type": "float" + } + } + }, + { + "score_norm_to_float": { + "match": "*_score_norm", + "mapping": { + "type": "float" + } + } + }, + { + "usage_to_float": { + "match": "usage", + "mapping": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + { + "location_to_geo_point": { + "match": "location", + "mapping": { + "type": "geo_point" + } + } + }, + { + "same_as_process_to_boolean": { + "match": "same_as_process", + "mapping": { + "type": "boolean" + } + } + }, + { + "established_to_boolean": { + "match": "established", + "mapping": { + "type": "boolean" + } + } + }, + { + "resumed_to_boolean": { + "match": "resumed", + "mapping": { + "type": "boolean" + } + } + }, + { + "max_bytes_per_process_exceeded_to_boolean": { + "match": "max_bytes_per_process_exceeded", + "mapping": { + "type": "boolean" + } + } + }, + { + "interactive_to_boolean": { + "match": "interactive", + "mapping": { + "type": "boolean" + } + } + }, + { + "exists_to_boolean": { + "match": "exists", + "mapping": { + "type": "boolean" + } + } + }, + { + "trusted_to_boolean": { + "match": "trusted", + "mapping": { + "type": "boolean" + } + } + }, + { + "valid_to_boolean": { + "match": "valid", + "mapping": { + "type": "boolean" + } + } + }, + { + "go_stripped_to_boolean": { + "match": "go_stripped", + "mapping": { + "type": "boolean" + } + } + }, + { + "coldstart_to_boolean": { + "match": "coldstart", + "mapping": { + "type": "boolean" + } + } + }, + { + "exports_to_flattened": { + "match": "exports", + "mapping": { + "type": "flattened" + } + } + }, + { + "structured_data_to_flattened": { + "match": "structured_data", + "mapping": { + "type": "flattened" + } + } + }, + { + "imports_to_flattened": { + "match": "*imports", + "mapping": { + "type": "flattened" + } + } + }, + { + "attachments_to_nested": { + "match": "attachments", + "mapping": { + "type": "nested" + } + } + }, + { + "segments_to_nested": { + "match": "segments", + "mapping": { + "type": "nested" + } + } + }, + { + "elf_sections_to_nested": { + "path_match": "*.elf.sections", + "mapping": { + "type": "nested" + } + } + }, + { + "pe_sections_to_nested": { + "path_match": "*.pe.sections", + "mapping": { + "type": "nested" + } + } + }, + { + "macho_sections_to_nested": { + "path_match": "*.macho.sections", + "mapping": { + "type": "nested" + } + } + }, + { + "trigger_to_nested": { + "match": "trigger", + "mapping": { + "type": "nested" + } + } + } + ] + } +} diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go new file mode 100644 index 000000000..356bf9ed9 --- /dev/null +++ b/internal/builder/dynamic_mappings.go @@ -0,0 +1,246 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package builder + +import ( + _ "embed" + "fmt" + "os" + "path/filepath" + "strconv" + + "github.com/Masterminds/semver/v3" + "github.com/pkg/errors" + "gopkg.in/yaml.v3" + + "github.com/elastic/elastic-package/internal/formatter" + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/packages/buildmanifest" +) + +//go:embed _static/ecs_mappings.json +var staticEcsMappings string + +const prefixMapping = "_embedded_ecs" + +var semver2_3_0 = semver.MustParse("2.3.0") + +type ecsTemplates struct { + Mappings struct { + DynamicTemplates []map[string]interface{} `yaml:"dynamic_templates"` + } `yaml:"mappings"` +} + +func addDynamicMappings(packageRoot, destinationDir string) error { + packageManifest := filepath.Join(destinationDir, packages.PackageManifestFile) + + m, err := packages.ReadPackageManifest(packageManifest) + if err != nil { + return err + } + + shouldImport, err := shouldImportEcsMappings(m.SpecVersion, packageRoot) + if err != nil { + return err + } + if !shouldImport { + return nil + } + + logger.Debug("Import ECS mappings into the built package") + + switch m.Type { + case "integration": + dataStreamManifests, err := filepath.Glob(filepath.Join(destinationDir, "data_stream", "*", packages.DataStreamManifestFile)) + if err != nil { + return err + } + + for _, datastream := range dataStreamManifests { + contents, err := addDynamicMappingElements(datastream) + if err != nil { + return err + } + err = os.WriteFile(datastream, contents, 0664) + if err != nil { + return err + } + } + case "input": + contents, err := addDynamicMappingElements(packageManifest) + if err != nil { + return err + } + os.WriteFile(packageManifest, contents, 0664) + if err != nil { + return err + } + } + + return nil +} + +func shouldImportEcsMappings(specVersion, packageRoot string) (bool, error) { + v, err := semver.NewVersion(specVersion) + if err != nil { + return false, errors.Wrap(err, "invalid spec version") + } + + if v.LessThan(semver2_3_0) { + logger.Debugf("Required spec version >= %s to import ECS mappings", semver2_3_0.String()) + return false, nil + } + + bm, ok, err := buildmanifest.ReadBuildManifest(packageRoot) + if err != nil { + return false, errors.Wrap(err, "can't read build manifest") + } + if !ok { + logger.Debug("Build manifest hasn't been defined for the package") + return false, nil + } + if !bm.ImportMappings() { + logger.Debug("Package doesn't have to import ECS mappings") + return false, nil + } + return true, nil +} + +func addDynamicMappingElements(path string) ([]byte, error) { + ecsMappings, err := loadEcsMappings() + if err != nil { + return nil, errors.New("can't load ecs mappings template") + } + + contents, err := os.ReadFile(path) + if err != nil { + return nil, errors.New("can't read manifest") + } + + var doc yaml.Node + err = yaml.Unmarshal(contents, &doc) + if err != nil { + return nil, err + } + + err = addEcsMappings(&doc, ecsMappings) + if err != nil { + return nil, err + } + + contents, err = formatResult(&doc) + if err != nil { + return nil, err + } + + return contents, nil +} + +func loadEcsMappings() (ecsTemplates, error) { + var ecsMappings ecsTemplates + err := yaml.Unmarshal([]byte(staticEcsMappings), &ecsMappings) + if err != nil { + return ecsMappings, err + } + return ecsMappings, nil +} + +func addEcsMappings(doc *yaml.Node, mappings ecsTemplates) error { + var templates yaml.Node + err := templates.Encode(mappings.Mappings.DynamicTemplates) + if err != nil { + return errors.Wrap(err, "failed to encode dynamic templates") + } + + renameMappingsNames(&templates) + + err = appendElements(doc, []string{"elasticsearch", "index_template", "mappings", "dynamic_templates"}, &templates) + if err != nil { + return errors.Wrap(err, "failed to append dynamic templates") + } + + return nil +} + +func renameMappingsNames(doc *yaml.Node) { + switch doc.Kind { + case yaml.MappingNode: + for i := 0; i < len(doc.Content); i += 2 { + doc.Content[i].Value = fmt.Sprintf("%s-%s", prefixMapping, doc.Content[i].Value) + } + case yaml.SequenceNode: + for i := 0; i < len(doc.Content); i++ { + renameMappingsNames(doc.Content[i]) + } + case yaml.DocumentNode: + renameMappingsNames(doc.Content[0]) + } +} + +func appendElements(root *yaml.Node, path []string, values *yaml.Node) error { + if len(path) == 0 { + contents := values.Content + if values.Kind == yaml.DocumentNode { + contents = values.Content[0].Content + } + root.Content = append(root.Content, contents...) + return nil + } + + key := path[0] + rest := path[1:] + + switch root.Kind { + case yaml.DocumentNode: + return appendElements(root.Content[0], path, values) + case yaml.MappingNode: + for i := 0; i < len(root.Content); i += 2 { + child := root.Content[i] + if child.Value == key { + return appendElements(root.Content[i+1], rest, values) + } + } + newContentNodes := newYamlNode(key) + + root.Content = append(root.Content, newContentNodes...) + return appendElements(newContentNodes[1], rest, values) + case yaml.SequenceNode: + index, err := strconv.Atoi(key) + if err != nil { + return err + } + if len(root.Content) >= index { + return errors.Errorf("index out of range in nodes from key %s", key) + } + + return appendElements(root.Content[index], rest, values) + } + return nil +} + +func newYamlNode(key string) []*yaml.Node { + keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key} + var childNode *yaml.Node + switch key { + case "dynamic_templates": + childNode = &yaml.Node{Kind: yaml.SequenceNode, Value: key} + default: + childNode = &yaml.Node{Kind: yaml.MappingNode, Value: key} + } + return []*yaml.Node{keyNode, childNode} +} + +func formatResult(result interface{}) ([]byte, error) { + d, err := yaml.Marshal(result) + if err != nil { + return nil, errors.New("failed to encode") + } + d, _, err = formatter.YAMLFormatter(d) + if err != nil { + return nil, errors.New("failed to format") + } + return d, nil +} diff --git a/internal/builder/dynamic_mappings_test.go b/internal/builder/dynamic_mappings_test.go new file mode 100644 index 000000000..3eca73aac --- /dev/null +++ b/internal/builder/dynamic_mappings_test.go @@ -0,0 +1,59 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package builder + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestAddEcsMappings(t *testing.T) { + cases := []struct { + title string + baseDocPath string + ecsToAddPath string + expectedPath string + }{ + { + title: "Add new mappings from scratch", + baseDocPath: "testdata/empty.yml", + ecsToAddPath: "testdata/ecs.template.yml", + expectedPath: "testdata/expected.empty.mappings.yml", + }, + { + title: "Append mappings", + baseDocPath: "testdata/existing.yml", + ecsToAddPath: "testdata/ecs.template.yml", + expectedPath: "testdata/expected.existing.mappings.yml", + }, + } + + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + contentsBase, _ := os.ReadFile(c.baseDocPath) + contentsToAdd, _ := os.ReadFile(c.ecsToAddPath) + contentsExpected, _ := os.ReadFile(c.expectedPath) + + var base yaml.Node + err := yaml.Unmarshal(contentsBase, &base) + require.NoError(t, err) + + var template ecsTemplates + err = yaml.Unmarshal(contentsToAdd, &template) + require.NoError(t, err) + + err = addEcsMappings(&base, template) + require.NoError(t, err) + + newYaml, _ := formatResult(&base) + + assert.Equal(t, string(contentsExpected), string(newYaml)) + }) + } +} diff --git a/internal/builder/packages.go b/internal/builder/packages.go index d4000873d..f7f40db17 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -180,6 +180,12 @@ func BuildPackage(options BuildOptions) (string, error) { return "", errors.Wrap(err, "resolving external fields failed") } + logger.Debug("Add dynamic mappings if needed") + err = addDynamicMappings(options.PackageRoot, destinationDir) + if err != nil { + return "", errors.Wrap(err, "adding dynamic mappings") + } + if options.CreateZip { return buildZippedPackage(options, destinationDir) } @@ -189,6 +195,7 @@ func BuildPackage(options BuildOptions) (string, error) { return destinationDir, nil } + logger.Debugf("Validating built package (path: %s)", destinationDir) err = validator.ValidateFromPath(destinationDir) if err != nil { return "", errors.Wrap(err, "invalid content found in built package") @@ -211,6 +218,7 @@ func buildZippedPackage(options BuildOptions, destinationDir string) (string, er if options.SkipValidation { logger.Debug("Skip validation of the built .zip package") } else { + logger.Debugf("Validating built .zip package (path: %s)", zippedPackagePath) err = validator.ValidateFromZip(zippedPackagePath) if err != nil { return "", errors.Wrapf(err, "invalid content found in built zip package") diff --git a/internal/builder/testdata/ecs.template.yml b/internal/builder/testdata/ecs.template.yml new file mode 100644 index 000000000..b8a0363fe --- /dev/null +++ b/internal/builder/testdata/ecs.template.yml @@ -0,0 +1,9 @@ +mappings: + properties: + datetime: + type: long + dynamic_templates: + - example_template: + patch_match: 'example.*' + mapping: + type: short diff --git a/internal/builder/testdata/empty.yml b/internal/builder/testdata/empty.yml new file mode 100644 index 000000000..670dee7f9 --- /dev/null +++ b/internal/builder/testdata/empty.yml @@ -0,0 +1 @@ +title: empty diff --git a/internal/builder/testdata/existing.yml b/internal/builder/testdata/existing.yml new file mode 100644 index 000000000..a17128725 --- /dev/null +++ b/internal/builder/testdata/existing.yml @@ -0,0 +1,13 @@ +title: existing + +elasticsearch: + index_template: + mappings: + properties: + example: + type: short + dynamic_templates: + - to_nested: + path: trigger + mapping: + type: nested diff --git a/internal/builder/testdata/expected.empty.mappings.yml b/internal/builder/testdata/expected.empty.mappings.yml new file mode 100644 index 000000000..706abec3b --- /dev/null +++ b/internal/builder/testdata/expected.empty.mappings.yml @@ -0,0 +1,9 @@ +title: empty +elasticsearch: + index_template: + mappings: + dynamic_templates: + - _embedded_ecs-example_template: + mapping: + type: short + patch_match: example.* diff --git a/internal/builder/testdata/expected.existing.mappings.yml b/internal/builder/testdata/expected.existing.mappings.yml new file mode 100644 index 000000000..28f168641 --- /dev/null +++ b/internal/builder/testdata/expected.existing.mappings.yml @@ -0,0 +1,16 @@ +title: existing +elasticsearch: + index_template: + mappings: + properties: + example: + type: short + dynamic_templates: + - to_nested: + path: trigger + mapping: + type: nested + - _embedded_ecs-example_template: + mapping: + type: short + patch_match: example.* diff --git a/internal/packages/buildmanifest/build_manifest.go b/internal/packages/buildmanifest/build_manifest.go index 956bada43..45f6345d9 100644 --- a/internal/packages/buildmanifest/build_manifest.go +++ b/internal/packages/buildmanifest/build_manifest.go @@ -8,9 +8,10 @@ import ( "os" "path/filepath" + "github.com/pkg/errors" + "github.com/elastic/go-ucfg" "github.com/elastic/go-ucfg/yaml" - "github.com/pkg/errors" ) // BuildManifest defines the manifest defining the building procedure. @@ -25,7 +26,8 @@ type Dependencies struct { // ECSDependency defines a dependency on ECS fields. type ECSDependency struct { - Reference string `config:"reference"` + Reference string `config:"reference"` + ImportMappings bool `config:"import_mappings"` } // HasDependencies function checks if there are any dependencies defined. @@ -33,6 +35,11 @@ func (bm *BuildManifest) HasDependencies() bool { return bm.Dependencies.ECS.Reference != "" } +// HasDependencies function checks if there are any dependencies defined. +func (bm *BuildManifest) ImportMappings() bool { + return bm.Dependencies.ECS.ImportMappings +} + // ReadBuildManifest function reads the package build manifest. func ReadBuildManifest(packageRoot string) (*BuildManifest, bool, error) { path := buildManifestPath(packageRoot) From 2794a56158509905f232362a94a406a94e4fdf63 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 13 Jan 2023 18:42:08 +0100 Subject: [PATCH 13/33] Add ECS mappings during test runner executions (#1090) Add ECS mappings from the ECS reference defined at "_dev/build/build.yml" during test runner executions to allow check that all fields in the sample event are defined. These mapping are just added in case "import_mappings" is enabled and the format_version of the package is greater than or equal to 2.3.0. A new test package has been added to check this new feature. --- internal/builder/dynamic_mappings.go | 2 +- internal/builder/packages.go | 1 - internal/fields/dependency_manager.go | 13 +++ internal/fields/validate.go | 44 +++++++++- internal/fields/validate_test.go | 46 ++++++++++ .../testrunner/runners/pipeline/runner.go | 1 + internal/testrunner/runners/static/runner.go | 1 + internal/testrunner/runners/system/runner.go | 1 + .../_dev/build/build.yml | 4 + .../_dev/build/docs/README.md | 9 ++ .../imported_mappings_tests/changelog.yml | 6 ++ .../first/agent/stream/stream.yml.hbs | 7 ++ .../elasticsearch/ingest_pipeline/default.yml | 10 +++ .../data_stream/first/fields/base-fields.yml | 12 +++ .../first/fields/histogram-fields.yml | 3 + .../data_stream/first/manifest.yml | 13 +++ .../data_stream/first/sample_event.json | 26 ++++++ .../second/agent/stream/stream.yml.hbs | 7 ++ .../elasticsearch/ingest_pipeline/default.yml | 10 +++ .../data_stream/second/fields/base-fields.yml | 12 +++ .../data_stream/second/fields/geo-fields.yml | 2 + .../second/fields/histogram-fields.yml | 3 + .../data_stream/second/manifest.yml | 13 +++ .../data_stream/second/sample_event.json | 26 ++++++ .../imported_mappings_tests/docs/README.md | 85 +++++++++++++++++++ .../imported_mappings_tests/manifest.yml | 20 +++++ 26 files changed, 373 insertions(+), 4 deletions(-) create mode 100644 test/packages/other/imported_mappings_tests/_dev/build/build.yml create mode 100644 test/packages/other/imported_mappings_tests/_dev/build/docs/README.md create mode 100644 test/packages/other/imported_mappings_tests/changelog.yml create mode 100644 test/packages/other/imported_mappings_tests/data_stream/first/agent/stream/stream.yml.hbs create mode 100644 test/packages/other/imported_mappings_tests/data_stream/first/elasticsearch/ingest_pipeline/default.yml create mode 100644 test/packages/other/imported_mappings_tests/data_stream/first/fields/base-fields.yml create mode 100644 test/packages/other/imported_mappings_tests/data_stream/first/fields/histogram-fields.yml create mode 100644 test/packages/other/imported_mappings_tests/data_stream/first/manifest.yml create mode 100644 test/packages/other/imported_mappings_tests/data_stream/first/sample_event.json create mode 100644 test/packages/other/imported_mappings_tests/data_stream/second/agent/stream/stream.yml.hbs create mode 100644 test/packages/other/imported_mappings_tests/data_stream/second/elasticsearch/ingest_pipeline/default.yml create mode 100644 test/packages/other/imported_mappings_tests/data_stream/second/fields/base-fields.yml create mode 100644 test/packages/other/imported_mappings_tests/data_stream/second/fields/geo-fields.yml create mode 100644 test/packages/other/imported_mappings_tests/data_stream/second/fields/histogram-fields.yml create mode 100644 test/packages/other/imported_mappings_tests/data_stream/second/manifest.yml create mode 100644 test/packages/other/imported_mappings_tests/data_stream/second/sample_event.json create mode 100644 test/packages/other/imported_mappings_tests/docs/README.md create mode 100644 test/packages/other/imported_mappings_tests/manifest.yml diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index 356bf9ed9..2cb85f0a1 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -50,7 +50,7 @@ func addDynamicMappings(packageRoot, destinationDir string) error { return nil } - logger.Debug("Import ECS mappings into the built package") + logger.Info("Import ECS mappings into the built package (technical preview)") switch m.Type { case "integration": diff --git a/internal/builder/packages.go b/internal/builder/packages.go index f7f40db17..9e28fd11c 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -180,7 +180,6 @@ func BuildPackage(options BuildOptions) (string, error) { return "", errors.Wrap(err, "resolving external fields failed") } - logger.Debug("Add dynamic mappings if needed") err = addDynamicMappings(options.PackageRoot, destinationDir) if err != nil { return "", errors.Wrap(err, "adding dynamic mappings") diff --git a/internal/fields/dependency_manager.go b/internal/fields/dependency_manager.go index 10a199fea..d9abc1245 100644 --- a/internal/fields/dependency_manager.go +++ b/internal/fields/dependency_manager.go @@ -233,6 +233,19 @@ func (dm *DependencyManager) ImportField(schemaName, fieldPath string) (FieldDef return *imported, nil } +// ImportAllFields method resolves all fields avaialble in the default ECS schema. +func (dm *DependencyManager) ImportAllFields(schemaName string) ([]FieldDefinition, error) { + if dm == nil { + return []FieldDefinition{}, fmt.Errorf(`importing all external fields: external fields not allowed because dependencies file "_dev/build/build.yml" is missing`) + } + schema, ok := dm.schema[schemaName] + if !ok { + return []FieldDefinition{}, fmt.Errorf(`schema "%s" is not defined as package depedency`, schemaName) + } + + return schema, nil +} + func buildFieldPath(root string, field common.MapStr) string { path := root if root != "" { diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 431e6d7dc..76eb564a3 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -26,7 +26,12 @@ import ( "github.com/elastic/elastic-package/internal/packages/buildmanifest" ) -var semver2_0_0 = semver.MustParse("2.0.0") +var ( + semver2_0_0 = semver.MustParse("2.0.0") + semver2_3_0 = semver.MustParse("2.3.0") + + defaultExternal = "ecs" +) // Validator is responsible for fields validation. type Validator struct { @@ -48,6 +53,8 @@ type Validator struct { enabledAllowedIPCheck bool allowedCIDRs []*net.IPNet + + enabledImportAllECSSchema bool } // ValidatorOption represents an optional flag that can be passed to CreateValidatorForDirectory. @@ -109,8 +116,31 @@ func WithExpectedDataset(dataset string) ValidatorOption { } } +// WithEnabledImportAllECSSchema configures the validator to check or not the fields with the complete ECS schema. +func WithEnabledImportAllECSSChema(importSchema bool) ValidatorOption { + return func(v *Validator) error { + v.enabledImportAllECSSchema = importSchema + return nil + } +} + +type packageRootFinder interface { + FindPackageRoot() (string, bool, error) +} + +type packageRoot struct{} + +func (p packageRoot) FindPackageRoot() (string, bool, error) { + return packages.FindPackageRoot() +} + // CreateValidatorForDirectory function creates a validator for the directory. func CreateValidatorForDirectory(fieldsParentDir string, opts ...ValidatorOption) (v *Validator, err error) { + p := packageRoot{} + return createValidatorForDirectoryAndPackageRoot(fieldsParentDir, p, opts...) +} + +func createValidatorForDirectoryAndPackageRoot(fieldsParentDir string, finder packageRootFinder, opts ...ValidatorOption) (v *Validator, err error) { v = new(Validator) for _, opt := range opts { if err := opt(v); err != nil { @@ -130,7 +160,7 @@ func CreateValidatorForDirectory(fieldsParentDir string, opts ...ValidatorOption return v, nil } - packageRoot, found, err := packages.FindPackageRoot() + packageRoot, found, err := finder.FindPackageRoot() if err != nil { return nil, errors.Wrap(err, "can't find package root") } @@ -156,6 +186,15 @@ func CreateValidatorForDirectory(fieldsParentDir string, opts ...ValidatorOption return nil, errors.Wrap(err, "can't create field dependency manager") } v.FieldDependencyManager = fdm + + if bm.ImportMappings() && !v.specVersion.LessThan(semver2_3_0) && v.enabledImportAllECSSchema { + ecsSchema, err := v.FieldDependencyManager.ImportAllFields(defaultExternal) + if err != nil { + return nil, err + } + v.Schema = append(v.Schema, ecsSchema...) + } + return v, nil } @@ -287,6 +326,7 @@ func (v *Validator) validateScalarElement(key string, val interface{}, doc commo if definition == nil && skipValidationForField(key) { return nil // generic field, let's skip validation for now } + if definition == nil { return fmt.Errorf(`field "%s" is undefined`, key) } diff --git a/internal/fields/validate_test.go b/internal/fields/validate_test.go index 42b60141f..592914a4a 100644 --- a/internal/fields/validate_test.go +++ b/internal/fields/validate_test.go @@ -7,6 +7,7 @@ package fields import ( "encoding/json" "os" + "sort" "testing" "github.com/stretchr/testify/assert" @@ -19,6 +20,14 @@ type results struct { Expected []json.RawMessage } +type packageRootTestFinder struct { + packageRootPath string +} + +func (p packageRootTestFinder) FindPackageRoot() (string, bool, error) { + return p.packageRootPath, true, nil +} + func TestValidate_NoWildcardFields(t *testing.T) { validator, err := CreateValidatorForDirectory("../../test/packages/parallel/aws/data_stream/elb_logs") require.NoError(t, err) @@ -64,6 +73,43 @@ func TestValidate_WithNumericKeywordFields(t *testing.T) { require.Empty(t, errs) } +func TestValidate_WithEnabledImportAllECSSchema(t *testing.T) { + finder := packageRootTestFinder{"../../test/packages/other/imported_mappings_tests"} + + validator, err := createValidatorForDirectoryAndPackageRoot("../../test/packages/other/imported_mappings_tests/data_stream/first", + finder, + WithSpecVersion("2.3.0"), + WithEnabledImportAllECSSChema(true)) + require.NoError(t, err) + require.NotNil(t, validator) + + e := readSampleEvent(t, "../../test/packages/other/imported_mappings_tests/data_stream/first/sample_event.json") + errs := validator.ValidateDocumentBody(e) + require.Empty(t, errs) +} + +func TestValidate_WithDisabledImportAllECSSchema(t *testing.T) { + finder := packageRootTestFinder{"../../test/packages/other/imported_mappings_tests"} + + validator, err := createValidatorForDirectoryAndPackageRoot("../../test/packages/other/imported_mappings_tests/data_stream/first", + finder, + WithSpecVersion("2.3.0"), + WithEnabledImportAllECSSChema(false)) + require.NoError(t, err) + require.NotNil(t, validator) + + e := readSampleEvent(t, "../../test/packages/other/imported_mappings_tests/data_stream/first/sample_event.json") + errs := validator.ValidateDocumentBody(e) + require.Len(t, errs, 4) + + errorMessages := []string{} + for _, err := range errs { + errorMessages = append(errorMessages, err.Error()) + } + sort.Strings(errorMessages) + require.Contains(t, errorMessages[0], `field "destination.geo.location.lat" is undefined`) +} + func TestValidate_constantKeyword(t *testing.T) { validator, err := CreateValidatorForDirectory("testdata") require.NoError(t, err) diff --git a/internal/testrunner/runners/pipeline/runner.go b/internal/testrunner/runners/pipeline/runner.go index 44f05be09..afdf5c1ff 100644 --- a/internal/testrunner/runners/pipeline/runner.go +++ b/internal/testrunner/runners/pipeline/runner.go @@ -157,6 +157,7 @@ func (r *runner) run() ([]testrunner.TestResult, error) { // since system tests can have dynamic public IPs fields.WithEnabledAllowedIPCheck(), fields.WithExpectedDataset(expectedDataset), + fields.WithEnabledImportAllECSSChema(true), ) if err != nil { return nil, errors.Wrapf(err, "creating fields validator for data stream failed (path: %s, test case file: %s)", dataStreamPath, testCaseFile) diff --git a/internal/testrunner/runners/static/runner.go b/internal/testrunner/runners/static/runner.go index 8c870f95b..762b5f4c0 100644 --- a/internal/testrunner/runners/static/runner.go +++ b/internal/testrunner/runners/static/runner.go @@ -100,6 +100,7 @@ func (r runner) verifySampleEvent(pkgManifest *packages.PackageManifest) []testr fields.WithSpecVersion(pkgManifest.SpecVersion), fields.WithDefaultNumericConversion(), fields.WithExpectedDataset(expectedDataset), + fields.WithEnabledImportAllECSSChema(true), ) if err != nil { results, _ := resultComposer.WithError(errors.Wrap(err, "creating fields validator for data stream failed")) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index efb8a9104..3de91edf8 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -472,6 +472,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext fields.WithSpecVersion(pkgManifest.SpecVersion), fields.WithNumericKeywordFields(config.NumericKeywordFields), fields.WithExpectedDataset(expectedDataset), + fields.WithEnabledImportAllECSSChema(true), ) if err != nil { return result.WithError(errors.Wrapf(err, "creating fields validator for data stream failed (path: %s)", serviceOptions.DataStreamRootPath)) diff --git a/test/packages/other/imported_mappings_tests/_dev/build/build.yml b/test/packages/other/imported_mappings_tests/_dev/build/build.yml new file mode 100644 index 000000000..8a08f65de --- /dev/null +++ b/test/packages/other/imported_mappings_tests/_dev/build/build.yml @@ -0,0 +1,4 @@ +dependencies: + ecs: + reference: git@v8.5.2 + import_mappings: true diff --git a/test/packages/other/imported_mappings_tests/_dev/build/docs/README.md b/test/packages/other/imported_mappings_tests/_dev/build/docs/README.md new file mode 100644 index 000000000..591d5fa57 --- /dev/null +++ b/test/packages/other/imported_mappings_tests/_dev/build/docs/README.md @@ -0,0 +1,9 @@ +# Imported Mappings Tests + +{{event "first"}} + +{{fields "first"}} + +{{event "second"}} + +{{fields "second"}} \ No newline at end of file diff --git a/test/packages/other/imported_mappings_tests/changelog.yml b/test/packages/other/imported_mappings_tests/changelog.yml new file mode 100644 index 000000000..bb0320a52 --- /dev/null +++ b/test/packages/other/imported_mappings_tests/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link diff --git a/test/packages/other/imported_mappings_tests/data_stream/first/agent/stream/stream.yml.hbs b/test/packages/other/imported_mappings_tests/data_stream/first/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..5845510de --- /dev/null +++ b/test/packages/other/imported_mappings_tests/data_stream/first/agent/stream/stream.yml.hbs @@ -0,0 +1,7 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$"] +processors: + - add_locale: ~ diff --git a/test/packages/other/imported_mappings_tests/data_stream/first/elasticsearch/ingest_pipeline/default.yml b/test/packages/other/imported_mappings_tests/data_stream/first/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000..81221adf3 --- /dev/null +++ b/test/packages/other/imported_mappings_tests/data_stream/first/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,10 @@ +--- +description: Pipeline for processing sample logs +processors: +- set: + field: sample_field + value: "1" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/test/packages/other/imported_mappings_tests/data_stream/first/fields/base-fields.yml b/test/packages/other/imported_mappings_tests/data_stream/first/fields/base-fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/other/imported_mappings_tests/data_stream/first/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/other/imported_mappings_tests/data_stream/first/fields/histogram-fields.yml b/test/packages/other/imported_mappings_tests/data_stream/first/fields/histogram-fields.yml new file mode 100644 index 000000000..128a0cb1c --- /dev/null +++ b/test/packages/other/imported_mappings_tests/data_stream/first/fields/histogram-fields.yml @@ -0,0 +1,3 @@ +- name: service.status.*.histogram + type: object + object_type: histogram diff --git a/test/packages/other/imported_mappings_tests/data_stream/first/manifest.yml b/test/packages/other/imported_mappings_tests/data_stream/first/manifest.yml new file mode 100644 index 000000000..979ef29d6 --- /dev/null +++ b/test/packages/other/imported_mappings_tests/data_stream/first/manifest.yml @@ -0,0 +1,13 @@ +title: "First" +type: logs +streams: + - input: logfile + title: Sample logs + description: Collect sample logs + vars: + - name: paths + type: text + title: Paths + multi: true + default: + - /var/log/*.log diff --git a/test/packages/other/imported_mappings_tests/data_stream/first/sample_event.json b/test/packages/other/imported_mappings_tests/data_stream/first/sample_event.json new file mode 100644 index 000000000..a242024f5 --- /dev/null +++ b/test/packages/other/imported_mappings_tests/data_stream/first/sample_event.json @@ -0,0 +1,26 @@ +{ + "source.geo.location": { + "lat": 1.0, + "lon": "2.0" + }, + "destination.geo.location.lat": 3.0, + "destination.geo.location.lon": 4.0, + "service.status.duration.histogram": { + "counts": [ + 8, + 17, + 8, + 7, + 6, + 2 + ], + "values": [ + 0.1, + 0.25, + 0.35, + 0.4, + 0.45, + 0.5 + ] + } +} \ No newline at end of file diff --git a/test/packages/other/imported_mappings_tests/data_stream/second/agent/stream/stream.yml.hbs b/test/packages/other/imported_mappings_tests/data_stream/second/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..5845510de --- /dev/null +++ b/test/packages/other/imported_mappings_tests/data_stream/second/agent/stream/stream.yml.hbs @@ -0,0 +1,7 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$"] +processors: + - add_locale: ~ diff --git a/test/packages/other/imported_mappings_tests/data_stream/second/elasticsearch/ingest_pipeline/default.yml b/test/packages/other/imported_mappings_tests/data_stream/second/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000..81221adf3 --- /dev/null +++ b/test/packages/other/imported_mappings_tests/data_stream/second/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,10 @@ +--- +description: Pipeline for processing sample logs +processors: +- set: + field: sample_field + value: "1" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/test/packages/other/imported_mappings_tests/data_stream/second/fields/base-fields.yml b/test/packages/other/imported_mappings_tests/data_stream/second/fields/base-fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/other/imported_mappings_tests/data_stream/second/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/other/imported_mappings_tests/data_stream/second/fields/geo-fields.yml b/test/packages/other/imported_mappings_tests/data_stream/second/fields/geo-fields.yml new file mode 100644 index 000000000..a618607d3 --- /dev/null +++ b/test/packages/other/imported_mappings_tests/data_stream/second/fields/geo-fields.yml @@ -0,0 +1,2 @@ +- name: destination.geo.location + external: ecs diff --git a/test/packages/other/imported_mappings_tests/data_stream/second/fields/histogram-fields.yml b/test/packages/other/imported_mappings_tests/data_stream/second/fields/histogram-fields.yml new file mode 100644 index 000000000..128a0cb1c --- /dev/null +++ b/test/packages/other/imported_mappings_tests/data_stream/second/fields/histogram-fields.yml @@ -0,0 +1,3 @@ +- name: service.status.*.histogram + type: object + object_type: histogram diff --git a/test/packages/other/imported_mappings_tests/data_stream/second/manifest.yml b/test/packages/other/imported_mappings_tests/data_stream/second/manifest.yml new file mode 100644 index 000000000..979ef29d6 --- /dev/null +++ b/test/packages/other/imported_mappings_tests/data_stream/second/manifest.yml @@ -0,0 +1,13 @@ +title: "First" +type: logs +streams: + - input: logfile + title: Sample logs + description: Collect sample logs + vars: + - name: paths + type: text + title: Paths + multi: true + default: + - /var/log/*.log diff --git a/test/packages/other/imported_mappings_tests/data_stream/second/sample_event.json b/test/packages/other/imported_mappings_tests/data_stream/second/sample_event.json new file mode 100644 index 000000000..a242024f5 --- /dev/null +++ b/test/packages/other/imported_mappings_tests/data_stream/second/sample_event.json @@ -0,0 +1,26 @@ +{ + "source.geo.location": { + "lat": 1.0, + "lon": "2.0" + }, + "destination.geo.location.lat": 3.0, + "destination.geo.location.lon": 4.0, + "service.status.duration.histogram": { + "counts": [ + 8, + 17, + 8, + 7, + 6, + 2 + ], + "values": [ + 0.1, + 0.25, + 0.35, + 0.4, + 0.45, + 0.5 + ] + } +} \ No newline at end of file diff --git a/test/packages/other/imported_mappings_tests/docs/README.md b/test/packages/other/imported_mappings_tests/docs/README.md new file mode 100644 index 000000000..6f99a892f --- /dev/null +++ b/test/packages/other/imported_mappings_tests/docs/README.md @@ -0,0 +1,85 @@ +# Imported Mappings Tests + +An example event for `first` looks as following: + +```json +{ + "source.geo.location": { + "lat": 1.0, + "lon": "2.0" + }, + "destination.geo.location.lat": 3.0, + "destination.geo.location.lon": 4.0, + "service.status.duration.histogram": { + "counts": [ + 8, + 17, + 8, + 7, + 6, + 2 + ], + "values": [ + 0.1, + 0.25, + 0.35, + 0.4, + 0.45, + 0.5 + ] + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| service.status.\*.histogram | | object | + + +An example event for `second` looks as following: + +```json +{ + "source.geo.location": { + "lat": 1.0, + "lon": "2.0" + }, + "destination.geo.location.lat": 3.0, + "destination.geo.location.lon": 4.0, + "service.status.duration.histogram": { + "counts": [ + 8, + 17, + 8, + 7, + 6, + 2 + ], + "values": [ + 0.1, + 0.25, + 0.35, + 0.4, + 0.45, + 0.5 + ] + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| destination.geo.location | Longitude and latitude. | geo_point | +| service.status.\*.histogram | | object | diff --git a/test/packages/other/imported_mappings_tests/manifest.yml b/test/packages/other/imported_mappings_tests/manifest.yml new file mode 100644 index 000000000..e5b556193 --- /dev/null +++ b/test/packages/other/imported_mappings_tests/manifest.yml @@ -0,0 +1,20 @@ +format_version: 2.3.0 +name: imported_mappings_tests +title: "Imported Mappings Tests" +version: 0.0.1 +description: "These are tests of field validation with imported mappings." +type: integration +categories: + - custom +conditions: + kibana.version: "^8.0.0" +policy_templates: + - name: sample + title: Sample logs + description: Collect sample logs + inputs: + - type: logfile + title: Collect sample logs from instances + description: Collecting sample logs +owner: + github: elastic/integrations From bbb1eeb09285923e32026cb335660a600c85f553 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 16 Jan 2023 15:25:33 +0100 Subject: [PATCH 14/33] Convert the ecs mappings static file to YAML (#1093) --- internal/builder/_static/ecs_mappings.json | 889 --------------------- internal/builder/_static/ecs_mappings.yaml | 453 +++++++++++ internal/builder/dynamic_mappings.go | 2 +- 3 files changed, 454 insertions(+), 890 deletions(-) delete mode 100644 internal/builder/_static/ecs_mappings.json create mode 100644 internal/builder/_static/ecs_mappings.yaml diff --git a/internal/builder/_static/ecs_mappings.json b/internal/builder/_static/ecs_mappings.json deleted file mode 100644 index f2f6f7660..000000000 --- a/internal/builder/_static/ecs_mappings.json +++ /dev/null @@ -1,889 +0,0 @@ -{ - "mappings": { - "dynamic_templates": [ - { - "ecs_timestamp": { - "path_match": "@timestamp", - "mapping": { - "type": "date", - "ignore_malformed": false - } - } - }, - { - "data_stream_to_constant": { - "path_match": "data_stream.*", - "mapping": { - "type": "constant_keyword" - } - } - }, - { - "resolved_ip_to_ip": { - "match": "resolved_ip", - "mapping": { - "type": "ip" - } - } - }, - { - "forwarded_ip_to_ip": { - "match_mapping_type": "string", - "match": "forwarded_ip", - "mapping": { - "type": "ip" - } - } - }, - { - "ip_to_ip": { - "match_mapping_type": "string", - "match": "ip", - "mapping": { - "type": "ip" - } - } - }, - { - "port_to_long": { - "match": "port", - "mapping": { - "type": "long" - } - } - }, - { - "thread_id_to_long": { - "path_match": "*.thread.id", - "mapping": { - "type": "long" - } - } - }, - { - "status_code_to_long": { - "match": "status_code", - "mapping": { - "type": "long" - } - } - }, - { - "line_to_long": { - "path_match": "*.file.line", - "mapping": { - "type": "long" - } - } - }, - { - "priority_to_long": { - "path_match": "log.syslog.priority", - "mapping": { - "type": "long" - } - } - }, - { - "code_to_long": { - "path_match": "*.facility.code", - "mapping": { - "type": "long" - } - } - }, - { - "code_to_long": { - "path_match": "*.severity.code", - "mapping": { - "type": "long" - } - } - }, - { - "bytes_to_long": { - "match": "bytes", - "path_unmatch": "*.data.bytes", - "mapping": { - "type": "long" - } - } - }, - { - "packets_to_long": { - "match": "packets", - "mapping": { - "type": "long" - } - } - }, - { - "public_key_exponent_to_long": { - "match": "public_key_exponent", - "mapping": { - "type": "long" - } - } - }, - { - "severity_to_long": { - "path_match": "event.severity", - "mapping": { - "type": "long" - } - } - }, - { - "duration_to_long": { - "path_match": "event.duration", - "mapping": { - "type": "long" - } - } - }, - { - "pid_to_long": { - "match": "pid", - "mapping": { - "type": "long" - } - } - }, - { - "uptime_to_long": { - "match": "uptime", - "mapping": { - "type": "long" - } - } - }, - { - "sequence_to_long": { - "match": "sequence", - "mapping": { - "type": "long" - } - } - }, - { - "entropy_to_long": { - "match": "*entropy", - "mapping": { - "type": "long" - } - } - }, - { - "size_to_long": { - "match": "*size", - "mapping": { - "type": "long" - } - } - }, - { - "entrypoint_to_long": { - "match": "entrypoint", - "mapping": { - "type": "long" - } - } - }, - { - "ttl_to_long": { - "match": "ttl", - "mapping": { - "type": "long" - } - } - }, - { - "major_to_long": { - "match": "major", - "mapping": { - "type": "long" - } - } - }, - { - "minor_to_long": { - "match": "minor", - "mapping": { - "type": "long" - } - } - }, - { - "as_number_to_long": { - "path_match": "*.as.number", - "mapping": { - "type": "long" - } - } - }, - { - "pgid_to_long": { - "match": "pgid", - "mapping": { - "type": "long" - } - } - }, - { - "exit_code_to_long": { - "match": "exit_code", - "mapping": { - "type": "long" - } - } - }, - { - "chi_to_long": { - "match": "chi2", - "mapping": { - "type": "long" - } - } - }, - { - "args_count_to_long": { - "match": "args_count", - "mapping": { - "type": "long" - } - } - }, - { - "virtual_address_to_long": { - "match": "virtual_address", - "mapping": { - "type": "long" - } - } - }, - { - "io_text_to_wildcard": { - "path_match": "*.io.text", - "mapping": { - "type": "wildcard" - } - } - }, - { - "strings_to_wildcard": { - "path_match": "registry.data.strings", - "mapping": { - "type": "wildcard" - } - } - }, - { - "path_to_wildcard": { - "path_match": "*url.path", - "mapping": { - "type": "wildcard" - } - } - }, - { - "message_id_to_wildcard": { - "match": "message_id", - "mapping": { - "type": "wildcard" - } - } - }, - { - "command_line_to_multifield": { - "match": "command_line", - "mapping": { - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "error_stack_trace_to_multifield": { - "match": "stack_trace", - "mapping": { - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "http_content_to_multifield": { - "path_match": "*.body.content", - "mapping": { - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "url_full_to_multifield": { - "path_match": "*.url.full", - "mapping": { - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "url_original_to_multifield": { - "path_match": "*.url.original", - "mapping": { - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "user_agent_original_to_multifield": { - "path_match": "user_agent.original", - "mapping": { - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "error_message_to_match_only": { - "path_match": "error.message", - "mapping": { - "type": "match_only_text" - } - } - }, - { - "message_match_only_text": { - "path_match": "message", - "mapping": { - "type": "match_only_text" - } - } - }, - { - "agent_name_to_keyword": { - "path_match": "agent.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "service_name_to_keyword": { - "path_match": "*.service.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "sections_name_to_keyword": { - "path_match": "*.sections.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "resource_name_to_keyword": { - "path_match": "*.resource.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "observer_name_to_keyword": { - "path_match": "observer.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "question_name_to_keyword": { - "path_match": "*.question.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "group_name_to_keyword": { - "path_match": "*.group.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "geo_name_to_keyword": { - "path_match": "*.geo.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "host_name_to_keyword": { - "path_match": "host.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "severity_name_to_keyword": { - "path_match": "*.severity.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "title_to_multifield": { - "match": "title", - "mapping": { - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "executable_to_multifield": { - "match": "executable", - "mapping": { - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "file_path_to_multifield": { - "path_match": "*.file.path", - "mapping": { - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "file_target_path_to_multifield": { - "path_match": "*.file.target_path", - "mapping": { - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "name_to_multifield": { - "match": "name", - "mapping": { - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "full_name_to_multifield": { - "match": "full_name", - "mapping": { - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "os_full_to_multifield": { - "path_match": "*.os.full", - "mapping": { - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "working_directory_to_multifield": { - "match": "working_directory", - "mapping": { - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "timestamp_to_date": { - "match": "timestamp", - "mapping": { - "type": "date" - } - } - }, - { - "delivery_timestamp_to_date": { - "match": "delivery_timestamp", - "mapping": { - "type": "date" - } - } - }, - { - "not_after_to_date": { - "match": "not_after", - "mapping": { - "type": "date" - } - } - }, - { - "not_before_to_date": { - "match": "not_before", - "mapping": { - "type": "date" - } - } - }, - { - "accessed_to_date": { - "match": "accessed", - "mapping": { - "type": "date" - } - } - }, - { - "origination_timestamp_to_date": { - "match": "origination_timestamp", - "mapping": { - "type": "date" - } - } - }, - { - "created_to_date": { - "match": "created", - "mapping": { - "type": "date" - } - } - }, - { - "installed_to_date": { - "match": "installed", - "mapping": { - "type": "date" - } - } - }, - { - "creation_date_to_date": { - "match": "creation_date", - "mapping": { - "type": "date" - } - } - }, - { - "ctime_to_date": { - "match": "ctime", - "mapping": { - "type": "date" - } - } - }, - { - "mtime_to_date": { - "match": "mtime", - "mapping": { - "type": "date" - } - } - }, - { - "ingested_to_date": { - "match": "ingested", - "mapping": { - "type": "date" - } - } - }, - { - "start_to_date": { - "match": "start", - "mapping": { - "type": "date" - } - } - }, - { - "end_to_date": { - "match": "end", - "mapping": { - "type": "date" - } - } - }, - { - "score_base_to_float": { - "path_match": "*.score.base", - "mapping": { - "type": "float" - } - } - }, - { - "score_temporal_to_float": { - "path_match": "*.score.temporal", - "mapping": { - "type": "float" - } - } - }, - { - "score_to_float": { - "match": "*_score", - "mapping": { - "type": "float" - } - } - }, - { - "score_norm_to_float": { - "match": "*_score_norm", - "mapping": { - "type": "float" - } - } - }, - { - "usage_to_float": { - "match": "usage", - "mapping": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - { - "location_to_geo_point": { - "match": "location", - "mapping": { - "type": "geo_point" - } - } - }, - { - "same_as_process_to_boolean": { - "match": "same_as_process", - "mapping": { - "type": "boolean" - } - } - }, - { - "established_to_boolean": { - "match": "established", - "mapping": { - "type": "boolean" - } - } - }, - { - "resumed_to_boolean": { - "match": "resumed", - "mapping": { - "type": "boolean" - } - } - }, - { - "max_bytes_per_process_exceeded_to_boolean": { - "match": "max_bytes_per_process_exceeded", - "mapping": { - "type": "boolean" - } - } - }, - { - "interactive_to_boolean": { - "match": "interactive", - "mapping": { - "type": "boolean" - } - } - }, - { - "exists_to_boolean": { - "match": "exists", - "mapping": { - "type": "boolean" - } - } - }, - { - "trusted_to_boolean": { - "match": "trusted", - "mapping": { - "type": "boolean" - } - } - }, - { - "valid_to_boolean": { - "match": "valid", - "mapping": { - "type": "boolean" - } - } - }, - { - "go_stripped_to_boolean": { - "match": "go_stripped", - "mapping": { - "type": "boolean" - } - } - }, - { - "coldstart_to_boolean": { - "match": "coldstart", - "mapping": { - "type": "boolean" - } - } - }, - { - "exports_to_flattened": { - "match": "exports", - "mapping": { - "type": "flattened" - } - } - }, - { - "structured_data_to_flattened": { - "match": "structured_data", - "mapping": { - "type": "flattened" - } - } - }, - { - "imports_to_flattened": { - "match": "*imports", - "mapping": { - "type": "flattened" - } - } - }, - { - "attachments_to_nested": { - "match": "attachments", - "mapping": { - "type": "nested" - } - } - }, - { - "segments_to_nested": { - "match": "segments", - "mapping": { - "type": "nested" - } - } - }, - { - "elf_sections_to_nested": { - "path_match": "*.elf.sections", - "mapping": { - "type": "nested" - } - } - }, - { - "pe_sections_to_nested": { - "path_match": "*.pe.sections", - "mapping": { - "type": "nested" - } - } - }, - { - "macho_sections_to_nested": { - "path_match": "*.macho.sections", - "mapping": { - "type": "nested" - } - } - }, - { - "trigger_to_nested": { - "match": "trigger", - "mapping": { - "type": "nested" - } - } - } - ] - } -} diff --git a/internal/builder/_static/ecs_mappings.yaml b/internal/builder/_static/ecs_mappings.yaml new file mode 100644 index 000000000..7417d07a0 --- /dev/null +++ b/internal/builder/_static/ecs_mappings.yaml @@ -0,0 +1,453 @@ +mappings: + dynamic_templates: + - ecs_timestamp: + mapping: + ignore_malformed: false + type: date + path_match: '@timestamp' + - data_stream_to_constant: + mapping: + type: constant_keyword + path_match: data_stream.* + - resolved_ip_to_ip: + mapping: + type: ip + match: resolved_ip + - forwarded_ip_to_ip: + mapping: + type: ip + match: forwarded_ip + match_mapping_type: string + - ip_to_ip: + mapping: + type: ip + match: ip + match_mapping_type: string + - port_to_long: + mapping: + type: long + match: port + - thread_id_to_long: + mapping: + type: long + path_match: '*.thread.id' + - status_code_to_long: + mapping: + type: long + match: status_code + - line_to_long: + mapping: + type: long + path_match: '*.file.line' + - priority_to_long: + mapping: + type: long + path_match: log.syslog.priority + - code_to_long: + mapping: + type: long + path_match: '*.facility.code' + - code_to_long: + mapping: + type: long + path_match: '*.severity.code' + - bytes_to_long: + mapping: + type: long + match: bytes + path_unmatch: '*.data.bytes' + - packets_to_long: + mapping: + type: long + match: packets + - public_key_exponent_to_long: + mapping: + type: long + match: public_key_exponent + - severity_to_long: + mapping: + type: long + path_match: event.severity + - duration_to_long: + mapping: + type: long + path_match: event.duration + - pid_to_long: + mapping: + type: long + match: pid + - uptime_to_long: + mapping: + type: long + match: uptime + - sequence_to_long: + mapping: + type: long + match: sequence + - entropy_to_long: + mapping: + type: long + match: '*entropy' + - size_to_long: + mapping: + type: long + match: '*size' + - entrypoint_to_long: + mapping: + type: long + match: entrypoint + - ttl_to_long: + mapping: + type: long + match: ttl + - major_to_long: + mapping: + type: long + match: major + - minor_to_long: + mapping: + type: long + match: minor + - as_number_to_long: + mapping: + type: long + path_match: '*.as.number' + - pgid_to_long: + mapping: + type: long + match: pgid + - exit_code_to_long: + mapping: + type: long + match: exit_code + - chi_to_long: + mapping: + type: long + match: chi2 + - args_count_to_long: + mapping: + type: long + match: args_count + - virtual_address_to_long: + mapping: + type: long + match: virtual_address + - io_text_to_wildcard: + mapping: + type: wildcard + path_match: '*.io.text' + - strings_to_wildcard: + mapping: + type: wildcard + path_match: registry.data.strings + - path_to_wildcard: + mapping: + type: wildcard + path_match: '*url.path' + - message_id_to_wildcard: + mapping: + type: wildcard + match: message_id + - command_line_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: wildcard + match: command_line + - error_stack_trace_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: wildcard + match: stack_trace + - http_content_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: wildcard + path_match: '*.body.content' + - url_full_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: wildcard + path_match: '*.url.full' + - url_original_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: wildcard + path_match: '*.url.original' + - user_agent_original_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: wildcard + path_match: user_agent.original + - error_message_to_match_only: + mapping: + type: match_only_text + path_match: error.message + - message_match_only_text: + mapping: + type: match_only_text + path_match: message + - agent_name_to_keyword: + mapping: + type: keyword + path_match: agent.name + - service_name_to_keyword: + mapping: + type: keyword + path_match: '*.service.name' + - sections_name_to_keyword: + mapping: + type: keyword + path_match: '*.sections.name' + - resource_name_to_keyword: + mapping: + type: keyword + path_match: '*.resource.name' + - observer_name_to_keyword: + mapping: + type: keyword + path_match: observer.name + - question_name_to_keyword: + mapping: + type: keyword + path_match: '*.question.name' + - group_name_to_keyword: + mapping: + type: keyword + path_match: '*.group.name' + - geo_name_to_keyword: + mapping: + type: keyword + path_match: '*.geo.name' + - host_name_to_keyword: + mapping: + type: keyword + path_match: host.name + - severity_name_to_keyword: + mapping: + type: keyword + path_match: '*.severity.name' + - title_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: keyword + match: title + - executable_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: keyword + match: executable + - file_path_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: keyword + path_match: '*.file.path' + - file_target_path_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: keyword + path_match: '*.file.target_path' + - name_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: keyword + match: name + - full_name_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: keyword + match: full_name + - os_full_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: keyword + path_match: '*.os.full' + - working_directory_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: keyword + match: working_directory + - timestamp_to_date: + mapping: + type: date + match: timestamp + - delivery_timestamp_to_date: + mapping: + type: date + match: delivery_timestamp + - not_after_to_date: + mapping: + type: date + match: not_after + - not_before_to_date: + mapping: + type: date + match: not_before + - accessed_to_date: + mapping: + type: date + match: accessed + - origination_timestamp_to_date: + mapping: + type: date + match: origination_timestamp + - created_to_date: + mapping: + type: date + match: created + - installed_to_date: + mapping: + type: date + match: installed + - creation_date_to_date: + mapping: + type: date + match: creation_date + - ctime_to_date: + mapping: + type: date + match: ctime + - mtime_to_date: + mapping: + type: date + match: mtime + - ingested_to_date: + mapping: + type: date + match: ingested + - start_to_date: + mapping: + type: date + match: start + - end_to_date: + mapping: + type: date + match: end + - score_base_to_float: + mapping: + type: float + path_match: '*.score.base' + - score_temporal_to_float: + mapping: + type: float + path_match: '*.score.temporal' + - score_to_float: + mapping: + type: float + match: '*_score' + - score_norm_to_float: + mapping: + type: float + match: '*_score_norm' + - usage_to_float: + mapping: + scaling_factor: 1000 + type: scaled_float + match: usage + - location_to_geo_point: + mapping: + type: geo_point + match: location + - same_as_process_to_boolean: + mapping: + type: boolean + match: same_as_process + - established_to_boolean: + mapping: + type: boolean + match: established + - resumed_to_boolean: + mapping: + type: boolean + match: resumed + - max_bytes_per_process_exceeded_to_boolean: + mapping: + type: boolean + match: max_bytes_per_process_exceeded + - interactive_to_boolean: + mapping: + type: boolean + match: interactive + - exists_to_boolean: + mapping: + type: boolean + match: exists + - trusted_to_boolean: + mapping: + type: boolean + match: trusted + - valid_to_boolean: + mapping: + type: boolean + match: valid + - go_stripped_to_boolean: + mapping: + type: boolean + match: go_stripped + - coldstart_to_boolean: + mapping: + type: boolean + match: coldstart + - exports_to_flattened: + mapping: + type: flattened + match: exports + - structured_data_to_flattened: + mapping: + type: flattened + match: structured_data + - imports_to_flattened: + mapping: + type: flattened + match: '*imports' + - attachments_to_nested: + mapping: + type: nested + match: attachments + - segments_to_nested: + mapping: + type: nested + match: segments + - elf_sections_to_nested: + mapping: + type: nested + path_match: '*.elf.sections' + - pe_sections_to_nested: + mapping: + type: nested + path_match: '*.pe.sections' + - macho_sections_to_nested: + mapping: + type: nested + path_match: '*.macho.sections' + - trigger_to_nested: + mapping: + type: nested + match: trigger diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index 2cb85f0a1..ead4a1093 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -21,7 +21,7 @@ import ( "github.com/elastic/elastic-package/internal/packages/buildmanifest" ) -//go:embed _static/ecs_mappings.json +//go:embed _static/ecs_mappings.yaml var staticEcsMappings string const prefixMapping = "_embedded_ecs" From 9afbb4c0f55d7551ae19135c3f1a816c5c5dcb92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jan 2023 17:36:57 +0000 Subject: [PATCH 15/33] Chore(deps): bump gotest.tools/gotestsum from 1.8.2 to 1.9.0 (#1095) Bumps [gotest.tools/gotestsum](https://github.com/gotestyourself/gotestsum) from 1.8.2 to 1.9.0. - [Release notes](https://github.com/gotestyourself/gotestsum/releases) - [Commits](https://github.com/gotestyourself/gotestsum/compare/v1.8.2...v1.9.0) --- updated-dependencies: - dependency-name: gotest.tools/gotestsum dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e82cc90ae..a6fa5634a 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( golang.org/x/oauth2 v0.4.0 golang.org/x/tools v0.5.0 gopkg.in/yaml.v3 v3.0.1 - gotest.tools/gotestsum v1.8.2 + gotest.tools/gotestsum v1.9.0 helm.sh/helm/v3 v3.10.3 honnef.co/go/tools v0.3.3 k8s.io/apimachinery v0.26.0 diff --git a/go.sum b/go.sum index 8d0f74c6b..bd994113b 100644 --- a/go.sum +++ b/go.sum @@ -612,8 +612,8 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/gotestsum v1.8.2 h1:szU3TaSz8wMx/uG+w/A2+4JUPwH903YYaMI9yOOYAyI= -gotest.tools/gotestsum v1.8.2/go.mod h1:6JHCiN6TEjA7Kaz23q1bH0e2Dc3YJjDUZ0DmctFZf+w= +gotest.tools/gotestsum v1.9.0 h1:Jbo/0k/sIOXIJu51IZxEAt27n77xspFEfL6SqKUR72A= +gotest.tools/gotestsum v1.9.0/go.mod h1:6JHCiN6TEjA7Kaz23q1bH0e2Dc3YJjDUZ0DmctFZf+w= gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= From 74bc7061097d3cab12506b78674e1bcd5ed39d50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Constan=C3=A7a=20Manteigas?= <113898685+constanca-m@users.noreply.github.com> Date: Tue, 17 Jan 2023 14:46:06 +0100 Subject: [PATCH 16/33] Add support for 1.26 K8s version. Update kind version. (#1094) --- .ci/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 278b4f8b7..b8d00bc13 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -15,8 +15,8 @@ pipeline { PIPELINE_LOG_LEVEL='INFO' AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' HOME = "${env.WORKSPACE}" - KIND_VERSION = 'v0.14.0' - K8S_VERSION = 'v1.25.0' + KIND_VERSION = 'v0.17.0' + K8S_VERSION = 'v1.26.0' JOB_GCS_BUCKET = 'fleet-ci-temp' JOB_GCS_BUCKET_INTERNAL = 'fleet-ci-temp-internal' JOB_GCS_CREDENTIALS = 'fleet-ci-gcs-plugin' From 9e65c9769f115b867c0765e9f9785a2018b8cf9c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 18 Jan 2023 12:28:52 +0100 Subject: [PATCH 17/33] Move spec version check for import mappings (#1099) --- internal/builder/dynamic_mappings.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index ead4a1093..f1e9d0967 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -89,11 +89,6 @@ func shouldImportEcsMappings(specVersion, packageRoot string) (bool, error) { return false, errors.Wrap(err, "invalid spec version") } - if v.LessThan(semver2_3_0) { - logger.Debugf("Required spec version >= %s to import ECS mappings", semver2_3_0.String()) - return false, nil - } - bm, ok, err := buildmanifest.ReadBuildManifest(packageRoot) if err != nil { return false, errors.Wrap(err, "can't read build manifest") @@ -106,6 +101,12 @@ func shouldImportEcsMappings(specVersion, packageRoot string) (bool, error) { logger.Debug("Package doesn't have to import ECS mappings") return false, nil } + + if v.LessThan(semver2_3_0) { + logger.Debugf("Required spec version >= %s to import ECS mappings", semver2_3_0.String()) + return false, nil + } + return true, nil } From 00343fbf1d1e4228e14ee7737a9d9e80643c97e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Jan 2023 17:38:12 +0000 Subject: [PATCH 18/33] Chore(deps): bump helm.sh/helm/v3 from 3.10.3 to 3.11.0 (#1101) Bumps [helm.sh/helm/v3](https://github.com/helm/helm) from 3.10.3 to 3.11.0. - [Release notes](https://github.com/helm/helm/releases) - [Commits](https://github.com/helm/helm/compare/v3.10.3...v3.11.0) --- updated-dependencies: - dependency-name: helm.sh/helm/v3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 9 +++------ go.sum | 28 ++++++---------------------- 2 files changed, 9 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index a6fa5634a..0c16cac24 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( golang.org/x/tools v0.5.0 gopkg.in/yaml.v3 v3.0.1 gotest.tools/gotestsum v1.9.0 - helm.sh/helm/v3 v3.10.3 + helm.sh/helm/v3 v3.11.0 honnef.co/go/tools v0.3.3 k8s.io/apimachinery v0.26.0 k8s.io/cli-runtime v0.26.0 @@ -41,7 +41,7 @@ require ( require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/BurntSushi/toml v1.1.0 // indirect + github.com/BurntSushi/toml v1.2.1 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Microsoft/go-winio v0.6.0 // indirect github.com/PaesslerAG/gval v1.2.1 // indirect @@ -129,10 +129,9 @@ require ( github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xlab/treeprint v1.1.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - go.etcd.io/etcd/api/v3 v3.5.6 // indirect go.mongodb.org/mongo-driver v1.11.1 // indirect go.starlark.net v0.0.0-20221205180719-3fd0dac74452 // indirect - golang.org/x/crypto v0.4.0 // indirect + golang.org/x/crypto v0.5.0 // indirect golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e // indirect golang.org/x/mod v0.7.0 // indirect golang.org/x/net v0.5.0 // indirect @@ -142,8 +141,6 @@ require ( golang.org/x/text v0.6.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 // indirect - google.golang.org/grpc v1.51.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index bd994113b..0c16ecc51 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,8 @@ github.com/AlecAivazis/survey/v2 v2.3.6/go.mod h1:4AuI9b7RjAR+G7v9+C4YSlX/YL3K3c github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= -github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= @@ -51,7 +51,6 @@ github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx2 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= @@ -66,8 +65,6 @@ github.com/cloudflare/circl v1.3.1/go.mod h1:+CauBF6R70Jqcyl8N2hC8pAXYbWkGIezuSb github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -102,9 +99,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -390,14 +385,11 @@ github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.etcd.io/etcd/api/v3 v3.5.6 h1:Cy2qx3npLcYqTKqGJzMypnMv2tiRyifZJ17BlWIWA7A= -go.etcd.io/etcd/api/v3 v3.5.6/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= go.mongodb.org/mongo-driver v1.11.1 h1:QP0znIRTuL0jf1oBQoAoM0C6ZJfBK4kx0Uumtv1A7w8= go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8= @@ -413,8 +405,8 @@ golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= -golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= +golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= +golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e h1:qyrTQ++p1afMkO4DPEeLGq/3oTsdlvdH4vqZUBWzUKM= @@ -541,7 +533,6 @@ golang.org/x/tools v0.0.0-20200526224456-8b020aee10d2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -559,21 +550,14 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 h1:jmIfw8+gSvXcZSgaFAGyInDXeWzUhvYH57G/5GKMn70= -google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= -google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -617,8 +601,8 @@ gotest.tools/gotestsum v1.9.0/go.mod h1:6JHCiN6TEjA7Kaz23q1bH0e2Dc3YJjDUZ0DmctFZ gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= -helm.sh/helm/v3 v3.10.3 h1:wL7IUZ7Zyukm5Kz0OUmIFZgKHuAgByCrUcJBtY0kDyw= -helm.sh/helm/v3 v3.10.3/go.mod h1:CXOcs02AYvrlPMWARNYNRgf2rNP7gLJQsi/Ubd4EDrI= +helm.sh/helm/v3 v3.11.0 h1:F+peaCQYbycY1FIqIQ6dAortHd/VzV5FkhMciv4Kf+c= +helm.sh/helm/v3 v3.11.0/go.mod h1:z/Bu/BylToGno/6dtNGuSmjRqxKq5gaH+FU0BPO+AQ8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.3.3 h1:oDx7VAwstgpYpb3wv0oxiZlxY+foCpRAwY7Vk6XpAgA= From af71fa30ed9a5b448adca10a5996985ad08486d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Jan 2023 17:39:05 +0000 Subject: [PATCH 19/33] Chore(deps): bump k8s.io/client-go from 0.26.0 to 0.26.1 (#1105) Bumps [k8s.io/client-go](https://github.com/kubernetes/client-go) from 0.26.0 to 0.26.1. - [Release notes](https://github.com/kubernetes/client-go/releases) - [Changelog](https://github.com/kubernetes/client-go/blob/master/CHANGELOG.md) - [Commits](https://github.com/kubernetes/client-go/compare/v0.26.0...v0.26.1) --- updated-dependencies: - dependency-name: k8s.io/client-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 0c16cac24..2efb395e7 100644 --- a/go.mod +++ b/go.mod @@ -34,9 +34,9 @@ require ( gotest.tools/gotestsum v1.9.0 helm.sh/helm/v3 v3.11.0 honnef.co/go/tools v0.3.3 - k8s.io/apimachinery v0.26.0 + k8s.io/apimachinery v0.26.1 k8s.io/cli-runtime v0.26.0 - k8s.io/client-go v0.26.0 + k8s.io/client-go v0.26.1 ) require ( @@ -146,7 +146,7 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gotest.tools/v3 v3.4.0 // indirect - k8s.io/api v0.26.0 // indirect + k8s.io/api v0.26.1 // indirect k8s.io/apiextensions-apiserver v0.26.0 // indirect k8s.io/component-base v0.26.0 // indirect k8s.io/klog/v2 v2.80.1 // indirect diff --git a/go.sum b/go.sum index 0c16ecc51..792ffca97 100644 --- a/go.sum +++ b/go.sum @@ -607,16 +607,16 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.3.3 h1:oDx7VAwstgpYpb3wv0oxiZlxY+foCpRAwY7Vk6XpAgA= honnef.co/go/tools v0.3.3/go.mod h1:jzwdWgg7Jdq75wlfblQxO4neNaFFSvgc1tD5Wv8U0Yw= -k8s.io/api v0.26.0 h1:IpPlZnxBpV1xl7TGk/X6lFtpgjgntCg8PJ+qrPHAC7I= -k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= +k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ= +k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg= k8s.io/apiextensions-apiserver v0.26.0 h1:Gy93Xo1eg2ZIkNX/8vy5xviVSxwQulsnUdQ00nEdpDo= k8s.io/apiextensions-apiserver v0.26.0/go.mod h1:7ez0LTiyW5nq3vADtK6C3kMESxadD51Bh6uz3JOlqWQ= -k8s.io/apimachinery v0.26.0 h1:1feANjElT7MvPqp0JT6F3Ss6TWDwmcjLypwoPpEf7zg= -k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= +k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ= +k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= k8s.io/cli-runtime v0.26.0 h1:aQHa1SyUhpqxAw1fY21x2z2OS5RLtMJOCj7tN4oq8mw= k8s.io/cli-runtime v0.26.0/go.mod h1:o+4KmwHzO/UK0wepE1qpRk6l3o60/txUZ1fEXWGIKTY= -k8s.io/client-go v0.26.0 h1:lT1D3OfO+wIi9UFolCrifbjUUgu7CpLca0AD8ghRLI8= -k8s.io/client-go v0.26.0/go.mod h1:I2Sh57A79EQsDmn7F7ASpmru1cceh3ocVT9KlX2jEZg= +k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU= +k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE= k8s.io/component-base v0.26.0 h1:0IkChOCohtDHttmKuz+EP3j3+qKmV55rM9gIFTXA7Vs= k8s.io/component-base v0.26.0/go.mod h1:lqHwlfV1/haa14F/Z5Zizk5QmzaVf23nQzCwVOQpfC8= k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= From aedabd57242fb70a594d4b302be825022c59b629 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jan 2023 12:25:31 +0000 Subject: [PATCH 20/33] Chore(deps): bump k8s.io/cli-runtime from 0.26.0 to 0.26.1 (#1104) Bumps [k8s.io/cli-runtime](https://github.com/kubernetes/cli-runtime) from 0.26.0 to 0.26.1. - [Release notes](https://github.com/kubernetes/cli-runtime/releases) - [Commits](https://github.com/kubernetes/cli-runtime/compare/v0.26.0...v0.26.1) --- updated-dependencies: - dependency-name: k8s.io/cli-runtime dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2efb395e7..1f7f7d755 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( helm.sh/helm/v3 v3.11.0 honnef.co/go/tools v0.3.3 k8s.io/apimachinery v0.26.1 - k8s.io/cli-runtime v0.26.0 + k8s.io/cli-runtime v0.26.1 k8s.io/client-go v0.26.1 ) diff --git a/go.sum b/go.sum index 792ffca97..011dc77d8 100644 --- a/go.sum +++ b/go.sum @@ -613,8 +613,8 @@ k8s.io/apiextensions-apiserver v0.26.0 h1:Gy93Xo1eg2ZIkNX/8vy5xviVSxwQulsnUdQ00n k8s.io/apiextensions-apiserver v0.26.0/go.mod h1:7ez0LTiyW5nq3vADtK6C3kMESxadD51Bh6uz3JOlqWQ= k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ= k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= -k8s.io/cli-runtime v0.26.0 h1:aQHa1SyUhpqxAw1fY21x2z2OS5RLtMJOCj7tN4oq8mw= -k8s.io/cli-runtime v0.26.0/go.mod h1:o+4KmwHzO/UK0wepE1qpRk6l3o60/txUZ1fEXWGIKTY= +k8s.io/cli-runtime v0.26.1 h1:f9+bRQ1V3elQsx37KmZy5fRAh56mVLbE9A7EMdlqVdI= +k8s.io/cli-runtime v0.26.1/go.mod h1:+e5Ym/ARySKscUhZ8K3hZ+ZBo/wYPIcg+7b5sFYi6Gg= k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU= k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE= k8s.io/component-base v0.26.0 h1:0IkChOCohtDHttmKuz+EP3j3+qKmV55rM9gIFTXA7Vs= From 30460b92ca4bdc5d762aaf7c9d119c7b22208803 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jan 2023 14:08:01 +0000 Subject: [PATCH 21/33] Chore(deps): bump github.com/elastic/package-spec/v2 from 2.3.0 to 2.4.0 (#1107) Bumps [github.com/elastic/package-spec/v2](https://github.com/elastic/package-spec) from 2.3.0 to 2.4.0. - [Release notes](https://github.com/elastic/package-spec/releases) - [Commits](https://github.com/elastic/package-spec/compare/v2.3.0...v2.4.0) --- updated-dependencies: - dependency-name: github.com/elastic/package-spec/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1f7f7d755..4960c9214 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.17.7 github.com/elastic/go-licenser v0.4.1 github.com/elastic/go-ucfg v0.8.6 - github.com/elastic/package-spec/v2 v2.3.0 + github.com/elastic/package-spec/v2 v2.4.0 github.com/fatih/color v1.13.0 github.com/go-git/go-billy/v5 v5.4.0 github.com/go-git/go-git/v5 v5.5.2 diff --git a/go.sum b/go.sum index 011dc77d8..de58339b3 100644 --- a/go.sum +++ b/go.sum @@ -88,8 +88,8 @@ github.com/elastic/go-ucfg v0.8.6 h1:stUeyh2goTgGX+/wb9gzKvTv0YB0231LTpKUgCKj4U0 github.com/elastic/go-ucfg v0.8.6/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul8OsQc= github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg= -github.com/elastic/package-spec/v2 v2.3.0 h1:7pWSZzyUVyHYpmHG7GYz9xKk/dKMPy8k71lNKkurABM= -github.com/elastic/package-spec/v2 v2.3.0/go.mod h1:bGpJHojIN9FYyNbqNySJ7D3mDVqOe1sL2l4m7BCSjWo= +github.com/elastic/package-spec/v2 v2.4.0 h1:3y2sp11HMZlVSfkprw1SXJldnrrCsHErKW9PYaIGPfQ= +github.com/elastic/package-spec/v2 v2.4.0/go.mod h1:bGpJHojIN9FYyNbqNySJ7D3mDVqOe1sL2l4m7BCSjWo= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= From a34fdc953f9432dc362b24c88fff7a375b12204e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jan 2023 15:04:55 +0000 Subject: [PATCH 22/33] Chore(deps): bump github.com/fatih/color from 1.13.0 to 1.14.0 (#1106) Bumps [github.com/fatih/color](https://github.com/fatih/color) from 1.13.0 to 1.14.0. - [Release notes](https://github.com/fatih/color/releases) - [Commits](https://github.com/fatih/color/compare/v1.13.0...v1.14.0) --- updated-dependencies: - dependency-name: github.com/fatih/color dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 4960c9214..6e572c3f9 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/elastic/go-licenser v0.4.1 github.com/elastic/go-ucfg v0.8.6 github.com/elastic/package-spec/v2 v2.4.0 - github.com/fatih/color v1.13.0 + github.com/fatih/color v1.14.0 github.com/go-git/go-billy/v5 v5.4.0 github.com/go-git/go-git/v5 v5.5.2 github.com/google/go-cmp v0.5.9 @@ -97,7 +97,7 @@ require ( github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect diff --git a/go.sum b/go.sum index de58339b3..d12c58d04 100644 --- a/go.sum +++ b/go.sum @@ -107,8 +107,9 @@ github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.14.0 h1:AD//feEuOKJSzxN81txMW47CNX1EW6kMVEPt4lePhtE= +github.com/fatih/color v1.14.0/go.mod h1:Ywr2WOhTEN4nsWMWU8I8GWIG5z8rhJEa0ukvJDOfSPY= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -264,8 +265,9 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= From 9d3f504d736db90f3cb07a6d759c6301b982388c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jan 2023 17:42:44 +0000 Subject: [PATCH 23/33] Chore(deps): bump github.com/fatih/color from 1.14.0 to 1.14.1 (#1109) Bumps [github.com/fatih/color](https://github.com/fatih/color) from 1.14.0 to 1.14.1. - [Release notes](https://github.com/fatih/color/releases) - [Commits](https://github.com/fatih/color/compare/v1.14.0...v1.14.1) --- updated-dependencies: - dependency-name: github.com/fatih/color dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6e572c3f9..a36025cc3 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/elastic/go-licenser v0.4.1 github.com/elastic/go-ucfg v0.8.6 github.com/elastic/package-spec/v2 v2.4.0 - github.com/fatih/color v1.14.0 + github.com/fatih/color v1.14.1 github.com/go-git/go-billy/v5 v5.4.0 github.com/go-git/go-git/v5 v5.5.2 github.com/google/go-cmp v0.5.9 diff --git a/go.sum b/go.sum index d12c58d04..12a3d247d 100644 --- a/go.sum +++ b/go.sum @@ -108,8 +108,8 @@ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2Vvl github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.14.0 h1:AD//feEuOKJSzxN81txMW47CNX1EW6kMVEPt4lePhtE= -github.com/fatih/color v1.14.0/go.mod h1:Ywr2WOhTEN4nsWMWU8I8GWIG5z8rhJEa0ukvJDOfSPY= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= From b07ca8b9b15fe345703b0123493756a83b8bc006 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Jan 2023 17:50:50 +0000 Subject: [PATCH 24/33] Chore(deps): bump github.com/ProtonMail/gopenpgp/v2 from 2.5.0 to 2.5.1 (#1113) Bumps [github.com/ProtonMail/gopenpgp/v2](https://github.com/ProtonMail/gopenpgp) from 2.5.0 to 2.5.1. - [Release notes](https://github.com/ProtonMail/gopenpgp/releases) - [Changelog](https://github.com/ProtonMail/gopenpgp/blob/main/CHANGELOG.md) - [Commits](https://github.com/ProtonMail/gopenpgp/compare/v2.5.0...v2.5.1) --- updated-dependencies: - dependency-name: github.com/ProtonMail/gopenpgp/v2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a36025cc3..e4d4d9173 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/AlecAivazis/survey/v2 v2.3.6 github.com/Masterminds/semver/v3 v3.2.0 - github.com/ProtonMail/gopenpgp/v2 v2.5.0 + github.com/ProtonMail/gopenpgp/v2 v2.5.1 github.com/aymerick/raymond v2.0.2+incompatible github.com/boumenot/gocover-cobertura v1.2.0 github.com/cespare/xxhash/v2 v2.2.0 diff --git a/go.sum b/go.sum index 12a3d247d..87e26d326 100644 --- a/go.sum +++ b/go.sum @@ -28,8 +28,8 @@ github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0A github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= github.com/ProtonMail/go-mime v0.0.0-20221031134845-8fd9bc37cf08 h1:dS7r5z4iGS0qCjM7UwWdsEMzQesUQbGcXdSm2/tWboA= github.com/ProtonMail/go-mime v0.0.0-20221031134845-8fd9bc37cf08/go.mod h1:qRZgbeASl2a9OwmsV85aWwRqic0NHPh+9ewGAzb4cgM= -github.com/ProtonMail/gopenpgp/v2 v2.5.0 h1:L+98m3xj/YerdWqpWNkZmwNjg1Bs0lKRGQyuONjOImw= -github.com/ProtonMail/gopenpgp/v2 v2.5.0/go.mod h1:N+W/oc/o6yqdevBw8vpgZ3JMyaMA/IwQIdnaGLCoA+w= +github.com/ProtonMail/gopenpgp/v2 v2.5.1 h1:NZrljYB/lpSANXpDZFKhRtZC9Gp7BmwS4HacVJcK4x4= +github.com/ProtonMail/gopenpgp/v2 v2.5.1/go.mod h1:yBn3IMz4CEjFj+tGgJLEPlxFIoC40R/GlUmxrAqnOnE= github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= @@ -414,13 +414,16 @@ golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9t golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e h1:qyrTQ++p1afMkO4DPEeLGq/3oTsdlvdH4vqZUBWzUKM= golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -530,6 +533,7 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200526224456-8b020aee10d2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= From bddde65fa20b46a842aebb03b5e29a6861dd75d1 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 26 Jan 2023 03:24:56 -0500 Subject: [PATCH 25/33] servicedeployer/compose.go - pass env vars when signalling (#1115) Pass environment variables when invoking docker-compose to send a signal to a service. Fixes #1114 --- .../testrunner/runners/system/servicedeployer/compose.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/servicedeployer/compose.go b/internal/testrunner/runners/system/servicedeployer/compose.go index ade566b3b..b7be23cd5 100644 --- a/internal/testrunner/runners/system/servicedeployer/compose.go +++ b/internal/testrunner/runners/system/servicedeployer/compose.go @@ -135,7 +135,12 @@ func (s *dockerComposeDeployedService) Signal(signal string) error { return errors.Wrap(err, "could not create Docker Compose project for service") } - opts := compose.CommandOptions{ExtraArgs: []string{"-s", signal}} + opts := compose.CommandOptions{ + Env: append( + []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, s.ctxt.Logs.Folder.Local)}, + s.sv.Env...), + ExtraArgs: []string{"-s", signal}, + } if s.ctxt.Name != "" { opts.Services = append(opts.Services, s.ctxt.Name) } From 9aca75872d19d85d7102dcd62188fd9435f3a4dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Jan 2023 13:46:22 +0000 Subject: [PATCH 26/33] Chore(deps): bump github.com/elastic/package-spec/v2 from 2.4.0 to 2.5.0 (#1120) Bumps [github.com/elastic/package-spec/v2](https://github.com/elastic/package-spec) from 2.4.0 to 2.5.0. - [Release notes](https://github.com/elastic/package-spec/releases) - [Commits](https://github.com/elastic/package-spec/compare/v2.4.0...v2.5.0) --- updated-dependencies: - dependency-name: github.com/elastic/package-spec/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e4d4d9173..f306417b7 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.17.7 github.com/elastic/go-licenser v0.4.1 github.com/elastic/go-ucfg v0.8.6 - github.com/elastic/package-spec/v2 v2.4.0 + github.com/elastic/package-spec/v2 v2.5.0 github.com/fatih/color v1.14.1 github.com/go-git/go-billy/v5 v5.4.0 github.com/go-git/go-git/v5 v5.5.2 diff --git a/go.sum b/go.sum index 87e26d326..0cf4fb8d0 100644 --- a/go.sum +++ b/go.sum @@ -88,8 +88,8 @@ github.com/elastic/go-ucfg v0.8.6 h1:stUeyh2goTgGX+/wb9gzKvTv0YB0231LTpKUgCKj4U0 github.com/elastic/go-ucfg v0.8.6/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul8OsQc= github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg= -github.com/elastic/package-spec/v2 v2.4.0 h1:3y2sp11HMZlVSfkprw1SXJldnrrCsHErKW9PYaIGPfQ= -github.com/elastic/package-spec/v2 v2.4.0/go.mod h1:bGpJHojIN9FYyNbqNySJ7D3mDVqOe1sL2l4m7BCSjWo= +github.com/elastic/package-spec/v2 v2.5.0 h1:vYz+wKBLxcROgYa9xAXNsC8knB0b4C1IrN+Gcdsvao4= +github.com/elastic/package-spec/v2 v2.5.0/go.mod h1:bGpJHojIN9FYyNbqNySJ7D3mDVqOe1sL2l4m7BCSjWo= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= From 85cde1564bfb7256f4ad5637e460156afe4bf41a Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 27 Jan 2023 18:23:49 +0100 Subject: [PATCH 27/33] Upgrade go to 1.19.5 (#1121) --- .go-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.go-version b/.go-version index 66e2ae6c2..83d5e73f0 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.19.1 +1.19.5 From e3a80a29c6c0b2491209c582f19325f0ab197483 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 30 Jan 2023 10:20:34 +0100 Subject: [PATCH 28/33] Update default stack version to 8.6.1 (#1118) Add also some options to try to reduce flakiness in elastic-package stack up: * Retry limits have been replaced with start periods. * Fleet-server and elastic-agent healthcheck retries have been increased. * When docker-compose up fails and elastic-agent container is the only container failing, retry once, this may happen if it tries to enroll while fleet-server is being restarted or reconfigured. Co-authored-by: Mario Rodriguez Molins --- internal/install/stack_version.go | 2 +- .../profile/_static/docker-compose-stack.yml | 18 +++++++----- internal/stack/boot.go | 29 +++++++++++++++++++ .../pipeline_benchmark/manifest.yml | 1 - .../use_pipeline_tests/manifest.yml | 1 - test/packages/other/multiinput/manifest.yml | 1 - .../other/pipeline_tests/manifest.yml | 1 - 7 files changed, 40 insertions(+), 13 deletions(-) diff --git a/internal/install/stack_version.go b/internal/install/stack_version.go index 1f742edd3..9e07dcf73 100644 --- a/internal/install/stack_version.go +++ b/internal/install/stack_version.go @@ -6,5 +6,5 @@ package install const ( // DefaultStackVersion is the default version of the stack - DefaultStackVersion = "8.5.1" + DefaultStackVersion = "8.6.1" ) diff --git a/internal/profile/_static/docker-compose-stack.yml b/internal/profile/_static/docker-compose-stack.yml index 827e1e62c..0ad5b8b3e 100644 --- a/internal/profile/_static/docker-compose-stack.yml +++ b/internal/profile/_static/docker-compose-stack.yml @@ -4,8 +4,8 @@ services: image: "${ELASTICSEARCH_IMAGE_REF}" healthcheck: test: "curl -s --cacert /usr/share/elasticsearch/config/certs/ca-cert.pem -f -u elastic:changeme https://127.0.0.1:9200/_cat/health | cut -f4 -d' ' | grep -E '(green|yellow)'" - retries: 300 - interval: 1s + start_period: 300s + interval: 5s environment: - "ES_JAVA_OPTS=-Xms1g -Xmx1g" - "ELASTIC_PASSWORD=changeme" @@ -32,8 +32,8 @@ services: condition: service_healthy healthcheck: test: "sh /usr/share/kibana/healthcheck.sh" - retries: 600 - interval: 1s + start_period: 600s + interval: 5s env_file: - "../certs/ca.env" environment: @@ -60,8 +60,8 @@ services: PROFILE: "${PROFILE_NAME}" healthcheck: test: ["CMD", "curl", "--cacert", "/etc/ssl/package-registry/ca-cert.pem", "-f", "https://localhost:8080"] - retries: 300 - interval: 1s + start_period: 300s + interval: 5s environment: - "EPR_LOG_LEVEL=debug" - "EPR_ADDRESS=0.0.0.0:8080" @@ -89,7 +89,7 @@ services: condition: service_healthy healthcheck: test: "curl --cacert /etc/ssl/elastic-agent/ca-cert.pem -f https://localhost:8220/api/status | grep -i healthy 2>&1 >/dev/null" - retries: 60 + start_period: 60s interval: 5s hostname: docker-fleet-server environment: @@ -124,8 +124,10 @@ services: condition: service_healthy healthcheck: test: "elastic-agent status" + timeout: 2s + start_period: 360s retries: 180 - interval: 1s + interval: 5s hostname: docker-fleet-agent env_file: "./elastic-agent.${STACK_VERSION_VARIANT}.env" volumes: diff --git a/internal/stack/boot.go b/internal/stack/boot.go index 61b1426bb..b2262a7ed 100644 --- a/internal/stack/boot.go +++ b/internal/stack/boot.go @@ -60,12 +60,41 @@ func BootUp(options Options) error { err = dockerComposeUp(options) if err != nil { + // At least starting on 8.6.0, fleet-server may be reconfigured or + // restarted after being healthy. If elastic-agent tries to enroll at + // this moment, it fails inmediately, stopping and making `docker-compose up` + // to fail too. + // As a workaround, try to give another chance to docker-compose if only + // elastic-agent failed. + if onlyElasticAgentFailed() { + fmt.Println("Elastic Agent failed to start, trying again.") + err = dockerComposeUp(options) + } return errors.Wrap(err, "running docker-compose failed") } return nil } +func onlyElasticAgentFailed() bool { + status, err := Status() + if err != nil { + fmt.Printf("Failed to check status of the stack after failure: %v\n", err) + return false + } + + for _, service := range status { + if strings.Contains(service.Name, "elastic-agent") { + continue + } + if !strings.HasPrefix(service.Status, "running") { + return false + } + } + + return true +} + // TearDown function takes down the testing stack. func TearDown(options Options) error { err := dockerComposeDown(options) diff --git a/test/packages/benchmarks/pipeline_benchmark/manifest.yml b/test/packages/benchmarks/pipeline_benchmark/manifest.yml index f7713fd52..bce8399a2 100644 --- a/test/packages/benchmarks/pipeline_benchmark/manifest.yml +++ b/test/packages/benchmarks/pipeline_benchmark/manifest.yml @@ -7,7 +7,6 @@ title: Pipeline benchmarks version: 999.999.999 description: Test for pipeline test runner categories: ["network"] -release: experimental license: basic type: integration conditions: diff --git a/test/packages/benchmarks/use_pipeline_tests/manifest.yml b/test/packages/benchmarks/use_pipeline_tests/manifest.yml index b30a02942..f145d7181 100644 --- a/test/packages/benchmarks/use_pipeline_tests/manifest.yml +++ b/test/packages/benchmarks/use_pipeline_tests/manifest.yml @@ -7,7 +7,6 @@ title: Use pipeline tests for the benchmark version: 999.999.999 description: Test for pipeline test runner categories: ["network"] -release: experimental license: basic type: integration conditions: diff --git a/test/packages/other/multiinput/manifest.yml b/test/packages/other/multiinput/manifest.yml index 7eb589115..7ffaf75b3 100644 --- a/test/packages/other/multiinput/manifest.yml +++ b/test/packages/other/multiinput/manifest.yml @@ -7,7 +7,6 @@ title: Multi-input test version: 999.999.999 description: Test for multiple input tests categories: ["network"] -release: experimental license: basic type: integration conditions: diff --git a/test/packages/other/pipeline_tests/manifest.yml b/test/packages/other/pipeline_tests/manifest.yml index 042c0d21f..96df3f523 100644 --- a/test/packages/other/pipeline_tests/manifest.yml +++ b/test/packages/other/pipeline_tests/manifest.yml @@ -7,7 +7,6 @@ title: Pipeline tests version: 999.999.999 description: Test for pipeline test runner categories: ["network"] -release: experimental license: basic type: integration conditions: From ed8b57f1bc760f05f7ea1cd6b4f83f9fdfde19ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Jan 2023 11:06:37 +0000 Subject: [PATCH 29/33] Chore(deps): bump github.com/ProtonMail/gopenpgp/v2 from 2.5.1 to 2.5.2 (#1117) Bumps [github.com/ProtonMail/gopenpgp/v2](https://github.com/ProtonMail/gopenpgp) from 2.5.1 to 2.5.2. - [Release notes](https://github.com/ProtonMail/gopenpgp/releases) - [Changelog](https://github.com/ProtonMail/gopenpgp/blob/main/CHANGELOG.md) - [Commits](https://github.com/ProtonMail/gopenpgp/compare/v2.5.1...v2.5.2) --- updated-dependencies: - dependency-name: github.com/ProtonMail/gopenpgp/v2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index f306417b7..82a46c0c4 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/AlecAivazis/survey/v2 v2.3.6 github.com/Masterminds/semver/v3 v3.2.0 - github.com/ProtonMail/gopenpgp/v2 v2.5.1 + github.com/ProtonMail/gopenpgp/v2 v2.5.2 github.com/aymerick/raymond v2.0.2+incompatible github.com/boumenot/gocover-cobertura v1.2.0 github.com/cespare/xxhash/v2 v2.2.0 @@ -46,7 +46,7 @@ require ( github.com/Microsoft/go-winio v0.6.0 // indirect github.com/PaesslerAG/gval v1.2.1 // indirect github.com/PaesslerAG/jsonpath v0.1.1 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230124153114-0acdc8ae009b // indirect github.com/ProtonMail/go-mime v0.0.0-20221031134845-8fd9bc37cf08 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect github.com/andybalholm/brotli v1.0.4 // indirect diff --git a/go.sum b/go.sum index 0cf4fb8d0..62f28074f 100644 --- a/go.sum +++ b/go.sum @@ -24,12 +24,13 @@ github.com/PaesslerAG/gval v1.2.1/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbV github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8= github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk= github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY= -github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I= github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= +github.com/ProtonMail/go-crypto v0.0.0-20230124153114-0acdc8ae009b h1:1DHH9haxfhaVM8owXQjLdn7UP4AkDfzSdiRoLdcSCqE= +github.com/ProtonMail/go-crypto v0.0.0-20230124153114-0acdc8ae009b/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= github.com/ProtonMail/go-mime v0.0.0-20221031134845-8fd9bc37cf08 h1:dS7r5z4iGS0qCjM7UwWdsEMzQesUQbGcXdSm2/tWboA= github.com/ProtonMail/go-mime v0.0.0-20221031134845-8fd9bc37cf08/go.mod h1:qRZgbeASl2a9OwmsV85aWwRqic0NHPh+9ewGAzb4cgM= -github.com/ProtonMail/gopenpgp/v2 v2.5.1 h1:NZrljYB/lpSANXpDZFKhRtZC9Gp7BmwS4HacVJcK4x4= -github.com/ProtonMail/gopenpgp/v2 v2.5.1/go.mod h1:yBn3IMz4CEjFj+tGgJLEPlxFIoC40R/GlUmxrAqnOnE= +github.com/ProtonMail/gopenpgp/v2 v2.5.2 h1:97SjlWNAxXl9P22lgwgrZRshQdiEfAht0g3ZoiA1GCw= +github.com/ProtonMail/gopenpgp/v2 v2.5.2/go.mod h1:52qDaCnto6r+CoWbuU50T77XQt99lIs46HtHtvgFO3o= github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= @@ -420,10 +421,9 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= +golang.org/x/mobile v0.0.0-20221110043201-43a038452099/go.mod h1:aAjjkJNdrh3PMckS4B10TGS2nag27cbKR1y2BpUxsiY= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -465,6 +465,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -533,7 +534,6 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200526224456-8b020aee10d2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= From 409ee852461b88bd9c9ee7d6c0109d4b22fb6bbe Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 1 Feb 2023 13:55:52 +0100 Subject: [PATCH 30/33] Add pipeline for buildkite with initial stages (#1124) Add the first Buildkite definition pipeline with the initial stages (check statics and run unit tests). --- .buildkite/pipeline.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .buildkite/pipeline.yml diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 000000000..cfdf57f39 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,28 @@ +steps: + - label: ":go: Run check-static" + command: "make check-static" + agents: + image: "golang:1.19.5" + cpu: "8" + memory: "4G" + + - label: ":go: Run unit tests" + command: "make test-go-ci" + artifact_paths: + - "build/test-results/*.xml" + - "build/test-coverage/*.xml" + agents: + image: "golang:1.19.5" + cpu: "8" + memory: "4G" + + - wait: ~ + continue_on_failure: true + + - label: "Junit annotate" + plugins: + - junit-annotate#v2.4.1: + artifacts: "build/test-results/*.xml" + agents: + provider: "gcp" + From 0440db5e29e540c2d5bee6538936981259c471fb Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 1 Feb 2023 14:44:44 +0100 Subject: [PATCH 31/33] Add variant for Kibana >= 8.7.0 with Fleet experimental toggles enabled (#1128) --- .ci/Jenkinsfile | 1 + Makefile | 8 ++- internal/profile/_static/kibana_config_86.yml | 61 +++++++++++++++++++ internal/profile/_static/kibana_config_8x.yml | 1 + internal/profile/profile.go | 3 + internal/profile/static.go | 37 +++++++++++ internal/stack/variants.go | 4 +- internal/stack/variants_test.go | 20 +++--- 8 files changed, 124 insertions(+), 11 deletions(-) create mode 100644 internal/profile/_static/kibana_config_86.yml diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index b8d00bc13..de7de1913 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -99,6 +99,7 @@ pipeline { 'stack-command-default': generateTestCommandStage(command: 'test-stack-command-default', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), 'stack-command-oldest': generateTestCommandStage(command: 'test-stack-command-oldest', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), 'stack-command-7x': generateTestCommandStage(command: 'test-stack-command-7x', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), + 'stack-command-86': generateTestCommandStage(command: 'test-stack-command-86', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), 'stack-command-8x': generateTestCommandStage(command: 'test-stack-command-8x', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), 'check-packages-with-kind': generateTestCommandStage(command: 'test-check-packages-with-kind', artifacts: ['build/test-results/*.xml', 'build/kubectl-dump.txt', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), 'check-packages-other': generateTestCommandStage(command: 'test-check-packages-other', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), diff --git a/Makefile b/Makefile index 23ee32d96..60f1559f5 100644 --- a/Makefile +++ b/Makefile @@ -58,10 +58,14 @@ test-stack-command-oldest: ./scripts/test-stack-command.sh 7.14.2 test-stack-command-7x: - ./scripts/test-stack-command.sh 7.17.3-SNAPSHOT + ./scripts/test-stack-command.sh 7.17.8 + +# Keeping a test for 8.6 because it has an specific configuration file. +test-stack-command-86: + ./scripts/test-stack-command.sh 8.6.1 test-stack-command-8x: - ./scripts/test-stack-command.sh 8.6.0-SNAPSHOT + ./scripts/test-stack-command.sh 8.7.0-SNAPSHOT test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x diff --git a/internal/profile/_static/kibana_config_86.yml b/internal/profile/_static/kibana_config_86.yml new file mode 100644 index 000000000..6d9564c00 --- /dev/null +++ b/internal/profile/_static/kibana_config_86.yml @@ -0,0 +1,61 @@ +server.name: kibana +server.host: "0.0.0.0" +server.ssl.enabled: true +server.ssl.certificate: "/usr/share/kibana/config/certs/cert.pem" +server.ssl.key: "/usr/share/kibana/config/certs/key.pem" +server.ssl.certificateAuthorities: ["/usr/share/kibana/config/certs/ca-cert.pem"] + +elasticsearch.hosts: [ "https://elasticsearch:9200" ] +elasticsearch.ssl.certificateAuthorities: "/usr/share/kibana/config/certs/ca-cert.pem" +elasticsearch.serviceAccountToken: "AAEAAWVsYXN0aWMva2liYW5hL2VsYXN0aWMtcGFja2FnZS1raWJhbmEtdG9rZW46b2x4b051SWNRa0tYMHdXazdLWmFBdw" + +monitoring.ui.container.elasticsearch.enabled: true + +xpack.fleet.registryUrl: "https://package-registry:8080" +xpack.fleet.agents.enabled: true +xpack.fleet.agents.elasticsearch.hosts: ["https://elasticsearch:9200"] +xpack.fleet.agents.fleet_server.hosts: ["https://fleet-server:8220"] + +xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" + +xpack.cloudSecurityPosture.enabled: true + +xpack.fleet.packages: + - name: system + version: latest + - name: elastic_agent + version: latest + - name: fleet_server + version: latest +xpack.fleet.agentPolicies: + - name: Elastic-Agent (elastic-package) + id: elastic-agent-managed-ep + is_default: true + is_managed: false + namespace: default + monitoring_enabled: + - logs + - metrics + package_policies: + - name: system-1 + id: default-system + package: + name: system + - name: Fleet Server (elastic-package) + id: fleet-server-policy + is_default_fleet_server: true + is_managed: false + namespace: default + package_policies: + - name: fleet_server-1 + id: default-fleet-server + package: + name: fleet_server +xpack.fleet.outputs: + - id: fleet-default-output + name: default + type: elasticsearch + hosts: [ https://elasticsearch:9200 ] + ca_trusted_fingerprint: "${ELASTIC_PACKAGE_CA_TRUSTED_FINGERPRINT}" + is_default: true + is_default_monitoring: true diff --git a/internal/profile/_static/kibana_config_8x.yml b/internal/profile/_static/kibana_config_8x.yml index 6d9564c00..72cead61c 100644 --- a/internal/profile/_static/kibana_config_8x.yml +++ b/internal/profile/_static/kibana_config_8x.yml @@ -15,6 +15,7 @@ xpack.fleet.registryUrl: "https://package-registry:8080" xpack.fleet.agents.enabled: true xpack.fleet.agents.elasticsearch.hosts: ["https://elasticsearch:9200"] xpack.fleet.agents.fleet_server.hosts: ["https://fleet-server:8220"] +xpack.fleet.enableExperimental: ["experimentalDataStreamSettings"] # Enable experimental toggles in Fleet UI xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" diff --git a/internal/profile/profile.go b/internal/profile/profile.go index 937834f21..73cebeec4 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -40,12 +40,15 @@ type configFile string var managedProfileFiles = map[configFile]NewConfig{ ElasticAgentDefaultEnvFile: newElasticAgentDefaultEnv, ElasticAgent8xEnvFile: newElasticAgent8xEnv, + ElasticAgent86EnvFile: newElasticAgent86Env, ElasticAgent80EnvFile: newElasticAgent80Env, ElasticsearchConfigDefaultFile: newElasticsearchConfigDefault, ElasticsearchConfig8xFile: newElasticsearchConfig8x, + ElasticsearchConfig86File: newElasticsearchConfig86, ElasticsearchConfig80File: newElasticsearchConfig80, KibanaConfigDefaultFile: newKibanaConfigDefault, KibanaConfig8xFile: newKibanaConfig8x, + KibanaConfig86File: newKibanaConfig86, KibanaConfig80File: newKibanaConfig80, PackageRegistryDockerfileFile: newPackageRegistryDockerfile, PackageRegistryConfigFile: newPackageRegistryConfig, diff --git a/internal/profile/static.go b/internal/profile/static.go index d1088c6d3..68b72f0e6 100644 --- a/internal/profile/static.go +++ b/internal/profile/static.go @@ -53,6 +53,20 @@ func newKibanaConfig8x(_ string, profilePath string) (*simpleFile, error) { }, nil } +// KibanaConfig86File is the Kibana config file for the 8.x stack family (8.2 to 8.6) +const KibanaConfig86File configFile = "kibana.config.86.yml" + +//go:embed _static/kibana_config_86.yml +var kibanaConfig86Yml string + +func newKibanaConfig86(_ string, profilePath string) (*simpleFile, error) { + return &simpleFile{ + name: string(KibanaConfig86File), + path: filepath.Join(profilePath, profileStackPath, string(KibanaConfig86File)), + body: kibanaConfig86Yml, + }, nil +} + // KibanaConfig80File is the Kibana config file for 8.0 stack family (8.0 to 8.1) const KibanaConfig80File configFile = "kibana.config.80.yml" @@ -95,6 +109,17 @@ func newElasticsearchConfig8x(_ string, profilePath string) (*simpleFile, error) }, nil } +// ElasticsearchConfig8xFile is the Elasticsearch config file for 8.x stack family (8.2 to 8.6) +const ElasticsearchConfig86File configFile = "elasticsearch.config.86.yml" + +func newElasticsearchConfig86(_ string, profilePath string) (*simpleFile, error) { + return &simpleFile{ + name: string(ElasticsearchConfig86File), + path: filepath.Join(profilePath, profileStackPath, string(ElasticsearchConfig86File)), + body: elasticsearchConfig8xYml, + }, nil +} + // ElasticsearchConfig80File is the Elasticsearch virtual config file name for 8.0 stack family (8.0 to 8.1) // This file does not exist in the source code, since it's identical to the 8x config file. const ElasticsearchConfig80File configFile = "elasticsearch.config.80.yml" @@ -154,6 +179,18 @@ func newElasticAgent80Env(_ string, profilePath string) (*simpleFile, error) { }, nil } +// ElasticAgent86EnvFile is the .env for the 8.6 stack. +// This file does not exist in the source code, since it's identical to the 8x env file. +const ElasticAgent86EnvFile configFile = "elastic-agent.86.env" + +func newElasticAgent86Env(_ string, profilePath string) (*simpleFile, error) { + return &simpleFile{ + name: string(ElasticAgent86EnvFile), + path: filepath.Join(profilePath, profileStackPath, string(ElasticAgent86EnvFile)), + body: elasticAgent8xEnv, + }, nil +} + // ElasticAgent8xEnvFile is the .env for the 8x stack. const ElasticAgent8xEnvFile configFile = "elastic-agent.8x.env" diff --git a/internal/stack/variants.go b/internal/stack/variants.go index 986c87569..15bdef78b 100644 --- a/internal/stack/variants.go +++ b/internal/stack/variants.go @@ -14,13 +14,15 @@ import ( // This map is used to deploy different versions of the Elastic stack with matching configurations. var configurationVariantMap = map[string]string{ "8.0-0 - 8.1.x-x": "80", - "^8.2-0": "8x", + "8.2-0 - 8.6.x-x": "86", + "^8.7-0": "8x", } // stackVariantAsEnv function returns a stack variant based on the given stack version. // We identified three variants: // * default, covers all of 7.x branches // * 80, covers stack versions 8.0.0 to 8.1.x +// * 86, covers stack versions 8.2.0 to 8.6.x, they don't support experimental toggles in Fleet // * 8x, supports different configuration options in Kibana, covers stack versions 8.2.0+ func stackVariantAsEnv(version string) string { return fmt.Sprintf("STACK_VERSION_VARIANT=%s", selectStackVersion(version)) diff --git a/internal/stack/variants_test.go b/internal/stack/variants_test.go index 5b5e5a6aa..143389a57 100644 --- a/internal/stack/variants_test.go +++ b/internal/stack/variants_test.go @@ -32,14 +32,18 @@ var tests = []struct { {"8.1.58", "80"}, {"8.1.99-beta", "80"}, {"8.1.999-SNAPSHOT", "80"}, - {"8.2-0", "8x"}, - {"8.2", "8x"}, - {"8.2.0-alpha", "8x"}, - {"8.2.0", "8x"}, - {"8.2.58", "8x"}, - {"8.2.99-gamma", "8x"}, - {"8.2.777-SNAPSHOT+arm64", "8x"}, - {"8.5", "8x"}, + {"8.2-0", "86"}, + {"8.2", "86"}, + {"8.2.0-alpha", "86"}, + {"8.2.0", "86"}, + {"8.2.58", "86"}, + {"8.2.99-gamma", "86"}, + {"8.2.777-SNAPSHOT+arm64", "86"}, + {"8.5", "86"}, + {"8.6.1", "86"}, + {"8.7.0", "8x"}, + {"8.7.0-SNAPSHOT", "8x"}, + {"8.7.1-SNAPSHOT", "8x"}, {"9", "default"}, } From 77594f097b6993ca6d565f9098131a7095fb4f44 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Feb 2023 17:39:20 +0000 Subject: [PATCH 32/33] Chore(deps): bump github.com/shirou/gopsutil/v3 from 3.22.12 to 3.23.1 (#1131) Bumps [github.com/shirou/gopsutil/v3](https://github.com/shirou/gopsutil) from 3.22.12 to 3.23.1. - [Release notes](https://github.com/shirou/gopsutil/releases) - [Commits](https://github.com/shirou/gopsutil/compare/v3.22.12...v3.23.1) --- updated-dependencies: - dependency-name: github.com/shirou/gopsutil/v3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 82a46c0c4..7ca0bdb0a 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 - github.com/shirou/gopsutil/v3 v3.22.12 + github.com/shirou/gopsutil/v3 v3.23.1 github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.1 golang.org/x/oauth2 v0.4.0 diff --git a/go.sum b/go.sum index 62f28074f..e0800e370 100644 --- a/go.sum +++ b/go.sum @@ -332,8 +332,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs= -github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= +github.com/shirou/gopsutil/v3 v3.23.1 h1:a9KKO+kGLKEvcPIs4W62v0nu3sciVDOOOPUD0Hz7z/4= +github.com/shirou/gopsutil/v3 v3.23.1/go.mod h1:NN6mnm5/0k8jw4cBfCnJtr5L7ErOTg18tMNpgFkn0hA= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= From 599941d9957cd76682752f5c7bb5014e75601db9 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 2 Feb 2023 10:24:34 +0100 Subject: [PATCH 33/33] Add pull request configuration for buildkite pr bot (#1129) --- .buildkite/pull-requests.json | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .buildkite/pull-requests.json diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json new file mode 100644 index 000000000..790ff0a57 --- /dev/null +++ b/.buildkite/pull-requests.json @@ -0,0 +1,20 @@ +{ + "jobs": [ + { + "enabled": true, + "pipelineSlug": "elastic-package", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": ["dependabot[bot]", "mergify[bot]"], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", + "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": [ ] + } + ] +}