Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

adding feature to support AWS Signing and creating new index per day #302

Merged
merged 6 commits into from
Jul 1, 2020
Merged

adding feature to support AWS Signing and creating new index per day #302

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2c1dddd
adding feature to support AWS Signing and creating new index per day …
Jun 30, 2020
ab4b4cc
adding feature to support AWS Signing and creating new index per day …
Jun 30, 2020
02d19da
Merge branch 'feature/support-aws-signing-for-es' of https://github.c…
Jun 30, 2020
65af86f
fmt fixes, removing default values for Elasticsearch config
Jun 30, 2020
cf3beff
gofmt fixes
Jul 1, 2020
a03bf1c
removing dupicate code for els return object
Jul 1, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions comm_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,13 @@ communications:
# Settings for ELS
elasticsearch:
enabled: false
server: 'ELASTICSEARCH_ADDRESS' # e.g https://example.com:9243
username: 'ELASTICSEARCH_USERNAME'
awsSigning:
enabled: false # enable awsSigning using IAM for Elastisearch hosted on AWS, if true make sure AWS encvironment variables are set. Refer https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
awsRegion: "us-east-1" # AWS region where Elasticsearch is deployed
awsAccessKey: ""
awsSecretKey: ""
server: 'ELASTICSEARCH_ADDRESS' # e.g https://example.com:9243
username: 'ELASTICSEARCH_USERNAME' # Basic Auth
password: 'ELASTICSEARCH_PASSWORD'
# ELS index settings
index:
Expand Down
6 changes: 3 additions & 3 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ module github.com/infracloudio/botkube

require (
github.com/Masterminds/squirrel v1.1.0 // indirect
github.com/aws/aws-sdk-go v1.32.12
github.com/blang/semver v3.5.1+incompatible // indirect
github.com/dyatlov/go-opengraph v0.0.0-20180429202543-816b6608b3c8 // indirect
github.com/evanphx/json-patch v4.5.0+incompatible // indirect
Expand All @@ -10,7 +11,6 @@ require (
github.com/go-gorp/gorp v2.0.0+incompatible // indirect
github.com/go-ldap/ldap v3.0.3+incompatible // indirect
github.com/go-redis/redis v6.15.2+incompatible // indirect
github.com/go-sql-driver/mysql v1.4.1 // indirect
github.com/gogo/protobuf v1.3.0 // indirect
github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc // indirect
github.com/google/go-cmp v0.3.1 // indirect
Expand All @@ -31,14 +31,14 @@ require (
github.com/pborman/uuid v1.2.0 // indirect
github.com/pelletier/go-toml v1.5.0 // indirect
github.com/prometheus/client_golang v1.2.1
github.com/sha1sum/aws_signing_client v0.0.0-20200229211254-f7815c59d5c1
github.com/sirupsen/logrus v1.4.2
github.com/stretchr/testify v1.4.0
github.com/stretchr/testify v1.5.1
github.com/ziutek/mymysql v1.5.4 // indirect
go.uber.org/atomic v1.4.0 // indirect
go.uber.org/multierr v1.2.0 // indirect
go.uber.org/zap v1.10.0 // indirect
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc // indirect
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb // indirect
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 // indirect
google.golang.org/appengine v1.6.5 // indirect
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
Expand Down
12 changes: 12 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/aws/aws-sdk-go v1.32.12 h1:l/djCeLI4ggBFWLlYUGTqkHraoLnVMubNlLXPdEtoYc=
github.com/aws/aws-sdk-go v1.32.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
Expand Down Expand Up @@ -106,6 +108,7 @@ github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDA
github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
Expand Down Expand Up @@ -168,6 +171,8 @@ github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI=
github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
Expand Down Expand Up @@ -253,6 +258,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
Expand All @@ -274,6 +281,8 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/sha1sum/aws_signing_client v0.0.0-20200229211254-f7815c59d5c1 h1:k3oIn0gu6A3olJwowlMKxFwiqTi2wm5UbzBVEomlJEY=
github.com/sha1sum/aws_signing_client v0.0.0-20200229211254-f7815c59d5c1/go.mod h1:hPj3jKAamv0ryZvssbqkCeOWYFmy9itWMSOD7tDsE3E=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
Expand All @@ -299,6 +308,7 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 h1:j2hhcujLRHAg872RWAV5yaUrEjHEObwDv3aImCaNLek=
github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8=
Expand Down Expand Up @@ -344,6 +354,8 @@ golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko=
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
Expand Down
17 changes: 12 additions & 5 deletions pkg/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,11 +116,18 @@ type Slack struct {

// ElasticSearch config auth settings
type ElasticSearch struct {
Enabled bool
Username string
Password string `yaml:",omitempty"`
Server string
Index Index
Enabled bool
Username string
Password string `yaml:",omitempty"`
Server string
AWSSigning AWSSigning `yaml:"awsSigning"`
Index Index
}

// AWSSigning contains AWS configurations
type AWSSigning struct {
Enabled bool
AWSRegion string `yaml:"awsRegion"`
}

// Index settings for ELS
Expand Down
51 changes: 42 additions & 9 deletions pkg/notify/elasticsearch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,22 @@ package notify
import (
"context"
"fmt"
"time"
kartik-moolya marked this conversation as resolved.
Show resolved Hide resolved

"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/signer/v4"
"github.com/infracloudio/botkube/pkg/config"
"github.com/infracloudio/botkube/pkg/events"
"github.com/infracloudio/botkube/pkg/log"
"github.com/olivere/elastic"
"github.com/sha1sum/aws_signing_client"
)

const (
// indexSuffixFormat is the date format that would be appended to the index name
indexSuffixFormat = "02-01-2006"
// awsService for the AWS client to authenticate against
awsService = "es"
)

// ElasticSearch contains auth cred and index setting
Expand All @@ -42,10 +53,32 @@ type ElasticSearch struct {

// NewElasticSearch returns new ElasticSearch object
func NewElasticSearch(c *config.Config) (Notifier, error) {
// create elasticsearch client
elsClient, err := elastic.NewClient(elastic.SetURL(c.Communications.ElasticSearch.Server), elastic.SetBasicAuth(c.Communications.ElasticSearch.Username, c.Communications.ElasticSearch.Password), elastic.SetSniff(false), elastic.SetHealthcheck(false), elastic.SetGzip(true))
if err != nil {
return nil, err
var elsClient *elastic.Client
var err error
if c.Communications.ElasticSearch.AWSSigning.Enabled {
// Get credentials from environment variables and create the AWS Signature Version 4 signer
creds := credentials.NewEnvCredentials()
signer := v4.NewSigner(creds)
awsClient, err := aws_signing_client.New(signer, nil, awsService, c.Communications.ElasticSearch.AWSSigning.AWSRegion)
if err != nil {
return nil, err
}
elsClient, err = elastic.NewClient(elastic.SetURL(c.Communications.ElasticSearch.Server), elastic.SetScheme("https"), elastic.SetHttpClient(awsClient), elastic.SetSniff(false), elastic.SetHealthcheck(false), elastic.SetGzip(false))
if err != nil {
return nil, err
}
} else {
// create elasticsearch client
elsClient, err = elastic.NewClient(
elastic.SetURL(c.Communications.ElasticSearch.Server),
elastic.SetBasicAuth(c.Communications.ElasticSearch.Username, c.Communications.ElasticSearch.Password),
elastic.SetSniff(false),
elastic.SetHealthcheck(false),
elastic.SetGzip(true),
)
if err != nil {
return nil, err
}
}
return &ElasticSearch{
ELSClient: elsClient,
Expand Down Expand Up @@ -78,7 +111,7 @@ func (e *ElasticSearch) SendEvent(event events.Event) (err error) {
event.Cluster = e.ClusterName

// Create index if not exists
exists, err := e.ELSClient.IndexExists(e.Index).Do(ctx)
exists, err := e.ELSClient.IndexExists(e.Index + "-" + time.Now().Format(indexSuffixFormat)).Do(ctx)
if err != nil {
log.Error(fmt.Sprintf("Failed to get index. Error:%s", err.Error()))
return err
Expand All @@ -93,25 +126,25 @@ func (e *ElasticSearch) SendEvent(event events.Event) (err error) {
},
},
}
_, err := e.ELSClient.CreateIndex(e.Index).BodyJson(mapping).Do(ctx)
_, err := e.ELSClient.CreateIndex(e.Index + "-" + time.Now().Format(indexSuffixFormat)).BodyJson(mapping).Do(ctx)
if err != nil {
log.Error(fmt.Sprintf("Failed to create index. Error:%s", err.Error()))
return err
}
}

// Send event to els
_, err = e.ELSClient.Index().Index(e.Index).Type(e.Type).BodyJson(event).Do(ctx)
_, err = e.ELSClient.Index().Index(e.Index + "-" + time.Now().Format(indexSuffixFormat)).Type(e.Type).BodyJson(event).Do(ctx)
if err != nil {
log.Error(fmt.Sprintf("Failed to post data to els. Error:%s", err.Error()))
return err
}
_, err = e.ELSClient.Flush().Index(e.Index).Do(ctx)
_, err = e.ELSClient.Flush().Index(e.Index + "-" + time.Now().Format(indexSuffixFormat)).Do(ctx)
if err != nil {
log.Error(fmt.Sprintf("Failed to flush data to els. Error:%s", err.Error()))
return err
}
log.Debugf("Event successfully sent to ElasticSearch index %s", e.Index)
log.Debugf("Event successfully sent to ElasticSearch index %s", e.Index+"-"+time.Now().Format(indexSuffixFormat))
return nil
}

Expand Down