Skip to content

Commit

Permalink
adding feature to support AWS Signing and creating new index per day (#…
Browse files Browse the repository at this point in the history 
…302)

##### ISSUE TYPE
<!--- Pick one below and delete the rest: -->
 - Feature Pull Request

##### SUMMARY
<!--- Describe the change, including rationale and design decisions -->

Allows user to specify if Elasticsearch is hosted on AWS
- configuration fields have been added to specify if Elasticsearch is hosted on AWS

<!---
If you are fixing an existing issue, please include "Fixes #nnn" in your
PR comment; and describe briefly what the change does.
-->
Includes feature which would allow botkube to send Kubernetes events to Elasticsearch hosted on AWS
* Uses AWS V4 signature to sign the http request
* Uses environment variables to pick AWS configurations

Secondly creates new index per day with index name suffixed as -DD-MM-YYYY
<!--- Please list dependencies added with your change also -->

Fixes #299 
Fixes #283
  • Loading branch information
@kartik-moolya
kartik-moolya authored Jul 1, 2020
1 parent cc3fc6d commit 5749185
Show file tree
Hide file tree
Showing 5 changed files with 76 additions and 19 deletions.
9 changes: 7 additions & 2 deletions comm_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,13 @@ communications:
# Settings for ELS
elasticsearch:
enabled: false
server: 'ELASTICSEARCH_ADDRESS' # e.g https://example.com:9243
username: 'ELASTICSEARCH_USERNAME'
awsSigning:
enabled: false # enable awsSigning using IAM for Elastisearch hosted on AWS, if true make sure AWS encvironment variables are set. Refer https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
awsRegion: "us-east-1" # AWS region where Elasticsearch is deployed
awsAccessKey: ""
awsSecretKey: ""
server: 'ELASTICSEARCH_ADDRESS' # e.g https://example.com:9243
username: 'ELASTICSEARCH_USERNAME' # Basic Auth
password: 'ELASTICSEARCH_PASSWORD'
# ELS index settings
index:
Expand Down
6 changes: 3 additions & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ module github.com/infracloudio/botkube

require (
github.com/Masterminds/squirrel v1.1.0 // indirect
github.com/aws/aws-sdk-go v1.32.12
github.com/blang/semver v3.5.1+incompatible // indirect
github.com/dyatlov/go-opengraph v0.0.0-20180429202543-816b6608b3c8 // indirect
github.com/evanphx/json-patch v4.5.0+incompatible // indirect
Expand All @@ -10,7 +11,6 @@ require (
github.com/go-gorp/gorp v2.0.0+incompatible // indirect
github.com/go-ldap/ldap v3.0.3+incompatible // indirect
github.com/go-redis/redis v6.15.2+incompatible // indirect
github.com/go-sql-driver/mysql v1.4.1 // indirect
github.com/gogo/protobuf v1.3.0 // indirect
github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc // indirect
github.com/google/go-cmp v0.3.1 // indirect
Expand All @@ -31,14 +31,14 @@ require (
github.com/pborman/uuid v1.2.0 // indirect
github.com/pelletier/go-toml v1.5.0 // indirect
github.com/prometheus/client_golang v1.2.1
github.com/sha1sum/aws_signing_client v0.0.0-20200229211254-f7815c59d5c1
github.com/sirupsen/logrus v1.4.2
github.com/stretchr/testify v1.4.0
github.com/stretchr/testify v1.5.1
github.com/ziutek/mymysql v1.5.4 // indirect
go.uber.org/atomic v1.4.0 // indirect
go.uber.org/multierr v1.2.0 // indirect
go.uber.org/zap v1.10.0 // indirect
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc // indirect
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb // indirect
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 // indirect
google.golang.org/appengine v1.6.5 // indirect
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
Expand Down
12 changes: 12 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/aws/aws-sdk-go v1.32.12 h1:l/djCeLI4ggBFWLlYUGTqkHraoLnVMubNlLXPdEtoYc=
github.com/aws/aws-sdk-go v1.32.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
Expand Down Expand Up @@ -106,6 +108,7 @@ github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDA
github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
Expand Down Expand Up @@ -168,6 +171,8 @@ github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI=
github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
Expand Down Expand Up @@ -253,6 +258,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
Expand All @@ -274,6 +281,8 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/sha1sum/aws_signing_client v0.0.0-20200229211254-f7815c59d5c1 h1:k3oIn0gu6A3olJwowlMKxFwiqTi2wm5UbzBVEomlJEY=
github.com/sha1sum/aws_signing_client v0.0.0-20200229211254-f7815c59d5c1/go.mod h1:hPj3jKAamv0ryZvssbqkCeOWYFmy9itWMSOD7tDsE3E=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
Expand All @@ -299,6 +308,7 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 h1:j2hhcujLRHAg872RWAV5yaUrEjHEObwDv3aImCaNLek=
github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8=
Expand Down Expand Up @@ -344,6 +354,8 @@ golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko=
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
Expand Down
17 changes: 12 additions & 5 deletions pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,11 +116,18 @@ type Slack struct {

// ElasticSearch config auth settings
type ElasticSearch struct {
Enabled bool
Username string
Password string `yaml:",omitempty"`
Server string
Index Index
Enabled bool
Username string
Password string `yaml:",omitempty"`
Server string
AWSSigning AWSSigning `yaml:"awsSigning"`
Index Index
}

// AWSSigning contains AWS configurations
type AWSSigning struct {
Enabled bool
AWSRegion string `yaml:"awsRegion"`
}

// Index settings for ELS
Expand Down
51 changes: 42 additions & 9 deletions pkg/notify/elasticsearch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,22 @@ package notify
import (
"context"
"fmt"
"time"

"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/signer/v4"
"github.com/infracloudio/botkube/pkg/config"
"github.com/infracloudio/botkube/pkg/events"
"github.com/infracloudio/botkube/pkg/log"
"github.com/olivere/elastic"
"github.com/sha1sum/aws_signing_client"
)

const (
// indexSuffixFormat is the date format that would be appended to the index name
indexSuffixFormat = "02-01-2006"
// awsService for the AWS client to authenticate against
awsService = "es"
)

// ElasticSearch contains auth cred and index setting
Expand All @@ -42,10 +53,32 @@ type ElasticSearch struct {

// NewElasticSearch returns new ElasticSearch object
func NewElasticSearch(c *config.Config) (Notifier, error) {
// create elasticsearch client
elsClient, err := elastic.NewClient(elastic.SetURL(c.Communications.ElasticSearch.Server), elastic.SetBasicAuth(c.Communications.ElasticSearch.Username, c.Communications.ElasticSearch.Password), elastic.SetSniff(false), elastic.SetHealthcheck(false), elastic.SetGzip(true))
if err != nil {
return nil, err
var elsClient *elastic.Client
var err error
if c.Communications.ElasticSearch.AWSSigning.Enabled {
// Get credentials from environment variables and create the AWS Signature Version 4 signer
creds := credentials.NewEnvCredentials()
signer := v4.NewSigner(creds)
awsClient, err := aws_signing_client.New(signer, nil, awsService, c.Communications.ElasticSearch.AWSSigning.AWSRegion)
if err != nil {
return nil, err
}
elsClient, err = elastic.NewClient(elastic.SetURL(c.Communications.ElasticSearch.Server), elastic.SetScheme("https"), elastic.SetHttpClient(awsClient), elastic.SetSniff(false), elastic.SetHealthcheck(false), elastic.SetGzip(false))
if err != nil {
return nil, err
}
} else {
// create elasticsearch client
elsClient, err = elastic.NewClient(
elastic.SetURL(c.Communications.ElasticSearch.Server),
elastic.SetBasicAuth(c.Communications.ElasticSearch.Username, c.Communications.ElasticSearch.Password),
elastic.SetSniff(false),
elastic.SetHealthcheck(false),
elastic.SetGzip(true),
)
if err != nil {
return nil, err
}
}
return &ElasticSearch{
ELSClient: elsClient,
Expand Down Expand Up @@ -78,7 +111,7 @@ func (e *ElasticSearch) SendEvent(event events.Event) (err error) {
event.Cluster = e.ClusterName

// Create index if not exists
exists, err := e.ELSClient.IndexExists(e.Index).Do(ctx)
exists, err := e.ELSClient.IndexExists(e.Index + "-" + time.Now().Format(indexSuffixFormat)).Do(ctx)
if err != nil {
log.Error(fmt.Sprintf("Failed to get index. Error:%s", err.Error()))
return err
Expand All @@ -93,25 +126,25 @@ func (e *ElasticSearch) SendEvent(event events.Event) (err error) {
},
},
}
_, err := e.ELSClient.CreateIndex(e.Index).BodyJson(mapping).Do(ctx)
_, err := e.ELSClient.CreateIndex(e.Index + "-" + time.Now().Format(indexSuffixFormat)).BodyJson(mapping).Do(ctx)
if err != nil {
log.Error(fmt.Sprintf("Failed to create index. Error:%s", err.Error()))
return err
}
}

// Send event to els
_, err = e.ELSClient.Index().Index(e.Index).Type(e.Type).BodyJson(event).Do(ctx)
_, err = e.ELSClient.Index().Index(e.Index + "-" + time.Now().Format(indexSuffixFormat)).Type(e.Type).BodyJson(event).Do(ctx)
if err != nil {
log.Error(fmt.Sprintf("Failed to post data to els. Error:%s", err.Error()))
return err
}
_, err = e.ELSClient.Flush().Index(e.Index).Do(ctx)
_, err = e.ELSClient.Flush().Index(e.Index + "-" + time.Now().Format(indexSuffixFormat)).Do(ctx)
if err != nil {
log.Error(fmt.Sprintf("Failed to flush data to els. Error:%s", err.Error()))
return err
}
log.Debugf("Event successfully sent to ElasticSearch index %s", e.Index)
log.Debugf("Event successfully sent to ElasticSearch index %s", e.Index+"-"+time.Now().Format(indexSuffixFormat))
return nil
}

Expand Down

0 comments on commit 5749185

Please sign in to comment.