From 4e860c48308864d73991188ca0afaf94fcc12c9e Mon Sep 17 00:00:00 2001 From: Victor Castell Date: Wed, 23 Jan 2019 21:39:40 +0100 Subject: [PATCH] This adds DynamoDB support in Dkron. The new addition to Valkeyrie allows dkron to use DynamoDB as backend store. This will provide a robust and reliable store with an availability rate of 99,999% using global tables. Signed-off-by: Victor Castell --- CHANGELOG.md | 139 ++++++++++++++++++++++ README.md | 2 +- dkron/agent.go | 2 +- dkron/config.go | 58 ++++----- dkron/store.go | 2 + go.mod | 2 +- go.sum | 6 + website/content/basics/getting-started.md | 5 +- 8 files changed, 182 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76bfe5e98..4014b5a1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,142 @@ +## Unreleased + +- Add DynamoDB support + +## 1.0.2 + +05e318b 💅 +72d9573 Allow sending mail without credentials +bd03a87 Fix docker tagging +7dd22e5 Fix to send mail +b7b739c Log plugin fix and improvements +1ef9356 More specific processor plugin usage logging +979138b Refactor: auth in notifier + +## 1.0.1 + +60b66f9 Conform to the new image_templates in goreleaser +d97559b Copy from the same directory structure in Dockerfile.hub for release +3770122 Don't dockerignore dist folder, it is needed by gorelease docker builder +2824449 Fix contact +24d17a5 Gen +fa323c2 Ignore node modules +2475b37 Move static assets to it's own directory inside static folder +d9ca0a4 Reformat +987dd5d Remove hash from url on modal close +455495c Remove node_modules +0c02ce0 Reorg asset generation in subpackages +6aced8d Use gofmt -s -w +138323a vfsgen is not a direct dep + +## 1.0.0 + +1c10d34 Button +c42884e Checksums for relase +0cb5eda Checksums for testing +c91852b Cookie consent +9865012 Do not install build tools +e280d31 Docker login +3229dc2 Ensure building static binaries +01e62b6 Error on test +69380f5 Ignore system files +3e34f5d No snapshot +5a594a7 Release script +a02a1ab Release with docker +e795210 Remove old dockerignore entries +c9c692c Remove unmarshalTags from dkron.go +c5f5de0 Report errors on unmarshal config +62e1e15 Sums for release +1cf235a UnmarshalTags belongs to the agent and should be public +36f9318 Update readme +80b2ab1 mail-port flag is uint + +## 1.0.0-rc4 + +913ee87 Bump mapstructure +5bd120f Remove legacy config loading +f20fbe5 Update mods + +## 1.0.0-rc3 + +4811e48 Fix UI run and delete +8695242 Redirect to dashboard + +## 1,0.0-rc2 + +d6dbb1a Add toggle to swagger +ffa4feb Deep linking to job views +fdc5344 Don't fury on make +236b5f4 Don't query jobs on interval in Dashboard +ea5e60b Fixes rescheduling on boltdb store +f55e2e3 Gardening and anchor links open modal +b22b362 Gen +6887c36 Logging info +d21cf16 Logging info and use store.Backend type instead of strings in config +28c130b Open modal with anchor links and gardening +1afb3df Several ui fixes introduced when migrating to glyphicons + + +## 1.0.0-rc1 + +ef86e13 Bump go-plugin +d09b942 Bump several dependencies +f96d622 GRPC +8e3b4b9 Ignore dist folder +1b7d4bc Issue template +caf4711 Logrus +5821c8c Mainly etcd +33a12c5 Revert "Bump several dependencies" +fb9460d Update cron-spec.md +706e65d Upgrade pflag + +## 0.11.3 + +723326f Add logging for pending executions response +df76e9c Add real examples to swagger spec +d1318a1 Add tags param to swagger spec +4da0b3b Big docs refactor +2d91a5e Break on errors +8fac831 Command to generate cli docs from cobra +31418de Docs +1622070 Docs +e223fc4 Docs +81fb69a Docs +cea8d20 Docs +b9ae4e0 Docs +5fc9dcf Docs +257f202 Docs for cli and target nodes +bdcd09c Don't use swagger2markup +253fe57 ECS and email pro docs +e89b353 Expvar dep +187190e Fix indentation +c8320b5 Fix testing +9037d65 Fix typo in getting started +9c60fe8 Formatting +f11ed84 Formatting +20be8e5 Integrate swagger-ui for a bit better API visualization +2cede00 Merge branch 'master' into boltdb +53d8464 Only query for pending executions when there is some +712be35 Remove extra useless locking introduced in 88c072c +dacb379 This should be TrimSuffix +dec6701 Update contacts +c21e565 Update getting-started.md +3fdba5f Use boltdb as default storage +70d9229 Wrong dash in example config file +9653bbc expvars are back and simple health endpoint + +## 0.11.2 + +7d88742 Add code of conduct +aed2f44 Proper serf debug logging +1226c93 Publish docker +a0b6f59 Publish docker +f1aaecc Reorg imports +8758bac Tests should use etcdv3 +fa3aaa4 Tests should use v3 client +5bcea4c Update create or update job api endpoint +39728d0 refactor: Methond name +1c64da4 refactor: Proper gin logging and mode + ## 0.11.1 (2018-10-07) - Add support for passing payload to command STDIN (@gustavosbarreto) diff --git a/README.md b/README.md index 2f8806c05..40e7bbbf7 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Next, run the included Docker Compose config: `docker-compose up` -This will start, etc, consul and Dkron instances. To add more Dkron instances to the clusters: +This will start Dkron instances. To add more Dkron instances to the clusters: `docker-compose scale dkron=4` diff --git a/dkron/agent.go b/dkron/agent.go index af1d04d38..616f723ae 100644 --- a/dkron/agent.go +++ b/dkron/agent.go @@ -294,7 +294,7 @@ func (a *Agent) SetConfig(c *Config) { func (a *Agent) StartServer() { if a.Store == nil { var sConfig *store.Config - if a.config.Backend == store.BOLTDB { + if a.config.Backend == store.BOLTDB || a.config.Backend == store.DYNAMODB { sConfig = &store.Config{Bucket: a.config.Keyspace} } a.Store = NewStore(a.config.Backend, a.config.BackendMachines, a, a.config.Keyspace, sConfig) diff --git a/dkron/config.go b/dkron/config.go index 3e568b68e..2e6b15f57 100644 --- a/dkron/config.go +++ b/dkron/config.go @@ -90,38 +90,38 @@ func ConfigFlagSet() *flag.FlagSet { c := DefaultConfig() cmdFlags := flag.NewFlagSet("agent flagset", flag.ContinueOnError) - cmdFlags.Bool("server", false, "This node is running in server mode.") - cmdFlags.String("node-name", c.NodeName, "Name of this node. Must be unique in the cluster.") - cmdFlags.String("bind-addr", c.BindAddr, "Address to bind network listeners to.") - cmdFlags.String("advertise-addr", "", "Address used to advertise to other nodes in the cluster. By default, the bind address is advertised.") - cmdFlags.String("http-addr", c.HTTPAddr, "Address to bind the UI web server to. Only used when server.") - cmdFlags.String("backend", string(c.Backend), "store backend") - cmdFlags.StringSlice("backend-machine", c.BackendMachines, "store backend machines addresses") - cmdFlags.String("profile", c.Profile, "Profile is used to control the timing profiles used. The default if not provided is lan.") - cmdFlags.StringSlice("join", []string{}, "An initial agent to join with. This flag can be specified multiple times.") - cmdFlags.StringSlice("tag", []string{}, "Tag can be specified multiple times to attach multiple key/value tag pairs to the given node. Specified as key=value") - cmdFlags.String("keyspace", c.Keyspace, "The keyspace to use. A prefix under all data is stored for this instance.") - cmdFlags.String("encrypt", "", "Key for encrypting network traffic. Must be a base64-encoded 16-byte key.") - cmdFlags.String("log-level", c.LogLevel, "Log level (debug, info, warn, error, fatal, panic), defaults to info") - cmdFlags.Int("rpc-port", c.RPCPort, "RPC Port used to communicate with clients. Only used when server. The RPC IP Address will be the same as the bind address.") - cmdFlags.Int("advertise-rpc-port", 0, "Use the value of rpc-port by default.") + cmdFlags.Bool("server", false, "This node is running in server mode") + cmdFlags.String("node-name", c.NodeName, "Name of this node. Must be unique in the cluster") + cmdFlags.String("bind-addr", c.BindAddr, "Address to bind network listeners to") + cmdFlags.String("advertise-addr", "", "Address used to advertise to other nodes in the cluster. By default, the bind address is advertised") + cmdFlags.String("http-addr", c.HTTPAddr, "Address to bind the UI web server to. Only used when server") + cmdFlags.String("backend", string(c.Backend), "Store backend (etcd|etcdv3|consul|zk|redis|boltdb|dynamodb)") + cmdFlags.StringSlice("backend-machine", c.BackendMachines, "Store backend machines addresses") + cmdFlags.String("profile", c.Profile, "Profile is used to control the timing profiles used") + cmdFlags.StringSlice("join", []string{}, "An initial agent to join with. This flag can be specified multiple times") + cmdFlags.StringSlice("tag", []string{}, "Tag can be specified multiple times to attach multiple key/value tag pairs to the given node, specified as key=value") + cmdFlags.String("keyspace", c.Keyspace, "The keyspace to use. A prefix under all data is stored for this instance") + cmdFlags.String("encrypt", "", "Key for encrypting network traffic. Must be a base64-encoded 16-byte key") + cmdFlags.String("log-level", c.LogLevel, "Log level (debug|info|warn|error|fatal|panic)") + cmdFlags.Int("rpc-port", c.RPCPort, "RPC Port used to communicate with clients. Only used when server. The RPC IP Address will be the same as the bind address") + cmdFlags.Int("advertise-rpc-port", 0, "Use the value of rpc-port by default") // Notifications - cmdFlags.String("mail-host", "", "Mail server host address to use for notifications.") - cmdFlags.Uint16("mail-port", 0, "Mail server port.") - cmdFlags.String("mail-username", "", "Mail server username used for authentication.") - cmdFlags.String("mail-password", "", "Mail server password to use.") - cmdFlags.String("mail-from", "", "From email address to use.") - cmdFlags.String("mail-payload", "", "Notification mail payload.") - cmdFlags.String("mail-subject-prefix", c.MailSubjectPrefix, "Notification mail subject prefix.") - - cmdFlags.String("webhook-url", "", "Webhook url to call for notifications.") - cmdFlags.String("webhook-payload", "", "Body of the POST request to send on webhook call.") - cmdFlags.StringSlice("webhook-header", []string{}, "Headers to use when calling the webhook URL. Can be specified multiple times.") - - cmdFlags.String("dog-statsd-addr", "", "DataDog Agent address.") + cmdFlags.String("mail-host", "", "Mail server host address to use for notifications") + cmdFlags.Uint16("mail-port", 0, "Mail server port") + cmdFlags.String("mail-username", "", "Mail server username used for authentication") + cmdFlags.String("mail-password", "", "Mail server password to use") + cmdFlags.String("mail-from", "", "From email address to use") + cmdFlags.String("mail-payload", "", "Notification mail payload") + cmdFlags.String("mail-subject-prefix", c.MailSubjectPrefix, "Notification mail subject prefix") + + cmdFlags.String("webhook-url", "", "Webhook url to call for notifications") + cmdFlags.String("webhook-payload", "", "Body of the POST request to send on webhook call") + cmdFlags.StringSlice("webhook-header", []string{}, "Headers to use when calling the webhook URL. Can be specified multiple times") + + cmdFlags.String("dog-statsd-addr", "", "DataDog Agent address") cmdFlags.StringSlice("dog-statsd-tags", []string{}, "Datadog tags, specified as key:value") - cmdFlags.String("statsd-addr", "", "Statsd Address.") + cmdFlags.String("statsd-addr", "", "Statsd address") return cmdFlags } diff --git a/dkron/store.go b/dkron/store.go index d0c77b408..eb1b82523 100644 --- a/dkron/store.go +++ b/dkron/store.go @@ -10,6 +10,7 @@ import ( "github.com/abronan/valkeyrie/store" "github.com/abronan/valkeyrie/store/boltdb" "github.com/abronan/valkeyrie/store/consul" + "github.com/abronan/valkeyrie/store/dynamodb" "github.com/abronan/valkeyrie/store/etcd/v2" "github.com/abronan/valkeyrie/store/etcd/v3" "github.com/abronan/valkeyrie/store/redis" @@ -57,6 +58,7 @@ func init() { zookeeper.Register() redis.Register() boltdb.Register() + dynamodb.Register() } func NewStore(backend store.Backend, machines []string, a *Agent, keyspace string, config *store.Config) *Store { diff --git a/go.mod b/go.mod index 9c4c7df0b..a4af3a263 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ require ( github.com/BurntSushi/toml v0.3.0 // indirect github.com/DataDog/datadog-go v0.0.0-20170427165718-0ddda6bee211 // indirect github.com/abronan/leadership v0.0.0-20171113102752-76df1b7fa384 - github.com/abronan/valkeyrie v0.0.0-20171113095143-2461fc53ec823c81911896e33b8f4253bb3977e2 + github.com/abronan/valkeyrie v0.0.0-20190123051312-61bbc4571a5d github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e github.com/armon/go-metrics v0.0.0-20171002182731-9a4b6e10bed6 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect diff --git a/go.sum b/go.sum index f9f6f7782..9edd950f6 100644 --- a/go.sum +++ b/go.sum @@ -7,10 +7,14 @@ github.com/abronan/leadership v0.0.0-20171113102752-76df1b7fa384 h1:RrhEAhkYDUHl github.com/abronan/leadership v0.0.0-20171113102752-76df1b7fa384/go.mod h1:rZiTOixhU4gU/mIAGIQImeqZJviSBl3/m0vepz3kQkU= github.com/abronan/valkeyrie v0.0.0-20171113095143-2461fc53ec823c81911896e33b8f4253bb3977e2 h1:GDG/JrB/aSFtVZgWcoeSWYqCqMDGAj0BDVSPC17nBtI= github.com/abronan/valkeyrie v0.0.0-20171113095143-2461fc53ec823c81911896e33b8f4253bb3977e2/go.mod h1:JIGF3QSHRw3C3mvFVynyswy1LEgy+3yDqw6czgY7R+M= +github.com/abronan/valkeyrie v0.0.0-20190123051312-61bbc4571a5d h1:zi7xxDWztTeG37nWfp9Sc9WI+BdVzRDr2aXrIrPa7+4= +github.com/abronan/valkeyrie v0.0.0-20190123051312-61bbc4571a5d/go.mod h1:A/IdrwsPb8nboz577OaOlF9StXVQHPXus26p6ZOEhZM= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20171002182731-9a4b6e10bed6 h1:rfTl4Lc+Gfud1YdmAFg99WN4rICM1S65KXBuADdLTms= github.com/armon/go-metrics v0.0.0-20171002182731-9a4b6e10bed6/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/aws/aws-sdk-go v1.16.23 h1:MwBOBeez0XEFVh6DCc888X+nHVBCjUDLnnWXSGGWUgM= +github.com/aws/aws-sdk-go v1.16.23/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= @@ -113,6 +117,8 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jordan-wright/email v0.0.0-20180115032944-94ae17dedda2 h1:BkuA0hfZuy4BoBCbU3ZUAyrgnnsbCVhdedKberVnfC0= diff --git a/website/content/basics/getting-started.md b/website/content/basics/getting-started.md index 378240c7f..a48156e06 100644 --- a/website/content/basics/getting-started.md +++ b/website/content/basics/getting-started.md @@ -29,14 +29,15 @@ Dkron relies on the key-value store for data storage, an instance of the distrib By default dkron will start with a file based, embedded KV store called BoltDB, it is functional for a single node demo but does not offers clustering or HA. {{% /notice %}} -It is compatible with etcd, Consul, Zookeeper, BoltDB and partially with Redis. For instructions on how to install and configure any one of these systems refer to their official sites: +It is compatible with etcd, Consul, Zookeeper, DynamoDB, BoltDB and partially with Redis. For instructions on how to install and configure any one of these systems refer to their official sites: - [etcd](https://coreos.com/etcd/docs/latest/) - [Consul](https://consul.io/intro/getting-started/install.html) - [ZooKeeper](https://zookeeper.apache.org/doc/r3.3.3/zookeeperStarted.html) - [Redis](https://redis.io/topics/quickstart) +- [DynamoDB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/SettingUp.html) -## Configuration +## Installation See the [installation](/basics/installation).