From 47df9e0724af57185cdb34a3c989f5108d257587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Fri, 21 Aug 2020 21:55:52 +0300 Subject: [PATCH 1/5] Add docs about sample-gitops --- README.md | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/README.md b/README.md index 0aff5087..66e13120 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,67 @@ See the [`pkg/gitdir`](pkg/gitdir) package for details. This package contains utilities used by the rest of the library. The most interesting thing here is the `Patcher` under [`pkg/util/patch`](pkg/util/patch), which can be used to apply patches to `pkg/runtime.Object` compliant types. +## Sample implementations + +All sample binaries in this repo are operating on a sample type called `Car`, that looks something like this: + +```yaml +apiVersion: sample-app.weave.works/v1alpha1 +kind: Car +metadata: + creationTimestamp: "2020-08-17T14:33:16Z" + name: foo + namespace: default +spec: + brand: SAAB + engine: The best one + yearModel: "2008" +status: + acceleration: 0 + distance: 12176941420362965433 + persons: 0 + speed: 53.37474583162469 +``` + +All binaries lets you access the data and fake a modification event using a sample webserver running on `localhost:8888`. + +### sample-gitops + +This is a sample binary that: + +a) clones a Git repo of your choice to a temporary directory, and authenticates using given `id_rsa` and `known_hosts` files. Create a Git repo with e.g. the sample file above, and set up SSH connection. +b) exposes all `Car`s in your Git repository at URL `GET localhost:8888/git/` +c) lets you fake a "reconciler spec/status write" event at path `PUT localhost:8888/git/`, where `name` is the name of the `Car` in your repo you want to modify +d) re-syncs every 10 seconds, and tries to pull the git repo +e) has an inotify watch on the temporary Git clone, so it will log all objects that have been changed as they happen in Git (e.g. from new commits) + +When you modify a the "desired state/current status" using e.g. `curl -sSL -X PUT localhost:8888/git/foo`, the following will happen: + +a) a `Transaction` will be started, which means `git pull` and `git checkout -b -update-` will happen underneath +b) `Storage.Get` for the `Car` with the given name will be requested +c) the Car's `.status.distance` and `.status.speed` fields are updated to random numbers, and `Storage.Update` is run +d) the transaction is "committed" by returning a `transaction.PullRequestResult` +e) when the transaction ends, `git commit -A -m `, `git push` and `git checkout
` will be ran. The `git pull` loop is resumed. +f) as a `transaction.PullRequestResult` was returned (and not `transaction.CommitResult`), the code will also use a `transaction.PullRequestProvider` to create a PR towards the repo. The configured provider is for now Github-only, and configured through the `GITHUB_TOKEN` variable. +g) the PR will be created for the given branch, with assignees, labels and a milestone as configured +h) once the PR is merged, the `git pull` loop will eventually download the new commit, and the inotify watch will tell which files were changed. + +#### Usage + +```console +$ make +... +$ bin/sample-gitops --help +Usage of bin/sample-gitops: + --author-email string Author email for Git commits (default "support@weave.works") + --author-name string Author name for Git commits (default "Weave libgitops") + --git-url string HTTPS Git URL; where the Git repository is, e.g. https://github.com/luxas/ignite-gitops + --identity-file string Path to where the SSH private key is + --version Show version information and exit +``` + +You also need to set `GITHUB_TOKEN` in order to be able to create the PR. + ## Getting Help If you have any questions about, feedback for or problems with `libgitops`: From feb581913fcbc9bec960eb352b5ae50546a5989a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Fri, 21 Aug 2020 22:02:59 +0300 Subject: [PATCH 2/5] Split sample-app and -watch --- Makefile | 2 +- cmd/sample-app/main.go | 61 +++------------------------ cmd/sample-gitops/main.go | 4 +- cmd/sample-watch/main.go | 89 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 59 deletions(-) create mode 100644 cmd/sample-watch/main.go diff --git a/Makefile b/Makefile index 3ce7ba50..4f3230b9 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SRC_PKGS := cmd pkg DOCKER_ARGS := --rm CACHE_DIR := $(shell pwd)/bin/cache API_DOCS := api/sample-app.md api/runtime.md -BINARIES := bin/sample-app bin/sample-gitops +BINARIES := bin/sample-app bin/sample-gitops bin/sample-watch # If we're not running in CI, run Docker interactively ifndef CI diff --git a/cmd/sample-app/main.go b/cmd/sample-app/main.go index 3103324c..729f19c6 100644 --- a/cmd/sample-app/main.go +++ b/cmd/sample-app/main.go @@ -3,13 +3,12 @@ package main import ( "bytes" "fmt" - "github.com/weaveworks/libgitops/pkg/storage/watch" - "github.com/weaveworks/libgitops/pkg/storage/watch/update" "net/http" "os" "github.com/labstack/echo" "github.com/sirupsen/logrus" + "github.com/spf13/pflag" "github.com/weaveworks/libgitops/cmd/common" "github.com/weaveworks/libgitops/cmd/sample-app/apis/sample/scheme" "github.com/weaveworks/libgitops/cmd/sample-app/apis/sample/v1alpha1" @@ -19,10 +18,7 @@ import ( "github.com/weaveworks/libgitops/pkg/storage" ) -const ( - ManifestDir = "/tmp/libgitops/manifest" - WatchDir = "/tmp/libgitops/watch" -) +var manifestDirFlag = pflag.String("data-dir", "/tmp/libgitops/manifest", "Where to store the JSON files") func main() { // Parse the version flag @@ -36,11 +32,8 @@ func main() { } func run() error { - // Create the manifest and watch directories - if err := os.MkdirAll(ManifestDir, 0755); err != nil { - return err - } - if err := os.MkdirAll(WatchDir, 0755); err != nil { + // Create the manifest directory + if err := os.MkdirAll(*manifestDirFlag, 0755); err != nil { return err } @@ -48,27 +41,12 @@ func run() error { logs.Logger.SetLevel(logrus.TraceLevel) plainStorage := storage.NewGenericStorage( - storage.NewGenericRawStorage(ManifestDir, v1alpha1.SchemeGroupVersion, serializer.ContentTypeYAML), + storage.NewGenericRawStorage(*manifestDirFlag, v1alpha1.SchemeGroupVersion, serializer.ContentTypeYAML), scheme.Serializer, []runtime.IdentifierFactory{runtime.Metav1NameIdentifier}, ) defer func() { _ = plainStorage.Close() }() - watchStorage, err := watch.NewManifestStorage(WatchDir, scheme.Serializer) - if err != nil { - return err - } - defer func() { _ = watchStorage.Close() }() - - updates := make(chan update.Update, 4096) - watchStorage.SetUpdateStream(updates) - - go func() { - for upd := range updates { - logrus.Infof("Got %s update for: %v %v", upd.Event, upd.PartialObject.GetObjectKind().GroupVersionKind(), upd.PartialObject.GetObjectMeta()) - } - }() - e := common.NewEcho() e.GET("/plain/:name", func(c echo.Context) error { @@ -112,34 +90,5 @@ func run() error { return c.String(200, "OK!") }) - e.GET("/watch/:name", func(c echo.Context) error { - name := c.Param("name") - if len(name) == 0 { - return echo.NewHTTPError(http.StatusBadRequest, "Please set name") - } - - obj, err := watchStorage.Get(common.CarKeyForName(name)) - if err != nil { - return err - } - var content bytes.Buffer - if err := scheme.Serializer.Encoder().Encode(serializer.NewJSONFrameWriter(&content), obj); err != nil { - return err - } - return c.JSONBlob(http.StatusOK, content.Bytes()) - }) - - e.PUT("/watch/:name", func(c echo.Context) error { - name := c.Param("name") - if len(name) == 0 { - return echo.NewHTTPError(http.StatusBadRequest, "Please set name") - } - - if err := common.SetNewCarStatus(plainStorage, common.CarKeyForName(name)); err != nil { - return err - } - return c.String(200, "OK!") - }) - return common.StartEcho(e) } diff --git a/cmd/sample-gitops/main.go b/cmd/sample-gitops/main.go index 8a53a37c..e8c21805 100644 --- a/cmd/sample-gitops/main.go +++ b/cmd/sample-gitops/main.go @@ -3,8 +3,6 @@ package main import ( "context" "fmt" - "github.com/weaveworks/libgitops/pkg/storage/watch" - "github.com/weaveworks/libgitops/pkg/storage/watch/update" "io/ioutil" "net/http" "os" @@ -23,6 +21,8 @@ import ( "github.com/weaveworks/libgitops/pkg/storage" "github.com/weaveworks/libgitops/pkg/storage/transaction" githubpr "github.com/weaveworks/libgitops/pkg/storage/transaction/pullrequest/github" + "github.com/weaveworks/libgitops/pkg/storage/watch" + "github.com/weaveworks/libgitops/pkg/storage/watch/update" ) var ( diff --git a/cmd/sample-watch/main.go b/cmd/sample-watch/main.go new file mode 100644 index 00000000..745862da --- /dev/null +++ b/cmd/sample-watch/main.go @@ -0,0 +1,89 @@ +package main + +import ( + "bytes" + "fmt" + "net/http" + "os" + + "github.com/labstack/echo" + "github.com/sirupsen/logrus" + "github.com/spf13/pflag" + "github.com/weaveworks/libgitops/cmd/common" + "github.com/weaveworks/libgitops/cmd/sample-app/apis/sample/scheme" + "github.com/weaveworks/libgitops/pkg/logs" + "github.com/weaveworks/libgitops/pkg/serializer" + "github.com/weaveworks/libgitops/pkg/storage/watch" + "github.com/weaveworks/libgitops/pkg/storage/watch/update" +) + +var watchDirFlag = pflag.String("watch-dir", "/tmp/libgitops/watch", "Where to watch for YAML/JSON manifests") + +func main() { + // Parse the version flag + common.ParseVersionFlag() + + // Run the application + if err := run(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func run() error { + // Create the watch directory + if err := os.MkdirAll(*watchDirFlag, 0755); err != nil { + return err + } + + // Set the log level + logs.Logger.SetLevel(logrus.TraceLevel) + + watchStorage, err := watch.NewManifestStorage(*watchDirFlag, scheme.Serializer) + if err != nil { + return err + } + defer func() { _ = watchStorage.Close() }() + + updates := make(chan update.Update, 4096) + watchStorage.SetUpdateStream(updates) + + go func() { + for upd := range updates { + logrus.Infof("Got %s update for: %v %v", upd.Event, upd.PartialObject.GetObjectKind().GroupVersionKind(), upd.PartialObject.GetObjectMeta()) + } + }() + + e := common.NewEcho() + + e.GET("/watch/:name", func(c echo.Context) error { + name := c.Param("name") + if len(name) == 0 { + return echo.NewHTTPError(http.StatusBadRequest, "Please set name") + } + + obj, err := watchStorage.Get(common.CarKeyForName(name)) + if err != nil { + return err + } + var content bytes.Buffer + if err := scheme.Serializer.Encoder().Encode(serializer.NewJSONFrameWriter(&content), obj); err != nil { + return err + } + return c.JSONBlob(http.StatusOK, content.Bytes()) + }) + + e.PUT("/watch/:name", func(c echo.Context) error { + name := c.Param("name") + if len(name) == 0 { + return echo.NewHTTPError(http.StatusBadRequest, "Please set name") + } + + if err := common.SetNewCarStatus(watchStorage, common.CarKeyForName(name)); err != nil { + return err + } + return c.String(200, "OK!") + }) + + return common.StartEcho(e) +} From c22691474e71c052fa7e760dbdf58543c98146c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Fri, 21 Aug 2020 22:16:33 +0300 Subject: [PATCH 3/5] flagify and use info loglevel --- cmd/sample-app/main.go | 4 ++-- cmd/sample-watch/main.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/sample-app/main.go b/cmd/sample-app/main.go index 729f19c6..ea119a95 100644 --- a/cmd/sample-app/main.go +++ b/cmd/sample-app/main.go @@ -18,7 +18,7 @@ import ( "github.com/weaveworks/libgitops/pkg/storage" ) -var manifestDirFlag = pflag.String("data-dir", "/tmp/libgitops/manifest", "Where to store the JSON files") +var manifestDirFlag = pflag.String("data-dir", "/tmp/libgitops/manifest", "Where to store the YAML files") func main() { // Parse the version flag @@ -38,7 +38,7 @@ func run() error { } // Set the log level - logs.Logger.SetLevel(logrus.TraceLevel) + logs.Logger.SetLevel(logrus.InfoLevel) plainStorage := storage.NewGenericStorage( storage.NewGenericRawStorage(*manifestDirFlag, v1alpha1.SchemeGroupVersion, serializer.ContentTypeYAML), diff --git a/cmd/sample-watch/main.go b/cmd/sample-watch/main.go index 745862da..ef1aec0a 100644 --- a/cmd/sample-watch/main.go +++ b/cmd/sample-watch/main.go @@ -37,7 +37,7 @@ func run() error { } // Set the log level - logs.Logger.SetLevel(logrus.TraceLevel) + logs.Logger.SetLevel(logrus.InfoLevel) watchStorage, err := watch.NewManifestStorage(*watchDirFlag, scheme.Serializer) if err != nil { From e267cde6c37b46b4650a80a6dddadd7c164d5801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Fri, 21 Aug 2020 22:16:55 +0300 Subject: [PATCH 4/5] document sample-app and watch --- README.md | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 519d55d2..049a5614 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ f) as a `transaction.PullRequestResult` was returned (and not `transaction.Commi g) the PR will be created for the given branch, with assignees, labels and a milestone as configured h) once the PR is merged, the `git pull` loop will eventually download the new commit, and the inotify watch will tell which files were changed. -#### Usage +#### sample-gitops Usage ```console $ make @@ -209,11 +209,55 @@ Usage of bin/sample-gitops: --author-name string Author name for Git commits (default "Weave libgitops") --git-url string HTTPS Git URL; where the Git repository is, e.g. https://github.com/luxas/ignite-gitops --identity-file string Path to where the SSH private key is + --pr-assignees strings What user logins to assign for the created PR. The user must have pull access to the repo. + --pr-milestone string What milestone to tag the PR with --version Show version information and exit ``` You also need to set `GITHUB_TOKEN` in order to be able to create the PR. +### sample-watch + +sample-watch demonstrates use of the inotify `GenericWatchStorage` on a customizable directory. + +When running it, create a file (e.g. the example above) anywhere in the folder you're watching. +You'll see it being noticed in the log. Once that's done, you can curl it like this: `curl -sSL localhost:8888/watch/`, where name equals `.metadata.name` of the object you just put in the dir. + +You can also write a new status using `curl -sSL -X PUT localhost:8888/watch/foo`, so that the next time you get it as per above, you can see the status has changed. + +#### sample-watch Usage + +```console +$ make +... +$ bin/sample-watch --help +Usage of bin/sample-watch: + --version Show version information and exit + --watch-dir string Where to watch for YAML/JSON manifests (default "/tmp/libgitops/watch") +``` + +### sample-app + +sample-app is using the `GenericStorage` and `GenericRawStorage` on the directory of your choice. The path where the objects are stored are of the form `///metadata.json`. + +Use it as follows: + +a) create a new `Car` using `curl -sSL -X POST localhost:8888/plain/foo` +b) see the object in e.g. using `cat /tmp/libgitops/manifest/Car/default/foo/metadata.yaml` +c) get it through the webserver using `curl -sSL localhost:8888/plain/foo` +d) update status through `curl -sSL -X PUT localhost:8888/plain/foo` + +#### sample-app Usage + +```console +$ make +... +$ bin/sample-app --help +Usage of bin/sample-app: + --data-dir string Where to store the YAML files (default "/tmp/libgitops/manifest") + --version Show version information and exit +``` + ## Getting Help If you have any questions about, feedback for or problems with `libgitops`: From 1ccaadedca35ef749bafb8237f42b78ed3a1e267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Fri, 21 Aug 2020 22:17:42 +0300 Subject: [PATCH 5/5] Apply suggestions from code review Co-authored-by: Dennis Marttinen <38858901+twelho@users.noreply.github.com> --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 049a5614..d64d180e 100644 --- a/README.md +++ b/README.md @@ -175,27 +175,27 @@ status: speed: 53.37474583162469 ``` -All binaries lets you access the data and fake a modification event using a sample webserver running on `localhost:8888`. +All binaries let you access the data and fake a modification event using a sample webserver running on `localhost:8888`. ### sample-gitops This is a sample binary that: -a) clones a Git repo of your choice to a temporary directory, and authenticates using given `id_rsa` and `known_hosts` files. Create a Git repo with e.g. the sample file above, and set up SSH connection. +a) clones a Git repo of your choice to a temporary directory, and authenticates using given `id_rsa` and `known_hosts` files. Create a Git repo with e.g. the sample file above, and set up SSH credentials. b) exposes all `Car`s in your Git repository at URL `GET localhost:8888/git/` c) lets you fake a "reconciler spec/status write" event at path `PUT localhost:8888/git/`, where `name` is the name of the `Car` in your repo you want to modify d) re-syncs every 10 seconds, and tries to pull the git repo e) has an inotify watch on the temporary Git clone, so it will log all objects that have been changed as they happen in Git (e.g. from new commits) -When you modify a the "desired state/current status" using e.g. `curl -sSL -X PUT localhost:8888/git/foo`, the following will happen: +When you modify the "desired state/current status" using e.g. `curl -sSL -X PUT localhost:8888/git/foo`, the following will happen: -a) a `Transaction` will be started, which means `git pull` and `git checkout -b -update-` will happen underneath +a) a `Transaction` will be started, which means `git pull` and `git checkout -b -update-` will be executed b) `Storage.Get` for the `Car` with the given name will be requested c) the Car's `.status.distance` and `.status.speed` fields are updated to random numbers, and `Storage.Update` is run d) the transaction is "committed" by returning a `transaction.PullRequestResult` -e) when the transaction ends, `git commit -A -m `, `git push` and `git checkout
` will be ran. The `git pull` loop is resumed. -f) as a `transaction.PullRequestResult` was returned (and not `transaction.CommitResult`), the code will also use a `transaction.PullRequestProvider` to create a PR towards the repo. The configured provider is for now Github-only, and configured through the `GITHUB_TOKEN` variable. -g) the PR will be created for the given branch, with assignees, labels and a milestone as configured +e) when the transaction ends, `git commit -A -m `, `git push` and `git checkout
` will be executed. The `git pull` loop is resumed. +f) as a `transaction.PullRequestResult` was returned (and not `transaction.CommitResult`), the code will also use a `transaction.PullRequestProvider` to create a PR towards the repo. The configured provider is for now GitHub-only, and configured through passing the `GITHUB_TOKEN` environment variable. +g) the PR will be created for the given branch, with the given assignees, labels and milestone h) once the PR is merged, the `git pull` loop will eventually download the new commit, and the inotify watch will tell which files were changed. #### sample-gitops Usage