From bc37f7c5ed604e9ae066c9f39707aeb466107105 Mon Sep 17 00:00:00 2001 From: Anmol Date: Fri, 25 Aug 2023 18:53:37 +0530 Subject: [PATCH] remove collector service completly, since it is dead code (#187) --- .bazelproject | 1 - .github/workflows/build.yaml | 27 -- charts/devnet/scripts/register_snapshots.sh | 24 -- .../templates/chains/cosmos/genesis.yaml | 4 - .../templates/chains/cosmos/validator.yaml | 4 - charts/devnet/templates/collector.yaml | 73 ------ charts/devnet/values.yaml | 3 - collector/BUILD.bazel | 53 ---- collector/Dockerfile | 27 -- collector/Makefile | 16 -- collector/app.go | 134 ---------- collector/config.go | 115 --------- collector/const.go | 21 -- collector/error.go | 64 ----- collector/fileserver.go | 243 ------------------ collector/go.mod | 22 -- collector/go.sum | 25 -- collector/handler.go | 112 -------- collector/main.go | 61 ----- collector/model.go | 62 ----- go.work | 3 +- tools/go.mod | 3 +- tools/go.sum | 3 +- 23 files changed, 5 insertions(+), 1095 deletions(-) delete mode 100644 charts/devnet/scripts/register_snapshots.sh delete mode 100644 charts/devnet/templates/collector.yaml delete mode 100644 collector/BUILD.bazel delete mode 100644 collector/Dockerfile delete mode 100644 collector/Makefile delete mode 100644 collector/app.go delete mode 100644 collector/config.go delete mode 100644 collector/const.go delete mode 100644 collector/error.go delete mode 100644 collector/fileserver.go delete mode 100644 collector/go.mod delete mode 100644 collector/go.sum delete mode 100644 collector/handler.go delete mode 100644 collector/main.go delete mode 100644 collector/model.go diff --git a/.bazelproject b/.bazelproject index 57b6ebbe..9959cac9 100644 --- a/.bazelproject +++ b/.bazelproject @@ -9,7 +9,6 @@ additional_languages: targets: //exposer:exposer //registry:registry - //collector:collector //tests:tests //cmd/starship:starship diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 33edb88e..a992592d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -66,30 +66,3 @@ jobs: - name: Build registry run: | bazel build //registry:registry - - build-collector: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Bazel cache - id: bazel-cache - uses: actions/cache@v3 - env: - version: 4.2.1 - with: - path: "~/.cache/bazel" - key: ${{ runner.os }}-${{ env.version }}-bazel-cache - - - name: Install bazelisk - run: | - curl -LO "https://github.com/bazelbuild/bazelisk/releases/download/v1.1.0/bazelisk-linux-amd64" - mkdir -p "${GITHUB_WORKSPACE}/bin/" - mv bazelisk-linux-amd64 "${GITHUB_WORKSPACE}/bin/bazel" - chmod +x "${GITHUB_WORKSPACE}/bin/bazel" - - - name: Build collector - run: | - bazel build //collector:collector \ No newline at end of file diff --git a/charts/devnet/scripts/register_snapshots.sh b/charts/devnet/scripts/register_snapshots.sh deleted file mode 100644 index b767e48a..00000000 --- a/charts/devnet/scripts/register_snapshots.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -CHAIN_ID="${CHAIN_ID:=osmosis-1}" -VAL_NAME="${VAL_NAME:=osmosis}" -CHAIN_DIR="${CHAIN_DIR:=$HOME/.osmosisd}" -COLLECTOR_HOST="${COLLECTOR_HOST}" - -set -euxo pipefail - -snapshot_name=data_${VAL_NAME}_$(date "+%F-%H-%M-%S") - -# Create the snapshot that will be uploaded -function create_snapshot { - tar -czvf /opt/${snapshot_name}.tar.gz $CHAIN_DIR/data -} - -# Register the snapshot to the collector service -function register_snapshot { - url=${COLLECTOR_HOST}/chains/${CHAIN_ID}/validators/${VAL_NAME}/snapshots/${snapshot_name}.tar.gz - curl -v -i ${url} -H'Content-Encoding: gzip' -H'Content-TYPE: application/gzip' --data-binary @/opt/${snapshot_name}.tar.gz -} - -create_snapshot -register_snapshot \ No newline at end of file diff --git a/charts/devnet/templates/chains/cosmos/genesis.yaml b/charts/devnet/templates/chains/cosmos/genesis.yaml index f0f90938..95e9cdc9 100644 --- a/charts/devnet/templates/chains/cosmos/genesis.yaml +++ b/charts/devnet/templates/chains/cosmos/genesis.yaml @@ -136,10 +136,6 @@ spec: env: {{- include "devnet.defaultEvnVars" $chain | indent 12 }} {{- include "devnet.evnVars" $chain | indent 12 }} - - name: COLLECTOR_SERVICE - value: collector - - name: COLLECTOR_PORT - value: "8070" command: - bash - "-c" diff --git a/charts/devnet/templates/chains/cosmos/validator.yaml b/charts/devnet/templates/chains/cosmos/validator.yaml index bcfb86d2..5461cc68 100644 --- a/charts/devnet/templates/chains/cosmos/validator.yaml +++ b/charts/devnet/templates/chains/cosmos/validator.yaml @@ -161,10 +161,6 @@ spec: {{- include "devnet.defaultEvnVars" $chain | indent 12 }} {{- include "devnet.evnVars" $chain | indent 12 }} {{- include "devnet.genesisVars" $dataExposer | indent 12}} - - name: COLLECTOR_SERVICE - value: collector - - name: COLLECTOR_PORT - value: "8070" command: - bash - "-c" diff --git a/charts/devnet/templates/collector.yaml b/charts/devnet/templates/collector.yaml deleted file mode 100644 index d2a69bf5..00000000 --- a/charts/devnet/templates/collector.yaml +++ /dev/null @@ -1,73 +0,0 @@ -{{- if .Values.collector.enabled }} ---- -apiVersion: v1 -kind: Service -metadata: - name: collector - labels: - app.kubernetes.io/name: collector -spec: - clusterIP: None - ports: - - name: http - port: 8070 - protocol: TCP - targetPort: 8070 - selector: - app.kubernetes.io/name: collector ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: collector -spec: - replicas: 1 - revisionHistoryLimit: 3 - selector: - matchLabels: - app.kubernetes.io/instance: collector - app.kubernetes.io/name: collector - template: - metadata: - annotations: - quality: release - role: api-gateway - sla: high - tier: gateway - labels: - app.kubernetes.io/instance: collector - app.kubernetes.io/type: collector - app.kubernetes.io/name: collector - app.kubernetes.io/version: {{ $.Chart.AppVersion }} - spec: - {{- include "imagePullSecrets" .Values.collector | indent 6 }} - containers: - - name: collector - image: {{ .Values.collector.image }} - imagePullPolicy: Always - env: - - name: COLLECTOR_ADDR - value: ":8070" - - name: COLLECTOR_DIR_PATH - value: /opt/collector - command: [ "collector" ] - resources: -{{- include "getResourceObject" .Values.collector.resources | indent 12 }} - volumeMounts: - - mountPath: /opt/collector - name: collector - readinessProbe: - tcpSocket: - port: 8070 - initialDelaySeconds: 60 - periodSeconds: 30 - livenessProbe: - tcpSocket: - port: 8070 - initialDelaySeconds: 60 - periodSeconds: 30 - volumes: - - name: collector - emptyDir: {} ---- -{{- end }} diff --git a/charts/devnet/values.yaml b/charts/devnet/values.yaml index 1eff7285..cb81d52a 100644 --- a/charts/devnet/values.yaml +++ b/charts/devnet/values.yaml @@ -191,6 +191,3 @@ registry: resources: cpu: "0.2" memory: "200M" - -collector: - enabled: false diff --git a/collector/BUILD.bazel b/collector/BUILD.bazel deleted file mode 100644 index d8a187b2..00000000 --- a/collector/BUILD.bazel +++ /dev/null @@ -1,53 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") - -go_library( - name = "lib", - srcs = [ - "app.go", - "config.go", - "const.go", - "error.go", - "fileserver.go", - "handler.go", - "main.go", - "model.go", - ], - importpath = "", - visibility = ["//visibility:public"], - deps = [ - "@com_github_go_chi_chi//:chi", - "@com_github_go_chi_chi//middleware", - "@com_github_go_chi_render//:render", - "@com_github_urfave_cli//:cli", - "@org_uber_go_zap//:zap", - ], -) - -go_binary( - name = "collector", - embed = [":collector_lib"], - visibility = ["//visibility:public"], -) - -go_library( - name = "collector_lib", - srcs = [ - "app.go", - "config.go", - "const.go", - "error.go", - "fileserver.go", - "handler.go", - "main.go", - "model.go", - ], - importpath = "github.com/cosmology-tech/starship/collector", - visibility = ["//visibility:public"], - deps = [ - "@com_github_go_chi_chi//:chi", - "@com_github_go_chi_chi//middleware", - "@com_github_go_chi_render//:render", - "@com_github_urfave_cli//:cli", - "@org_uber_go_zap//:zap", - ], -) diff --git a/collector/Dockerfile b/collector/Dockerfile deleted file mode 100644 index 8a1ec16f..00000000 --- a/collector/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM golang:1.19-alpine AS build-env - -# Set up dependencies -ENV PACKAGES curl make git libc-dev bash gcc linux-headers - -# Set working directory for the build -WORKDIR /usr/local/share/app - -# Add source files -COPY . . - -# Install minimum necessary dependencies, build Cosmos SDK, remove packages -RUN apk add --no-cache $PACKAGES && go build -mod readonly -o collector ./... - -# Final image -FROM alpine:3.16 - -# Install ca-certificates -RUN apk add --update ca-certificates jq bash curl -WORKDIR /usr/local/share/app - -RUN ls /usr/bin - -# Copy over binaries from the build-env -COPY --from=build-env /usr/local/share/app/collector /usr/bin/collector - -EXPOSE 8081 diff --git a/collector/Makefile b/collector/Makefile deleted file mode 100644 index 06a3dca3..00000000 --- a/collector/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -DOCKER := $(shell which docker) -DOCKER_REPO_NAME := anmol1696 -DOCKER_IMAGE := collector -DOCKER_TAG_NAME := latest - -DOCKER_ARGS += --platform linux/amd64 - -docker-build: - $(DOCKER) buildx build $(DOCKER_ARGS) \ - -t $(DOCKER_REPO_NAME)/$(DOCKER_IMAGE):$(DOCKER_TAG_NAME) . - -docker-build-push: docker-build - $(DOCKER) push $(DOCKER_REPO_NAME)/$(DOCKER_IMAGE):$(DOCKER_TAG_NAME) - -docker-run: - $(DOCKER) run --rm -it --entrypoint /bin/bash $(DOCKER_REPO_NAME)/$(DOCKER_IMAGE):$(DOCKER_TAG_NAME) diff --git a/collector/app.go b/collector/app.go deleted file mode 100644 index ff970008..00000000 --- a/collector/app.go +++ /dev/null @@ -1,134 +0,0 @@ -package main - -import ( - "fmt" - "net/http" - "time" - - "github.com/go-chi/chi" - "github.com/go-chi/chi/middleware" - "github.com/go-chi/render" - "go.uber.org/zap" -) - -type AppServer struct { - config *Config - db *FileDB - logger *zap.Logger - server *http.Server - router http.Handler -} - -func NewAppServer(config *Config) (*AppServer, error) { - log, err := NewLogger(config) - if err != nil { - return nil, err - } - log.Info( - "Starting the service", - zap.String("prog", Prog), - zap.String("version", Version), - zap.Any("config", config), - ) - - app := &AppServer{ - config: config, - logger: log, - db: NewFileDB(log, config.DirPath), - } - - // Setup routes - router, err := app.Router() - if err != nil { - log.Error("Error setting up routes", zap.Error(err)) - return nil, err - } - app.router = router - - return app, err -} - -func (a *AppServer) Router() (*chi.Mux, error) { - router := chi.NewRouter() - router.MethodNotAllowed(MethodNotAllowed) - router.NotFound(NotFound) - - // Set middleware - router.Use(a.panicRecovery) - router.Use(render.SetContentType(render.ContentTypeJSON)) - - // Setup routes - // handler of export states - router.Get("/chains", a.GetChains) - router.Route("/chains/{chain}/validators/{validator}", func(r chi.Router) { - r.Get("/exports", a.GetChainExports) - r.Get("/exports/{id}", a.GetChainExport) - r.Post("/exports/{id}", a.SetChainExport) - r.Get("/snapshots", a.GetChainSnapshots) - r.Get("/snapshots/{id}", a.GetChainSnapshot) - r.Post("/snapshots/{id}", a.SetChainSnapshot) - }) - - return router, nil -} - -func (a *AppServer) loggingMiddleware(next http.Handler) http.Handler { - fn := func(w http.ResponseWriter, r *http.Request) { - ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor) - start := time.Now() - defer func() { - a.logger.Info("client request", - zap.Duration("latency", time.Since(start)), - zap.Int("status", ww.Status()), - zap.Int("bytes", ww.BytesWritten()), - zap.String("client_ip", r.RemoteAddr), - zap.String("method", r.Method), - zap.String("path", r.URL.Path), - zap.String("request-id", middleware.GetReqID(r.Context()))) - }() - - next.ServeHTTP(ww, r) - } - return http.HandlerFunc(fn) -} - -func (a *AppServer) panicRecovery(next http.Handler) http.Handler { - fn := func(w http.ResponseWriter, r *http.Request) { - defer func() { - if rc := recover(); rc != nil { - err, ok := rc.(error) - if !ok { - err = fmt.Errorf("panic: %v", rc) - } - a.logger.Error("panic error", - zap.String("request-id", middleware.GetReqID(r.Context())), - zap.Error(err)) - - render.Render(w, r, ErrInternalServer) - return - } - }() - next.ServeHTTP(w, r) - } - return http.HandlerFunc(fn) -} - -func (a *AppServer) Run() error { - a.logger.Info("App starting", zap.Any("Config", a.config)) - - // Setup server - server := &http.Server{ - Addr: a.config.Addr, - Handler: a.router, - } - a.server = server - - // Start http server as long-running go routine - go func() { - if err := server.ListenAndServe(); err != nil { - a.logger.Error("failed to start the App HTTP server", zap.Error(err)) - } - }() - - return nil -} diff --git a/collector/config.go b/collector/config.go deleted file mode 100644 index a8bdf076..00000000 --- a/collector/config.go +++ /dev/null @@ -1,115 +0,0 @@ -package main - -import ( - "fmt" - "reflect" - - "github.com/urfave/cli" - "go.uber.org/zap" -) - -func NewDefaultConfig() *Config { - return &Config{ - Addr: ":8070", - DirPath: "/opt/collector", - SnapshotExt: "tar.gz", - ExportExt: "json", - } -} - -type Config struct { - // Addr is the interface and port to bind the HTTP service on - Addr string `name:"addr" json:"addr" env:"ADDR" usage:"IP address and port to listen on"` - // DirPath is the directory which is used by the fileserver to store files - DirPath string `name:"dir-path" json:"dir_path" env:"DIR_PATH" usage:"directory where the files are stored"` - // SnapshotExt is the extensions of the snapshot - SnapshotExt string `name:"snapshot-ext" json:"snapshot_ext" env:"SNAPSHOT_EXT" usage:"extenstion of the snapshots"` - // ExportExt is the extensions of the exports - ExportExt string `name:"export-ext" json:"export_ext" env:"EXPORT_EXT" usage:"extenstion of the exports"` - // Verbose switches on debug logging - Verbose bool `name:"verbose" json:"verbose" usage:"switch on debug / verbose logging"` - // OnlyFatalLog set log level as fatal to ignore logs - OnlyFatalLog bool `name:"only-fatal-log" json:"only-fatal-log" usage:"used while running test"` -} - -func GetCommandLineOptions() []cli.Flag { - defaults := NewDefaultConfig() - var flags []cli.Flag - count := reflect.TypeOf(Config{}).NumField() - for i := 0; i < count; i++ { - field := reflect.TypeOf(Config{}).Field(i) - usage, found := field.Tag.Lookup("usage") - if !found { - continue - } - envName := field.Tag.Get("env") - if envName != "" { - envName = envPrefix + envName - } - optName := field.Tag.Get("name") - - switch t := field.Type; t.Kind() { - case reflect.Bool: - dv := reflect.ValueOf(defaults).Elem().FieldByName(field.Name).Bool() - msg := fmt.Sprintf("%s (default: %t)", usage, dv) - flags = append(flags, cli.BoolTFlag{ - Name: optName, - Usage: msg, - EnvVar: envName, - }) - case reflect.String: - defaultValue := reflect.ValueOf(defaults).Elem().FieldByName(field.Name).String() - flags = append(flags, cli.StringFlag{ - Name: optName, - Usage: usage, - EnvVar: envName, - Value: defaultValue, - }) - } - } - - return flags -} - -func ParseCLIOptions(cx *cli.Context, config *Config) (err error) { - // iterate the Config and grab command line options via reflection - count := reflect.TypeOf(config).Elem().NumField() - for i := 0; i < count; i++ { - field := reflect.TypeOf(config).Elem().Field(i) - name := field.Tag.Get("name") - - if cx.IsSet(name) { - switch field.Type.Kind() { - case reflect.Bool: - reflect.ValueOf(config).Elem().FieldByName(field.Name).SetBool(cx.Bool(name)) - case reflect.String: - reflect.ValueOf(config).Elem().FieldByName(field.Name).SetString(cx.String(name)) - } - } - } - return nil -} - -func NewLogger(config *Config) (*zap.Logger, error) { - c := zap.NewProductionConfig() - c.DisableCaller = true - // c.Encoding = "console" - - if config.Verbose { - c.DisableCaller = false - c.Development = true - c.DisableStacktrace = true // Disable stack trace for development - c.Level = zap.NewAtomicLevelAt(zap.DebugLevel) - } - - if config.OnlyFatalLog { - c.Level = zap.NewAtomicLevelAt(zap.FatalLevel) - } - - log, err := c.Build() - if err != nil { - return nil, err - } - zap.ReplaceGlobals(log) // Set zap global logger - return log, err -} diff --git a/collector/const.go b/collector/const.go deleted file mode 100644 index 56e49fdf..00000000 --- a/collector/const.go +++ /dev/null @@ -1,21 +0,0 @@ -package main - -var ( - Version = "v0" - RequestIdCtxKey = &contextKey{"RequestId"} -) - -const ( - Prog = "collector" - Description = "is a service that aggregates the information of starship" - envPrefix = "COLLECTOR_" -) - -// copied and modified from net/http/http.go -// contextKey is a value for use with context.WithValue. It's used as -// a pointer, so it fits in an interface{} without allocation. -type contextKey struct { - name string -} - -func (k *contextKey) String() string { return Prog + " context value " + k.name } diff --git a/collector/error.go b/collector/error.go deleted file mode 100644 index 62e86f4c..00000000 --- a/collector/error.go +++ /dev/null @@ -1,64 +0,0 @@ -package main - -import ( - "net/http" - - "github.com/go-chi/render" - "go.uber.org/zap" -) - -type ErrResponse struct { - Err error `json:"-"` - HTTPStatusCode int `json:"-"` - - MessageText string `json:"message"` -} - -func (e *ErrResponse) Render(w http.ResponseWriter, r *http.Request) error { - render.Status(r, e.HTTPStatusCode) - return nil -} - -// NewErrResponse create http aware errors from custom errors -func NewErrResponse(err error) *ErrResponse { - return &ErrResponse{ - Err: err, - HTTPStatusCode: http.StatusInternalServerError, - MessageText: err.Error(), - } -} - -var ( - ErrValidation = &ErrResponse{HTTPStatusCode: http.StatusBadRequest, MessageText: "Validation error."} - ErrNotFound = &ErrResponse{HTTPStatusCode: http.StatusNotFound, MessageText: "Resource not found."} - ErrNotImplemented = &ErrResponse{HTTPStatusCode: http.StatusNotImplemented, MessageText: "Not Implemented."} - ErrMethodNotAllowed = &ErrResponse{HTTPStatusCode: http.StatusMethodNotAllowed, MessageText: "Method not allowed."} - ErrRequestBind = &ErrResponse{HTTPStatusCode: http.StatusBadRequest, MessageText: "Unable to bind request body."} - ErrInternalServer = &ErrResponse{HTTPStatusCode: http.StatusInternalServerError, MessageText: "Internal server error."} -) - -// NotFound Method to render Json respose, used by middleware -func NotFound(w http.ResponseWriter, r *http.Request) { - _ = render.Render(w, r, ErrNotFound) -} - -// MethodNotAllowed Method to render Json respose, used by middleware -func MethodNotAllowed(w http.ResponseWriter, r *http.Request) { - _ = render.Render(w, r, ErrMethodNotAllowed) -} - -func (a *AppServer) renderError(w http.ResponseWriter, r *http.Request, err error, msg ...string) { - log := a.logger - errResp := NewErrResponse(err) - // Logging error at different levels depending on status - switch code := errResp.HTTPStatusCode; { - case code < http.StatusInternalServerError: - log.Warn(err.Error()) - default: - log.Error( - "Internal server error", - zap.Error(err), - ) - } - _ = render.Render(w, r, errResp) -} diff --git a/collector/fileserver.go b/collector/fileserver.go deleted file mode 100644 index 5751c23a..00000000 --- a/collector/fileserver.go +++ /dev/null @@ -1,243 +0,0 @@ -package main - -import ( - "fmt" - "go.uber.org/zap" - "os" - "path/filepath" -) - -type FileDB struct { - logger *zap.Logger - path string -} - -func NewFileDB(logger *zap.Logger, path string) *FileDB { - dirPath := filepath.Dir(path) - if err := os.MkdirAll(dirPath, os.ModeDir); err != nil { - panic(fmt.Sprintf("Unable to create root directory at: %s", dirPath)) - } - logger.Info("Initialized directory used for storing files", - zap.String("path", dirPath)) - - return &FileDB{ - path: dirPath, - logger: logger, - } -} - -func (f *FileDB) getDirs(paths ...string) ([]string, error) { - path := filepath.Join(f.path, filepath.Join(paths...)) - - files, err := os.ReadDir(path) - if err != nil { - return nil, err - } - - var dirNames []string - for _, file := range files { - fileInfo, err := file.Info() - if err != nil { - return nil, err - } - if fileInfo.IsDir() { - dirNames = append(dirNames, fileInfo.Name()) - } - } - - return dirNames, nil -} - -func (f *FileDB) getFiles(paths ...string) ([]string, error) { - path := filepath.Join(f.path, filepath.Join(paths...)) - - files, err := os.ReadDir(path) - if err != nil { - return nil, err - } - - var fileNames []string - for _, file := range files { - fileInfo, err := file.Info() - if err != nil { - return nil, err - } - if !fileInfo.IsDir() { - fileNames = append(fileNames, fileInfo.Name()) - } - } - - return fileNames, nil -} - -// GetChains will return a list of chains reading from files -func (f *FileDB) GetChains() ([]Chain, error) { - chainDirs, err := f.getDirs("/") - if err != nil { - return nil, err - } - - var chains []Chain - for _, chainDir := range chainDirs { - validators, err := f.GetChainValidators(chainDir) - if err != nil { - return nil, err - } - chains = append(chains, Chain{ - Name: chainDir, - Validators: validators, - }) - } - - return chains, nil -} - -func (f *FileDB) IsChain(name string) bool { - chains, err := f.GetChains() - if err != nil { - return false - } - - for _, chain := range chains { - if chain.Name == name { - return true - } - } - - return false -} - -func (f *FileDB) CreateChain(name string) error { - err := os.Mkdir(filepath.Join(f.path, name), os.ModeDir) - if os.IsExist(err) { - return nil - } - - return err -} - -// GetChainValidators will return a list of validators in the chain -func (f *FileDB) GetChainValidators(chain string) ([]Validator, error) { - valDirs, err := f.getDirs(chain) - if err != nil { - return nil, err - } - - var vals []Validator - for _, valDir := range valDirs { - vals = append(vals, Validator{ - Name: valDir, - Moniker: valDir, - }) - } - - return vals, nil -} - -func (f *FileDB) IsValidator(chain string, name string) bool { - vals, err := f.GetChainValidators(chain) - if err != nil { - return false - } - - for _, val := range vals { - if val.Name == name { - return true - } - } - - return false -} - -func (f *FileDB) CreateValidator(chain string, name string) error { - err := os.Mkdir(filepath.Join(f.path, chain, name), os.ModeDir) - if os.IsExist(err) { - return nil - } - - return err -} - -func (f *FileDB) ListSnapshots(chain string, validator string) ([]string, error) { - if !f.IsChain(chain) { - return nil, fmt.Errorf("chain %s does not exists", chain) - } - if !f.IsValidator(chain, validator) { - return nil, fmt.Errorf("validator %s does not exists for %s", chain, validator) - } - - files, err := f.getFiles(chain, validator, "snapshots") - if err != nil { - return nil, err - } - - return files, nil -} - -// GetSnapshot return snapshot stored for the chain and validator -func (f *FileDB) GetSnapshot(chain string, validator string, snapshot string) ([]byte, error) { - filePath := filepath.Join(f.path, chain, validator, "snapshots", snapshot) - data, err := os.ReadFile(filePath) - if err != nil { - return nil, err - } - - return data, nil -} - -func (f *FileDB) StoreSnapshot(chain string, validator string, snapshot string, data []byte) error { - err := os.MkdirAll(filepath.Join(f.path, chain, validator, "snapshots"), os.ModeDir) - if err != nil { - return err - } - - filePath := filepath.Join(f.path, chain, validator, "snapshots", snapshot) - err = os.WriteFile(filePath, data, 0644) - if err != nil { - return fmt.Errorf("unable to write snapshot to %s, with err: %s", filePath, err) - } - - return nil -} - -func (f *FileDB) ListExports(chain string, validator string) ([]string, error) { - if !f.IsChain(chain) { - return nil, fmt.Errorf("chain %s does not exists", chain) - } - if !f.IsValidator(chain, validator) { - return nil, fmt.Errorf("validator %s does not exists for %s", chain, validator) - } - - files, err := f.getFiles(chain, validator, "exports") - if err != nil { - return nil, err - } - - return files, nil -} - -// GetExport return snapshot stored for the chain and validator -func (f *FileDB) GetExport(chain string, validator string, export string) ([]byte, error) { - filePath := filepath.Join(f.path, chain, validator, "exports", export) - data, err := os.ReadFile(filePath) - if err != nil { - return nil, err - } - - return data, nil -} - -func (f *FileDB) StoreExport(chain string, validator string, export string, data []byte) error { - err := os.MkdirAll(filepath.Join(f.path, chain, validator, "exports"), os.ModeDir) - if err != nil { - return err - } - - filePath := filepath.Join(f.path, chain, validator, "exports", export) - err = os.WriteFile(filePath, data, 0644) - if err != nil { - return fmt.Errorf("unable to write snapshot to %s, with err: %s", filePath, err) - } - - return nil -} diff --git a/collector/go.mod b/collector/go.mod deleted file mode 100644 index 4da28bbb..00000000 --- a/collector/go.mod +++ /dev/null @@ -1,22 +0,0 @@ -module exposer - -go 1.19 - -require ( - github.com/go-chi/chi v1.5.4 - github.com/go-chi/render v1.0.2 - github.com/urfave/cli v1.22.12 - go.uber.org/zap v1.24.0 -) - -require ( - github.com/ajg/form v1.5.1 // indirect - github.com/benbjohnson/clock v1.3.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/stretchr/testify v1.8.2 // indirect - go.uber.org/atomic v1.10.0 // indirect - go.uber.org/goleak v1.2.1 // indirect - go.uber.org/multierr v1.10.0 // indirect -) diff --git a/collector/go.sum b/collector/go.sum deleted file mode 100644 index 2bc32cc6..00000000 --- a/collector/go.sum +++ /dev/null @@ -1,25 +0,0 @@ -github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= -github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs= -github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg= -github.com/go-chi/render v1.0.2 h1:4ER/udB0+fMWB2Jlf15RV3F4A2FDuYi/9f+lFttR/Lg= -github.com/go-chi/render v1.0.2/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/collector/handler.go b/collector/handler.go deleted file mode 100644 index f6c55ccb..00000000 --- a/collector/handler.go +++ /dev/null @@ -1,112 +0,0 @@ -package main - -import ( - "fmt" - "github.com/go-chi/chi" - "github.com/go-chi/render" - "io" - "net/http" - "os" - "strings" - - "go.uber.org/zap" -) - -func (a *AppServer) renderJSONFile(w http.ResponseWriter, r *http.Request, filePath string) { - jsonFile, err := os.Open(filePath) - if err != nil { - a.logger.Error("Error opening file", - zap.String("file", filePath), - zap.Error(err)) - a.renderError(w, r, fmt.Errorf("error opening json file: %s", filePath)) - } - - byteValue, _ := io.ReadAll(jsonFile) - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write(byteValue) -} - -func (a *AppServer) GetChains(w http.ResponseWriter, r *http.Request) { - chains, err := a.db.GetChains() - if err != nil { - a.renderError(w, r, err, "unable to get chains") - return - } - - render.JSON(w, r, NewItemsResponse(chains)) -} - -func (a *AppServer) GetChainExports(w http.ResponseWriter, r *http.Request) { - render.Render(w, r, ErrNotImplemented) -} - -func (a *AppServer) SetChainExport(w http.ResponseWriter, r *http.Request) { - render.Render(w, r, ErrNotImplemented) -} - -func (a *AppServer) GetChainExport(w http.ResponseWriter, r *http.Request) { - render.Render(w, r, ErrNotImplemented) -} - -func (a *AppServer) GetChainSnapshots(w http.ResponseWriter, r *http.Request) { - chainID := chi.URLParam(r, "chain") - valID := chi.URLParam(r, "validator") - - snapshotsIDs, err := a.db.ListSnapshots(chainID, valID) - if err != nil { - a.renderError(w, r, err) - return - } - - render.JSON(w, r, NewItemsResponse(snapshotsIDs)) -} - -func (a *AppServer) SetChainSnapshot(w http.ResponseWriter, r *http.Request) { - chainID := chi.URLParam(r, "chain") - valID := chi.URLParam(r, "validator") - snapshotID := chi.URLParam(r, "id") - - if !strings.HasSuffix(snapshotID, a.config.SnapshotExt) { - a.renderError(w, r, fmt.Errorf("snapshot id must contain suffix :%s", a.config.SnapshotExt)) - return - } - - data, err := io.ReadAll(r.Body) - if err != nil { - a.logger.Warn(ErrRequestBind.MessageText, zap.Error(err)) - render.Render(w, r, ErrRequestBind) - return - } - - err = a.db.StoreSnapshot(chainID, valID, snapshotID, data) - if err != nil { - a.logger.Warn("unable to store snapshot", - zap.String("snapshotID", snapshotID), - zap.Error(err)) - a.renderError(w, r, err) - return - } - - render.Status(r, http.StatusCreated) // Set response status to 201 -} - -func (a *AppServer) GetChainSnapshot(w http.ResponseWriter, r *http.Request) { - chainID := chi.URLParam(r, "chain") - valID := chi.URLParam(r, "validator") - snapshotID := chi.URLParam(r, "id") - - data, err := a.db.GetSnapshot(chainID, valID, snapshotID) - if err != nil { - a.logger.Warn("unable to read snapshot", - zap.String("snapshotID", snapshotID), - zap.Error(err)) - a.renderError(w, r, err) - return - } - - w.Header().Set("Content-Type", "application/gzip") - w.WriteHeader(http.StatusOK) - w.Write(data) -} diff --git a/collector/main.go b/collector/main.go deleted file mode 100644 index 9eda6426..00000000 --- a/collector/main.go +++ /dev/null @@ -1,61 +0,0 @@ -package main - -import ( - "os" - "os/signal" - "runtime" - "syscall" - "time" - - "github.com/urfave/cli" -) - -func init() { - time.LoadLocation("UTC") // ensure all time is in UTC - runtime.GOMAXPROCS(runtime.NumCPU()) // set the core -} - -// NewApp creates a cli app that can be setup using args flags -// gracefull shutdown using termination signals -func NewApp() *cli.App { - conf := NewDefaultConfig() - app := cli.NewApp() - app.Name = Prog - app.Usage = Description - app.Version = Version - app.Flags = GetCommandLineOptions() - app.UsageText = "exposer [options]" - - app.Action = func(ctx *cli.Context) error { - if err := ParseCLIOptions(ctx, conf); err != nil { - return cli.NewExitError(err.Error(), 1) - } - - // Alternative Sentry Setup point where we can actually pass along configuration options - // SetupSentry(conf) - - server, err := NewAppServer(conf) - if err != nil { - return cli.NewExitError(err.Error(), 1) - } - - defer server.logger.Sync() - if err := server.Run(); err != nil { - return cli.NewExitError(err.Error(), 1) - } - - // Setup the termination signals - signalChannel := make(chan os.Signal) - signal.Notify(signalChannel, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) - <-signalChannel - - return nil - } - - return app -} - -func main() { - app := NewApp() - app.Run(os.Args) -} diff --git a/collector/model.go b/collector/model.go deleted file mode 100644 index 0041c8fc..00000000 --- a/collector/model.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - -type ItemsResponse struct { - Items interface{} `json:"items"` - TotalItems int `json:"total_items"` -} - -func NewItemsResponse(items interface{}) ItemsResponse { - if items == nil { - return ItemsResponse{ - Items: []string{}, - TotalItems: 0, - } - } - - listItems, ok := items.([]interface{}) - if !ok { - return ItemsResponse{ - Items: []interface{}{items}, - TotalItems: 1, - } - } - - return ItemsResponse{ - Items: listItems, - TotalItems: len(listItems), - } -} - -type Chain struct { - Name string `json:"name,omitempty"` - Type string `json:"type,omitempty"` - Validators []Validator `json:"validators,omitempty"` -} - -type Validator struct { - Name string `json:"name,omitempty"` - Moniker string `json:"moniker,omitempty"` - Address string `json:"address,omitempty"` -} - -type State struct { - ID string `json:"id,omitempty"` - Height string `json:"height,omitempty"` - DataType string `json:"data_type,omitempty"` -} - -func NewExportState(id string, height string) State { - return State{ - ID: id, - Height: height, - DataType: "json", - } -} - -func NewSnapshotState(id string, height string) State { - return State{ - ID: id, - Height: height, - DataType: "tar", - } -} diff --git a/go.work b/go.work index 1cc68063..42c036ef 100644 --- a/go.work +++ b/go.work @@ -3,11 +3,10 @@ go 1.19 use ( ./clients/go/client ./cmd/starship - ./collector ./examples/upgrade-test ./exposer ./registry - ./tools ./tests/e2e + ./tools ) diff --git a/tools/go.mod b/tools/go.mod index d23a93a2..b685646f 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -85,7 +85,8 @@ require ( github.com/spf13/cobra v1.6.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.0 // indirect - github.com/urfave/cli/v2 v2.2.0 // indirect + github.com/urfave/cli/v2 v2.25.3 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.opentelemetry.io/otel v1.14.0 // indirect go.opentelemetry.io/otel/sdk v1.14.0 // indirect go.opentelemetry.io/otel/trace v1.14.0 // indirect diff --git a/tools/go.sum b/tools/go.sum index 2770f714..e512e141 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -376,13 +376,14 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/urfave/cli/v2 v2.25.3 h1:VJkt6wvEBOoSjPFQvOkv6iWIrsJyCrKGtCtxXWwmGeY= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=