From 96842d15ca758f79608ca44917d038977084e35e Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Tue, 16 Jul 2024 14:30:55 +0200 Subject: [PATCH 01/31] Add explorer custom api --- Dockerfile.api | 10 ++ api/api.go | 84 +++++++++++++++++ api/cache/cache.go | 12 +++ api/handler/epoch.go | 30 ++++++ api/handler/handler.go | 33 +++++++ api/handler/layer.go | 30 ++++++ api/handler/overview.go | 71 ++++++++++++++ api/handler/smesher.go | 73 +++++++++++++++ api/router/router.go | 15 +++ api/storage/accounts.go | 15 +++ api/storage/activations.go | 14 +++ api/storage/epoch.go | 102 ++++++++++++++++++++ api/storage/layer.go | 78 ++++++++++++++++ api/storage/rewards.go | 14 +++ api/storage/smesher.go | 110 ++++++++++++++++++++++ api/storage/storage.go | 29 ++++++ api/storage/transactions.go | 179 ++++++++++++++++++++++++++++++++++++ cmd/api/main.go | 113 +++++++++++++++++++++++ go.mod | 20 ++-- go.sum | 38 ++++---- 20 files changed, 1045 insertions(+), 25 deletions(-) create mode 100644 Dockerfile.api create mode 100644 api/api.go create mode 100644 api/cache/cache.go create mode 100644 api/handler/epoch.go create mode 100644 api/handler/handler.go create mode 100644 api/handler/layer.go create mode 100644 api/handler/overview.go create mode 100644 api/handler/smesher.go create mode 100644 api/router/router.go create mode 100644 api/storage/accounts.go create mode 100644 api/storage/activations.go create mode 100644 api/storage/epoch.go create mode 100644 api/storage/layer.go create mode 100644 api/storage/rewards.go create mode 100644 api/storage/smesher.go create mode 100644 api/storage/storage.go create mode 100644 api/storage/transactions.go create mode 100644 cmd/api/main.go diff --git a/Dockerfile.api b/Dockerfile.api new file mode 100644 index 0000000..5fd8f8a --- /dev/null +++ b/Dockerfile.api @@ -0,0 +1,10 @@ +FROM golang:1.22.3-alpine AS build +WORKDIR /src +COPY . . +RUN apk add --no-cache gcc musl-dev +RUN go build -o explorer-custom-api ./cmd/api/ + +FROM alpine:3.17 +COPY --from=build /src/explorer-custom-api /bin/ +EXPOSE 5000 +ENTRYPOINT ["/bin/explorer-custom-api"] diff --git a/api/api.go b/api/api.go new file mode 100644 index 0000000..df18b59 --- /dev/null +++ b/api/api.go @@ -0,0 +1,84 @@ +package api + +import ( + "context" + "fmt" + "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" + "github.com/spacemeshos/explorer-backend/api/handler" + "github.com/spacemeshos/explorer-backend/api/router" + "github.com/spacemeshos/explorer-backend/api/storage" + "github.com/spacemeshos/go-spacemesh/log" + "github.com/spacemeshos/go-spacemesh/sql" + "os" + "os/signal" + "syscall" + "time" +) + +type Api struct { + Echo *echo.Echo +} + +func Init(db *sql.Database, dbClient storage.DatabaseClient, + allowedOrigins []string, debug bool, layersPerEpoch int64) *Api { + + e := echo.New() + e.Use(middleware.Recover()) + e.Use(func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + cc := &handler.ApiContext{ + Context: c, + Storage: db, + StorageClient: dbClient, + LayersPerEpoch: layersPerEpoch, + } + return next(cc) + } + }) + e.HideBanner = true + e.HidePort = true + e.Use(middleware.CORSWithConfig(middleware.CORSConfig{ + AllowOrigins: allowedOrigins, + })) + + e.Use(middleware.RequestLoggerWithConfig(middleware.RequestLoggerConfig{ + LogStatus: true, + LogURI: true, + LogValuesFunc: func(c echo.Context, v middleware.RequestLoggerValues) error { + log.Info("%s [%d] - %s", time.Now().Format(time.RFC3339), c.Response().Status, c.Request().URL.Path) + return nil + }, + })) + + if debug { + e.Debug = true + e.Use(middleware.Logger()) + } + + router.Init(e) + + return &Api{ + Echo: e, + } +} + +func (a *Api) Run(address string) { + log.Info("server is running. For exit ") + if err := a.Echo.Start(address); err != nil { + log.Err(fmt.Errorf("server stopped: %s", err)) + } + + sysSignal := make(chan os.Signal, 1) + signal.Notify(sysSignal, syscall.SIGINT, syscall.SIGTERM) + + s := <-sysSignal + log.Info("exiting, got signal %v", s) + if err := a.Shutdown(); err != nil { + log.Err(fmt.Errorf("error on shutdown: %v", err)) + } +} + +func (a *Api) Shutdown() error { + return a.Echo.Shutdown(context.TODO()) +} diff --git a/api/cache/cache.go b/api/cache/cache.go new file mode 100644 index 0000000..1705d24 --- /dev/null +++ b/api/cache/cache.go @@ -0,0 +1,12 @@ +package cache + +import ( + "github.com/patrickmn/go-cache" + "time" +) + +var Cache *cache.Cache + +func Init() { + Cache = cache.New(1*time.Hour, 10*time.Minute) +} diff --git a/api/handler/epoch.go b/api/handler/epoch.go new file mode 100644 index 0000000..da97108 --- /dev/null +++ b/api/handler/epoch.go @@ -0,0 +1,30 @@ +package handler + +import ( + "github.com/labstack/echo/v4" + "github.com/spacemeshos/explorer-backend/api/cache" + "github.com/spacemeshos/go-spacemesh/log" + "net/http" + "strconv" +) + +func EpochStats(c echo.Context) error { + cc := c.(*ApiContext) + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + return c.NoContent(http.StatusBadRequest) + } + + if cached, ok := cache.Cache.Get("epochStats" + c.Param("id")); ok { + return c.JSON(http.StatusOK, cached) + } + + epochStats, err := cc.StorageClient.GetEpochStats(cc.Storage, int64(id), cc.LayersPerEpoch) + if err != nil { + log.Warning("failed to get layer stats: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + cache.Cache.Set("epochStats"+c.Param("id"), epochStats, 0) + return c.JSON(http.StatusOK, epochStats) +} diff --git a/api/handler/handler.go b/api/handler/handler.go new file mode 100644 index 0000000..6e787ce --- /dev/null +++ b/api/handler/handler.go @@ -0,0 +1,33 @@ +package handler + +import ( + "github.com/labstack/echo/v4" + "github.com/spacemeshos/explorer-backend/api/storage" + "github.com/spacemeshos/go-spacemesh/sql" + "strconv" +) + +type ApiContext struct { + echo.Context + Storage *sql.Database + StorageClient storage.DatabaseClient + LayersPerEpoch int64 +} + +func GetPagination(c echo.Context) (limit, offset int64) { + limit = 20 + offset = 0 + if page := c.QueryParam("limit"); page != "" { + limit, _ = strconv.ParseInt(page, 10, 32) + if limit <= 0 { + limit = 0 + } + } + if size := c.QueryParam("offset"); size != "" { + offset, _ = strconv.ParseInt(size, 10, 32) + if offset <= 0 { + offset = 0 + } + } + return limit, offset +} diff --git a/api/handler/layer.go b/api/handler/layer.go new file mode 100644 index 0000000..d98f09c --- /dev/null +++ b/api/handler/layer.go @@ -0,0 +1,30 @@ +package handler + +import ( + "github.com/labstack/echo/v4" + "github.com/spacemeshos/explorer-backend/api/cache" + "github.com/spacemeshos/go-spacemesh/log" + "net/http" + "strconv" +) + +func LayerStats(c echo.Context) error { + cc := c.(*ApiContext) + lid, err := strconv.Atoi(c.Param("id")) + if err != nil { + return c.NoContent(http.StatusBadRequest) + } + + if cached, ok := cache.Cache.Get("layerStats" + c.Param("id")); ok { + return c.JSON(http.StatusOK, cached) + } + + layerStats, err := cc.StorageClient.GetLayerStats(cc.Storage, int64(lid)) + if err != nil { + log.Warning("failed to get layer stats: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + cache.Cache.Set("layerStats"+c.Param("id"), layerStats, 0) + return c.JSON(http.StatusOK, layerStats) +} diff --git a/api/handler/overview.go b/api/handler/overview.go new file mode 100644 index 0000000..338ea2a --- /dev/null +++ b/api/handler/overview.go @@ -0,0 +1,71 @@ +package handler + +import ( + "github.com/labstack/echo/v4" + "github.com/spacemeshos/explorer-backend/api/cache" + "github.com/spacemeshos/go-spacemesh/log" + "net/http" +) + +type OverviewResp struct { + AccountsCount uint64 `json:"accounts_count"` + SmeshersCount uint64 `json:"smeshers_count"` + LayersCount uint64 `json:"layers_count"` + RewardsCount uint64 `json:"rewards_count"` + TransactionsCount uint64 `json:"transactions_count"` + NumUnits uint64 `json:"num_units"` +} + +func Overview(c echo.Context) error { + cc := c.(*ApiContext) + + if cached, ok := cache.Cache.Get("overview"); ok { + return c.JSON(http.StatusOK, cached) + } + + overviewResp := OverviewResp{} + accountsCount, err := cc.StorageClient.GetAccountsCount(cc.Storage) + if err != nil { + log.Warning("failed to get accounts count: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + overviewResp.AccountsCount = accountsCount + + smeshersCount, err := cc.StorageClient.GetSmeshersCount(cc.Storage) + if err != nil { + log.Warning("failed to get smeshers count: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + overviewResp.SmeshersCount = smeshersCount + + layersCount, err := cc.StorageClient.GetLayersCount(cc.Storage) + if err != nil { + log.Warning("failed to get layers count: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + overviewResp.LayersCount = layersCount + + rewardsCount, err := cc.StorageClient.GetRewardsCount(cc.Storage) + if err != nil { + log.Warning("failed to get rewards count: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + overviewResp.RewardsCount = rewardsCount + + transactionsCount, err := cc.StorageClient.GetTransactionsCount(cc.Storage) + if err != nil { + log.Warning("failed to get transactions count: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + overviewResp.TransactionsCount = transactionsCount + + numUnits, err := cc.StorageClient.GetTotalNumUnits(cc.Storage) + if err != nil { + log.Warning("failed to get num units count: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + overviewResp.NumUnits = numUnits + + cache.Cache.Set("overview", overviewResp, 0) + return c.JSON(http.StatusOK, overviewResp) +} diff --git a/api/handler/smesher.go b/api/handler/smesher.go new file mode 100644 index 0000000..42e8ff1 --- /dev/null +++ b/api/handler/smesher.go @@ -0,0 +1,73 @@ +package handler + +import ( + "fmt" + "github.com/labstack/echo/v4" + "github.com/spacemeshos/explorer-backend/api/cache" + "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/log" + "net/http" + "strconv" +) + +func Smeshers(c echo.Context) error { + cc := c.(*ApiContext) + limit, offset := GetPagination(c) + + if cached, ok := cache.Cache.Get(fmt.Sprintf("smeshers-%d-%d", limit, offset)); ok { + return c.JSON(http.StatusOK, cached) + } + + smeshers, err := cc.StorageClient.GetSmeshers(cc.Storage, uint64(limit), uint64(offset)) + if err != nil { + log.Warning("failed to get smeshers: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + cache.Cache.Set(fmt.Sprintf("smeshers-%d-%d", limit, offset), smeshers, 0) + return c.JSON(http.StatusOK, smeshers) +} + +func SmeshersByEpoch(c echo.Context) error { + cc := c.(*ApiContext) + + epochId, err := strconv.Atoi(c.Param("epoch")) + if err != nil || epochId < 0 { + return c.NoContent(http.StatusBadRequest) + } + + limit, offset := GetPagination(c) + + if cached, ok := cache.Cache.Get(fmt.Sprintf("smeshers-epoch-%d-%d-%d", epochId, limit, offset)); ok { + return c.JSON(http.StatusOK, cached) + } + + smeshers, err := cc.StorageClient.GetSmeshersByEpoch(cc.Storage, uint64(limit), uint64(offset), uint64(epochId)) + if err != nil { + log.Warning("failed to get smeshers: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + cache.Cache.Set(fmt.Sprintf("smeshers-epoch-%d-%d-%d", epochId, limit, offset), smeshers, 0) + return c.JSON(http.StatusOK, smeshers) +} + +func Smesher(c echo.Context) error { + cc := c.(*ApiContext) + + smesherId := c.Param("smesherId") + hash := types.HexToHash32(smesherId) + + if cached, ok := cache.Cache.Get("smesher-" + smesherId); ok { + return c.JSON(http.StatusOK, cached) + } + + smesher, err := cc.StorageClient.GetSmesher(cc.Storage, hash.Bytes()) + if err != nil { + log.Warning("failed to get smesher: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + cache.Cache.Set("smesher-"+smesherId, smesher, 0) + + return c.JSON(http.StatusOK, smesher) +} diff --git a/api/router/router.go b/api/router/router.go new file mode 100644 index 0000000..4c63766 --- /dev/null +++ b/api/router/router.go @@ -0,0 +1,15 @@ +package router + +import ( + "github.com/labstack/echo/v4" + "github.com/spacemeshos/explorer-backend/api/handler" +) + +func Init(e *echo.Echo) { + e.GET("/stats/layer/:id", handler.LayerStats) + e.GET("/stats/epoch/:id", handler.EpochStats) + e.GET("/smeshers/:epoch", handler.SmeshersByEpoch) + e.GET("/smeshers", handler.Smeshers) + e.GET("/smesher/:smesherId", handler.Smesher) + e.GET("/overview", handler.Overview) +} diff --git a/api/storage/accounts.go b/api/storage/accounts.go new file mode 100644 index 0000000..b606a2c --- /dev/null +++ b/api/storage/accounts.go @@ -0,0 +1,15 @@ +package storage + +import "github.com/spacemeshos/go-spacemesh/sql" + +func (c *Client) GetAccountsCount(db *sql.Database) (uint64, error) { + var total uint64 + _, err := db.Exec(`SELECT COUNT(DISTINCT address) FROM accounts`, + func(stmt *sql.Statement) { + }, + func(stmt *sql.Statement) bool { + total = uint64(stmt.ColumnInt64(0)) + return true + }) + return total, err +} diff --git a/api/storage/activations.go b/api/storage/activations.go new file mode 100644 index 0000000..fdc7170 --- /dev/null +++ b/api/storage/activations.go @@ -0,0 +1,14 @@ +package storage + +import "github.com/spacemeshos/go-spacemesh/sql" + +func (c *Client) GetTotalNumUnits(db *sql.Database) (count uint64, err error) { + _, err = db.Exec(`SELECT SUM(effective_num_units) FROM atxs;`, + func(stmt *sql.Statement) { + }, + func(stmt *sql.Statement) bool { + count = uint64(stmt.ColumnInt64(0)) + return true + }) + return +} diff --git a/api/storage/epoch.go b/api/storage/epoch.go new file mode 100644 index 0000000..4029c3c --- /dev/null +++ b/api/storage/epoch.go @@ -0,0 +1,102 @@ +package storage + +import ( + "github.com/spacemeshos/go-spacemesh/log" + "github.com/spacemeshos/go-spacemesh/sql" + "github.com/spacemeshos/go-spacemesh/sql/atxs" + "github.com/spacemeshos/go-spacemesh/sql/builder" +) + +type EpochStats struct { + TransactionsCount uint64 `json:"transactions_count"` + ActivationsCount uint64 `json:"activations_count"` + RewardsCount uint64 `json:"rewards_count"` + RewardsSum uint64 `json:"rewards_sum"` + NumUnits uint64 `json:"num_units"` + SmeshersCount uint64 `json:"smeshers_count"` +} + +func (c *Client) GetEpochStats(db *sql.Database, epoch int64, layersPerEpoch int64) (*EpochStats, error) { + stats := &EpochStats{ + TransactionsCount: 0, + ActivationsCount: 0, + RewardsCount: 0, + RewardsSum: 0, + } + + start := epoch * layersPerEpoch + end := start + layersPerEpoch - 1 + + _, err := db.Exec(`SELECT COUNT(*) +FROM ( + SELECT distinct id + FROM transactions + LEFT JOIN transactions_results_addresses + ON transactions.id = transactions_results_addresses.tid + WHERE layer >= ?1 and layer <= ?2 +);`, + func(stmt *sql.Statement) { + stmt.BindInt64(1, start) + stmt.BindInt64(2, end) + }, + func(stmt *sql.Statement) bool { + stats.TransactionsCount = uint64(stmt.ColumnInt64(0)) + return true + }) + if err != nil { + return nil, err + } + + ops := builder.Operations{ + Filter: []builder.Op{ + { + Field: builder.Epoch, + Token: builder.Eq, + Value: epoch - 1, + }, + }, + } + count, err := atxs.CountAtxsByOps(db, ops) + if err != nil { + log.Err(err) + return nil, err + } + stats.ActivationsCount = uint64(count) + + _, err = db.Exec(`SELECT COUNT(*), SUM(total_reward) FROM rewards WHERE layer >= ?1 and layer <= ?2`, + func(stmt *sql.Statement) { + stmt.BindInt64(1, start) + stmt.BindInt64(2, end) + }, + func(stmt *sql.Statement) bool { + stats.RewardsCount = uint64(stmt.ColumnInt64(0)) + stats.RewardsSum = uint64(stmt.ColumnInt64(1)) + return true + }) + if err != nil { + return nil, err + } + + _, err = db.Exec(`SELECT SUM(effective_num_units) FROM (SELECT effective_num_units FROM atxs WHERE epoch = ?1)`, + func(stmt *sql.Statement) { + stmt.BindInt64(1, epoch-1) + }, + func(stmt *sql.Statement) bool { + stats.NumUnits = uint64(stmt.ColumnInt64(0)) + return true + }) + if err != nil { + return nil, err + } + + _, err = db.Exec(`SELECT COUNT(*) FROM (SELECT DISTINCT pubkey FROM atxs WHERE epoch = ?1)`, + func(stmt *sql.Statement) { + stmt.BindInt64(1, epoch-1) + }, + func(stmt *sql.Statement) bool { + stats.SmeshersCount = uint64(stmt.ColumnInt64(0)) + return true + }) + + return stats, err +} diff --git a/api/storage/layer.go b/api/storage/layer.go new file mode 100644 index 0000000..2f6ba2f --- /dev/null +++ b/api/storage/layer.go @@ -0,0 +1,78 @@ +package storage + +import ( + "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/sql" + "github.com/spacemeshos/go-spacemesh/sql/builder" + "github.com/spacemeshos/go-spacemesh/sql/transactions" +) + +type LayerStats struct { + TransactionsCount uint64 `json:"transactions_count"` + TransactionsSum uint64 `json:"transactions_sum"` + RewardsCount uint64 `json:"rewards_count"` + RewardsSum uint64 `json:"rewards_sum"` +} + +func (c *Client) GetLayerStats(db *sql.Database, lid int64) (*LayerStats, error) { + stats := &LayerStats{ + TransactionsCount: 0, + TransactionsSum: 0, + RewardsCount: 0, + RewardsSum: 0, + } + ops := builder.Operations{ + Filter: []builder.Op{ + { + Field: builder.Layer, + Token: builder.Eq, + Value: lid, + }, + }, + } + err := transactions.IterateTransactionsOps(db, ops, func(tx *types.MeshTransaction, + result *types.TransactionResult, + ) bool { + contents, _, err := toTxContents(tx.Raw) + if err != nil { + return false + } + + if contents.GetSend() != nil { + stats.TransactionsSum += contents.GetSend().GetAmount() + } + + stats.TransactionsCount++ + return true + }) + if err != nil { + return nil, err + } + + _, err = db.Exec(`SELECT COUNT(*), SUM(total_reward) FROM rewards WHERE layer=?1`, + func(stmt *sql.Statement) { + stmt.BindInt64(1, lid) + }, + func(stmt *sql.Statement) bool { + stats.RewardsCount = uint64(stmt.ColumnInt64(0)) + stats.RewardsSum = uint64(stmt.ColumnInt64(1)) + return true + }) + if err != nil { + return nil, err + } + + return stats, err +} + +func (c *Client) GetLayersCount(db *sql.Database) (count uint64, err error) { + _, err = db.Exec(`SELECT COUNT(*) FROM layers`, + func(stmt *sql.Statement) { + }, + func(stmt *sql.Statement) bool { + count = uint64(stmt.ColumnInt64(0)) + return true + }) + return + +} diff --git a/api/storage/rewards.go b/api/storage/rewards.go new file mode 100644 index 0000000..e5bb2f0 --- /dev/null +++ b/api/storage/rewards.go @@ -0,0 +1,14 @@ +package storage + +import "github.com/spacemeshos/go-spacemesh/sql" + +func (c *Client) GetRewardsCount(db *sql.Database) (count uint64, err error) { + _, err = db.Exec(`SELECT COUNT(*) FROM rewards`, + func(stmt *sql.Statement) { + }, + func(stmt *sql.Statement) bool { + count = uint64(stmt.ColumnInt64(0)) + return true + }) + return +} diff --git a/api/storage/smesher.go b/api/storage/smesher.go new file mode 100644 index 0000000..e179c93 --- /dev/null +++ b/api/storage/smesher.go @@ -0,0 +1,110 @@ +package storage + +import ( + "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/sql" +) + +type SmesherList struct { + Smeshers []Smesher `json:"smeshers"` +} + +type Smesher struct { + Pubkey types.NodeID `json:"pubkey"` + Coinbase string `json:"coinbase,omitempty"` + NumUnits uint64 `json:"num_units,omitempty"` + Atxs uint64 `json:"atxs"` + RewardsCount uint64 `json:"rewards_count,omitempty"` + RewardsSum uint64 `json:"rewards_sum,omitempty"` +} + +func (c *Client) GetSmeshers(db *sql.Database, limit, offset uint64) (*SmesherList, error) { + smesherList := &SmesherList{ + Smeshers: []Smesher{}, + } + + _, err := db.Exec(`SELECT pubkey, COUNT(*) as atxs FROM atxs GROUP BY pubkey ORDER BY pubkey ASC, epoch DESC LIMIT ?1 OFFSET ?2;`, + func(stmt *sql.Statement) { + stmt.BindInt64(1, int64(limit)) + stmt.BindInt64(2, int64(offset)) + }, + func(stmt *sql.Statement) bool { + var smesher Smesher + stmt.ColumnBytes(0, smesher.Pubkey[:]) + smesher.Atxs = uint64(stmt.ColumnInt64(1)) + smesherList.Smeshers = append(smesherList.Smeshers, smesher) + return true + }) + if err != nil { + return nil, err + } + + return smesherList, err +} + +func (c *Client) GetSmeshersByEpoch(db *sql.Database, limit, offset, epoch uint64) (*SmesherList, error) { + smesherList := &SmesherList{ + Smeshers: []Smesher{}, + } + + _, err := db.Exec(`SELECT DISTINCT pubkey, COUNT(*) as atxs FROM atxs WHERE epoch = ?1 GROUP BY pubkey ORDER BY pubkey ASC, epoch DESC LIMIT ?2 OFFSET ?3;`, + func(stmt *sql.Statement) { + stmt.BindInt64(1, int64(epoch-1)) + stmt.BindInt64(2, int64(limit)) + stmt.BindInt64(3, int64(offset)) + }, + func(stmt *sql.Statement) bool { + var smesher Smesher + stmt.ColumnBytes(0, smesher.Pubkey[:]) + smesher.Atxs = uint64(stmt.ColumnInt64(1)) + smesherList.Smeshers = append(smesherList.Smeshers, smesher) + return true + }) + if err != nil { + return nil, err + } + + return smesherList, err +} + +func (c *Client) GetSmeshersCount(db *sql.Database) (count uint64, err error) { + _, err = db.Exec(`SELECT COUNT(*) FROM (SELECT DISTINCT pubkey FROM atxs)`, + func(stmt *sql.Statement) { + }, + func(stmt *sql.Statement) bool { + count = uint64(stmt.ColumnInt64(0)) + return true + }) + return +} + +func (c *Client) GetSmesher(db *sql.Database, pubkey []byte) (smesher *Smesher, err error) { + smesher = &Smesher{} + _, err = db.Exec(`SELECT pubkey, coinbase, effective_num_units, COUNT(*) as atxs FROM atxs WHERE pubkey = ?1 GROUP BY pubkey ORDER BY epoch DESC LIMIT 1;`, + func(stmt *sql.Statement) { + stmt.BindBytes(1, pubkey) + }, + func(stmt *sql.Statement) bool { + stmt.ColumnBytes(0, smesher.Pubkey[:]) + var coinbase types.Address + stmt.ColumnBytes(1, coinbase[:]) + smesher.Coinbase = coinbase.String() + smesher.NumUnits = uint64(stmt.ColumnInt64(2)) + smesher.Atxs = uint64(stmt.ColumnInt64(3)) + return true + }) + if err != nil { + return + } + + _, err = db.Exec(`SELECT COUNT(*), SUM(total_reward) FROM rewards WHERE pubkey=?1`, + func(stmt *sql.Statement) { + stmt.BindBytes(1, pubkey) + }, + func(stmt *sql.Statement) bool { + smesher.RewardsCount = uint64(stmt.ColumnInt64(0)) + smesher.RewardsSum = uint64(stmt.ColumnInt64(1)) + return true + }) + return +} diff --git a/api/storage/storage.go b/api/storage/storage.go new file mode 100644 index 0000000..621ef13 --- /dev/null +++ b/api/storage/storage.go @@ -0,0 +1,29 @@ +package storage + +import ( + "fmt" + "github.com/spacemeshos/go-spacemesh/sql" +) + +type DatabaseClient interface { + GetLayerStats(db *sql.Database, lid int64) (*LayerStats, error) + GetEpochStats(db *sql.Database, epoch int64, layersPerEpoch int64) (*EpochStats, error) + GetSmeshers(db *sql.Database, limit, offset uint64) (*SmesherList, error) + GetSmeshersByEpoch(db *sql.Database, limit, offset, epoch uint64) (*SmesherList, error) + GetSmesher(db *sql.Database, pubkey []byte) (*Smesher, error) + + GetAccountsCount(db *sql.Database) (uint64, error) + GetSmeshersCount(db *sql.Database) (uint64, error) + GetLayersCount(db *sql.Database) (uint64, error) + GetRewardsCount(db *sql.Database) (uint64, error) + GetTransactionsCount(db *sql.Database) (uint64, error) + GetTotalNumUnits(db *sql.Database) (uint64, error) +} + +type Client struct{} + +func Setup(path string) (db *sql.Database, err error) { + db, err = sql.Open(fmt.Sprintf("file:%s?mode=ro", path), + sql.WithConnections(16), sql.WithMigrations(nil)) + return +} diff --git a/api/storage/transactions.go b/api/storage/transactions.go new file mode 100644 index 0000000..3f626f5 --- /dev/null +++ b/api/storage/transactions.go @@ -0,0 +1,179 @@ +package storage + +import ( + "bytes" + "fmt" + spacemeshv2alpha1 "github.com/spacemeshos/api/release/go/spacemesh/v2alpha1" + "github.com/spacemeshos/go-scale" + "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/genvm/core" + "github.com/spacemeshos/go-spacemesh/genvm/registry" + "github.com/spacemeshos/go-spacemesh/genvm/templates/multisig" + "github.com/spacemeshos/go-spacemesh/genvm/templates/vault" + "github.com/spacemeshos/go-spacemesh/genvm/templates/vesting" + "github.com/spacemeshos/go-spacemesh/genvm/templates/wallet" + "github.com/spacemeshos/go-spacemesh/sql" +) + +func (c *Client) GetTransactionsCount(db *sql.Database) (count uint64, err error) { + _, err = db.Exec(`SELECT COUNT(*) +FROM ( + SELECT distinct id + FROM transactions + LEFT JOIN transactions_results_addresses + ON transactions.id = transactions_results_addresses.tid +);`, + func(stmt *sql.Statement) { + }, + func(stmt *sql.Statement) bool { + count = uint64(stmt.ColumnInt64(0)) + return true + }) + return +} + +func decodeTxArgs(decoder *scale.Decoder) (uint8, *core.Address, scale.Encodable, error) { + reg := registry.New() + wallet.Register(reg) + multisig.Register(reg) + vesting.Register(reg) + vault.Register(reg) + + _, _, err := scale.DecodeCompact8(decoder) + if err != nil { + return 0, nil, nil, fmt.Errorf("%w: failed to decode version %w", core.ErrMalformed, err) + } + + var principal core.Address + if _, err := principal.DecodeScale(decoder); err != nil { + return 0, nil, nil, fmt.Errorf("%w failed to decode principal: %w", core.ErrMalformed, err) + } + + method, _, err := scale.DecodeCompact8(decoder) + if err != nil { + return 0, nil, nil, fmt.Errorf("%w: failed to decode method selector %w", core.ErrMalformed, err) + } + + var templateAddress *core.Address + var handler core.Handler + switch method { + case core.MethodSpawn: + templateAddress = &core.Address{} + if _, err := templateAddress.DecodeScale(decoder); err != nil { + return 0, nil, nil, fmt.Errorf("%w failed to decode template address %w", core.ErrMalformed, err) + } + case vesting.MethodDrainVault: + templateAddress = &vesting.TemplateAddress + default: + templateAddress = &wallet.TemplateAddress + } + + handler = reg.Get(*templateAddress) + if handler == nil { + return 0, nil, nil, fmt.Errorf("%w: unknown template %s", core.ErrMalformed, *templateAddress) + } + + var p core.Payload + if _, err = p.DecodeScale(decoder); err != nil { + return 0, nil, nil, fmt.Errorf("%w: %w", core.ErrMalformed, err) + } + + args := handler.Args(method) + if args == nil { + return 0, nil, nil, fmt.Errorf("%w: unknown method %s %d", core.ErrMalformed, *templateAddress, method) + } + if _, err := args.DecodeScale(decoder); err != nil { + return 0, nil, nil, fmt.Errorf("%w failed to decode method arguments %w", core.ErrMalformed, err) + } + + return method, templateAddress, args, nil +} + +func toTxContents(rawTx []byte) (*spacemeshv2alpha1.TransactionContents, + spacemeshv2alpha1.Transaction_TransactionType, error, +) { + res := &spacemeshv2alpha1.TransactionContents{} + txType := spacemeshv2alpha1.Transaction_TRANSACTION_TYPE_UNSPECIFIED + + r := bytes.NewReader(rawTx) + method, template, txArgs, err := decodeTxArgs(scale.NewDecoder(r)) + if err != nil { + return res, txType, err + } + + switch method { + case core.MethodSpawn: + switch *template { + case wallet.TemplateAddress: + args := txArgs.(*wallet.SpawnArguments) + res.Contents = &spacemeshv2alpha1.TransactionContents_SingleSigSpawn{ + SingleSigSpawn: &spacemeshv2alpha1.ContentsSingleSigSpawn{ + Pubkey: args.PublicKey.String(), + }, + } + txType = spacemeshv2alpha1.Transaction_TRANSACTION_TYPE_SINGLE_SIG_SPAWN + case multisig.TemplateAddress: + args := txArgs.(*multisig.SpawnArguments) + contents := &spacemeshv2alpha1.TransactionContents_MultiSigSpawn{ + MultiSigSpawn: &spacemeshv2alpha1.ContentsMultiSigSpawn{ + Required: uint32(args.Required), + }, + } + contents.MultiSigSpawn.Pubkey = make([]string, len(args.PublicKeys)) + for i := range args.PublicKeys { + contents.MultiSigSpawn.Pubkey[i] = args.PublicKeys[i].String() + } + res.Contents = contents + txType = spacemeshv2alpha1.Transaction_TRANSACTION_TYPE_MULTI_SIG_SPAWN + case vesting.TemplateAddress: + args := txArgs.(*multisig.SpawnArguments) + contents := &spacemeshv2alpha1.TransactionContents_VestingSpawn{ + VestingSpawn: &spacemeshv2alpha1.ContentsMultiSigSpawn{ + Required: uint32(args.Required), + }, + } + contents.VestingSpawn.Pubkey = make([]string, len(args.PublicKeys)) + for i := range args.PublicKeys { + contents.VestingSpawn.Pubkey[i] = args.PublicKeys[i].String() + } + res.Contents = contents + txType = spacemeshv2alpha1.Transaction_TRANSACTION_TYPE_VESTING_SPAWN + case vault.TemplateAddress: + args := txArgs.(*vault.SpawnArguments) + res.Contents = &spacemeshv2alpha1.TransactionContents_VaultSpawn{ + VaultSpawn: &spacemeshv2alpha1.ContentsVaultSpawn{ + Owner: args.Owner.String(), + TotalAmount: args.TotalAmount, + InitialUnlockAmount: args.InitialUnlockAmount, + VestingStart: args.VestingStart.Uint32(), + VestingEnd: args.VestingEnd.Uint32(), + }, + } + txType = spacemeshv2alpha1.Transaction_TRANSACTION_TYPE_VAULT_SPAWN + } + case core.MethodSpend: + args := txArgs.(*wallet.SpendArguments) + res.Contents = &spacemeshv2alpha1.TransactionContents_Send{ + Send: &spacemeshv2alpha1.ContentsSend{ + Destination: args.Destination.String(), + Amount: args.Amount, + }, + } + txType = spacemeshv2alpha1.Transaction_TRANSACTION_TYPE_SINGLE_SIG_SEND + if r.Len() > types.EdSignatureSize { + txType = spacemeshv2alpha1.Transaction_TRANSACTION_TYPE_MULTI_SIG_SEND + } + case vesting.MethodDrainVault: + args := txArgs.(*vesting.DrainVaultArguments) + res.Contents = &spacemeshv2alpha1.TransactionContents_DrainVault{ + DrainVault: &spacemeshv2alpha1.ContentsDrainVault{ + Vault: args.Vault.String(), + Destination: args.Destination.String(), + Amount: args.Amount, + }, + } + txType = spacemeshv2alpha1.Transaction_TRANSACTION_TYPE_DRAIN_VAULT + } + + return res, txType, nil +} diff --git a/cmd/api/main.go b/cmd/api/main.go new file mode 100644 index 0000000..e278077 --- /dev/null +++ b/cmd/api/main.go @@ -0,0 +1,113 @@ +package main + +import ( + "fmt" + "github.com/spacemeshos/address" + "github.com/spacemeshos/explorer-backend/api" + "github.com/spacemeshos/explorer-backend/api/cache" + "github.com/spacemeshos/explorer-backend/api/storage" + "github.com/spacemeshos/go-spacemesh/log" + "github.com/urfave/cli/v2" + "os" +) + +var ( + version string + commit string + branch string +) + +var ( + listenStringFlag string + testnetBoolFlag bool + allowedOrigins = cli.NewStringSlice("*") + debug bool + sqlitePathStringFlag string + layersPerEpoch int64 +) + +var flags = []cli.Flag{ + &cli.StringFlag{ + Name: "listen", + Usage: "Explorer API listen string in format :", + Required: false, + Destination: &listenStringFlag, + Value: ":5000", + EnvVars: []string{"SPACEMESH_API_LISTEN"}, + }, + &cli.BoolFlag{ + Name: "testnet", + Usage: `Use this flag to enable testnet preset ("stest" instead of "sm" for wallet addresses)`, + Required: false, + Destination: &testnetBoolFlag, + EnvVars: []string{"SPACEMESH_TESTNET"}, + }, + &cli.StringSliceFlag{ + Name: "allowed-origins", + Usage: `Use this flag to set allowed origins for CORS (default: "*")`, + Destination: allowedOrigins, + EnvVars: []string{"ALLOWED_ORIGINS"}, + }, + &cli.BoolFlag{ + Name: "debug", + Usage: "Use this flag to enable echo debug option along with logger middleware", + Required: false, + Destination: &debug, + EnvVars: []string{"DEBUG"}, + }, + &cli.StringFlag{ + Name: "sqlite", + Usage: "Path to node sqlite file", + Required: false, + Destination: &sqlitePathStringFlag, + Value: "explorer.sql", + EnvVars: []string{"SPACEMESH_SQLITE"}, + }, + &cli.Int64Flag{ + Name: "layers-per-epoch", + Usage: "Number of layers per epoch", + Required: false, + Destination: &layersPerEpoch, + Value: 4032, + EnvVars: []string{"SPACEMESH_LAYERS_PER_EPOCH"}, + }, +} + +func main() { + app := cli.NewApp() + app.Name = "Spacemesh Explorer REST API Server" + app.Version = fmt.Sprintf("%s, commit '%s', branch '%s'", version, commit, branch) + app.Flags = flags + app.Writer = os.Stderr + + app.Action = func(ctx *cli.Context) error { + if testnetBoolFlag { + address.SetAddressConfig("stest") + log.Info(`network HRP set to "stest"`) + } + + cache.Init() + + db, err := storage.Setup(sqlitePathStringFlag) + if err != nil { + log.Info("SQLite storage open error %v", err) + return err + } + dbClient := &storage.Client{} + + server := api.Init(db, dbClient, allowedOrigins.Value(), debug, layersPerEpoch) + + log.Info(fmt.Sprintf("starting server on %s", listenStringFlag)) + server.Run(listenStringFlag) + + log.Info("server is shutdown") + return nil + } + + if err := app.Run(os.Args); err != nil { + log.Info("%v", err) + os.Exit(1) + } + + os.Exit(0) +} diff --git a/go.mod b/go.mod index 7073cc7..c52a877 100644 --- a/go.mod +++ b/go.mod @@ -10,18 +10,20 @@ require ( github.com/golang/protobuf v1.5.4 github.com/gorilla/websocket v1.5.1 github.com/labstack/echo/v4 v4.9.1 + github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a + github.com/patrickmn/go-cache v2.1.0+incompatible github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/prometheus/client_golang v1.19.1 github.com/spacemeshos/address v0.0.0-20220829090052-44ab32617871 - github.com/spacemeshos/api/release/go v1.49.0 + github.com/spacemeshos/api/release/go v1.50.0 github.com/spacemeshos/go-scale v1.2.0 - github.com/spacemeshos/go-spacemesh v1.6.0 + github.com/spacemeshos/go-spacemesh v1.6.2 github.com/stretchr/testify v1.9.0 github.com/urfave/cli/v2 v2.27.2 go.mongodb.org/mongo-driver v1.10.1 golang.org/x/net v0.26.0 golang.org/x/sync v0.7.0 - google.golang.org/grpc v1.64.1 + google.golang.org/grpc v1.65.0 ) require ( @@ -32,7 +34,7 @@ require ( github.com/andybalholm/brotli v1.0.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/c0mm4nd/go-ripemd v0.0.0-20200326052756-bd1759ad7d10 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cosmos/btcutil v1.0.5 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -51,17 +53,17 @@ require ( github.com/mattn/go-runewidth v0.0.15 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect - github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.54.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spacemeshos/merkle-tree v0.2.3 // indirect github.com/spacemeshos/poet v0.10.3 // indirect - github.com/spacemeshos/post v0.12.6 // indirect + github.com/spacemeshos/post v0.12.7 // indirect github.com/spacemeshos/sha256-simd v0.1.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.51.0 // indirect @@ -81,7 +83,7 @@ require ( golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 85dc7e2..275161b 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,8 @@ github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c h1:FUUopH4brHNO2kJoN github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= github.com/c0mm4nd/go-ripemd v0.0.0-20200326052756-bd1759ad7d10 h1:wJ2csnFApV9G1jgh5KmYdxVOQMi+fihIggVTjcbM7ts= github.com/c0mm4nd/go-ripemd v0.0.0-20200326052756-bd1759ad7d10/go.mod h1:mYPR+a1fzjnHY3VFH5KL3PkEjMlVfGXP7c8rbWlkLJg= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cosmos/btcutil v1.0.5 h1:t+ZFcX77LpKtDBhjucvnOH8C2l2ioGsBNEQ3jef8xFk= github.com/cosmos/btcutil v1.0.5/go.mod h1:IyB7iuqZMJlthe2tkIFL33xPyzbFYP0XVdS8P5lUPis= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= @@ -100,8 +100,12 @@ github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a h1:dlRvE5fWabOchtH7znfiFCcOvmIYgOeAS5ifBXBlh9Q= github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a/go.mod h1:hVoHR2EVESiICEMbg137etN/Lx+lSrHPTD39Z/uE+2s= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= @@ -114,10 +118,10 @@ github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQ github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8= -github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -129,18 +133,20 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/spacemeshos/address v0.0.0-20220829090052-44ab32617871 h1:7cFCSnK/XIbyFPNprR0BZWOpcF/6Ja7JSfJxfEczXeE= github.com/spacemeshos/address v0.0.0-20220829090052-44ab32617871/go.mod h1:99pkrI1qHZje8hwQphmzRi6FQrJwgM1HvWVRC5GE1Fo= -github.com/spacemeshos/api/release/go v1.49.0 h1:1hRPztB5aTX1ebnjS8Fh86Q2JQPDzM4+MbU40rrbK5A= -github.com/spacemeshos/api/release/go v1.49.0/go.mod h1:8pxGN6/di8iBpQReiOgY+Cppi7bhJ+qJ3QiRQtJfoag= +github.com/spacemeshos/api/release/go v1.50.0 h1:M7Usg/LxymscwqYO7/Doyb+sU4lS1e+JIsSgqTDGk/0= +github.com/spacemeshos/api/release/go v1.50.0/go.mod h1:PvgDpjfwkZLVVNExYG7wDNzgMqT3p+ppfTU2UESSF9U= +github.com/spacemeshos/economics v0.1.3 h1:ACkq3mTebIky4Zwbs9SeSSRZrUCjU/Zk0wq9Z0BTh2A= +github.com/spacemeshos/economics v0.1.3/go.mod h1:FH7u0FzTIm6Kpk+X5HOZDvpkgNYBKclmH86rVwYaDAo= github.com/spacemeshos/go-scale v1.2.0 h1:ZlA2L1ILym2gmyJUwUdLTiyP1ZIG0U4xE9nFVFLi83M= github.com/spacemeshos/go-scale v1.2.0/go.mod h1:HV6e3/X5h9u2aFpYKJxt7PY/fBuLBegEKWgeZJ+/5jE= -github.com/spacemeshos/go-spacemesh v1.6.0 h1:ZIGtrJonUQZ81Z6JB4IltgHwf0SVOy6LlLncI2UBT2Q= -github.com/spacemeshos/go-spacemesh v1.6.0/go.mod h1:jyKKOOez+6n5X4vG6R1q3N9gfbe1BqGmyiOQPnZaq+s= +github.com/spacemeshos/go-spacemesh v1.6.2 h1:79Dyf80DG03xoL4NZnkqiGhRkW4+2NSRduFGofbw8/I= +github.com/spacemeshos/go-spacemesh v1.6.2/go.mod h1:IwY+BaIPewz3B2ujqB2I2fSSq5chEy4QWEb5he22wDQ= github.com/spacemeshos/merkle-tree v0.2.3 h1:zGEgOR9nxAzJr0EWjD39QFngwFEOxfxMloEJZtAysas= github.com/spacemeshos/merkle-tree v0.2.3/go.mod h1:VomOcQ5pCBXz7goiWMP5hReyqOfDXGSKbrH2GB9Htww= github.com/spacemeshos/poet v0.10.3 h1:ZDPqihukDphdM+Jr/3xgn7vXadheaRMa6wF70Zsv4fg= github.com/spacemeshos/poet v0.10.3/go.mod h1:TPZ/aX+YIgIqs/bvYTcJIwUWEUzvZw6jueFPxdhCGpY= -github.com/spacemeshos/post v0.12.6 h1:BtKK4n8qa7d0APtQZ2KBx9fQpx1B5LSt2OD7XIgE8g4= -github.com/spacemeshos/post v0.12.6/go.mod h1:NEstvZ4BKHuiGTcb+H+cQsZiNSh0G7GOLjZv6jjnHxM= +github.com/spacemeshos/post v0.12.7 h1:0pLD19TWM6EktFhnd+7QW8ifvdVH952EKliGUN49gFk= +github.com/spacemeshos/post v0.12.7/go.mod h1:WzfVgaa1wxgrsytC4EVKkG8rwoUxjyoyQL0ZSxs56Y0= github.com/spacemeshos/sha256-simd v0.1.0 h1:G7Mfu5RYdQiuE+wu4ZyJ7I0TI74uqLhFnKblEnSpjYI= github.com/spacemeshos/sha256-simd v0.1.0/go.mod h1:O8CClVIilId7RtuCMV2+YzMj6qjVn75JsxOxaE8vcfM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -220,10 +226,10 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= -google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= -google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 h1:Di6ANFilr+S60a4S61ZM00vLdw0IrQOSMS2/6mrnOU0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 29f8f2ffe6b9efd8535c44d2a13485765096d24e Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Tue, 16 Jul 2024 14:32:08 +0200 Subject: [PATCH 02/31] Push dev image to dockerhub --- .github/workflows/ci.yaml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b97ede1..93fd964 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -95,4 +95,29 @@ jobs: - name: start db run: make ci_up - name: unit test_pkg - run: make test_pkg \ No newline at end of file + run: make test_pkg + + docker-push: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Docker meta custom api + id: meta_customapi + uses: docker/metadata-action@v4 + with: + images: spacemeshos/explorer-custom-api-dev + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - uses: docker/build-push-action@v2 + with: + context: . + repository: spacemeshos/explorer-custom-api-dev + file: ./Dockerfile.api + push: true + tags: ${{ steps.meta_customapi.outputs.tags }} \ No newline at end of file From de2352b3eeed4ce34e80f61cee5c02dad2c16de4 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Tue, 16 Jul 2024 16:04:34 +0200 Subject: [PATCH 03/31] Disable cache for layer stats --- api/handler/layer.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/api/handler/layer.go b/api/handler/layer.go index d98f09c..c075101 100644 --- a/api/handler/layer.go +++ b/api/handler/layer.go @@ -2,7 +2,6 @@ package handler import ( "github.com/labstack/echo/v4" - "github.com/spacemeshos/explorer-backend/api/cache" "github.com/spacemeshos/go-spacemesh/log" "net/http" "strconv" @@ -15,16 +14,11 @@ func LayerStats(c echo.Context) error { return c.NoContent(http.StatusBadRequest) } - if cached, ok := cache.Cache.Get("layerStats" + c.Param("id")); ok { - return c.JSON(http.StatusOK, cached) - } - layerStats, err := cc.StorageClient.GetLayerStats(cc.Storage, int64(lid)) if err != nil { log.Warning("failed to get layer stats: %v", err) return c.NoContent(http.StatusInternalServerError) } - cache.Cache.Set("layerStats"+c.Param("id"), layerStats, 0) return c.JSON(http.StatusOK, layerStats) } From 26d8fb31732b58ebe1ccb4830bfb6f52f91554cb Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Wed, 17 Jul 2024 10:03:20 +0200 Subject: [PATCH 04/31] Add rewards sum to overview --- api/handler/overview.go | 4 +++- api/storage/rewards.go | 5 +++-- api/storage/smesher.go | 12 ++++++++++++ api/storage/storage.go | 3 ++- cmd/api/main.go | 3 +++ 5 files changed, 23 insertions(+), 4 deletions(-) diff --git a/api/handler/overview.go b/api/handler/overview.go index 338ea2a..7365179 100644 --- a/api/handler/overview.go +++ b/api/handler/overview.go @@ -12,6 +12,7 @@ type OverviewResp struct { SmeshersCount uint64 `json:"smeshers_count"` LayersCount uint64 `json:"layers_count"` RewardsCount uint64 `json:"rewards_count"` + RewardsSum uint64 `json:"rewards_sum"` TransactionsCount uint64 `json:"transactions_count"` NumUnits uint64 `json:"num_units"` } @@ -45,11 +46,12 @@ func Overview(c echo.Context) error { } overviewResp.LayersCount = layersCount - rewardsCount, err := cc.StorageClient.GetRewardsCount(cc.Storage) + rewardsSum, rewardsCount, err := cc.StorageClient.GetRewardsSum(cc.Storage) if err != nil { log.Warning("failed to get rewards count: %v", err) return c.NoContent(http.StatusInternalServerError) } + overviewResp.RewardsSum = rewardsSum overviewResp.RewardsCount = rewardsCount transactionsCount, err := cc.StorageClient.GetTransactionsCount(cc.Storage) diff --git a/api/storage/rewards.go b/api/storage/rewards.go index e5bb2f0..9285317 100644 --- a/api/storage/rewards.go +++ b/api/storage/rewards.go @@ -2,12 +2,13 @@ package storage import "github.com/spacemeshos/go-spacemesh/sql" -func (c *Client) GetRewardsCount(db *sql.Database) (count uint64, err error) { - _, err = db.Exec(`SELECT COUNT(*) FROM rewards`, +func (c *Client) GetRewardsSum(db *sql.Database) (sum uint64, count uint64, err error) { + _, err = db.Exec(`SELECT COUNT(*), SUM(total_reward) FROM rewards`, func(stmt *sql.Statement) { }, func(stmt *sql.Statement) bool { count = uint64(stmt.ColumnInt64(0)) + sum = uint64(stmt.ColumnInt64(1)) return true }) return diff --git a/api/storage/smesher.go b/api/storage/smesher.go index e179c93..5bda2c5 100644 --- a/api/storage/smesher.go +++ b/api/storage/smesher.go @@ -78,6 +78,18 @@ func (c *Client) GetSmeshersCount(db *sql.Database) (count uint64, err error) { return } +func (c *Client) GetSmeshersByEpochCount(db *sql.Database, epoch uint64) (count uint64, err error) { + _, err = db.Exec(`SELECT COUNT(*) FROM (SELECT DISTINCT pubkey FROM atxs WHERE epoch = ?1)`, + func(stmt *sql.Statement) { + stmt.BindInt64(1, int64(epoch-1)) + }, + func(stmt *sql.Statement) bool { + count = uint64(stmt.ColumnInt64(0)) + return true + }) + return +} + func (c *Client) GetSmesher(db *sql.Database, pubkey []byte) (smesher *Smesher, err error) { smesher = &Smesher{} _, err = db.Exec(`SELECT pubkey, coinbase, effective_num_units, COUNT(*) as atxs FROM atxs WHERE pubkey = ?1 GROUP BY pubkey ORDER BY epoch DESC LIMIT 1;`, diff --git a/api/storage/storage.go b/api/storage/storage.go index 621ef13..5e163c5 100644 --- a/api/storage/storage.go +++ b/api/storage/storage.go @@ -14,8 +14,9 @@ type DatabaseClient interface { GetAccountsCount(db *sql.Database) (uint64, error) GetSmeshersCount(db *sql.Database) (uint64, error) + GetSmeshersByEpochCount(db *sql.Database, epoch uint64) (uint64, error) GetLayersCount(db *sql.Database) (uint64, error) - GetRewardsCount(db *sql.Database) (uint64, error) + GetRewardsSum(db *sql.Database) (uint64, uint64, error) GetTransactionsCount(db *sql.Database) (uint64, error) GetTotalNumUnits(db *sql.Database) (uint64, error) } diff --git a/cmd/api/main.go b/cmd/api/main.go index e278077..950df19 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -85,6 +85,9 @@ func main() { address.SetAddressConfig("stest") log.Info(`network HRP set to "stest"`) } + log.Info("layers per epoch: %d", layersPerEpoch) + log.Info("debug: %v", debug) + log.Info("sqlite path: %s", sqlitePathStringFlag) cache.Init() From f809223f8c5c9431c3f4c620357a273de853d7f0 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Wed, 17 Jul 2024 10:46:49 +0200 Subject: [PATCH 05/31] Return 404 if smesher is not found --- api/handler/smesher.go | 4 ++++ api/storage/smesher.go | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/api/handler/smesher.go b/api/handler/smesher.go index 42e8ff1..d04579c 100644 --- a/api/handler/smesher.go +++ b/api/handler/smesher.go @@ -64,6 +64,10 @@ func Smesher(c echo.Context) error { smesher, err := cc.StorageClient.GetSmesher(cc.Storage, hash.Bytes()) if err != nil { + if err.Error() == "smesher not found" { + return c.NoContent(http.StatusNotFound) + } + log.Warning("failed to get smesher: %v", err) return c.NoContent(http.StatusInternalServerError) } diff --git a/api/storage/smesher.go b/api/storage/smesher.go index 5bda2c5..64a820f 100644 --- a/api/storage/smesher.go +++ b/api/storage/smesher.go @@ -1,6 +1,7 @@ package storage import ( + "fmt" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/sql" ) @@ -91,12 +92,12 @@ func (c *Client) GetSmeshersByEpochCount(db *sql.Database, epoch uint64) (count } func (c *Client) GetSmesher(db *sql.Database, pubkey []byte) (smesher *Smesher, err error) { - smesher = &Smesher{} _, err = db.Exec(`SELECT pubkey, coinbase, effective_num_units, COUNT(*) as atxs FROM atxs WHERE pubkey = ?1 GROUP BY pubkey ORDER BY epoch DESC LIMIT 1;`, func(stmt *sql.Statement) { stmt.BindBytes(1, pubkey) }, func(stmt *sql.Statement) bool { + smesher = &Smesher{} stmt.ColumnBytes(0, smesher.Pubkey[:]) var coinbase types.Address stmt.ColumnBytes(1, coinbase[:]) @@ -108,6 +109,9 @@ func (c *Client) GetSmesher(db *sql.Database, pubkey []byte) (smesher *Smesher, if err != nil { return } + if smesher == nil { + return nil, fmt.Errorf("smesher not found") + } _, err = db.Exec(`SELECT COUNT(*), SUM(total_reward) FROM rewards WHERE pubkey=?1`, func(stmt *sql.Statement) { From 11a39239d97199b54d829a4e0b2b08b3d58a0e88 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Wed, 17 Jul 2024 12:11:27 +0200 Subject: [PATCH 06/31] Add AccountStats --- api/handler/account.go | 27 ++++++++++++++++++ api/router/router.go | 1 + api/storage/accounts.go | 63 ++++++++++++++++++++++++++++++++++++++++- api/storage/storage.go | 9 +++++- cmd/api/main.go | 2 ++ 5 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 api/handler/account.go diff --git a/api/handler/account.go b/api/handler/account.go new file mode 100644 index 0000000..a2fdf8e --- /dev/null +++ b/api/handler/account.go @@ -0,0 +1,27 @@ +package handler + +import ( + "github.com/labstack/echo/v4" + "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/log" + "net/http" +) + +func AccountStats(c echo.Context) error { + cc := c.(*ApiContext) + + address := c.Param("address") + addr, err := types.StringToAddress(address) + if err != nil { + log.Warning("failed to parse account address: %v", err) + return c.NoContent(http.StatusBadRequest) + } + + accountStats, err := cc.StorageClient.GetAccountsStats(cc.Storage, addr) + if err != nil { + log.Warning("failed to get account stats: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + return c.JSON(http.StatusOK, accountStats) +} diff --git a/api/router/router.go b/api/router/router.go index 4c63766..c0a4bbe 100644 --- a/api/router/router.go +++ b/api/router/router.go @@ -8,6 +8,7 @@ import ( func Init(e *echo.Echo) { e.GET("/stats/layer/:id", handler.LayerStats) e.GET("/stats/epoch/:id", handler.EpochStats) + e.GET("/stats/account/:address", handler.AccountStats) e.GET("/smeshers/:epoch", handler.SmeshersByEpoch) e.GET("/smeshers", handler.Smeshers) e.GET("/smesher/:smesherId", handler.Smesher) diff --git a/api/storage/accounts.go b/api/storage/accounts.go index b606a2c..4e32ab1 100644 --- a/api/storage/accounts.go +++ b/api/storage/accounts.go @@ -1,6 +1,11 @@ package storage -import "github.com/spacemeshos/go-spacemesh/sql" +import ( + "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/sql" + "github.com/spacemeshos/go-spacemesh/sql/builder" + "github.com/spacemeshos/go-spacemesh/sql/transactions" +) func (c *Client) GetAccountsCount(db *sql.Database) (uint64, error) { var total uint64 @@ -13,3 +18,59 @@ func (c *Client) GetAccountsCount(db *sql.Database) (uint64, error) { }) return total, err } + +type AccountStats struct { + Account string `json:"account"` + Received uint64 `json:"received"` + Sent uint64 `json:"sent"` +} + +func (c *Client) GetAccountsStats(db *sql.Database, addr types.Address) (*AccountStats, error) { + stats := &AccountStats{ + Account: addr.String(), + Received: 0, + Sent: 0, + } + + ops := builder.Operations{ + Filter: []builder.Op{ + { + Group: []builder.Op{ + { + Field: builder.Address, + Token: builder.Eq, + Value: addr.Bytes(), + }, + { + Field: builder.Principal, + Token: builder.Eq, + Value: addr.Bytes(), + }, + }, + GroupOperator: builder.Or, + }, + }, + } + err := transactions.IterateTransactionsOps(db, ops, func(tx *types.MeshTransaction, + result *types.TransactionResult, + ) bool { + contents, _, err := toTxContents(tx.Raw) + if err != nil { + return false + } + + if contents.GetSend() != nil { + if contents.GetSend().GetDestination() == addr.String() { + stats.Received += contents.GetSend().GetAmount() + } else { + stats.Sent += contents.GetSend().GetAmount() + } + } + return true + }) + if err != nil { + return nil, err + } + + return stats, nil +} diff --git a/api/storage/storage.go b/api/storage/storage.go index 5e163c5..13f8d97 100644 --- a/api/storage/storage.go +++ b/api/storage/storage.go @@ -2,21 +2,28 @@ package storage import ( "fmt" + "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/sql" ) type DatabaseClient interface { GetLayerStats(db *sql.Database, lid int64) (*LayerStats, error) + GetLayersCount(db *sql.Database) (uint64, error) + GetEpochStats(db *sql.Database, epoch int64, layersPerEpoch int64) (*EpochStats, error) + GetSmeshers(db *sql.Database, limit, offset uint64) (*SmesherList, error) GetSmeshersByEpoch(db *sql.Database, limit, offset, epoch uint64) (*SmesherList, error) GetSmesher(db *sql.Database, pubkey []byte) (*Smesher, error) GetAccountsCount(db *sql.Database) (uint64, error) + GetAccountsStats(db *sql.Database, addr types.Address) (*AccountStats, error) + GetSmeshersCount(db *sql.Database) (uint64, error) GetSmeshersByEpochCount(db *sql.Database, epoch uint64) (uint64, error) - GetLayersCount(db *sql.Database) (uint64, error) + GetRewardsSum(db *sql.Database) (uint64, uint64, error) + GetTransactionsCount(db *sql.Database) (uint64, error) GetTotalNumUnits(db *sql.Database) (uint64, error) } diff --git a/cmd/api/main.go b/cmd/api/main.go index 950df19..f79ed88 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -6,6 +6,7 @@ import ( "github.com/spacemeshos/explorer-backend/api" "github.com/spacemeshos/explorer-backend/api/cache" "github.com/spacemeshos/explorer-backend/api/storage" + "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/log" "github.com/urfave/cli/v2" "os" @@ -83,6 +84,7 @@ func main() { app.Action = func(ctx *cli.Context) error { if testnetBoolFlag { address.SetAddressConfig("stest") + types.SetNetworkHRP("stest") log.Info(`network HRP set to "stest"`) } log.Info("layers per epoch: %d", layersPerEpoch) From 1cd825ec87049cbf4e83d0c651048d2be6686eaa Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Thu, 18 Jul 2024 13:11:20 +0200 Subject: [PATCH 07/31] Clean-up --- api/api.go | 7 +++-- api/cache/cache.go | 14 +++++---- api/handler/epoch.go | 11 +++++-- api/handler/handler.go | 2 ++ api/handler/overview.go | 59 ++++++------------------------------- api/handler/smesher.go | 18 ++++-------- api/storage/accounts.go | 27 +++++++++++++---- api/storage/overview.go | 64 +++++++++++++++++++++++++++++++++++++++++ api/storage/rewards.go | 18 +++++++++++- api/storage/storage.go | 3 ++ cmd/api/main.go | 4 +-- go.mod | 6 ++++ go.sum | 31 ++++++++++++++++++++ 13 files changed, 180 insertions(+), 84 deletions(-) create mode 100644 api/storage/overview.go diff --git a/api/api.go b/api/api.go index df18b59..b1deabf 100644 --- a/api/api.go +++ b/api/api.go @@ -3,6 +3,7 @@ package api import ( "context" "fmt" + "github.com/eko/gocache/lib/v4/marshaler" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" "github.com/spacemeshos/explorer-backend/api/handler" @@ -20,8 +21,8 @@ type Api struct { Echo *echo.Echo } -func Init(db *sql.Database, dbClient storage.DatabaseClient, - allowedOrigins []string, debug bool, layersPerEpoch int64) *Api { +func Init(db *sql.Database, dbClient storage.DatabaseClient, allowedOrigins []string, debug bool, layersPerEpoch int64, + marshaler *marshaler.Marshaler) *Api { e := echo.New() e.Use(middleware.Recover()) @@ -32,12 +33,12 @@ func Init(db *sql.Database, dbClient storage.DatabaseClient, Storage: db, StorageClient: dbClient, LayersPerEpoch: layersPerEpoch, + Cache: marshaler, } return next(cc) } }) e.HideBanner = true - e.HidePort = true e.Use(middleware.CORSWithConfig(middleware.CORSConfig{ AllowOrigins: allowedOrigins, })) diff --git a/api/cache/cache.go b/api/cache/cache.go index 1705d24..4cb29ce 100644 --- a/api/cache/cache.go +++ b/api/cache/cache.go @@ -1,12 +1,16 @@ package cache import ( - "github.com/patrickmn/go-cache" + "github.com/eko/gocache/lib/v4/cache" + "github.com/eko/gocache/lib/v4/marshaler" + gocacheStore "github.com/eko/gocache/store/go_cache/v4" + gocache "github.com/patrickmn/go-cache" "time" ) -var Cache *cache.Cache - -func Init() { - Cache = cache.New(1*time.Hour, 10*time.Minute) +func New() *marshaler.Marshaler { + client := gocache.New(gocache.NoExpiration, 6*time.Hour) + s := gocacheStore.NewGoCache(client) + manager := cache.New[any](s) + return marshaler.New(manager) } diff --git a/api/handler/epoch.go b/api/handler/epoch.go index da97108..f2a53f8 100644 --- a/api/handler/epoch.go +++ b/api/handler/epoch.go @@ -1,8 +1,9 @@ package handler import ( + "context" "github.com/labstack/echo/v4" - "github.com/spacemeshos/explorer-backend/api/cache" + "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/log" "net/http" "strconv" @@ -15,7 +16,7 @@ func EpochStats(c echo.Context) error { return c.NoContent(http.StatusBadRequest) } - if cached, ok := cache.Cache.Get("epochStats" + c.Param("id")); ok { + if cached, err := cc.Cache.Get(context.Background(), "epochStats"+c.Param("id"), new(*storage.EpochStats)); err == nil { return c.JSON(http.StatusOK, cached) } @@ -25,6 +26,10 @@ func EpochStats(c echo.Context) error { return c.NoContent(http.StatusInternalServerError) } - cache.Cache.Set("epochStats"+c.Param("id"), epochStats, 0) + if err = cc.Cache.Set(context.Background(), "epochStats"+c.Param("id"), epochStats); err != nil { + log.Warning("failed to cache epoch stats: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + return c.JSON(http.StatusOK, epochStats) } diff --git a/api/handler/handler.go b/api/handler/handler.go index 6e787ce..70a2829 100644 --- a/api/handler/handler.go +++ b/api/handler/handler.go @@ -1,6 +1,7 @@ package handler import ( + "github.com/eko/gocache/lib/v4/marshaler" "github.com/labstack/echo/v4" "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/sql" @@ -12,6 +13,7 @@ type ApiContext struct { Storage *sql.Database StorageClient storage.DatabaseClient LayersPerEpoch int64 + Cache *marshaler.Marshaler } func GetPagination(c echo.Context) (limit, offset int64) { diff --git a/api/handler/overview.go b/api/handler/overview.go index 7365179..202eea1 100644 --- a/api/handler/overview.go +++ b/api/handler/overview.go @@ -1,73 +1,30 @@ package handler import ( + "context" "github.com/labstack/echo/v4" - "github.com/spacemeshos/explorer-backend/api/cache" + "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/log" "net/http" ) -type OverviewResp struct { - AccountsCount uint64 `json:"accounts_count"` - SmeshersCount uint64 `json:"smeshers_count"` - LayersCount uint64 `json:"layers_count"` - RewardsCount uint64 `json:"rewards_count"` - RewardsSum uint64 `json:"rewards_sum"` - TransactionsCount uint64 `json:"transactions_count"` - NumUnits uint64 `json:"num_units"` -} - func Overview(c echo.Context) error { cc := c.(*ApiContext) - if cached, ok := cache.Cache.Get("overview"); ok { + if cached, err := cc.Cache.Get(context.Background(), "overview", new(storage.Overview)); err == nil { return c.JSON(http.StatusOK, cached) } - overviewResp := OverviewResp{} - accountsCount, err := cc.StorageClient.GetAccountsCount(cc.Storage) - if err != nil { - log.Warning("failed to get accounts count: %v", err) - return c.NoContent(http.StatusInternalServerError) - } - overviewResp.AccountsCount = accountsCount - - smeshersCount, err := cc.StorageClient.GetSmeshersCount(cc.Storage) - if err != nil { - log.Warning("failed to get smeshers count: %v", err) - return c.NoContent(http.StatusInternalServerError) - } - overviewResp.SmeshersCount = smeshersCount - - layersCount, err := cc.StorageClient.GetLayersCount(cc.Storage) + overview, err := cc.StorageClient.Overview(cc.Storage) if err != nil { - log.Warning("failed to get layers count: %v", err) + log.Warning("failed to get overview: %v", err) return c.NoContent(http.StatusInternalServerError) } - overviewResp.LayersCount = layersCount - rewardsSum, rewardsCount, err := cc.StorageClient.GetRewardsSum(cc.Storage) - if err != nil { - log.Warning("failed to get rewards count: %v", err) - return c.NoContent(http.StatusInternalServerError) - } - overviewResp.RewardsSum = rewardsSum - overviewResp.RewardsCount = rewardsCount - - transactionsCount, err := cc.StorageClient.GetTransactionsCount(cc.Storage) - if err != nil { - log.Warning("failed to get transactions count: %v", err) - return c.NoContent(http.StatusInternalServerError) - } - overviewResp.TransactionsCount = transactionsCount - - numUnits, err := cc.StorageClient.GetTotalNumUnits(cc.Storage) - if err != nil { - log.Warning("failed to get num units count: %v", err) + if err = cc.Cache.Set(context.Background(), "overview", overview); err != nil { + log.Warning("failed to cache overview: %v", err) return c.NoContent(http.StatusInternalServerError) } - overviewResp.NumUnits = numUnits - cache.Cache.Set("overview", overviewResp, 0) - return c.JSON(http.StatusOK, overviewResp) + return c.JSON(http.StatusOK, overview) } diff --git a/api/handler/smesher.go b/api/handler/smesher.go index d04579c..b2002e8 100644 --- a/api/handler/smesher.go +++ b/api/handler/smesher.go @@ -1,9 +1,10 @@ package handler import ( + "context" "fmt" "github.com/labstack/echo/v4" - "github.com/spacemeshos/explorer-backend/api/cache" + "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/log" "net/http" @@ -14,17 +15,12 @@ func Smeshers(c echo.Context) error { cc := c.(*ApiContext) limit, offset := GetPagination(c) - if cached, ok := cache.Cache.Get(fmt.Sprintf("smeshers-%d-%d", limit, offset)); ok { - return c.JSON(http.StatusOK, cached) - } - smeshers, err := cc.StorageClient.GetSmeshers(cc.Storage, uint64(limit), uint64(offset)) if err != nil { log.Warning("failed to get smeshers: %v", err) return c.NoContent(http.StatusInternalServerError) } - cache.Cache.Set(fmt.Sprintf("smeshers-%d-%d", limit, offset), smeshers, 0) return c.JSON(http.StatusOK, smeshers) } @@ -38,7 +34,8 @@ func SmeshersByEpoch(c echo.Context) error { limit, offset := GetPagination(c) - if cached, ok := cache.Cache.Get(fmt.Sprintf("smeshers-epoch-%d-%d-%d", epochId, limit, offset)); ok { + if cached, err := cc.Cache.Get(context.Background(), + fmt.Sprintf("smeshers-epoch-%d-%d-%d", epochId, limit, offset), new(storage.SmesherList)); err == nil { return c.JSON(http.StatusOK, cached) } @@ -48,7 +45,7 @@ func SmeshersByEpoch(c echo.Context) error { return c.NoContent(http.StatusInternalServerError) } - cache.Cache.Set(fmt.Sprintf("smeshers-epoch-%d-%d-%d", epochId, limit, offset), smeshers, 0) + cc.Cache.Set(context.Background(), fmt.Sprintf("smeshers-epoch-%d-%d-%d", epochId, limit, offset), smeshers) return c.JSON(http.StatusOK, smeshers) } @@ -58,10 +55,6 @@ func Smesher(c echo.Context) error { smesherId := c.Param("smesherId") hash := types.HexToHash32(smesherId) - if cached, ok := cache.Cache.Get("smesher-" + smesherId); ok { - return c.JSON(http.StatusOK, cached) - } - smesher, err := cc.StorageClient.GetSmesher(cc.Storage, hash.Bytes()) if err != nil { if err.Error() == "smesher not found" { @@ -71,7 +64,6 @@ func Smesher(c echo.Context) error { log.Warning("failed to get smesher: %v", err) return c.NoContent(http.StatusInternalServerError) } - cache.Cache.Set("smesher-"+smesherId, smesher, 0) return c.JSON(http.StatusOK, smesher) } diff --git a/api/storage/accounts.go b/api/storage/accounts.go index 4e32ab1..27be6a3 100644 --- a/api/storage/accounts.go +++ b/api/storage/accounts.go @@ -20,16 +20,22 @@ func (c *Client) GetAccountsCount(db *sql.Database) (uint64, error) { } type AccountStats struct { - Account string `json:"account"` - Received uint64 `json:"received"` - Sent uint64 `json:"sent"` + Account string `json:"account"` + Received uint64 `json:"received"` + Sent uint64 `json:"sent"` + TransactionsCount uint64 `json:"transactions_count"` + RewardsCount uint64 `json:"rewards_count"` + RewardsSum uint64 `json:"rewards_sum"` } func (c *Client) GetAccountsStats(db *sql.Database, addr types.Address) (*AccountStats, error) { stats := &AccountStats{ - Account: addr.String(), - Received: 0, - Sent: 0, + Account: addr.String(), + Received: 0, + Sent: 0, + TransactionsCount: 0, + RewardsCount: 0, + RewardsSum: 0, } ops := builder.Operations{ @@ -66,11 +72,20 @@ func (c *Client) GetAccountsStats(db *sql.Database, addr types.Address) (*Accoun stats.Sent += contents.GetSend().GetAmount() } } + + stats.TransactionsCount++ return true }) if err != nil { return nil, err } + sum, count, err := c.GetRewardsSumByAddress(db, addr) + if err != nil { + return nil, err + } + stats.RewardsSum = sum + stats.RewardsCount = count + return stats, nil } diff --git a/api/storage/overview.go b/api/storage/overview.go new file mode 100644 index 0000000..ad6f3d6 --- /dev/null +++ b/api/storage/overview.go @@ -0,0 +1,64 @@ +package storage + +import ( + "github.com/spacemeshos/go-spacemesh/log" + "github.com/spacemeshos/go-spacemesh/sql" +) + +type Overview struct { + AccountsCount uint64 `json:"accounts_count"` + SmeshersCount uint64 `json:"smeshers_count"` + LayersCount uint64 `json:"layers_count"` + RewardsCount uint64 `json:"rewards_count"` + RewardsSum uint64 `json:"rewards_sum"` + TransactionsCount uint64 `json:"transactions_count"` + NumUnits uint64 `json:"num_units"` +} + +func (c *Client) Overview(db *sql.Database) (*Overview, error) { + overview := &Overview{} + accountsCount, err := c.GetAccountsCount(db) + if err != nil { + log.Warning("failed to get accounts count: %v", err) + return nil, err + } + overview.AccountsCount = accountsCount + + smeshersCount, err := c.GetSmeshersCount(db) + if err != nil { + log.Warning("failed to get smeshers count: %v", err) + return nil, err + } + overview.SmeshersCount = smeshersCount + + layersCount, err := c.GetLayersCount(db) + if err != nil { + log.Warning("failed to get layers count: %v", err) + return nil, err + } + overview.LayersCount = layersCount + + rewardsSum, rewardsCount, err := c.GetRewardsSum(db) + if err != nil { + log.Warning("failed to get rewards count: %v", err) + return nil, err + } + overview.RewardsSum = rewardsSum + overview.RewardsCount = rewardsCount + + transactionsCount, err := c.GetTransactionsCount(db) + if err != nil { + log.Warning("failed to get transactions count: %v", err) + return nil, err + } + overview.TransactionsCount = transactionsCount + + numUnits, err := c.GetTotalNumUnits(db) + if err != nil { + log.Warning("failed to get num units count: %v", err) + return nil, err + } + overview.NumUnits = numUnits + + return overview, nil +} diff --git a/api/storage/rewards.go b/api/storage/rewards.go index 9285317..a3a508a 100644 --- a/api/storage/rewards.go +++ b/api/storage/rewards.go @@ -1,6 +1,9 @@ package storage -import "github.com/spacemeshos/go-spacemesh/sql" +import ( + "github.com/spacemeshos/go-spacemesh/common/types" + "github.com/spacemeshos/go-spacemesh/sql" +) func (c *Client) GetRewardsSum(db *sql.Database) (sum uint64, count uint64, err error) { _, err = db.Exec(`SELECT COUNT(*), SUM(total_reward) FROM rewards`, @@ -13,3 +16,16 @@ func (c *Client) GetRewardsSum(db *sql.Database) (sum uint64, count uint64, err }) return } + +func (c *Client) GetRewardsSumByAddress(db *sql.Database, addr types.Address) (sum uint64, count uint64, err error) { + _, err = db.Exec(`SELECT COUNT(*), SUM(total_reward) FROM rewards WHERE coinbase = ?1`, + func(stmt *sql.Statement) { + stmt.BindBytes(1, addr.Bytes()) + }, + func(stmt *sql.Statement) bool { + count = uint64(stmt.ColumnInt64(0)) + sum = uint64(stmt.ColumnInt64(1)) + return true + }) + return +} diff --git a/api/storage/storage.go b/api/storage/storage.go index 13f8d97..aea6733 100644 --- a/api/storage/storage.go +++ b/api/storage/storage.go @@ -7,6 +7,8 @@ import ( ) type DatabaseClient interface { + Overview(db *sql.Database) (*Overview, error) + GetLayerStats(db *sql.Database, lid int64) (*LayerStats, error) GetLayersCount(db *sql.Database) (uint64, error) @@ -23,6 +25,7 @@ type DatabaseClient interface { GetSmeshersByEpochCount(db *sql.Database, epoch uint64) (uint64, error) GetRewardsSum(db *sql.Database) (uint64, uint64, error) + GetRewardsSumByAddress(db *sql.Database, addr types.Address) (sum uint64, count uint64, err error) GetTransactionsCount(db *sql.Database) (uint64, error) GetTotalNumUnits(db *sql.Database) (uint64, error) diff --git a/cmd/api/main.go b/cmd/api/main.go index f79ed88..04111c4 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -91,7 +91,7 @@ func main() { log.Info("debug: %v", debug) log.Info("sqlite path: %s", sqlitePathStringFlag) - cache.Init() + c := cache.New() db, err := storage.Setup(sqlitePathStringFlag) if err != nil { @@ -100,7 +100,7 @@ func main() { } dbClient := &storage.Client{} - server := api.Init(db, dbClient, allowedOrigins.Value(), debug, layersPerEpoch) + server := api.Init(db, dbClient, allowedOrigins.Value(), debug, layersPerEpoch, c) log.Info(fmt.Sprintf("starting server on %s", listenStringFlag)) server.Run(listenStringFlag) diff --git a/go.mod b/go.mod index c52a877..f85f076 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,8 @@ go 1.22.3 toolchain go1.22.4 require ( + github.com/eko/gocache/lib/v4 v4.1.6 + github.com/eko/gocache/store/go_cache/v4 v4.2.2 github.com/go-llsqlite/crawshaw v0.5.3 github.com/gofiber/fiber/v2 v2.52.5 github.com/golang/protobuf v1.5.4 @@ -39,6 +41,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang/mock v1.6.0 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 // indirect @@ -69,6 +72,8 @@ require ( github.com/valyala/fasthttp v1.51.0 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect github.com/valyala/tcplisten v1.0.0 // indirect + github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.1 // indirect github.com/xdg-go/stringprep v1.0.3 // indirect @@ -79,6 +84,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.24.0 // indirect + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect diff --git a/go.sum b/go.sum index 275161b..e3a0d37 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,10 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eko/gocache/lib/v4 v4.1.6 h1:5WWIGISKhE7mfkyF+SJyWwqa4Dp2mkdX8QsZpnENqJI= +github.com/eko/gocache/lib/v4 v4.1.6/go.mod h1:HFxC8IiG2WeRotg09xEnPD72sCheJiTSr4Li5Ameg7g= +github.com/eko/gocache/store/go_cache/v4 v4.2.2 h1:tAI9nl6TLoJyKG1ujF0CS0n/IgTEMl+NivxtR5R3/hw= +github.com/eko/gocache/store/go_cache/v4 v4.2.2/go.mod h1:T9zkHokzr8K9EiC7RfMbDg6HSwaV6rv3UdcNu13SGcA= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= @@ -43,6 +47,8 @@ github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yG github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -168,6 +174,10 @@ github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -179,6 +189,7 @@ github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= @@ -195,17 +206,30 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -216,6 +240,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= @@ -223,7 +249,12 @@ golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 h1:Di6ANFilr+S60a4S61ZM00vLdw0IrQOSMS2/6mrnOU0= From 4155fd68a4f7f9ad0172ea060088ee99a8df59ee Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Fri, 19 Jul 2024 16:14:34 +0200 Subject: [PATCH 08/31] Add refresh api server, cache other endpoints --- Dockerfile.api | 1 + api/api.go | 5 ++- api/handler/account.go | 17 +++++++++- api/handler/epoch.go | 25 ++++++++++++-- api/handler/handler.go | 12 +++---- api/handler/layer.go | 17 +++++++++- api/handler/overview.go | 17 ++++++++++ api/handler/smesher.go | 75 ++++++++++++++++++++++++++++++++++++++++- api/router/router.go | 16 ++++++--- cmd/api/main.go | 43 ++++++++++++++++++----- 10 files changed, 201 insertions(+), 27 deletions(-) diff --git a/Dockerfile.api b/Dockerfile.api index 5fd8f8a..3404e66 100644 --- a/Dockerfile.api +++ b/Dockerfile.api @@ -7,4 +7,5 @@ RUN go build -o explorer-custom-api ./cmd/api/ FROM alpine:3.17 COPY --from=build /src/explorer-custom-api /bin/ EXPOSE 5000 +EXPOSE 5050 ENTRYPOINT ["/bin/explorer-custom-api"] diff --git a/api/api.go b/api/api.go index b1deabf..616f268 100644 --- a/api/api.go +++ b/api/api.go @@ -7,7 +7,6 @@ import ( "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" "github.com/spacemeshos/explorer-backend/api/handler" - "github.com/spacemeshos/explorer-backend/api/router" "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/log" "github.com/spacemeshos/go-spacemesh/sql" @@ -22,7 +21,7 @@ type Api struct { } func Init(db *sql.Database, dbClient storage.DatabaseClient, allowedOrigins []string, debug bool, layersPerEpoch int64, - marshaler *marshaler.Marshaler) *Api { + marshaler *marshaler.Marshaler, routes func(e *echo.Echo)) *Api { e := echo.New() e.Use(middleware.Recover()) @@ -57,7 +56,7 @@ func Init(db *sql.Database, dbClient storage.DatabaseClient, allowedOrigins []st e.Use(middleware.Logger()) } - router.Init(e) + routes(e) return &Api{ Echo: e, diff --git a/api/handler/account.go b/api/handler/account.go index a2fdf8e..5379fc5 100644 --- a/api/handler/account.go +++ b/api/handler/account.go @@ -1,13 +1,17 @@ package handler import ( + "context" + "github.com/eko/gocache/lib/v4/store" "github.com/labstack/echo/v4" + "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/log" "net/http" + "time" ) -func AccountStats(c echo.Context) error { +func Account(c echo.Context) error { cc := c.(*ApiContext) address := c.Param("address") @@ -17,11 +21,22 @@ func AccountStats(c echo.Context) error { return c.NoContent(http.StatusBadRequest) } + if cached, err := cc.Cache.Get(context.Background(), "accountStats"+address, + new(*storage.AccountStats)); err == nil { + return c.JSON(http.StatusOK, cached) + } + accountStats, err := cc.StorageClient.GetAccountsStats(cc.Storage, addr) if err != nil { log.Warning("failed to get account stats: %v", err) return c.NoContent(http.StatusInternalServerError) } + if err = cc.Cache.Set(context.Background(), "accountStats"+address, accountStats, + store.WithExpiration(1*time.Minute)); err != nil { + log.Warning("failed to cache account stats: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + return c.JSON(http.StatusOK, accountStats) } diff --git a/api/handler/epoch.go b/api/handler/epoch.go index f2a53f8..f8e6ada 100644 --- a/api/handler/epoch.go +++ b/api/handler/epoch.go @@ -9,7 +9,7 @@ import ( "strconv" ) -func EpochStats(c echo.Context) error { +func Epoch(c echo.Context) error { cc := c.(*ApiContext) id, err := strconv.Atoi(c.Param("id")) if err != nil { @@ -22,7 +22,7 @@ func EpochStats(c echo.Context) error { epochStats, err := cc.StorageClient.GetEpochStats(cc.Storage, int64(id), cc.LayersPerEpoch) if err != nil { - log.Warning("failed to get layer stats: %v", err) + log.Warning("failed to get epoch stats: %v", err) return c.NoContent(http.StatusInternalServerError) } @@ -33,3 +33,24 @@ func EpochStats(c echo.Context) error { return c.JSON(http.StatusOK, epochStats) } + +func EpochRefresh(c echo.Context) error { + cc := c.(*ApiContext) + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + return c.NoContent(http.StatusBadRequest) + } + + epochStats, err := cc.StorageClient.GetEpochStats(cc.Storage, int64(id), cc.LayersPerEpoch) + if err != nil { + log.Warning("failed to get epoch stats: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + if err = cc.Cache.Set(context.Background(), "epochStats"+c.Param("id"), epochStats); err != nil { + log.Warning("failed to cache epoch stats: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + return c.NoContent(http.StatusOK) +} diff --git a/api/handler/handler.go b/api/handler/handler.go index 70a2829..c6ef4ef 100644 --- a/api/handler/handler.go +++ b/api/handler/handler.go @@ -19,12 +19,12 @@ type ApiContext struct { func GetPagination(c echo.Context) (limit, offset int64) { limit = 20 offset = 0 - if page := c.QueryParam("limit"); page != "" { - limit, _ = strconv.ParseInt(page, 10, 32) - if limit <= 0 { - limit = 0 - } - } + //if page := c.QueryParam("limit"); page != "" { + // limit, _ = strconv.ParseInt(page, 10, 32) + // if limit <= 0 { + // limit = 0 + // } + //} if size := c.QueryParam("offset"); size != "" { offset, _ = strconv.ParseInt(size, 10, 32) if offset <= 0 { diff --git a/api/handler/layer.go b/api/handler/layer.go index c075101..4113634 100644 --- a/api/handler/layer.go +++ b/api/handler/layer.go @@ -1,24 +1,39 @@ package handler import ( + "context" + "github.com/eko/gocache/lib/v4/store" "github.com/labstack/echo/v4" + "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/log" "net/http" "strconv" + "time" ) -func LayerStats(c echo.Context) error { +func Layer(c echo.Context) error { cc := c.(*ApiContext) lid, err := strconv.Atoi(c.Param("id")) if err != nil { return c.NoContent(http.StatusBadRequest) } + if cached, err := cc.Cache.Get(context.Background(), "layerStats"+c.Param("id"), + new(*storage.LayerStats)); err == nil { + return c.JSON(http.StatusOK, cached) + } + layerStats, err := cc.StorageClient.GetLayerStats(cc.Storage, int64(lid)) if err != nil { log.Warning("failed to get layer stats: %v", err) return c.NoContent(http.StatusInternalServerError) } + if err = cc.Cache.Set(context.Background(), "layerStats"+c.Param("id"), + layerStats, store.WithExpiration(2*time.Minute)); err != nil { + log.Warning("failed to cache layer stats: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + return c.JSON(http.StatusOK, layerStats) } diff --git a/api/handler/overview.go b/api/handler/overview.go index 202eea1..3651ed2 100644 --- a/api/handler/overview.go +++ b/api/handler/overview.go @@ -28,3 +28,20 @@ func Overview(c echo.Context) error { return c.JSON(http.StatusOK, overview) } + +func OverviewRefresh(c echo.Context) error { + cc := c.(*ApiContext) + + overview, err := cc.StorageClient.Overview(cc.Storage) + if err != nil { + log.Warning("failed to get overview: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + if err = cc.Cache.Set(context.Background(), "overview", overview); err != nil { + log.Warning("failed to cache overview: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + return c.NoContent(http.StatusOK) +} diff --git a/api/handler/smesher.go b/api/handler/smesher.go index b2002e8..661cdc5 100644 --- a/api/handler/smesher.go +++ b/api/handler/smesher.go @@ -3,27 +3,60 @@ package handler import ( "context" "fmt" + "github.com/eko/gocache/lib/v4/store" "github.com/labstack/echo/v4" "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/log" "net/http" "strconv" + "time" ) func Smeshers(c echo.Context) error { cc := c.(*ApiContext) limit, offset := GetPagination(c) + if cached, err := cc.Cache.Get(context.Background(), fmt.Sprintf("smeshers-%d-%d", limit, offset), + new(*storage.SmesherList)); err == nil { + return c.JSON(http.StatusOK, cached) + } + smeshers, err := cc.StorageClient.GetSmeshers(cc.Storage, uint64(limit), uint64(offset)) if err != nil { log.Warning("failed to get smeshers: %v", err) return c.NoContent(http.StatusInternalServerError) } + if err = cc.Cache.Set(context.Background(), fmt.Sprintf("smeshers-%d-%d", limit, offset), smeshers); err != nil { + log.Warning("failed to cache smeshers: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + return c.JSON(http.StatusOK, smeshers) } +func SmeshersRefresh(c echo.Context) error { + cc := c.(*ApiContext) + + smeshers, err := cc.StorageClient.GetSmeshers(cc.Storage, 1000, 0) + if err != nil { + log.Warning("failed to get smeshers: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + for i := 0; i < len(smeshers.Smeshers); i += 20 { + if err = cc.Cache.Set(context.Background(), fmt.Sprintf("smeshers-%d-%d", 20, i), &storage.SmesherList{ + Smeshers: smeshers.Smeshers[i : i+20], + }); err != nil { + log.Warning("failed to cache smeshers: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + } + + return c.NoContent(http.StatusOK) +} + func SmeshersByEpoch(c echo.Context) error { cc := c.(*ApiContext) @@ -45,7 +78,37 @@ func SmeshersByEpoch(c echo.Context) error { return c.NoContent(http.StatusInternalServerError) } - cc.Cache.Set(context.Background(), fmt.Sprintf("smeshers-epoch-%d-%d-%d", epochId, limit, offset), smeshers) + if err = cc.Cache.Set(context.Background(), fmt.Sprintf("smeshers-epoch-%d-%d-%d", epochId, limit, offset), smeshers); err != nil { + log.Warning("failed to cache smeshers: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + return c.JSON(http.StatusOK, smeshers) +} + +func SmeshersByEpochRefresh(c echo.Context) error { + cc := c.(*ApiContext) + + epochId, err := strconv.Atoi(c.Param("epoch")) + if err != nil || epochId < 0 { + return c.NoContent(http.StatusBadRequest) + } + + smeshers, err := cc.StorageClient.GetSmeshersByEpoch(cc.Storage, 1000, 0, uint64(epochId)) + if err != nil { + log.Warning("failed to get smeshers: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + for i := 0; i < len(smeshers.Smeshers); i += 20 { + if err = cc.Cache.Set(context.Background(), fmt.Sprintf("smeshers-epoch-%d-%d-%d", epochId, 20, i), &storage.SmesherList{ + Smeshers: smeshers.Smeshers[i : i+20], + }); err != nil { + log.Warning("failed to cache smeshers: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + } + return c.JSON(http.StatusOK, smeshers) } @@ -55,6 +118,10 @@ func Smesher(c echo.Context) error { smesherId := c.Param("smesherId") hash := types.HexToHash32(smesherId) + if cached, err := cc.Cache.Get(context.Background(), "smesher-"+smesherId, new(*storage.Smesher)); err == nil { + return c.JSON(http.StatusOK, cached) + } + smesher, err := cc.StorageClient.GetSmesher(cc.Storage, hash.Bytes()) if err != nil { if err.Error() == "smesher not found" { @@ -65,5 +132,11 @@ func Smesher(c echo.Context) error { return c.NoContent(http.StatusInternalServerError) } + if err = cc.Cache.Set(context.Background(), "smesher-"+smesherId, smesher, + store.WithExpiration(10*time.Minute)); err != nil { + log.Warning("failed to cache smesher: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + return c.JSON(http.StatusOK, smesher) } diff --git a/api/router/router.go b/api/router/router.go index c0a4bbe..b79f5b3 100644 --- a/api/router/router.go +++ b/api/router/router.go @@ -5,12 +5,20 @@ import ( "github.com/spacemeshos/explorer-backend/api/handler" ) -func Init(e *echo.Echo) { - e.GET("/stats/layer/:id", handler.LayerStats) - e.GET("/stats/epoch/:id", handler.EpochStats) - e.GET("/stats/account/:address", handler.AccountStats) +func Router(e *echo.Echo) { + e.GET("/layer/:id", handler.Layer) + e.GET("/epoch/:id", handler.Epoch) + e.GET("/account/:address", handler.Account) e.GET("/smeshers/:epoch", handler.SmeshersByEpoch) e.GET("/smeshers", handler.Smeshers) e.GET("/smesher/:smesherId", handler.Smesher) e.GET("/overview", handler.Overview) } + +func RefreshRouter(e *echo.Echo) { + g := e.Group("/refresh") + g.GET("/epoch/:id", handler.EpochRefresh) + g.GET("/overview", handler.OverviewRefresh) + g.GET("/smeshers/:epoch", handler.SmeshersByEpochRefresh) + g.GET("/smeshers", handler.SmeshersRefresh) +} diff --git a/cmd/api/main.go b/cmd/api/main.go index 04111c4..336c31c 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -5,11 +5,13 @@ import ( "github.com/spacemeshos/address" "github.com/spacemeshos/explorer-backend/api" "github.com/spacemeshos/explorer-backend/api/cache" + "github.com/spacemeshos/explorer-backend/api/router" "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/log" "github.com/urfave/cli/v2" "os" + "sync" ) var ( @@ -19,12 +21,13 @@ var ( ) var ( - listenStringFlag string - testnetBoolFlag bool - allowedOrigins = cli.NewStringSlice("*") - debug bool - sqlitePathStringFlag string - layersPerEpoch int64 + listenStringFlag string + refreshListenStringFlag string + testnetBoolFlag bool + allowedOrigins = cli.NewStringSlice("*") + debug bool + sqlitePathStringFlag string + layersPerEpoch int64 ) var flags = []cli.Flag{ @@ -36,6 +39,14 @@ var flags = []cli.Flag{ Value: ":5000", EnvVars: []string{"SPACEMESH_API_LISTEN"}, }, + &cli.StringFlag{ + Name: "listen-refresh", + Usage: "Explorer refresh API listen string in format :", + Required: false, + Destination: &refreshListenStringFlag, + Value: ":5050", + EnvVars: []string{"SPACEMESH_REFRESH_API_LISTEN"}, + }, &cli.BoolFlag{ Name: "testnet", Usage: `Use this flag to enable testnet preset ("stest" instead of "sm" for wallet addresses)`, @@ -100,10 +111,24 @@ func main() { } dbClient := &storage.Client{} - server := api.Init(db, dbClient, allowedOrigins.Value(), debug, layersPerEpoch, c) + var wg sync.WaitGroup + wg.Add(2) + // start api server + server := api.Init(db, dbClient, allowedOrigins.Value(), debug, layersPerEpoch, c, router.Router) + go func() { + defer wg.Done() + log.Info(fmt.Sprintf("starting api server on %s", listenStringFlag)) + server.Run(listenStringFlag) + }() - log.Info(fmt.Sprintf("starting server on %s", listenStringFlag)) - server.Run(listenStringFlag) + // start refresh api server + refreshServer := api.Init(db, dbClient, allowedOrigins.Value(), debug, layersPerEpoch, c, router.RefreshRouter) + go func() { + defer wg.Done() + log.Info(fmt.Sprintf("starting refresh api server on %s", refreshListenStringFlag)) + refreshServer.Run(refreshListenStringFlag) + }() + wg.Wait() log.Info("server is shutdown") return nil From 264760b38f9cf43fae8e477989e992168c8c8de9 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Wed, 31 Jul 2024 15:49:00 +0200 Subject: [PATCH 09/31] Add circulation and decentral endpoints --- api/api.go | 4 +- api/handler/circulation.go | 30 +++++++++++++++ api/handler/epoch.go | 50 ++++++++++++++++++++++++- api/router/router.go | 3 ++ api/storage/circulation.go | 30 +++++++++++++++ api/storage/epoch.go | 56 ++++++++++++++++++++++++--- api/storage/storage.go | 11 +++++- cmd/api/main.go | 77 ++++++++++++++++++++++++++++++++++++-- go.mod | 4 +- go.sum | 5 ++- test/testseed/epoch.go | 2 +- test/testseed/generator.go | 4 +- utils/decentral.go | 2 +- utils/gini.go | 64 ++++++++++++++++--------------- 14 files changed, 291 insertions(+), 51 deletions(-) create mode 100644 api/handler/circulation.go create mode 100644 api/storage/circulation.go diff --git a/api/api.go b/api/api.go index 616f268..43e0a22 100644 --- a/api/api.go +++ b/api/api.go @@ -20,9 +20,7 @@ type Api struct { Echo *echo.Echo } -func Init(db *sql.Database, dbClient storage.DatabaseClient, allowedOrigins []string, debug bool, layersPerEpoch int64, - marshaler *marshaler.Marshaler, routes func(e *echo.Echo)) *Api { - +func Init(db *sql.Database, dbClient storage.DatabaseClient, allowedOrigins []string, debug bool, layersPerEpoch int64, marshaler *marshaler.Marshaler, routes func(e *echo.Echo)) *Api { e := echo.New() e.Use(middleware.Recover()) e.Use(func(next echo.HandlerFunc) echo.HandlerFunc { diff --git a/api/handler/circulation.go b/api/handler/circulation.go new file mode 100644 index 0000000..5948a68 --- /dev/null +++ b/api/handler/circulation.go @@ -0,0 +1,30 @@ +package handler + +import ( + "context" + "github.com/labstack/echo/v4" + "github.com/spacemeshos/explorer-backend/api/storage" + "github.com/spacemeshos/go-spacemesh/log" + "net/http" +) + +func Circulation(c echo.Context) error { + cc := c.(*ApiContext) + + if cached, err := cc.Cache.Get(context.Background(), "circulation", new(*storage.Circulation)); err == nil { + return c.JSON(http.StatusOK, cached) + } + + circulation, err := cc.StorageClient.GetCirculation(cc.Storage) + if err != nil { + log.Warning("failed to get circulation: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + if err = cc.Cache.Set(context.Background(), "circulation", circulation); err != nil { + log.Warning("failed to cache circulation: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + return c.JSON(http.StatusOK, circulation) +} diff --git a/api/handler/epoch.go b/api/handler/epoch.go index f8e6ada..4cfc0ef 100644 --- a/api/handler/epoch.go +++ b/api/handler/epoch.go @@ -2,11 +2,13 @@ package handler import ( "context" + "github.com/eko/gocache/lib/v4/store" "github.com/labstack/echo/v4" "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/log" "net/http" "strconv" + "time" ) func Epoch(c echo.Context) error { @@ -26,7 +28,7 @@ func Epoch(c echo.Context) error { return c.NoContent(http.StatusInternalServerError) } - if err = cc.Cache.Set(context.Background(), "epochStats"+c.Param("id"), epochStats); err != nil { + if err = cc.Cache.Set(context.Background(), "epochStats"+c.Param("id"), epochStats, store.WithExpiration(1*time.Minute)); err != nil { log.Warning("failed to cache epoch stats: %v", err) return c.NoContent(http.StatusInternalServerError) } @@ -54,3 +56,49 @@ func EpochRefresh(c echo.Context) error { return c.NoContent(http.StatusOK) } + +func EpochDecentral(c echo.Context) error { + cc := c.(*ApiContext) + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + return c.NoContent(http.StatusBadRequest) + } + + if cached, err := cc.Cache.Get(context.Background(), "epochStatsDecentral"+c.Param("id"), new(*storage.EpochStats)); err == nil { + return c.JSON(http.StatusOK, cached) + } + + epochStats, err := cc.StorageClient.GetEpochDecentralRatio(cc.Storage, int64(id)) + if err != nil { + log.Warning("failed to get epoch stats: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + if err = cc.Cache.Set(context.Background(), "epochStatsDecentral"+c.Param("id"), epochStats, store.WithExpiration(1*time.Minute)); err != nil { + log.Warning("failed to cache epoch stats: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + return c.JSON(http.StatusOK, epochStats) +} + +func EpochDecentralRefresh(c echo.Context) error { + cc := c.(*ApiContext) + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + return c.NoContent(http.StatusBadRequest) + } + + epochStats, err := cc.StorageClient.GetEpochDecentralRatio(cc.Storage, int64(id)) + if err != nil { + log.Warning("failed to get epoch stats: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + if err = cc.Cache.Set(context.Background(), "epochStatsDecentral"+c.Param("id"), epochStats); err != nil { + log.Warning("failed to cache epoch stats: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + return c.NoContent(http.StatusOK) +} diff --git a/api/router/router.go b/api/router/router.go index b79f5b3..a9c53b8 100644 --- a/api/router/router.go +++ b/api/router/router.go @@ -8,16 +8,19 @@ import ( func Router(e *echo.Echo) { e.GET("/layer/:id", handler.Layer) e.GET("/epoch/:id", handler.Epoch) + e.GET("/epoch/:id/decentral", handler.EpochDecentral) e.GET("/account/:address", handler.Account) e.GET("/smeshers/:epoch", handler.SmeshersByEpoch) e.GET("/smeshers", handler.Smeshers) e.GET("/smesher/:smesherId", handler.Smesher) e.GET("/overview", handler.Overview) + e.GET("/circulation", handler.Circulation) } func RefreshRouter(e *echo.Echo) { g := e.Group("/refresh") g.GET("/epoch/:id", handler.EpochRefresh) + e.GET("/epoch/:id/decentral", handler.EpochDecentralRefresh) g.GET("/overview", handler.OverviewRefresh) g.GET("/smeshers/:epoch", handler.SmeshersByEpochRefresh) g.GET("/smeshers", handler.SmeshersRefresh) diff --git a/api/storage/circulation.go b/api/storage/circulation.go new file mode 100644 index 0000000..e6b5892 --- /dev/null +++ b/api/storage/circulation.go @@ -0,0 +1,30 @@ +package storage + +import ( + "github.com/spacemeshos/economics/vesting" + "github.com/spacemeshos/go-spacemesh/log" + "github.com/spacemeshos/go-spacemesh/sql" +) + +type Circulation struct { + Circulation uint64 `json:"circulation"` +} + +func (c *Client) GetCirculation(db *sql.Database) (*Circulation, error) { + circulation := &Circulation{ + Circulation: 0, + } + if !c.Testnet { + accumulatedVest := vesting.AccumulatedVestAtLayer(c.NodeClock.CurrentLayer().Uint32()) + circulation.Circulation = accumulatedVest + } + + rewardsSum, _, err := c.GetRewardsSum(db) + if err != nil { + log.Warning("failed to get rewards count: %v", err) + return nil, err + } + circulation.Circulation += rewardsSum + + return circulation, nil +} diff --git a/api/storage/epoch.go b/api/storage/epoch.go index 4029c3c..58717cf 100644 --- a/api/storage/epoch.go +++ b/api/storage/epoch.go @@ -1,19 +1,23 @@ package storage import ( + "github.com/spacemeshos/explorer-backend/utils" + "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/log" "github.com/spacemeshos/go-spacemesh/sql" "github.com/spacemeshos/go-spacemesh/sql/atxs" "github.com/spacemeshos/go-spacemesh/sql/builder" + "math" ) type EpochStats struct { - TransactionsCount uint64 `json:"transactions_count"` - ActivationsCount uint64 `json:"activations_count"` - RewardsCount uint64 `json:"rewards_count"` - RewardsSum uint64 `json:"rewards_sum"` - NumUnits uint64 `json:"num_units"` - SmeshersCount uint64 `json:"smeshers_count"` + TransactionsCount uint64 `json:"transactions_count,omitempty"` + ActivationsCount uint64 `json:"activations_count,omitempty"` + RewardsCount uint64 `json:"rewards_count,omitempty"` + RewardsSum uint64 `json:"rewards_sum,omitempty"` + NumUnits uint64 `json:"num_units,omitempty"` + SmeshersCount uint64 `json:"smeshers_count,omitempty"` + Decentral uint64 `json:"decentral,omitempty"` } func (c *Client) GetEpochStats(db *sql.Database, epoch int64, layersPerEpoch int64) (*EpochStats, error) { @@ -100,3 +104,43 @@ FROM ( return stats, err } + +func (c *Client) GetEpochDecentralRatio(db *sql.Database, epoch int64) (*EpochStats, error) { + stats := &EpochStats{ + Decentral: 0, + } + + _, err := db.Exec(`SELECT COUNT(*) FROM (SELECT DISTINCT pubkey FROM atxs WHERE epoch = ?1)`, + func(stmt *sql.Statement) { + stmt.BindInt64(1, epoch-1) + }, + func(stmt *sql.Statement) bool { + stats.SmeshersCount = uint64(stmt.ColumnInt64(0)) + return true + }) + if err != nil { + return nil, err + } + + a := math.Min(float64(stats.SmeshersCount), 1e4) + // pubkey: commitment size + smeshers := make(map[string]uint64) + _, err = db.Exec(`SELECT pubkey, effective_num_units FROM atxs WHERE epoch = ?1`, + func(stmt *sql.Statement) { + stmt.BindInt64(1, epoch-1) + }, + func(stmt *sql.Statement) bool { + var smesher types.NodeID + stmt.ColumnBytes(0, smesher[:]) + smeshers[smesher.String()] = uint64(stmt.ColumnInt64(1)) * ((c.BitsPerLabel * c.LabelsPerUnit) / 8) + return true + }) + if err != nil { + return nil, err + } + + stats.Decentral = uint64(100.0 * (0.5*(a*a)/1e8 + 0.5*(1.0-utils.Gini(smeshers)))) + + return stats, nil + +} diff --git a/api/storage/storage.go b/api/storage/storage.go index aea6733..a4818fc 100644 --- a/api/storage/storage.go +++ b/api/storage/storage.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/sql" + "github.com/spacemeshos/go-spacemesh/timesync" ) type DatabaseClient interface { @@ -13,6 +14,7 @@ type DatabaseClient interface { GetLayersCount(db *sql.Database) (uint64, error) GetEpochStats(db *sql.Database, epoch int64, layersPerEpoch int64) (*EpochStats, error) + GetEpochDecentralRatio(db *sql.Database, epoch int64) (*EpochStats, error) GetSmeshers(db *sql.Database, limit, offset uint64) (*SmesherList, error) GetSmeshersByEpoch(db *sql.Database, limit, offset, epoch uint64) (*SmesherList, error) @@ -29,9 +31,16 @@ type DatabaseClient interface { GetTransactionsCount(db *sql.Database) (uint64, error) GetTotalNumUnits(db *sql.Database) (uint64, error) + + GetCirculation(db *sql.Database) (*Circulation, error) } -type Client struct{} +type Client struct { + NodeClock *timesync.NodeClock + Testnet bool + LabelsPerUnit uint64 + BitsPerLabel uint64 +} func Setup(path string) (db *sql.Database, err error) { db, err = sql.Open(fmt.Sprintf("file:%s?mode=ro", path), diff --git a/cmd/api/main.go b/cmd/api/main.go index 336c31c..c3cdb03 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -9,9 +9,12 @@ import ( "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/log" + "github.com/spacemeshos/go-spacemesh/timesync" "github.com/urfave/cli/v2" + "go.uber.org/zap" "os" "sync" + "time" ) var ( @@ -28,6 +31,10 @@ var ( debug bool sqlitePathStringFlag string layersPerEpoch int64 + genesisTimeStringFlag string + layerDuration time.Duration + labelsPerUnit uint64 + bitsPerLabel uint64 ) var flags = []cli.Flag{ @@ -83,6 +90,38 @@ var flags = []cli.Flag{ Value: 4032, EnvVars: []string{"SPACEMESH_LAYERS_PER_EPOCH"}, }, + &cli.StringFlag{ + Name: "genesis-time", + Usage: "Genesis time in RFC3339 format", + Required: true, + Destination: &genesisTimeStringFlag, + Value: "2024-06-21T13:00:00.000Z", + EnvVars: []string{"SPACEMESH_GENESIS_TIME"}, + }, + &cli.DurationFlag{ + Name: "layer-duration", + Usage: "Duration of a single layer", + Required: false, + Destination: &layerDuration, + Value: 30 * time.Second, + EnvVars: []string{"SPACEMESH_LAYER_DURATION"}, + }, + &cli.Uint64Flag{ + Name: "labels-per-unit", + Usage: "Number of labels per unit", + Required: false, + Destination: &labelsPerUnit, + Value: 1024, + EnvVars: []string{"SPACEMESH_LABELS_PER_UNIT"}, + }, + &cli.Uint64Flag{ + Name: "bits-per-label", + Usage: "Number of bits per label", + Required: false, + Destination: &bitsPerLabel, + Value: 128, + EnvVars: []string{"SPACEMESH_BITS_PER_LABEL"}, + }, } func main() { @@ -104,17 +143,43 @@ func main() { c := cache.New() + gTime, err := time.Parse(time.RFC3339, genesisTimeStringFlag) + if err != nil { + return fmt.Errorf("cannot parse genesis time %s: %w", genesisTimeStringFlag, err) + } + + clock, err := timesync.NewClock( + timesync.WithLayerDuration(layerDuration), + timesync.WithTickInterval(1*time.Second), + timesync.WithGenesisTime(gTime), + timesync.WithLogger(zap.NewNop()), + ) + if err != nil { + return fmt.Errorf("cannot create clock: %w", err) + } + db, err := storage.Setup(sqlitePathStringFlag) if err != nil { log.Info("SQLite storage open error %v", err) return err } - dbClient := &storage.Client{} + dbClient := &storage.Client{ + NodeClock: clock, + Testnet: testnetBoolFlag, + LabelsPerUnit: labelsPerUnit, + BitsPerLabel: bitsPerLabel, + } var wg sync.WaitGroup wg.Add(2) // start api server - server := api.Init(db, dbClient, allowedOrigins.Value(), debug, layersPerEpoch, c, router.Router) + server := api.Init(db, + dbClient, + allowedOrigins.Value(), + debug, + layersPerEpoch, + c, + router.Router) go func() { defer wg.Done() log.Info(fmt.Sprintf("starting api server on %s", listenStringFlag)) @@ -122,7 +187,13 @@ func main() { }() // start refresh api server - refreshServer := api.Init(db, dbClient, allowedOrigins.Value(), debug, layersPerEpoch, c, router.RefreshRouter) + refreshServer := api.Init(db, + dbClient, + allowedOrigins.Value(), + debug, + layersPerEpoch, + c, + router.RefreshRouter) go func() { defer wg.Done() log.Info(fmt.Sprintf("starting refresh api server on %s", refreshListenStringFlag)) diff --git a/go.mod b/go.mod index f85f076..2ccb6e2 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/prometheus/client_golang v1.19.1 github.com/spacemeshos/address v0.0.0-20220829090052-44ab32617871 github.com/spacemeshos/api/release/go v1.50.0 + github.com/spacemeshos/economics v0.1.3 github.com/spacemeshos/go-scale v1.2.0 github.com/spacemeshos/go-spacemesh v1.6.2 github.com/stretchr/testify v1.9.0 @@ -48,6 +49,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huandu/xstrings v1.2.0 // indirect + github.com/jonboulle/clockwork v0.4.0 // indirect github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/labstack/gommon v0.4.0 // indirect @@ -62,7 +64,7 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/rivo/uniseg v0.2.0 // indirect + github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spacemeshos/merkle-tree v0.2.3 // indirect github.com/spacemeshos/poet v0.10.3 // indirect diff --git a/go.sum b/go.sum index e3a0d37..8f1e040 100644 --- a/go.sum +++ b/go.sum @@ -73,6 +73,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyf github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0= github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= +github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= +github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= @@ -128,8 +130,9 @@ github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= diff --git a/test/testseed/epoch.go b/test/testseed/epoch.go index 00ca557..75f798a 100644 --- a/test/testseed/epoch.go +++ b/test/testseed/epoch.go @@ -48,7 +48,7 @@ type SeedEpoch struct { Rewards map[string]*model.Reward Blocks map[string]*model.Block Smeshers map[string]*model.Smesher - SmeshersCommitment map[string]int64 + SmeshersCommitment map[string]uint64 Activations map[string]*model.Activation } diff --git a/test/testseed/generator.go b/test/testseed/generator.go index 7f1c1b0..30a0d99 100644 --- a/test/testseed/generator.go +++ b/test/testseed/generator.go @@ -69,7 +69,7 @@ func (s *SeedGenerator) GenerateEpoches(count int) error { Transactions: map[string]*model.Transaction{}, Rewards: map[string]*model.Reward{}, Smeshers: map[string]*model.Smesher{}, - SmeshersCommitment: map[string]int64{}, + SmeshersCommitment: map[string]uint64{}, Activations: map[string]*model.Activation{}, Blocks: map[string]*model.Block{}, } @@ -211,7 +211,7 @@ func (s *SeedGenerator) fillLayer(layerID, epochID int32, seedEpoch *SeedEpoch) tmpSm := s.generateSmesher(tmpLayer.Number, from, uint64(atxNumUnits)*s.seed.GetPostUnitsSize()) layerContainer.Smeshers[tmpSm.Id] = &tmpSm seedEpoch.Epoch.Stats.Current.Smeshers++ - seedEpoch.SmeshersCommitment[tmpSm.Id] += int64(tmpSm.CommitmentSize) + seedEpoch.SmeshersCommitment[tmpSm.Id] += tmpSm.CommitmentSize tmpAtx := s.generateActivation(tmpLayer.Number, atxNumUnits, &tmpSm, s.seed.GetPostUnitsSize(), uint32(epochID)) seedEpoch.Activations[tmpAtx.Id] = &tmpAtx diff --git a/utils/decentral.go b/utils/decentral.go index 803771c..e16e4a9 100644 --- a/utils/decentral.go +++ b/utils/decentral.go @@ -5,7 +5,7 @@ import ( ) // CalcDecentralCoefficient calc decentral coefficient for epoch stat. -func CalcDecentralCoefficient(smeshers map[string]int64) int64 { +func CalcDecentralCoefficient(smeshers map[string]uint64) int64 { a := math.Min(float64(len(smeshers)), 1e4) return int64(100.0 * (0.5*(a*a)/1e8 + 0.5*(1.0-Gini(smeshers)))) } diff --git a/utils/gini.go b/utils/gini.go index 5d95367..6127a56 100644 --- a/utils/gini.go +++ b/utils/gini.go @@ -1,43 +1,45 @@ package utils import ( - "sort" + "sort" ) -type CommitmentSizes []int64 +type CommitmentSizes []uint64 func (a CommitmentSizes) Len() int { return len(a) } func (a CommitmentSizes) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a CommitmentSizes) Less(i, j int) bool { return a[i] < a[j] } + /* - 1 sum((n + 1 - i)*y[i]) - G = -(n + 1 - 2--------------------- - n sum(y[i]) + 1 sum((n + 1 - i)*y[i]) + +G = -(n + 1 - 2--------------------- + + n sum(y[i]) */ -func Gini(smeshers map[string]int64) float64 { - var n int - var sum float64 - if len(smeshers) > 0 { - data := make(CommitmentSizes, len(smeshers)) - for _, commitment_size := range smeshers { - data[n] = commitment_size - if data[n] == 0 { - data[n] = 1 - } - sum += float64(data[n]) - n++ - } - if sum > 0 && n > 0 { - sort.Sort(data) - var top float64 - for i, y := range data { - top += float64(int64(n - i) * y) - } - c := (float64(n) + 1.0 - 2.0 * top / sum) / float64(n) -// log.Info("gini: top = %v, n = %v, sum = %v, coef = %v", top, n, sum, c) - return c - } - } - return 1 +func Gini(smeshers map[string]uint64) float64 { + var n int + var sum float64 + if len(smeshers) > 0 { + data := make(CommitmentSizes, len(smeshers)) + for _, commitment_size := range smeshers { + data[n] = commitment_size + if data[n] == 0 { + data[n] = 1 + } + sum += float64(data[n]) + n++ + } + if sum > 0 && n > 0 { + sort.Sort(data) + var top float64 + for i, y := range data { + top += float64(uint64(n-i) * y) + } + c := (float64(n) + 1.0 - 2.0*top/sum) / float64(n) + // log.Info("gini: top = %v, n = %v, sum = %v, coef = %v", top, n, sum, c) + return c + } + } + return 1 } - From 9c8df8fff995ce17d3b8428ffacf4d20607dc863 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Tue, 6 Aug 2024 14:29:13 +0200 Subject: [PATCH 10/31] Fix tests, change image name to explorer-stats-api-dev --- .github/workflows/ci.yaml | 10 +++++----- Dockerfile.api | 6 +++--- storage/epoch.go | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 93fd964..10a80a2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -102,11 +102,11 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Docker meta custom api - id: meta_customapi + - name: Docker meta stats api + id: meta_statsapi uses: docker/metadata-action@v4 with: - images: spacemeshos/explorer-custom-api-dev + images: spacemeshos/explorer-stats-api-dev - name: Login to DockerHub uses: docker/login-action@v1 @@ -117,7 +117,7 @@ jobs: - uses: docker/build-push-action@v2 with: context: . - repository: spacemeshos/explorer-custom-api-dev + repository: spacemeshos/explorer-stats-api-dev file: ./Dockerfile.api push: true - tags: ${{ steps.meta_customapi.outputs.tags }} \ No newline at end of file + tags: ${{ steps.meta_statsapi.outputs.tags }} \ No newline at end of file diff --git a/Dockerfile.api b/Dockerfile.api index 3404e66..6731632 100644 --- a/Dockerfile.api +++ b/Dockerfile.api @@ -2,10 +2,10 @@ FROM golang:1.22.3-alpine AS build WORKDIR /src COPY . . RUN apk add --no-cache gcc musl-dev -RUN go build -o explorer-custom-api ./cmd/api/ +RUN go build -o explorer-stats-api ./cmd/api/ FROM alpine:3.17 -COPY --from=build /src/explorer-custom-api /bin/ +COPY --from=build /src/explorer-stats-api /bin/ EXPOSE 5000 EXPOSE 5050 -ENTRYPOINT ["/bin/explorer-custom-api"] +ENTRYPOINT ["/bin/explorer-stats-api"] diff --git a/storage/epoch.go b/storage/epoch.go index b0d79f2..eece7ca 100644 --- a/storage/epoch.go +++ b/storage/epoch.go @@ -378,9 +378,9 @@ func (s *Storage) computeStatistics(epoch *model.Epoch) { } atxs, _ := s.GetActivations(context.Background(), &bson.D{{Key: "targetEpoch", Value: epoch.Number}}) if atxs != nil { - smeshers := make(map[string]int64) + smeshers := make(map[string]uint64) for _, atx := range atxs { - var commitmentSize int64 + var commitmentSize uint64 var smesher string for _, e := range atx { if e.Key == "smesher" { @@ -389,15 +389,15 @@ func (s *Storage) computeStatistics(epoch *model.Epoch) { } if e.Key == "commitmentSize" { if value, ok := e.Value.(int64); ok { - commitmentSize = value + commitmentSize = uint64(value) } else if value, ok := e.Value.(int32); ok { - commitmentSize = int64(value) + commitmentSize = uint64(value) } } } if smesher != "" { smeshers[smesher] += commitmentSize - epoch.Stats.Current.Security += commitmentSize + epoch.Stats.Current.Security += int64(commitmentSize) } } epoch.Stats.Current.Smeshers = int64(len(smeshers)) From ba25b7d36aac8c8820b5219b5fb381f03e62c720 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Tue, 6 Aug 2024 16:32:26 +0200 Subject: [PATCH 11/31] Add metrics --- api/router/router.go | 3 +++ cmd/api/main.go | 25 ++++++++++++++++++++++++- go.mod | 9 +++++---- go.sum | 21 ++++++++------------- 4 files changed, 40 insertions(+), 18 deletions(-) diff --git a/api/router/router.go b/api/router/router.go index a9c53b8..fb19565 100644 --- a/api/router/router.go +++ b/api/router/router.go @@ -1,11 +1,13 @@ package router import ( + "github.com/labstack/echo-contrib/echoprometheus" "github.com/labstack/echo/v4" "github.com/spacemeshos/explorer-backend/api/handler" ) func Router(e *echo.Echo) { + e.Use(echoprometheus.NewMiddleware("spacemesh_explorer_stats_api")) e.GET("/layer/:id", handler.Layer) e.GET("/epoch/:id", handler.Epoch) e.GET("/epoch/:id/decentral", handler.EpochDecentral) @@ -18,6 +20,7 @@ func Router(e *echo.Echo) { } func RefreshRouter(e *echo.Echo) { + e.Use(echoprometheus.NewMiddleware("spacemesh_explorer_stats_api_refresh")) g := e.Group("/refresh") g.GET("/epoch/:id", handler.EpochRefresh) e.GET("/epoch/:id/decentral", handler.EpochDecentralRefresh) diff --git a/cmd/api/main.go b/cmd/api/main.go index c3cdb03..d44f081 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -1,7 +1,10 @@ package main import ( + "errors" "fmt" + "github.com/labstack/echo-contrib/echoprometheus" + "github.com/labstack/echo/v4" "github.com/spacemeshos/address" "github.com/spacemeshos/explorer-backend/api" "github.com/spacemeshos/explorer-backend/api/cache" @@ -12,6 +15,7 @@ import ( "github.com/spacemeshos/go-spacemesh/timesync" "github.com/urfave/cli/v2" "go.uber.org/zap" + "net/http" "os" "sync" "time" @@ -35,6 +39,7 @@ var ( layerDuration time.Duration labelsPerUnit uint64 bitsPerLabel uint64 + metricsPortFlag string ) var flags = []cli.Flag{ @@ -122,6 +127,14 @@ var flags = []cli.Flag{ Value: 128, EnvVars: []string{"SPACEMESH_BITS_PER_LABEL"}, }, + &cli.StringFlag{ + Name: "metricsPort", + Usage: ``, + Required: false, + Value: ":5070", + Destination: &metricsPortFlag, + EnvVars: []string{"SPACEMESH_METRICS_PORT"}, + }, } func main() { @@ -171,7 +184,7 @@ func main() { } var wg sync.WaitGroup - wg.Add(2) + wg.Add(3) // start api server server := api.Init(db, dbClient, @@ -199,6 +212,16 @@ func main() { log.Info(fmt.Sprintf("starting refresh api server on %s", refreshListenStringFlag)) refreshServer.Run(refreshListenStringFlag) }() + + go func() { + defer wg.Done() + metrics := echo.New() + metrics.GET("/metrics", echoprometheus.NewHandler()) + if err := metrics.Start(metricsPortFlag); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Fatal("%v", err) + } + }() + wg.Wait() log.Info("server is shutdown") diff --git a/go.mod b/go.mod index 2ccb6e2..3159dc6 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,8 @@ require ( github.com/gofiber/fiber/v2 v2.52.5 github.com/golang/protobuf v1.5.4 github.com/gorilla/websocket v1.5.1 - github.com/labstack/echo/v4 v4.9.1 + github.com/labstack/echo-contrib v0.17.1 + github.com/labstack/echo/v4 v4.12.0 github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a github.com/patrickmn/go-cache v2.1.0+incompatible github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 @@ -24,6 +25,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/urfave/cli/v2 v2.27.2 go.mongodb.org/mongo-driver v1.10.1 + go.uber.org/zap v1.27.0 golang.org/x/net v0.26.0 golang.org/x/sync v0.7.0 google.golang.org/grpc v1.65.0 @@ -52,7 +54,7 @@ require ( github.com/jonboulle/clockwork v0.4.0 // indirect github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect - github.com/labstack/gommon v0.4.0 // indirect + github.com/labstack/gommon v0.4.2 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect @@ -72,7 +74,7 @@ require ( github.com/spacemeshos/sha256-simd v0.1.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.51.0 // indirect - github.com/valyala/fasttemplate v1.2.1 // indirect + github.com/valyala/fasttemplate v1.2.2 // indirect github.com/valyala/tcplisten v1.0.0 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect @@ -84,7 +86,6 @@ require ( github.com/zeebo/blake3 v0.2.3 // indirect go.uber.org/mock v0.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect golang.org/x/sys v0.21.0 // indirect diff --git a/go.sum b/go.sum index 8f1e040..bcbcf08 100644 --- a/go.sum +++ b/go.sum @@ -90,14 +90,14 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/labstack/echo/v4 v4.9.1 h1:GliPYSpzGKlyOhqIbG8nmHBo3i1saKWFOgh41AN3b+Y= -github.com/labstack/echo/v4 v4.9.1/go.mod h1:Pop5HLc+xoc4qhTZ1ip6C0RtP7Z+4VzRLWZZFKqbbjo= -github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= -github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= -github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/labstack/echo-contrib v0.17.1 h1:7I/he7ylVKsDUieaGRZ9XxxTYOjfQwVzHzUYrNykfCU= +github.com/labstack/echo-contrib v0.17.1/go.mod h1:SnsCZtwHBAZm5uBSAtQtXQHI3wqEA73hvTn0bYMKnZA= +github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0= +github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM= +github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= +github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= @@ -161,7 +161,6 @@ github.com/spacemeshos/sha256-simd v0.1.0/go.mod h1:O8CClVIilId7RtuCMV2+YzMj6qjV github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= @@ -173,8 +172,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= -github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= -github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= +github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= @@ -234,9 +233,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -271,6 +267,5 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From b81848009d522a2aa934511d4068a53beac0a7a4 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Wed, 7 Aug 2024 10:25:15 +0200 Subject: [PATCH 12/31] Add metrics to Dockerfile --- Dockerfile.api | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.api b/Dockerfile.api index 6731632..57514e3 100644 --- a/Dockerfile.api +++ b/Dockerfile.api @@ -8,4 +8,5 @@ FROM alpine:3.17 COPY --from=build /src/explorer-stats-api /bin/ EXPOSE 5000 EXPOSE 5050 +EXPOSE 5070 ENTRYPOINT ["/bin/explorer-stats-api"] From c3941b195e4bdd8f17210314e29efff1880a66c8 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Wed, 7 Aug 2024 11:28:41 +0200 Subject: [PATCH 13/31] Update README --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 868e12d..a29db71 100644 --- a/README.md +++ b/README.md @@ -33,3 +33,23 @@ The API is not properly documented yet. The best way to identity the supported A - Get mainnet current network info: https://mainnet-explorer-api.spacemesh.network/ +### Explorer Stats API + +```shell +GLOBAL OPTIONS: + --listen value Explorer API listen string in format : (default: ":5000") [$SPACEMESH_API_LISTEN] + --listen-refresh value Explorer refresh API listen string in format : (default: ":5050") [$SPACEMESH_REFRESH_API_LISTEN] + --testnet Use this flag to enable testnet preset ("stest" instead of "sm" for wallet addresses) (default: false) [$SPACEMESH_TESTNET] + --allowed-origins value [ --allowed-origins value ] Use this flag to set allowed origins for CORS (default: "*") [$ALLOWED_ORIGINS] + --debug Use this flag to enable echo debug option along with logger middleware (default: false) [$DEBUG] + --sqlite value Path to node sqlite file (default: "explorer.sql") [$SPACEMESH_SQLITE] + --layers-per-epoch value Number of layers per epoch (default: 4032) [$SPACEMESH_LAYERS_PER_EPOCH] + --genesis-time value Genesis time in RFC3339 format (default: "2024-06-21T13:00:00.000Z") [$SPACEMESH_GENESIS_TIME] + --layer-duration value Duration of a single layer (default: 30s) [$SPACEMESH_LAYER_DURATION] + --labels-per-unit value Number of labels per unit (default: 1024) [$SPACEMESH_LABELS_PER_UNIT] + --bits-per-label value Number of bits per label (default: 128) [$SPACEMESH_BITS_PER_LABEL] + --metricsPort value (default: ":5070") [$SPACEMESH_METRICS_PORT] + --help, -h show help + --version, -v print the version + +``` \ No newline at end of file From 42a1ce756b7a796e8ea5b7c8fc06d37d1e1f2f99 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Wed, 7 Aug 2024 12:21:07 +0200 Subject: [PATCH 14/31] Add cache-ttl and short-cache-ttl to set cache expiration times --- api/cache/cache.go | 5 ++++- api/handler/account.go | 4 ++-- api/handler/circulation.go | 17 +++++++++++++++++ api/handler/epoch.go | 12 ++++++------ api/handler/handler.go | 6 ------ api/handler/layer.go | 4 ++-- api/handler/smesher.go | 14 ++++++++------ api/router/router.go | 1 + cmd/api/main.go | 16 ++++++++++++++++ 9 files changed, 56 insertions(+), 23 deletions(-) diff --git a/api/cache/cache.go b/api/cache/cache.go index 4cb29ce..9477f25 100644 --- a/api/cache/cache.go +++ b/api/cache/cache.go @@ -8,8 +8,11 @@ import ( "time" ) +var Expiration time.Duration = 0 +var ShortExpiration = 5 * time.Minute + func New() *marshaler.Marshaler { - client := gocache.New(gocache.NoExpiration, 6*time.Hour) + client := gocache.New(Expiration, 6*time.Hour) s := gocacheStore.NewGoCache(client) manager := cache.New[any](s) return marshaler.New(manager) diff --git a/api/handler/account.go b/api/handler/account.go index 5379fc5..e51ec05 100644 --- a/api/handler/account.go +++ b/api/handler/account.go @@ -4,11 +4,11 @@ import ( "context" "github.com/eko/gocache/lib/v4/store" "github.com/labstack/echo/v4" + "github.com/spacemeshos/explorer-backend/api/cache" "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/log" "net/http" - "time" ) func Account(c echo.Context) error { @@ -33,7 +33,7 @@ func Account(c echo.Context) error { } if err = cc.Cache.Set(context.Background(), "accountStats"+address, accountStats, - store.WithExpiration(1*time.Minute)); err != nil { + store.WithExpiration(cache.ShortExpiration)); err != nil { log.Warning("failed to cache account stats: %v", err) return c.NoContent(http.StatusInternalServerError) } diff --git a/api/handler/circulation.go b/api/handler/circulation.go index 5948a68..eb116f6 100644 --- a/api/handler/circulation.go +++ b/api/handler/circulation.go @@ -28,3 +28,20 @@ func Circulation(c echo.Context) error { return c.JSON(http.StatusOK, circulation) } + +func CirculationRefresh(c echo.Context) error { + cc := c.(*ApiContext) + + circulation, err := cc.StorageClient.GetCirculation(cc.Storage) + if err != nil { + log.Warning("failed to get circulation: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + if err = cc.Cache.Set(context.Background(), "circulation", circulation); err != nil { + log.Warning("failed to cache circulation: %v", err) + return c.NoContent(http.StatusInternalServerError) + } + + return c.NoContent(http.StatusOK) +} diff --git a/api/handler/epoch.go b/api/handler/epoch.go index 4cfc0ef..702e403 100644 --- a/api/handler/epoch.go +++ b/api/handler/epoch.go @@ -2,13 +2,11 @@ package handler import ( "context" - "github.com/eko/gocache/lib/v4/store" "github.com/labstack/echo/v4" "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/log" "net/http" "strconv" - "time" ) func Epoch(c echo.Context) error { @@ -18,7 +16,8 @@ func Epoch(c echo.Context) error { return c.NoContent(http.StatusBadRequest) } - if cached, err := cc.Cache.Get(context.Background(), "epochStats"+c.Param("id"), new(*storage.EpochStats)); err == nil { + if cached, err := cc.Cache.Get(context.Background(), "epochStats"+c.Param("id"), + new(*storage.EpochStats)); err == nil { return c.JSON(http.StatusOK, cached) } @@ -28,7 +27,7 @@ func Epoch(c echo.Context) error { return c.NoContent(http.StatusInternalServerError) } - if err = cc.Cache.Set(context.Background(), "epochStats"+c.Param("id"), epochStats, store.WithExpiration(1*time.Minute)); err != nil { + if err = cc.Cache.Set(context.Background(), "epochStats"+c.Param("id"), epochStats); err != nil { log.Warning("failed to cache epoch stats: %v", err) return c.NoContent(http.StatusInternalServerError) } @@ -64,7 +63,8 @@ func EpochDecentral(c echo.Context) error { return c.NoContent(http.StatusBadRequest) } - if cached, err := cc.Cache.Get(context.Background(), "epochStatsDecentral"+c.Param("id"), new(*storage.EpochStats)); err == nil { + if cached, err := cc.Cache.Get(context.Background(), "epochStatsDecentral"+c.Param("id"), + new(*storage.EpochStats)); err == nil { return c.JSON(http.StatusOK, cached) } @@ -74,7 +74,7 @@ func EpochDecentral(c echo.Context) error { return c.NoContent(http.StatusInternalServerError) } - if err = cc.Cache.Set(context.Background(), "epochStatsDecentral"+c.Param("id"), epochStats, store.WithExpiration(1*time.Minute)); err != nil { + if err = cc.Cache.Set(context.Background(), "epochStatsDecentral"+c.Param("id"), epochStats); err != nil { log.Warning("failed to cache epoch stats: %v", err) return c.NoContent(http.StatusInternalServerError) } diff --git a/api/handler/handler.go b/api/handler/handler.go index c6ef4ef..915d4e3 100644 --- a/api/handler/handler.go +++ b/api/handler/handler.go @@ -19,12 +19,6 @@ type ApiContext struct { func GetPagination(c echo.Context) (limit, offset int64) { limit = 20 offset = 0 - //if page := c.QueryParam("limit"); page != "" { - // limit, _ = strconv.ParseInt(page, 10, 32) - // if limit <= 0 { - // limit = 0 - // } - //} if size := c.QueryParam("offset"); size != "" { offset, _ = strconv.ParseInt(size, 10, 32) if offset <= 0 { diff --git a/api/handler/layer.go b/api/handler/layer.go index 4113634..92b3111 100644 --- a/api/handler/layer.go +++ b/api/handler/layer.go @@ -4,11 +4,11 @@ import ( "context" "github.com/eko/gocache/lib/v4/store" "github.com/labstack/echo/v4" + "github.com/spacemeshos/explorer-backend/api/cache" "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/log" "net/http" "strconv" - "time" ) func Layer(c echo.Context) error { @@ -30,7 +30,7 @@ func Layer(c echo.Context) error { } if err = cc.Cache.Set(context.Background(), "layerStats"+c.Param("id"), - layerStats, store.WithExpiration(2*time.Minute)); err != nil { + layerStats, store.WithExpiration(cache.ShortExpiration)); err != nil { log.Warning("failed to cache layer stats: %v", err) return c.NoContent(http.StatusInternalServerError) } diff --git a/api/handler/smesher.go b/api/handler/smesher.go index 661cdc5..c7ef5d7 100644 --- a/api/handler/smesher.go +++ b/api/handler/smesher.go @@ -5,12 +5,12 @@ import ( "fmt" "github.com/eko/gocache/lib/v4/store" "github.com/labstack/echo/v4" + "github.com/spacemeshos/explorer-backend/api/cache" "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/log" "net/http" "strconv" - "time" ) func Smeshers(c echo.Context) error { @@ -78,7 +78,8 @@ func SmeshersByEpoch(c echo.Context) error { return c.NoContent(http.StatusInternalServerError) } - if err = cc.Cache.Set(context.Background(), fmt.Sprintf("smeshers-epoch-%d-%d-%d", epochId, limit, offset), smeshers); err != nil { + if err = cc.Cache.Set(context.Background(), + fmt.Sprintf("smeshers-epoch-%d-%d-%d", epochId, limit, offset), smeshers); err != nil { log.Warning("failed to cache smeshers: %v", err) return c.NoContent(http.StatusInternalServerError) } @@ -101,9 +102,10 @@ func SmeshersByEpochRefresh(c echo.Context) error { } for i := 0; i < len(smeshers.Smeshers); i += 20 { - if err = cc.Cache.Set(context.Background(), fmt.Sprintf("smeshers-epoch-%d-%d-%d", epochId, 20, i), &storage.SmesherList{ - Smeshers: smeshers.Smeshers[i : i+20], - }); err != nil { + if err = cc.Cache.Set(context.Background(), + fmt.Sprintf("smeshers-epoch-%d-%d-%d", epochId, 20, i), &storage.SmesherList{ + Smeshers: smeshers.Smeshers[i : i+20], + }); err != nil { log.Warning("failed to cache smeshers: %v", err) return c.NoContent(http.StatusInternalServerError) } @@ -133,7 +135,7 @@ func Smesher(c echo.Context) error { } if err = cc.Cache.Set(context.Background(), "smesher-"+smesherId, smesher, - store.WithExpiration(10*time.Minute)); err != nil { + store.WithExpiration(cache.ShortExpiration)); err != nil { log.Warning("failed to cache smesher: %v", err) return c.NoContent(http.StatusInternalServerError) } diff --git a/api/router/router.go b/api/router/router.go index fb19565..919b73c 100644 --- a/api/router/router.go +++ b/api/router/router.go @@ -27,4 +27,5 @@ func RefreshRouter(e *echo.Echo) { g.GET("/overview", handler.OverviewRefresh) g.GET("/smeshers/:epoch", handler.SmeshersByEpochRefresh) g.GET("/smeshers", handler.SmeshersRefresh) + g.GET("/circulation", handler.CirculationRefresh) } diff --git a/cmd/api/main.go b/cmd/api/main.go index d44f081..6695abe 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -135,6 +135,22 @@ var flags = []cli.Flag{ Destination: &metricsPortFlag, EnvVars: []string{"SPACEMESH_METRICS_PORT"}, }, + &cli.DurationFlag{ + Name: "cache-ttl", + Usage: "Cache TTL for resources like overview, epochs, cumulative stats etc.", + Required: false, + Value: 0, + Destination: &cache.Expiration, + EnvVars: []string{"SPACEMESH_CACHE_TTL"}, + }, + &cli.DurationFlag{ + Name: "short-cache-ttl", + Usage: "Short Cache TTL for resources like layers, accounts etc.", + Required: false, + Value: 5 * time.Minute, + Destination: &cache.ShortExpiration, + EnvVars: []string{"SPACEMESH_SHORT_CACHE_TTL"}, + }, } func main() { From a20c8fac233f237b89531d2b458b3a0dee286a98 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Wed, 7 Aug 2024 14:25:49 +0200 Subject: [PATCH 15/31] Remove bits-per-label --- cmd/api/main.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 6695abe..3d0bda3 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -13,6 +13,7 @@ import ( "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/log" "github.com/spacemeshos/go-spacemesh/timesync" + "github.com/spacemeshos/post/config" "github.com/urfave/cli/v2" "go.uber.org/zap" "net/http" @@ -38,7 +39,6 @@ var ( genesisTimeStringFlag string layerDuration time.Duration labelsPerUnit uint64 - bitsPerLabel uint64 metricsPortFlag string ) @@ -119,14 +119,6 @@ var flags = []cli.Flag{ Value: 1024, EnvVars: []string{"SPACEMESH_LABELS_PER_UNIT"}, }, - &cli.Uint64Flag{ - Name: "bits-per-label", - Usage: "Number of bits per label", - Required: false, - Destination: &bitsPerLabel, - Value: 128, - EnvVars: []string{"SPACEMESH_BITS_PER_LABEL"}, - }, &cli.StringFlag{ Name: "metricsPort", Usage: ``, @@ -196,7 +188,7 @@ func main() { NodeClock: clock, Testnet: testnetBoolFlag, LabelsPerUnit: labelsPerUnit, - BitsPerLabel: bitsPerLabel, + BitsPerLabel: config.BitsPerLabel, } var wg sync.WaitGroup From a0bfbb7ea8bd427e1ab94c337c797f7038c901aa Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Wed, 7 Aug 2024 14:29:43 +0200 Subject: [PATCH 16/31] Fix --- cmd/api/main.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 3d0bda3..ba178cd 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -13,7 +13,6 @@ import ( "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/log" "github.com/spacemeshos/go-spacemesh/timesync" - "github.com/spacemeshos/post/config" "github.com/urfave/cli/v2" "go.uber.org/zap" "net/http" @@ -188,7 +187,7 @@ func main() { NodeClock: clock, Testnet: testnetBoolFlag, LabelsPerUnit: labelsPerUnit, - BitsPerLabel: config.BitsPerLabel, + BitsPerLabel: 128, } var wg sync.WaitGroup From b3c2697e9bdb7f6c1accc9215f3a2d6e505339d8 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Tue, 20 Aug 2024 15:07:48 +0200 Subject: [PATCH 17/31] Add redis cache store --- api/cache/cache.go | 21 ++++++++++++++++++--- cmd/api/main.go | 8 ++++++++ docker-compose.yml | 14 ++++++++++---- go.mod | 3 +++ go.sum | 10 ++++++++++ 5 files changed, 49 insertions(+), 7 deletions(-) diff --git a/api/cache/cache.go b/api/cache/cache.go index 9477f25..75d9840 100644 --- a/api/cache/cache.go +++ b/api/cache/cache.go @@ -3,17 +3,32 @@ package cache import ( "github.com/eko/gocache/lib/v4/cache" "github.com/eko/gocache/lib/v4/marshaler" + "github.com/eko/gocache/lib/v4/store" gocacheStore "github.com/eko/gocache/store/go_cache/v4" + redis_store "github.com/eko/gocache/store/redis/v4" gocache "github.com/patrickmn/go-cache" + "github.com/redis/go-redis/v9" + "github.com/spacemeshos/go-spacemesh/log" "time" ) +var RedisAddress = "" var Expiration time.Duration = 0 var ShortExpiration = 5 * time.Minute func New() *marshaler.Marshaler { - client := gocache.New(Expiration, 6*time.Hour) - s := gocacheStore.NewGoCache(client) - manager := cache.New[any](s) + var manager *cache.Cache[any] + if RedisAddress != "" { + log.Info("using redis cache") + redisStore := redis_store.NewRedis(redis.NewClient(&redis.Options{ + Addr: RedisAddress, + }), store.WithExpiration(Expiration)) + manager = cache.New[any](redisStore) + } else { + log.Info("using memory cahe") + client := gocache.New(Expiration, 6*time.Hour) + s := gocacheStore.NewGoCache(client) + manager = cache.New[any](s) + } return marshaler.New(manager) } diff --git a/cmd/api/main.go b/cmd/api/main.go index ba178cd..2952dd4 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -142,6 +142,14 @@ var flags = []cli.Flag{ Destination: &cache.ShortExpiration, EnvVars: []string{"SPACEMESH_SHORT_CACHE_TTL"}, }, + &cli.StringFlag{ + Name: "redis", + Usage: "Redis address for cache / if not set memory cache will be used", + Required: false, + Value: "", + Destination: &cache.RedisAddress, + EnvVars: []string{"SPACEMESH_REDIS"}, + }, } func main() { diff --git a/docker-compose.yml b/docker-compose.yml index aa8a2c7..c1e9cce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,14 @@ version: '3' services: - dbMongo: - image: mongo:5 - container_name: sm_explorer_db + redis: + image: redis:6 + restart: always ports: - - "27017-27019:27017-27019" \ No newline at end of file + - "6379:6379" + volumes: + - redis-data:/data + +volumes: + redis-data: + driver: local \ No newline at end of file diff --git a/go.mod b/go.mod index 3159dc6..43de800 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ toolchain go1.22.4 require ( github.com/eko/gocache/lib/v4 v4.1.6 github.com/eko/gocache/store/go_cache/v4 v4.2.2 + github.com/eko/gocache/store/redis/v4 v4.2.2 github.com/go-llsqlite/crawshaw v0.5.3 github.com/gofiber/fiber/v2 v2.52.5 github.com/golang/protobuf v1.5.4 @@ -43,6 +44,7 @@ require ( github.com/cosmos/btcutil v1.0.5 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/mock v1.6.0 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -66,6 +68,7 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/redis/go-redis/v9 v9.0.2 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spacemeshos/merkle-tree v0.2.3 // indirect diff --git a/go.sum b/go.sum index bcbcf08..53f70bd 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,10 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c h1:FUUopH4brHNO2kJoNN3pV+OBEYmgraLT/KHZrMM69r0= github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= +github.com/bsm/ginkgo/v2 v2.5.0 h1:aOAnND1T40wEdAtkGSkvSICWeQ8L3UASX7YVCqQx+eQ= +github.com/bsm/ginkgo/v2 v2.5.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w= +github.com/bsm/gomega v1.20.0 h1:JhAwLmtRzXFTx2AkALSLa8ijZafntmhSoU63Ok18Uq8= +github.com/bsm/gomega v1.20.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk= github.com/c0mm4nd/go-ripemd v0.0.0-20200326052756-bd1759ad7d10 h1:wJ2csnFApV9G1jgh5KmYdxVOQMi+fihIggVTjcbM7ts= github.com/c0mm4nd/go-ripemd v0.0.0-20200326052756-bd1759ad7d10/go.mod h1:mYPR+a1fzjnHY3VFH5KL3PkEjMlVfGXP7c8rbWlkLJg= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -31,12 +35,16 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eko/gocache/lib/v4 v4.1.6 h1:5WWIGISKhE7mfkyF+SJyWwqa4Dp2mkdX8QsZpnENqJI= github.com/eko/gocache/lib/v4 v4.1.6/go.mod h1:HFxC8IiG2WeRotg09xEnPD72sCheJiTSr4Li5Ameg7g= github.com/eko/gocache/store/go_cache/v4 v4.2.2 h1:tAI9nl6TLoJyKG1ujF0CS0n/IgTEMl+NivxtR5R3/hw= github.com/eko/gocache/store/go_cache/v4 v4.2.2/go.mod h1:T9zkHokzr8K9EiC7RfMbDg6HSwaV6rv3UdcNu13SGcA= +github.com/eko/gocache/store/redis/v4 v4.2.2 h1:Thw31fzGuH3WzJywsdbMivOmP550D6JS7GDHhvCJPA0= +github.com/eko/gocache/store/redis/v4 v4.2.2/go.mod h1:LaTxLKx9TG/YUEybQvPMij++D7PBTIJ4+pzvk0ykz0w= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= @@ -130,6 +138,8 @@ github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/redis/go-redis/v9 v9.0.2 h1:BA426Zqe/7r56kCcvxYLWe1mkaz71LKF77GwgFzSxfE= +github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= From 01e9be09529908cb6ffb36c3303859d382af57f8 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Tue, 20 Aug 2024 15:19:28 +0200 Subject: [PATCH 18/31] Add mongo to docker-compose --- docker-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index c1e9cce..c4a52f4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,11 @@ version: '3' services: + dbMongo: + image: mongo:5 + container_name: sm_explorer_db + ports: + - "27017-27019:27017-27019" redis: image: redis:6 restart: always From 8c346e4bf9816e66829caaac408c89f8b9c89870 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Thu, 22 Aug 2024 15:18:37 +0200 Subject: [PATCH 19/31] Remove v1 codebase, fix linter & ci, bump go to v1.23.0, clean up a little bit --- .github/workflows/ci.yaml | 60 +- .github/workflows/publish.yaml | 30 +- .golangci.yml | 864 ++++++++++++++++++ Dockerfile.api => Dockerfile | 2 +- Dockerfile.apiserver | 10 - Dockerfile.collector | 9 - Makefile | 47 +- api/api.go | 14 +- api/cache/cache.go | 12 +- api/handler/account.go | 4 +- api/handler/circulation.go | 4 +- api/handler/epoch.go | 6 +- api/handler/handler.go | 4 +- api/handler/layer.go | 6 +- api/handler/overview.go | 4 +- api/handler/smesher.go | 6 +- api/storage/circulation.go | 1 + api/storage/epoch.go | 7 +- api/storage/layer.go | 1 - api/storage/rewards.go | 4 +- api/storage/smesher.go | 20 +- api/storage/storage.go | 5 +- api/storage/transactions.go | 2 + cmd/api/main.go | 14 +- cmd/apiserver/main.go | 112 --- cmd/collector/main.go | 229 ----- collector/atxs_test.go | 32 - collector/blocks_test.go | 28 - collector/collector.go | 218 ----- collector/collector_test.go | 107 --- collector/epochs_test.go | 71 -- collector/global.go | 31 - collector/http.go | 199 ---- collector/layers_test.go | 31 - collector/mesh.go | 266 ------ collector/node.go | 79 -- collector/rewards_test.go | 35 - collector/smeshers_test.go | 38 - collector/sql/accounts.go | 11 - collector/sql/atxs.go | 154 ---- collector/sql/layers.go | 185 ---- collector/sql/rewards.go | 50 - collector/sql/sql.go | 28 - collector/txs.go | 59 -- collector/txs_test.go | 30 - docker-compose.yml | 5 - explorer-backend.go | 1 - go.mod | 40 +- go.sum | 56 -- internal/api/api.go | 89 -- internal/api/handler/accounts.go | 70 -- internal/api/handler/accounts_test.go | 87 -- internal/api/handler/activations.go | 38 - internal/api/handler/activations_test.go | 42 - internal/api/handler/blocks.go | 25 - internal/api/handler/blocks_test.go | 21 - internal/api/handler/epochs.go | 81 -- internal/api/handler/epochs_test.go | 151 --- internal/api/handler/handler.go | 37 - internal/api/handler/handler_test.go | 138 --- internal/api/handler/layers.go | 82 -- internal/api/handler/layers_test.go | 138 --- internal/api/handler/network_info.go | 96 -- internal/api/handler/network_info_test.go | 76 -- internal/api/handler/pagination.go | 56 -- internal/api/handler/rewards.go | 72 -- internal/api/handler/rewards_test.go | 61 -- internal/api/handler/search.go | 23 - internal/api/handler/search_test.go | 60 -- internal/api/handler/smeshers.go | 69 -- internal/api/handler/smeshers_test.go | 72 -- internal/api/handler/transactions.go | 38 - internal/api/handler/transactions_test.go | 35 - internal/api/router/router.go | 45 - internal/service/abstract.go | 29 - internal/service/account.go | 120 --- internal/service/activation.go | 47 - internal/service/app.go | 44 - internal/service/block.go | 39 - internal/service/epoch.go | 114 --- internal/service/layer.go | 132 --- internal/service/reward.go | 58 -- internal/service/search.go | 73 -- internal/service/service.go | 115 --- internal/service/smesher.go | 70 -- internal/service/transaction.go | 47 - internal/storage/storagereader/abstract.go | 59 -- internal/storage/storagereader/accounts.go | 187 ---- internal/storage/storagereader/activation.go | 33 - internal/storage/storagereader/apps.go | 33 - internal/storage/storagereader/blocks.go | 34 - internal/storage/storagereader/epochs.go | 89 -- internal/storage/storagereader/layers.go | 184 ---- internal/storage/storagereader/rewards.go | 166 ---- internal/storage/storagereader/smeshers.go | 123 --- internal/storage/storagereader/storage.go | 63 -- .../storage/storagereader/transactions.go | 156 ---- model/account.go | 45 - model/app.go | 14 - model/atx.go | 53 -- model/block.go | 19 - model/epoch.go | 43 - model/layer.go | 91 -- model/malfeasance_proof.go | 22 - model/network_info.go | 21 - model/reward.go | 40 - model/smesher.go | 30 - model/tx.go | 142 --- pkg/transactionparser/transaction.go | 66 -- pkg/transactionparser/transaction/abstract.go | 40 - .../transaction/transaction_types.go | 15 - pkg/transactionparser/transaction_test.go | 305 ------- pkg/transactionparser/v0/parser.go | 113 --- pkg/transactionparser/v0/types.go | 380 -------- pkg/transactionparser/v0/types_scale.go | 856 ----------------- storage/account.go | 225 ----- storage/atx.go | 199 ---- storage/block.go | 122 --- storage/epoch.go | 419 --------- storage/layer.go | 149 --- storage/malfaesance_proof.go | 31 - storage/network_info.go | 76 -- storage/reward.go | 201 ---- storage/smesher.go | 203 ---- storage/storage.go | 661 -------------- storage/tx.go | 341 ------- test/testseed/account.go | 39 - test/testseed/db.go | 220 ----- test/testseed/epoch.go | 103 --- test/testseed/generator.go | 406 -------- test/testseed/layers.go | 30 - test/testserver/fake_node.go | 335 ------- test/testserver/fake_private_node.go | 45 - test/testserver/test_api.go | 92 -- test/testutils/response.go | 39 - utils/bson.go | 90 -- utils/bytes-to-string.go | 9 - utils/capacity.go | 8 - utils/decentral.go | 11 - utils/layer.go | 12 - utils/print.go | 69 -- utils/string-to-bytes.go | 17 - 142 files changed, 970 insertions(+), 12425 deletions(-) create mode 100644 .golangci.yml rename Dockerfile.api => Dockerfile (87%) delete mode 100644 Dockerfile.apiserver delete mode 100644 Dockerfile.collector delete mode 100644 cmd/apiserver/main.go delete mode 100644 cmd/collector/main.go delete mode 100644 collector/atxs_test.go delete mode 100644 collector/blocks_test.go delete mode 100644 collector/collector.go delete mode 100644 collector/collector_test.go delete mode 100644 collector/epochs_test.go delete mode 100644 collector/global.go delete mode 100644 collector/http.go delete mode 100644 collector/layers_test.go delete mode 100644 collector/mesh.go delete mode 100644 collector/node.go delete mode 100644 collector/rewards_test.go delete mode 100644 collector/smeshers_test.go delete mode 100644 collector/sql/accounts.go delete mode 100644 collector/sql/atxs.go delete mode 100644 collector/sql/layers.go delete mode 100644 collector/sql/rewards.go delete mode 100644 collector/sql/sql.go delete mode 100644 collector/txs.go delete mode 100644 collector/txs_test.go delete mode 100644 internal/api/api.go delete mode 100644 internal/api/handler/accounts.go delete mode 100644 internal/api/handler/accounts_test.go delete mode 100644 internal/api/handler/activations.go delete mode 100644 internal/api/handler/activations_test.go delete mode 100644 internal/api/handler/blocks.go delete mode 100644 internal/api/handler/blocks_test.go delete mode 100644 internal/api/handler/epochs.go delete mode 100644 internal/api/handler/epochs_test.go delete mode 100644 internal/api/handler/handler.go delete mode 100644 internal/api/handler/handler_test.go delete mode 100644 internal/api/handler/layers.go delete mode 100644 internal/api/handler/layers_test.go delete mode 100644 internal/api/handler/network_info.go delete mode 100644 internal/api/handler/network_info_test.go delete mode 100644 internal/api/handler/pagination.go delete mode 100644 internal/api/handler/rewards.go delete mode 100644 internal/api/handler/rewards_test.go delete mode 100644 internal/api/handler/search.go delete mode 100644 internal/api/handler/search_test.go delete mode 100644 internal/api/handler/smeshers.go delete mode 100644 internal/api/handler/smeshers_test.go delete mode 100644 internal/api/handler/transactions.go delete mode 100644 internal/api/handler/transactions_test.go delete mode 100644 internal/api/router/router.go delete mode 100644 internal/service/abstract.go delete mode 100644 internal/service/account.go delete mode 100644 internal/service/activation.go delete mode 100644 internal/service/app.go delete mode 100644 internal/service/block.go delete mode 100644 internal/service/epoch.go delete mode 100644 internal/service/layer.go delete mode 100644 internal/service/reward.go delete mode 100644 internal/service/search.go delete mode 100644 internal/service/service.go delete mode 100644 internal/service/smesher.go delete mode 100644 internal/service/transaction.go delete mode 100644 internal/storage/storagereader/abstract.go delete mode 100644 internal/storage/storagereader/accounts.go delete mode 100644 internal/storage/storagereader/activation.go delete mode 100644 internal/storage/storagereader/apps.go delete mode 100644 internal/storage/storagereader/blocks.go delete mode 100644 internal/storage/storagereader/epochs.go delete mode 100644 internal/storage/storagereader/layers.go delete mode 100644 internal/storage/storagereader/rewards.go delete mode 100644 internal/storage/storagereader/smeshers.go delete mode 100644 internal/storage/storagereader/storage.go delete mode 100644 internal/storage/storagereader/transactions.go delete mode 100644 model/account.go delete mode 100644 model/app.go delete mode 100644 model/atx.go delete mode 100644 model/block.go delete mode 100644 model/epoch.go delete mode 100644 model/layer.go delete mode 100644 model/malfeasance_proof.go delete mode 100644 model/network_info.go delete mode 100644 model/reward.go delete mode 100644 model/smesher.go delete mode 100644 model/tx.go delete mode 100644 pkg/transactionparser/transaction.go delete mode 100644 pkg/transactionparser/transaction/abstract.go delete mode 100644 pkg/transactionparser/transaction/transaction_types.go delete mode 100644 pkg/transactionparser/transaction_test.go delete mode 100644 pkg/transactionparser/v0/parser.go delete mode 100644 pkg/transactionparser/v0/types.go delete mode 100644 pkg/transactionparser/v0/types_scale.go delete mode 100644 storage/account.go delete mode 100644 storage/atx.go delete mode 100644 storage/block.go delete mode 100644 storage/epoch.go delete mode 100644 storage/layer.go delete mode 100644 storage/malfaesance_proof.go delete mode 100644 storage/network_info.go delete mode 100644 storage/reward.go delete mode 100644 storage/smesher.go delete mode 100644 storage/storage.go delete mode 100644 storage/tx.go delete mode 100644 test/testseed/account.go delete mode 100644 test/testseed/db.go delete mode 100644 test/testseed/epoch.go delete mode 100644 test/testseed/generator.go delete mode 100644 test/testseed/layers.go delete mode 100644 test/testserver/fake_node.go delete mode 100644 test/testserver/fake_private_node.go delete mode 100644 test/testserver/test_api.go delete mode 100644 test/testutils/response.go delete mode 100644 utils/bson.go delete mode 100644 utils/bytes-to-string.go delete mode 100644 utils/capacity.go delete mode 100644 utils/decentral.go delete mode 100644 utils/layer.go delete mode 100644 utils/print.go delete mode 100644 utils/string-to-bytes.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 10a80a2..57d9f60 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,7 +1,7 @@ name: CI env: - go-version: '1.22.3' + go-version: '1.23.0' # Trigger the workflow on all pull requests, and on push to specific branches on: @@ -31,7 +31,6 @@ jobs: runs-on: ubuntu-latest needs: filter-changes if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }} - # should not take more than 4-6 mins timeout-minutes: 10 steps: - uses: actions/setup-go@v3 @@ -42,60 +41,9 @@ jobs: uses: golangci/golangci-lint-action@v3 with: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: v1.57.1 + version: v1.59.0 only-new-issues: true - args: --timeout=3m - - unittests_api: - runs-on: ubuntu-latest - needs: filter-changes - if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }} - timeout-minutes: 20 - steps: - - name: checkout - uses: actions/checkout@v2 - - name: set up go - uses: actions/setup-go@v2 - with: - go-version: ${{ env.go-version }} - - name: start db - run: make ci_up - - name: unit test_api - run: make test_api - - unittests_collector: - runs-on: ubuntu-latest - needs: filter-changes - if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }} - timeout-minutes: 20 - steps: - - name: checkout - uses: actions/checkout@v2 - - name: set up go - uses: actions/setup-go@v2 - with: - go-version: ${{ env.go-version }} - - name: start db - run: make ci_up - - name: unit test_collector - run: make test_collector - - unittests_pkg: - runs-on: ubuntu-latest - needs: filter-changes - if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }} - timeout-minutes: 20 - steps: - - name: checkout - uses: actions/checkout@v2 - - name: set up go - uses: actions/setup-go@v2 - with: - go-version: ${{ env.go-version }} - - name: start db - run: make ci_up - - name: unit test_pkg - run: make test_pkg + args: --timeout=10m docker-push: runs-on: ubuntu-latest @@ -118,6 +66,6 @@ jobs: with: context: . repository: spacemeshos/explorer-stats-api-dev - file: ./Dockerfile.api + file: ./Dockerfile push: true tags: ${{ steps.meta_statsapi.outputs.tags }} \ No newline at end of file diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 920f7b9..f586b01 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -5,23 +5,16 @@ on: types: [published] jobs: - - publish: + docker-push: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Docker meta apiserver - id: meta_apiserver + - name: Docker meta stats api + id: meta_statsapi uses: docker/metadata-action@v4 with: - images: spacemeshos/explorer-apiserver - - - name: Docker meta collector - id: meta_collector - uses: docker/metadata-action@v4 - with: - images: spacemeshos/explorer-collector + images: spacemeshos/explorer-stats-api - name: Login to DockerHub uses: docker/login-action@v1 @@ -32,16 +25,7 @@ jobs: - uses: docker/build-push-action@v2 with: context: . - repository: spacemeshos/explorer-apiserver - file: ./Dockerfile.apiserver - push: true - tags: ${{ steps.meta_apiserver.outputs.tags }} - - - uses: docker/build-push-action@v2 - with: - context: . - repository: spacemeshos/explorer-collector - file: ./Dockerfile.collector + repository: spacemeshos/explorer-stats-api + file: ./Dockerfile push: true - tags: ${{ steps.meta_collector.outputs.tags }} - + tags: ${{ steps.meta_statsapi.outputs.tags }} diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..a259114 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,864 @@ +# This file contains all available configuration options +# with their default values (in comments). +# +# This file is not a configuration example, +# it contains the exhaustive configuration with explanations of the options. + +# Options for analysis running. +run: + # Number of operating system threads (`GOMAXPROCS`) that can execute golangci-lint simultaneously. + # If it is explicitly set to 0 (i.e. not the default) then golangci-lint will automatically set the value to match Linux container CPU quota. + # Default: the number of logical CPUs in the machine + # concurrency: 4 + + # Timeout for analysis, e.g. 30s, 5m. + # Default: 1m + timeout: 5m + + # Exit code when at least one issue was found. + # Default: 1 + # issues-exit-code: 1 + + # Include test files or not. + # Default: true + # tests: true + + # List of build tags, all linters use it. + # Default: [] + # build-tags: + # - mytag + + # If set, we pass it to "go list -mod={option}". From "go help modules": + # If invoked with -mod=readonly, the go command is disallowed from the implicit + # automatic updating of go.mod described above. Instead, it fails when any changes + # to go.mod are needed. This setting is most useful to check that go.mod does + # not need updates, such as in a continuous integration and testing system. + # If invoked with -mod=vendor, the go command assumes that the vendor + # directory holds the correct copies of dependencies and ignores + # the dependency descriptions in go.mod. + # + # Allowed values: readonly|vendor|mod + # Default: "" + modules-download-mode: readonly + + # Allow multiple parallel golangci-lint instances running. + # If false, golangci-lint acquires file lock on start. + # Default: false + # allow-parallel-runners: true + + # Allow multiple golangci-lint instances running, but serialize them around a lock. + # If false, golangci-lint exits with an error if it fails to acquire file lock on start. + # Default: false + # allow-serial-runners: true + + # Define the Go version limit. + # Mainly related to generics support since go1.18. + # Default: use Go version from the go.mod file, fallback on the env var `GOVERSION`, fallback on 1.17 + # go: '1.19' + +# output configuration options +output: + # The formats used to render issues. + # Format: `colored-line-number`, `line-number`, `json`, `colored-tab`, `tab`, `checkstyle`, `code-climate`, `junit-xml`, `github-actions`, `teamcity` + # Output path can be either `stdout`, `stderr` or path to the file to write to. + # + # For the CLI flag (`--out-format`), multiple formats can be specified by separating them by comma. + # The output can be specified for each of them by separating format name and path by colon symbol. + # Example: "--out-format=checkstyle:report.xml,json:stdout,colored-line-number" + # The CLI flag (`--out-format`) override the configuration file. + # + # Default: + # formats: + # - format: colored-line-number + # path: stdout + formats: + # - format: json + # path: stderr + # - format: checkstyle + # path: report.xml + - format: colored-line-number + + # Print lines of code with issue. + # Default: true + # print-issued-lines: false + + # Print linter name in the end of issue text. + # Default: true + # print-linter-name: false + + # Make issues output unique by line. + # Default: true + # uniq-by-line: false + + # Add a prefix to the output file references. + # Default: "" + # path-prefix: "" + + # Sort results by the order defined in `sort-order`. + # Default: false + sort-results: true + + # Order to use when sorting results. + # Require `sort-results` to `true`. + # Possible values: `file`, `linter`, and `severity`. + # + # If the severity values are inside the following list, they are ordered in this order: + # 1. error + # 2. warning + # 3. high + # 4. medium + # 5. low + # Either they are sorted alphabetically. + # + # Default: ["file"] + sort-order: + - linter + - severity + - file # filepath, line, and column. + + # Show statistics per linter. + # Default: false + # show-stats: true + +linters: + # Disable all linters. + # Default: false + disable-all: true + # Enable specific linter + # https://golangci-lint.run/usage/linters/#enabled-by-default + enable: + # - asasalint + # - asciicheck + # - bidichk + # - bodyclose + # - containedctx + # - contextcheck + - copyloopvar + # - cyclop + # - decorder + - depguard + # - dogsled + # - dupl + # - dupword + # - durationcheck + # - errcheck + # - errchkjson + # - errname + # - errorlint + # - execinquery + # - exhaustive + # - exhaustruct + # - exportloopref + # - forbidigo + # - forcetypeassert + # - funlen + - gci + # - ginkgolinter + # - gocheckcompilerdirectives + # - gochecknoglobals + # - gochecknoinits + - gochecksumtype + # - gocognit + # - goconst + # - gocritic + # - gocyclo + - godot + # - godox + # - goerr113 + - gofmt + - gofumpt + # - goheader + # - goimports + # - gomnd + # - gomoddirectives + # - gomodguard + # - goprintffuncname + # - gosec + - gosimple + # - gosmopolitan + - govet + # - grouper + - importas + # - inamedparam + - ineffassign + # - interfacebloat + # - intrange + # - ireturn + - lll + # - loggercheck + # - maintidx + # - makezero + # - mirror + - misspell + # - musttag + - nakedret + - nestif + # - nilerr + # - nilnil + # - nlreturn + # - noctx + # - nolintlint + # - nonamedreturns + # - nosprintfhostport + # - paralleltest + - perfsprint + # - prealloc + # - predeclared + # - promlinter + # - protogetter + # - reassign + - revive + # - rowserrcheck + # - sloglint + - spancheck + # - sqlclosecheck + - staticcheck + # - stylecheck + # - tagalign + # - tagliatelle + # - tenv + # - testableexamples + - testifylint + # - testpackage + # - thelper + # - tparallel + - typecheck + # - unconvert + # - unparam + - unused + # - usestdlibvars + # - varnamelen + # - wastedassign + # - whitespace + # - wrapcheck + # - wsl + # - zerologlint + + # Enable all available linters. + # Default: false + # enable-all: true + # Disable specific linter + # https://golangci-lint.run/usage/linters/#disabled-by-default + # disable: + # - asasalint + # - asciicheck + # - bidichk + # - bodyclose + # - containedctx + # - contextcheck + # - copyloopvar + # - cyclop + # - decorder + # - depguard + # - dogsled + # - dupl + # - dupword + # - durationcheck + # - errcheck + # - errchkjson + # - errname + # - errorlint + # - execinquery + # - exhaustive + # - exhaustruct + # - exportloopref + # - forbidigo + # - forcetypeassert + # - funlen + # - gci + # - ginkgolinter + # - gocheckcompilerdirectives + # - gochecknoglobals + # - gochecknoinits + # - gochecksumtype + # - gocognit + # - goconst + # - gocritic + # - gocyclo + # - godot + # - godox + # - goerr113 + # - gofmt + # - gofumpt + # - goheader + # - goimports + # - gomnd + # - gomoddirectives + # - gomodguard + # - goprintffuncname + # - gosec + # - gosimple + # - gosmopolitan + # - govet + # - grouper + # - importas + # - inamedparam + # - ineffassign + # - interfacebloat + # - intrange + # - ireturn + # - lll + # - loggercheck + # - maintidx + # - makezero + # - mirror + # - misspell + # - musttag + # - nakedret + # - nestif + # - nilerr + # - nilnil + # - nlreturn + # - noctx + # - nolintlint + # - nonamedreturns + # - nosprintfhostport + # - paralleltest + # - perfsprint + # - prealloc + # - predeclared + # - promlinter + # - protogetter + # - reassign + # - revive + # - rowserrcheck + # - sloglint + # - spancheck + # - sqlclosecheck + # - staticcheck + # - stylecheck + # - tagalign + # - tagliatelle + # - tenv + # - testableexamples + # - testifylint + # - testpackage + # - thelper + # - tparallel + # - typecheck + # - unconvert + # - unparam + # - unused + # - usestdlibvars + # - varnamelen + # - wastedassign + # - whitespace + # - wrapcheck + # - wsl + # - zerologlint + + # Enable presets. + # https://golangci-lint.run/usage/linters + # Default: [] + # presets: + # - bugs + # - comment + # - complexity + # - error + # - format + # - import + # - metalinter + # - module + # - performance + # - sql + # - style + # - test + # - unused + + # Enable only fast linters from enabled linters set (first run won't be fast) + # Default: false + # fast: true + +# All available settings of specific linters. +linters-settings: + depguard: + # Rules to apply. + # + # Variables: + # - File Variables + # you can still use and exclamation mark ! in front of a variable to say not to use it. + # Example !$test will match any file that is not a go test file. + # + # `$all` - matches all go files + # `$test` - matches all go test files + # + # - Package Variables + # + # `$gostd` - matches all of go's standard library (Pulled from `GOROOT`) + # + # Default: Only allow $gostd in all files. + rules: + # Name of a rule. + main: + # Used to determine the package matching priority. + # There are three different modes: `original`, `strict`, and `lax`. + # Default: "original" + # list-mode: lax + # List of file globs that will match this list of settings to compare against. + # Default: $all + # files: + # - "!**/*_a _file.go" + # List of allowed packages. + # allow: + # - $gostd + # - github.com/OpenPeeDeeP + # Packages that are not allowed where the value is a suggestion. + deny: + - pkg: "io/ioutil" + desc: Use os instead + - pkg: "github.com/pkg/errors" + desc: Should be replaced by standard lib errors package + - pkg: "golang.org/x/xerrors" + desc: Should be replaced by standard lib errors package + - pkg: "golang.org/x/net/context" + desc: Should be replaced by standard lib context package + - pkg: "golang.org/x/crypto/ed25519" + desc: Should be replaced by standard lib ed25519 package + + gci: + # Section configuration to compare against. + # Section names are case-insensitive and may contain parameters in (). + # The default order of sections is `standard > default > custom > blank > dot > alias`, + # If `custom-order` is `true`, it follows the order of `sections` option. + # Default: ["standard", "default"] + sections: + - standard # Standard section: captures all standard packages. + - default # Default section: contains all imports that could not be matched to another section type. + - prefix(github.com/spacemeshos/go-spacemesh) # Custom section: groups all imports with the specified Prefix. + # - blank # Blank section: contains all blank imports. This section is not present unless explicitly enabled. + # - dot # Dot section: contains all dot imports. This section is not present unless explicitly enabled. + # - alias # Alias section: contains all alias imports. This section is not present unless explicitly enabled. + + # Skip generated files. + # Default: true + # skip-generated: false + + # Enable custom order of sections. + # If `true`, make the section order the same as the order of `sections`. + # Default: false + # custom-order: true + + gofmt: + # Simplify code: gofmt with `-s` option. + # Default: true + # simplify: false + # Apply the rewrite rules to the source before reformatting. + # https://pkg.go.dev/cmd/gofmt + # Default: [] + rewrite-rules: + - pattern: "interface{}" + replacement: "any" + - pattern: "a[b:len(a)]" + replacement: "a[b:]" + + gofumpt: + # Module path which contains the source code being formatted. + # Default: "" + # module-path: github.com/org/project + + # Choose whether to use the extra rules. + # Default: false + extra-rules: true + + gosimple: + # Sxxxx checks in https://staticcheck.io/docs/configuration/options/#checks + # Default: ["*"] + checks: ["all"] + + govet: + # Disable all analyzers. + # Default: false + disable-all: false + # Enable analyzers by name. + # (in addition to default: + # appends, asmdecl, assign, atomic, bools, buildtag, cgocall, composites, copylocks, defers, directive, errorsas, + # framepointer, httpresponse, ifaceassert, loopclosure, lostcancel, nilfunc, printf, shift, sigchanyzer, slog, + # stdmethods, stringintconv, structtag, testinggoroutine, tests, timeformat, unmarshal, unreachable, unsafeptr, + # unusedresult + # ). + # Run `GL_DEBUG=govet golangci-lint run --enable=govet` to see default, all available analyzers, and enabled analyzers. + # Default: [] + # enable: + + # Enable all analyzers. + # Default: false + enable-all: false + # Disable analyzers by name. + # (in addition to default + # atomicalign, deepequalerrors, fieldalignment, findcall, nilness, reflectvaluecompare, shadow, sortslice, + # timeformat, unusedwrite + # ). + # Run `GL_DEBUG=govet golangci-lint run --enable=govet` to see default, all available analyzers, and enabled analyzers. + # Default: [] + # disable: + + importas: + # Do not allow unaliased imports of aliased packages. + # Default: false + # no-unaliased: true + # Do not allow non-required aliases. + # Default: false + # no-extra-aliases: true + # List of aliases + # Default: [] + alias: + - pkg: "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + alias: chaos + - pkg: "github.com/hashicorp/golang-lru/v2" + alias: lru + - pkg: "github.com/grpc-ecosystem/go-grpc-middleware" + alias: grpcmw + - pkg: "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap" + alias: grpczap + - pkg: "github.com/grpc-ecosystem/go-grpc-middleware/tags" + alias: grpctags + - pkg: "github.com/libp2p/go-libp2p-pubsub" + alias: pubsub + - pkg: "github.com/libp2p/go-libp2p-pubsub/pb" + alias: pubsubpb + - pkg: "github.com/libp2p/go-libp2p/p2p/net/mock" + alias: mocknet + - pkg: "github.com/libp2p/go-libp2p-testing/netutil" + alias: p2putil + - pkg: "github.com/multiformats/go-multiaddr" + alias: ma + - pkg: "github.com/multiformats/go-multiaddr/net" + alias: manet + - pkg: "github.com/spacemeshos/api/release/go/spacemesh/v1" + alias: pb + - pkg: "github.com/spacemeshos/go-spacemesh/genvm" + alias: vm + - pkg: "github.com/spacemeshos/go-spacemesh/p2p/metrics" + alias: p2pmetrics + - pkg: "github.com/spacemeshos/go-spacemesh/sql/metrics" + alias: dbmetrics + - pkg: "github.com/spacemeshos/go-spacemesh/txs/types" + alias: txtypes + - pkg: "google.golang.org/genproto/googleapis/rpc/status" + alias: rpcstatus + - pkg: "k8s.io/apimachinery/pkg/apis/meta/v1" + alias: apimetav1 + - pkg: "k8s.io/api/apps/v1" + alias: apiappsv1 + - pkg: "k8s.io/api/core/v1" + alias: apiv1 + - pkg: "k8s.io/client-go/applyconfigurations/apps/v1" + alias: appsv1 + - pkg: "k8s.io/client-go/applyconfigurations/core/v1" + alias: corev1 + - pkg: "k8s.io/client-go/applyconfigurations/meta/v1" + alias: metav1 + + lll: + # Max line length, lines longer will be reported. + # '\t' is counted as 1 character by default, and can be changed with the tab-width option. + # Default: 120. + line-length: 120 + # Tab width in spaces. + # Default: 1 + tab-width: 4 + + misspell: + # Correct spellings using locale preferences for US or UK. + # Setting locale to US will correct the British spelling of 'colour' to 'color'. + # Default is to use a neutral variety of English. + locale: US + # Typos to ignore. + # Should be in lower case. + # Default: [] + # ignore-words: + # - someword + # Extra word corrections. + # `typo` and `correction` should only contain letters. + # The words are case-insensitive. + # Default: [] + extra-words: + - typo: "iff" + correction: "if" + - typo: "cancelation" + correction: "cancellation" + # Mode of the analysis: + # - default: checks all the file content. + # - restricted: checks only comments. + # Default: "" + mode: restricted + + nakedret: + # Make an issue if func has more lines of code than this setting, and it has naked returns. + # Default: 30 + max-func-lines: 30 + + nestif: + # Minimal complexity of if statements to report. + # Default: 5 + min-complexity: 15 + + perfsprint: + # Optimizes even if it requires an int or uint type cast. + # Default: true + # int-conversion: false + # Optimizes into `err.Error()` even if it is only equivalent for non-nil errors. + # Default: false + err-error: true + # Optimizes `fmt.Errorf`. + # Default: true + # errorf: false + # Optimizes `fmt.Sprintf` with only one argument. + # Default: true + # sprintf1: false + # Optimizes into strings concatenation. + # Default: true + strconcat: false + + revive: + # Maximum number of open files at the same time. + # See https://github.com/mgechev/revive#command-line-flags + # Defaults to unlimited. + max-open-files: 2048 + + # When set to false, ignores files with "GENERATED" header, similar to golint. + # See https://github.com/mgechev/revive#available-rules for details. + # Default: false + ignore-generated-header: true + + # Sets the default severity. + # See https://github.com/mgechev/revive#configuration + # Default: warning + # severity: error + + # Enable all available rules. + # Default: false + # enable-all-rules: true + + # Sets the default failure confidence. + # This means that linting errors with less than 0.8 confidence will be ignored. + # Default: 0.8 + # confidence: 0.8 + + spancheck: + # Checks to enable. + # Options include: + # - `end`: check that `span.End()` is called + # - `record-error`: check that `span.RecordError(err)` is called when an error is returned + # - `set-status`: check that `span.SetStatus(codes.Error, msg)` is called when an error is returned + # Default: ["end"] + checks: + - end + - record-error + - set-status + # A list of regexes for function signatures that silence `record-error` and `set-status` reports + # if found in the call path to a returned error. + # https://github.com/jjti/go-spancheck#ignore-check-signatures + # Default: [] + ignore-check-signatures: + - "telemetry.RecordError" + + staticcheck: + # SAxxxx checks in https://staticcheck.io/docs/configuration/options/#checks + # Default: ["*"] + checks: ["all"] + + testifylint: + # Enable all checkers (https://github.com/Antonboom/testifylint#checkers). + # Default: false + enable-all: true + # Disable checkers by name + # (in addition to default + # suite-thelper + # ). + # disable: + # - blank-import + # - bool-compare + # - compares + # - empty + # - error-is-as + # - error-nil + # - expected-actual + # - go-require + # - float-compare + # - len + # - nil-compare + # - require-error + # - suite-dont-use-pkg + # - suite-extra-assert-call + # - suite-thelper + # - useless-assert + + # Disable all checkers (https://github.com/Antonboom/testifylint#checkers). + # Default: false + # disable-all: true + # Enable checkers by name + # (in addition to default + # blank-import, bool-compare, compares, empty, error-is-as, error-nil, expected-actual, go-require, float-compare, + # len, nil-compare, require-error, suite-dont-use-pkg, suite-extra-assert-call, useless-assert + # ). + # enable: + # - blank-import + # - bool-compare + # - compares + # - empty + # - error-is-as + # - error-nil + # - expected-actual + # - go-require + # - float-compare + # - len + # - nil-compare + # - require-error + # - suite-dont-use-pkg + # - suite-extra-assert-call + # - suite-thelper + # - useless-assert + + unused: + exported-fields-are-used: false + local-variables-are-used: false + +issues: + # List of regexps of issue texts to exclude. + # + # But independently of this option we use default exclude patterns, + # it can be disabled by `exclude-use-default: false`. + # To list all excluded by default patterns execute `golangci-lint run --help` + # + # Default: https://golangci-lint.run/usage/false-positives/#default-exclusions + # exclude: + # - abcdef + + # Excluding configuration per-path, per-linter, per-text and per-source + # exclude-rules: + + # Independently of option `exclude` we use default exclude patterns, + # it can be disabled by this option. + # To list all excluded by default patterns execute `golangci-lint run --help`. + # Default: true + exclude-use-default: false + + # If set to true, `exclude` and `exclude-rules` regular expressions become case-sensitive. + # Default: false + exclude-case-sensitive: false + + # Which dirs to exclude: issues from them won't be reported. + # Can use regexp here: `generated.*`, regexp is applied on full path, + # including the path prefix if one is set. + # Default dirs are skipped independently of this option's value (see exclude-dirs-use-default). + # "/" will be replaced by current OS file path separator to properly work on Windows. + # Default: [] + # exclude-dirs: + # - src/external_libs + # - autogenerated_by_my_lib + + # Enables exclude of directories: + # - vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + # Default: true + exclude-dirs-use-default: false + + # Which files to exclude: they will be analyzed, but issues from them won't be reported. + # There is no need to include all autogenerated files, + # we confidently recognize autogenerated files. + # If it's not, please let us know. + # "/" will be replaced by current OS file path separator to properly work on Windows. + # Default: [] + exclude-files: + - "^mock_*\\.go$" + + # To follow strictly the Go generated file convention. + # + # If set to true, source files that have lines matching only the following regular expression will be excluded: + # `^// Code generated .* DO NOT EDIT\.$` + # This line must appear before the first non-comment, non-blank text in the file. + # https://go.dev/s/generatedcode + # + # By default, a lax pattern is applied: + # sources are excluded if they contain lines `autogenerated file`, `code generated`, `do not edit`, etc. + # Default: false + # exclude-generated-strict: true + + # The list of ids of default excludes to include or disable. + # https://golangci-lint.run/usage/false-positives/#default-exclusions + # Default: [] + # include: + # - EXC0001 + # - EXC0002 + # - EXC0003 + # - EXC0004 + # - EXC0005 + # - EXC0006 + # - EXC0007 + # - EXC0008 + # - EXC0009 + # - EXC0010 + # - EXC0011 + # - EXC0012 + # - EXC0013 + # - EXC0014 + # - EXC0015 + + # Maximum issues count per one linter. + # Set to 0 to disable. + # Default: 50 + max-issues-per-linter: 0 + + # Maximum count of issues with the same text. + # Set to 0 to disable. + # Default: 3 + max-same-issues: 0 + + # Show only new issues: if there are unstaged changes or untracked files, + # only those changes are analyzed, else only changes in HEAD~ are analyzed. + # It's a super-useful option for integration of golangci-lint into existing large codebase. + # It's not practical to fix all existing issues at the moment of integration: + # much better don't allow issues in new code. + # + # Default: false + # new: true + + # Show only new issues created after git revision `REV`. + # Default: "" + # new-from-rev: HEAD + + # Show only new issues created in git patch with set file path. + # Default: "" + # new-from-patch: path/to/patch/file + + # Fix found issues (if it's supported by the linter). + # Default: false + # fix: true + + # Show issues in any part of update files (requires new-from-rev or new-from-patch). + # Default: false + whole-files: true + +severity: + # Set the default severity for issues. + # + # If severity rules are defined and the issues do not match or no severity is provided to the rule + # this will be the default severity applied. + # Severities should match the supported severity names of the selected out format. + # - Code climate: https://docs.codeclimate.com/docs/issues#issue-severity + # - Checkstyle: https://checkstyle.sourceforge.io/property_types.html#SeverityLevel + # - GitHub: https://help.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-error-message + # - TeamCity: https://www.jetbrains.com/help/teamcity/service-messages.html#Inspection+Instance + # + # `@linter` can be used as severity value to keep the severity from linters (e.g. revive, gosec, ...) + # + # Default: "" + default-severity: error + + # If set to true `severity-rules` regular expressions become case-sensitive. + # Default: false + case-sensitive: true + + # When a list of severity rules are provided, severity information will be added to lint issues. + # Severity rules have the same filtering capability as exclude rules + # except you are allowed to specify one matcher per severity rule. + # + # `@linter` can be used as severity value to keep the severity from linters (e.g. revive, gosec, ...) + # + # Only affects out formats that support setting severity information. + # + # Default: [] + # rules: + # - linters: + # - dupl + # severity: info diff --git a/Dockerfile.api b/Dockerfile similarity index 87% rename from Dockerfile.api rename to Dockerfile index 57514e3..3830616 100644 --- a/Dockerfile.api +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22.3-alpine AS build +FROM golang:1.23.0-alpine AS build WORKDIR /src COPY . . RUN apk add --no-cache gcc musl-dev diff --git a/Dockerfile.apiserver b/Dockerfile.apiserver deleted file mode 100644 index d80b93a..0000000 --- a/Dockerfile.apiserver +++ /dev/null @@ -1,10 +0,0 @@ -FROM golang:1.22.3-alpine AS build -WORKDIR /src -COPY . . -RUN apk add --no-cache gcc musl-dev -RUN go build -o explorer-api ./cmd/apiserver/ - -FROM alpine:3.17 -COPY --from=build /src/explorer-api /bin/ -EXPOSE 5000 -ENTRYPOINT ["/bin/explorer-api"] diff --git a/Dockerfile.collector b/Dockerfile.collector deleted file mode 100644 index b96205b..0000000 --- a/Dockerfile.collector +++ /dev/null @@ -1,9 +0,0 @@ -FROM golang:1.22.3-alpine AS build -WORKDIR /src -COPY . . -RUN apk add --no-cache gcc musl-dev -RUN go build -o explorer-collector ./cmd/collector/ - -FROM alpine:3.17 -COPY --from=build /src/explorer-collector /bin/ -ENTRYPOINT ["/bin/explorer-collector"] diff --git a/Makefile b/Makefile index 0f0a3dd..8e3eb92 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -VERSION := 0.1.0 +VERSION ?= $(shell git describe --tags) COMMIT = $(shell git rev-parse HEAD) SHA = $(shell git rev-parse --short HEAD) CURR_DIR = $(shell pwd) @@ -7,7 +7,9 @@ BIN_DIR = $(CURR_DIR)/build BIN_DIR_WIN = $(CURR_DIR_WIN)/build export GO111MODULE = on -BRANCH := $(shell bash -c 'if [ "$$TRAVIS_PULL_REQUEST" == "false" ]; then echo $$TRAVIS_BRANCH; else echo $$TRAVIS_PULL_REQUEST_BRANCH; fi') +BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) + +GOLANGCI_LINT_VERSION := v1.59.0 # Set BRANCH when running make manually ifeq ($(BRANCH),) @@ -16,58 +18,33 @@ endif # Setup the -ldflags option to pass vars defined here to app vars LDFLAGS = -ldflags "-X main.version=${VERSION} -X main.commit=${COMMIT} -X main.branch=${BRANCH}" - PKGS = $(shell go list ./...) PLATFORMS := windows linux darwin os = $(word 1, $@) +.PHONY: install +install: + go mod download + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s $(GOLANGCI_LINT_VERSION) + .PHONY: all all: -.PHONY: apiserver -apiserver: -ifeq ($(OS),Windows_NT) - cd cmd/apiserver ; go build -o $(BIN_DIR_WIN)/apiserver.exe; cd .. -else - cd cmd/apiserver ; go build -o $(BIN_DIR)/apiserver; cd .. -endif - - -.PHONY: collector -collector: -ifeq ($(OS),Windows_NT) - cd cmd/collector ; go build -o $(BIN_DIR_WIN)/collector.exe; cd .. -else - cd cmd/collector ; go build -o $(BIN_DIR)/collector; cd .. -endif - .PHONY: lint-ci lint-ci: golangci-lint run ./... .PHONY: lint lint: - golangci-lint run --timeout=3m ./... + ./bin/golangci-lint run --config .golangci.yml .PHONY: lint-fix lint-fix: - golangci-lint run --fix - -.PHONY: test_collector -test_collector: - go test ./collector/... - -.PHONY: test_api -test_api: - go test ./internal/api/... - -.PHONY: test_pkg -test_pkg: - go test ./pkg/... + ./bin/golangci-lint run --config .golangci.yml --fix .PHONY: test -test: vet lint test_api test_collector test_pkg +test: vet lint .PHONY: vet vet: diff --git a/api/api.go b/api/api.go index 43e0a22..03a5c6b 100644 --- a/api/api.go +++ b/api/api.go @@ -3,24 +3,28 @@ package api import ( "context" "fmt" + "os" + "os/signal" + "syscall" + "time" + "github.com/eko/gocache/lib/v4/marshaler" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" "github.com/spacemeshos/explorer-backend/api/handler" "github.com/spacemeshos/explorer-backend/api/storage" + "github.com/spacemeshos/go-spacemesh/log" "github.com/spacemeshos/go-spacemesh/sql" - "os" - "os/signal" - "syscall" - "time" ) type Api struct { Echo *echo.Echo } -func Init(db *sql.Database, dbClient storage.DatabaseClient, allowedOrigins []string, debug bool, layersPerEpoch int64, marshaler *marshaler.Marshaler, routes func(e *echo.Echo)) *Api { +func Init(db *sql.Database, dbClient storage.DatabaseClient, allowedOrigins []string, + debug bool, layersPerEpoch int64, marshaler *marshaler.Marshaler, routes func(e *echo.Echo), +) *Api { e := echo.New() e.Use(middleware.Recover()) e.Use(func(next echo.HandlerFunc) echo.HandlerFunc { diff --git a/api/cache/cache.go b/api/cache/cache.go index 75d9840..2873762 100644 --- a/api/cache/cache.go +++ b/api/cache/cache.go @@ -1,6 +1,8 @@ package cache import ( + "time" + "github.com/eko/gocache/lib/v4/cache" "github.com/eko/gocache/lib/v4/marshaler" "github.com/eko/gocache/lib/v4/store" @@ -8,13 +10,15 @@ import ( redis_store "github.com/eko/gocache/store/redis/v4" gocache "github.com/patrickmn/go-cache" "github.com/redis/go-redis/v9" + "github.com/spacemeshos/go-spacemesh/log" - "time" ) -var RedisAddress = "" -var Expiration time.Duration = 0 -var ShortExpiration = 5 * time.Minute +var ( + RedisAddress = "" + Expiration time.Duration = 0 + ShortExpiration = 5 * time.Minute +) func New() *marshaler.Marshaler { var manager *cache.Cache[any] diff --git a/api/handler/account.go b/api/handler/account.go index e51ec05..9cb6afd 100644 --- a/api/handler/account.go +++ b/api/handler/account.go @@ -2,13 +2,15 @@ package handler import ( "context" + "net/http" + "github.com/eko/gocache/lib/v4/store" "github.com/labstack/echo/v4" "github.com/spacemeshos/explorer-backend/api/cache" "github.com/spacemeshos/explorer-backend/api/storage" + "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/log" - "net/http" ) func Account(c echo.Context) error { diff --git a/api/handler/circulation.go b/api/handler/circulation.go index eb116f6..3b3f0a6 100644 --- a/api/handler/circulation.go +++ b/api/handler/circulation.go @@ -2,10 +2,12 @@ package handler import ( "context" + "net/http" + "github.com/labstack/echo/v4" "github.com/spacemeshos/explorer-backend/api/storage" + "github.com/spacemeshos/go-spacemesh/log" - "net/http" ) func Circulation(c echo.Context) error { diff --git a/api/handler/epoch.go b/api/handler/epoch.go index 702e403..b179cbe 100644 --- a/api/handler/epoch.go +++ b/api/handler/epoch.go @@ -2,11 +2,13 @@ package handler import ( "context" + "net/http" + "strconv" + "github.com/labstack/echo/v4" "github.com/spacemeshos/explorer-backend/api/storage" + "github.com/spacemeshos/go-spacemesh/log" - "net/http" - "strconv" ) func Epoch(c echo.Context) error { diff --git a/api/handler/handler.go b/api/handler/handler.go index 915d4e3..0fc2cf0 100644 --- a/api/handler/handler.go +++ b/api/handler/handler.go @@ -1,11 +1,13 @@ package handler import ( + "strconv" + "github.com/eko/gocache/lib/v4/marshaler" "github.com/labstack/echo/v4" "github.com/spacemeshos/explorer-backend/api/storage" + "github.com/spacemeshos/go-spacemesh/sql" - "strconv" ) type ApiContext struct { diff --git a/api/handler/layer.go b/api/handler/layer.go index 92b3111..4f1d9d0 100644 --- a/api/handler/layer.go +++ b/api/handler/layer.go @@ -2,13 +2,15 @@ package handler import ( "context" + "net/http" + "strconv" + "github.com/eko/gocache/lib/v4/store" "github.com/labstack/echo/v4" "github.com/spacemeshos/explorer-backend/api/cache" "github.com/spacemeshos/explorer-backend/api/storage" + "github.com/spacemeshos/go-spacemesh/log" - "net/http" - "strconv" ) func Layer(c echo.Context) error { diff --git a/api/handler/overview.go b/api/handler/overview.go index 3651ed2..fb9ad83 100644 --- a/api/handler/overview.go +++ b/api/handler/overview.go @@ -2,10 +2,12 @@ package handler import ( "context" + "net/http" + "github.com/labstack/echo/v4" "github.com/spacemeshos/explorer-backend/api/storage" + "github.com/spacemeshos/go-spacemesh/log" - "net/http" ) func Overview(c echo.Context) error { diff --git a/api/handler/smesher.go b/api/handler/smesher.go index c7ef5d7..3683794 100644 --- a/api/handler/smesher.go +++ b/api/handler/smesher.go @@ -3,14 +3,16 @@ package handler import ( "context" "fmt" + "net/http" + "strconv" + "github.com/eko/gocache/lib/v4/store" "github.com/labstack/echo/v4" "github.com/spacemeshos/explorer-backend/api/cache" "github.com/spacemeshos/explorer-backend/api/storage" + "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/log" - "net/http" - "strconv" ) func Smeshers(c echo.Context) error { diff --git a/api/storage/circulation.go b/api/storage/circulation.go index e6b5892..a1e2df5 100644 --- a/api/storage/circulation.go +++ b/api/storage/circulation.go @@ -2,6 +2,7 @@ package storage import ( "github.com/spacemeshos/economics/vesting" + "github.com/spacemeshos/go-spacemesh/log" "github.com/spacemeshos/go-spacemesh/sql" ) diff --git a/api/storage/epoch.go b/api/storage/epoch.go index 58717cf..975236e 100644 --- a/api/storage/epoch.go +++ b/api/storage/epoch.go @@ -1,13 +1,15 @@ package storage import ( + "math" + "github.com/spacemeshos/explorer-backend/utils" + "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/log" "github.com/spacemeshos/go-spacemesh/sql" "github.com/spacemeshos/go-spacemesh/sql/atxs" "github.com/spacemeshos/go-spacemesh/sql/builder" - "math" ) type EpochStats struct { @@ -20,7 +22,7 @@ type EpochStats struct { Decentral uint64 `json:"decentral,omitempty"` } -func (c *Client) GetEpochStats(db *sql.Database, epoch int64, layersPerEpoch int64) (*EpochStats, error) { +func (c *Client) GetEpochStats(db *sql.Database, epoch, layersPerEpoch int64) (*EpochStats, error) { stats := &EpochStats{ TransactionsCount: 0, ActivationsCount: 0, @@ -142,5 +144,4 @@ func (c *Client) GetEpochDecentralRatio(db *sql.Database, epoch int64) (*EpochSt stats.Decentral = uint64(100.0 * (0.5*(a*a)/1e8 + 0.5*(1.0-utils.Gini(smeshers)))) return stats, nil - } diff --git a/api/storage/layer.go b/api/storage/layer.go index 2f6ba2f..907f0b1 100644 --- a/api/storage/layer.go +++ b/api/storage/layer.go @@ -74,5 +74,4 @@ func (c *Client) GetLayersCount(db *sql.Database) (count uint64, err error) { return true }) return - } diff --git a/api/storage/rewards.go b/api/storage/rewards.go index a3a508a..3932238 100644 --- a/api/storage/rewards.go +++ b/api/storage/rewards.go @@ -5,7 +5,7 @@ import ( "github.com/spacemeshos/go-spacemesh/sql" ) -func (c *Client) GetRewardsSum(db *sql.Database) (sum uint64, count uint64, err error) { +func (c *Client) GetRewardsSum(db *sql.Database) (sum, count uint64, err error) { _, err = db.Exec(`SELECT COUNT(*), SUM(total_reward) FROM rewards`, func(stmt *sql.Statement) { }, @@ -17,7 +17,7 @@ func (c *Client) GetRewardsSum(db *sql.Database) (sum uint64, count uint64, err return } -func (c *Client) GetRewardsSumByAddress(db *sql.Database, addr types.Address) (sum uint64, count uint64, err error) { +func (c *Client) GetRewardsSumByAddress(db *sql.Database, addr types.Address) (sum, count uint64, err error) { _, err = db.Exec(`SELECT COUNT(*), SUM(total_reward) FROM rewards WHERE coinbase = ?1`, func(stmt *sql.Statement) { stmt.BindBytes(1, addr.Bytes()) diff --git a/api/storage/smesher.go b/api/storage/smesher.go index 64a820f..02263d4 100644 --- a/api/storage/smesher.go +++ b/api/storage/smesher.go @@ -1,7 +1,8 @@ package storage import ( - "fmt" + "errors" + "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/sql" ) @@ -24,7 +25,8 @@ func (c *Client) GetSmeshers(db *sql.Database, limit, offset uint64) (*SmesherLi Smeshers: []Smesher{}, } - _, err := db.Exec(`SELECT pubkey, COUNT(*) as atxs FROM atxs GROUP BY pubkey ORDER BY pubkey ASC, epoch DESC LIMIT ?1 OFFSET ?2;`, + _, err := db.Exec(`SELECT pubkey, COUNT(*) as atxs FROM atxs + GROUP BY pubkey ORDER BY pubkey ASC, epoch DESC LIMIT ?1 OFFSET ?2;`, func(stmt *sql.Statement) { stmt.BindInt64(1, int64(limit)) stmt.BindInt64(2, int64(offset)) @@ -48,7 +50,9 @@ func (c *Client) GetSmeshersByEpoch(db *sql.Database, limit, offset, epoch uint6 Smeshers: []Smesher{}, } - _, err := db.Exec(`SELECT DISTINCT pubkey, COUNT(*) as atxs FROM atxs WHERE epoch = ?1 GROUP BY pubkey ORDER BY pubkey ASC, epoch DESC LIMIT ?2 OFFSET ?3;`, + _, err := db.Exec(`SELECT DISTINCT pubkey, COUNT(*) as atxs FROM atxs + WHERE epoch = ?1 GROUP BY pubkey + ORDER BY pubkey ASC, epoch DESC LIMIT ?2 OFFSET ?3;`, func(stmt *sql.Statement) { stmt.BindInt64(1, int64(epoch-1)) stmt.BindInt64(2, int64(limit)) @@ -92,7 +96,9 @@ func (c *Client) GetSmeshersByEpochCount(db *sql.Database, epoch uint64) (count } func (c *Client) GetSmesher(db *sql.Database, pubkey []byte) (smesher *Smesher, err error) { - _, err = db.Exec(`SELECT pubkey, coinbase, effective_num_units, COUNT(*) as atxs FROM atxs WHERE pubkey = ?1 GROUP BY pubkey ORDER BY epoch DESC LIMIT 1;`, + _, err = db.Exec(`SELECT pubkey, coinbase, effective_num_units, COUNT(*) as atxs FROM atxs + WHERE pubkey = ?1 GROUP BY pubkey + ORDER BY epoch DESC LIMIT 1;`, func(stmt *sql.Statement) { stmt.BindBytes(1, pubkey) }, @@ -107,10 +113,10 @@ func (c *Client) GetSmesher(db *sql.Database, pubkey []byte) (smesher *Smesher, return true }) if err != nil { - return + return smesher, err } if smesher == nil { - return nil, fmt.Errorf("smesher not found") + return nil, errors.New("smesher not found") } _, err = db.Exec(`SELECT COUNT(*), SUM(total_reward) FROM rewards WHERE pubkey=?1`, @@ -122,5 +128,5 @@ func (c *Client) GetSmesher(db *sql.Database, pubkey []byte) (smesher *Smesher, smesher.RewardsSum = uint64(stmt.ColumnInt64(1)) return true }) - return + return smesher, err } diff --git a/api/storage/storage.go b/api/storage/storage.go index a4818fc..a64c657 100644 --- a/api/storage/storage.go +++ b/api/storage/storage.go @@ -2,6 +2,7 @@ package storage import ( "fmt" + "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/sql" "github.com/spacemeshos/go-spacemesh/timesync" @@ -13,7 +14,7 @@ type DatabaseClient interface { GetLayerStats(db *sql.Database, lid int64) (*LayerStats, error) GetLayersCount(db *sql.Database) (uint64, error) - GetEpochStats(db *sql.Database, epoch int64, layersPerEpoch int64) (*EpochStats, error) + GetEpochStats(db *sql.Database, epoch, layersPerEpoch int64) (*EpochStats, error) GetEpochDecentralRatio(db *sql.Database, epoch int64) (*EpochStats, error) GetSmeshers(db *sql.Database, limit, offset uint64) (*SmesherList, error) @@ -27,7 +28,7 @@ type DatabaseClient interface { GetSmeshersByEpochCount(db *sql.Database, epoch uint64) (uint64, error) GetRewardsSum(db *sql.Database) (uint64, uint64, error) - GetRewardsSumByAddress(db *sql.Database, addr types.Address) (sum uint64, count uint64, err error) + GetRewardsSumByAddress(db *sql.Database, addr types.Address) (sum, count uint64, err error) GetTransactionsCount(db *sql.Database) (uint64, error) GetTotalNumUnits(db *sql.Database) (uint64, error) diff --git a/api/storage/transactions.go b/api/storage/transactions.go index 3f626f5..954ccb7 100644 --- a/api/storage/transactions.go +++ b/api/storage/transactions.go @@ -3,8 +3,10 @@ package storage import ( "bytes" "fmt" + spacemeshv2alpha1 "github.com/spacemeshos/api/release/go/spacemesh/v2alpha1" "github.com/spacemeshos/go-scale" + "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/genvm/core" "github.com/spacemeshos/go-spacemesh/genvm/registry" diff --git a/cmd/api/main.go b/cmd/api/main.go index 2952dd4..8765009 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -3,6 +3,11 @@ package main import ( "errors" "fmt" + "net/http" + "os" + "sync" + "time" + "github.com/labstack/echo-contrib/echoprometheus" "github.com/labstack/echo/v4" "github.com/spacemeshos/address" @@ -10,15 +15,12 @@ import ( "github.com/spacemeshos/explorer-backend/api/cache" "github.com/spacemeshos/explorer-backend/api/router" "github.com/spacemeshos/explorer-backend/api/storage" + "github.com/urfave/cli/v2" + "go.uber.org/zap" + "github.com/spacemeshos/go-spacemesh/common/types" "github.com/spacemeshos/go-spacemesh/log" "github.com/spacemeshos/go-spacemesh/timesync" - "github.com/urfave/cli/v2" - "go.uber.org/zap" - "net/http" - "os" - "sync" - "time" ) var ( diff --git a/cmd/apiserver/main.go b/cmd/apiserver/main.go deleted file mode 100644 index c6646cf..0000000 --- a/cmd/apiserver/main.go +++ /dev/null @@ -1,112 +0,0 @@ -package main - -import ( - "context" - "fmt" - "github.com/spacemeshos/address" - "github.com/spacemeshos/explorer-backend/internal/api" - appService "github.com/spacemeshos/explorer-backend/internal/service" - "github.com/spacemeshos/explorer-backend/internal/storage/storagereader" - "github.com/spacemeshos/go-spacemesh/log" - "github.com/urfave/cli/v2" - "os" - "time" -) - -var ( - version string - commit string - branch string -) - -var ( - listenStringFlag string - mongoDbURLStringFlag string - mongoDbNameStringFlag string - testnetBoolFlag bool - allowedOrigins = cli.NewStringSlice("*") - debug bool -) - -var flags = []cli.Flag{ - &cli.StringFlag{ - Name: "listen", - Usage: "Explorer API listen string in format :", - Required: false, - Destination: &listenStringFlag, - Value: ":5000", - EnvVars: []string{"SPACEMESH_API_LISTEN"}, - }, - &cli.StringFlag{ - Name: "mongodb", - Usage: "Explorer MongoDB Uri string in format mongodb://:", - Required: false, - Destination: &mongoDbURLStringFlag, - Value: "mongodb://localhost:27017", - EnvVars: []string{"SPACEMESH_MONGO_URI"}, - }, - &cli.StringFlag{ - Name: "db", - Usage: "MongoDB Explorer database name string", - Required: false, - Destination: &mongoDbNameStringFlag, - Value: "explorer", - EnvVars: []string{"SPACEMESH_MONGO_DB"}, - }, - &cli.BoolFlag{ - Name: "testnet", - Usage: `Use this flag to enable testnet preset ("stest" instead of "sm" for wallet addresses)`, - Required: false, - Destination: &testnetBoolFlag, - EnvVars: []string{"SPACEMESH_TESTNET"}, - }, - &cli.StringSliceFlag{ - Name: "allowed-origins", - Usage: `Use this flag to set allowed origins for CORS (default: "*")`, - Destination: allowedOrigins, - EnvVars: []string{"ALLOWED_ORIGINS"}, - }, - &cli.BoolFlag{ - Name: "debug", - Usage: "Use this flag to enable echo debug option along with logger middleware", - Required: false, - Destination: &debug, - EnvVars: []string{"DEBUG"}, - }, -} - -func main() { - app := cli.NewApp() - app.Name = "Spacemesh Explorer REST API Server" - app.Version = fmt.Sprintf("%s, commit '%s', branch '%s'", version, commit, branch) - app.Flags = flags - app.Writer = os.Stderr - - app.Action = func(ctx *cli.Context) error { - if testnetBoolFlag { - address.SetAddressConfig("stest") - log.Info(`network HRP set to "stest"`) - } - - dbReader, err := storagereader.NewStorageReader(context.Background(), mongoDbURLStringFlag, mongoDbNameStringFlag) - if err != nil { - return fmt.Errorf("error init storage reader: %w", err) - } - - service := appService.NewService(dbReader, time.Minute) - server := api.Init(service, allowedOrigins.Value(), debug) - - log.Info(fmt.Sprintf("starting server on %s", listenStringFlag)) - server.Run(listenStringFlag) - - log.Info("server is shutdown") - return nil - } - - if err := app.Run(os.Args); err != nil { - log.Info("%v", err) - os.Exit(1) - } - - os.Exit(0) -} diff --git a/cmd/collector/main.go b/cmd/collector/main.go deleted file mode 100644 index c7befa9..0000000 --- a/cmd/collector/main.go +++ /dev/null @@ -1,229 +0,0 @@ -package main - -import ( - "context" - "fmt" - "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/spacemeshos/address" - "github.com/spacemeshos/explorer-backend/collector" - "github.com/spacemeshos/explorer-backend/collector/sql" - "github.com/spacemeshos/explorer-backend/storage" - "github.com/spacemeshos/go-spacemesh/common/types" - "github.com/spacemeshos/go-spacemesh/log" - "github.com/urfave/cli/v2" - "net/http" - "os" - "os/signal" - "syscall" - "time" -) - -var ( - version string - commit string - branch string -) - -var ( - nodePublicAddressStringFlag string - nodePrivateAddressStringFlag string - mongoDbUrlStringFlag string - mongoDbNameStringFlag string - testnetBoolFlag bool - syncFromLayerFlag int - syncMissingLayersBoolFlag bool - sqlitePathStringFlag string - metricsPortFlag int - apiHostFlag string - apiPortFlag int - recalculateEpochStatsBoolFlag bool - atxSyncFlag bool -) - -var flags = []cli.Flag{ - &cli.StringFlag{ - Name: "node-public", - Usage: "Spacemesh public node API address string in format :", - Required: false, - Destination: &nodePublicAddressStringFlag, - Value: "localhost:9092", - EnvVars: []string{"SPACEMESH_NODE_PUBLIC"}, - }, - &cli.StringFlag{ - Name: "node-private", - Usage: "Spacemesh private node API address string in format :", - Required: false, - Destination: &nodePrivateAddressStringFlag, - Value: "localhost:9093", - EnvVars: []string{"SPACEMESH_NODE_PRIVATE"}, - }, - &cli.StringFlag{ - Name: "mongodb", - Usage: "Explorer MongoDB Uri string in format mongodb://:", - Required: false, - Destination: &mongoDbUrlStringFlag, - Value: "mongodb://localhost:27017", - EnvVars: []string{"SPACEMESH_MONGO_URI"}, - }, - &cli.StringFlag{ - Name: "db", - Usage: "MongoDB Explorer database name string", - Required: false, - Destination: &mongoDbNameStringFlag, - Value: "explorer", - EnvVars: []string{"SPACEMESH_MONGO_DB"}, - }, - &cli.BoolFlag{ - Name: "testnet", - Usage: `Use this flag to enable testnet preset ("stest" instead of "sm" for wallet addresses)`, - Required: false, - Destination: &testnetBoolFlag, - EnvVars: []string{"SPACEMESH_TESTNET"}, - }, - &cli.IntFlag{ - Name: "syncFromLayer", - Usage: ``, - Required: false, - Value: 0, - Destination: &syncFromLayerFlag, - EnvVars: []string{"SPACEMESH_SYNC_FROM_LAYER"}, - }, - &cli.BoolFlag{ - Name: "syncMissingLayers", - Usage: `Use this flag to disable missing layers sync`, - Required: false, - Destination: &syncMissingLayersBoolFlag, - Value: true, - EnvVars: []string{"SPACEMESH_SYNC_MISSING_LAYERS"}, - }, - &cli.StringFlag{ - Name: "sqlite", - Usage: "Path to node sqlite file", - Required: false, - Destination: &sqlitePathStringFlag, - Value: "explorer.sql", - EnvVars: []string{"SPACEMESH_SQLITE"}, - }, - &cli.IntFlag{ - Name: "metricsPort", - Usage: ``, - Required: false, - Value: 9090, - Destination: &metricsPortFlag, - EnvVars: []string{"SPACEMESH_METRICS_PORT"}, - }, - &cli.BoolFlag{ - Name: "recalculateEpochStats", - Usage: `Use this flag to recalculate epoch stats`, - Required: false, - Destination: &recalculateEpochStatsBoolFlag, - Value: false, - EnvVars: []string{"SPACEMESH_RECALCULATE_EPOCH_STATS"}, - }, - &cli.StringFlag{ - Name: "apiHost", - Usage: ``, - Required: false, - Value: "127.0.0.1", - Destination: &apiHostFlag, - EnvVars: []string{"SPACEMESH_API_HOST"}, - }, - &cli.IntFlag{ - Name: "apiPort", - Usage: ``, - Required: false, - Value: 8080, - Destination: &apiPortFlag, - EnvVars: []string{"SPACEMESH_API_PORT"}, - }, - &cli.BoolFlag{ - Name: "atxSync", - Usage: ``, - Required: false, - Value: true, - Destination: &atxSyncFlag, - EnvVars: []string{"SPACEMESH_ATX_SYNC"}, - }, -} - -func main() { - app := cli.NewApp() - app.Name = "Spacemesh Explorer Collector" - app.Version = fmt.Sprintf("%s, commit '%s', branch '%s'", version, commit, branch) - app.Flags = flags - app.Writer = os.Stderr - - app.Action = func(ctx *cli.Context) error { - var pidFile *os.File - - if testnetBoolFlag { - address.SetAddressConfig("stest") - types.SetNetworkHRP("stest") - log.Info(`Network HRP set to "stest"`) - } - - mongoStorage, err := storage.New(context.Background(), mongoDbUrlStringFlag, mongoDbNameStringFlag) - if err != nil { - log.Info("MongoDB storage open error %v", err) - return err - } - - db, err := sql.Setup(sqlitePathStringFlag) - if err != nil { - log.Info("SQLite storage open error %v", err) - return err - } - dbClient := &sql.Client{} - - c := collector.NewCollector(nodePublicAddressStringFlag, nodePrivateAddressStringFlag, - syncMissingLayersBoolFlag, syncFromLayerFlag, recalculateEpochStatsBoolFlag, mongoStorage, db, dbClient, atxSyncFlag) - mongoStorage.AccountUpdater = c - - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - - pidFile, err = os.OpenFile("/var/run/explorer-collector", os.O_RDWR|os.O_CREATE, 0644) - if err == nil { - _, err := pidFile.Write([]byte("started")) - if err != nil { - return err - } - err = pidFile.Close() - if err != nil { - return err - } - } - - go func() { - <-sigs - os.Remove("/var/run/explorer-collector") - os.Exit(0) - }() - - go func() { - for { - if err := c.Run(); err != nil { - fmt.Println(err) - time.Sleep(5 * time.Second) - } - } - }() - - go func() { - // expose metrics endpoint - http.Handle("/metrics", promhttp.Handler()) - http.ListenAndServe(fmt.Sprintf(":%d", metricsPortFlag), nil) - }() - - go c.StartHttpServer(apiHostFlag, apiPortFlag) - - select {} - } - - if err := app.Run(os.Args); err != nil { - log.Info("%+v", err) - os.Exit(1) - } - - os.Exit(0) -} diff --git a/collector/atxs_test.go b/collector/atxs_test.go deleted file mode 100644 index 9eb5ebe..0000000 --- a/collector/atxs_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package collector_test - -import ( - "context" - "encoding/json" - "strings" - "testing" - - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - - "github.com/spacemeshos/explorer-backend/model" -) - -func TestAtxs(t *testing.T) { - t.Parallel() - atxs, err := storageDB.GetActivations(context.TODO(), &bson.D{}) - require.NoError(t, err) - require.Equal(t, len(generator.Activations), len(atxs)) - for _, atx := range atxs { - // temporary hack until storage return data as slice of bson.B, not an struct. - atxEncoded, err := json.Marshal(atx.Map()) - require.NoError(t, err) - var tmpAtx model.Activation - require.NoError(t, json.Unmarshal(atxEncoded, &tmpAtx)) - atxGen, ok := generator.Activations[tmpAtx.Id] - require.True(t, ok) - tmpAtx.Coinbase = strings.ToLower(tmpAtx.Coinbase) - atxGen.Coinbase = strings.ToLower(atxGen.Coinbase) - require.Equal(t, *atxGen, tmpAtx) - } -} diff --git a/collector/blocks_test.go b/collector/blocks_test.go deleted file mode 100644 index 56b7f23..0000000 --- a/collector/blocks_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package collector_test - -import ( - "context" - "encoding/json" - "testing" - - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - - "github.com/spacemeshos/explorer-backend/model" -) - -func TestBlocks(t *testing.T) { - t.Parallel() - blocks, err := storageDB.GetBlocks(context.TODO(), &bson.D{}) - require.NoError(t, err) - for _, block := range blocks { - // temporary hack, until storage return data as slice of bson.B not an struct. - blockEncoded, err := json.Marshal(block.Map()) - require.NoError(t, err) - var tmpBlock model.Block - require.NoError(t, json.Unmarshal(blockEncoded, &tmpBlock)) - generated := generator.Blocks[tmpBlock.Id] - require.NotNil(t, generated) - require.Equal(t, *generated, tmpBlock) - } -} diff --git a/collector/collector.go b/collector/collector.go deleted file mode 100644 index 4fff178..0000000 --- a/collector/collector.go +++ /dev/null @@ -1,218 +0,0 @@ -package collector - -import ( - "context" - "errors" - "github.com/spacemeshos/explorer-backend/collector/sql" - "github.com/spacemeshos/explorer-backend/model" - "github.com/spacemeshos/go-spacemesh/common/types" - sql2 "github.com/spacemeshos/go-spacemesh/sql" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - "golang.org/x/sync/errgroup" - "google.golang.org/grpc/keepalive" - "time" - - "google.golang.org/grpc/credentials/insecure" - - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" - "google.golang.org/grpc" - - "github.com/spacemeshos/go-spacemesh/log" -) - -const ( - streamType_node_SyncStatus int = 1 - //streamType_mesh_Layer int = 2 - streamType_transactions int = 2 - streamType_mesh_Malfeasance int = 3 - - streamType_count int = 3 -) - -type Listener interface { - OnNetworkInfo(genesisId string, genesisTime uint64, epochNumLayers uint32, maxTransactionsPerSecond uint64, layerDuration uint64, postUnitSize uint64) - OnNodeStatus(connectedPeers uint64, isSynced bool, syncedLayer uint32, topLayer uint32, verifiedLayer uint32) - OnLayer(layer *pb.Layer) - OnAccounts(accounts []*types.Account) - OnReward(reward *pb.Reward) - OnMalfeasanceProof(proof *pb.MalfeasanceProof) - OnTransactionResult(res *pb.TransactionResult, state *pb.TransactionState) - GetLastLayer(parent context.Context) uint32 - LayersInQueue() int - IsLayerInQueue(layer *pb.Layer) bool - GetEpochNumLayers() uint32 - GetTransactions(parent context.Context, query *bson.D, opts ...*options.FindOptions) ([]model.Transaction, error) - UpdateTransactionState(parent context.Context, id string, state int32) error - UpdateEpochStats(layer uint32) - OnActivation(atx *types.ActivationTx) - GetLastActivationReceived() int64 - RecalculateEpochStats() - OnActivations(atxs []*model.Activation) -} - -type Collector struct { - apiPublicUrl string - apiPrivateUrl string - syncMissingLayersFlag bool - recalculateEpochStatsFlag bool - syncFromLayerFlag uint32 - atxSyncFlag bool - - listener Listener - db *sql2.Database - dbClient sql.DatabaseClient - - nodeClient pb.NodeServiceClient - meshClient pb.MeshServiceClient - globalClient pb.GlobalStateServiceClient - transactionsClient pb.TransactionServiceClient - debugClient pb.DebugServiceClient - smesherClient pb.SmesherServiceClient - - streams [streamType_count]bool - activeStreams int - connecting bool - online bool - closing bool - - // Stream status changed. - notify chan int -} - -func NewCollector(nodePublicAddress string, nodePrivateAddress string, syncMissingLayersFlag bool, - syncFromLayerFlag int, recalculateEpochStatsFlag bool, - listener Listener, db *sql2.Database, dbClient sql.DatabaseClient, atxSyncFlag bool) *Collector { - return &Collector{ - apiPublicUrl: nodePublicAddress, - apiPrivateUrl: nodePrivateAddress, - syncMissingLayersFlag: syncMissingLayersFlag, - recalculateEpochStatsFlag: recalculateEpochStatsFlag, - syncFromLayerFlag: uint32(syncFromLayerFlag), - listener: listener, - notify: make(chan int), - db: db, - dbClient: dbClient, - atxSyncFlag: atxSyncFlag, - } -} - -func (c *Collector) Run() error { - log.Info("dial node %v and %v", c.apiPublicUrl, c.apiPrivateUrl) - c.connecting = true - - //TODO: move to env - keepaliveOpts := keepalive.ClientParameters{ - Time: 4 * time.Minute, - Timeout: 2 * time.Minute, - PermitWithoutStream: true, - } - - publicConn, err := grpc.Dial(c.apiPublicUrl, grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithKeepaliveParams(keepaliveOpts), - grpc.WithBlock(), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(50*1024*1024))) - if err != nil { - return errors.Join(errors.New("cannot dial node"), err) - } - defer publicConn.Close() - - privateConn, err := grpc.Dial(c.apiPrivateUrl, grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithKeepaliveParams(keepaliveOpts), - grpc.WithBlock(), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(50*1024*1024))) - if err != nil { - return errors.Join(errors.New("cannot dial node"), err) - } - defer privateConn.Close() - - c.nodeClient = pb.NewNodeServiceClient(publicConn) - c.meshClient = pb.NewMeshServiceClient(publicConn) - c.globalClient = pb.NewGlobalStateServiceClient(publicConn) - c.transactionsClient = pb.NewTransactionServiceClient(publicConn) - c.debugClient = pb.NewDebugServiceClient(publicConn) - c.smesherClient = pb.NewSmesherServiceClient(privateConn) - - err = c.getNetworkInfo() - if err != nil { - return errors.Join(errors.New("cannot get network info"), err) - } - - if c.atxSyncFlag { - err = c.syncActivations() - if err != nil { - return errors.Join(errors.New("cannot sync activations"), err) - } - } - - if c.syncMissingLayersFlag { - err = c.syncMissingLayers() - if err != nil { - return errors.Join(errors.New("cannot sync missing layers"), err) - } - } - - if c.recalculateEpochStatsFlag { - c.listener.RecalculateEpochStats() - } - - g := new(errgroup.Group) - g.Go(func() error { - err := c.syncStatusPump() - if err != nil { - return errors.Join(errors.New("cannot start sync status pump"), err) - } - return nil - }) - - g.Go(func() error { - err := c.transactionsPump() - if err != nil { - return errors.Join(errors.New("cannot start transactions pump"), err) - } - return nil - }) - - g.Go(func() error { - err := c.malfeasancePump() - if err != nil { - return errors.Join(errors.New("cannot start sync malfeasance pump"), err) - } - return nil - }) - - g.Go(func() error { - for c.connecting || c.closing || c.online { - state := <-c.notify - log.Info("stream notify %v", state) - switch { - case state > 0: - c.streams[state-1] = true - c.activeStreams++ - log.Info("stream connected %v", state) - case state < 0: - c.streams[(-state)-1] = false - c.activeStreams-- - if c.activeStreams == 0 { - c.closing = false - } - log.Info("stream disconnected %v", state) - } - if c.activeStreams == streamType_count { - c.connecting = false - c.online = true - log.Info("all streams synchronized!") - } - if c.online && c.activeStreams < streamType_count { - log.Info("streams desynchronized!!!") - c.online = false - c.closing = true - } - } - return nil - }) - - if err := g.Wait(); err != nil { - return err - } - - return nil -} diff --git a/collector/collector_test.go b/collector/collector_test.go deleted file mode 100644 index 734d9f0..0000000 --- a/collector/collector_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package collector_test - -import ( - "context" - "fmt" - "github.com/spacemeshos/explorer-backend/collector" - "github.com/spacemeshos/go-spacemesh/sql" - "os" - "testing" - "time" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/storage" - "github.com/spacemeshos/explorer-backend/test/testseed" - "github.com/spacemeshos/explorer-backend/test/testserver" -) - -const testAPIServiceDB = "explorer_test" - -var ( - dbPort = 27017 - generator *testseed.SeedGenerator - node *testserver.FakeNode - collectorApp *collector.Collector - storageDB *storage.Storage -) - -func TestMain(m *testing.M) { - mongoURL := fmt.Sprintf("mongodb://localhost:%d", dbPort) - client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(mongoURL)) - if err != nil { - fmt.Println("failed to connect to mongo", err) - os.Exit(1) - } - - database := client.Database(testAPIServiceDB) - if database != nil { - if err = database.Drop(context.TODO()); err != nil { - fmt.Println("failed to drop db", err) - os.Exit(1) - } - } - - storageDB, err = storage.New(context.TODO(), mongoURL, testAPIServiceDB) - if err != nil { - fmt.Println("failed to init storage to mongo", err) - os.Exit(1) - } - - sqlDb, err := sql.Open("file:test.db?cache=shared&mode=memory", sql.WithConnections(16), sql.WithMigrations(nil)) - seed := testseed.GetServerSeed() - generator = testseed.NewSeedGenerator(seed) - if err = generator.GenerateEpoches(10); err != nil { - fmt.Println("failed to generate epochs", err) - os.Exit(1) - } - - dbClient := &testseed.Client{SeedGen: generator} - - node, err = testserver.CreateFakeSMNode(generator.FirstLayerTime, generator, seed) - if err != nil { - fmt.Println("failed to generate fake node", err) - os.Exit(1) - } - defer node.Stop() - go func() { - if err = node.Start(); err != nil { - fmt.Println("failed to start fake node", err) - os.Exit(1) - } - }() - - privateNode, err := testserver.CreateFakeSMPrivateNode(generator.FirstLayerTime, generator, seed) - if err != nil { - fmt.Println("failed to generate fake private node", err) - os.Exit(1) - } - defer privateNode.Stop() - go func() { - if err = privateNode.Start(); err != nil { - fmt.Println("failed to start private fake node", err) - os.Exit(1) - } - }() - - collectorApp = collector.NewCollector(fmt.Sprintf("localhost:%d", node.NodePort), - fmt.Sprintf("localhost:%d", privateNode.NodePort), false, - 0, false, storageDB, sqlDb, dbClient, true) - storageDB.AccountUpdater = collectorApp - defer storageDB.Close() - go collectorApp.Run() - - ticker := time.NewTicker(1 * time.Second) - defer ticker.Stop() - for range ticker.C { - num := storageDB.GetRewardsCount(context.TODO(), &bson.D{}) - if int(num) == len(generator.Rewards) { - break - } - } - println("init done, start collector tests") - code := m.Run() - os.Exit(code) -} diff --git a/collector/epochs_test.go b/collector/epochs_test.go deleted file mode 100644 index 218b70e..0000000 --- a/collector/epochs_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package collector_test - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - - "github.com/spacemeshos/explorer-backend/model" -) - -func TestEpochs(t *testing.T) { - t.Parallel() - epochs, err := storageDB.GetEpochs(context.TODO(), &bson.D{}) - require.NoError(t, err) - require.Equal(t, len(generator.Epochs)+1, len(epochs)) - data := make(map[int32]*model.Epoch) - for _, epoch := range generator.Epochs { - data[epoch.Epoch.Number] = &epoch.Epoch - } - for _, epoch := range epochs[:len(epochs)-1] { - // temporary hack, until storage return data as slice of bson.B, not an struct. - epochMap := epoch.Map() - generatedEpoch, ok := data[epochMap["number"].(int32)] - require.True(t, ok) - - require.Equal(t, int64(generatedEpoch.LayerStart), epochMap["layerstart"].(int64)) - require.Equal(t, int64(generatedEpoch.LayerEnd), epochMap["layerend"].(int64)) - require.Equal(t, int64(generatedEpoch.Layers), epochMap["layers"].(int64)) - require.Equal(t, int64(generatedEpoch.Start), epochMap["start"].(int64)) - require.Equal(t, int64(generatedEpoch.End), epochMap["end"].(int64)) - - // todo check stats - println("epoch num", generatedEpoch.Number) - for k, values := range epochMap["stats"].(primitive.D).Map() { - v := values.(primitive.D).Map() - if k == "current" { - require.Equal(t, generatedEpoch.Stats.Current.Transactions, v["transactions"].(int64)) - require.Equal(t, generatedEpoch.Stats.Current.TxsAmount, v["txsamount"].(int64)) - require.Equal(t, generatedEpoch.Stats.Current.Smeshers, v["smeshers"].(int64)) - // TODO: should be fixed, cause current accounts count is not correct - //require.Equal(t, generatedEpoch.Stats.Current.Accounts, v["accounts"].(int64)) - //require.Equalf(t, generatedEpoch.Stats.Current.RewardsNumber, v["rewardsnumber"].(int64), "rewards number not equal") - //require.Equal(t, generatedEpoch.Stats.Current.Rewards, v["rewards"].(int64), "rewards sum mismatch") - require.Equal(t, generatedEpoch.Stats.Current.Security, v["security"].(int64)) - require.Equal(t, generatedEpoch.Stats.Current.Capacity, v["capacity"].(int64)) - //require.Equal(t, generatedEpoch.Stats.Current.Circulation, v["circulation"].(int64), "circulation sum mismatch") - - // todo should be fixed, cause current stat calc not correct get data about commitmentSize from db - // require.Equal(t, generatedEpoch.Stats.Current.Decentral, v["decentral"].(int64), "decentral sum mismatch") - } else if k == "cumulative" { - // t.Skip("todo test cumulative stats") - require.Equal(t, generatedEpoch.Stats.Cumulative.Transactions, v["transactions"].(int64)) - require.Equal(t, generatedEpoch.Stats.Cumulative.TxsAmount, v["txsamount"].(int64)) - require.Equal(t, generatedEpoch.Stats.Cumulative.Smeshers, v["smeshers"].(int64)) - // TODO: should be fixed, cause current accounts count is not correct - //require.Equal(t, generatedEpoch.Stats.Cumulative.Accounts, v["accounts"].(int64)) - //require.Equalf(t, generatedEpoch.Stats.Cumulative.RewardsNumber, v["rewardsnumber"].(int64), "rewards number not equal") - //require.Equal(t, generatedEpoch.Stats.Cumulative.Rewards, v["rewards"].(int64), "rewards sum mismatch") - require.Equal(t, generatedEpoch.Stats.Cumulative.Security, v["security"].(int64)) - require.Equal(t, generatedEpoch.Stats.Cumulative.Capacity, v["capacity"].(int64)) - //require.Equal(t, generatedEpoch.Stats.Cumulative.Circulation, v["circulation"].(int64), "circulation sum mismatch") - - // todo should be fixed, cause current stat calc not correct get data about commitmentSize from db - // require.Equal(t, generatedEpoch.Stats.Cumulative.Decentral, v["decentral"].(int64), "decentral sum mismatch") - } - } - } -} diff --git a/collector/global.go b/collector/global.go deleted file mode 100644 index 67227e1..0000000 --- a/collector/global.go +++ /dev/null @@ -1,31 +0,0 @@ -package collector - -import ( - "context" - "errors" - "fmt" - "time" - - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" - "github.com/spacemeshos/go-spacemesh/log" -) - -func (c *Collector) GetAccountState(address string) (uint64, uint64, error) { - req := &pb.AccountRequest{AccountId: &pb.AccountId{Address: address}} - - // set timeout - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - res, err := c.globalClient.Account(ctx, req) - if err != nil { - log.Err(fmt.Errorf("cannot get account info: %v", err)) - return 0, 0, err - } - - if res.AccountWrapper == nil || res.AccountWrapper.StateCurrent == nil || res.AccountWrapper.StateCurrent.Balance == nil { - return 0, 0, errors.New("Bad result") - } - - return res.AccountWrapper.StateCurrent.Balance.Value, res.AccountWrapper.StateCurrent.Counter, nil -} diff --git a/collector/http.go b/collector/http.go deleted file mode 100644 index 0976b64..0000000 --- a/collector/http.go +++ /dev/null @@ -1,199 +0,0 @@ -package collector - -import ( - "fmt" - "github.com/labstack/echo/v4" - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" - "github.com/spacemeshos/explorer-backend/model" - "github.com/spacemeshos/go-spacemesh/common/types" - "github.com/spacemeshos/go-spacemesh/log" - "net/http" - "strconv" -) - -func (c *Collector) StartHttpServer(apiHost string, apiPort int) { - e := echo.New() - - e.GET("/sync/atx/:id", func(ctx echo.Context) error { - id := ctx.Param("id") - - log.Info("http syncing atx %s", id) - go func() { - atx, err := c.dbClient.GetAtxById(c.db, id) - if err != nil { - log.Warning("syncing atx %s failed with error %d", id, err) - return - } - if atx != nil { - c.listener.OnActivation(atx) - c.listener.RecalculateEpochStats() - } - }() - - return ctx.NoContent(http.StatusOK) - }) - - e.GET("/sync/atxs/ts/:ts", func(ctx echo.Context) error { - ts := ctx.Param("ts") - timestamp, err := strconv.ParseInt(ts, 10, 64) - if err != nil { - return ctx.String(http.StatusBadRequest, "Invalid parameter") - } - - log.Info("http syncing atxs from %d", timestamp) - go func() { - err = c.dbClient.GetAtxsReceivedAfter(c.db, timestamp, func(atx *types.ActivationTx) bool { - c.listener.OnActivation(atx) - return true - }) - if err != nil { - log.Warning("syncing atxs from %s failed with error %d", ts, err) - return - } - c.listener.RecalculateEpochStats() - }() - - return ctx.NoContent(http.StatusOK) - }) - - e.GET("/sync/bulk/atxs/ts/:ts", func(ctx echo.Context) error { - ts := ctx.Param("ts") - timestamp, err := strconv.ParseInt(ts, 10, 64) - if err != nil { - return ctx.String(http.StatusBadRequest, "Invalid parameter") - } - - log.Info("http syncing atxs from %d", timestamp) - go func() { - var atxs []*model.Activation - err = c.dbClient.GetAtxsReceivedAfter(c.db, timestamp, func(atx *types.ActivationTx) bool { - atxs = append(atxs, model.NewActivation(atx)) - return true - }) - if err != nil { - log.Warning("syncing atxs from %s failed with error %d", ts, err) - return - } - c.listener.OnActivations(atxs) - c.listener.RecalculateEpochStats() - }() - - return ctx.NoContent(http.StatusOK) - }) - - e.GET("/sync/atxs/:epoch", func(ctx echo.Context) error { - epoch := ctx.Param("epoch") - epochId, err := strconv.ParseInt(epoch, 10, 64) - if err != nil { - return ctx.String(http.StatusBadRequest, "Invalid parameter") - } - - log.Info("http syncing atxs for epoch %s", epoch) - go func() { - err = c.dbClient.GetAtxsByEpoch(c.db, epochId, func(atx *types.ActivationTx) bool { - c.listener.OnActivation(atx) - return true - }) - if err != nil { - log.Warning("syncing atxs for %s failed with error %d", epoch, err) - return - } - c.listener.RecalculateEpochStats() - }() - - return ctx.NoContent(http.StatusOK) - }) - - e.GET("/sync/bulk/atxs/:epoch", func(ctx echo.Context) error { - epoch := ctx.Param("epoch") - epochId, err := strconv.ParseInt(epoch, 10, 64) - if err != nil { - return ctx.String(http.StatusBadRequest, "Invalid parameter") - } - - log.Info("http syncing atxs for epoch %s", epoch) - go func() { - count, err := c.dbClient.CountAtxsByEpoch(c.db, epochId) - if err != nil { - log.Warning("syncing atxs for %s failed with error %d", epoch, err) - return - } - batchSize := 100000 - totalPages := (count + batchSize - 1) / batchSize - for page := 0; page < totalPages; page++ { - offset := page * batchSize - var atxs []*model.Activation - err = c.dbClient.GetAtxsByEpochPaginated(c.db, epochId, int64(batchSize), int64(offset), func(atx *types.ActivationTx) bool { - atxs = append(atxs, model.NewActivation(atx)) - return true - }) - if err != nil { - log.Warning("syncing atxs for %s failed with error %d", epoch, err) - return - } - c.listener.OnActivations(atxs) - atxs = nil - } - c.listener.RecalculateEpochStats() - }() - - return ctx.NoContent(http.StatusOK) - }) - - e.GET("/sync/layer/:layer", func(ctx echo.Context) error { - layer := ctx.Param("layer") - layerId, err := strconv.ParseInt(layer, 10, 64) - if err != nil { - return ctx.String(http.StatusBadRequest, "Invalid parameter") - } - lid := types.LayerID(layerId) - - go func() { - l, err := c.dbClient.GetLayer(c.db, lid, c.listener.GetEpochNumLayers()) - if err != nil { - log.Warning("%v", err) - return - } - - log.Info("http syncing layer: %d", l.Number.Number) - c.listener.OnLayer(l) - }() - - return ctx.NoContent(http.StatusOK) - }) - - e.GET("/sync/rewards/:layer", func(ctx echo.Context) error { - layer := ctx.Param("layer") - layerId, err := strconv.ParseInt(layer, 10, 64) - if err != nil { - return ctx.String(http.StatusBadRequest, "Invalid parameter") - } - lid := types.LayerID(layerId) - - go func() { - log.Info("http syncing rewards for layer: %d", lid.Uint32()) - rewards, err := c.dbClient.GetLayerRewards(c.db, lid) - if err != nil { - log.Warning("%v", err) - return - } - - for _, reward := range rewards { - r := &pb.Reward{ - Layer: &pb.LayerNumber{Number: reward.Layer.Uint32()}, - Total: &pb.Amount{Value: reward.TotalReward}, - LayerReward: &pb.Amount{Value: reward.LayerReward}, - Coinbase: &pb.AccountId{Address: reward.Coinbase.String()}, - Smesher: &pb.SmesherId{Id: reward.SmesherID.Bytes()}, - } - c.listener.OnReward(r) - } - - c.listener.UpdateEpochStats(lid.Uint32()) - }() - - return ctx.NoContent(http.StatusOK) - }) - - e.Logger.Fatal(e.Start(fmt.Sprintf("%s:%d", apiHost, apiPort))) -} diff --git a/collector/layers_test.go b/collector/layers_test.go deleted file mode 100644 index f7f57ad..0000000 --- a/collector/layers_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package collector_test - -import ( - "context" - "encoding/json" - "testing" - - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - - "github.com/spacemeshos/explorer-backend/model" -) - -func TestLayers(t *testing.T) { - t.Parallel() - layers, err := storageDB.GetLayers(context.TODO(), &bson.D{}) - require.NoError(t, err) - require.Equal(t, len(generator.Layers), len(layers)) - for _, layer := range layers { - // temporary hack, until storage return data as slice of bson.B not an struct. - layerEncoded, err := json.Marshal(layer.Map()) - require.NoError(t, err) - var tmpLayer model.Layer - require.NoError(t, json.Unmarshal(layerEncoded, &tmpLayer)) - generatedLayer, ok := generator.Layers[tmpLayer.Number] - require.True(t, ok) - tmpLayer.Rewards = generatedLayer.Rewards // todo should fill data from proto api - tmpLayer.Hash = tmpLayer.Hash[2:] // contain string like `0x...`, cut 0x - require.Equal(t, *generatedLayer, tmpLayer) - } -} diff --git a/collector/mesh.go b/collector/mesh.go deleted file mode 100644 index e1922ed..0000000 --- a/collector/mesh.go +++ /dev/null @@ -1,266 +0,0 @@ -package collector - -import ( - "context" - "fmt" - "github.com/spacemeshos/explorer-backend/model" - "github.com/spacemeshos/explorer-backend/utils" - "github.com/spacemeshos/go-spacemesh/common/types" - "go.mongodb.org/mongo-driver/bson" - "io" - "time" - - empty "github.com/golang/protobuf/ptypes/empty" - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" - "github.com/spacemeshos/go-spacemesh/log" -) - -func (c *Collector) getNetworkInfo() error { - // set timeout - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - genesisTime, err := c.meshClient.GenesisTime(ctx, &pb.GenesisTimeRequest{}) - if err != nil { - log.Err(fmt.Errorf("cannot get GenesisTime: %v", err)) - return err - } - - genesisId, err := c.meshClient.GenesisID(ctx, &pb.GenesisIDRequest{}) - if err != nil { - log.Err(fmt.Errorf("cannot get NetId: %v", err)) - } - - epochNumLayers, err := c.meshClient.EpochNumLayers(ctx, &pb.EpochNumLayersRequest{}) - if err != nil { - log.Err(fmt.Errorf("cannot get EpochNumLayers: %v", err)) - return err - } - - maxTransactionsPerSecond, err := c.meshClient.MaxTransactionsPerSecond(ctx, &pb.MaxTransactionsPerSecondRequest{}) - if err != nil { - log.Err(fmt.Errorf("cannot get MaxTransactionsPerSecond: %v", err)) - return err - } - - layerDuration, err := c.meshClient.LayerDuration(ctx, &pb.LayerDurationRequest{}) - if err != nil { - log.Err(fmt.Errorf("cannot get LayerDuration: %v", err)) - return err - } - - res, err := c.smesherClient.PostConfig(ctx, &empty.Empty{}) - if err != nil { - log.Err(fmt.Errorf("cannot get POST config: %v", err)) - return err - } - - c.listener.OnNetworkInfo( - utils.BytesToHex(genesisId.GetGenesisId()), - genesisTime.GetUnixtime().GetValue(), - epochNumLayers.GetNumlayers().GetNumber(), - maxTransactionsPerSecond.GetMaxTxsPerSecond().GetValue(), - layerDuration.GetDuration().GetValue(), - (uint64(res.BitsPerLabel)*uint64(res.LabelsPerUnit))/8, - ) - - return nil -} - -func (c *Collector) syncMissingLayers() error { - status, err := c.nodeClient.Status(context.Background(), &pb.StatusRequest{}) - if err != nil { - log.Err(fmt.Errorf("cannot receive node status: %v", err)) - return err - } - syncedLayerNum := status.Status.VerifiedLayer.Number - lastLayer := c.listener.GetLastLayer(context.TODO()) - - if syncedLayerNum == lastLayer { - return nil - } - - log.Info("Syncing missing layers %d...%d", lastLayer+1, syncedLayerNum) - - for i := lastLayer + 1; i <= syncedLayerNum; i++ { - err := c.syncLayer(types.LayerID(i)) - if err != nil { - log.Warning("syncMissingLayers error: %v", err) - } - } - - log.Info("Waiting for layers queue to be empty") - for { - layersInQueue := c.listener.LayersInQueue() - if layersInQueue > 0 { - log.Info("%d layers in queue. Waiting", layersInQueue) - time.Sleep(15 * time.Second) - } else { - break - } - } - - return nil -} - -func (c *Collector) malfeasancePump() error { - var req = pb.MalfeasanceStreamRequest{} - - log.Info("Start mesh malfeasance pump") - defer func() { - c.notify <- -streamType_mesh_Malfeasance - log.Info("Stop mesh malfeasance pump") - }() - - c.notify <- +streamType_mesh_Malfeasance - - stream, err := c.meshClient.MalfeasanceStream(context.Background(), &req) - if err != nil { - log.Err(fmt.Errorf("cannot get malfeasance stream: %v", err)) - return err - } - - for { - response, err := stream.Recv() - if err == io.EOF { - return err - } - if err != nil { - log.Err(fmt.Errorf("cannot receive malfeasance proof: %v", err)) - return err - } - proof := response.GetProof() - c.listener.OnMalfeasanceProof(proof) - } -} - -func (c *Collector) syncLayer(lid types.LayerID) error { - layer, err := c.dbClient.GetLayer(c.db, lid, c.listener.GetEpochNumLayers()) - if err != nil { - return err - } - - if c.listener.IsLayerInQueue(layer) { - log.Info("layer %d is already in queue", layer.Number.Number) - return nil - } - - if lastLayer := c.listener.GetLastLayer(context.TODO()); lastLayer >= layer.Number.Number { - log.Info("layer %d is already in database", layer.Number.Number) - return nil - } - - log.Info("syncing layer: %d", layer.Number.Number) - c.listener.OnLayer(layer) - - log.Info("syncing accounts for layer: %d", layer.Number.Number) - accounts, err := c.dbClient.AccountsSnapshot(c.db, lid) - if err != nil { - log.Warning("%v\n", err) - } - c.listener.OnAccounts(accounts) - - log.Info("syncing rewards for layer: %d", layer.Number.Number) - rewards, err := c.dbClient.GetLayerRewards(c.db, lid) - if err != nil { - log.Warning("%v\n", err) - } - - for _, reward := range rewards { - r := &pb.Reward{ - Layer: &pb.LayerNumber{Number: reward.Layer.Uint32()}, - Total: &pb.Amount{Value: reward.TotalReward}, - LayerReward: &pb.Amount{Value: reward.LayerReward}, - Coinbase: &pb.AccountId{Address: reward.Coinbase.String()}, - Smesher: &pb.SmesherId{Id: reward.SmesherID.Bytes()}, - } - c.listener.OnReward(r) - } - - c.listener.UpdateEpochStats(layer.Number.Number) - - return nil -} - -func (c *Collector) syncNotProcessedTxs() error { - txs, err := c.listener.GetTransactions(context.TODO(), &bson.D{{Key: "state", Value: 0}}) - if err != nil { - return err - } - - for _, tx := range txs { - txId, err := utils.StringToBytes(tx.Id) - if err != nil { - return err - } - - state, err := c.transactionsClient.TransactionsState(context.TODO(), &pb.TransactionsStateRequest{ - TransactionId: []*pb.TransactionId{{Id: txId}}, - IncludeTransactions: false, - }) - if err != nil { - return err - } - - txState := state.TransactionsState[0] - - if txState != nil { - err := c.listener.UpdateTransactionState(context.TODO(), tx.Id, int32(txState.State)) - if err != nil { - return err - } - } - } - - return nil -} - -func (c *Collector) syncAllRewards() error { - rewards, err := c.dbClient.GetAllRewards(c.db) - if err != nil { - return fmt.Errorf("%v\n", err) - } - - for _, reward := range rewards { - r := &pb.Reward{ - Layer: &pb.LayerNumber{Number: reward.Layer.Uint32()}, - Total: &pb.Amount{Value: reward.TotalReward}, - LayerReward: &pb.Amount{Value: reward.LayerReward}, - Coinbase: &pb.AccountId{Address: reward.Coinbase.String()}, - Smesher: &pb.SmesherId{Id: reward.SmesherID.Bytes()}, - } - c.listener.OnReward(r) - } - - return nil -} - -func (c *Collector) syncActivations() error { - received := c.listener.GetLastActivationReceived() - log.Info("Syncing activations from %d", received) - - var atxs []*model.Activation - err := c.dbClient.GetAtxsReceivedAfter(c.db, received, func(atx *types.ActivationTx) bool { - atxs = append(atxs, model.NewActivation(atx)) - return true - }) - if err != nil { - return err - } - - c.listener.OnActivations(atxs) - - return nil -} - -func (c *Collector) createFutureEpoch() error { - lastLayer := c.listener.GetLastLayer(context.Background()) - epochNumLayers := c.listener.GetEpochNumLayers() - - if epochNumLayers > 0 { - futureEpoch := lastLayer/epochNumLayers + 1 - c.listener.UpdateEpochStats(futureEpoch*epochNumLayers + 1) - } - - return nil -} diff --git a/collector/node.go b/collector/node.go deleted file mode 100644 index 2bafbdd..0000000 --- a/collector/node.go +++ /dev/null @@ -1,79 +0,0 @@ -package collector - -import ( - "context" - "fmt" - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" - "github.com/spacemeshos/go-spacemesh/common/types" - "io" - - "github.com/spacemeshos/go-spacemesh/log" -) - -func (c *Collector) syncStatusPump() error { - req := pb.StatusStreamRequest{} - - log.Info("Start node sync status pump") - defer func() { - c.notify <- -streamType_node_SyncStatus - log.Info("Stop node sync status pump") - }() - - c.notify <- +streamType_node_SyncStatus - - stream, err := c.nodeClient.StatusStream(context.Background(), &req) - if err != nil { - log.Err(fmt.Errorf("cannot get sync status stream: %v", err)) - return err - } - - for { - res, err := stream.Recv() - if err == io.EOF { - log.Info("syncStatusPump: EOF") - return err - } - if err != nil { - log.Err(fmt.Errorf("cannot receive sync status: %v", err)) - return err - } - - status := res.GetStatus() - log.Info("Node sync status: %v", status) - - lastLayer := c.listener.GetLastLayer(context.TODO()) - if lastLayer != status.GetVerifiedLayer().GetNumber() { - for i := lastLayer + 1; i <= status.GetVerifiedLayer().GetNumber(); i++ { - err := c.syncLayer(types.LayerID(i)) - if err != nil { - log.Warning("syncLayer error: %v", err) - } - - err = c.syncNotProcessedTxs() - if err != nil { - log.Warning("syncNotProcessedTxs error: %v", err) - } - - if c.atxSyncFlag { - err = c.syncActivations() - if err != nil { - log.Warning("syncActivations error: %v", err) - } - } - - err = c.createFutureEpoch() - if err != nil { - log.Warning("createFutureEpoch error: %v", err) - } - } - } - - c.listener.OnNodeStatus( - status.GetConnectedPeers(), - status.GetIsSynced(), - status.GetSyncedLayer().GetNumber(), - status.GetTopLayer().GetNumber(), - status.GetVerifiedLayer().GetNumber(), - ) - } -} diff --git a/collector/rewards_test.go b/collector/rewards_test.go deleted file mode 100644 index ce2677a..0000000 --- a/collector/rewards_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package collector_test - -import ( - "context" - "encoding/json" - "github.com/spacemeshos/explorer-backend/model" - "strings" - "testing" - - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" -) - -func TestRewards(t *testing.T) { - t.Parallel() - rewards, err := storageDB.GetRewards(context.TODO(), &bson.D{}) - require.NoError(t, err) - require.Equal(t, len(generator.Rewards), len(rewards)) - - for _, reward := range rewards { - // temporary hack, until storage return data as slice of bson.B not an struct. - rewardEncoded, err := json.Marshal(reward.Map()) - require.NoError(t, err) - var tmpReward model.Reward - require.NoError(t, json.Unmarshal(rewardEncoded, &tmpReward)) - generatedReward, ok := generator.Rewards[strings.ToLower(tmpReward.Smesher)] - require.True(t, ok, "reward not found") - generatedReward.Smesher = strings.ToLower(generatedReward.Smesher) - tmpReward.Smesher = strings.ToLower(tmpReward.Smesher) - tmpReward.Coinbase = strings.ToLower(tmpReward.Coinbase) - generatedReward.Coinbase = strings.ToLower(generatedReward.Coinbase) - tmpReward.ID = "" // id is internal mongo id. before insert to db we do not know it. - require.Equal(t, *generatedReward, tmpReward) - } -} diff --git a/collector/smeshers_test.go b/collector/smeshers_test.go deleted file mode 100644 index af13ad7..0000000 --- a/collector/smeshers_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package collector_test - -import ( - "context" - "encoding/json" - "strings" - "testing" - - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - - "github.com/spacemeshos/explorer-backend/model" -) - -func TestSmeshers(t *testing.T) { - t.Parallel() - smeshers, err := storageDB.GetSmeshers(context.TODO(), &bson.D{}) - require.NoError(t, err) - require.Equal(t, len(generator.Smeshers), len(smeshers)) - for _, smesher := range smeshers { - // temporary hack, until storage return data as slice of bson.B not an struct. - smesherEncoded, err := json.Marshal(smesher.Map()) - require.NoError(t, err) - var tmpSmesher model.Smesher - require.NoError(t, json.Unmarshal(smesherEncoded, &tmpSmesher)) - generatedSmesher, ok := generator.Smeshers[strings.ToLower(tmpSmesher.Id)] - require.True(t, ok) - generatedSmesher.Id = strings.ToLower(generatedSmesher.Id) - size, ok := smesher.Map()["cSize"].(int64) - require.True(t, ok) - generatedSmesher.CommitmentSize = uint64(size) - tmpSmesher.Coinbase = strings.ToLower(tmpSmesher.Coinbase) - generatedSmesher.Coinbase = strings.ToLower(generatedSmesher.Coinbase) - tmpSmesher.Rewards = generatedSmesher.Rewards - tmpSmesher.Epochs = generatedSmesher.Epochs // this is 0 cause it calculates from special mthod on api. - require.Equal(t, *generatedSmesher, tmpSmesher) - } -} diff --git a/collector/sql/accounts.go b/collector/sql/accounts.go deleted file mode 100644 index 7212ecd..0000000 --- a/collector/sql/accounts.go +++ /dev/null @@ -1,11 +0,0 @@ -package sql - -import ( - "github.com/spacemeshos/go-spacemesh/common/types" - "github.com/spacemeshos/go-spacemesh/sql" - "github.com/spacemeshos/go-spacemesh/sql/accounts" -) - -func (c *Client) AccountsSnapshot(db *sql.Database, lid types.LayerID) (rst []*types.Account, err error) { - return accounts.Snapshot(db, lid) -} diff --git a/collector/sql/atxs.go b/collector/sql/atxs.go deleted file mode 100644 index 25fc702..0000000 --- a/collector/sql/atxs.go +++ /dev/null @@ -1,154 +0,0 @@ -package sql - -import ( - sqlite "github.com/go-llsqlite/crawshaw" - "github.com/spacemeshos/explorer-backend/utils" - "github.com/spacemeshos/go-spacemesh/common/types" - "github.com/spacemeshos/go-spacemesh/sql" - "github.com/spacemeshos/go-spacemesh/sql/atxs" - "time" -) - -// Query to retrieve ATXs. -// Can't use inner join for the ATX blob here b/c this will break -// filters that refer to the id column. -const fieldsQuery = `select -atxs.id, atxs.nonce, atxs.base_tick_height, atxs.tick_count, atxs.pubkey, atxs.effective_num_units, -atxs.received, atxs.epoch, atxs.sequence, atxs.coinbase, atxs.validity, atxs.prev_id, atxs.commitment_atx` - -const fullQuery = fieldsQuery + ` from atxs` - -type decoderCallback func(*types.ActivationTx) bool - -func decoder(fn decoderCallback) sql.Decoder { - return func(stmt *sql.Statement) bool { - var ( - a types.ActivationTx - id types.ATXID - ) - stmt.ColumnBytes(0, id[:]) - a.SetID(id) - a.VRFNonce = types.VRFPostIndex(stmt.ColumnInt64(1)) - a.BaseTickHeight = uint64(stmt.ColumnInt64(2)) - a.TickCount = uint64(stmt.ColumnInt64(3)) - stmt.ColumnBytes(4, a.SmesherID[:]) - a.NumUnits = uint32(stmt.ColumnInt32(5)) - // Note: received is assigned `0` for checkpointed ATXs. - // We treat `0` as 'zero time'. - // We could use `NULL` instead, but the column has "NOT NULL" constraint. - // In future, consider changing the schema to allow `NULL` for received. - if received := stmt.ColumnInt64(6); received == 0 { - a.SetGolden() - } else { - a.SetReceived(time.Unix(0, received).Local()) - } - a.PublishEpoch = types.EpochID(uint32(stmt.ColumnInt(7))) - a.Sequence = uint64(stmt.ColumnInt64(8)) - stmt.ColumnBytes(9, a.Coinbase[:]) - a.SetValidity(types.Validity(stmt.ColumnInt(10))) - if stmt.ColumnType(11) != sqlite.SQLITE_NULL { - stmt.ColumnBytes(11, a.PrevATXID[:]) - } - if stmt.ColumnType(12) != sqlite.SQLITE_NULL { - a.CommitmentATX = new(types.ATXID) - stmt.ColumnBytes(12, a.CommitmentATX[:]) - } - - return fn(&a) - } -} - -func (c *Client) GetAtxsReceivedAfter(db *sql.Database, ts int64, fn func(tx *types.ActivationTx) bool) error { - var derr error - _, err := db.Exec( - fullQuery+` WHERE received > ?1`, - func(stmt *sql.Statement) { - stmt.BindInt64(1, ts) - }, - decoder(func(atx *types.ActivationTx) bool { - if atx != nil { - return fn(atx) - } - return true - }), - ) - if err != nil { - return err - } - return derr -} - -func (c *Client) GetAtxsByEpoch(db *sql.Database, epoch int64, fn func(tx *types.ActivationTx) bool) error { - var derr error - _, err := db.Exec( - fullQuery+` WHERE epoch = ?1 ORDER BY epoch asc, id asc`, - func(stmt *sql.Statement) { - stmt.BindInt64(1, epoch) - }, - decoder(func(atx *types.ActivationTx) bool { - if atx != nil { - return fn(atx) - } - return true - }), - ) - if err != nil { - return err - } - return derr -} - -func (c *Client) CountAtxsByEpoch(db *sql.Database, epoch int64) (int, error) { - var totalCount int - _, err := db.Exec( - `SELECT COUNT(*) FROM atxs WHERE epoch = ?1`, - func(stmt *sql.Statement) { - stmt.BindInt64(1, epoch) - }, func(stmt *sql.Statement) bool { - totalCount = stmt.ColumnInt(0) - return true - }) - if err != nil { - return 0, err - } - return totalCount, nil -} - -func (c *Client) GetAtxsByEpochPaginated(db *sql.Database, epoch, limit, offset int64, fn func(tx *types.ActivationTx) bool) error { - var derr error - _, err := db.Exec( - fullQuery+` WHERE epoch = ?1 ORDER BY epoch asc, id asc LIMIT ?2 OFFSET ?3`, - func(stmt *sql.Statement) { - stmt.BindInt64(1, epoch) - stmt.BindInt64(2, limit) - stmt.BindInt64(3, offset) - }, - decoder(func(atx *types.ActivationTx) bool { - if atx != nil { - return fn(atx) - } - return true - }), - ) - if err != nil { - return err - } - return derr -} - -func (c *Client) GetAtxById(db *sql.Database, id string) (*types.ActivationTx, error) { - idBytes, err := utils.StringToBytes(id) - if err != nil { - return nil, err - } - - var atxId types.ATXID - copy(atxId[:], idBytes) - - atx, err := atxs.Get(db, atxId) - if err != nil { - return nil, err - } - - return atx, nil -} diff --git a/collector/sql/layers.go b/collector/sql/layers.go deleted file mode 100644 index f26e00a..0000000 --- a/collector/sql/layers.go +++ /dev/null @@ -1,185 +0,0 @@ -package sql - -import ( - "context" - "errors" - "fmt" - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" - "github.com/spacemeshos/go-spacemesh/common/types" - "github.com/spacemeshos/go-spacemesh/log" - "github.com/spacemeshos/go-spacemesh/sql" - "github.com/spacemeshos/go-spacemesh/sql/atxs" - "github.com/spacemeshos/go-spacemesh/sql/ballots" - "github.com/spacemeshos/go-spacemesh/sql/blocks" - "github.com/spacemeshos/go-spacemesh/sql/layers" - "github.com/spacemeshos/go-spacemesh/sql/transactions" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func (c *Client) GetLayer(db *sql.Database, lid types.LayerID, numLayers uint32) (*pb.Layer, error) { - var bs []*pb.Block - var activations []types.ATXID - - blts, err := ballots.Layer(db, lid) - if err != nil { - return nil, err - } - - blks, err := blocks.Layer(db, lid) - if err != nil { - return nil, err - } - - layer := types.NewExistingLayer(lid, blts, blks) - - for _, b := range layer.Blocks() { - if b == nil { - continue - } - mtxs, missing := getMeshTransactions(db, b.TxIDs) - if len(missing) != 0 { - return nil, status.Errorf(codes.Internal, "error retrieving tx data") - } - - pbTxs := make([]*pb.Transaction, 0, len(mtxs)) - for _, t := range mtxs { - pbTxs = append(pbTxs, castTransaction(&t.Transaction)) - } - - bs = append(bs, &pb.Block{ - Id: types.Hash20(b.ID()).Bytes(), - Transactions: pbTxs, - }) - } - - epoch := lid.Uint32() / numLayers - if lid.Uint32()%numLayers == 0 { - atxsId, err := atxs.GetIDsByEpoch(context.Background(), db, types.EpochID(epoch-1)) - if err != nil { - return nil, err - } - - activations = append(activations, atxsId...) - } - - // Extract ATX data from block data - var pbActivations []*pb.Activation - - // Add unique ATXIDs - atxids, matxs := GetATXs(db, activations) - if len(matxs) != 0 { - return nil, status.Errorf(codes.Internal, "error retrieving activations data") - } - for _, atx := range atxids { - pbActivations = append(pbActivations, convertActivation(atx)) - } - - stateRoot, err := layers.GetStateHash(db, layer.Index()) - if err != nil { - // This is expected. We can only retrieve state root for a layer that was applied to state, - // which only happens after it's approved/confirmed. - log.Debug("no state root for layer", err) - } - - hash, err := layers.GetAggregatedHash(db, lid) - if err != nil { - // This is expected. We can only retrieve state root for a layer that was applied to state, - // which only happens after it's approved/confirmed. - log.Debug("no mesh hash at layer", err) - } - return &pb.Layer{ - Number: &pb.LayerNumber{Number: layer.Index().Uint32()}, - Status: pb.Layer_LAYER_STATUS_CONFIRMED, - Blocks: bs, - Activations: pbActivations, - Hash: hash.Bytes(), - RootStateHash: stateRoot.Bytes(), - }, nil -} - -func getMeshTransactions(db *sql.Database, ids []types.TransactionID) ([]*types.MeshTransaction, map[types.TransactionID]struct{}) { - if ids == nil { - return []*types.MeshTransaction{}, map[types.TransactionID]struct{}{} - } - missing := make(map[types.TransactionID]struct{}) - mtxs := make([]*types.MeshTransaction, 0, len(ids)) - for _, tid := range ids { - var ( - mtx *types.MeshTransaction - err error - ) - if mtx, err = transactions.Get(db, tid); err != nil { - missing[tid] = struct{}{} - } else { - mtxs = append(mtxs, mtx) - } - } - return mtxs, missing -} - -func GetATXs(db *sql.Database, atxIds []types.ATXID) (map[types.ATXID]*types.ActivationTx, []types.ATXID) { - var mIds []types.ATXID - a := make(map[types.ATXID]*types.ActivationTx, len(atxIds)) - for _, id := range atxIds { - t, err := getFullAtx(db, id) - if err != nil { - mIds = append(mIds, id) - } else { - a[t.ID()] = t - } - } - return a, mIds -} - -func getFullAtx(db *sql.Database, id types.ATXID) (*types.ActivationTx, error) { - if id == types.EmptyATXID { - return nil, errors.New("trying to fetch empty atx id") - } - - atx, err := atxs.Get(db, id) - if err != nil { - return nil, fmt.Errorf("get ATXs from DB: %w", err) - } - - return atx, nil -} - -func castTransaction(t *types.Transaction) *pb.Transaction { - tx := &pb.Transaction{ - Id: t.ID[:], - Raw: t.Raw, - } - if t.TxHeader != nil { - tx.Principal = &pb.AccountId{ - Address: t.Principal.String(), - } - tx.Template = &pb.AccountId{ - Address: t.TemplateAddress.String(), - } - tx.Method = uint32(t.Method) - tx.Nonce = &pb.Nonce{ - Counter: t.Nonce, - } - tx.Limits = &pb.LayerLimits{ - Min: t.LayerLimits.Min, - Max: t.LayerLimits.Max, - } - tx.MaxGas = t.MaxGas - tx.GasPrice = t.GasPrice - tx.MaxSpend = t.MaxSpend - } - return tx -} - -func convertActivation(a *types.ActivationTx) *pb.Activation { - return &pb.Activation{ - Id: &pb.ActivationId{Id: a.ID().Bytes()}, - Layer: &pb.LayerNumber{Number: a.PublishEpoch.Uint32()}, - SmesherId: &pb.SmesherId{Id: a.SmesherID.Bytes()}, - Coinbase: &pb.AccountId{Address: a.Coinbase.String()}, - PrevAtx: &pb.ActivationId{Id: a.PrevATXID.Bytes()}, - NumUnits: a.NumUnits, - Sequence: a.Sequence, - } -} diff --git a/collector/sql/rewards.go b/collector/sql/rewards.go deleted file mode 100644 index 63c422b..0000000 --- a/collector/sql/rewards.go +++ /dev/null @@ -1,50 +0,0 @@ -package sql - -import ( - "github.com/spacemeshos/go-spacemesh/common/types" - "github.com/spacemeshos/go-spacemesh/sql" -) - -func (c *Client) GetLayerRewards(db *sql.Database, lid types.LayerID) (rst []*types.Reward, err error) { - _, err = db.Exec("select coinbase, layer, total_reward, layer_reward, pubkey from rewards where layer = ?1;", - func(stmt *sql.Statement) { - stmt.BindInt64(1, int64(lid)) - }, func(stmt *sql.Statement) bool { - addrBytes := stmt.ColumnViewBytes(0) - - var addr types.Address - copy(addr[:], addrBytes) - - reward := &types.Reward{ - Coinbase: addr, - Layer: types.LayerID(uint32(stmt.ColumnInt64(1))), - TotalReward: uint64(stmt.ColumnInt64(2)), - LayerReward: uint64(stmt.ColumnInt64(3)), - SmesherID: types.BytesToNodeID(stmt.ColumnViewBytes(4)), - } - rst = append(rst, reward) - return true - }) - return -} - -func (c *Client) GetAllRewards(db *sql.Database) (rst []*types.Reward, err error) { - _, err = db.Exec("select coinbase, layer, total_reward, layer_reward, pubkey from rewards;", - nil, func(stmt *sql.Statement) bool { - addrBytes := stmt.ColumnViewBytes(0) - - var addr types.Address - copy(addr[:], addrBytes) - - reward := &types.Reward{ - Coinbase: addr, - Layer: types.LayerID(uint32(stmt.ColumnInt64(1))), - TotalReward: uint64(stmt.ColumnInt64(2)), - LayerReward: uint64(stmt.ColumnInt64(3)), - SmesherID: types.BytesToNodeID(stmt.ColumnViewBytes(4)), - } - rst = append(rst, reward) - return true - }) - return -} diff --git a/collector/sql/sql.go b/collector/sql/sql.go deleted file mode 100644 index 163f96f..0000000 --- a/collector/sql/sql.go +++ /dev/null @@ -1,28 +0,0 @@ -package sql - -import ( - "fmt" - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" - "github.com/spacemeshos/go-spacemesh/common/types" - "github.com/spacemeshos/go-spacemesh/sql" -) - -type DatabaseClient interface { - GetLayer(db *sql.Database, lid types.LayerID, numLayers uint32) (*pb.Layer, error) - GetLayerRewards(db *sql.Database, lid types.LayerID) (rst []*types.Reward, err error) - GetAllRewards(db *sql.Database) (rst []*types.Reward, err error) - AccountsSnapshot(db *sql.Database, lid types.LayerID) (rst []*types.Account, err error) - GetAtxsReceivedAfter(db *sql.Database, ts int64, fn func(tx *types.ActivationTx) bool) error - GetAtxsByEpoch(db *sql.Database, epoch int64, fn func(tx *types.ActivationTx) bool) error - CountAtxsByEpoch(db *sql.Database, epoch int64) (int, error) - GetAtxsByEpochPaginated(db *sql.Database, epoch, limit, offset int64, fn func(tx *types.ActivationTx) bool) error - GetAtxById(db *sql.Database, id string) (*types.ActivationTx, error) -} - -type Client struct{} - -func Setup(path string) (db *sql.Database, err error) { - db, err = sql.Open(fmt.Sprintf("file:%s?mode=ro", path), - sql.WithConnections(16), sql.WithMigrations(nil)) - return -} diff --git a/collector/txs.go b/collector/txs.go deleted file mode 100644 index 9f39593..0000000 --- a/collector/txs.go +++ /dev/null @@ -1,59 +0,0 @@ -package collector - -import ( - "context" - "fmt" - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" - "github.com/spacemeshos/go-spacemesh/log" - "io" -) - -func (c *Collector) transactionsPump() error { - lastLayer := c.listener.GetLastLayer(context.Background()) - - req := pb.TransactionResultsRequest{ - Start: lastLayer - 500, - Watch: true, - } - - log.Info("Start transactions pump") - defer func() { - c.notify <- -streamType_transactions - log.Info("Stop transactions pump") - }() - - c.notify <- +streamType_transactions - - stream, err := c.transactionsClient.StreamResults(context.Background(), &req) - if err != nil { - log.Err(fmt.Errorf("cannot get transactions stream results: %v", err)) - return err - } - - for { - response, err := stream.Recv() - if err == io.EOF { - return err - } - if err != nil { - log.Err(fmt.Errorf("cannot receive transaction result: %v", err)) - return err - } - if response == nil { - continue - } - - state, err := c.transactionsClient.TransactionsState(context.TODO(), &pb.TransactionsStateRequest{ - TransactionId: []*pb.TransactionId{{Id: response.Tx.Id}}, - IncludeTransactions: false, - }) - if err != nil { - log.Err(fmt.Errorf("cannot receive transaction state: %v", err)) - return err - } - - if len(state.GetTransactionsState()) > 0 { - c.listener.OnTransactionResult(response, state.GetTransactionsState()[0]) - } - } -} diff --git a/collector/txs_test.go b/collector/txs_test.go deleted file mode 100644 index 9510070..0000000 --- a/collector/txs_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package collector_test - -import ( - "context" - "strings" - "testing" - - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" -) - -func TestTransactions(t *testing.T) { - t.Parallel() - txs, err := storageDB.GetTransactions(context.TODO(), &bson.D{}) - require.NoError(t, err) - require.Equal(t, len(generator.Transactions), len(txs)) - for _, tx := range txs { - require.NoError(t, err) - generatedTx, ok := generator.Transactions[tx.Id] - require.True(t, ok) - tx.Receiver = strings.ToLower(tx.Receiver) - tx.Sender = strings.ToLower(tx.Sender) - generatedTx.Receiver = strings.ToLower(generatedTx.Receiver) - generatedTx.Sender = strings.ToLower(generatedTx.Sender) - generatedTx.PublicKey = "" // we do not encode it to send tx, omit this. - generatedTx.Signature = "" // we generate sign on emulation of pb stream. - tx.Signature = "" // we generate sign on emulation of pb stream. - require.Equal(t, *generatedTx, tx) - } -} diff --git a/docker-compose.yml b/docker-compose.yml index c4a52f4..c1e9cce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,6 @@ version: '3' services: - dbMongo: - image: mongo:5 - container_name: sm_explorer_db - ports: - - "27017-27019:27017-27019" redis: image: redis:6 restart: always diff --git a/explorer-backend.go b/explorer-backend.go index bd66189..11c512d 100644 --- a/explorer-backend.go +++ b/explorer-backend.go @@ -1,2 +1 @@ package explorer_backend - diff --git a/go.mod b/go.mod index 43de800..2156c7c 100644 --- a/go.mod +++ b/go.mod @@ -1,35 +1,22 @@ module github.com/spacemeshos/explorer-backend -go 1.22.3 - -toolchain go1.22.4 +go 1.23.0 require ( github.com/eko/gocache/lib/v4 v4.1.6 github.com/eko/gocache/store/go_cache/v4 v4.2.2 github.com/eko/gocache/store/redis/v4 v4.2.2 - github.com/go-llsqlite/crawshaw v0.5.3 - github.com/gofiber/fiber/v2 v2.52.5 - github.com/golang/protobuf v1.5.4 - github.com/gorilla/websocket v1.5.1 github.com/labstack/echo-contrib v0.17.1 github.com/labstack/echo/v4 v4.12.0 - github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a github.com/patrickmn/go-cache v2.1.0+incompatible - github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 - github.com/prometheus/client_golang v1.19.1 + github.com/redis/go-redis/v9 v9.0.2 github.com/spacemeshos/address v0.0.0-20220829090052-44ab32617871 github.com/spacemeshos/api/release/go v1.50.0 github.com/spacemeshos/economics v0.1.3 github.com/spacemeshos/go-scale v1.2.0 github.com/spacemeshos/go-spacemesh v1.6.2 - github.com/stretchr/testify v1.9.0 github.com/urfave/cli/v2 v2.27.2 - go.mongodb.org/mongo-driver v1.10.1 go.uber.org/zap v1.27.0 - golang.org/x/net v0.26.0 - golang.org/x/sync v0.7.0 - google.golang.org/grpc v1.65.0 ) require ( @@ -37,65 +24,54 @@ require ( github.com/anacrolix/missinggo v1.2.1 // indirect github.com/anacrolix/missinggo/perf v1.0.0 // indirect github.com/anacrolix/sync v0.3.0 // indirect - github.com/andybalholm/brotli v1.0.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/c0mm4nd/go-ripemd v0.0.0-20200326052756-bd1759ad7d10 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cosmos/btcutil v1.0.5 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/go-llsqlite/crawshaw v0.5.3 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/mock v1.6.0 // indirect - github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huandu/xstrings v1.2.0 // indirect github.com/jonboulle/clockwork v0.4.0 // indirect - github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.15 // indirect github.com/minio/sha256-simd v1.0.1 // indirect - github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a // indirect + github.com/prometheus/client_golang v1.19.1 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/redis/go-redis/v9 v9.0.2 // indirect - github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spacemeshos/merkle-tree v0.2.3 // indirect github.com/spacemeshos/poet v0.10.3 // indirect github.com/spacemeshos/post v0.12.7 // indirect github.com/spacemeshos/sha256-simd v0.1.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.51.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect - github.com/valyala/tcplisten v1.0.0 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.1.1 // indirect - github.com/xdg-go/stringprep v1.0.3 // indirect github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect - github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect github.com/zeebo/blake3 v0.2.3 // indirect go.uber.org/mock v0.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/grpc v1.65.0 // indirect google.golang.org/protobuf v1.34.2 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 53f70bd..98f85d6 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,6 @@ github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5ur github.com/anacrolix/sync v0.3.0 h1:ZPjTrkqQWEfnYVGTQHh5qNjokWaXnjsyXTJSMsKY0TA= github.com/anacrolix/sync v0.3.0/go.mod h1:BbecHL6jDSExojhNtgTFSBcdGerzNc64tz3DCOj/I0g= github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= -github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= -github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= @@ -32,7 +30,6 @@ github.com/cosmos/btcutil v1.0.5/go.mod h1:IyB7iuqZMJlthe2tkIFL33xPyzbFYP0XVdS8P github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= @@ -51,20 +48,12 @@ github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-llsqlite/crawshaw v0.5.3 h1:PmHXJjjxHodMoFWs3dEcJ1I1m9a+J0TCMNiHjGbXy1o= github.com/go-llsqlite/crawshaw v0.5.3/go.mod h1:/YJdV7uBQaYDE0fwe4z3wwJIZBJxdYzd38ICggWqtaE= -github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo= -github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= @@ -72,8 +61,6 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -84,18 +71,12 @@ github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63 github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/echo-contrib v0.17.1 h1:7I/he7ylVKsDUieaGRZ9XxxTYOjfQwVzHzUYrNykfCU= @@ -109,12 +90,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= -github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -122,11 +99,7 @@ github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a h1:dl github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a/go.mod h1:hVoHR2EVESiICEMbg137etN/Lx+lSrHPTD39Z/uE+2s= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= -github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -140,9 +113,6 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/redis/go-redis/v9 v9.0.2 h1:BA426Zqe/7r56kCcvxYLWe1mkaz71LKF77GwgFzSxfE= github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -173,34 +143,20 @@ github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI= github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= -github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= -github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E= -github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= -github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs= -github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw= github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= @@ -208,8 +164,6 @@ github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= -go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4= -go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= @@ -220,7 +174,6 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= @@ -229,7 +182,6 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -240,9 +192,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -251,8 +201,6 @@ golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= @@ -262,7 +210,6 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= @@ -273,9 +220,6 @@ google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjr google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/api/api.go b/internal/api/api.go deleted file mode 100644 index b967dec..0000000 --- a/internal/api/api.go +++ /dev/null @@ -1,89 +0,0 @@ -package api - -import ( - "context" - "fmt" - "github.com/labstack/echo/v4" - "github.com/labstack/echo/v4/middleware" - "github.com/spacemeshos/explorer-backend/internal/api/handler" - "github.com/spacemeshos/explorer-backend/internal/api/router" - "github.com/spacemeshos/explorer-backend/internal/service" - "github.com/spacemeshos/go-spacemesh/log" - "net/http" - "os" - "os/signal" - "syscall" - "time" -) - -type Api struct { - Echo *echo.Echo -} - -func Init(appService service.AppService, allowedOrigins []string, debug bool) *Api { - e := echo.New() - e.Use(middleware.Recover()) - e.Use(func(next echo.HandlerFunc) echo.HandlerFunc { - return func(c echo.Context) error { - cc := &handler.ApiContext{ - Context: c, - Service: appService, - } - return next(cc) - } - }) - e.HideBanner = true - e.HidePort = true - e.Use(middleware.CORSWithConfig(middleware.CORSConfig{ - AllowOrigins: allowedOrigins, - })) - handler.Upgrader.CheckOrigin = func(r *http.Request) bool { - origin := r.Header.Get("origin") - for _, val := range allowedOrigins { - if origin == val || val == "*" { - return true - } - } - return false - } - - e.Use(middleware.RequestLoggerWithConfig(middleware.RequestLoggerConfig{ - LogStatus: true, - LogURI: true, - LogValuesFunc: func(c echo.Context, v middleware.RequestLoggerValues) error { - log.Info("%s [%d] - %s", time.Now().Format(time.RFC3339), c.Response().Status, c.Request().URL.Path) - return nil - }, - })) - - if debug { - e.Debug = true - e.Use(middleware.Logger()) - } - - router.Init(e) - - return &Api{ - Echo: e, - } -} - -func (a *Api) Run(address string) { - log.Info("server is running. For exit ") - if err := a.Echo.Start(address); err != nil { - log.Err(fmt.Errorf("server stopped: %s", err)) - } - - sysSignal := make(chan os.Signal, 1) - signal.Notify(sysSignal, syscall.SIGINT, syscall.SIGTERM) - - s := <-sysSignal - log.Info("exiting, got signal %v", s) - if err := a.Shutdown(); err != nil { - log.Err(fmt.Errorf("error on shutdown: %v", err)) - } -} - -func (a *Api) Shutdown() error { - return a.Echo.Shutdown(context.TODO()) -} diff --git a/internal/api/handler/accounts.go b/internal/api/handler/accounts.go deleted file mode 100644 index a780cae..0000000 --- a/internal/api/handler/accounts.go +++ /dev/null @@ -1,70 +0,0 @@ -package handler - -import ( - "context" - "fmt" - "github.com/labstack/echo/v4" - "github.com/spacemeshos/explorer-backend/internal/service" - "github.com/spacemeshos/explorer-backend/model" - "net/http" -) - -func Accounts(c echo.Context) error { - cc := c.(*ApiContext) - - pageNum, pageSize := GetPagination(c) - accounts, total, err := cc.Service.GetAccounts(context.TODO(), pageNum, pageSize) - if err != nil { - return fmt.Errorf("failed to get accounts list: %w", err) - } - - return c.JSON(http.StatusOK, PaginatedDataResponse{ - Data: accounts, - Pagination: GetPaginationMetadata(total, pageNum, pageSize), - }) -} - -func Account(c echo.Context) error { - cc := c.(*ApiContext) - - account, err := cc.Service.GetAccount(context.TODO(), c.Param("id")) - if err != nil { - if err == service.ErrNotFound { - return echo.ErrNotFound - } - return fmt.Errorf("failed to get account `%s` info: %w", c.Param("id"), err) - } - - return c.JSON(http.StatusOK, DataResponse{Data: []*model.Account{account}}) -} - -func AccountDetails(c echo.Context) error { - cc := c.(*ApiContext) - pageNum, pageSize := GetPagination(cc) - accountID := c.Param("id") - var ( - response interface{} - err error - total int64 - ) - - switch c.Param("entity") { - case txs: - response, total, err = cc.Service.GetAccountTransactions(context.TODO(), accountID, pageNum, pageSize) - case rewards: - response, total, err = cc.Service.GetAccountRewards(context.TODO(), accountID, pageNum, pageSize) - default: - return echo.NewHTTPError(http.StatusNotFound, "entity not found") - } - if err != nil { - if err == service.ErrNotFound { - return echo.ErrNotFound - } - return fmt.Errorf("failed to get account entity `%s` list: %w", c.Param("entity"), err) - } - - return c.JSON(http.StatusOK, PaginatedDataResponse{ - Data: response, - Pagination: GetPaginationMetadata(total, pageNum, pageSize), - }) -} diff --git a/internal/api/handler/accounts_test.go b/internal/api/handler/accounts_test.go deleted file mode 100644 index 79b3270..0000000 --- a/internal/api/handler/accounts_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package handler_test - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestAccounts(t *testing.T) { // accounts - t.Parallel() - res := apiServer.Get(t, apiPrefix+"/accounts?pagesize=1000") - res.RequireOK(t) - var resp accountResp - res.RequireUnmarshal(t, &resp) - require.Equal(t, len(generator.Accounts), len(resp.Data)) - for _, acc := range resp.Data { - accGenerated, ok := generator.Accounts[strings.ToLower(acc.Address)] - require.True(t, ok) - // this not calculated on list endpoints - acc.LastActivity = accGenerated.Account.LastActivity - acc.Received = accGenerated.Account.Received - acc.Sent = accGenerated.Account.Sent - acc.Fees = accGenerated.Account.Fees - acc.Txs = accGenerated.Account.Txs - acc.Awards = accGenerated.Account.Awards - require.Equal(t, accGenerated.Account, acc) - } -} - -func TestAccount(t *testing.T) { // /accounts/{id} - t.Parallel() - for _, acc := range generator.Accounts { - res := apiServer.Get(t, apiPrefix+"/accounts/"+acc.Account.Address) - res.RequireOK(t) - var resp accountResp - res.RequireUnmarshal(t, &resp) - // this not calculated on list endpoints - resp.Data[0].LastActivity = acc.Account.LastActivity - resp.Data[0].Received = acc.Account.Received - resp.Data[0].Sent = acc.Account.Sent - resp.Data[0].Fees = acc.Account.Fees - resp.Data[0].Txs = acc.Account.Txs - resp.Data[0].Awards = acc.Account.Awards - require.Equal(t, 1, len(resp.Data)) - require.Equal(t, acc.Account, resp.Data[0]) - } -} - -func TestAccountTransactions(t *testing.T) { // /accounts/{id}/txs - t.Parallel() - for _, acc := range generator.Accounts { - res := apiServer.Get(t, apiPrefix+"/accounts/"+acc.Account.Address+"/txs?pagesize=1000") - res.RequireOK(t) - var resp transactionResp - res.RequireUnmarshal(t, &resp) - require.Equal(t, len(acc.Transactions), len(resp.Data)) - if len(acc.Transactions) == 0 { - continue - } - for _, tx := range resp.Data { - generatedTx, ok := acc.Transactions[tx.Id] - require.True(t, ok) - require.Equal(t, *generatedTx, tx) - } - } -} - -func TestAccountRewards(t *testing.T) { // /accounts/{id}/rewards - t.Parallel() - for _, acc := range generator.Accounts { - res := apiServer.Get(t, apiPrefix+"/accounts/"+acc.Account.Address+"/rewards?pagesize=1000") - res.RequireOK(t) - var resp rewardResp - res.RequireUnmarshal(t, &resp) - require.Equal(t, len(acc.Rewards), len(resp.Data)) - if len(acc.Rewards) == 0 { - continue - } - for _, rw := range resp.Data { - generatedRw, ok := acc.Rewards[rw.Smesher] - require.True(t, ok) - generatedRw.ID = rw.ID - require.Equal(t, *generatedRw, rw) - } - } -} diff --git a/internal/api/handler/activations.go b/internal/api/handler/activations.go deleted file mode 100644 index 64d5c29..0000000 --- a/internal/api/handler/activations.go +++ /dev/null @@ -1,38 +0,0 @@ -package handler - -import ( - "context" - "fmt" - "github.com/labstack/echo/v4" - "github.com/spacemeshos/explorer-backend/internal/service" - "net/http" - - "github.com/spacemeshos/explorer-backend/model" -) - -func Activations(c echo.Context) error { - cc := c.(*ApiContext) - pageNum, pageSize := GetPagination(c) - atxs, total, err := cc.Service.GetActivations(context.TODO(), pageNum, pageSize) - if err != nil { - return fmt.Errorf("failed to get apps info: %w", err) - } - - return c.JSON(http.StatusOK, PaginatedDataResponse{ - Data: atxs, - Pagination: GetPaginationMetadata(total, pageNum, pageSize), - }) -} - -func Activation(c echo.Context) error { - cc := c.(*ApiContext) - atx, err := cc.Service.GetActivation(context.TODO(), c.Param("id")) - if err != nil { - if err == service.ErrNotFound { - return echo.ErrNotFound - } - return fmt.Errorf("failed to get activation %s info: %w", c.Param("id"), err) - } - - return c.JSON(http.StatusOK, DataResponse{Data: []*model.Activation{atx}}) -} diff --git a/internal/api/handler/activations_test.go b/internal/api/handler/activations_test.go deleted file mode 100644 index 4f0ff73..0000000 --- a/internal/api/handler/activations_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package handler_test - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestActivations(t *testing.T) { // /atxs - t.Parallel() - insertedAtxs := generator.Epochs.GetActivations() - res := apiServer.Get(t, apiPrefix+"/atxs?pagesize=1000") - res.RequireOK(t) - var resp atxResp - res.RequireUnmarshal(t, &resp) - require.Equal(t, len(insertedAtxs), len(resp.Data)) - for _, atx := range resp.Data { - generatedAtx, ok := insertedAtxs[atx.Id] - require.True(t, ok) - require.Equal(t, generatedAtx, &atx) - } -} - -func TestActivation(t *testing.T) { // /atxs/{id} - t.Parallel() - insertedAtxs := generator.Epochs.GetActivations() - res := apiServer.Get(t, apiPrefix+"/atxs?pagesize=1000") - res.RequireOK(t) - var resp atxResp - res.RequireUnmarshal(t, &resp) - require.Equal(t, len(insertedAtxs), len(resp.Data)) - for _, atx := range resp.Data { - response := apiServer.Get(t, apiPrefix+"/atxs/"+atx.Id) - response.RequireOK(t) - var respLoop atxResp - response.RequireUnmarshal(t, &respLoop) - require.Equal(t, 1, len(respLoop.Data)) - generatedAtx, ok := insertedAtxs[atx.Id] - require.True(t, ok) - require.Equal(t, *generatedAtx, respLoop.Data[0]) - } -} diff --git a/internal/api/handler/blocks.go b/internal/api/handler/blocks.go deleted file mode 100644 index 56efbe0..0000000 --- a/internal/api/handler/blocks.go +++ /dev/null @@ -1,25 +0,0 @@ -package handler - -import ( - "context" - "fmt" - "github.com/labstack/echo/v4" - "github.com/spacemeshos/explorer-backend/internal/service" - "github.com/spacemeshos/go-spacemesh/log" - "net/http" - - "github.com/spacemeshos/explorer-backend/model" -) - -func Block(c echo.Context) error { - cc := c.(*ApiContext) - block, err := cc.Service.GetBlock(context.TODO(), c.Param("id")) - if err != nil { - if err == service.ErrNotFound { - return echo.ErrNotFound - } - log.Err(fmt.Errorf("failed to get block `%v` info: %s", block, err)) - return err - } - return c.JSON(http.StatusOK, DataResponse{Data: []*model.Block{block}}) -} diff --git a/internal/api/handler/blocks_test.go b/internal/api/handler/blocks_test.go deleted file mode 100644 index 07e7862..0000000 --- a/internal/api/handler/blocks_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package handler_test - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestBlocks(t *testing.T) { // /blocks/{id} - t.Parallel() - for _, epoch := range generator.Epochs { - for _, block := range epoch.Blocks { - res := apiServer.Get(t, apiPrefix+"/blocks/"+block.Id) - res.RequireOK(t) - var resp blockResp - res.RequireUnmarshal(t, &resp) - require.Equal(t, 1, len(resp.Data)) - require.Equal(t, block, &resp.Data[0]) - } - } -} diff --git a/internal/api/handler/epochs.go b/internal/api/handler/epochs.go deleted file mode 100644 index ebed0e4..0000000 --- a/internal/api/handler/epochs.go +++ /dev/null @@ -1,81 +0,0 @@ -package handler - -import ( - "context" - "fmt" - "github.com/labstack/echo/v4" - "github.com/spacemeshos/explorer-backend/internal/service" - "net/http" - "strconv" - - "github.com/gofiber/fiber/v2" - - "github.com/spacemeshos/explorer-backend/model" -) - -func Epochs(c echo.Context) error { - cc := c.(*ApiContext) - pageNum, pageSize := GetPagination(c) - epochs, total, err := cc.Service.GetEpochs(context.TODO(), pageNum, pageSize) - if err != nil { - return fmt.Errorf("failed to get epoch list: %w", err) - } - - return c.JSON(http.StatusOK, PaginatedDataResponse{ - Data: epochs, - Pagination: GetPaginationMetadata(total, pageNum, pageSize), - }) -} - -func Epoch(c echo.Context) error { - cc := c.(*ApiContext) - layerNum, err := strconv.Atoi(c.Param("id")) - if err != nil { - return fiber.ErrBadRequest - } - epochs, err := cc.Service.GetEpoch(context.TODO(), layerNum) - if err != nil { - if err == service.ErrNotFound { - return echo.ErrNotFound - } - return fmt.Errorf("failed to get epoch info: %w", err) - } - - return c.JSON(http.StatusOK, DataResponse{Data: []*model.Epoch{epochs}}) -} - -func EpochDetails(c echo.Context) error { - cc := c.(*ApiContext) - pageNum, pageSize := GetPagination(c) - epochID, err := strconv.Atoi(c.Param("id")) - if err != nil { - return fmt.Errorf("wrong epoch id: %w", err) - } - var ( - response interface{} - total int64 - ) - - switch c.Param("entity") { - case layers: - response, total, err = cc.Service.GetEpochLayers(context.TODO(), epochID, pageNum, pageSize) - case txs: - response, total, err = cc.Service.GetEpochTransactions(context.TODO(), epochID, pageNum, pageSize) - case smeshers: - response, total, err = cc.Service.GetEpochSmeshers(context.TODO(), epochID, pageNum, pageSize) - case rewards: - response, total, err = cc.Service.GetEpochRewards(context.TODO(), epochID, pageNum, pageSize) - case atxs: - response, total, err = cc.Service.GetEpochActivations(context.TODO(), epochID, pageNum, pageSize) - default: - return fiber.NewError(fiber.StatusNotFound, "entity not found") - } - if err != nil { - return fmt.Errorf("failed to get epoch entity `%s` list: %w", c.Param("entity"), err) - } - - return c.JSON(http.StatusOK, PaginatedDataResponse{ - Data: response, - Pagination: GetPaginationMetadata(total, pageNum, pageSize), - }) -} diff --git a/internal/api/handler/epochs_test.go b/internal/api/handler/epochs_test.go deleted file mode 100644 index 2b5c30e..0000000 --- a/internal/api/handler/epochs_test.go +++ /dev/null @@ -1,151 +0,0 @@ -package handler_test - -import ( - "fmt" - "sort" - "strings" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/spacemeshos/explorer-backend/model" -) - -func TestEpochsHandler(t *testing.T) { - t.Parallel() - responseEpochs := make([]model.Epoch, 0, 100) - pageSize := 10 - for i := 1; i <= 10; i++ { - url := apiPrefix + fmt.Sprintf("/epochs?pagesize=%d", pageSize) - if i > 0 { - url += fmt.Sprintf("&page=%d", i) - } - var loopResult epochResp - res := apiServer.Get(t, url) - res.RequireOK(t) - res.RequireUnmarshal(t, &loopResult) - responseEpochs = append(responseEpochs, loopResult.Data...) - } - require.Equal(t, len(generator.Epochs), len(responseEpochs)) - sort.Slice(responseEpochs, func(i, j int) bool { - return responseEpochs[i].Number < responseEpochs[j].Number - }) - inserted := make([]model.Epoch, 0, len(generator.Epochs)) - for i := range generator.Epochs { - generator.Epochs[i].Epoch.Stats.Current.Rewards = responseEpochs[i].Stats.Current.Rewards - generator.Epochs[i].Epoch.Stats.Cumulative.Rewards = responseEpochs[i].Stats.Cumulative.Rewards - generator.Epochs[i].Epoch.Stats.Current.RewardsNumber = responseEpochs[i].Stats.Current.RewardsNumber - generator.Epochs[i].Epoch.Stats.Cumulative.RewardsNumber = responseEpochs[i].Stats.Cumulative.RewardsNumber - inserted = append(inserted, generator.Epochs[i].Epoch) - } - require.Equal(t, responseEpochs, inserted) -} - -func TestEpochHandler(t *testing.T) { - t.Parallel() - for _, ep := range generator.Epochs { - res := apiServer.Get(t, apiPrefix+fmt.Sprintf("/epochs/%d", ep.Epoch.Number)) - res.RequireOK(t) - var loopResult epochResp - res.RequireOK(t) - res.RequireUnmarshal(t, &loopResult) - require.Equal(t, 1, len(loopResult.Data)) - - ep.Epoch.Stats.Current.Rewards = loopResult.Data[0].Stats.Current.Rewards - ep.Epoch.Stats.Cumulative.Rewards = loopResult.Data[0].Stats.Cumulative.Rewards - ep.Epoch.Stats.Current.RewardsNumber = loopResult.Data[0].Stats.Current.RewardsNumber - ep.Epoch.Stats.Cumulative.RewardsNumber = loopResult.Data[0].Stats.Cumulative.RewardsNumber - require.Equal(t, ep.Epoch, loopResult.Data[0]) - } -} - -func TestEpochLayersHandler(t *testing.T) { - t.Parallel() - for _, ep := range generator.Epochs { - data := make(map[uint32]model.Layer, len(ep.Layers)) - for _, l := range ep.Layers { - data[l.Layer.Number] = l.Layer - } - - res := apiServer.Get(t, apiPrefix+fmt.Sprintf("/epochs/%d/layers", ep.Epoch.Number)) - res.RequireOK(t) - var loopResult layerResp - res.RequireUnmarshal(t, &loopResult) - require.Equal(t, len(ep.Layers), len(loopResult.Data)) - for _, l := range loopResult.Data { - generatedLayer, ok := data[l.Number] - generatedLayer.Rewards = l.Rewards - require.True(t, ok) - require.Equal(t, generatedLayer, l) - } - } -} - -func TestEpochTxsHandler(t *testing.T) { - t.Parallel() - for _, ep := range generator.Epochs { - res := apiServer.Get(t, apiPrefix+fmt.Sprintf("/epochs/%d/txs?pagesize=100", ep.Epoch.Number)) - res.RequireOK(t) - var loopResult transactionResp - res.RequireUnmarshal(t, &loopResult) - require.Equal(t, len(ep.Transactions), len(loopResult.Data)) - for _, tx := range loopResult.Data { - generatedTx, ok := ep.Transactions[tx.Id] - require.True(t, ok) - require.Equal(t, generatedTx, &tx) - } - } -} - -func TestEpochSmeshersHandler(t *testing.T) { - t.Parallel() - for _, ep := range generator.Epochs { - res := apiServer.Get(t, apiPrefix+fmt.Sprintf("/epochs/%d/smeshers?pagesize=1000", ep.Epoch.Number)) - res.RequireOK(t) - var loopResult smesherResp - res.RequireUnmarshal(t, &loopResult) - require.Equal(t, len(ep.Smeshers), len(loopResult.Data)) - for _, smesher := range loopResult.Data { - generatedSmesher, ok := ep.Smeshers[strings.ToLower(smesher.Id)] - require.True(t, ok) - smesher.Rewards = generatedSmesher.Rewards // this not calculated on list endpoints, simply set as 0. - smesher.Timestamp = generatedSmesher.Timestamp - smesher.AtxLayer = generatedSmesher.AtxLayer - smesher.Epochs = generatedSmesher.Epochs - require.Equal(t, *generatedSmesher, smesher) - } - } -} - -func TestEpochRewardsHandler(t *testing.T) { - t.Parallel() - for _, ep := range generator.Epochs { - res := apiServer.Get(t, apiPrefix+fmt.Sprintf("/epochs/%d/rewards?pagesize=100", ep.Epoch.Number)) - res.RequireOK(t) - var loopResult rewardResp - res.RequireUnmarshal(t, &loopResult) - require.Equal(t, len(ep.Rewards), len(loopResult.Data)) - for _, rw := range loopResult.Data { - generatedRw, ok := ep.Rewards[rw.Smesher] - require.True(t, ok) - generatedRw.ID = rw.ID - require.Equal(t, *generatedRw, rw) - } - } -} - -func TestEpochAtxsHandler(t *testing.T) { - t.Parallel() - for _, ep := range generator.Epochs { - res := apiServer.Get(t, apiPrefix+fmt.Sprintf("/epochs/%d/atxs?pagesize=100", ep.Epoch.Number)) - res.RequireOK(t) - var loopResult atxResp - res.RequireUnmarshal(t, &loopResult) - require.Equal(t, len(ep.Activations), len(loopResult.Data)) - for _, atx := range loopResult.Data { - generatedAtx, ok := ep.Activations[atx.Id] - require.True(t, ok) - require.Equal(t, *generatedAtx, atx) - } - } -} diff --git a/internal/api/handler/handler.go b/internal/api/handler/handler.go deleted file mode 100644 index 856d868..0000000 --- a/internal/api/handler/handler.go +++ /dev/null @@ -1,37 +0,0 @@ -package handler - -import ( - "github.com/gorilla/websocket" - "github.com/labstack/echo/v4" - "github.com/spacemeshos/explorer-backend/internal/service" -) - -const ( - // list of items to search from GET request. - txs = "txs" - atxs = "atxs" - blocks = "blocks" - layers = "layers" - rewards = "rewards" - smeshers = "smeshers" -) - -var Upgrader = websocket.Upgrader{} - -type ApiContext struct { - echo.Context - Service service.AppService -} - -type DataResponse struct { - Data interface{} `json:"data"` -} - -type PaginatedDataResponse struct { - Data interface{} `json:"data"` - Pagination PaginationMetadata `json:"pagination"` -} - -type RedirectResponse struct { - Redirect string `json:"redirect"` -} diff --git a/internal/api/handler/handler_test.go b/internal/api/handler/handler_test.go deleted file mode 100644 index 972aebc..0000000 --- a/internal/api/handler/handler_test.go +++ /dev/null @@ -1,138 +0,0 @@ -package handler_test - -import ( - "context" - "fmt" - "github.com/spacemeshos/explorer-backend/model" - "os" - "testing" - "time" - - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/internal/storage/storagereader" - "github.com/spacemeshos/explorer-backend/storage" - "github.com/spacemeshos/explorer-backend/test/testseed" - "github.com/spacemeshos/explorer-backend/test/testserver" -) - -const testAPIServiceDB = "explorer_test" - -var ( - apiServer *testserver.TestAPIService - generator *testseed.SeedGenerator - seed *testseed.TestServerSeed - dbPort = 27017 -) - -func TestMain(m *testing.M) { - mongoURL := fmt.Sprintf("mongodb://localhost:%d", dbPort) - client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(mongoURL)) - if err != nil { - fmt.Println("failed to connect to mongo", err) - os.Exit(1) - } - database := client.Database(testAPIServiceDB) - if database != nil { - if err = database.Drop(context.Background()); err != nil { - fmt.Println("failed to drop db", err) - os.Exit(1) - } - } - - db, err := storage.New(context.Background(), mongoURL, testAPIServiceDB) - if err != nil { - fmt.Println("failed to init storage to mongo", err) - os.Exit(1) - } - seed = testseed.GetServerSeed() - db.OnNetworkInfo(string(seed.GenesisID), seed.GenesisTime, seed.EpochNumLayers, seed.MaxTransactionPerSecond, seed.LayersDuration, seed.GetPostUnitsSize()) - - dbReader, err := storagereader.NewStorageReader(context.Background(), mongoURL, testAPIServiceDB) - if err != nil { - fmt.Println("failed to init storage to mongo", err) - os.Exit(1) - } - - apiServer, err = testserver.StartTestAPIServiceV2(db, dbReader) - // old version of app here apiServer, err = testserver.StartTestAPIService(dbPort, db) - if err != nil { - fmt.Println("failed to start test api service", err) - os.Exit(1) - } - generator = testseed.NewSeedGenerator(testseed.GetServerSeed()) - if err = generator.GenerateEpoches(10); err != nil { - fmt.Println("failed to generate epochs", err) - os.Exit(1) - } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - if err = generator.SaveEpoches(ctx, db); err != nil { - fmt.Println("failed to save generated epochs", err) - os.Exit(1) - } - - code := m.Run() - db.Close() - os.Exit(code) -} - -type layerResp struct { - Data []model.Layer `json:"data"` - Pagination pagination `json:"pagination"` -} - -type epochResp struct { - Data []model.Epoch `json:"data"` - Pagination pagination `json:"pagination"` -} - -type transactionResp struct { - Data []model.Transaction `json:"data"` - Pagination pagination `json:"pagination"` -} - -type smesherResp struct { - Data []model.Smesher `json:"data"` - Pagination pagination `json:"pagination"` -} - -type rewardResp struct { - Data []model.Reward `json:"data"` - Pagination pagination `json:"pagination"` -} - -type accountResp struct { - Data []model.Account `json:"data"` - Pagination pagination `json:"pagination"` -} - -type atxResp struct { - Data []model.Activation `json:"data"` - Pagination pagination `json:"pagination"` -} -type blockResp struct { - Data []model.Block `json:"data"` - Pagination pagination `json:"pagination"` -} - -type appResp struct { - Data []model.App `json:"data"` - Pagination pagination `json:"pagination"` -} - -type redirect struct { - Redirect string `json:"redirect"` -} - -type pagination struct { - TotalCount int `json:"totalCount"` - PageCount int `json:"pageCount"` - PerPage int `json:"perPage"` - Next int `json:"next"` - HasNext bool `json:"hasNext"` - Current int `json:"current"` - Previous int `json:"previous"` - HasPrevious bool `json:"hasPrevious"` -} diff --git a/internal/api/handler/layers.go b/internal/api/handler/layers.go deleted file mode 100644 index 4ad6551..0000000 --- a/internal/api/handler/layers.go +++ /dev/null @@ -1,82 +0,0 @@ -package handler - -import ( - "context" - "fmt" - "github.com/labstack/echo/v4" - "github.com/spacemeshos/explorer-backend/internal/service" - "net/http" - "strconv" - - "github.com/gofiber/fiber/v2" - - "github.com/spacemeshos/explorer-backend/model" -) - -func Layers(c echo.Context) error { - cc := c.(*ApiContext) - pageNum, pageSize := GetPagination(c) - layersList, total, err := cc.Service.GetLayers(context.TODO(), pageNum, pageSize) - if err != nil { - return fmt.Errorf("failed to get epoch list: %w", err) - } - - return c.JSON(http.StatusOK, PaginatedDataResponse{ - Data: layersList, - Pagination: GetPaginationMetadata(total, pageNum, pageSize), - }) -} - -func Layer(c echo.Context) error { - cc := c.(*ApiContext) - layerID, err := strconv.Atoi(c.Param("id")) - if err != nil { - return c.NoContent(http.StatusBadRequest) - } - - layer, err := cc.Service.GetLayer(context.TODO(), layerID) - if err != nil { - if err == service.ErrNotFound { - return echo.ErrNotFound - } - return fmt.Errorf("failed to get layer info: %w", err) - } - - return c.JSON(http.StatusOK, DataResponse{Data: []*model.Layer{layer}}) -} - -func LayerDetails(c echo.Context) error { - cc := c.(*ApiContext) - pageNum, pageSize := GetPagination(c) - layerID, err := strconv.Atoi(c.Param("id")) - if err != nil { - return fmt.Errorf("wrong layer id: %w", err) - } - var ( - response interface{} - total int64 - ) - - switch c.Param("entity") { - case blocks: - response, total, err = cc.Service.GetLayerBlocks(context.TODO(), layerID, pageNum, pageSize) - case txs: - response, total, err = cc.Service.GetLayerTransactions(context.TODO(), layerID, pageNum, pageSize) - case smeshers: - response, total, err = cc.Service.GetLayerSmeshers(context.TODO(), layerID, pageNum, pageSize) - case rewards: - response, total, err = cc.Service.GetLayerRewards(context.TODO(), layerID, pageNum, pageSize) - case atxs: - response, total, err = cc.Service.GetLayerActivations(context.TODO(), layerID, pageNum, pageSize) - default: - return fiber.NewError(fiber.StatusNotFound, "entity not found") - } - if err != nil { - return fmt.Errorf("failed to get layer entity `%s` list: %w", c.Param("entity"), err) - } - - return c.JSON(http.StatusOK, PaginatedDataResponse{ - Data: response, - Pagination: GetPaginationMetadata(total, pageNum, pageSize), - }) -} diff --git a/internal/api/handler/layers_test.go b/internal/api/handler/layers_test.go deleted file mode 100644 index 5297a9f..0000000 --- a/internal/api/handler/layers_test.go +++ /dev/null @@ -1,138 +0,0 @@ -package handler_test - -import ( - "fmt" - "strings" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/spacemeshos/explorer-backend/model" -) - -func TestLayers(t *testing.T) { // /layers - t.Parallel() - var result layerResp - insertedLayers := generator.Epochs.GetLayers() - res := apiServer.Get(t, apiPrefix+"/layers?pagesize=100") - res.RequireOK(t) - res.RequireUnmarshal(t, &result) - for i := range insertedLayers { - insertedLayers[i].Rewards = result.Data[i].Rewards - } - require.Equal(t, insertedLayers, result.Data) -} - -func TestLayer(t *testing.T) { // /layers/{id:[0-9]+} - t.Parallel() - insertedLayers := generator.Epochs.GetLayers() - for _, layer := range insertedLayers { - res := apiServer.Get(t, apiPrefix+fmt.Sprintf("/layers/%d", layer.Number)) - res.RequireOK(t) - var loopResp layerResp - res.RequireUnmarshal(t, &loopResp) - require.Equal(t, 1, len(loopResp.Data)) - } -} - -func TestLayerTxs(t *testing.T) { // /layers/{id:[0-9]+}/txs - t.Parallel() - for _, epoch := range generator.Epochs { - for _, layerContainer := range epoch.Layers { - res := apiServer.Get(t, apiPrefix+fmt.Sprintf("/layers/%d/txs", layerContainer.Layer.Number)) - res.RequireOK(t) - var loopResp transactionResp - res.RequireUnmarshal(t, &loopResp) - layerTx := make(map[string]model.Transaction, len(epoch.Transactions)) - for _, tx := range epoch.Transactions { - if tx.Layer != layerContainer.Layer.Number { - continue - } - layerTx[tx.Id] = *tx - } - require.Equal(t, len(layerTx), len(loopResp.Data)) - if len(layerTx) == 0 { - continue - } - for _, tx := range loopResp.Data { - generatedTx, ok := layerTx[tx.Id] - require.True(t, ok) - require.Equal(t, generatedTx, tx) - } - } - } -} - -func TestLayerSmeshers(t *testing.T) { // /layers/{id:[0-9]+}/smeshers - t.Parallel() - for _, epoch := range generator.Epochs { - for _, layerContainer := range epoch.Layers { - res := apiServer.Get(t, apiPrefix+fmt.Sprintf("/layers/%d/smeshers", layerContainer.Layer.Number)) - res.RequireOK(t) - var loopResp smesherResp - res.RequireUnmarshal(t, &loopResp) - for _, smesher := range loopResp.Data { - generatedSmesher, ok := epoch.Smeshers[strings.ToLower(smesher.Id)] - require.True(t, ok) - smesher.Rewards = generatedSmesher.Rewards // this not calculated on list endpoints, simply set as 0. - smesher.Timestamp = generatedSmesher.Timestamp - smesher.AtxLayer = generatedSmesher.AtxLayer - require.Equal(t, *generatedSmesher, smesher) - } - } - } -} - -func TestLayerBlocks(t *testing.T) { // /layers/{id:[0-9]+}/blocks - t.Parallel() - for _, epoch := range generator.Epochs { - for _, layerContainer := range epoch.Layers { - res := apiServer.Get(t, apiPrefix+fmt.Sprintf("/layers/%d/blocks", layerContainer.Layer.Number)) - res.RequireOK(t) - var loopResp blockResp - res.RequireUnmarshal(t, &loopResp) - for _, block := range loopResp.Data { - generatedBlock, ok := epoch.Blocks[block.Id] - require.True(t, ok) - require.Equal(t, generatedBlock, &block) - } - } - } -} - -func TestLayerRewards(t *testing.T) { // /layers/{id:[0-9]+}/rewards - t.Parallel() - for _, epoch := range generator.Epochs { - for _, layerContainer := range epoch.Layers { - res := apiServer.Get(t, apiPrefix+fmt.Sprintf("/layers/%d/rewards", layerContainer.Layer.Number)) - res.RequireOK(t) - var loopResp rewardResp - res.RequireUnmarshal(t, &loopResp) - for _, tx := range loopResp.Data { - generatedRw, ok := epoch.Rewards[tx.Smesher] - require.True(t, ok) - generatedRw.ID = tx.ID - require.Equal(t, generatedRw, &tx) - } - } - } -} - -func TestLayerAtxs(t *testing.T) { // /layers/{id:[0-9]+}/atxs - t.Parallel() - for _, epoch := range generator.Epochs { - for _, layerContainer := range epoch.Layers { - res := apiServer.Get(t, apiPrefix+fmt.Sprintf("/layers/%d/atxs", layerContainer.Layer.Number)) - res.RequireOK(t) - var loopResp atxResp - res.RequireUnmarshal(t, &loopResp) - for _, tx := range loopResp.Data { - generatedAtx, ok := epoch.Activations[tx.Id] - require.True(t, ok) - // TargetEpoch is not stored in db - tx.TargetEpoch = 0 - require.Equal(t, generatedAtx, &tx) - } - } - } -} diff --git a/internal/api/handler/network_info.go b/internal/api/handler/network_info.go deleted file mode 100644 index 217f24b..0000000 --- a/internal/api/handler/network_info.go +++ /dev/null @@ -1,96 +0,0 @@ -package handler - -import ( - "context" - "errors" - "fmt" - "github.com/gorilla/websocket" - "github.com/labstack/echo/v4" - "github.com/spacemeshos/go-spacemesh/log" - "net/http" - "syscall" - "time" - - "github.com/gofiber/fiber/v2" -) - -func HealthzHandler(c echo.Context) error { - cc := c.(*ApiContext) - if err := cc.Service.Ping(context.TODO()); err != nil { - return fiber.NewError(fiber.StatusServiceUnavailable, err.Error()) - } - return c.String(http.StatusOK, "OK") -} - -func Synced(c echo.Context) error { - cc := c.(*ApiContext) - networkInfo, err := cc.Service.GetNetworkInfo(context.TODO()) - if err != nil { - return fmt.Errorf("failed to check is synced: %w", err) - } - - if !networkInfo.IsSynced { - return c.String(http.StatusTooEarly, "SYNCING") - } - - return c.String(http.StatusOK, "SYNCED") -} - -func NetworkInfo(c echo.Context) error { - cc := c.(*ApiContext) - networkInfo, epoch, layer, err := cc.Service.GetState(context.TODO()) - if err != nil { - return fmt.Errorf("failed to get current state info: %w", err) - } - - return c.JSON(http.StatusOK, map[string]interface{}{ - "network": networkInfo, - "layer": layer, - "epoch": epoch, - }) -} - -func NetworkInfoWS(c echo.Context) error { - ws, err := Upgrader.Upgrade(c.Response(), c.Request(), nil) - if err != nil { - log.Err(fmt.Errorf("NetworkInfoWS: upgrade error: %w\n", err)) - return nil - } - defer ws.Close() - - ticker := time.NewTicker(30 * time.Second) - defer ticker.Stop() - - for ; true; <-ticker.C { - if err := serveNetworkInfo(c, ws); err != nil { - if !errors.Is(err, syscall.EPIPE) { - log.Err(fmt.Errorf("NetworkInfoWS: serve network info: %s", err)) - return nil - } - } - } - - return nil -} - -func serveNetworkInfo(c echo.Context, ws *websocket.Conn) error { - cc := c.(*ApiContext) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - networkInfo, epoch, layer, err := cc.Service.GetState(ctx) - if err != nil { - return fmt.Errorf("failed to get current state info: %w", err) - } - - if err = ws.WriteJSON(map[string]interface{}{ - "network": networkInfo, - "layer": layer, - "epoch": epoch, - }); err != nil { - return fmt.Errorf("serve network info: %w", err) - } - - return nil -} diff --git a/internal/api/handler/network_info_test.go b/internal/api/handler/network_info_test.go deleted file mode 100644 index 0932163..0000000 --- a/internal/api/handler/network_info_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package handler_test - -import ( - "encoding/json" - "net/http" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -const apiPrefix = "" // will be replaced to v2 after some endpoints will be refactored. -type networkResp struct { - Network struct { - Genesisid string `json:"genesisid"` - Genesis uint64 `json:"genesis"` - Layers uint32 `json:"layers"` - Maxtx uint64 `json:"maxtx"` - Duration uint64 `json:"duration"` - Lastlayer int `json:"lastlayer"` - Lastlayerts int `json:"lastlayerts"` - Lastapprovedlayer int `json:"lastapprovedlayer"` - Lastconfirmedlayer int `json:"lastconfirmedlayer"` - Connectedpeers int `json:"connectedpeers"` - Issynced bool `json:"issynced"` - Syncedlayer int `json:"syncedlayer"` - Toplayer int `json:"toplayer"` - Verifiedlayer int `json:"verifiedlayer"` - } `json:"network"` -} - -func TestNetworkInfoHandler(t *testing.T) { - res := apiServer.Get(t, apiPrefix+"/network-info") - var networkInfo networkResp - res.RequireOK(t) - res.RequireUnmarshal(t, &networkInfo) - compareNetworkInfo(t, networkInfo) -} - -func TestSyncedHandler(t *testing.T) { - res := apiServer.Get(t, apiPrefix+"/synced") - res.RequireTooEarly(t) - apiServer.Storage.OnNodeStatus(10, true, 11, 12, 13) - require.Eventually(t, func() bool { - res = apiServer.Get(t, apiPrefix+"/synced") - return res.Res.StatusCode == http.StatusOK - }, 4*time.Second, 1*time.Second) -} - -func TestWSNetworkInfoHandler(t *testing.T) { - t.Parallel() - res := apiServer.GetReadWS(t, apiPrefix+"/ws/network-info") - counter := 0 -loop: - for { - select { - case c := <-res: - var networkInfo networkResp - require.NoError(t, json.Unmarshal(c, &networkInfo)) - compareNetworkInfo(t, networkInfo) - counter++ - - case <-time.After(2 * time.Second): - break loop - } - } - require.NotZero(t, counter) -} - -func compareNetworkInfo(t *testing.T, networkInfo networkResp) { - require.Equal(t, string(seed.GenesisID), networkInfo.Network.Genesisid) - require.Equal(t, seed.GenesisTime, networkInfo.Network.Genesis) - require.Equal(t, seed.EpochNumLayers, networkInfo.Network.Layers) - require.Equal(t, seed.MaxTransactionPerSecond, networkInfo.Network.Maxtx) - require.Equal(t, seed.LayersDuration, networkInfo.Network.Duration) -} diff --git a/internal/api/handler/pagination.go b/internal/api/handler/pagination.go deleted file mode 100644 index af586fe..0000000 --- a/internal/api/handler/pagination.go +++ /dev/null @@ -1,56 +0,0 @@ -package handler - -import ( - "github.com/labstack/echo/v4" - "strconv" -) - -type PaginationMetadata struct { - TotalCount int64 `json:"totalCount"` - PageCount int64 `json:"pageCount"` - PerPage int64 `json:"perPage"` - Next int64 `json:"next"` - HasNext bool `json:"hasNext"` - HasPrevious bool `json:"hasPrevious"` - Current int64 `json:"current"` - Previous int64 `json:"previous"` -} - -func GetPagination(c echo.Context) (pageNumber, pageSize int64) { - pageNumber = 1 - pageSize = 20 - if page := c.QueryParam("page"); page != "" { - pageNumber, _ = strconv.ParseInt(page, 10, 32) - if pageNumber <= 0 { - pageNumber = 1 - } - } - if size := c.QueryParam("pagesize"); size != "" { - pageSize, _ = strconv.ParseInt(size, 10, 32) - if pageSize <= 0 { - pageSize = 20 - } - } - return pageNumber, pageSize -} - -func GetPaginationMetadata(total int64, pageNumber int64, pageSize int64) PaginationMetadata { - pageCount := (total + pageSize - 1) / pageSize - result := PaginationMetadata{ - TotalCount: total, - PageCount: pageNumber, - PerPage: pageSize, - Next: pageCount, - Current: pageNumber, - Previous: 1, - } - if pageNumber < pageCount { - result.Next = pageNumber + 1 - result.HasNext = true - } - if pageNumber > 1 { - result.Previous = pageNumber - 1 - result.HasPrevious = true - } - return result -} diff --git a/internal/api/handler/rewards.go b/internal/api/handler/rewards.go deleted file mode 100644 index 9c90fc7..0000000 --- a/internal/api/handler/rewards.go +++ /dev/null @@ -1,72 +0,0 @@ -package handler - -import ( - "context" - "fmt" - "github.com/labstack/echo/v4" - "github.com/spacemeshos/explorer-backend/internal/service" - "go.mongodb.org/mongo-driver/bson" - "net/http" - "strconv" - - "github.com/spacemeshos/explorer-backend/model" -) - -func Rewards(c echo.Context) error { - cc := c.(*ApiContext) - pageNum, pageSize := GetPagination(c) - rewardsList, total, err := cc.Service.GetRewards(context.TODO(), pageNum, pageSize) - if err != nil { - return fmt.Errorf("failed to get rewards info: %w", err) - } - - return c.JSON(http.StatusOK, PaginatedDataResponse{ - Data: rewardsList, - Pagination: GetPaginationMetadata(total, pageNum, pageSize), - }) -} - -func Reward(c echo.Context) error { - cc := c.(*ApiContext) - reward, err := cc.Service.GetReward(context.TODO(), c.Param("id")) - if err != nil { - if err == service.ErrNotFound { - return echo.ErrNotFound - } - return fmt.Errorf("failed to get reward `%s` info: %w", c.Param("id"), err) - } - - return c.JSON(http.StatusOK, DataResponse{Data: []*model.Reward{reward}}) -} - -func RewardV2(c echo.Context) error { - cc := c.(*ApiContext) - layer := c.Param("layer") - layerId, err := strconv.Atoi(layer) - if err != nil { - return c.NoContent(http.StatusBadRequest) - } - reward, err := cc.Service.GetRewardV2(context.TODO(), c.Param("smesherId"), uint32(layerId)) - if err != nil { - if err == service.ErrNotFound { - return echo.ErrNotFound - } - return fmt.Errorf("failed to get reward `%s` info: %w", c.Param("id"), err) - } - - return c.JSON(http.StatusOK, DataResponse{Data: []*model.Reward{reward}}) -} - -func TotalRewards(c echo.Context) error { - cc := c.(*ApiContext) - - total, count, err := cc.Service.GetTotalRewards(context.TODO(), &bson.D{}) - if err != nil { - return fmt.Errorf("failed to get total rewards. info: %w", err) - } - - return c.JSON(http.StatusOK, DataResponse{Data: map[string]interface{}{ - "rewards": total, - "count": count, - }}) -} diff --git a/internal/api/handler/rewards_test.go b/internal/api/handler/rewards_test.go deleted file mode 100644 index b049af3..0000000 --- a/internal/api/handler/rewards_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package handler_test - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestRewards(t *testing.T) { //"/rewards" - t.Parallel() - insertedRewards := generator.Epochs.GetRewards() - res := apiServer.Get(t, apiPrefix+"/rewards?pagesize=1000") - res.RequireOK(t) - var resp rewardResp - res.RequireUnmarshal(t, &resp) - require.Equal(t, len(insertedRewards), len(resp.Data)) - for _, reward := range resp.Data { - rw, ok := insertedRewards[reward.Smesher] - require.True(t, ok) - rw.ID = "" - reward.ID = "" - require.Equal(t, rw, &reward) - } -} - -func TestReward(t *testing.T) { //"/rewards/{id}" - t.Parallel() - type rewardRespWithID struct { - Data []struct { - ID string `json:"_id"` - Layer int `json:"layer"` - Total int64 `json:"total"` - LayerReward int `json:"layerReward"` - LayerComputed int `json:"layerComputed"` - Coinbase string `json:"coinbase"` - Smesher string `json:"smesher"` - Space int `json:"space"` - Timestamp int `json:"timestamp"` - } `json:"data"` - Pagination pagination `json:"pagination"` - } - insertedRewards := generator.Epochs.GetRewards() - - res := apiServer.Get(t, apiPrefix+"/rewards?pagesize=100") - res.RequireOK(t) - var resp rewardRespWithID - res.RequireUnmarshal(t, &resp) - for _, reward := range resp.Data { - require.True(t, reward.ID != "") - resSingleReward := apiServer.Get(t, apiPrefix+"/rewards/"+reward.ID) - resSingleReward.RequireOK(t) - var respLoop rewardResp - resSingleReward.RequireUnmarshal(t, &respLoop) - require.Equal(t, 1, len(respLoop.Data)) - rw, ok := insertedRewards[reward.Smesher] - require.True(t, ok) - rw.ID = "" - respLoop.Data[0].ID = "" // this id generates by mongo, reset it. // todo probably bag, reward should have db independed id - require.Equal(t, rw, &respLoop.Data[0]) - } -} diff --git a/internal/api/handler/search.go b/internal/api/handler/search.go deleted file mode 100644 index 7d73b2c..0000000 --- a/internal/api/handler/search.go +++ /dev/null @@ -1,23 +0,0 @@ -package handler - -import ( - "context" - "fmt" - "github.com/labstack/echo/v4" - "net/http" - "strings" -) - -func Search(c echo.Context) error { - cc := c.(*ApiContext) - - search := strings.ToLower(c.Param("id")) - redirectURL, err := cc.Service.Search(context.TODO(), search) - if err != nil { - return fmt.Errorf("error search `%s`: %w", search, err) - } - - return c.JSON(http.StatusOK, RedirectResponse{ - Redirect: redirectURL, - }) -} diff --git a/internal/api/handler/search_test.go b/internal/api/handler/search_test.go deleted file mode 100644 index ba23b2d..0000000 --- a/internal/api/handler/search_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package handler_test - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestSearch(t *testing.T) { // /search/{id} - t.Parallel() - for _, epoch := range generator.Epochs { - // transactions - for _, tx := range epoch.Transactions { - res := apiServer.Get(t, apiPrefix+fmt.Sprintf("/search/%s", tx.Id)) - res.RequireOK(t) - var loopResp redirect - res.RequireUnmarshal(t, &loopResp) - res = apiServer.Get(t, loopResp.Redirect) - res.RequireOK(t) - var txResp transactionResp - res.RequireUnmarshal(t, &txResp) - require.Equal(t, 1, len(txResp.Data)) - require.Equal(t, tx, &txResp.Data[0]) - } - - // layer - for _, layerContainer := range epoch.Layers { - res := apiServer.Get(t, apiPrefix+fmt.Sprintf("/search/%d", layerContainer.Layer.Number)) - res.RequireOK(t) - var loopResp redirect - res.RequireUnmarshal(t, &loopResp) - res = apiServer.Get(t, loopResp.Redirect) - res.RequireOK(t) - var resp layerResp - res.RequireUnmarshal(t, &resp) - require.Equal(t, 1, len(resp.Data)) - layerContainer.Layer.Rewards = resp.Data[0].Rewards - require.Equal(t, layerContainer.Layer, resp.Data[0]) - } - } - // account - for _, acc := range generator.Accounts { - res := apiServer.Get(t, apiPrefix+fmt.Sprintf("/search/%s", acc.Account.Address)) - res.RequireOK(t) - var loopResp redirect - res.RequireUnmarshal(t, &loopResp) - res = apiServer.Get(t, loopResp.Redirect) - res.RequireOK(t) - var resp accountResp - res.RequireUnmarshal(t, &resp) - require.Equal(t, 1, len(resp.Data)) - resp.Data[0].LastActivity = 0 - resp.Data[0].Awards = 0 - resp.Data[0].Fees = 0 - resp.Data[0].Sent = 0 - resp.Data[0].Received = 0 - require.Equal(t, acc.Account, resp.Data[0]) - } -} diff --git a/internal/api/handler/smeshers.go b/internal/api/handler/smeshers.go deleted file mode 100644 index 4b51a77..0000000 --- a/internal/api/handler/smeshers.go +++ /dev/null @@ -1,69 +0,0 @@ -package handler - -import ( - "context" - "fmt" - "github.com/labstack/echo/v4" - "github.com/spacemeshos/explorer-backend/internal/service" - "net/http" - - "github.com/gofiber/fiber/v2" - "github.com/spacemeshos/go-spacemesh/log" - - "github.com/spacemeshos/explorer-backend/model" -) - -func Smeshers(c echo.Context) error { - cc := c.(*ApiContext) - pageNum, pageSize := GetPagination(c) - smeshersList, total, err := cc.Service.GetSmeshers(context.TODO(), pageNum, pageSize) - if err != nil { - log.Err(fmt.Errorf("failed to get smeshers list: %s", err)) - return err - } - - return c.JSON(http.StatusOK, PaginatedDataResponse{ - Data: smeshersList, - Pagination: GetPaginationMetadata(total, pageNum, pageSize), - }) -} - -func Smesher(c echo.Context) error { - cc := c.(*ApiContext) - smesher, err := cc.Service.GetSmesher(context.TODO(), c.Param("id")) - if err != nil { - if err == service.ErrNotFound { - return echo.ErrNotFound - } - return fmt.Errorf("failed to get smesher: %w", err) - } - - return c.JSON(http.StatusOK, DataResponse{Data: []*model.Smesher{smesher}}) -} - -func SmesherDetails(c echo.Context) error { - cc := c.(*ApiContext) - var ( - response interface{} - err error - total int64 - ) - pageNum, pageSize := GetPagination(c) - switch c.Param("entity") { - case atxs: - response, total, err = cc.Service.GetSmesherActivations(context.TODO(), c.Param("id"), pageNum, pageSize) - case rewards: - response, total, err = cc.Service.GetSmesherRewards(context.TODO(), c.Param("id"), pageNum, pageSize) - default: - return fiber.NewError(fiber.StatusNotFound, "entity not found") - } - if err != nil { - log.Err(fmt.Errorf("failed to get smesher entity `%s` details: %s", c.Param("entity"), err)) - return err - } - - return c.JSON(http.StatusOK, PaginatedDataResponse{ - Data: response, - Pagination: GetPaginationMetadata(total, pageNum, pageSize), - }) -} diff --git a/internal/api/handler/smeshers_test.go b/internal/api/handler/smeshers_test.go deleted file mode 100644 index d37999e..0000000 --- a/internal/api/handler/smeshers_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package handler_test - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestSmeshersHandler(t *testing.T) { // /smeshers - t.Parallel() - res := apiServer.Get(t, apiPrefix+"/smeshers?pagesize=1000") - res.RequireOK(t) - var resp smesherResp - res.RequireUnmarshal(t, &resp) - require.Equal(t, len(generator.Smeshers), len(resp.Data)) - for _, smesher := range resp.Data { - generatedSmesher, ok := generator.Smeshers[strings.ToLower(smesher.Id)] - require.True(t, ok) - smesher.Rewards = generatedSmesher.Rewards // for this endpoint we not calculate extra values, cause not use this field - smesher.Timestamp = generatedSmesher.Timestamp - smesher.AtxLayer = generatedSmesher.AtxLayer - smesher.Epochs = generatedSmesher.Epochs - require.Equal(t, generatedSmesher, &smesher) - } -} - -func TestSmesherHandler(t *testing.T) { // /smeshers/{id} - t.Parallel() - for _, smesher := range generator.Smeshers { - res := apiServer.Get(t, apiPrefix+"/smeshers/"+smesher.Id) - res.RequireOK(t) - var resp smesherResp - res.RequireUnmarshal(t, &resp) - require.Equal(t, 1, len(resp.Data)) - smesher.Epochs = resp.Data[0].Epochs - require.Equal(t, *smesher, resp.Data[0]) - } -} - -func TestSmesherAtxsHandler(t *testing.T) { // /smeshers/{id}/atxs - t.Parallel() - for _, epoch := range generator.Epochs { - for _, smesher := range epoch.Smeshers { - res := apiServer.Get(t, apiPrefix+"/smeshers/"+smesher.Id+"/atxs") - res.RequireOK(t) - var resp atxResp - res.RequireUnmarshal(t, &resp) - require.Equal(t, 1, len(resp.Data)) - atx, ok := epoch.Activations[resp.Data[0].Id] - require.True(t, ok) - require.Equal(t, *atx, resp.Data[0]) - } - } -} - -func TestSmesherRewardsHandler(t *testing.T) { // /smeshers/{id}/rewards - t.Parallel() - for _, epoch := range generator.Epochs { - for _, smesher := range epoch.Smeshers { - res := apiServer.Get(t, apiPrefix+"/smeshers/"+smesher.Id+"/rewards") - res.RequireOK(t) - var resp rewardResp - res.RequireUnmarshal(t, &resp) - require.Equal(t, 1, len(resp.Data)) - rw, ok := generator.Rewards[resp.Data[0].Smesher] - require.True(t, ok) - rw.ID = resp.Data[0].ID - require.Equal(t, *rw, resp.Data[0]) - } - } -} diff --git a/internal/api/handler/transactions.go b/internal/api/handler/transactions.go deleted file mode 100644 index d556d08..0000000 --- a/internal/api/handler/transactions.go +++ /dev/null @@ -1,38 +0,0 @@ -package handler - -import ( - "context" - "fmt" - "github.com/labstack/echo/v4" - "github.com/spacemeshos/explorer-backend/internal/service" - "net/http" - - "github.com/spacemeshos/explorer-backend/model" -) - -func Transactions(c echo.Context) error { - cc := c.(*ApiContext) - pageNum, pageSize := GetPagination(c) - txs, total, err := cc.Service.GetTransactions(context.TODO(), pageNum, pageSize) - if err != nil { - return fmt.Errorf("failed to get transactions list: %w", err) - } - - return c.JSON(http.StatusOK, PaginatedDataResponse{ - Data: txs, - Pagination: GetPaginationMetadata(total, pageNum, pageSize), - }) -} - -func Transaction(c echo.Context) error { - cc := c.(*ApiContext) - tx, err := cc.Service.GetTransaction(context.TODO(), c.Param("id")) - if err != nil { - if err == service.ErrNotFound { - return echo.ErrNotFound - } - return fmt.Errorf("failed to get transaction %s list: %s", c.Param("id"), err) - } - - return c.JSON(http.StatusOK, DataResponse{Data: []*model.Transaction{tx}}) -} diff --git a/internal/api/handler/transactions_test.go b/internal/api/handler/transactions_test.go deleted file mode 100644 index 841b6b1..0000000 --- a/internal/api/handler/transactions_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package handler_test - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestTransactions(t *testing.T) { // /txs - t.Parallel() - insertedTxs := generator.Epochs.GetTransactions() - res := apiServer.Get(t, apiPrefix+"/txs?pagesize=1000") - res.RequireOK(t) - var resp transactionResp - res.RequireUnmarshal(t, &resp) - require.Equal(t, len(insertedTxs), len(resp.Data)) - for _, tx := range resp.Data { - generatedTx, ok := insertedTxs[tx.Id] - require.True(t, ok) - require.Equal(t, *generatedTx, tx) - } -} - -func TestTransaction(t *testing.T) { // /txs/{id} - t.Parallel() - insertedTxs := generator.Epochs.GetTransactions() - for _, tx := range insertedTxs { - res := apiServer.Get(t, apiPrefix+"/txs/"+tx.Id) - res.RequireOK(t) - var resp transactionResp - res.RequireUnmarshal(t, &resp) - require.Equal(t, 1, len(resp.Data)) - require.Equal(t, *tx, resp.Data[0]) - } -} diff --git a/internal/api/router/router.go b/internal/api/router/router.go deleted file mode 100644 index 3ec5520..0000000 --- a/internal/api/router/router.go +++ /dev/null @@ -1,45 +0,0 @@ -package router - -import ( - "github.com/labstack/echo/v4" - "github.com/spacemeshos/explorer-backend/internal/api/handler" -) - -func Init(e *echo.Echo) { - e.GET("/healthz", handler.HealthzHandler) - e.GET("/synced", handler.Synced) - - e.GET("/network-info", handler.NetworkInfo) - e.GET("/ws/network-info", handler.NetworkInfoWS) - - e.GET("/epochs", handler.Epochs) - e.GET("/epochs/:id", handler.Epoch) - e.GET("/epochs/:id/:entity", handler.EpochDetails) - - e.GET("/layers", handler.Layers) - e.GET("/layers/:id", handler.Layer) - e.GET("/layers/:id/:entity", handler.LayerDetails) - - e.GET("/smeshers", handler.Smeshers) - e.GET("/smeshers/:id", handler.Smesher) - e.GET("/smeshers/:id/:entity", handler.SmesherDetails) - - e.GET("/atxs", handler.Activations) - e.GET("/atxs/:id", handler.Activation) - - e.GET("/txs", handler.Transactions) - e.GET("/txs/:id", handler.Transaction) - - e.GET("/rewards", handler.Rewards) - e.GET("/rewards/total", handler.TotalRewards) - e.GET("/rewards/:id", handler.Reward) - e.GET("/v2/rewards/:smesherId/:layer", handler.RewardV2) - - e.GET("/accounts", handler.Accounts) - e.GET("/accounts/:id", handler.Account) - e.GET("/accounts/:id/:entity", handler.AccountDetails) - - e.GET("/blocks/:id", handler.Block) - - e.GET("/search/:id", handler.Search) -} diff --git a/internal/service/abstract.go b/internal/service/abstract.go deleted file mode 100644 index 7553f8a..0000000 --- a/internal/service/abstract.go +++ /dev/null @@ -1,29 +0,0 @@ -package service - -import ( - "context" - "errors" - - "github.com/spacemeshos/explorer-backend/model" -) - -// ErrNotFound is returned when a resource is not found. Router will serve 404 error if this is returned. -var ErrNotFound = errors.New("not found") - -// AppService is an interface for interacting with the app collection. -type AppService interface { - GetState(ctx context.Context) (*model.NetworkInfo, *model.Epoch, *model.Layer, error) - GetNetworkInfo(ctx context.Context) (*model.NetworkInfo, error) - Search(ctx context.Context, search string) (string, error) - Ping(ctx context.Context) error - - model.EpochService - model.LayerService - model.SmesherService - model.AccountService - model.RewardService - model.TransactionService - model.ActivationService - model.AppService - model.BlockService -} diff --git a/internal/service/account.go b/internal/service/account.go deleted file mode 100644 index 8843500..0000000 --- a/internal/service/account.go +++ /dev/null @@ -1,120 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "github.com/spacemeshos/go-spacemesh/log" - - "github.com/spacemeshos/address" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// GetAccount returns account by id. -func (e *Service) GetAccount(ctx context.Context, accountID string) (*model.Account, error) { - addr, err := address.StringToAddress(accountID) - if err != nil { - log.Err(fmt.Errorf("GetAccount error: %v", err)) - return nil, ErrNotFound - } - - filter := &bson.D{{Key: "address", Value: addr.String()}} - accs, total, err := e.getAccounts(ctx, filter, options.Find().SetSort(bson.D{{Key: "created", Value: 1}}).SetLimit(1).SetProjection(bson.D{ - {Key: "_id", Value: 0}, - {Key: "layer", Value: 0}, - })) - if err != nil { - return nil, fmt.Errorf("error find account: %w", err) - } - if total == 0 { - return nil, ErrNotFound - } - acc := accs[0] - summary, err := e.storage.GetAccountSummary(ctx, acc.Address) - if err != nil { - return nil, fmt.Errorf("error get account summary: %w", err) - } - - if summary != nil { - acc.Sent = summary.Sent - acc.Received = summary.Received - acc.Awards = summary.Awards - acc.Fees = summary.Fees - acc.LastActivity = summary.LastActivity - } - - if acc.LastActivity == 0 { - net, err := e.GetNetworkInfo(ctx) - if err != nil { - return nil, fmt.Errorf("error get network info for acc summury: %w", err) - } - acc.LastActivity = int32(net.GenesisTime) - } - - acc.Txs, err = e.storage.CountTransactions(ctx, &bson.D{ - {Key: "$or", Value: bson.A{ - bson.D{{Key: "sender", Value: acc.Address}}, - bson.D{{Key: "receiver", Value: acc.Address}}, - }}, - }) - if err != nil { - return nil, fmt.Errorf("error count transactions: %w", err) - } - return acc, nil -} - -// GetAccounts returns accounts by filter. -func (e *Service) GetAccounts(ctx context.Context, page, perPage int64) ([]*model.Account, int64, error) { - return e.getAccounts(ctx, &bson.D{}, e.getFindOptions("layer", page, perPage).SetProjection(bson.D{ - {Key: "_id", Value: 0}, - {Key: "layer", Value: 0}, - })) -} - -// GetAccountTransactions returns transactions by account id. -func (e *Service) GetAccountTransactions(ctx context.Context, accountID string, page, perPage int64) ([]*model.Transaction, int64, error) { - addr, err := address.StringToAddress(accountID) - if err != nil { - return nil, 0, ErrNotFound - } - - filter := &bson.D{ - {Key: "$or", Value: bson.A{ - bson.D{{Key: "sender", Value: addr.String()}}, - bson.D{{Key: "receiver", Value: addr.String()}}, - }}, - } - - return e.getTransactions(ctx, filter, e.getFindOptionsSort(bson.D{ - {Key: "layer", Value: -1}, {Key: "blockIndex", Value: -1}, - }, page, perPage)) -} - -// GetAccountRewards returns rewards by account id. -func (e *Service) GetAccountRewards(ctx context.Context, accountID string, page, perPage int64) ([]*model.Reward, int64, error) { - addr, err := address.StringToAddress(accountID) - if err != nil { - return nil, 0, ErrNotFound - } - opts := e.getFindOptions("layer", page, perPage) - opts.SetProjection(bson.D{}) - return e.getRewards(ctx, &bson.D{{Key: "coinbase", Value: addr.String()}}, opts) -} - -func (e *Service) getAccounts(ctx context.Context, filter *bson.D, options *options.FindOptions) (accs []*model.Account, total int64, err error) { - total, err = e.storage.CountAccounts(ctx, filter) - if err != nil { - return nil, 0, fmt.Errorf("error count accounts: %w", err) - } - if total == 0 { - return []*model.Account{}, 0, nil - } - accs, err = e.storage.GetAccounts(ctx, filter, options) - if err != nil { - return nil, 0, fmt.Errorf("error get accounts: %w", err) - } - return accs, total, nil -} diff --git a/internal/service/activation.go b/internal/service/activation.go deleted file mode 100644 index 2c8de6e..0000000 --- a/internal/service/activation.go +++ /dev/null @@ -1,47 +0,0 @@ -package service - -import ( - "context" - "fmt" - "strings" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// GetActivations returns atxs by filter. -func (e *Service) GetActivations(ctx context.Context, page, perPage int64) (atxs []*model.Activation, total int64, err error) { - return e.getActivations(ctx, &bson.D{}, e.getFindOptions("layer", page, perPage)) -} - -// GetActivation returns atx by id. -func (e *Service) GetActivation(ctx context.Context, activationID string) (*model.Activation, error) { - filter := &bson.D{{Key: "id", Value: strings.ToLower(activationID)}} - atx, total, err := e.getActivations(ctx, filter, options.Find().SetLimit(1).SetProjection(bson.D{{Key: "_id", Value: 0}})) - if err != nil { - return nil, fmt.Errorf("error find atx: %w", err) - } - if total == 0 { - return nil, ErrNotFound - } - return atx[0], nil -} - -func (e *Service) getActivations(ctx context.Context, filter *bson.D, options *options.FindOptions) (atxs []*model.Activation, total int64, err error) { - total, err = e.storage.CountActivations(ctx, filter) - if err != nil { - return nil, 0, fmt.Errorf("error count atxs: %w", err) - } - if total == 0 { - return []*model.Activation{}, 0, nil - } - - atxs, err = e.storage.GetActivations(ctx, filter, options) - if err != nil { - return nil, 0, fmt.Errorf("error get atxs: %w", err) - } - - return atxs, total, nil -} diff --git a/internal/service/app.go b/internal/service/app.go deleted file mode 100644 index 4f7a47f..0000000 --- a/internal/service/app.go +++ /dev/null @@ -1,44 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// GetApps returns apps by filter. -func (e *Service) GetApps(ctx context.Context, page, pageSize int64) (apps []*model.App, total int64, err error) { - return e.getApps(ctx, &bson.D{}, e.getFindOptions("address", page, pageSize)) -} - -// GetApp returns app by address. -func (e *Service) GetApp(ctx context.Context, appID string) (*model.App, error) { - filter := &bson.D{{Key: "address", Value: appID}} - apps, _, err := e.getApps(ctx, filter, options.Find().SetLimit(1).SetProjection(bson.D{{Key: "_id", Value: 0}})) - if err != nil { - return nil, fmt.Errorf("error find app: %w", err) - } - if len(apps) == 0 { - return nil, ErrNotFound - } - return apps[0], nil -} - -func (e *Service) getApps(ctx context.Context, filter *bson.D, options *options.FindOptions) (apps []*model.App, total int64, err error) { - total, err = e.storage.CountApps(ctx, filter) - if err != nil { - return nil, 0, fmt.Errorf("error count apps: %w", err) - } - if total == 0 { - return []*model.App{}, 0, nil - } - apps, err = e.storage.GetApps(ctx, filter, options) - if err != nil { - return nil, 0, fmt.Errorf("error get apps: %w", err) - } - return apps, total, nil -} diff --git a/internal/service/block.go b/internal/service/block.go deleted file mode 100644 index 85ae10c..0000000 --- a/internal/service/block.go +++ /dev/null @@ -1,39 +0,0 @@ -package service - -import ( - "context" - "fmt" - "strings" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// GetBlock returns block by id. -func (e *Service) GetBlock(ctx context.Context, blockID string) (*model.Block, error) { - blocks, _, err := e.getBlocks(ctx, &bson.D{{Key: "id", Value: strings.ToLower(blockID)}}, options.Find().SetLimit(1).SetProjection(bson.D{{Key: "_id", Value: 0}})) - if err != nil { - return nil, fmt.Errorf("error get block by `%s`: %w", blockID, err) - } - if len(blocks) == 0 { - return nil, ErrNotFound - } - return blocks[0], nil -} - -func (e *Service) getBlocks(ctx context.Context, filter *bson.D, options *options.FindOptions) (blocks []*model.Block, total int64, err error) { - total, err = e.storage.CountBlocks(ctx, filter) - if err != nil { - return nil, 0, fmt.Errorf("error count blocks: %w", err) - } - if total == 0 { - return []*model.Block{}, 0, nil - } - blocks, err = e.storage.GetBlocks(ctx, filter, options) - if err != nil { - return nil, 0, fmt.Errorf("error get blocks: %w", err) - } - return blocks, total, nil -} diff --git a/internal/service/epoch.go b/internal/service/epoch.go deleted file mode 100644 index 99e0c6f..0000000 --- a/internal/service/epoch.go +++ /dev/null @@ -1,114 +0,0 @@ -package service - -import ( - "context" - "fmt" - "time" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// GetCurrentEpoch returns current epoch. -func (e *Service) GetCurrentEpoch(ctx context.Context) (*model.Epoch, error) { - e.currentEpochMU.RLock() - epoch := e.currentEpoch - loadTime := e.currentEpochLoaded - e.currentEpochMU.RUnlock() - if epoch == nil || loadTime.Add(e.cacheTTL).Unix() < time.Now().Unix() { - now := time.Now().Unix() - epochs, err := e.storage.GetEpochs(ctx, &bson.D{{Key: "start", Value: bson.D{{Key: "$lte", Value: now}}}}, options.Find().SetSort(bson.D{{Key: "number", Value: -1}}).SetLimit(1).SetProjection(bson.D{{Key: "_id", Value: 0}})) - if err != nil { - return nil, fmt.Errorf("failed to get epoch: %w", err) - } - if len(epochs) == 0 { - return nil, nil - } - epoch = epochs[0] - - e.currentEpochMU.Lock() - e.currentEpoch = epoch - e.currentEpochLoaded = time.Now() - e.currentEpochMU.Unlock() - } - return epoch, nil -} - -// GetEpoch get epoch by number. -func (e *Service) GetEpoch(ctx context.Context, epochNum int) (*model.Epoch, error) { - epoch, err := e.storage.GetEpoch(ctx, epochNum) - if err != nil { - return nil, fmt.Errorf("failed to get epoch `%d`: %w", epochNum, err) - } - if epoch == nil { - return nil, ErrNotFound - } - return epoch, nil -} - -// GetEpochs returns list of epochs. -func (e *Service) GetEpochs(ctx context.Context, page, perPage int64) ([]*model.Epoch, int64, error) { - total, err := e.storage.CountEpochs(ctx, &bson.D{}) - if err != nil { - return nil, 0, fmt.Errorf("failed to count total epochs: %w", err) - } - epochs, err := e.storage.GetEpochs(ctx, &bson.D{}, e.getFindOptions("number", page, perPage)) - if err != nil { - return nil, 0, fmt.Errorf("failed to get epochs: %w", err) - } - return epochs, total, nil -} - -// GetEpochLayers returns layers for the given epoch. -func (e *Service) GetEpochLayers(ctx context.Context, epochNum int, page, perPage int64) (layers []*model.Layer, total int64, err error) { - layerStart, layerEnd := e.getEpochLayers(epochNum) - filter := &bson.D{{Key: "number", Value: bson.D{{Key: "$gte", Value: layerStart}, {Key: "$lte", Value: layerEnd}}}} - total, err = e.storage.CountLayers(ctx, filter) - if err != nil { - return nil, 0, fmt.Errorf("failed to count layers for epoch `%d`: %w", epochNum, err) - } - if total == 0 { - return []*model.Layer{}, 0, nil - } - - layers, err = e.storage.GetLayers(ctx, filter, e.getFindOptions("number", page, perPage)) - if err != nil { - return nil, 0, fmt.Errorf("failed to get layers for epoch `%d`: %w", epochNum, err) - } - - return layers, total, nil -} - -// GetEpochTransactions returns transactions for the given epoch. -func (e *Service) GetEpochTransactions(ctx context.Context, epochNum int, page, perPage int64) (txs []*model.Transaction, total int64, err error) { - layerStart, layerEnd := e.getEpochLayers(epochNum) - filter := &bson.D{{Key: "layer", Value: bson.D{{Key: "$gte", Value: layerStart}, {Key: "$lte", Value: layerEnd}}}} - return e.getTransactions(ctx, filter, e.getFindOptions("layer", page, perPage)) -} - -// GetEpochSmeshers returns smeshers for the given epoch. -func (e *Service) GetEpochSmeshers(ctx context.Context, epochNum int, page, perPage int64) (smeshers []*model.Smesher, total int64, err error) { - filter := &bson.D{{ - Key: "epochs", Value: epochNum, - }} - return e.getSmeshers(ctx, filter, e.getFindOptions("timestamp", page, perPage).SetProjection(bson.D{ - {Key: "epochs", Value: 0}, - })) -} - -// GetEpochRewards returns rewards for the given epoch. -func (e *Service) GetEpochRewards(ctx context.Context, epochNum int, page, perPage int64) (rewards []*model.Reward, total int64, err error) { - layerStart, layerEnd := e.getEpochLayers(epochNum) - filter := &bson.D{{Key: "layer", Value: bson.D{{Key: "$gte", Value: layerStart}, {Key: "$lte", Value: layerEnd}}}} - opts := e.getFindOptions("layer", page, perPage) - opts.SetProjection(bson.D{}) - return e.getRewards(ctx, filter, opts) -} - -// GetEpochActivations returns activations for the given epoch. -func (e *Service) GetEpochActivations(ctx context.Context, epochNum int, page, perPage int64) (atxs []*model.Activation, total int64, err error) { - filter := &bson.D{{Key: "targetEpoch", Value: epochNum}} - return e.getActivations(ctx, filter, e.getFindOptions("layer", page, perPage)) -} diff --git a/internal/service/layer.go b/internal/service/layer.go deleted file mode 100644 index 22beabe..0000000 --- a/internal/service/layer.go +++ /dev/null @@ -1,132 +0,0 @@ -package service - -import ( - "context" - "fmt" - "time" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// GetCurrentLayer returns current layer. -func (e *Service) GetCurrentLayer(ctx context.Context) (*model.Layer, error) { - e.currentLayerMU.RLock() - layer := e.currentLayer - loadTime := e.currentLayerLoaded - e.currentLayerMU.RUnlock() - if layer == nil || loadTime.Add(e.cacheTTL).Unix() < time.Now().Unix() { - layers, err := e.storage.GetLayers(ctx, &bson.D{}, options.Find().SetSort(bson.D{{Key: "number", Value: -1}}).SetLimit(1).SetProjection(bson.D{{Key: "_id", Value: 0}})) - if err != nil { - return nil, fmt.Errorf("error get layers: %s", err) - } - if len(layers) == 0 { - return nil, nil - } - layer = layers[0] - - e.currentLayerMU.Lock() - e.currentLayer = layer - e.currentLayerLoaded = time.Now() - e.currentLayerMU.Unlock() - } - return layer, nil -} - -// GetLayer returns layer by number. -func (e *Service) GetLayer(ctx context.Context, layerNum int) (*model.Layer, error) { - layer, err := e.storage.GetLayer(ctx, layerNum) - if err != nil { - return nil, fmt.Errorf("error get layer %d: %w", layerNum, err) - } - if layer == nil { - return nil, ErrNotFound - } - - txs, derr := e.storage.GetTransactions(ctx, &bson.D{{Key: "layer", Value: layer.Number}}) - if derr != nil { - return nil, fmt.Errorf("failed to count txs for layer %d: %w", layer.Number, derr) - } - layer.Txs = uint32(len(txs)) - layer.TxsAmount = 0 - for _, tx := range txs { - layer.TxsAmount += tx.Amount - } - - return layer, nil -} - -// GetLayerByHash returns layer by hash. -//func (e *Service) GetLayerByHash(ctx context.Context, layerHash string) (*model.Layer, error) { -// layers, err := e.storage.GetLayers(ctx, &bson.D{{Key: "hash", Value: layerHash}}) -// if err != nil { -// return nil, fmt.Errorf("error get layer by hash `%s`: %w", layerHash, err) -// } -// if len(layers) == 0 { -// return nil, ErrNotFound -// } -// return layers[0], nil -//} - -// GetLayers returns layers. -func (e *Service) GetLayers(ctx context.Context, page, perPage int64) (layers []*model.Layer, total int64, err error) { - total, err = e.storage.CountLayers(ctx, &bson.D{}) - if err != nil { - return nil, 0, fmt.Errorf("failed to count total layers: %w", err) - } - layers, err = e.storage.GetLayers(ctx, &bson.D{}, e.getFindOptions("number", page, perPage)) - if err != nil { - return nil, 0, fmt.Errorf("failed to get layers: %w", err) - } - - for _, layer := range layers { - txs, derr := e.storage.GetTransactions(ctx, &bson.D{{Key: "layer", Value: layer.Number}}) - if derr != nil { - return nil, 0, fmt.Errorf("failed to count txs for layer %d: %w", layer.Number, derr) - } - layer.Txs = uint32(len(txs)) - layer.TxsAmount = 0 - for _, tx := range txs { - layer.TxsAmount += tx.Amount - } - } - return layers, total, nil -} - -// GetLayerTransactions returns transactions for layer. -func (e *Service) GetLayerTransactions(ctx context.Context, layerNum int, page, perPage int64) (txs []*model.Transaction, total int64, err error) { - return e.getTransactions(ctx, &bson.D{{Key: "layer", Value: layerNum}}, e.getFindOptionsSort(bson.D{ - {Key: "blockIndex", Value: 1}, - }, page, perPage)) -} - -// GetLayerSmeshers returns smeshers for layer. -func (e *Service) GetLayerSmeshers(ctx context.Context, layerNum int, page, perPage int64) (smeshers []*model.Smesher, total int64, err error) { - filter := &bson.D{{Key: "layer", Value: layerNum}} - return e.getSmeshers(ctx, filter, e.getFindOptions("id", page, perPage).SetProjection(bson.D{ - {Key: "id", Value: 0}, - {Key: "layer", Value: 0}, - {Key: "coinbase", Value: 0}, - {Key: "prevAtx", Value: 0}, - {Key: "cSize", Value: 0}, - })) -} - -// GetLayerRewards returns rewards for layer. -func (e *Service) GetLayerRewards(ctx context.Context, layerNum int, page, perPage int64) (rewards []*model.Reward, total int64, err error) { - opts := e.getFindOptions("layer", page, perPage) - opts.SetProjection(bson.D{}) - return e.getRewards(ctx, &bson.D{{Key: "layer", Value: layerNum}}, opts) -} - -// GetLayerActivations returns activations for layer. -func (e *Service) GetLayerActivations(ctx context.Context, layerNum int, page, perPage int64) (atxs []*model.Activation, total int64, err error) { - return e.getActivations(ctx, &bson.D{{Key: "layer", Value: layerNum}}, e.getFindOptions("id", page, perPage)) -} - -// GetLayerBlocks returns blocks for layer. -func (e *Service) GetLayerBlocks(ctx context.Context, layerNum int, page, perPage int64) (blocks []*model.Block, total int64, err error) { - return e.getBlocks(ctx, &bson.D{{Key: "layer", Value: layerNum}}, e.getFindOptions("id", page, perPage)) -} diff --git a/internal/service/reward.go b/internal/service/reward.go deleted file mode 100644 index 7727fec..0000000 --- a/internal/service/reward.go +++ /dev/null @@ -1,58 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// GetReward returns reward by id. -func (e *Service) GetReward(ctx context.Context, rewardID string) (*model.Reward, error) { - reward, err := e.storage.GetReward(ctx, rewardID) - if err != nil { - return nil, fmt.Errorf("error get reward: %w", err) - } - if reward == nil { - return nil, ErrNotFound - } - return reward, nil -} - -func (e *Service) GetRewardV2(ctx context.Context, smesherID string, layer uint32) (*model.Reward, error) { - reward, err := e.storage.GetRewardV2(ctx, smesherID, layer) - if err != nil { - return nil, fmt.Errorf("error while getting reward: %w", err) - } - if reward == nil { - return nil, ErrNotFound - } - return reward, nil -} - -// GetRewards returns rewards by filter. -func (e *Service) GetRewards(ctx context.Context, page, perPage int64) ([]*model.Reward, int64, error) { - return e.getRewards(ctx, &bson.D{}, options.Find().SetSort(bson.D{{Key: "layer", Value: -1}}).SetLimit(perPage).SetSkip((page-1)*perPage)) -} - -func (e *Service) getRewards(ctx context.Context, filter *bson.D, options *options.FindOptions) (rewards []*model.Reward, total int64, err error) { - total, err = e.storage.CountRewards(ctx, filter) - if err != nil { - return nil, 0, fmt.Errorf("error count rewards: %w", err) - } - if total == 0 { - return []*model.Reward{}, 0, nil - } - rewards, err = e.storage.GetRewards(ctx, filter, options) - if err != nil { - return nil, 0, fmt.Errorf("error get rewards: %w", err) - } - return rewards, total, nil -} - -func (e *Service) GetTotalRewards(ctx context.Context, filter *bson.D) (int64, int64, error) { - return e.storage.GetTotalRewards(ctx, filter) -} diff --git a/internal/service/search.go b/internal/service/search.go deleted file mode 100644 index 484abbc..0000000 --- a/internal/service/search.go +++ /dev/null @@ -1,73 +0,0 @@ -package service - -import ( - "context" - "fmt" - "strconv" -) - -const ( - // addressTestLength is the expected length of an address with testnet hrp `sm`. - addressTestLength = 51 - // addressLength is the expected length of an address with mainet hrp `sm`. - addressLength = 48 - // blockIDLength is the expected length of a block id. - blockIDLength = 42 - // idLength is the expected length of a transactionID | activation | smesher. - idLength = 66 -) - -// Search try guess entity to search and find related one. -func (e *Service) Search(ctx context.Context, search string) (string, error) { - switch len(search) { - case addressLength, addressTestLength: - if acc, _ := e.GetAccount(ctx, search); acc != nil { - return "/accounts/" + search, nil - } - case blockIDLength: - if block, _ := e.GetBlock(ctx, search); block != nil { - return "/blocks/" + search, nil - } - case idLength: - if tx, _ := e.GetTransaction(ctx, search); tx != nil { - return "/txs/" + search, nil - } - if atx, _ := e.GetActivation(ctx, search); atx != nil { - return "/atxs/" + search, nil - } - if smesher, _ := e.GetSmesher(ctx, search); smesher != nil { - return "/smeshers/" + search, nil - } - //if layer, _ := e.GetLayerByHash(ctx, search); layer != nil { - // return fmt.Sprintf("/smeshers/%d", layer.Number), nil - //} - default: - if reward, _ := e.GetReward(ctx, search); reward != nil { - return "rewards/" + search, nil - } - id, err := strconv.Atoi(search) - if err != nil { - return "", ErrNotFound - } - layer, err := e.GetCurrentLayer(ctx) - if err != nil { - return "", fmt.Errorf("error get current layer for search: %w", err) - } - epoch, err := e.GetCurrentEpoch(ctx) - if err != nil { - return "", fmt.Errorf("error get current epoch for search: %w", err) - } - if epoch == nil { - return fmt.Sprintf("/layers/%d", id), nil - } - - if id > int(epoch.Number)+1 { - if id <= int(layer.Number) && id > 0 { - return fmt.Sprintf("/layers/%d", id), nil - } - } else if id > 0 { - return fmt.Sprintf("/epochs/%d", id), nil - } - } - return "", ErrNotFound -} diff --git a/internal/service/service.go b/internal/service/service.go deleted file mode 100644 index ee83d31..0000000 --- a/internal/service/service.go +++ /dev/null @@ -1,115 +0,0 @@ -package service - -import ( - "context" - "fmt" - "sync" - "time" - - "github.com/spacemeshos/go-spacemesh/log" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/internal/storage/storagereader" - "github.com/spacemeshos/explorer-backend/model" -) - -// Service main app service which working with database. -type Service struct { - networkInfo *model.NetworkInfo - networkInfoMU *sync.RWMutex - networkInfoLoaded time.Time - - currentEpoch *model.Epoch - currentEpochMU *sync.RWMutex - currentEpochLoaded time.Time - - currentLayer *model.Layer - currentLayerMU *sync.RWMutex - currentLayerLoaded time.Time - - cacheTTL time.Duration - storage storagereader.StorageReader -} - -// NewService creates new service instance. -func NewService(reader storagereader.StorageReader, cacheTTL time.Duration) *Service { - service := &Service{ - storage: reader, - cacheTTL: cacheTTL, - networkInfoMU: &sync.RWMutex{}, - currentEpochMU: &sync.RWMutex{}, - currentLayerMU: &sync.RWMutex{}, - } - - if _, err := service.GetNetworkInfo(context.Background()); err != nil { - log.Err(fmt.Errorf("error load network info: %w", err)) - } - return service -} - -// GetState returns state of the network, current layer and epoch. -func (e *Service) GetState(ctx context.Context) (*model.NetworkInfo, *model.Epoch, *model.Layer, error) { - net, err := e.GetNetworkInfo(ctx) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get network info: %w", err) - } - epoch, err := e.GetCurrentEpoch(ctx) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get epoch: %w", err) - } - layer, err := e.GetCurrentLayer(ctx) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get layer: %w", err) - } - return net, epoch, layer, nil -} - -// GetNetworkInfo returns actual network info. Caches data for some time (see networkInfoTTL). -func (e *Service) GetNetworkInfo(ctx context.Context) (net *model.NetworkInfo, err error) { - e.networkInfoMU.RLock() - net = e.networkInfo - loadTime := e.networkInfoLoaded - e.networkInfoMU.RUnlock() - if net == nil || loadTime.Add(e.cacheTTL).Unix() < time.Now().Unix() { - net, err = e.storage.GetNetworkInfo(ctx) - if err != nil { - return nil, fmt.Errorf("failed get networkInfo: %w", err) - } - e.networkInfoMU.Lock() - e.networkInfo = net - e.networkInfoLoaded = time.Now() - e.networkInfoMU.Unlock() - } - return net, nil -} - -func (e *Service) getFindOptions(key string, page, perPage int64) *options.FindOptions { - return options.Find(). - SetSort(bson.D{{Key: key, Value: -1}}). - SetLimit(perPage). - SetSkip((page - 1) * perPage). - SetProjection(bson.D{{Key: "_id", Value: 0}}) -} - -func (e *Service) getFindOptionsSort(sort bson.D, page, perPage int64) *options.FindOptions { - return options.Find(). - SetSort(sort). - SetLimit(perPage). - SetSkip((page - 1) * perPage). - SetProjection(bson.D{{Key: "_id", Value: 0}}) -} - -func (e *Service) getEpochLayers(epoch int) (uint32, uint32) { - e.networkInfoMU.RLock() - net := e.networkInfo - e.networkInfoMU.RUnlock() - start := uint32(epoch) * net.EpochNumLayers - end := start + net.EpochNumLayers - 1 - return start, end -} - -// Ping checks if the database is reachable. -func (e *Service) Ping(ctx context.Context) error { - return e.storage.Ping(ctx) -} diff --git a/internal/service/smesher.go b/internal/service/smesher.go deleted file mode 100644 index 93fb253..0000000 --- a/internal/service/smesher.go +++ /dev/null @@ -1,70 +0,0 @@ -package service - -import ( - "context" - "fmt" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// GetSmesher returns smesher by id. -func (e *Service) GetSmesher(ctx context.Context, smesherID string) (*model.Smesher, error) { - smesher, err := e.storage.GetSmesher(ctx, smesherID) - if err != nil { - return nil, err - } - if smesher == nil { - return nil, ErrNotFound - } - smesher.Rewards, _, err = e.CountSmesherRewards(ctx, smesherID) - return smesher, err -} - -// GetSmeshers returns smeshers by filter. -func (e *Service) GetSmeshers(ctx context.Context, page, perPage int64) (smeshers []*model.Smesher, total int64, err error) { - total, err = e.storage.CountSmeshers(ctx, &bson.D{}) - if err != nil { - return nil, 0, fmt.Errorf("failed to count total smeshers: %w", err) - } - if total == 0 { - return []*model.Smesher{}, 0, nil - } - smeshers, err = e.storage.GetSmeshers(ctx, &bson.D{}, e.getFindOptions("timestamp", page, perPage)) - if err != nil { - return nil, 0, fmt.Errorf("failed to get smeshers: %w", err) - } - return smeshers, total, nil -} - -// GetSmesherActivations returns smesher activations by filter. -func (e *Service) GetSmesherActivations(ctx context.Context, smesherID string, page, perPage int64) (atxs []*model.Activation, total int64, err error) { - return e.getActivations(ctx, &bson.D{{Key: "smesher", Value: smesherID}}, e.getFindOptions("layer", page, perPage)) -} - -// GetSmesherRewards returns smesher rewards by filter. -func (e *Service) GetSmesherRewards(ctx context.Context, smesherID string, page, perPage int64) (rewards []*model.Reward, total int64, err error) { - opts := e.getFindOptions("layer", page, perPage) - opts.SetProjection(bson.D{}) - return e.getRewards(ctx, &bson.D{{Key: "smesher", Value: smesherID}}, opts) -} - -// CountSmesherRewards returns smesher rewards count by filter. -func (e *Service) CountSmesherRewards(ctx context.Context, smesherID string) (total, count int64, err error) { - return e.storage.CountSmesherRewards(ctx, smesherID) -} - -func (e *Service) getSmeshers(ctx context.Context, filter *bson.D, options *options.FindOptions) (smeshers []*model.Smesher, total int64, err error) { - total, err = e.storage.CountEpochSmeshers(ctx, filter) - if err != nil { - return []*model.Smesher{}, 0, err - } - - smeshers, err = e.storage.GetEpochSmeshers(ctx, filter, options) - if err != nil { - return nil, 0, fmt.Errorf("error load smeshers: %w", err) - } - - return smeshers, total, nil -} diff --git a/internal/service/transaction.go b/internal/service/transaction.go deleted file mode 100644 index 846eafb..0000000 --- a/internal/service/transaction.go +++ /dev/null @@ -1,47 +0,0 @@ -package service - -import ( - "context" - "fmt" - "strings" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// GetTransaction returns tx by id. -func (e *Service) GetTransaction(ctx context.Context, txID string) (*model.Transaction, error) { - filter := &bson.D{{Key: "id", Value: strings.ToLower(txID)}} - txs, total, err := e.getTransactions(ctx, filter, options.Find().SetLimit(1).SetProjection(bson.D{{Key: "_id", Value: 0}})) - if err != nil { - return nil, fmt.Errorf("error get transaction: %w", err) - } - if total == 0 { - return nil, ErrNotFound - } - return txs[0], nil -} - -// GetTransactions returns txs by filter. -func (e *Service) GetTransactions(ctx context.Context, page, perPage int64) (txs []*model.Transaction, total int64, err error) { - return e.getTransactions(ctx, &bson.D{}, e.getFindOptionsSort(bson.D{ - {Key: "layer", Value: -1}, {Key: "blockIndex", Value: -1}, - }, page, perPage)) -} - -func (e *Service) getTransactions(ctx context.Context, filter *bson.D, options *options.FindOptions) (txs []*model.Transaction, total int64, err error) { - total, err = e.storage.CountTransactions(ctx, filter) - if err != nil { - return nil, 0, fmt.Errorf("error count txs: %w", err) - } - if total == 0 { - return []*model.Transaction{}, 0, nil - } - txs, err = e.storage.GetTransactions(ctx, filter, options) - if err != nil { - return nil, 0, fmt.Errorf("error get txs: %w", err) - } - return txs, total, nil -} diff --git a/internal/storage/storagereader/abstract.go b/internal/storage/storagereader/abstract.go deleted file mode 100644 index 781559b..0000000 --- a/internal/storage/storagereader/abstract.go +++ /dev/null @@ -1,59 +0,0 @@ -package storagereader - -import ( - "context" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// StorageReader is the interface for the storage reader. Providing ReadOnly methods. -type StorageReader interface { - Ping(ctx context.Context) error - GetNetworkInfo(ctx context.Context) (*model.NetworkInfo, error) - - CountTransactions(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) - GetTransactions(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Transaction, error) - CountSentTransactions(ctx context.Context, address string) (amount, fees, count int64, err error) - CountReceivedTransactions(ctx context.Context, address string) (amount, count int64, err error) - GetLatestTransaction(ctx context.Context, address string) (*model.Transaction, error) - GetFirstSentTransaction(ctx context.Context, address string) (*model.Transaction, error) - - CountApps(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) - GetApps(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.App, error) - - CountAccounts(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) - GetAccounts(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Account, error) - GetAccountSummary(ctx context.Context, address string) (*model.AccountSummary, error) - - CountActivations(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) - GetActivations(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Activation, error) - - CountBlocks(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) - GetBlocks(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Block, error) - - CountEpochs(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) - GetEpochs(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Epoch, error) - GetEpoch(ctx context.Context, epochNumber int) (*model.Epoch, error) - - CountLayers(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) - GetLayers(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Layer, error) - GetLayer(ctx context.Context, layerNumber int) (*model.Layer, error) - - CountRewards(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) - CountCoinbaseRewards(ctx context.Context, coinbase string) (total, count int64, err error) - GetRewards(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Reward, error) - GetReward(ctx context.Context, rewardID string) (*model.Reward, error) - GetRewardV2(ctx context.Context, smesherID string, layer uint32) (*model.Reward, error) - GetLatestReward(ctx context.Context, coinbase string) (*model.Reward, error) - GetTotalRewards(ctx context.Context, filter *bson.D) (total, count int64, err error) - - CountSmeshers(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) - GetSmeshers(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Smesher, error) - GetSmesher(ctx context.Context, smesherID string) (*model.Smesher, error) - CountEpochSmeshers(ctx context.Context, query *bson.D) (int64, error) - GetEpochSmeshers(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Smesher, error) - CountSmesherRewards(ctx context.Context, smesherID string) (total, count int64, err error) -} diff --git a/internal/storage/storagereader/accounts.go b/internal/storage/storagereader/accounts.go deleted file mode 100644 index 83fb18c..0000000 --- a/internal/storage/storagereader/accounts.go +++ /dev/null @@ -1,187 +0,0 @@ -package storagereader - -import ( - "context" - "fmt" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// CountAccounts returns the number of accounts matching the query. -func (s *Reader) CountAccounts(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) { - return s.db.Collection("accounts").CountDocuments(ctx, query, opts...) -} - -// GetAccounts returns the accounts matching the query. -func (s *Reader) GetAccounts(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Account, error) { - skip := int64(0) - if opts[0].Skip != nil { - skip = *opts[0].Skip - } - - pipeline := bson.A{ - bson.D{ - {"$lookup", - bson.D{ - {"from", "txs"}, - {"let", bson.D{{"addr", "$address"}}}, - {"pipeline", - bson.A{ - bson.D{ - {"$match", - bson.D{ - {"$or", - bson.A{ - bson.D{ - {"$expr", - bson.D{ - {"$eq", - bson.A{ - "$sender", - "$$addr", - }, - }, - }, - }, - }, - bson.D{ - {"$expr", - bson.D{ - {"$eq", - bson.A{ - "$receiver", - "$$addr", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - bson.D{{"$sort", bson.D{{"layer", 1}}}}, - bson.D{{"$limit", 1}}, - bson.D{ - {"$project", - bson.D{ - {"_id", 0}, - {"layer", 1}, - }, - }, - }, - }, - }, - {"as", "createdLayerRst"}, - }, - }, - }, - bson.D{ - {"$addFields", - bson.D{ - {"createdLayer", - bson.D{ - {"$arrayElemAt", - bson.A{ - "$createdLayerRst.layer", - 0, - }, - }, - }, - }, - }, - }, - }, - bson.D{{"$project", bson.D{{"createdLayerRst", 0}}}}, - bson.D{{Key: "$sort", Value: bson.D{{Key: "createdLayer", Value: -1}}}}, - bson.D{{Key: "$skip", Value: skip}}, - bson.D{{Key: "$limit", Value: *opts[0].Limit}}, - } - - if query != nil { - pipeline = append(bson.A{ - bson.D{{Key: "$match", Value: *query}}, - }, pipeline...) - } - - cursor, err := s.db.Collection("accounts").Aggregate(ctx, pipeline) - if err != nil { - return nil, fmt.Errorf("failed to get accounts: %w", err) - } - var docs []*model.Account - if err = cursor.All(ctx, &docs); err != nil { - return nil, fmt.Errorf("error decode accounts: %w", err) - } - - for _, doc := range docs { - summary, err := s.GetAccountSummary(ctx, doc.Address) - if err != nil { - return nil, fmt.Errorf("failed to get account summary: %w", err) - } - if summary == nil { - continue - } - - doc.Sent = summary.Sent - doc.Received = summary.Received - doc.Awards = summary.Awards - doc.Fees = summary.Fees - doc.LastActivity = summary.LastActivity - } - return docs, nil -} - -// GetAccountSummary returns the summary of the accounts matching the query. Not all accounts from api have filled this data. -func (s *Reader) GetAccountSummary(ctx context.Context, address string) (*model.AccountSummary, error) { - var accSummary model.AccountSummary - - totalRewards, _, err := s.CountCoinbaseRewards(ctx, address) - if err != nil { - return nil, fmt.Errorf("error occured while getting sum of rewards: %w", err) - } - accSummary.Awards = uint64(totalRewards) - - received, _, err := s.CountReceivedTransactions(ctx, address) - if err != nil { - if err != nil { - return nil, fmt.Errorf("error occured while getting sum of received txs: %w", err) - } - } - accSummary.Received = uint64(received) - - sent, fees, _, err := s.CountSentTransactions(ctx, address) - if err != nil { - if err != nil { - return nil, fmt.Errorf("error occured while getting sum of sent txs: %w", err) - } - } - accSummary.Sent = uint64(sent) - accSummary.Fees = uint64(fees) - - latestTx, err := s.GetLatestTransaction(ctx, address) - if err != nil { - return nil, fmt.Errorf("error occured while getting latest sent txs: %w", err) - } - - latestReward, err := s.GetLatestReward(ctx, address) - if err != nil { - return nil, fmt.Errorf("error occured while getting latest reawrd: %w", err) - } - - if latestTx != nil { - if latestReward != nil && latestReward.Layer > latestTx.Layer { - accSummary.LastActivity = int32(s.GetLayerTimestamp(latestReward.Layer)) - } else { - accSummary.LastActivity = int32(s.GetLayerTimestamp(latestTx.Layer)) - } - } else { - if latestReward != nil { - accSummary.LastActivity = int32(s.GetLayerTimestamp(latestReward.Layer)) - } - } - - return &accSummary, nil -} diff --git a/internal/storage/storagereader/activation.go b/internal/storage/storagereader/activation.go deleted file mode 100644 index 7caf5c5..0000000 --- a/internal/storage/storagereader/activation.go +++ /dev/null @@ -1,33 +0,0 @@ -package storagereader - -import ( - "context" - "fmt" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// CountActivations returns the number of activations matching the query. -func (s *Reader) CountActivations(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) { - count, err := s.db.Collection("activations").CountDocuments(ctx, query, opts...) - if err != nil { - return 0, fmt.Errorf("error count activations: %w", err) - } - return count, nil -} - -// GetActivations returns the activations matching the query. -func (s *Reader) GetActivations(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Activation, error) { - cursor, err := s.db.Collection("activations").Find(ctx, query, opts...) - if err != nil { - return nil, fmt.Errorf("failed to get activations: %w", err) - } - var docs []*model.Activation - if err = cursor.All(ctx, &docs); err != nil { - return nil, fmt.Errorf("error decode activations: %w", err) - } - return docs, nil -} diff --git a/internal/storage/storagereader/apps.go b/internal/storage/storagereader/apps.go deleted file mode 100644 index 941ab84..0000000 --- a/internal/storage/storagereader/apps.go +++ /dev/null @@ -1,33 +0,0 @@ -package storagereader - -import ( - "context" - "fmt" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// CountApps returns the number of apps matching the query. -func (s *Reader) CountApps(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) { - count, err := s.db.Collection("apps").CountDocuments(ctx, query, opts...) - if err != nil { - return 0, fmt.Errorf("error count apps: %w", err) - } - return count, nil -} - -// GetApps returns the apps matching the query. -func (s *Reader) GetApps(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.App, error) { - cursor, err := s.db.Collection("apps").Find(ctx, query, opts...) - if err != nil { - return nil, fmt.Errorf("failed to get apps: %w", err) - } - var docs []*model.App - if err = cursor.All(ctx, &docs); err != nil { - return nil, fmt.Errorf("error decode apps: %w", err) - } - return docs, nil -} diff --git a/internal/storage/storagereader/blocks.go b/internal/storage/storagereader/blocks.go deleted file mode 100644 index 2ae730e..0000000 --- a/internal/storage/storagereader/blocks.go +++ /dev/null @@ -1,34 +0,0 @@ -package storagereader - -import ( - "context" - "fmt" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// CountBlocks returns the number of blocks matching the query. -func (s *Reader) CountBlocks(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) { - count, err := s.db.Collection("blocks").CountDocuments(ctx, query, opts...) - if err != nil { - return 0, fmt.Errorf("error count blocks: %w", err) - } - return count, nil -} - -// GetBlocks returns the blocks matching the query. -func (s *Reader) GetBlocks(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Block, error) { - cursor, err := s.db.Collection("blocks").Find(ctx, query, opts...) - if err != nil { - return nil, fmt.Errorf("error get blocks: %w", err) - } - - var blocks []*model.Block - if err = cursor.All(ctx, &blocks); err != nil { - return nil, fmt.Errorf("error decode blocks: %w", err) - } - return blocks, nil -} diff --git a/internal/storage/storagereader/epochs.go b/internal/storage/storagereader/epochs.go deleted file mode 100644 index 2d34e80..0000000 --- a/internal/storage/storagereader/epochs.go +++ /dev/null @@ -1,89 +0,0 @@ -package storagereader - -import ( - "context" - "fmt" - bson "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// CountEpochs returns the number of epochs matching the query. -func (s *Reader) CountEpochs(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) { - count, err := s.db.Collection("epochs").CountDocuments(ctx, query, opts...) - if err != nil { - return 0, fmt.Errorf("error count epochs: %w", err) - } - return count, nil -} - -// GetEpochs returns the epochs matching the query. -func (s *Reader) GetEpochs(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Epoch, error) { - cursor, err := s.db.Collection("epochs").Find(ctx, query, opts...) - if err != nil { - return nil, fmt.Errorf("error get epochs: %w", err) - } - var epochs []*model.Epoch - if err = cursor.All(ctx, &epochs); err != nil { - return nil, err - } - - for _, epoch := range epochs { - total, count, err := s.GetTotalRewards(context.TODO(), &bson.D{{Key: "layer", Value: bson.D{ - {Key: "$gte", Value: epoch.LayerStart}, {Key: "$lte", Value: epoch.LayerEnd}}}, - }) - if err != nil { - return nil, fmt.Errorf("error get total rewards for epoch %d: %w", epoch.Number, err) - } - - epoch.Stats.Current.Rewards = total - epoch.Stats.Current.RewardsNumber = count - epoch.Stats.Cumulative.Rewards = total - epoch.Stats.Cumulative.RewardsNumber = count - - atxCount, err := s.CountActivations(context.Background(), &bson.D{{Key: "targetEpoch", Value: epoch.Number}}) - if err != nil { - return nil, err - } - epoch.Stats.Current.Smeshers = atxCount - } - - return epochs, nil -} - -// GetEpoch returns the epoch matching the query. -func (s *Reader) GetEpoch(ctx context.Context, epochNumber int) (*model.Epoch, error) { - cursor, err := s.db.Collection("epochs").Find(ctx, bson.D{{Key: "number", Value: epochNumber}}) - if err != nil { - return nil, fmt.Errorf("error get epoch `%d`: %w", epochNumber, err) - } - if !cursor.Next(ctx) { - return nil, nil - } - var epoch *model.Epoch - if err = cursor.Decode(&epoch); err != nil { - return nil, fmt.Errorf("error decode epoch `%d`: %w", epochNumber, err) - } - - total, count, err := s.GetTotalRewards(context.TODO(), &bson.D{{Key: "layer", Value: bson.D{ - {Key: "$gte", Value: epoch.LayerStart}, {Key: "$lte", Value: epoch.LayerEnd}}}, - }) - if err != nil { - return nil, err - } - - epoch.Stats.Current.Rewards = total - epoch.Stats.Current.RewardsNumber = count - epoch.Stats.Cumulative.Rewards = total - epoch.Stats.Cumulative.RewardsNumber = count - - atxCount, err := s.CountActivations(context.Background(), &bson.D{{Key: "targetEpoch", Value: epoch.Number}}) - if err != nil { - return nil, err - } - epoch.Stats.Current.Smeshers = atxCount - epoch.Stats.Cumulative.Smeshers = atxCount - - return epoch, nil -} diff --git a/internal/storage/storagereader/layers.go b/internal/storage/storagereader/layers.go deleted file mode 100644 index d07aea7..0000000 --- a/internal/storage/storagereader/layers.go +++ /dev/null @@ -1,184 +0,0 @@ -package storagereader - -import ( - "context" - "fmt" - "github.com/spacemeshos/go-spacemesh/log" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// CountLayers returns the number of layers matching the query. -func (s *Reader) CountLayers(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) { - count, err := s.db.Collection("layers").CountDocuments(ctx, query, opts...) - if err != nil { - return 0, fmt.Errorf("error count layers: %w", err) - } - return count, nil -} - -// GetLayers returns the layers matching the query. -func (s *Reader) GetLayers(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Layer, error) { - skip := int64(0) - if opts[0].Skip != nil { - skip = *opts[0].Skip - } - - pipeline := bson.A{ - bson.D{{Key: "$sort", Value: bson.D{{Key: "number", Value: -1}}}}, - bson.D{{Key: "$skip", Value: skip}}, - bson.D{{Key: "$limit", Value: *opts[0].Limit}}, - bson.D{ - {Key: "$lookup", - Value: bson.D{ - {Key: "from", Value: "rewards"}, - {Key: "localField", Value: "number"}, - {Key: "foreignField", Value: "layer"}, - {Key: "as", Value: "rewardsData"}, - }, - }, - }, - bson.D{{Key: "$unwind", Value: bson.D{ - {Key: "path", Value: "$rewardsData"}, - {Key: "preserveNullAndEmptyArrays", Value: true}, - }}}, - bson.D{ - {Key: "$group", - Value: bson.D{ - {Key: "_id", Value: "$_id"}, - {Key: "layerData", Value: bson.D{{Key: "$first", Value: "$$ROOT"}}}, - {Key: "rewards", Value: bson.D{{Key: "$sum", Value: "$rewardsData.total"}}}, - }, - }, - }, - bson.D{ - {Key: "$project", - Value: bson.D{ - {Key: "layerData", Value: 1}, - {Key: "rewards", Value: 1}, - }, - }, - }, - bson.D{ - {Key: "$replaceRoot", - Value: bson.D{ - {Key: "newRoot", - Value: bson.D{ - {Key: "$mergeObjects", - Value: bson.A{ - "$layerData", - bson.D{{Key: "rewards", Value: "$rewards"}}, - }, - }, - }, - }, - }, - }, - }, - bson.D{{Key: "$project", Value: bson.D{ - {Key: "rewardsData", Value: 0}, - {Key: "_id", Value: 0}, - }}}, - bson.D{{Key: "$sort", Value: bson.D{{Key: "number", Value: -1}}}}, - } - - if query != nil { - pipeline = append(bson.A{ - bson.D{{Key: "$match", Value: *query}}, - }, pipeline...) - } - - cursor, err := s.db.Collection("layers").Aggregate(ctx, pipeline) - if err != nil { - return nil, fmt.Errorf("error get layers: %s", err) - } - - var layers []*model.Layer - if err = cursor.All(ctx, &layers); err != nil { - return nil, fmt.Errorf("error decode layers: %s", err) - } - return layers, nil -} - -// GetLayer returns the layer matching the query. -func (s *Reader) GetLayer(ctx context.Context, layerNumber int) (*model.Layer, error) { - pipeline := bson.A{ - bson.D{{Key: "$match", Value: bson.D{{Key: "number", Value: layerNumber}}}}, - bson.D{ - {Key: "$lookup", - Value: bson.D{ - {Key: "from", Value: "rewards"}, - {Key: "localField", Value: "number"}, - {Key: "foreignField", Value: "layer"}, - {Key: "as", Value: "rewardsData"}, - }, - }, - }, - bson.D{{Key: "$unwind", Value: bson.D{ - {Key: "path", Value: "$rewardsData"}, - {Key: "preserveNullAndEmptyArrays", Value: true}, - }}}, - bson.D{ - {Key: "$group", - Value: bson.D{ - {Key: "_id", Value: "$_id"}, - {Key: "layerData", Value: bson.D{{Key: "$first", Value: "$$ROOT"}}}, - {Key: "rewards", Value: bson.D{{Key: "$sum", Value: "$rewardsData.total"}}}, - }, - }, - }, - bson.D{ - {Key: "$project", - Value: bson.D{ - {Key: "layerData", Value: 1}, - {Key: "rewards", Value: 1}, - }, - }, - }, - bson.D{ - {Key: "$replaceRoot", - Value: bson.D{ - {Key: "newRoot", - Value: bson.D{ - {Key: "$mergeObjects", - Value: bson.A{ - "$layerData", - bson.D{{Key: "rewards", Value: "$rewards"}}, - }, - }, - }, - }, - }, - }, - }, - bson.D{{Key: "$project", Value: bson.D{{Key: "rewardsData", Value: 0}}}}, - } - - cursor, err := s.db.Collection("layers").Aggregate(ctx, pipeline) - if err != nil { - return nil, fmt.Errorf("error get layer `%d`: %w", layerNumber, err) - } - if !cursor.Next(ctx) { - return nil, nil - } - var layer *model.Layer - if err = cursor.Decode(&layer); err != nil { - return nil, fmt.Errorf("error decode layer `%d`: %w", layerNumber, err) - } - return layer, nil -} - -func (s *Reader) GetLayerTimestamp(layer uint32) uint32 { - networkInfo, err := s.GetNetworkInfo(context.TODO()) - if err != nil { - log.Err(fmt.Errorf("getLayerTimestamp: %w", err)) - return 0 - } - - if layer == 0 { - return networkInfo.GenesisTime - } - return networkInfo.GenesisTime + (layer-1)*networkInfo.LayerDuration -} diff --git a/internal/storage/storagereader/rewards.go b/internal/storage/storagereader/rewards.go deleted file mode 100644 index c939225..0000000 --- a/internal/storage/storagereader/rewards.go +++ /dev/null @@ -1,166 +0,0 @@ -package storagereader - -import ( - "context" - "fmt" - "github.com/spacemeshos/explorer-backend/utils" - "go.mongodb.org/mongo-driver/mongo" - "strings" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// CountRewards returns the number of rewards matching the query. -func (s *Reader) CountRewards(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) { - count, err := s.db.Collection("rewards").CountDocuments(ctx, query, opts...) - if err != nil { - return 0, fmt.Errorf("error count transactions: %w", err) - } - return count, nil -} - -// GetRewards returns the rewards matching the query. -func (s *Reader) GetRewards(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Reward, error) { - cursor, err := s.db.Collection("rewards").Find(ctx, query, opts...) - if err != nil { - return nil, fmt.Errorf("error get rewards: %w", err) - } - - var rewards []*model.Reward - if err = cursor.All(ctx, &rewards); err != nil { - return nil, fmt.Errorf("error decode rewards: %w", err) - } - return rewards, nil -} - -// GetReward returns the reward matching the query. -func (s *Reader) GetReward(ctx context.Context, rewardID string) (*model.Reward, error) { - id, err := primitive.ObjectIDFromHex(strings.ToLower(rewardID)) - if err != nil { - return nil, fmt.Errorf("error create objectID from string `%s`: %w", rewardID, err) - } - cursor, err := s.db.Collection("rewards").Find(ctx, &bson.D{{Key: "_id", Value: id}}) - if err != nil { - return nil, fmt.Errorf("error get reward `%s`: %w", rewardID, err) - } - if !cursor.Next(ctx) { - return nil, nil - } - var reward *model.Reward - if err = cursor.Decode(&reward); err != nil { - return nil, fmt.Errorf("error decode reward `%s`: %w", rewardID, err) - } - return reward, nil -} - -func (s *Reader) GetRewardV2(ctx context.Context, smesherID string, layer uint32) (*model.Reward, error) { - cursor, err := s.db.Collection("rewards").Find(ctx, &bson.D{{Key: "smesher", Value: smesherID}, {Key: "layer", Value: layer}}) - if err != nil { - return nil, fmt.Errorf("error while getting reward by smesher `%s` and layer `%d`: %w", smesherID, layer, err) - } - if !cursor.Next(ctx) { - return nil, nil - } - var reward *model.Reward - if err = cursor.Decode(&reward); err != nil { - return nil, fmt.Errorf("error while decoding reward smesher `%s` layer `%d`: %w", smesherID, layer, err) - } - return reward, nil -} - -// CountCoinbaseRewards returns the number of rewards for given coinbase address. -func (s *Reader) CountCoinbaseRewards(ctx context.Context, coinbase string) (total, count int64, err error) { - matchStage := bson.D{{Key: "$match", Value: bson.D{{Key: "coinbase", Value: coinbase}}}} - groupStage := bson.D{ - {Key: "$group", Value: bson.D{ - {Key: "_id", Value: ""}, - {Key: "total", Value: bson.D{ - {Key: "$sum", Value: "$total"}, - }}, - {Key: "layerReward", Value: bson.D{ - {Key: "$sum", Value: "$layerReward"}, - }}, - {Key: "count", Value: bson.D{ - {Key: "$sum", Value: 1}, - }}, - }}, - } - cursor, err := s.db.Collection("rewards").Aggregate(ctx, mongo.Pipeline{ - matchStage, - groupStage, - }) - if err != nil { - return 0, 0, fmt.Errorf("error get coinbase rewards: %w", err) - } - if !cursor.Next(ctx) { - return 0, 0, nil - } - doc := cursor.Current - return utils.GetAsInt64(doc.Lookup("total")), utils.GetAsInt64(doc.Lookup("count")), nil -} - -// GetTotalRewards returns the total number of rewards. -func (s *Reader) GetTotalRewards(ctx context.Context, filter *bson.D) (total, count int64, err error) { - groupStage := bson.D{ - {Key: "$group", Value: bson.D{ - {Key: "_id", Value: ""}, - {Key: "total", Value: bson.D{ - {Key: "$sum", Value: "$total"}, - }}, - {Key: "count", Value: bson.D{ - {Key: "$sum", Value: 1}, - }}, - }}, - } - - pipeline := bson.A{ - groupStage, - } - - if filter != nil { - pipeline = append(bson.A{ - bson.D{{Key: "$match", Value: *filter}}, - }, pipeline...) - } - - cursor, err := s.db.Collection("rewards").Aggregate(ctx, pipeline) - if err != nil { - return 0, 0, fmt.Errorf("error get total rewards: %w", err) - } - if !cursor.Next(ctx) { - return 0, 0, nil - } - doc := cursor.Current - return utils.GetAsInt64(doc.Lookup("total")), utils.GetAsInt64(doc.Lookup("count")), nil -} - -// GetLatestReward returns the latest reward for given coinbase -func (s *Reader) GetLatestReward(ctx context.Context, coinbase string) (*model.Reward, error) { - matchStage := bson.D{{Key: "$match", Value: bson.D{{Key: "coinbase", Value: coinbase}}}} - groupStage := bson.D{ - {Key: "$group", Value: bson.D{ - {Key: "_id", Value: ""}, - {Key: "layer", Value: bson.D{ - {Key: "$max", Value: "$layer"}, - }}, - }}, - } - - cursor, err := s.db.Collection("rewards").Aggregate(ctx, mongo.Pipeline{matchStage, groupStage}) - if err != nil { - return nil, fmt.Errorf("error occured while getting latest reward: %w", err) - } - if !cursor.Next(ctx) { - return nil, nil - } - - var reward *model.Reward - if err = cursor.Decode(&reward); err != nil { - return nil, fmt.Errorf("error decode reward: %w", err) - } - return reward, nil -} diff --git a/internal/storage/storagereader/smeshers.go b/internal/storage/storagereader/smeshers.go deleted file mode 100644 index 6b49066..0000000 --- a/internal/storage/storagereader/smeshers.go +++ /dev/null @@ -1,123 +0,0 @@ -package storagereader - -import ( - "context" - "fmt" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" - "github.com/spacemeshos/explorer-backend/utils" -) - -// CountSmeshers returns the number of smeshers matching the query. -func (s *Reader) CountSmeshers(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) { - count, err := s.db.Collection("smeshers").CountDocuments(ctx, query, opts...) - if err != nil { - return 0, fmt.Errorf("error count transactions: %w", err) - } - return count, nil -} - -// GetSmeshers returns the smeshers matching the query. -func (s *Reader) GetSmeshers(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Smesher, error) { - cursor, err := s.db.Collection("smeshers").Find(ctx, query, opts...) - if err != nil { - return nil, fmt.Errorf("error get smeshers: %w", err) - } - - var smeshers []*model.Smesher - if err = cursor.All(ctx, &smeshers); err != nil { - return nil, fmt.Errorf("error decode smeshers: %w", err) - } - - return smeshers, nil -} - -// GetEpochSmeshers returns the smeshers for specific epoch -func (s *Reader) CountEpochSmeshers(ctx context.Context, query *bson.D) (int64, error) { - count, err := s.db.Collection("smeshers").CountDocuments(ctx, query) - if err != nil { - return 0, fmt.Errorf("error get smeshers: %w", err) - } - return count, nil -} - -// GetEpochSmeshers returns the smeshers for specific epoch -func (s *Reader) GetEpochSmeshers(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Smesher, error) { - cursor, err := s.db.Collection("smeshers").Find(ctx, query, opts...) - if err != nil { - return nil, fmt.Errorf("error get smeshers: %w", err) - } - - var smeshers []*model.Smesher - if err = cursor.All(ctx, &smeshers); err != nil { - return nil, fmt.Errorf("error decode smeshers: %w", err) - } - - return smeshers, nil -} - -// GetSmesher returns the smesher matching the query. -func (s *Reader) GetSmesher(ctx context.Context, smesherID string) (*model.Smesher, error) { - matchStage := bson.D{{Key: "$match", Value: bson.D{{Key: "id", Value: smesherID}}}} - lookupStage := bson.D{ - {Key: "$lookup", - Value: bson.D{ - {Key: "from", Value: "malfeasance_proofs"}, - {Key: "localField", Value: "id"}, - {Key: "foreignField", Value: "smesher"}, - {Key: "as", Value: "proofs"}, - }, - }, - } - cursor, err := s.db.Collection("smeshers").Aggregate(ctx, mongo.Pipeline{ - matchStage, - lookupStage, - }) - if err != nil { - return nil, fmt.Errorf("error get smesher `%s`: %w", smesherID, err) - } - if !cursor.Next(ctx) { - return nil, nil - } - - var smesher *model.Smesher - if err = cursor.Decode(&smesher); err != nil { - return nil, fmt.Errorf("error decode smesher `%s`: %w", smesherID, err) - } - - return smesher, nil -} - -// CountSmesherRewards returns the number of smesher rewards matching the query. -func (s *Reader) CountSmesherRewards(ctx context.Context, smesherID string) (total, count int64, err error) { - matchStage := bson.D{{Key: "$match", Value: bson.D{{Key: "smesher", Value: smesherID}}}} - groupStage := bson.D{ - {Key: "$group", Value: bson.D{ - {Key: "_id", Value: ""}, - {Key: "total", Value: bson.D{ - {Key: "$sum", Value: "$total"}, - }}, - {Key: "layerReward", Value: bson.D{ - {Key: "$sum", Value: "$layerReward"}, - }}, - {Key: "count", Value: bson.D{ - {Key: "$sum", Value: 1}, - }}, - }}, - } - cursor, err := s.db.Collection("rewards").Aggregate(ctx, mongo.Pipeline{ - matchStage, - groupStage, - }) - if err != nil { - return 0, 0, fmt.Errorf("error get smesher rewards: %w", err) - } - if !cursor.Next(ctx) { - return 0, 0, nil - } - doc := cursor.Current - return utils.GetAsInt64(doc.Lookup("total")), utils.GetAsInt64(doc.Lookup("count")), nil -} diff --git a/internal/storage/storagereader/storage.go b/internal/storage/storagereader/storage.go deleted file mode 100644 index 24307c7..0000000 --- a/internal/storage/storagereader/storage.go +++ /dev/null @@ -1,63 +0,0 @@ -package storagereader - -import ( - "context" - "errors" - "fmt" - "time" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// Reader is a wrapper around a mongo client. This client is read-only. -type Reader struct { - client *mongo.Client - db *mongo.Database -} - -// NewStorageReader creates a new storage reader. -func NewStorageReader(ctx context.Context, dbURL string, dbName string) (*Reader, error) { - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - client, err := mongo.Connect(ctx, options.Client().ApplyURI(dbURL)) - if err != nil { - return nil, fmt.Errorf("error connect to db: %s", err) - } - - if err = client.Ping(ctx, nil); err != nil { - return nil, fmt.Errorf("error ping to db: %s", err) - } - reader := &Reader{ - client: client, - db: client.Database(dbName), - } - return reader, nil -} - -// GetNetworkInfo returns the network info matching the query. -func (s *Reader) GetNetworkInfo(ctx context.Context) (*model.NetworkInfo, error) { - cursor, err := s.db.Collection("networkinfo").Find(ctx, bson.D{{Key: "id", Value: 1}}) - if err != nil { - return nil, fmt.Errorf("error get network info: %s", err) - } - if !cursor.Next(ctx) { - return nil, fmt.Errorf("error get network info: %s", errors.New("empty result")) - } - var result model.NetworkInfo - if err = cursor.Decode(&result); err != nil { - return nil, fmt.Errorf("error decode network info: %s", err) - } - return &result, nil -} - -// Ping checks if the database is reachable. -func (s *Reader) Ping(ctx context.Context) error { - if s.client == nil { - return errors.New("storage not initialized") - } - return s.client.Ping(ctx, nil) -} diff --git a/internal/storage/storagereader/transactions.go b/internal/storage/storagereader/transactions.go deleted file mode 100644 index 39268cf..0000000 --- a/internal/storage/storagereader/transactions.go +++ /dev/null @@ -1,156 +0,0 @@ -package storagereader - -import ( - "context" - "fmt" - "github.com/spacemeshos/explorer-backend/utils" - "go.mongodb.org/mongo-driver/mongo" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" -) - -// CountTransactions returns the number of transactions matching the query. -func (s *Reader) CountTransactions(ctx context.Context, query *bson.D, opts ...*options.CountOptions) (int64, error) { - count, err := s.db.Collection("txs").CountDocuments(ctx, query, opts...) - if err != nil { - return 0, fmt.Errorf("error count transactions: %w", err) - } - return count, nil -} - -// GetTransactions returns the transactions matching the query. -func (s *Reader) GetTransactions(ctx context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Transaction, error) { - cursor, err := s.db.Collection("txs").Find(ctx, query, opts...) - if err != nil { - return nil, fmt.Errorf("error get txs: %w", err) - } - - var txs []*model.Transaction - if err = cursor.All(ctx, &txs); err != nil { - return nil, fmt.Errorf("error decode txs: %w", err) - } - return txs, nil -} - -func (s *Reader) CountSentTransactions(ctx context.Context, address string) (amount, fees, count int64, err error) { - matchStage := bson.D{{Key: "$match", Value: bson.D{{Key: "sender", Value: address}}}} - groupStage := bson.D{ - {Key: "$group", Value: bson.D{ - {Key: "_id", Value: ""}, - {Key: "amount", Value: bson.D{ - {Key: "$sum", Value: "$amount"}, - }}, - {Key: "fees", Value: bson.D{ - {Key: "$sum", Value: "$fee"}, - }}, - {Key: "count", Value: bson.D{ - {Key: "$sum", Value: 1}, - }}, - }}, - } - cursor, err := s.db.Collection("txs").Aggregate(ctx, mongo.Pipeline{ - matchStage, - groupStage, - }) - if err != nil { - return 0, 0, 0, fmt.Errorf("error get sent txs: %w", err) - } - if !cursor.Next(ctx) { - return 0, 0, 0, nil - } - doc := cursor.Current - return utils.GetAsInt64(doc.Lookup("amount")), - utils.GetAsInt64(doc.Lookup("fees")), utils.GetAsInt64(doc.Lookup("count")), nil -} - -func (s *Reader) CountReceivedTransactions(ctx context.Context, address string) (amount, count int64, err error) { - matchStage := bson.D{{Key: "$match", Value: bson.D{{Key: "receiver", Value: address}}}} - groupStage := bson.D{ - {Key: "$group", Value: bson.D{ - {Key: "_id", Value: ""}, - {Key: "amount", Value: bson.D{ - {Key: "$sum", Value: "$amount"}, - }}, - {Key: "fees", Value: bson.D{ - {Key: "$sum", Value: "$fee"}, - }}, - {Key: "count", Value: bson.D{ - {Key: "$sum", Value: 1}, - }}, - }}, - } - cursor, err := s.db.Collection("txs").Aggregate(ctx, mongo.Pipeline{ - matchStage, - groupStage, - }) - if err != nil { - return 0, 0, fmt.Errorf("error get received txs: %w", err) - } - if !cursor.Next(ctx) { - return 0, 0, nil - } - doc := cursor.Current - return utils.GetAsInt64(doc.Lookup("amount")), utils.GetAsInt64(doc.Lookup("count")), nil -} - -// GetLatestTransaction returns the latest tx for given address -func (s *Reader) GetLatestTransaction(ctx context.Context, address string) (*model.Transaction, error) { - matchStage := bson.D{{Key: "$match", Value: bson.D{ - {Key: "$or", Value: bson.A{ - bson.D{{"sender", address}}, - bson.D{{"receiver", address}}, - }}}, - }} - groupStage := bson.D{ - {Key: "$group", Value: bson.D{ - {Key: "_id", Value: ""}, - {Key: "layer", Value: bson.D{ - {Key: "$max", Value: "$layer"}, - }}, - }}, - } - - cursor, err := s.db.Collection("txs").Aggregate(ctx, mongo.Pipeline{matchStage, groupStage}) - if err != nil { - return nil, fmt.Errorf("error occured while getting latest reward: %w", err) - } - if !cursor.Next(ctx) { - return nil, nil - } - - var tx *model.Transaction - if err = cursor.Decode(&tx); err != nil { - return nil, fmt.Errorf("error decode reward: %w", err) - } - return tx, nil -} - -// GetFirstSentTransaction returns the first sent tx for given address -func (s *Reader) GetFirstSentTransaction(ctx context.Context, address string) (*model.Transaction, error) { - matchStage := bson.D{{Key: "$match", Value: bson.D{{Key: "sender", Value: address}}}} - groupStage := bson.D{ - {Key: "$group", Value: bson.D{ - {Key: "_id", Value: ""}, - {Key: "layer", Value: bson.D{ - {Key: "$min", Value: "$layer"}, - }}, - }}, - } - - cursor, err := s.db.Collection("txs").Aggregate(ctx, mongo.Pipeline{matchStage, groupStage}) - if err != nil { - return nil, fmt.Errorf("error occured while getting latest reward: %w", err) - } - if !cursor.Next(ctx) { - return nil, nil - } - - var tx *model.Transaction - if err = cursor.Decode(&tx); err != nil { - return nil, fmt.Errorf("error decode reward: %w", err) - } - return tx, nil -} diff --git a/model/account.go b/model/account.go deleted file mode 100644 index 4d7085b..0000000 --- a/model/account.go +++ /dev/null @@ -1,45 +0,0 @@ -package model - -import ( - "context" - - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" -) - -type Account struct { - Address string `json:"address" bson:"address"` // account public address - Balance uint64 `json:"balance" bson:"balance"` // known account balance - Counter uint64 `json:"counter" bson:"counter"` - Created uint64 `json:"created" bson:"created"` - // get from ledger collection - Sent uint64 `json:"sent" bson:"-"` - Received uint64 `json:"received" bson:"-"` - Awards uint64 `json:"awards" bson:"-"` - Fees uint64 `json:"fees" bson:"-"` - Txs int64 `json:"txs" bson:"-"` - LastActivity int32 `json:"lastActivity" bson:"-"` -} - -// AccountSummary data taken from `ledger` collection. Not all accounts from api have filled this data. -type AccountSummary struct { - Sent uint64 `json:"sent" bson:"sent"` - Received uint64 `json:"received" bson:"received"` - Awards uint64 `json:"awards" bson:"awards"` - Fees uint64 `json:"fees" bson:"fees"` - LastActivity int32 `json:"lastActivity" bson:"-"` -} - -type AccountService interface { - GetAccount(ctx context.Context, accountID string) (*Account, error) - GetAccounts(ctx context.Context, page, perPage int64) ([]*Account, int64, error) - GetAccountTransactions(ctx context.Context, accountID string, page, perPage int64) ([]*Transaction, int64, error) - GetAccountRewards(ctx context.Context, accountID string, page, perPage int64) ([]*Reward, int64, error) -} - -func NewAccount(in *pb.Account) *Account { - return &Account{ - Address: in.GetAccountId().GetAddress(), - Balance: in.GetStateCurrent().GetBalance().GetValue(), - Counter: in.GetStateCurrent().GetCounter(), - } -} diff --git a/model/app.go b/model/app.go deleted file mode 100644 index e77662e..0000000 --- a/model/app.go +++ /dev/null @@ -1,14 +0,0 @@ -package model - -import ( - "context" -) - -type App struct { - Address string `json:"address" bson:"address"` -} - -type AppService interface { - GetApps(ctx context.Context, pageNum, pageSize int64) (apps []*App, total int64, err error) - GetApp(ctx context.Context, appID string) (*App, error) -} diff --git a/model/atx.go b/model/atx.go deleted file mode 100644 index cba502f..0000000 --- a/model/atx.go +++ /dev/null @@ -1,53 +0,0 @@ -package model - -import ( - "context" - "github.com/spacemeshos/go-spacemesh/common/types" - - "github.com/spacemeshos/explorer-backend/utils" -) - -type Activation struct { - Id string `json:"id" bson:"id"` //nolint will fix it later. - SmesherId string `json:"smesher" bson:"smesher"` //nolint will fix it later // id of smesher who created the ATX - Coinbase string `json:"coinbase" bson:"coinbase"` // coinbase account id - PrevAtx string `json:"prevAtx" bson:"prevAtx"` // previous ATX pointed to - NumUnits uint32 `json:"numunits" bson:"numunits"` // number of PoST data commitment units - CommitmentSize uint64 `json:"commitmentSize" bson:"commitmentSize"` - PublishEpoch uint32 `json:"publishEpoch" bson:"publishEpoch"` - TargetEpoch uint32 `json:"targetEpoch" bson:"targetEpoch"` - TickCount uint64 `json:"tickCount" bson:"tickCount"` - Weight uint64 `json:"weight" bson:"weight"` - EffectiveNumUnits uint32 `json:"effectiveNumUnits" bson:"effectiveNumUnits"` - Received int64 `json:"received" bson:"received"` -} - -type ActivationService interface { - GetActivations(ctx context.Context, page, perPage int64) (atxs []*Activation, total int64, err error) - GetActivation(ctx context.Context, activationID string) (*Activation, error) -} - -func NewActivation(atx *types.ActivationTx) *Activation { - return &Activation{ - Id: utils.BytesToHex(atx.ID().Bytes()), - PublishEpoch: atx.PublishEpoch.Uint32(), - TargetEpoch: atx.PublishEpoch.Uint32() + 1, - SmesherId: utils.BytesToHex(atx.SmesherID.Bytes()), - Coinbase: atx.Coinbase.String(), - PrevAtx: utils.BytesToHex(atx.PrevATXID.Bytes()), - NumUnits: atx.NumUnits, - TickCount: atx.TickCount, - Weight: atx.GetWeight(), - EffectiveNumUnits: atx.NumUnits, - Received: atx.Received().UnixNano(), - } -} - -func (atx *Activation) GetSmesher(unitSize uint64) *Smesher { - return &Smesher{ - Id: atx.SmesherId, - Coinbase: atx.Coinbase, - Timestamp: uint64(atx.Received), - CommitmentSize: uint64(atx.NumUnits) * unitSize, - } -} diff --git a/model/block.go b/model/block.go deleted file mode 100644 index edc9807..0000000 --- a/model/block.go +++ /dev/null @@ -1,19 +0,0 @@ -package model - -import ( - "context" -) - -type Block struct { - Id string `json:"id" bson:"id"` // nolint will fix it later - Layer uint32 `json:"layer" bson:"layer"` - Epoch uint32 `json:"epoch" bson:"epoch"` - Start uint32 `json:"start" bson:"start"` - End uint32 `json:"end" bson:"end"` - TxsNumber uint32 `json:"txsnumber" bson:"txsnumber"` - TxsValue uint64 `json:"txsvalue" bson:"txsvalue"` -} - -type BlockService interface { - GetBlock(ctx context.Context, blockID string) (*Block, error) -} diff --git a/model/epoch.go b/model/epoch.go deleted file mode 100644 index 7416a28..0000000 --- a/model/epoch.go +++ /dev/null @@ -1,43 +0,0 @@ -package model - -import ( - "context" -) - -type Statistics struct { - Capacity int64 `json:"capacity" bson:"capacity"` // Average tx/s rate over capacity considering all layers in the current epoch. - Decentral int64 `json:"decentral" bson:"decentral"` // Distribution of storage between all active smeshers. - Smeshers int64 `json:"smeshers" bson:"smeshers"` // Number of active smeshers in the current epoch. - Transactions int64 `json:"transactions" bson:"transactions"` // Total number of transactions processed by the state transition function. - Accounts int64 `json:"accounts" bson:"accounts"` // Total number of on-mesh accounts with a non-zero coin balance as of the current epoch. - Circulation int64 `json:"circulation" bson:"circulation"` // Total number of Smesh coins in circulation. This is the total balances of all on-mesh accounts. - Rewards int64 `json:"rewards" bson:"rewards"` // Total amount of Smesh minted as mining rewards as of the last known reward distribution event. - RewardsNumber int64 `json:"rewardsnumber" bson:"rewardsnumber"` - Security int64 `json:"security" bson:"security"` // Total amount of storage committed to the network based on the ATXs in the previous epoch. - TxsAmount int64 `json:"txsamount" bson:"txsamount"` // Total amount of coin transferred between accounts in the epoch. Incl coin transactions and smart wallet transactions. -} - -type Stats struct { - Current Statistics `json:"current"` - Cumulative Statistics `json:"cumulative"` -} - -type Epoch struct { - Number int32 `json:"number" bson:"number"` - Start uint32 `json:"start" bson:"start"` - End uint32 `json:"end" bson:"end"` - LayerStart uint32 `json:"layerstart" bson:"layerstart"` - LayerEnd uint32 `json:"layerend" bson:"layerend"` - Layers uint32 `json:"layers" bson:"layers"` - Stats Stats `json:"stats"` -} - -type EpochService interface { - GetEpoch(ctx context.Context, epochNum int) (*Epoch, error) - GetEpochs(ctx context.Context, page, perPage int64) (epochs []*Epoch, total int64, err error) - GetEpochLayers(ctx context.Context, epochNum int, page, perPage int64) (layers []*Layer, total int64, err error) - GetEpochTransactions(ctx context.Context, epochNum int, page, perPage int64) (txs []*Transaction, total int64, err error) - GetEpochSmeshers(ctx context.Context, epochNum int, page, perPage int64) (smeshers []*Smesher, total int64, err error) - GetEpochRewards(ctx context.Context, epochNum int, page, perPage int64) (rewards []*Reward, total int64, err error) - GetEpochActivations(ctx context.Context, epochNum int, page, perPage int64) (atxs []*Activation, total int64, err error) -} diff --git a/model/layer.go b/model/layer.go deleted file mode 100644 index ff47120..0000000 --- a/model/layer.go +++ /dev/null @@ -1,91 +0,0 @@ -package model - -import ( - "context" - "fmt" - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" - "github.com/spacemeshos/go-spacemesh/log" - - "github.com/spacemeshos/explorer-backend/utils" -) - -type Layer struct { - Number uint32 `json:"number" bson:"number"` - Status int `json:"status" bson:"status"` - Txs uint32 `json:"txs" bson:"txs"` - Start uint32 `json:"start" bson:"start"` - End uint32 `json:"end" bson:"end"` - TxsAmount uint64 `json:"txsamount" bson:"txsamount"` - Rewards uint64 `json:"rewards" bson:"rewards"` - Epoch uint32 `json:"epoch" bson:"epoch"` - Hash string `json:"hash" bson:"hash"` - BlocksNumber uint32 `json:"blocksnumber" bson:"blocksnumber"` -} - -type LayerService interface { - GetLayer(ctx context.Context, layerNum int) (*Layer, error) - //GetLayerByHash(ctx context.Context, layerHash string) (*Layer, error) - GetLayers(ctx context.Context, page, perPage int64) (layers []*Layer, total int64, err error) - GetLayerTransactions(ctx context.Context, layerNum int, pageNum, pageSize int64) (txs []*Transaction, total int64, err error) - GetLayerSmeshers(ctx context.Context, layerNum int, pageNum, pageSize int64) (smeshers []*Smesher, total int64, err error) - GetLayerRewards(ctx context.Context, layerNum int, pageNum, pageSize int64) (rewards []*Reward, total int64, err error) - GetLayerActivations(ctx context.Context, layerNum int, pageNum, pageSize int64) (atxs []*Activation, total int64, err error) - GetLayerBlocks(ctx context.Context, layerNum int, pageNum, pageSize int64) (blocks []*Block, total int64, err error) -} - -func NewLayer(in *pb.Layer, networkInfo *NetworkInfo) (*Layer, []*Block, []*Activation, map[string]*Transaction) { - pbBlocks := in.GetBlocks() - pbAtxs := in.GetActivations() - layer := &Layer{ - Number: in.Number.Number, - Status: int(in.GetStatus()), - Epoch: in.Number.Number / networkInfo.EpochNumLayers, - BlocksNumber: uint32(len(pbBlocks)), - Hash: utils.BytesToHex(in.Hash), - } - if layer.Number == 0 { - layer.Start = networkInfo.GenesisTime - } else { - layer.Start = networkInfo.GenesisTime + layer.Number*networkInfo.LayerDuration - } - layer.End = layer.Start + networkInfo.LayerDuration - 1 - - blocks := make([]*Block, len(pbBlocks)) - atxs := make([]*Activation, len(pbAtxs)) - txs := make(map[string]*Transaction) - - for i, b := range pbBlocks { - blocks[i] = &Block{ - Id: utils.NBytesToHex(b.GetId(), 20), - Layer: layer.Number, - Epoch: layer.Epoch, - Start: layer.Start, - End: layer.End, - TxsNumber: uint32(len(b.Transactions)), - } - for j, t := range b.Transactions { - tx, err := NewTransaction(t, layer.Number, blocks[i].Id, layer.Start, uint32(j)) - if err != nil { - log.Err(fmt.Errorf("cannot create transaction: %v", err)) - continue - } - txs[tx.Id] = tx - blocks[i].TxsValue += tx.Amount - } - } - - layer.Txs = uint32(len(txs)) - for _, tx := range txs { - layer.TxsAmount += tx.Amount - } - - return layer, blocks, atxs, txs -} - -func IsApprovedLayer(l *pb.Layer) bool { - return l.GetStatus() >= pb.Layer_LAYER_STATUS_APPROVED -} - -func IsConfirmedLayer(l *pb.Layer) bool { - return l.GetStatus() == pb.Layer_LAYER_STATUS_CONFIRMED -} diff --git a/model/malfeasance_proof.go b/model/malfeasance_proof.go deleted file mode 100644 index 4f76ec1..0000000 --- a/model/malfeasance_proof.go +++ /dev/null @@ -1,22 +0,0 @@ -package model - -import ( - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" - "github.com/spacemeshos/explorer-backend/utils" -) - -type MalfeasanceProof struct { - Smesher string `json:"smesher" bson:"smesher"` - Layer uint32 `json:"layer" bson:"layer"` - Kind string `json:"kind" bson:"kind"` - DebugInfo string `json:"debugInfo" bson:"debugInfo"` -} - -func NewMalfeasanceProof(in *pb.MalfeasanceProof) *MalfeasanceProof { - return &MalfeasanceProof{ - Smesher: utils.BytesToHex(in.GetSmesherId().GetId()), - Layer: in.Layer.GetNumber(), - Kind: in.Kind.String(), - DebugInfo: in.DebugInfo, - } -} diff --git a/model/network_info.go b/model/network_info.go deleted file mode 100644 index c5adaee..0000000 --- a/model/network_info.go +++ /dev/null @@ -1,21 +0,0 @@ -package model - -type NetworkInfo struct { - GenesisId string `json:"genesisid" bson:"genesisid"` // nolint will fix it later - GenesisTime uint32 `json:"genesis" bson:"genesis"` - EpochNumLayers uint32 `json:"layers" bson:"layers"` - MaxTransactionsPerSecond uint32 `json:"maxtx" bson:"maxtx"` - LayerDuration uint32 `json:"duration" bson:"duration"` - PostUnitSize uint64 `json:"postUnitSize" bson:"postUnitSize"` - - LastLayer uint32 `json:"lastlayer" bson:"lastlayer"` - LastLayerTimestamp uint32 `json:"lastlayerts" bson:"lastlayerts"` - LastApprovedLayer uint32 `json:"lastapprovedlayer" bson:"lastapprovedlayer"` - LastConfirmedLayer uint32 `json:"lastconfirmedlayer" bson:"lastconfirmedlayer"` - - ConnectedPeers uint64 `json:"connectedpeers" bson:"connectedpeers"` - IsSynced bool `json:"issynced" bson:"issynced"` - SyncedLayer uint32 `json:"syncedlayer" bson:"syncedlayer"` - TopLayer uint32 `json:"toplayer" bson:"toplayer"` - VerifiedLayer uint32 `json:"verifiedlayer" bson:"verifiedlayer"` -} diff --git a/model/reward.go b/model/reward.go deleted file mode 100644 index 00ff3d3..0000000 --- a/model/reward.go +++ /dev/null @@ -1,40 +0,0 @@ -package model - -import ( - "context" - "go.mongodb.org/mongo-driver/bson" - - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" - - "github.com/spacemeshos/explorer-backend/utils" -) - -type Reward struct { - ID string `json:"_id" bson:"_id"` - Layer uint32 `json:"layer" bson:"layer"` - Total uint64 `json:"total" bson:"total"` - LayerReward uint64 `json:"layerReward" bson:"layerReward"` - LayerComputed uint32 `json:"layerComputed" bson:"layerComputed"` // layer number of the layer when reward was computed - // tx_fee = total - layer_reward - Coinbase string `json:"coinbase" bson:"coinbase"` // account awarded this reward - Smesher string `json:"smesher" bson:"smesher"` - Timestamp uint32 `json:"timestamp" bson:"timestamp"` -} - -type RewardService interface { - GetReward(ctx context.Context, rewardID string) (*Reward, error) - GetRewardV2(ctx context.Context, smesherID string, layer uint32) (*Reward, error) - GetRewards(ctx context.Context, page, perPage int64) ([]*Reward, int64, error) - GetTotalRewards(ctx context.Context, filter *bson.D) (int64, int64, error) -} - -func NewReward(reward *pb.Reward) *Reward { - return &Reward{ - Layer: reward.GetLayer().GetNumber(), - Total: reward.GetTotal().GetValue(), - LayerReward: reward.GetLayerReward().GetValue(), - LayerComputed: reward.GetLayerComputed().GetNumber(), - Coinbase: reward.GetCoinbase().GetAddress(), - Smesher: utils.BytesToHex(reward.GetSmesher().GetId()), - } -} diff --git a/model/smesher.go b/model/smesher.go deleted file mode 100644 index 0ef66d5..0000000 --- a/model/smesher.go +++ /dev/null @@ -1,30 +0,0 @@ -package model - -import ( - "context" -) - -type Geo struct { - Name string `json:"name"` - Coordinates [2]float64 `json:"coordinates"` -} - -type Smesher struct { - Id string `json:"id" bson:"id"` //nolint will fix it later. - CommitmentSize uint64 `json:"cSize" bson:"cSize"` - Coinbase string `json:"coinbase" bson:"coinbase"` - AtxCount uint32 `json:"atxcount" bson:"atxcount"` - Timestamp uint64 `json:"timestamp" bson:"timestamp"` - Rewards int64 `json:"rewards" bson:"-"` - AtxLayer uint32 `json:"atxLayer" bson:"atxLayer"` - Proofs []MalfeasanceProof `json:"proofs,omitempty" bson:"proofs,omitempty"` - Epochs []uint32 `json:"epochs,omitempty" bson:"epochs,omitempty"` -} - -type SmesherService interface { - GetSmesher(ctx context.Context, smesherID string) (*Smesher, error) - GetSmeshers(ctx context.Context, page, perPage int64) (smeshers []*Smesher, total int64, err error) - GetSmesherActivations(ctx context.Context, smesherID string, page, perPage int64) (atxs []*Activation, total int64, err error) - GetSmesherRewards(ctx context.Context, smesherID string, page, perPage int64) (rewards []*Reward, total int64, err error) - CountSmesherRewards(ctx context.Context, smesherID string) (total, count int64, err error) -} diff --git a/model/tx.go b/model/tx.go deleted file mode 100644 index 8870880..0000000 --- a/model/tx.go +++ /dev/null @@ -1,142 +0,0 @@ -package model - -import ( - "context" - "fmt" - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" - "github.com/spacemeshos/explorer-backend/pkg/transactionparser" - "github.com/spacemeshos/explorer-backend/pkg/transactionparser/transaction" - "github.com/spacemeshos/explorer-backend/utils" - "strings" -) - -type Transaction struct { - Id string `json:"id" bson:"id"` //nolint will fix it later - - Layer uint32 `json:"layer" bson:"layer"` - Block string `json:"block" bson:"block"` - BlockIndex uint32 `json:"blockIndex" bson:"blockIndex"` - Index uint32 `json:"index" bson:"index"` // the index of the tx in the ordered list of txs to be executed by stf in the layer - State int `json:"state" bson:"state"` - Result int `json:"result" bson:"result"` - Timestamp uint32 `json:"timestamp" bson:"timestamp"` - - MaxGas uint64 `json:"maxGas" bson:"maxGas"` - GasPrice uint64 `json:"gasPrice" bson:"gasPrice"` - GasUsed uint64 `bson:"gasUsed" json:"gasUsed"` // gas units used by the transaction (gas price in tx) - Fee uint64 `json:"fee" bson:"fee"` // transaction fee charged for the transaction - - Amount uint64 `json:"amount" bson:"amount"` // amount of coin transferred in this tx by sender - Counter uint64 `json:"counter" bson:"counter"` // tx counter aka nonce - - Type int `json:"type" bson:"type"` - Signature string `json:"signature" bson:"signature"` // the signature itself - Signatures []SignaturePart `json:"signatures" bson:"signatures"` // the signature itself - PublicKey string `json:"pubKey" bson:"pubKey"` // included in schemes which require signer to provide a public key - - Sender string `json:"sender" bson:"sender"` // tx originator, should match signer inside Signature - Receiver string `json:"receiver" bson:"receiver"` - SvmData string `json:"svmData" bson:"svmData"` // svm binary data. Decode with svm-codec - - Message string `json:"message" bson:"message"` - TouchedAddresses []string `json:"touchedAddresses" bson:"touchedAddresses"` - - Vault string `json:"vault" bson:"vault"` - VaultOwner string `json:"vaultOwner" bson:"vaultOwner"` - VaultTotalAmount uint64 `json:"vaultTotalAmount" bson:"vaultTotalAmount"` - VaultInitialUnlockAmount uint64 `json:"vaultInitialUnlockAmount" bson:"vaultInitialUnlockAmount"` - VaultVestingStart uint32 `json:"vaultVestingStart" bson:"vaultVestingStart"` - VaultVestingEnd uint32 `json:"vaultVestingEnd" bson:"vaultVestingEnd"` -} - -type SignaturePart struct { - Ref uint32 `json:"ref" bson:"ref"` - Signature string `json:"signature" bson:"signature"` -} - -type TransactionReceipt struct { - Id string //nolint will fix it later - Result int - Message string - GasUsed uint64 // gas units used by the transaction (gas price in tx) - Fee uint64 // transaction fee charged for the transaction - Layer uint32 - Block string - TouchedAddresses []string -} - -type TransactionService interface { - GetTransaction(ctx context.Context, txID string) (*Transaction, error) - GetTransactions(ctx context.Context, page, perPage int64) (txs []*Transaction, total int64, err error) -} - -func NewTransactionResult(res *pb.TransactionResult, state *pb.TransactionState, networkInfo NetworkInfo) (*Transaction, error) { - layerStart := networkInfo.GenesisTime + res.GetLayer()*networkInfo.LayerDuration - tx, err := NewTransaction(res.GetTx(), res.GetLayer(), utils.NBytesToHex(res.GetBlock(), 20), layerStart, 0) - if err != nil { - return nil, err - } - - tx.State = int(state.State) - tx.Fee = res.GetFee() - tx.GasUsed = res.GetGasConsumed() - tx.Message = res.GetMessage() - tx.TouchedAddresses = res.GetTouchedAddresses() - tx.Result = int(res.Status) - - return tx, nil -} - -// NewTransaction try to parse the transaction and return a new Transaction struct. -func NewTransaction(in *pb.Transaction, layer uint32, blockID string, timestamp uint32, blockIndex uint32) (*Transaction, error) { - txDecoded, err := transactionparser.Parse(in.GetRaw()) - if err != nil { - return nil, fmt.Errorf("failed to parse transaction: %w", err) - } - tx := &Transaction{ - Id: utils.BytesToHex(in.GetId()), - Sender: txDecoded.Tx.GetPrincipal().String(), - Amount: txDecoded.Tx.GetAmount(), - Counter: txDecoded.Tx.GetCounter(), - Layer: layer, - Block: blockID, - BlockIndex: blockIndex, - State: int(pb.TransactionState_TRANSACTION_STATE_UNSPECIFIED), - Timestamp: timestamp, - MaxGas: in.GetMaxGas(), - GasPrice: txDecoded.Tx.GetGasPrice(), - Fee: in.GetMaxGas() * txDecoded.Tx.GetGasPrice(), - Type: txDecoded.Type, - Receiver: txDecoded.Tx.GetReceiver().String(), - } - keys := make([]string, 0, len(txDecoded.Tx.GetPublicKeys())) - for i := range txDecoded.Tx.GetPublicKeys() { - keys = append(keys, utils.BytesToHex(txDecoded.Tx.GetPublicKeys()[i])) - } - tx.PublicKey = strings.Join(keys, ",") - - if txDecoded.Signatures != nil { - for _, sig := range *txDecoded.Signatures { - tx.Signatures = append(tx.Signatures, SignaturePart{ - Ref: uint32(sig.Ref), - Signature: utils.BytesToHex(sig.Sig.Bytes()), - }) - } - } else { - tx.Signature = utils.BytesToHex(txDecoded.Sig.Bytes()) - } - - if txDecoded.Type == transaction.TypeDrainVault { - tx.Vault = txDecoded.Vault.GetVault().String() - } - - if txDecoded.Type == transaction.TypeVaultSpawn { - tx.VaultOwner = txDecoded.Vault.GetOwner().String() - tx.VaultTotalAmount = txDecoded.Vault.GetTotalAmount() - tx.VaultInitialUnlockAmount = txDecoded.Vault.GetInitialUnlockAmount() - tx.VaultVestingStart = txDecoded.Vault.GetVestingStart().Uint32() - tx.VaultVestingEnd = txDecoded.Vault.GetVestingEnd().Uint32() - } - - return tx, nil -} diff --git a/pkg/transactionparser/transaction.go b/pkg/transactionparser/transaction.go deleted file mode 100644 index 1693f2f..0000000 --- a/pkg/transactionparser/transaction.go +++ /dev/null @@ -1,66 +0,0 @@ -package transactionparser - -import ( - "bytes" - "fmt" - "github.com/spacemeshos/go-scale" - "github.com/spacemeshos/go-spacemesh/genvm/core" - "github.com/spacemeshos/go-spacemesh/genvm/registry" - "github.com/spacemeshos/go-spacemesh/genvm/templates/multisig" - "github.com/spacemeshos/go-spacemesh/genvm/templates/vault" - "github.com/spacemeshos/go-spacemesh/genvm/templates/vesting" - "github.com/spacemeshos/go-spacemesh/genvm/templates/wallet" - - "github.com/spacemeshos/explorer-backend/pkg/transactionparser/transaction" - v0 "github.com/spacemeshos/explorer-backend/pkg/transactionparser/v0" -) - -// Parse parses transaction from raw bytes and returns its type. -func Parse(rawTx []byte) (*transaction.TransactionData, error) { - version, method, templateAddress, err := decodeHeader(scale.NewDecoder(bytes.NewReader(rawTx))) - if err != nil { - return nil, err - } - switch version { - case 0: - return v0.ParseTransaction(bytes.NewBuffer(rawTx), method, templateAddress) - default: - return nil, fmt.Errorf("%w: unsupported version %d", core.ErrMalformed, version) - } -} - -// decodeHeader decodes version, method and template address from *scale.Decoder -func decodeHeader(decoder *scale.Decoder) (uint8, uint8, *core.Address, error) { - reg := registry.New() - wallet.Register(reg) - multisig.Register(reg) - vesting.Register(reg) - vault.Register(reg) - - version, _, err := scale.DecodeCompact8(decoder) - if err != nil { - return 0, 0, nil, fmt.Errorf("%w: failed to decode version %w", core.ErrMalformed, err) - } - - var principal core.Address - if _, err := principal.DecodeScale(decoder); err != nil { - return 0, 0, nil, fmt.Errorf("%w failed to decode principal: %w", core.ErrMalformed, err) - } - - method, _, err := scale.DecodeCompact8(decoder) - if err != nil { - return 0, 0, nil, fmt.Errorf("%w: failed to decode method selector %w", core.ErrMalformed, err) - } - - var templateAddress *core.Address - if method == core.MethodSpawn { - templateAddress = &core.Address{} - if _, err := templateAddress.DecodeScale(decoder); err != nil { - return 0, 0, nil, fmt.Errorf("%w failed to decode template address %w", core.ErrMalformed, err) - } - } else { - templateAddress = &wallet.TemplateAddress - } - - return version, method, templateAddress, nil -} diff --git a/pkg/transactionparser/transaction/abstract.go b/pkg/transactionparser/transaction/abstract.go deleted file mode 100644 index 7a2fdfe..0000000 --- a/pkg/transactionparser/transaction/abstract.go +++ /dev/null @@ -1,40 +0,0 @@ -package transaction - -import ( - "github.com/spacemeshos/address" - "github.com/spacemeshos/go-spacemesh/genvm/core" - "github.com/spacemeshos/go-spacemesh/genvm/templates/multisig" -) - -// DecodedTransactioner is an interface for transaction decoded from raw bytes. -type DecodedTransactioner interface { - GetType() uint8 - GetAmount() uint64 - GetCounter() uint64 - GetReceiver() address.Address - GetGasPrice() uint64 - GetPrincipal() address.Address - GetPublicKeys() [][]byte -} - -type DecodedSignature interface { - GetSignature() []byte - GetSignatures() []multisig.Part -} - -type TransactionData struct { - Tx DecodedTransactioner - Sig *core.Signature - Signatures *multisig.Signatures - Vault DecodedVault - Type int -} - -type DecodedVault interface { - GetVault() core.Address - GetOwner() core.Address - GetTotalAmount() uint64 - GetInitialUnlockAmount() uint64 - GetVestingStart() core.LayerID - GetVestingEnd() core.LayerID -} diff --git a/pkg/transactionparser/transaction/transaction_types.go b/pkg/transactionparser/transaction/transaction_types.go deleted file mode 100644 index 887274a..0000000 --- a/pkg/transactionparser/transaction/transaction_types.go +++ /dev/null @@ -1,15 +0,0 @@ -package transaction - -const ( - // TypeSpawn is type of the spawn transaction. - TypeSpawn = 1 + iota - // TypeMultisigSpawn is type of the multisig spawn transaction. - TypeMultisigSpawn - // TypeSpend is type of the spend transaction. - TypeSpend - // TypeMultisigSpend is type of the multisig spend transaction. - TypeMultisigSpend - TypeVestingSpawn - TypeVaultSpawn - TypeDrainVault -) diff --git a/pkg/transactionparser/transaction_test.go b/pkg/transactionparser/transaction_test.go deleted file mode 100644 index fd297da..0000000 --- a/pkg/transactionparser/transaction_test.go +++ /dev/null @@ -1,305 +0,0 @@ -package transactionparser_test - -import ( - "github.com/oasisprotocol/curve25519-voi/primitives/ed25519" - "github.com/spacemeshos/explorer-backend/pkg/transactionparser/transaction" - "github.com/spacemeshos/go-spacemesh/common/types" - "github.com/spacemeshos/go-spacemesh/genvm/core" - "github.com/spacemeshos/go-spacemesh/genvm/sdk" - multisig2 "github.com/spacemeshos/go-spacemesh/genvm/sdk/multisig" - "github.com/spacemeshos/go-spacemesh/genvm/sdk/vesting" - sdkWallet "github.com/spacemeshos/go-spacemesh/genvm/sdk/wallet" - "github.com/spacemeshos/go-spacemesh/genvm/templates/multisig" - "github.com/spacemeshos/go-spacemesh/genvm/templates/vault" - "github.com/spacemeshos/go-spacemesh/genvm/templates/wallet" - "github.com/spacemeshos/go-spacemesh/signing" - "github.com/stretchr/testify/require" - "math/rand" - "testing" - "time" - - "github.com/spacemeshos/explorer-backend/pkg/transactionparser" -) - -func TestSpawn(t *testing.T) { - table := []struct { - name string - gasPrice uint64 - opts []sdk.Opt - }{ - { - name: "default gas price", - gasPrice: 1, - opts: []sdk.Opt{}, - }, - { - name: "non default gasPrice", - gasPrice: 2, - opts: []sdk.Opt{sdk.WithGasPrice(2)}, - }, - } - for _, tc := range table { - testCase := tc - t.Run(testCase.name, func(t *testing.T) { - t.Parallel() - signer, _ := signing.NewEdSigner() - rawTx := sdkWallet.SelfSpawn(signer.PrivateKey(), core.Nonce(0), testCase.opts...) - args := wallet.SpawnArguments{} - copy(args.PublicKey[:], signer.PublicKey().Bytes()) - principal := core.ComputePrincipal(wallet.TemplateAddress, &args) - - decodedTx, err := transactionparser.Parse(rawTx) - require.NoError(t, err) - - require.Equal(t, testCase.gasPrice, decodedTx.Tx.GetGasPrice()) - require.Equal(t, principal.String(), decodedTx.Tx.GetPrincipal().String()) - require.Equal(t, transaction.TypeSpawn, decodedTx.Type) - }) - } -} - -func TestSpend(t *testing.T) { - table := []struct { - name string - to types.Address - gasPrice uint64 - amount uint64 - opts []sdk.Opt - nonce types.Nonce - }{ - { - name: "default gas price", - amount: 123, - gasPrice: 1, - to: types.GenerateAddress(generatePublicKey()), - opts: []sdk.Opt{}, - nonce: types.Nonce(0), - }, - { - name: "non default gasPrice", - amount: 723, - gasPrice: 2, - to: types.GenerateAddress(generatePublicKey()), - opts: []sdk.Opt{sdk.WithGasPrice(2)}, - nonce: types.Nonce(0), - }, - } - for _, tc := range table { - testCase := tc - t.Run(testCase.name, func(t *testing.T) { - t.Parallel() - signer, _ := signing.NewEdSigner() - rawTx := sdkWallet.Spend(signer.PrivateKey(), testCase.to, testCase.amount, testCase.nonce, testCase.opts...) - args := wallet.SpawnArguments{} - copy(args.PublicKey[:], signing.Public(signer.PrivateKey())) - accAddress := core.ComputePrincipal(wallet.TemplateAddress, &args) - - decodedTx, err := transactionparser.Parse(rawTx) - require.NoError(t, err) - - require.Equal(t, testCase.gasPrice, decodedTx.Tx.GetGasPrice()) - require.Equal(t, testCase.to.String(), decodedTx.Tx.GetReceiver().String()) - require.Equal(t, testCase.amount, decodedTx.Tx.GetAmount()) - require.Equal(t, testCase.nonce, decodedTx.Tx.GetCounter()) - require.Equal(t, accAddress.String(), decodedTx.Tx.GetPrincipal().String()) - require.Equal(t, transaction.TypeSpend, decodedTx.Type) - }) - } -} - -func TestSpawnMultisig(t *testing.T) { - table := []struct { - name string - gasPrice uint64 - ref uint8 - opts []sdk.Opt - nonce types.Nonce - }{ - { - name: "default gas price", - gasPrice: 1, - ref: 3, - opts: []sdk.Opt{}, - nonce: types.Nonce(1), - }, - { - name: "non default gasPrice", - gasPrice: 2, - ref: 5, - opts: []sdk.Opt{sdk.WithGasPrice(2)}, - nonce: types.Nonce(2), - }, - } - for _, tc := range table { - testCase := tc - t.Run(testCase.name, func(t *testing.T) { - t.Parallel() - - var pubs []ed25519.PublicKey - pks := make([]ed25519.PrivateKey, 0, 3) - for i := 0; i < 3; i++ { - pub, pk, err := ed25519.GenerateKey(rand.New(rand.NewSource(time.Now().UnixNano()))) - require.NoError(t, err) - pubs = append(pubs, pub) - pks = append(pks, pk) - } - - var agg *multisig2.Aggregator - for i := 0; i < len(pks); i++ { - part := multisig2.SelfSpawn(uint8(i), pks[i], multisig.TemplateAddress, 1, pubs, testCase.nonce, testCase.opts...) - if agg == nil { - agg = part - } else { - agg.Add(*part.Part(uint8(i))) - } - } - rawTx := agg.Raw() - - decodedTx, err := transactionparser.Parse(rawTx) - require.NoError(t, err) - require.Equal(t, testCase.gasPrice, decodedTx.Tx.GetGasPrice()) - require.Equal(t, transaction.TypeMultisigSpawn, decodedTx.Type) - for i := 0; i < len(pks); i++ { - require.Equal(t, []byte(pubs[i]), decodedTx.Tx.GetPublicKeys()[i]) - } - }) - } -} - -func TestSpendMultisig(t *testing.T) { - table := []struct { - name string - to types.Address - gasPrice uint64 - amount uint64 - opts []sdk.Opt - nonce types.Nonce - ref uint8 - }{ - { - name: "default gas price", - amount: 123, - gasPrice: 1, - to: types.GenerateAddress(generatePublicKey()), - opts: []sdk.Opt{}, - ref: 3, - nonce: types.Nonce(1), - }, - { - name: "non default gasPrice", - amount: 723, - gasPrice: 2, - ref: 5, - to: types.GenerateAddress(generatePublicKey()), - opts: []sdk.Opt{sdk.WithGasPrice(2)}, - nonce: types.Nonce(2), - }, - } - for _, tc := range table { - testCase := tc - t.Run(testCase.name, func(t *testing.T) { - t.Parallel() - - pubs := make([][]byte, 0, 3) - pks := make([]ed25519.PrivateKey, 0, 3) - for i := 0; i < 3; i++ { - pub, pk, err := ed25519.GenerateKey(rand.New(rand.NewSource(time.Now().UnixNano()))) - require.NoError(t, err) - pubs = append(pubs, pub) - pks = append(pks, pk) - } - principal := multisig2.Address(multisig.TemplateAddress, 3, pubs...) - agg := multisig2.Spend(0, pks[0], principal, testCase.to, testCase.amount, testCase.nonce, testCase.opts...) - for i := 1; i < len(pks); i++ { - part := multisig2.Spend(uint8(i), pks[i], principal, testCase.to, testCase.amount, testCase.nonce, testCase.opts...) - agg.Add(*part.Part(uint8(i))) - } - rawTx := agg.Raw() - - decodedTx, err := transactionparser.Parse(rawTx) - require.NoError(t, err) - require.Equal(t, testCase.gasPrice, decodedTx.Tx.GetGasPrice()) - require.Equal(t, testCase.to.String(), decodedTx.Tx.GetReceiver().String()) - require.Equal(t, testCase.amount, decodedTx.Tx.GetAmount()) - require.Equal(t, principal.String(), decodedTx.Tx.GetPrincipal().String()) - require.Equal(t, transaction.TypeMultisigSpend, decodedTx.Type) - - }) - } -} - -func TestDrainVault(t *testing.T) { - pubs := make([][]byte, 0, 3) - pks := make([]ed25519.PrivateKey, 0, 3) - for i := 0; i < 3; i++ { - pub, pk, err := ed25519.GenerateKey(rand.New(rand.NewSource(time.Now().UnixNano()))) - require.NoError(t, err) - pubs = append(pubs, pub) - pks = append(pks, pk) - } - principal := multisig2.Address(multisig.TemplateAddress, 3, pubs...) - to := types.GenerateAddress(generatePublicKey()) - vaultAddr := types.GenerateAddress(generatePublicKey()) - agg := vesting.DrainVault( - 0, - pks[0], - principal, - vaultAddr, - to, - 100, - types.Nonce(1)) - for i := 1; i < len(pks); i++ { - part := vesting.DrainVault(uint8(i), pks[i], principal, vaultAddr, to, 100, types.Nonce(1)) - agg.Add(*part.Part(uint8(i))) - } - txRaw := agg.Raw() - - decodedTx, err := transactionparser.Parse(txRaw) - require.NoError(t, err) - require.Equal(t, transaction.TypeDrainVault, decodedTx.Type) - require.Equal(t, vaultAddr.String(), decodedTx.Vault.GetVault().String()) -} - -func TestVaultSpawn(t *testing.T) { - pks := make([]ed25519.PrivateKey, 0, 3) - for i := 0; i < 3; i++ { - _, pk, err := ed25519.GenerateKey(rand.New(rand.NewSource(time.Now().UnixNano()))) - require.NoError(t, err) - pks = append(pks, pk) - } - owner := types.GenerateAddress(generatePublicKey()) - vaultArgs := &vault.SpawnArguments{ - Owner: owner, - InitialUnlockAmount: uint64(1000), - TotalAmount: uint64(1001), - VestingStart: 105120, - VestingEnd: 4 * 105120, - } - vaultAddr := core.ComputePrincipal(vault.TemplateAddress, vaultArgs) - - var agg *multisig2.Aggregator - for i := 0; i < len(pks); i++ { - part := multisig2.Spawn(uint8(i), pks[i], vaultAddr, vault.TemplateAddress, vaultArgs, types.Nonce(0)) - if agg == nil { - agg = part - } else { - agg.Add(*part.Part(uint8(i))) - } - } - rawTx := agg.Raw() - decodedTx, err := transactionparser.Parse(rawTx) - require.NoError(t, err) - - require.Equal(t, vaultArgs.Owner.String(), decodedTx.Vault.GetOwner().String()) - require.Equal(t, vaultArgs.InitialUnlockAmount, decodedTx.Vault.GetInitialUnlockAmount()) - require.Equal(t, vaultArgs.TotalAmount, decodedTx.Vault.GetTotalAmount()) - require.Equal(t, vaultArgs.VestingStart, decodedTx.Vault.GetVestingStart()) - require.Equal(t, vaultArgs.VestingEnd, decodedTx.Vault.GetVestingEnd()) - require.Equal(t, transaction.TypeVaultSpawn, decodedTx.Type) - -} - -func generatePublicKey() []byte { - signer, _ := signing.NewEdSigner() - return signer.PublicKey().Bytes() -} diff --git a/pkg/transactionparser/v0/parser.go b/pkg/transactionparser/v0/parser.go deleted file mode 100644 index 2d6165a..0000000 --- a/pkg/transactionparser/v0/parser.go +++ /dev/null @@ -1,113 +0,0 @@ -package v0 - -import ( - "bytes" - "fmt" - "github.com/spacemeshos/go-spacemesh/codec" - "github.com/spacemeshos/go-spacemesh/common/types" - "github.com/spacemeshos/go-spacemesh/genvm/core" - "github.com/spacemeshos/go-spacemesh/genvm/registry" - "github.com/spacemeshos/go-spacemesh/genvm/templates/multisig" - "github.com/spacemeshos/go-spacemesh/genvm/templates/vault" - "github.com/spacemeshos/go-spacemesh/genvm/templates/vesting" - "github.com/spacemeshos/go-spacemesh/genvm/templates/wallet" - - "github.com/spacemeshos/explorer-backend/pkg/transactionparser/transaction" -) - -const ( - methodSpawn = 0 - methodSend = 16 -) - -// ParseTransaction parses a transaction encoded in version 0. -// possible two types of transaction: -// 1. spawn transaction - `&sdk.TxVersion, &principal, &sdk.MethodSpawn, &wallet.TemplateAddress, &wallet.SpawnPayload` -// 2. spend transaction - `&sdk.TxVersion, &principal, &sdk.MethodSpend, &wallet.SpendPayload. -// every transaction can be multisig also. -func ParseTransaction(rawTx *bytes.Buffer, method uint8, template *core.Address) (*transaction.TransactionData, error) { - reg := registry.New() - wallet.Register(reg) - multisig.Register(reg) - vesting.Register(reg) - vault.Register(reg) - - txData := &transaction.TransactionData{} - switch method { - case methodSpawn: - switch *template { - case wallet.TemplateAddress: - var spawnTx SpawnTransaction - if _, err := codec.DecodeFrom(rawTx, &spawnTx); err != nil { - return nil, err - } - - txData.Tx = &spawnTx - txData.Type = transaction.TypeSpawn - case multisig.TemplateAddress: - var spawnMultisigTx SpawnMultisigTransaction - if _, err := codec.DecodeFrom(rawTx, &spawnMultisigTx); err != nil { - return nil, err - } - txData.Tx = &spawnMultisigTx - txData.Type = transaction.TypeMultisigSpawn - case vault.TemplateAddress: - var spawnVaultTx SpawnVaultTransaction - if _, err := codec.DecodeFrom(rawTx, &spawnVaultTx); err != nil { - return nil, err - } - txData.Tx = &spawnVaultTx - txData.Vault = &spawnVaultTx - txData.Type = transaction.TypeVaultSpawn - case vesting.TemplateAddress: - var spawnMultisigTx SpawnMultisigTransaction - if _, err := codec.DecodeFrom(rawTx, &spawnMultisigTx); err != nil { - return nil, err - } - txData.Tx = &spawnMultisigTx - txData.Type = transaction.TypeVestingSpawn - } - case methodSend: - var spendTx SpendTransaction - if _, err := codec.DecodeFrom(rawTx, &spendTx); err != nil { - return nil, err - } - txData.Tx = &spendTx - txData.Type = transaction.TypeSpend - case vesting.MethodDrainVault: - var drainVaultTx DrainVaultTransaction - if _, err := codec.DecodeFrom(rawTx, &drainVaultTx); err != nil { - return nil, err - } - txData.Tx = &drainVaultTx - txData.Vault = &drainVaultTx - txData.Type = transaction.TypeDrainVault - default: - return nil, fmt.Errorf("%w: unsupported method %d", core.ErrMalformed, method) - } - - // decode signature or signatures - if rawTx.Len() <= types.EdSignatureSize { - var sig core.Signature - if _, err := codec.DecodeFrom(rawTx, &sig); err != nil { - return nil, err - } - txData.Sig = &sig - } else { - var signatures multisig.Signatures - for rawTx.Len() > 0 { - var part multisig.Part - if _, err := codec.DecodeFrom(rawTx, &part); err != nil { - return nil, err - } - signatures = append(signatures, part) - } - txData.Signatures = &signatures - - if txData.Type == transaction.TypeSpend { - txData.Type = transaction.TypeMultisigSpend - } - } - - return txData, nil -} diff --git a/pkg/transactionparser/v0/types.go b/pkg/transactionparser/v0/types.go deleted file mode 100644 index 530c6f6..0000000 --- a/pkg/transactionparser/v0/types.go +++ /dev/null @@ -1,380 +0,0 @@ -package v0 - -import ( - "github.com/spacemeshos/address" - "github.com/spacemeshos/go-scale" - "github.com/spacemeshos/go-spacemesh/genvm/core" - "github.com/spacemeshos/go-spacemesh/genvm/templates/multisig" - "github.com/spacemeshos/go-spacemesh/genvm/templates/wallet" - "github.com/spacemeshos/go-spacemesh/hash" - - "github.com/spacemeshos/explorer-backend/pkg/transactionparser/transaction" -) - -//go:generate scalegen SpawnTransaction SpawnMultisigTransaction SpendTransaction - -// PublicKey is a public key of the transaction. -type PublicKey [32]byte - -// EncodeScale implements scale codec interface. -func (h *PublicKey) EncodeScale(e *scale.Encoder) (int, error) { - return scale.EncodeByteArray(e, h[:]) -} - -// DecodeScale implements scale codec interface. -func (h *PublicKey) DecodeScale(d *scale.Decoder) (int, error) { - return scale.DecodeByteArray(d, h[:]) -} - -// ComputePrincipal address as the last 20 bytes from blake3(scale(template || args)). -func ComputePrincipal(template core.Address, args scale.Encodable) address.Address { - hasher := hash.New() - encoder := scale.NewEncoder(hasher) - _, _ = template.EncodeScale(encoder) - _, _ = args.EncodeScale(encoder) - hs := hasher.Sum(nil) - return address.GenerateAddress(hs[12:]) -} - -// SpawnTransaction initial transaction for wallet. -type SpawnTransaction struct { - Type uint8 - Principal address.Address - Method uint8 - Template address.Address - Payload SpawnPayload -} - -// SpawnPayload provides arguments for spawn transaction. -type SpawnPayload struct { - Nonce core.Nonce - GasPrice uint64 - Arguments SpawnArguments -} - -// SpawnArguments is the arguments of the spawn transaction. -type SpawnArguments struct { - PublicKey PublicKey -} - -// GetType returns type of the transaction. -func (t *SpawnTransaction) GetType() uint8 { - return transaction.TypeSpawn -} - -// GetAmount returns amount of the transaction. Always zero for spawn transaction. -func (t *SpawnTransaction) GetAmount() uint64 { - return 0 -} - -// GetCounter returns counter of the transaction. Always zero for spawn transaction. -func (t *SpawnTransaction) GetCounter() uint64 { - return 0 -} - -// GetReceiver returns receiver address of the transaction. -func (t *SpawnTransaction) GetReceiver() address.Address { - args := SpawnArguments{} - copy(args.PublicKey[:], t.Payload.Arguments.PublicKey[:]) - return ComputePrincipal(wallet.TemplateAddress, &args) -} - -// GetGasPrice returns gas price of the transaction. -func (t *SpawnTransaction) GetGasPrice() uint64 { - return t.Payload.GasPrice -} - -// GetPrincipal returns the principal address who pay for gas for this transaction. -func (t *SpawnTransaction) GetPrincipal() address.Address { - return t.Principal -} - -// GetPublicKeys returns public keys of the transaction. -func (t *SpawnTransaction) GetPublicKeys() [][]byte { - return [][]byte{t.Payload.Arguments.PublicKey[:]} -} - -// SpawnMultisigTransaction initial transaction for multisig wallet. -type SpawnMultisigTransaction struct { - Type uint8 - Principal address.Address - Method uint8 - Template address.Address - Payload SpawnMultisigPayload -} - -// SpawnMultisigPayload payload of the multisig spawn transaction. -type SpawnMultisigPayload struct { - Nonce core.Nonce - GasPrice uint64 - Arguments SpawnMultisigArguments -} - -// SpawnMultisigArguments arguments for multisig spawn transaction. -type SpawnMultisigArguments struct { - Required uint8 - PublicKeys []PublicKey `scale:"max=10"` -} - -// GetType returns type of the transaction. -func (t *SpawnMultisigTransaction) GetType() uint8 { - return transaction.TypeMultisigSpawn -} - -// GetAmount returns amount of the transaction. Always zero for spawn transaction. -func (t *SpawnMultisigTransaction) GetAmount() uint64 { - return 0 -} - -// GetCounter returns counter of the transaction. Always zero for spawn transaction. -func (t *SpawnMultisigTransaction) GetCounter() uint64 { - return 0 -} - -// GetReceiver returns receiver address of the transaction. -func (t *SpawnMultisigTransaction) GetReceiver() address.Address { - args := SpawnMultisigArguments{PublicKeys: make([]PublicKey, len(t.Payload.Arguments.PublicKeys))} - for i := range t.Payload.Arguments.PublicKeys { - copy(args.PublicKeys[i][:], t.Payload.Arguments.PublicKeys[i][:]) - } - return ComputePrincipal(multisig.TemplateAddress, &args) -} - -// GetGasPrice returns gas price of the transaction. -func (t *SpawnMultisigTransaction) GetGasPrice() uint64 { - return t.Payload.GasPrice -} - -// GetPrincipal returns the principal address who pay for gas for this transaction. -func (t *SpawnMultisigTransaction) GetPrincipal() address.Address { - return t.Principal -} - -// GetPublicKeys returns all public keys of the multisig transaction. -func (t *SpawnMultisigTransaction) GetPublicKeys() [][]byte { - result := make([][]byte, 0, len(t.Payload.Arguments.PublicKeys)) - for i := range t.Payload.Arguments.PublicKeys { - result = append(result, t.Payload.Arguments.PublicKeys[i][:]) - } - return result -} - -// SpendTransaction coin transfer transaction. also includes multisig. -type SpendTransaction struct { - Type uint8 - Principal address.Address - Method uint8 - Payload SpendPayload -} - -// SpendArguments arguments of the spend transaction. -type SpendArguments struct { - Destination address.Address - Amount uint64 -} - -// SpendPayload payload of the spend transaction. -type SpendPayload struct { - Nonce core.Nonce - GasPrice uint64 - Arguments SpendArguments -} - -// GetType returns transaction type. -func (t *SpendTransaction) GetType() uint8 { - return transaction.TypeSpend -} - -// GetAmount returns the amount of the transaction. -func (t *SpendTransaction) GetAmount() uint64 { - return t.Payload.Arguments.Amount -} - -// GetCounter returns the counter of the transaction. -func (t *SpendTransaction) GetCounter() uint64 { - return t.Payload.Nonce -} - -// GetReceiver returns receiver address. -func (t *SpendTransaction) GetReceiver() address.Address { - return t.Payload.Arguments.Destination -} - -// GetGasPrice returns gas price of the transaction. -func (t *SpendTransaction) GetGasPrice() uint64 { - return t.Payload.GasPrice -} - -// GetPrincipal return address which spend gas. -func (t *SpendTransaction) GetPrincipal() address.Address { - return t.Principal -} - -// GetPublicKeys returns nil. -func (t *SpendTransaction) GetPublicKeys() [][]byte { - return nil // todo we do not encode publickeys in the transaction -} - -// SpawnVaultTransaction initial transaction for vault. -type SpawnVaultTransaction struct { - Type uint8 - Principal address.Address - Method uint8 - Template address.Address - Payload SpawnVaultPayload -} - -// SpawnVaultPayload provides arguments for spawn vault transaction. -type SpawnVaultPayload struct { - Nonce core.Nonce - GasPrice uint64 - Arguments SpawnVaultArguments -} - -// SpawnVaultArguments is the arguments of the spawn vault transaction. -type SpawnVaultArguments struct { - Owner core.Address - TotalAmount uint64 - InitialUnlockAmount uint64 - VestingStart core.LayerID - VestingEnd core.LayerID -} - -// GetType returns type of the transaction. -func (t *SpawnVaultTransaction) GetType() uint8 { - return transaction.TypeSpawn -} - -// GetAmount returns amount of the transaction. Always zero for spawn transaction. -func (t *SpawnVaultTransaction) GetAmount() uint64 { - return 0 -} - -// GetCounter returns counter of the transaction. Always zero for spawn transaction. -func (t *SpawnVaultTransaction) GetCounter() uint64 { - return 0 -} - -// GetReceiver returns receiver address of the transaction. -func (t *SpawnVaultTransaction) GetReceiver() address.Address { - return address.Address{} -} - -// GetGasPrice returns gas price of the transaction. -func (t *SpawnVaultTransaction) GetGasPrice() uint64 { - return t.Payload.GasPrice -} - -// GetPrincipal returns the principal address who pay for gas for this transaction. -func (t *SpawnVaultTransaction) GetPrincipal() address.Address { - return t.Principal -} - -// GetPublicKeys returns public keys of the transaction. -func (t *SpawnVaultTransaction) GetPublicKeys() [][]byte { - return [][]byte{} -} - -func (t *SpawnVaultTransaction) GetOwner() core.Address { - return t.Payload.Arguments.Owner -} - -func (t *SpawnVaultTransaction) GetTotalAmount() uint64 { - return t.Payload.Arguments.TotalAmount -} - -func (t *SpawnVaultTransaction) GetInitialUnlockAmount() uint64 { - return t.Payload.Arguments.InitialUnlockAmount -} - -func (t *SpawnVaultTransaction) GetVestingStart() core.LayerID { - return t.Payload.Arguments.VestingStart -} - -func (t *SpawnVaultTransaction) GetVestingEnd() core.LayerID { - return t.Payload.Arguments.VestingEnd -} - -func (t *SpawnVaultTransaction) GetVault() core.Address { - return core.Address{} -} - -// DrainVaultTransaction initial transaction for vault. -type DrainVaultTransaction struct { - Type uint8 - Principal address.Address - Method uint8 - Payload DrainVaultPayload -} - -// DrainVaultPayload provides arguments for drain vault transaction. -type DrainVaultPayload struct { - Nonce core.Nonce - GasPrice uint64 - Arguments DrainVaultArguments -} - -// DrainVaultArguments is the arguments of the drain vault transaction. -type DrainVaultArguments struct { - Vault core.Address - SpendArguments -} - -// GetType returns type of the transaction. -func (t *DrainVaultTransaction) GetType() uint8 { - return transaction.TypeDrainVault -} - -// GetAmount returns amount of the transaction. Always zero for spawn transaction. -func (t *DrainVaultTransaction) GetAmount() uint64 { - return t.Payload.Arguments.Amount -} - -// GetCounter returns counter of the transaction. Always zero for spawn transaction. -func (t *DrainVaultTransaction) GetCounter() uint64 { - return t.Payload.Nonce -} - -// GetReceiver returns receiver address of the transaction. -func (t *DrainVaultTransaction) GetReceiver() address.Address { - return t.Payload.Arguments.Destination -} - -// GetGasPrice returns gas price of the transaction. -func (t *DrainVaultTransaction) GetGasPrice() uint64 { - return t.Payload.GasPrice -} - -// GetPrincipal returns the principal address who pay for gas for this transaction. -func (t *DrainVaultTransaction) GetPrincipal() address.Address { - return t.Principal -} - -// GetPublicKeys returns public keys of the transaction. -func (t *DrainVaultTransaction) GetPublicKeys() [][]byte { - return [][]byte{} -} - -func (t *DrainVaultTransaction) GetOwner() core.Address { - return core.Address{} -} - -func (t *DrainVaultTransaction) GetTotalAmount() uint64 { - return 0 -} - -func (t *DrainVaultTransaction) GetInitialUnlockAmount() uint64 { - return 0 -} - -func (t *DrainVaultTransaction) GetVestingStart() core.LayerID { - return core.LayerID(0) -} - -func (t *DrainVaultTransaction) GetVestingEnd() core.LayerID { - return core.LayerID(0) -} - -func (t *DrainVaultTransaction) GetVault() core.Address { - return t.Payload.Arguments.Vault -} diff --git a/pkg/transactionparser/v0/types_scale.go b/pkg/transactionparser/v0/types_scale.go deleted file mode 100644 index 4903226..0000000 --- a/pkg/transactionparser/v0/types_scale.go +++ /dev/null @@ -1,856 +0,0 @@ -// Code generated by github.com/spacemeshos/go-scale/scalegen. DO NOT EDIT. - -// nolint -package v0 - -import ( - "github.com/spacemeshos/go-scale" - "github.com/spacemeshos/go-spacemesh/common/types" -) - -func (t *SpawnTransaction) EncodeScale(enc *scale.Encoder) (total int, err error) { - { - n, err := scale.EncodeCompact8(enc, uint8(t.Type)) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeByteArray(enc, t.Principal[:]) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeCompact8(enc, uint8(t.Method)) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeByteArray(enc, t.Template[:]) - if err != nil { - return total, err - } - total += n - } - { - n, err := t.Payload.EncodeScale(enc) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpawnTransaction) DecodeScale(dec *scale.Decoder) (total int, err error) { - { - field, n, err := scale.DecodeCompact8(dec) - if err != nil { - return total, err - } - total += n - t.Type = uint8(field) - } - { - n, err := scale.DecodeByteArray(dec, t.Principal[:]) - if err != nil { - return total, err - } - total += n - } - { - field, n, err := scale.DecodeCompact8(dec) - if err != nil { - return total, err - } - total += n - t.Method = uint8(field) - } - { - n, err := scale.DecodeByteArray(dec, t.Template[:]) - if err != nil { - return total, err - } - total += n - } - { - n, err := t.Payload.DecodeScale(dec) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpawnPayload) EncodeScale(enc *scale.Encoder) (total int, err error) { - { - n, err := scale.EncodeCompact64(enc, uint64(t.Nonce)) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeCompact64(enc, uint64(t.GasPrice)) - if err != nil { - return total, err - } - total += n - } - { - n, err := t.Arguments.EncodeScale(enc) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpawnPayload) DecodeScale(dec *scale.Decoder) (total int, err error) { - { - field, n, err := scale.DecodeCompact64(dec) - if err != nil { - return total, err - } - total += n - t.Nonce = uint64(field) - } - { - field, n, err := scale.DecodeCompact64(dec) - if err != nil { - return total, err - } - total += n - t.GasPrice = uint64(field) - } - { - n, err := t.Arguments.DecodeScale(dec) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpawnArguments) EncodeScale(enc *scale.Encoder) (total int, err error) { - { - n, err := scale.EncodeByteArray(enc, t.PublicKey[:]) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpawnArguments) DecodeScale(dec *scale.Decoder) (total int, err error) { - { - n, err := scale.DecodeByteArray(dec, t.PublicKey[:]) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpawnMultisigTransaction) EncodeScale(enc *scale.Encoder) (total int, err error) { - { - n, err := scale.EncodeCompact8(enc, uint8(t.Type)) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeByteArray(enc, t.Principal[:]) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeCompact8(enc, uint8(t.Method)) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeByteArray(enc, t.Template[:]) - if err != nil { - return total, err - } - total += n - } - { - n, err := t.Payload.EncodeScale(enc) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpawnMultisigTransaction) DecodeScale(dec *scale.Decoder) (total int, err error) { - { - field, n, err := scale.DecodeCompact8(dec) - if err != nil { - return total, err - } - total += n - t.Type = uint8(field) - } - { - n, err := scale.DecodeByteArray(dec, t.Principal[:]) - if err != nil { - return total, err - } - total += n - } - { - field, n, err := scale.DecodeCompact8(dec) - if err != nil { - return total, err - } - total += n - t.Method = uint8(field) - } - { - n, err := scale.DecodeByteArray(dec, t.Template[:]) - if err != nil { - return total, err - } - total += n - } - { - n, err := t.Payload.DecodeScale(dec) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpawnMultisigPayload) EncodeScale(enc *scale.Encoder) (total int, err error) { - { - n, err := scale.EncodeCompact64(enc, uint64(t.Nonce)) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeCompact64(enc, uint64(t.GasPrice)) - if err != nil { - return total, err - } - total += n - } - { - n, err := t.Arguments.EncodeScale(enc) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpawnMultisigPayload) DecodeScale(dec *scale.Decoder) (total int, err error) { - { - field, n, err := scale.DecodeCompact64(dec) - if err != nil { - return total, err - } - total += n - t.Nonce = uint64(field) - } - { - field, n, err := scale.DecodeCompact64(dec) - if err != nil { - return total, err - } - total += n - t.GasPrice = uint64(field) - } - { - n, err := t.Arguments.DecodeScale(dec) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpawnMultisigArguments) EncodeScale(enc *scale.Encoder) (total int, err error) { - { - n, err := scale.EncodeCompact8(enc, uint8(t.Required)) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeStructSliceWithLimit(enc, t.PublicKeys, 10) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpawnMultisigArguments) DecodeScale(dec *scale.Decoder) (total int, err error) { - { - field, n, err := scale.DecodeCompact8(dec) - if err != nil { - return total, err - } - total += n - t.Required = uint8(field) - } - { - field, n, err := scale.DecodeStructSliceWithLimit[PublicKey](dec, 10) - if err != nil { - return total, err - } - total += n - t.PublicKeys = field - } - return total, nil -} - -func (t *SpendTransaction) EncodeScale(enc *scale.Encoder) (total int, err error) { - { - n, err := scale.EncodeCompact8(enc, uint8(t.Type)) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeByteArray(enc, t.Principal[:]) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeCompact8(enc, uint8(t.Method)) - if err != nil { - return total, err - } - total += n - } - { - n, err := t.Payload.EncodeScale(enc) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpendTransaction) DecodeScale(dec *scale.Decoder) (total int, err error) { - { - field, n, err := scale.DecodeCompact8(dec) - if err != nil { - return total, err - } - total += n - t.Type = uint8(field) - } - { - n, err := scale.DecodeByteArray(dec, t.Principal[:]) - if err != nil { - return total, err - } - total += n - } - { - field, n, err := scale.DecodeCompact8(dec) - if err != nil { - return total, err - } - total += n - t.Method = uint8(field) - } - { - n, err := t.Payload.DecodeScale(dec) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpendArguments) EncodeScale(enc *scale.Encoder) (total int, err error) { - { - n, err := scale.EncodeByteArray(enc, t.Destination[:]) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeCompact64(enc, uint64(t.Amount)) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpendArguments) DecodeScale(dec *scale.Decoder) (total int, err error) { - { - n, err := scale.DecodeByteArray(dec, t.Destination[:]) - if err != nil { - return total, err - } - total += n - } - { - field, n, err := scale.DecodeCompact64(dec) - if err != nil { - return total, err - } - total += n - t.Amount = uint64(field) - } - return total, nil -} - -func (t *SpendPayload) EncodeScale(enc *scale.Encoder) (total int, err error) { - { - n, err := scale.EncodeCompact64(enc, uint64(t.Nonce)) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeCompact64(enc, uint64(t.GasPrice)) - if err != nil { - return total, err - } - total += n - } - { - n, err := t.Arguments.EncodeScale(enc) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpendPayload) DecodeScale(dec *scale.Decoder) (total int, err error) { - { - field, n, err := scale.DecodeCompact64(dec) - if err != nil { - return total, err - } - total += n - t.Nonce = uint64(field) - } - { - field, n, err := scale.DecodeCompact64(dec) - if err != nil { - return total, err - } - total += n - t.GasPrice = uint64(field) - } - { - n, err := t.Arguments.DecodeScale(dec) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpawnVaultTransaction) EncodeScale(enc *scale.Encoder) (total int, err error) { - { - n, err := scale.EncodeCompact8(enc, uint8(t.Type)) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeByteArray(enc, t.Principal[:]) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeCompact8(enc, uint8(t.Method)) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeByteArray(enc, t.Template[:]) - if err != nil { - return total, err - } - total += n - } - { - n, err := t.Payload.EncodeScale(enc) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpawnVaultTransaction) DecodeScale(dec *scale.Decoder) (total int, err error) { - { - field, n, err := scale.DecodeCompact8(dec) - if err != nil { - return total, err - } - total += n - t.Type = uint8(field) - } - { - n, err := scale.DecodeByteArray(dec, t.Principal[:]) - if err != nil { - return total, err - } - total += n - } - { - field, n, err := scale.DecodeCompact8(dec) - if err != nil { - return total, err - } - total += n - t.Method = uint8(field) - } - { - n, err := scale.DecodeByteArray(dec, t.Template[:]) - if err != nil { - return total, err - } - total += n - } - { - n, err := t.Payload.DecodeScale(dec) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpawnVaultPayload) EncodeScale(enc *scale.Encoder) (total int, err error) { - { - n, err := scale.EncodeCompact64(enc, uint64(t.Nonce)) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeCompact64(enc, uint64(t.GasPrice)) - if err != nil { - return total, err - } - total += n - } - { - n, err := t.Arguments.EncodeScale(enc) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpawnVaultPayload) DecodeScale(dec *scale.Decoder) (total int, err error) { - { - field, n, err := scale.DecodeCompact64(dec) - if err != nil { - return total, err - } - total += n - t.Nonce = uint64(field) - } - { - field, n, err := scale.DecodeCompact64(dec) - if err != nil { - return total, err - } - total += n - t.GasPrice = uint64(field) - } - { - n, err := t.Arguments.DecodeScale(dec) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpawnVaultArguments) EncodeScale(enc *scale.Encoder) (total int, err error) { - { - n, err := scale.EncodeByteArray(enc, t.Owner[:]) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeCompact64(enc, uint64(t.TotalAmount)) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeCompact64(enc, uint64(t.InitialUnlockAmount)) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeCompact32(enc, uint32(t.VestingStart)) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeCompact32(enc, uint32(t.VestingEnd)) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *SpawnVaultArguments) DecodeScale(dec *scale.Decoder) (total int, err error) { - { - n, err := scale.DecodeByteArray(dec, t.Owner[:]) - if err != nil { - return total, err - } - total += n - } - { - field, n, err := scale.DecodeCompact64(dec) - if err != nil { - return total, err - } - total += n - t.TotalAmount = uint64(field) - } - { - field, n, err := scale.DecodeCompact64(dec) - if err != nil { - return total, err - } - total += n - t.InitialUnlockAmount = uint64(field) - } - { - field, n, err := scale.DecodeCompact32(dec) - if err != nil { - return total, err - } - total += n - t.VestingStart = types.LayerID(field) - } - { - field, n, err := scale.DecodeCompact32(dec) - if err != nil { - return total, err - } - total += n - t.VestingEnd = types.LayerID(field) - } - return total, nil -} - -func (t *DrainVaultTransaction) EncodeScale(enc *scale.Encoder) (total int, err error) { - { - n, err := scale.EncodeCompact8(enc, uint8(t.Type)) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeByteArray(enc, t.Principal[:]) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeCompact8(enc, uint8(t.Method)) - if err != nil { - return total, err - } - total += n - } - { - n, err := t.Payload.EncodeScale(enc) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *DrainVaultTransaction) DecodeScale(dec *scale.Decoder) (total int, err error) { - { - field, n, err := scale.DecodeCompact8(dec) - if err != nil { - return total, err - } - total += n - t.Type = uint8(field) - } - { - n, err := scale.DecodeByteArray(dec, t.Principal[:]) - if err != nil { - return total, err - } - total += n - } - { - field, n, err := scale.DecodeCompact8(dec) - if err != nil { - return total, err - } - total += n - t.Method = uint8(field) - } - { - n, err := t.Payload.DecodeScale(dec) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *DrainVaultPayload) EncodeScale(enc *scale.Encoder) (total int, err error) { - { - n, err := scale.EncodeCompact64(enc, uint64(t.Nonce)) - if err != nil { - return total, err - } - total += n - } - { - n, err := scale.EncodeCompact64(enc, uint64(t.GasPrice)) - if err != nil { - return total, err - } - total += n - } - { - n, err := t.Arguments.EncodeScale(enc) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *DrainVaultPayload) DecodeScale(dec *scale.Decoder) (total int, err error) { - { - field, n, err := scale.DecodeCompact64(dec) - if err != nil { - return total, err - } - total += n - t.Nonce = uint64(field) - } - { - field, n, err := scale.DecodeCompact64(dec) - if err != nil { - return total, err - } - total += n - t.GasPrice = uint64(field) - } - { - n, err := t.Arguments.DecodeScale(dec) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *DrainVaultArguments) EncodeScale(enc *scale.Encoder) (total int, err error) { - { - n, err := scale.EncodeByteArray(enc, t.Vault[:]) - if err != nil { - return total, err - } - total += n - } - { - n, err := t.SpendArguments.EncodeScale(enc) - if err != nil { - return total, err - } - total += n - } - return total, nil -} - -func (t *DrainVaultArguments) DecodeScale(dec *scale.Decoder) (total int, err error) { - { - n, err := scale.DecodeByteArray(dec, t.Vault[:]) - if err != nil { - return total, err - } - total += n - } - { - n, err := t.SpendArguments.DecodeScale(dec) - if err != nil { - return total, err - } - total += n - } - return total, nil -} diff --git a/storage/account.go b/storage/account.go deleted file mode 100644 index de6d2d1..0000000 --- a/storage/account.go +++ /dev/null @@ -1,225 +0,0 @@ -package storage - -import ( - "context" - "errors" - "time" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/go-spacemesh/log" - - "github.com/spacemeshos/explorer-backend/model" - "github.com/spacemeshos/explorer-backend/utils" -) - -func (s *Storage) InitAccountsStorage(ctx context.Context) error { - if _, err := s.db.Collection("accounts").Indexes().CreateOne(ctx, mongo.IndexModel{ - Keys: bson.D{{Key: "address", Value: 1}}, - Options: options.Index().SetName("addressIndex").SetUnique(true)}); err != nil { - return err - } - - if _, err := s.db.Collection("accounts").Indexes().CreateOne(ctx, mongo.IndexModel{ - Keys: bson.D{{Key: "created", Value: 1}}, - Options: options.Index().SetName("createIndex").SetUnique(false)}); err != nil { - return err - } - - if _, err := s.db.Collection("accounts").Indexes().CreateOne(ctx, mongo.IndexModel{ - Keys: bson.D{{Key: "layer", Value: -1}}, - Options: options.Index().SetName("modifiedIndex").SetUnique(false)}); err != nil { - return err - } - - return nil -} - -func (s *Storage) GetAccount(parent context.Context, query *bson.D) (*model.Account, error) { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - cursor, err := s.db.Collection("accounts").Find(ctx, query) - if err != nil { - log.Info("GetAccount: %v", err) - return nil, err - } - if !cursor.Next(ctx) { - log.Info("GetAccount: Empty result", err) - return nil, errors.New("Empty result") - } - doc := cursor.Current - account := &model.Account{ - Address: utils.GetAsString(doc.Lookup("address")), - Balance: utils.GetAsUInt64(doc.Lookup("balance")), - Counter: utils.GetAsUInt64(doc.Lookup("counter")), - } - return account, nil -} - -func (s *Storage) GetAccountsCount(parent context.Context, query *bson.D, opts ...*options.CountOptions) int64 { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - count, err := s.db.Collection("accounts").CountDocuments(ctx, query, opts...) - if err != nil { - log.Info("GetAccountsCount: %v", err) - return 0 - } - return count -} - -func (s *Storage) GetAccounts(parent context.Context, query *bson.D, opts ...*options.FindOptions) ([]bson.D, error) { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - cursor, err := s.db.Collection("accounts").Find(ctx, query, opts...) - if err != nil { - log.Info("GetAccounts: %v", err) - return nil, err - } - var docs interface{} = []bson.D{} - err = cursor.All(ctx, &docs) - if err != nil { - log.Info("GetAccounts: %v", err) - return nil, err - } - if len(docs.([]bson.D)) == 0 { - log.Info("GetAccounts: Empty result") - return nil, nil - } - return docs.([]bson.D), nil -} - -func (s *Storage) AddAccount(parent context.Context, layer uint32, address string, balance uint64) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - - acc := bson.D{ - {Key: "$set", - Value: bson.D{ - {Key: "address", Value: address}, - {Key: "layer", Value: layer}, - {Key: "balance", Value: balance}, - {Key: "counter", Value: uint64(0)}, - }, - }, - {Key: "$setOnInsert", - Value: bson.D{ - {Key: "created", Value: layer}, - }, - }, - } - - opts := options.Update().SetUpsert(true) - _, err := s.db.Collection("accounts").UpdateOne(ctx, bson.D{{Key: "address", Value: address}}, acc, opts) - if err != nil { - log.Info("AddAccount: %v", err) - } - return nil -} - -func (s *Storage) AddAccountQuery(layer uint32, address string, balance uint64) *mongo.UpdateOneModel { - filter := bson.D{{Key: "address", Value: address}} - acc := bson.D{ - {Key: "$set", - Value: bson.D{ - {Key: "address", Value: address}, - {Key: "layer", Value: layer}, - {Key: "balance", Value: balance}, - {Key: "counter", Value: uint64(0)}, - }, - }, - {Key: "$setOnInsert", - Value: bson.D{ - {Key: "created", Value: layer}, - }, - }, - } - - accountModel := mongo.NewUpdateOneModel() - accountModel.SetFilter(filter) - accountModel.SetUpdate(acc) - accountModel.SetUpsert(true) - - return accountModel -} - -func (s *Storage) SaveAccount(parent context.Context, layer uint32, in *model.Account) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - _, err := s.db.Collection("accounts").UpdateOne(ctx, bson.D{{Key: "address", Value: in.Address}}, bson.D{ - {Key: "$set", - Value: bson.D{ - {Key: "address", Value: in.Address}, - {Key: "layer", Value: layer}, - {Key: "balance", Value: in.Balance}, - {Key: "counter", Value: in.Counter}, - }}, - {Key: "$setOnInsert", - Value: bson.D{ - {Key: "created", Value: layer}, - }, - }, - }, options.Update().SetUpsert(true)) - if err != nil { - log.Info("SaveAccount: %v", err) - } - return nil -} - -func (s *Storage) UpdateAccount(parent context.Context, address string, balance uint64, counter uint64) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - _, err := s.db.Collection("accounts").UpdateOne(ctx, bson.D{{Key: "address", Value: address}}, bson.D{ - {Key: "$set", Value: bson.D{ - {Key: "balance", Value: balance}, - {Key: "counter", Value: counter}, - }}, - }, options.Update().SetUpsert(true)) - if err != nil { - log.Info("UpdateAccount: %v", err) - } - return nil -} - -func (s *Storage) AddAccountSent(parent context.Context, layer uint32, address string, amount uint64, fee uint64) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - _, err := s.db.Collection("accounts").UpdateOne(ctx, bson.D{{Key: "address", Value: address}}, bson.D{ - {Key: "$set", Value: bson.D{ - {Key: "layer", Value: layer}, - }}, - }) - if err != nil { - log.Info("AddAccountSent: update account touch error %v", err) - } - return nil -} - -func (s *Storage) AddAccountReceived(parent context.Context, layer uint32, address string, amount uint64) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - _, err := s.db.Collection("accounts").UpdateOne(ctx, bson.D{{Key: "address", Value: address}}, bson.D{ - {Key: "$set", Value: bson.D{ - {Key: "layer", Value: layer}, - }}, - }) - if err != nil { - log.Info("AddAccountReceived: update account touch error %v", err) - } - return nil -} - -func (s *Storage) AddAccountReward(parent context.Context, layer uint32, address string, reward uint64, fee uint64) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - _, err := s.db.Collection("accounts").UpdateOne(ctx, bson.D{{Key: "address", Value: address}}, bson.D{ - {Key: "$set", Value: bson.D{ - {Key: "layer", Value: layer}, - }}, - }) - if err != nil { - log.Info("AddAccountReward: update account touch error %v", err) - } - return nil -} diff --git a/storage/atx.go b/storage/atx.go deleted file mode 100644 index eccb36d..0000000 --- a/storage/atx.go +++ /dev/null @@ -1,199 +0,0 @@ -package storage - -import ( - "context" - "errors" - "fmt" - "time" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/go-spacemesh/log" - - "github.com/spacemeshos/explorer-backend/model" - "github.com/spacemeshos/explorer-backend/utils" -) - -func (s *Storage) InitActivationsStorage(ctx context.Context) error { - models := []mongo.IndexModel{ - {Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index().SetName("idIndex").SetUnique(true)}, - {Keys: bson.D{{Key: "layer", Value: 1}}, Options: options.Index().SetName("layerIndex").SetUnique(false)}, - {Keys: bson.D{{Key: "smesher", Value: 1}}, Options: options.Index().SetName("smesherIndex").SetUnique(false)}, - {Keys: bson.D{{Key: "coinbase", Value: 1}}, Options: options.Index().SetName("coinbaseIndex").SetUnique(false)}, - {Keys: bson.D{{Key: "targetEpoch", Value: 1}}, Options: options.Index().SetName("targetEpochIndex").SetUnique(false)}, - } - _, err := s.db.Collection("activations").Indexes().CreateMany(ctx, models, options.CreateIndexes().SetMaxTime(20*time.Second)) - return err -} - -func (s *Storage) GetActivation(parent context.Context, query *bson.D) (*model.Activation, error) { - ctx, cancel := context.WithTimeout(parent, 5*time.Minute) - defer cancel() - cursor, err := s.db.Collection("activations").Find(ctx, query) - if err != nil { - log.Info("GetActivation: %v", err) - return nil, err - } - if !cursor.Next(ctx) { - log.Info("GetActivation: Empty result") - return nil, errors.New("empty result") - } - doc := cursor.Current - account := &model.Activation{ - Id: utils.GetAsString(doc.Lookup("id")), - SmesherId: utils.GetAsString(doc.Lookup("smesher")), - Coinbase: utils.GetAsString(doc.Lookup("coinbase")), - PrevAtx: utils.GetAsString(doc.Lookup("prevAtx")), - NumUnits: utils.GetAsUInt32(doc.Lookup("numunits")), - CommitmentSize: utils.GetAsUInt64(doc.Lookup("commitmentSize")), - PublishEpoch: utils.GetAsUInt32(doc.Lookup("publishEpoch")), - TargetEpoch: utils.GetAsUInt32(doc.Lookup("targetEpoch")), - Received: utils.GetAsInt64(doc.Lookup("received")), - TickCount: utils.GetAsUInt64(doc.Lookup("tickCount")), - Weight: utils.GetAsUInt64(doc.Lookup("weight")), - EffectiveNumUnits: utils.GetAsUInt32(doc.Lookup("effectiveNumUnits")), - } - return account, nil -} - -func (s *Storage) GetActivationsCount(parent context.Context, query *bson.D, opts ...*options.CountOptions) int64 { - ctx, cancel := context.WithTimeout(parent, 5*time.Minute) - defer cancel() - count, err := s.db.Collection("activations").CountDocuments(ctx, query, opts...) - if err != nil { - log.Info("GetActivationsCount: %v", err) - return 0 - } - return count -} - -func (s *Storage) GetActivations(parent context.Context, query *bson.D, opts ...*options.FindOptions) ([]bson.D, error) { - ctx, cancel := context.WithTimeout(parent, 5*time.Minute) - defer cancel() - cursor, err := s.db.Collection("activations").Find(ctx, query, opts...) - if err != nil { - log.Info("GetActivations: %v", err) - return nil, err - } - var docs interface{} = []bson.D{} - err = cursor.All(ctx, &docs) - if err != nil { - log.Info("GetActivations: %v", err) - return nil, err - } - if len(docs.([]bson.D)) == 0 { - log.Info("GetActivations: Empty result") - return nil, nil - } - return docs.([]bson.D), nil -} - -func (s *Storage) SaveActivation(parent context.Context, in *model.Activation) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Minute) - defer cancel() - _, err := s.db.Collection("activations").UpdateOne(ctx, bson.D{{Key: "id", Value: in.Id}}, bson.D{{ - Key: "$set", - Value: bson.D{ - {Key: "id", Value: in.Id}, - {Key: "smesher", Value: in.SmesherId}, - {Key: "coinbase", Value: in.Coinbase}, - {Key: "prevAtx", Value: in.PrevAtx}, - {Key: "numunits", Value: in.NumUnits}, - {Key: "commitmentSize", Value: int64(in.NumUnits) * int64(s.postUnitSize)}, - {Key: "received", Value: in.Received}, - {Key: "publishEpoch", Value: in.PublishEpoch}, - {Key: "targetEpoch", Value: in.TargetEpoch}, - {Key: "tickCount", Value: in.TickCount}, - {Key: "weight", Value: in.Weight}, - {Key: "effectiveNumUnits", Value: in.EffectiveNumUnits}, - }, - }}, options.Update().SetUpsert(true)) - if err != nil { - log.Info("SaveActivation: %v", err) - } - return err -} - -func (s *Storage) SaveOrUpdateActivation(parent context.Context, atx *model.Activation) error { - filter := bson.D{{Key: "id", Value: atx.Id}} - update := bson.D{ - {Key: "$set", Value: bson.D{ - {Key: "id", Value: atx.Id}, - {Key: "smesher", Value: atx.SmesherId}, - {Key: "coinbase", Value: atx.Coinbase}, - {Key: "prevAtx", Value: atx.PrevAtx}, - {Key: "numunits", Value: atx.NumUnits}, - {Key: "commitmentSize", Value: int64(atx.NumUnits) * int64(s.postUnitSize)}, - {Key: "received", Value: atx.Received}, - {Key: "publishEpoch", Value: atx.PublishEpoch}, - {Key: "targetEpoch", Value: atx.TargetEpoch}, - {Key: "tickCount", Value: atx.TickCount}, - {Key: "weight", Value: atx.Weight}, - {Key: "effectiveNumUnits", Value: atx.EffectiveNumUnits}, - }}, - } - - _, err := s.db.Collection("activations").UpdateOne(parent, filter, update, options.Update().SetUpsert(true)) - if err != nil { - log.Info("SaveOrUpdateActivation: %v", err) - return err - } - - return nil -} - -func (s *Storage) SaveOrUpdateActivations(parent context.Context, atxs []*model.Activation) error { - var updateOps []mongo.WriteModel - - for _, atx := range atxs { - filter := bson.D{{Key: "id", Value: atx.Id}} - update := bson.D{ - {Key: "$set", Value: bson.D{ - {Key: "id", Value: atx.Id}, - {Key: "smesher", Value: atx.SmesherId}, - {Key: "coinbase", Value: atx.Coinbase}, - {Key: "prevAtx", Value: atx.PrevAtx}, - {Key: "numunits", Value: atx.NumUnits}, - {Key: "commitmentSize", Value: int64(atx.NumUnits) * int64(s.postUnitSize)}, - {Key: "received", Value: atx.Received}, - {Key: "publishEpoch", Value: atx.PublishEpoch}, - {Key: "targetEpoch", Value: atx.TargetEpoch}, - {Key: "tickCount", Value: atx.TickCount}, - {Key: "weight", Value: atx.Weight}, - {Key: "effectiveNumUnits", Value: atx.EffectiveNumUnits}, - }}, - } - - updateModel := mongo.NewUpdateOneModel() - updateModel.Filter = filter - updateModel.Update = update - updateModel.SetUpsert(true) - - updateOps = append(updateOps, updateModel) - } - - if len(updateOps) > 0 { - _, err := s.db.Collection("activations").BulkWrite(context.TODO(), updateOps) - if err != nil { - log.Err(fmt.Errorf("SaveOrUpdateActivations: error atxs write %v", err)) - } - } - - return nil -} - -func (s *Storage) GetLastActivationReceived() int64 { - cursor, err := s.db.Collection("activations").Find(context.Background(), bson.D{}, options.Find().SetSort(bson.D{{Key: "received", Value: -1}}).SetLimit(1)) - if err != nil { - log.Info("GetLastActivationReceived: %v", err) - return 0 - } - if !cursor.Next(context.Background()) { - log.Info("GetLastActivationReceived: Empty result", err) - return 0 - } - doc := cursor.Current - return utils.GetAsInt64(doc.Lookup("received")) -} diff --git a/storage/block.go b/storage/block.go deleted file mode 100644 index cec2574..0000000 --- a/storage/block.go +++ /dev/null @@ -1,122 +0,0 @@ -package storage - -import ( - "context" - "errors" - "time" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/go-spacemesh/log" - - "github.com/spacemeshos/explorer-backend/model" - "github.com/spacemeshos/explorer-backend/utils" -) - -func (s *Storage) InitBlocksStorage(ctx context.Context) error { - _, err := s.db.Collection("blocks").Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index().SetName("idIndex").SetUnique(true)}) - return err -} - -func (s *Storage) GetBlock(parent context.Context, query *bson.D) (*model.Block, error) { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - cursor, err := s.db.Collection("blocks").Find(ctx, query) - if err != nil { - log.Info("GetBlock: %v", err) - return nil, err - } - if !cursor.Next(ctx) { - log.Info("GetBlock: Empty result", err) - return nil, errors.New("Empty result") - } - doc := cursor.Current - account := &model.Block{ - Id: utils.GetAsString(doc.Lookup("id")), - Layer: utils.GetAsUInt32(doc.Lookup("layer")), - Epoch: utils.GetAsUInt32(doc.Lookup("epoch")), - Start: utils.GetAsUInt32(doc.Lookup("start")), - End: utils.GetAsUInt32(doc.Lookup("end")), - TxsNumber: utils.GetAsUInt32(doc.Lookup("txsnumber")), - TxsValue: utils.GetAsUInt64(doc.Lookup("txsvalue")), - } - return account, nil -} - -func (s *Storage) GetBlocksCount(parent context.Context, query *bson.D, opts ...*options.CountOptions) int64 { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - count, err := s.db.Collection("blocks").CountDocuments(ctx, query, opts...) - if err != nil { - log.Info("GetBlocksCount: %v", err) - return 0 - } - return count -} - -func (s *Storage) GetBlocks(parent context.Context, query *bson.D, opts ...*options.FindOptions) ([]bson.D, error) { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - cursor, err := s.db.Collection("blocks").Find(ctx, query, opts...) - if err != nil { - log.Info("GetBlocks: %v", err) - return nil, err - } - var docs interface{} = []bson.D{} - err = cursor.All(ctx, &docs) - if err != nil { - log.Info("GetBlocks: %v", err) - return nil, err - } - if len(docs.([]bson.D)) == 0 { - log.Info("GetBlocks: Empty result") - return nil, nil - } - return docs.([]bson.D), nil -} - -func (s *Storage) SaveBlock(parent context.Context, in *model.Block) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - _, err := s.db.Collection("blocks").UpdateOne(ctx, bson.D{{Key: "id", Value: in.Id}}, bson.D{{ - Key: "$set", - Value: bson.D{ - {Key: "id", Value: in.Id}, - {Key: "layer", Value: in.Layer}, - {Key: "epoch", Value: in.Epoch}, - {Key: "start", Value: in.Start}, - {Key: "end", Value: in.End}, - {Key: "txsnumber", Value: in.TxsNumber}, - {Key: "txsvalue", Value: in.TxsValue}, - }, - }}, options.Update().SetUpsert(true)) - if err != nil { - log.Info("SaveBlock: %v", err) - } - return err -} - -func (s *Storage) SaveOrUpdateBlocks(parent context.Context, in []*model.Block) error { - ctx, cancel := context.WithTimeout(parent, 10*time.Second) - defer cancel() - for _, block := range in { - _, err := s.db.Collection("blocks").UpdateOne(ctx, bson.D{{Key: "id", Value: block.Id}}, bson.D{ - {Key: "$set", Value: bson.D{ - {Key: "id", Value: block.Id}, - {Key: "layer", Value: block.Layer}, - {Key: "epoch", Value: block.Epoch}, - {Key: "start", Value: block.Start}, - {Key: "end", Value: block.End}, - {Key: "txsnumber", Value: block.TxsNumber}, - {Key: "txsvalue", Value: block.TxsValue}, - }}, - }, options.Update().SetUpsert(true)) - if err != nil { - log.Info("SaveOrUpdateBlocks: %v", err) - return err - } - } - return nil -} diff --git a/storage/epoch.go b/storage/epoch.go deleted file mode 100644 index eece7ca..0000000 --- a/storage/epoch.go +++ /dev/null @@ -1,419 +0,0 @@ -package storage - -import ( - "context" - "errors" - "fmt" - "go.mongodb.org/mongo-driver/bson/primitive" - "math" - "time" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/go-spacemesh/log" - - "github.com/spacemeshos/explorer-backend/model" - "github.com/spacemeshos/explorer-backend/utils" -) - -func (s *Storage) InitEpochsStorage(ctx context.Context) error { - _, err := s.db.Collection("epochs").Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "number", Value: 1}}, Options: options.Index().SetName("numberIndex").SetUnique(true)}) - return err -} - -func (s *Storage) GetEpochByNumber(parent context.Context, epochNumber int32) (*model.Epoch, error) { - return s.GetEpoch(parent, &bson.D{{Key: "number", Value: epochNumber}}) -} - -func (s *Storage) GetEpoch(parent context.Context, query *bson.D) (*model.Epoch, error) { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - cursor, err := s.db.Collection("epochs").Find(ctx, query) - if err != nil { - log.Info("GetEpoch: %v", err) - return nil, err - } - if !cursor.Next(ctx) { - log.Info("GetEpoch: Empty result", err) - return nil, errors.New("Empty result") - } - doc := cursor.Current - epoch := &model.Epoch{ - Number: utils.GetAsInt32(doc.Lookup("number")), - Start: utils.GetAsUInt32(doc.Lookup("start")), - End: utils.GetAsUInt32(doc.Lookup("end")), - LayerStart: utils.GetAsUInt32(doc.Lookup("layerstart")), - LayerEnd: utils.GetAsUInt32(doc.Lookup("layerend")), - Layers: utils.GetAsUInt32(doc.Lookup("layers")), - } - stats := doc.Lookup("stats").Document() - current := stats.Lookup("current").Document() - epoch.Stats.Current.Capacity = utils.GetAsInt64(current.Lookup("capacity")) - epoch.Stats.Current.Decentral = utils.GetAsInt64(current.Lookup("decentral")) - epoch.Stats.Current.Smeshers = utils.GetAsInt64(current.Lookup("smeshers")) - epoch.Stats.Current.Transactions = utils.GetAsInt64(current.Lookup("transactions")) - epoch.Stats.Current.Accounts = utils.GetAsInt64(current.Lookup("accounts")) - epoch.Stats.Current.Circulation = utils.GetAsInt64(current.Lookup("circulation")) - epoch.Stats.Current.Rewards = utils.GetAsInt64(current.Lookup("rewards")) - epoch.Stats.Current.RewardsNumber = utils.GetAsInt64(current.Lookup("rewardsnumber")) - epoch.Stats.Current.Security = utils.GetAsInt64(current.Lookup("security")) - epoch.Stats.Current.TxsAmount = utils.GetAsInt64(current.Lookup("txsamount")) - cumulative := stats.Lookup("cumulative").Document() - epoch.Stats.Cumulative.Capacity = utils.GetAsInt64(cumulative.Lookup("capacity")) - epoch.Stats.Cumulative.Decentral = utils.GetAsInt64(cumulative.Lookup("decentral")) - epoch.Stats.Cumulative.Smeshers = utils.GetAsInt64(cumulative.Lookup("smeshers")) - epoch.Stats.Cumulative.Transactions = utils.GetAsInt64(cumulative.Lookup("transactions")) - epoch.Stats.Cumulative.Accounts = utils.GetAsInt64(cumulative.Lookup("accounts")) - epoch.Stats.Cumulative.Circulation = utils.GetAsInt64(cumulative.Lookup("circulation")) - epoch.Stats.Cumulative.Rewards = utils.GetAsInt64(cumulative.Lookup("rewards")) - epoch.Stats.Cumulative.RewardsNumber = utils.GetAsInt64(cumulative.Lookup("rewardsnumber")) - epoch.Stats.Cumulative.Security = utils.GetAsInt64(cumulative.Lookup("security")) - epoch.Stats.Cumulative.TxsAmount = utils.GetAsInt64(cumulative.Lookup("txsamount")) - return epoch, nil -} - -func (s *Storage) GetEpochsData(parent context.Context, query *bson.D, opts ...*options.FindOptions) ([]*model.Epoch, error) { - skip := int64(0) - if opts[0].Skip != nil { - skip = *opts[0].Skip - } - - pipeline := bson.A{ - bson.D{{Key: "$sort", Value: bson.D{{Key: "number", Value: -1}}}}, - bson.D{{Key: "$skip", Value: skip}}, - bson.D{{Key: "$limit", Value: *opts[0].Limit}}, - bson.D{ - {Key: "$lookup", - Value: bson.D{ - {Key: "from", Value: "rewards"}, - {Key: "let", - Value: bson.D{ - {Key: "start", Value: "$layerstart"}, - {Key: "end", Value: "$layerend"}, - }, - }, - {Key: "pipeline", - Value: bson.A{ - bson.D{ - {Key: "$match", - Value: bson.D{ - {Key: "$expr", - Value: bson.D{ - {Key: "$and", - Value: bson.A{ - bson.D{ - {Key: "$gte", - Value: bson.A{ - "$layer", - "$$start", - }, - }, - }, - bson.D{ - {Key: "$lte", - Value: bson.A{ - "$layer", - "$$end", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - {Key: "as", Value: "rewardsData"}, - }, - }, - }, - bson.D{ - {Key: "$unwind", - Value: bson.D{ - {Key: "path", Value: "$rewardsData"}, - {Key: "preserveNullAndEmptyArrays", Value: true}, - }, - }, - }, - bson.D{ - {Key: "$group", - Value: bson.D{ - {Key: "_id", Value: "$_id"}, - {Key: "epochData", Value: bson.D{{Key: "$first", Value: "$$ROOT"}}}, - {Key: "totalRewards", Value: bson.D{{Key: "$sum", Value: "$rewardsData.total"}}}, - {Key: "totalRewardsCount", - Value: bson.D{ - {Key: "$sum", - Value: bson.D{ - {Key: "$cond", - Value: bson.A{ - bson.D{ - {Key: "$gt", - Value: bson.A{ - "$rewardsData", - primitive.Null{}, - }, - }, - }, - 1, - 0, - }, - }, - }, - }, - }, - }, - }, - }, - }, - bson.D{{Key: "$project", Value: bson.D{{Key: "epochData.rewardsData", Value: 0}}}}, - bson.D{ - {Key: "$addFields", - Value: bson.D{ - {Key: "epochData.stats.current.rewards", Value: "$totalRewards"}, - {Key: "epochData.stats.current.rewardsnumber", Value: "$totalRewardsCount"}, - {Key: "epochData.stats.cumulative.rewards", Value: "$totalRewards"}, - {Key: "epochData.stats.cumulative.rewardsnumber", Value: "$totalRewardsCount"}, - {Key: "totalRewards", Value: "$$REMOVE"}, - {Key: "totalRewardsCount", Value: "$$REMOVE"}, - }, - }, - }, - bson.D{{Key: "$replaceRoot", Value: bson.D{{Key: "newRoot", Value: "$epochData"}}}}, - bson.D{{Key: "$sort", Value: bson.D{{Key: "number", Value: -1}}}}, - } - - if query != nil { - pipeline = append(bson.A{ - bson.D{{Key: "$match", Value: *query}}, - }, pipeline...) - } - - cursor, err := s.db.Collection("epochs").Aggregate(parent, pipeline) - if err != nil { - return nil, fmt.Errorf("error get epochs: %w", err) - } - var epochs []*model.Epoch - if err = cursor.All(parent, &epochs); err != nil { - return nil, err - } - return epochs, nil -} - -func (s *Storage) GetCirculation(parent context.Context) (int64, error) { - pipeline := bson.A{ - bson.D{ - {Key: "$group", - Value: bson.D{ - {Key: "_id", Value: primitive.Null{}}, - {Key: "circulation", Value: bson.D{{Key: "$sum", Value: "$total"}}}, - }, - }, - }, - } - - cursor, err := s.db.Collection("rewards").Aggregate(parent, pipeline) - if err != nil { - return 0, fmt.Errorf("error get circulation: %w", err) - } - - var result struct { - Circulation int64 `bson:"circulation"` - } - - if cursor.Next(context.Background()) { - if err := cursor.Decode(&result); err != nil { - return 0, fmt.Errorf("error decode circulation: %w", err) - } - return result.Circulation, nil - } - - return 0, nil -} - -func (s *Storage) GetEpochsCount(parent context.Context, query *bson.D, opts ...*options.CountOptions) int64 { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - count, err := s.db.Collection("epochs").CountDocuments(ctx, query, opts...) - if err != nil { - log.Info("GetEpochsCount: %v", err) - return 0 - } - return count -} - -func (s *Storage) GetEpochs(parent context.Context, query *bson.D, opts ...*options.FindOptions) ([]bson.D, error) { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - cursor, err := s.db.Collection("epochs").Find(ctx, query, opts...) - if err != nil { - log.Info("GetEpochs: %v", err) - return nil, err - } - var docs interface{} = []bson.D{} - err = cursor.All(ctx, &docs) - if err != nil { - log.Info("GetEpochs: %v", err) - return nil, err - } - if len(docs.([]bson.D)) == 0 { - log.Info("GetEpochs: Empty result", err) - return nil, nil - } - return docs.([]bson.D), nil -} - -func (s *Storage) SaveEpoch(parent context.Context, epoch *model.Epoch) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - _, err := s.db.Collection("epochs").UpdateOne(ctx, bson.D{{Key: "number", Value: epoch.Number}}, bson.D{{ - Key: "$set", - Value: bson.D{ - {Key: "number", Value: epoch.Number}, - {Key: "start", Value: epoch.Start}, - {Key: "end", Value: epoch.End}, - {Key: "layerstart", Value: epoch.LayerStart}, - {Key: "layerend", Value: epoch.LayerEnd}, - {Key: "layers", Value: epoch.Layers}, - {Key: "stats", Value: bson.D{ - {Key: "current", Value: bson.D{ - {Key: "capacity", Value: epoch.Stats.Current.Capacity}, - {Key: "decentral", Value: epoch.Stats.Current.Decentral}, - {Key: "smeshers", Value: epoch.Stats.Current.Smeshers}, - {Key: "transactions", Value: epoch.Stats.Current.Transactions}, - {Key: "accounts", Value: epoch.Stats.Current.Accounts}, - {Key: "circulation", Value: epoch.Stats.Current.Circulation}, - {Key: "rewards", Value: epoch.Stats.Current.Rewards}, - {Key: "rewardsnumber", Value: epoch.Stats.Current.RewardsNumber}, - {Key: "security", Value: epoch.Stats.Current.Security}, - {Key: "txsamount", Value: epoch.Stats.Current.TxsAmount}, - }}, - {Key: "cumulative", Value: bson.D{ - {Key: "capacity", Value: epoch.Stats.Cumulative.Capacity}, - {Key: "decentral", Value: epoch.Stats.Cumulative.Decentral}, - {Key: "smeshers", Value: epoch.Stats.Cumulative.Smeshers}, - {Key: "transactions", Value: epoch.Stats.Cumulative.Transactions}, - {Key: "accounts", Value: epoch.Stats.Cumulative.Accounts}, - {Key: "circulation", Value: epoch.Stats.Cumulative.Circulation}, - {Key: "rewards", Value: epoch.Stats.Cumulative.Rewards}, - {Key: "rewardsnumber", Value: epoch.Stats.Cumulative.RewardsNumber}, - {Key: "security", Value: epoch.Stats.Cumulative.Security}, - {Key: "txsamount", Value: epoch.Stats.Cumulative.TxsAmount}, - }}, - }}, - }, - }}, options.Update().SetUpsert(true)) - if err != nil { - log.Info("SaveEpoch: %v", err) - } - return err -} - -func (s *Storage) SaveOrUpdateEpoch(parent context.Context, epoch *model.Epoch) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - status, err := s.db.Collection("epochs").UpdateOne(ctx, bson.D{{Key: "number", Value: epoch.Number}}, bson.D{ - {Key: "$set", Value: bson.D{ - {Key: "number", Value: epoch.Number}, - {Key: "start", Value: epoch.Start}, - {Key: "end", Value: epoch.End}, - {Key: "layerstart", Value: epoch.LayerStart}, - {Key: "layerend", Value: epoch.LayerEnd}, - {Key: "layers", Value: epoch.Layers}, - {Key: "stats", Value: bson.D{ - {Key: "current", Value: bson.D{ - {Key: "capacity", Value: epoch.Stats.Current.Capacity}, - {Key: "decentral", Value: epoch.Stats.Current.Decentral}, - {Key: "smeshers", Value: epoch.Stats.Current.Smeshers}, - {Key: "transactions", Value: epoch.Stats.Current.Transactions}, - {Key: "accounts", Value: epoch.Stats.Current.Accounts}, - {Key: "circulation", Value: epoch.Stats.Current.Circulation}, - {Key: "rewards", Value: epoch.Stats.Current.Rewards}, - {Key: "rewardsnumber", Value: epoch.Stats.Current.RewardsNumber}, - {Key: "security", Value: epoch.Stats.Current.Security}, - {Key: "txsamount", Value: epoch.Stats.Current.TxsAmount}, - }}, - {Key: "cumulative", Value: bson.D{ - {Key: "capacity", Value: epoch.Stats.Cumulative.Capacity}, - {Key: "decentral", Value: epoch.Stats.Cumulative.Decentral}, - {Key: "smeshers", Value: epoch.Stats.Cumulative.Smeshers}, - {Key: "transactions", Value: epoch.Stats.Cumulative.Transactions}, - {Key: "accounts", Value: epoch.Stats.Cumulative.Accounts}, - {Key: "circulation", Value: epoch.Stats.Cumulative.Circulation}, - {Key: "rewards", Value: epoch.Stats.Cumulative.Rewards}, - {Key: "rewardsnumber", Value: epoch.Stats.Cumulative.RewardsNumber}, - {Key: "security", Value: epoch.Stats.Cumulative.Security}, - {Key: "txsamount", Value: epoch.Stats.Cumulative.TxsAmount}, - }}, - }}, - }}, - }, options.Update().SetUpsert(true)) - if err != nil { - log.Info("SaveOrUpdateEpoch: %+v, %v", status, err) - } - return err -} - -func (s *Storage) computeStatistics(epoch *model.Epoch) { - layerStart, layerEnd := s.GetEpochLayers(epoch.Number) - if epoch.Start == 0 { - epoch.LayerStart = layerStart - epoch.Start = s.getLayerTimestamp(layerStart) - } - epoch.LayerEnd = layerEnd - epoch.End = s.getLayerTimestamp(layerEnd) + s.NetworkInfo.LayerDuration - 1 - epoch.Layers = epoch.LayerEnd - epoch.LayerStart + 1 - duration := float64(s.NetworkInfo.LayerDuration) * float64(s.GetLayersCount(context.Background(), s.GetEpochLayersFilter(epoch.Number, "number"))) - layerFilter := s.GetEpochLayersFilter(epoch.Number, "layer") - epoch.Stats.Current.Transactions = s.GetTransactionsCount(context.Background(), layerFilter) - epoch.Stats.Current.TxsAmount = s.GetTransactionsAmount(context.Background(), layerFilter) - if duration > 0 && s.NetworkInfo.MaxTransactionsPerSecond > 0 { - // todo replace to utils.CalcEpochCapacity - epoch.Stats.Current.Capacity = int64(math.Round(((float64(epoch.Stats.Current.Transactions) / duration) / float64(s.NetworkInfo.MaxTransactionsPerSecond)) * 100.0)) - } - atxs, _ := s.GetActivations(context.Background(), &bson.D{{Key: "targetEpoch", Value: epoch.Number}}) - if atxs != nil { - smeshers := make(map[string]uint64) - for _, atx := range atxs { - var commitmentSize uint64 - var smesher string - for _, e := range atx { - if e.Key == "smesher" { - smesher, _ = e.Value.(string) - continue - } - if e.Key == "commitmentSize" { - if value, ok := e.Value.(int64); ok { - commitmentSize = uint64(value) - } else if value, ok := e.Value.(int32); ok { - commitmentSize = uint64(value) - } - } - } - if smesher != "" { - smeshers[smesher] += commitmentSize - epoch.Stats.Current.Security += int64(commitmentSize) - } - } - epoch.Stats.Current.Smeshers = int64(len(smeshers)) - // degree_of_decentralization is defined as: 0.5 * (min(n,1e4)^2/1e8) + 0.5 * (1 - gini_coeff(last_100_epochs)) - a := math.Min(float64(epoch.Stats.Current.Smeshers), 1e4) - // todo replace to utils.CalcDecentralCoefficient - epoch.Stats.Current.Decentral = int64(100.0 * (0.5*(a*a)/1e8 + 0.5*(1.0-utils.Gini(smeshers)))) - } - epoch.Stats.Current.Accounts = s.GetAccountsCount(context.Background(), &bson.D{{Key: "created", Value: bson.D{{Key: "$lte", Value: layerEnd}}}}) - //epoch.Stats.Cumulative.Circulation, _ = s.GetLayersRewards(context.Background(), 0, layerEnd) - //epoch.Stats.Current.Rewards, epoch.Stats.Current.RewardsNumber = s.GetLayersRewards(context.Background(), layerStart, layerEnd) -} - -func (s *Storage) RecalculateEpochStats() { - currentEpoch := s.NetworkInfo.VerifiedLayer / s.NetworkInfo.EpochNumLayers - for i := 0; i <= int(currentEpoch+1); i++ { - s.UpdateEpochStats(uint32(i) * s.NetworkInfo.EpochNumLayers) - } -} diff --git a/storage/layer.go b/storage/layer.go deleted file mode 100644 index 80ed1d4..0000000 --- a/storage/layer.go +++ /dev/null @@ -1,149 +0,0 @@ -package storage - -import ( - "context" - "errors" - "time" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/go-spacemesh/log" - - "github.com/spacemeshos/explorer-backend/model" - "github.com/spacemeshos/explorer-backend/utils" -) - -func (s *Storage) InitLayersStorage(ctx context.Context) error { - _, err := s.db.Collection("layers").Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "number", Value: 1}}, Options: options.Index().SetName("numberIndex").SetUnique(true)}) - //_, err = s.db.Collection("layers").Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "hash", Value: 1}}, Options: options.Index().SetName("hashIndex").SetUnique(true)}) - return err -} - -func (s *Storage) GetLayerByNumber(parent context.Context, layerNumber uint32) (*model.Layer, error) { - return s.GetLayer(parent, &bson.D{{Key: "number", Value: layerNumber}}) -} - -func (s *Storage) GetLayer(parent context.Context, query *bson.D) (*model.Layer, error) { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - cursor, err := s.db.Collection("layers").Find(ctx, query) - if err != nil { - log.Info("GetLayer: %v", err) - return nil, err - } - if !cursor.Next(ctx) { - log.Info("GetLayer: Empty result", err) - return nil, errors.New("Empty result") - } - doc := cursor.Current - account := &model.Layer{ - Number: utils.GetAsUInt32(doc.Lookup("number")), - Status: utils.GetAsInt(doc.Lookup("status")), - Txs: utils.GetAsUInt32(doc.Lookup("txs")), - Start: utils.GetAsUInt32(doc.Lookup("start")), - End: utils.GetAsUInt32(doc.Lookup("end")), - TxsAmount: utils.GetAsUInt64(doc.Lookup("txsamount")), - Rewards: utils.GetAsUInt64(doc.Lookup("rewards")), - Epoch: utils.GetAsUInt32(doc.Lookup("epoch")), - Hash: utils.GetAsString(doc.Lookup("hash")), - BlocksNumber: utils.GetAsUInt32(doc.Lookup("blocksnumber")), - } - return account, nil -} - -func (s *Storage) GetLayersCount(parent context.Context, query *bson.D, opts ...*options.CountOptions) int64 { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - count, err := s.db.Collection("layers").CountDocuments(ctx, query, opts...) - if err != nil { - log.Info("GetLayersCount: %v", err) - return 0 - } - return count -} - -func (s *Storage) GetLastLayer(parent context.Context) uint32 { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - cursor, err := s.db.Collection("layers").Find(ctx, bson.D{}, options.Find().SetSort(bson.D{{Key: "number", Value: -1}}).SetLimit(1)) - if err != nil { - log.Info("GetLastLayer: %v", err) - return 0 - } - if !cursor.Next(ctx) { - log.Info("GetLastLayer: Empty result", err) - return 0 - } - doc := cursor.Current - return utils.GetAsUInt32(doc.Lookup("number")) -} - -func (s *Storage) GetLayers(parent context.Context, query *bson.D, opts ...*options.FindOptions) ([]bson.D, error) { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - cursor, err := s.db.Collection("layers").Find(ctx, query, opts...) - if err != nil { - log.Info("GetLayers: %v", err) - return nil, err - } - var docs interface{} = []bson.D{} - err = cursor.All(ctx, &docs) - if err != nil { - log.Info("GetLayers: %v", err) - return nil, err - } - if len(docs.([]bson.D)) == 0 { - log.Info("GetLayers: Empty result", err) - return nil, nil - } - return docs.([]bson.D), nil -} - -func (s *Storage) SaveLayer(parent context.Context, in *model.Layer) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - _, err := s.db.Collection("layers").UpdateOne(ctx, bson.D{{Key: "number", Value: in.Number}}, bson.D{{ - Key: "$set", - Value: bson.D{ - {Key: "number", Value: in.Number}, - {Key: "status", Value: in.Status}, - {Key: "txs", Value: in.Txs}, - {Key: "start", Value: in.Start}, - {Key: "end", Value: in.End}, - {Key: "txsamount", Value: in.TxsAmount}, - {Key: "rewards", Value: in.Rewards}, - {Key: "epoch", Value: in.Epoch}, - {Key: "hash", Value: in.Hash}, - {Key: "blocksnumber", Value: in.BlocksNumber}, - }, - }}, options.Update().SetUpsert(true)) - if err != nil { - log.Info("SaveLayer: %v", err) - } - return err -} - -func (s *Storage) SaveOrUpdateLayer(parent context.Context, in *model.Layer) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - _, err := s.db.Collection("layers").UpdateOne(ctx, bson.D{{Key: "number", Value: in.Number}}, bson.D{ - {Key: "$set", Value: bson.D{ - {Key: "number", Value: in.Number}, - {Key: "status", Value: in.Status}, - {Key: "txs", Value: in.Txs}, - {Key: "start", Value: in.Start}, - {Key: "end", Value: in.End}, - {Key: "txsamount", Value: in.TxsAmount}, - {Key: "rewards", Value: in.Rewards}, - {Key: "epoch", Value: in.Epoch}, - {Key: "hash", Value: in.Hash}, - {Key: "blocksnumber", Value: in.BlocksNumber}, - }}, - }, options.Update().SetUpsert(true)) - if err != nil { - log.Info("SaveOrUpdateLayer: %v", err) - } - return err -} diff --git a/storage/malfaesance_proof.go b/storage/malfaesance_proof.go deleted file mode 100644 index a6411e4..0000000 --- a/storage/malfaesance_proof.go +++ /dev/null @@ -1,31 +0,0 @@ -package storage - -import ( - "context" - "github.com/spacemeshos/explorer-backend/model" - "github.com/spacemeshos/go-spacemesh/log" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - "time" -) - -func (s *Storage) SaveMalfeasanceProof(parent context.Context, in *model.MalfeasanceProof) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - _, err := s.db.Collection("malfeasance_proofs").UpdateOne(ctx, bson.D{ - {Key: "smesher", Value: in.Smesher}, - {Key: "layer", Value: in.Layer}, - {Key: "kind", Value: in.Kind}, - }, bson.D{ - {Key: "$setOnInsert", Value: bson.D{ - {Key: "smesher", Value: in.Smesher}, - {Key: "layer", Value: in.Layer}, - {Key: "kind", Value: in.Kind}, - {Key: "debugInfo", Value: in.DebugInfo}, - }}, - }, options.Update().SetUpsert(true)) - if err != nil { - log.Info("SaveMalfeasanceProof: %v", err) - } - return err -} diff --git a/storage/network_info.go b/storage/network_info.go deleted file mode 100644 index bad8ac7..0000000 --- a/storage/network_info.go +++ /dev/null @@ -1,76 +0,0 @@ -package storage - -import ( - "context" - "errors" - "time" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/go-spacemesh/log" - - "github.com/spacemeshos/explorer-backend/model" - "github.com/spacemeshos/explorer-backend/utils" -) - -func (s *Storage) GetNetworkInfo(parent context.Context) (*model.NetworkInfo, error) { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - cursor, err := s.db.Collection("networkinfo").Find(ctx, bson.D{{Key: "id", Value: 1}}) - if err != nil { - log.Info("GetNetworkInfo: %v", err) - return nil, err - } - if !cursor.Next(ctx) { - log.Info("GetNetworkInfo: Empty result") - return nil, errors.New("Empty result") - } - doc := cursor.Current - info := &model.NetworkInfo{ - GenesisId: utils.GetAsString(doc.Lookup("genesisid")), - GenesisTime: utils.GetAsUInt32(doc.Lookup("genesis")), - EpochNumLayers: utils.GetAsUInt32(doc.Lookup("layers")), - MaxTransactionsPerSecond: utils.GetAsUInt32(doc.Lookup("maxtx")), - LayerDuration: utils.GetAsUInt32(doc.Lookup("duration")), - LastLayer: utils.GetAsUInt32(doc.Lookup("lastlayer")), - LastLayerTimestamp: utils.GetAsUInt32(doc.Lookup("lastlayerts")), - LastApprovedLayer: utils.GetAsUInt32(doc.Lookup("lastapprovedlayer")), - LastConfirmedLayer: utils.GetAsUInt32(doc.Lookup("lastconfirmedlayer")), - ConnectedPeers: utils.GetAsUInt64(doc.Lookup("connectedpeers")), - IsSynced: utils.GetAsBool(doc.Lookup("issynced")), - SyncedLayer: utils.GetAsUInt32(doc.Lookup("syncedlayer")), - TopLayer: utils.GetAsUInt32(doc.Lookup("toplayer")), - VerifiedLayer: utils.GetAsUInt32(doc.Lookup("verifiedlayer")), - } - return info, nil -} - -func (s *Storage) SaveOrUpdateNetworkInfo(parent context.Context, in *model.NetworkInfo) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - _, err := s.db.Collection("networkinfo").UpdateOne(ctx, bson.D{{Key: "id", Value: 1}}, bson.D{ - {Key: "$set", Value: bson.D{ - {Key: "id", Value: 1}, - {Key: "genesisid", Value: in.GenesisId}, - {Key: "genesis", Value: in.GenesisTime}, - {Key: "layers", Value: in.EpochNumLayers}, - {Key: "maxtx", Value: in.MaxTransactionsPerSecond}, - {Key: "duration", Value: in.LayerDuration}, - {Key: "postUnitSize", Value: in.PostUnitSize}, - {Key: "lastlayer", Value: in.LastLayer}, - {Key: "lastlayerts", Value: in.LastLayerTimestamp}, - {Key: "lastapprovedlayer", Value: in.LastApprovedLayer}, - {Key: "lastconfirmedlayer", Value: in.LastConfirmedLayer}, - {Key: "connectedpeers", Value: in.ConnectedPeers}, - {Key: "issynced", Value: in.IsSynced}, - {Key: "syncedlayer", Value: in.SyncedLayer}, - {Key: "toplayer", Value: in.TopLayer}, - {Key: "verifiedlayer", Value: in.VerifiedLayer}, - }}, - }, options.Update().SetUpsert(true)) - if err != nil { - log.Info("SaveOrUpdateNetworkInfo: %v", err) - } - return err -} diff --git a/storage/reward.go b/storage/reward.go deleted file mode 100644 index c01e3fa..0000000 --- a/storage/reward.go +++ /dev/null @@ -1,201 +0,0 @@ -package storage - -import ( - "context" - "errors" - "time" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/go-spacemesh/log" - - "github.com/spacemeshos/explorer-backend/model" - "github.com/spacemeshos/explorer-backend/utils" -) - -func (s *Storage) InitRewardsStorage(ctx context.Context) error { - models := []mongo.IndexModel{ - {Keys: bson.D{{Key: "layer", Value: 1}}, Options: options.Index().SetName("layerIndex").SetUnique(false)}, - {Keys: bson.D{{Key: "smesher", Value: 1}}, Options: options.Index().SetName("smesherIndex").SetUnique(false)}, - {Keys: bson.D{{Key: "coinbase", Value: 1}}, Options: options.Index().SetName("coinbaseIndex").SetUnique(false)}, - {Keys: bson.D{{Key: "layer", Value: 1}, {Key: "smesher", Value: 1}, {Key: "coinbase", Value: 1}}, Options: options.Index().SetName("rewardIndex").SetUnique(false)}, - {Keys: bson.D{{Key: "layer", Value: 1}, {Key: "total", Value: 1}, {Key: "layerReward", Value: 1}}, Options: options.Index().SetName("layerRewards").SetUnique(false)}, - {Keys: bson.D{{Key: "smesher", Value: 1}, {Key: "layer", Value: 1}}, Options: options.Index().SetName("keyIndex").SetUnique(true)}, - } - _, err := s.db.Collection("rewards").Indexes().CreateMany(ctx, models, options.CreateIndexes().SetMaxTime(20*time.Second)) - return err -} - -func (s *Storage) GetReward(parent context.Context, query *bson.D) (*model.Reward, error) { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - cursor, err := s.db.Collection("rewards").Find(ctx, query) - if err != nil { - log.Info("GetReward: %v", err) - return nil, err - } - if !cursor.Next(ctx) { - log.Info("GetReward: Empty result") - return nil, errors.New("Empty result") - } - doc := cursor.Current - account := &model.Reward{ - Layer: utils.GetAsUInt32(doc.Lookup("layer")), - Total: utils.GetAsUInt64(doc.Lookup("total")), - LayerReward: utils.GetAsUInt64(doc.Lookup("layerReward")), - LayerComputed: utils.GetAsUInt32(doc.Lookup("layerComputed")), - Coinbase: utils.GetAsString(doc.Lookup("coinbase")), - Smesher: utils.GetAsString(doc.Lookup("smesher")), - Timestamp: utils.GetAsUInt32(doc.Lookup("timestamp")), - } - return account, nil -} - -func (s *Storage) GetRewardsCount(parent context.Context, query *bson.D, opts ...*options.CountOptions) int64 { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - count, err := s.db.Collection("rewards").CountDocuments(ctx, query, opts...) - if err != nil { - log.Info("GetRewardsCount: %v", err) - return 0 - } - return count -} - -func (s *Storage) GetLayersRewards(parent context.Context, layerStart uint32, layerEnd uint32) (int64, int64) { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - matchStage := bson.D{ - {Key: "$match", Value: bson.D{ - {Key: "layer", Value: bson.D{{Key: "$gte", Value: layerStart}, {Key: "$lte", Value: layerEnd}}}, - }}, - } - groupStage := bson.D{ - {Key: "$group", Value: bson.D{ - {Key: "_id", Value: ""}, - {Key: "total", Value: bson.D{ - {Key: "$sum", Value: "$total"}, - }}, - {Key: "layerReward", Value: bson.D{ - {Key: "$sum", Value: "$layerReward"}, - }}, - {Key: "count", Value: bson.D{ - {Key: "$sum", Value: 1}, - }}, - }}, - } - cursor, err := s.db.Collection("rewards").Aggregate(ctx, mongo.Pipeline{ - matchStage, - groupStage, - }) - if err != nil { - log.Info("GetLayersRewards: %v", err) - return 0, 0 - } - if !cursor.Next(ctx) { - log.Info("GetLayersRewards: Empty result") - return 0, 0 - } - doc := cursor.Current - return utils.GetAsInt64(doc.Lookup("total")), utils.GetAsInt64(doc.Lookup("count")) -} - -func (s *Storage) GetSmesherRewards(parent context.Context, smesher string) (int64, int64) { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - matchStage := bson.D{{Key: "$match", Value: bson.D{{Key: "smesher", Value: smesher}}}} - groupStage := bson.D{ - {Key: "$group", Value: bson.D{ - {Key: "_id", Value: ""}, - {Key: "total", Value: bson.D{ - {Key: "$sum", Value: "$total"}, - }}, - {Key: "layerReward", Value: bson.D{ - {Key: "$sum", Value: "$layerReward"}, - }}, - {Key: "count", Value: bson.D{ - {Key: "$sum", Value: 1}, - }}, - }}, - } - cursor, err := s.db.Collection("rewards").Aggregate(ctx, mongo.Pipeline{ - matchStage, - groupStage, - }) - if err != nil { - log.Info("GetSmesherRewards: %v", err) - return 0, 0 - } - if !cursor.Next(ctx) { - log.Info("GetSmesherRewards: Empty result") - return 0, 0 - } - doc := cursor.Current - return utils.GetAsInt64(doc.Lookup("total")), utils.GetAsInt64(doc.Lookup("count")) -} - -func (s *Storage) GetRewards(parent context.Context, query *bson.D, opts ...*options.FindOptions) ([]bson.D, error) { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - cursor, err := s.db.Collection("rewards").Find(ctx, query, opts...) - if err != nil { - log.Info("GetRewards: %v", err) - return nil, err - } - var docs interface{} = []bson.D{} - err = cursor.All(ctx, &docs) - if err != nil { - log.Info("GetRewards: %v", err) - return nil, err - } - if len(docs.([]bson.D)) == 0 { - return nil, nil - } - return docs.([]bson.D), nil -} - -func (s *Storage) SaveReward(parent context.Context, in *model.Reward) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - _, err := s.db.Collection("rewards").UpdateOne(ctx, bson.D{{Key: "smesher", Value: in.Smesher}, {Key: "layer", Value: in.Layer}}, bson.D{{ - Key: "$set", - Value: bson.D{ - {Key: "layer", Value: in.Layer}, - {Key: "total", Value: in.Total}, - {Key: "layerReward", Value: in.LayerReward}, - {Key: "layerComputed", Value: in.LayerComputed}, - {Key: "coinbase", Value: in.Coinbase}, - {Key: "smesher", Value: in.Smesher}, - {Key: "timestamp", Value: in.Timestamp}, - }, - }}, options.Update().SetUpsert(true)) - if err != nil { - log.Info("SaveReward: %v", err) - } - return err -} - -func (s *Storage) GetRewardsTotalSum(ctx context.Context) (int64, error) { - groupStage := bson.D{ - {Key: "$group", Value: bson.D{ - {Key: "_id", Value: ""}, - {Key: "totalRewards", Value: bson.D{ - {Key: "$sum", Value: "$total"}, - }, - }, - }}, - } - cursor, err := s.db.Collection("rewards").Aggregate(ctx, mongo.Pipeline{groupStage}) - if err != nil { - return 0, err - } - - if !cursor.Next(ctx) { - log.Info("GetSmesherRewards: Empty result") - return 0, nil - } - doc := cursor.Current - return utils.GetAsInt64(doc.Lookup("totalRewards")), nil -} diff --git a/storage/smesher.go b/storage/smesher.go deleted file mode 100644 index ad8d940..0000000 --- a/storage/smesher.go +++ /dev/null @@ -1,203 +0,0 @@ -package storage - -import ( - "context" - "errors" - "fmt" - "time" - - "github.com/spacemeshos/go-spacemesh/log" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/explorer-backend/model" - "github.com/spacemeshos/explorer-backend/utils" -) - -func (s *Storage) InitSmeshersStorage(ctx context.Context) error { - _, err := s.db.Collection("smeshers").Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index().SetName("idIndex").SetUnique(true)}) - if err != nil { - return fmt.Errorf("error init `smeshers` collection: %w", err) - } - _, err = s.db.Collection("coinbases").Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "smesherId", Value: 1}}, Options: options.Index().SetName("smesherIdIndex").SetUnique(true)}) - if err != nil { - return fmt.Errorf("error init `coinbases` collection: %w", err) - } - return nil -} - -func (s *Storage) GetSmesher(parent context.Context, query *bson.D) (*model.Smesher, error) { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - cursor, err := s.db.Collection("smeshers").Find(ctx, query) - if err != nil { - log.Info("GetSmesher: %v", err) - return nil, err - } - if !cursor.Next(ctx) { - log.Info("GetSmesher: Empty result") - return nil, errors.New("Empty result") - } - doc := cursor.Current - smesher := &model.Smesher{ - Id: utils.GetAsString(doc.Lookup("id")), - CommitmentSize: utils.GetAsUInt64(doc.Lookup("cSize")), - Coinbase: utils.GetAsString(doc.Lookup("coinbase")), - AtxCount: utils.GetAsUInt32(doc.Lookup("atxcount")), - Timestamp: utils.GetAsUInt64(doc.Lookup("timestamp")), - } - return smesher, nil -} - -func (s *Storage) GetSmeshersCount(parent context.Context, query *bson.D, opts ...*options.CountOptions) int64 { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - count, err := s.db.Collection("smeshers").CountDocuments(ctx, query, opts...) - if err != nil { - log.Info("GetSmeshersCount: %v", err) - return 0 - } - return count -} - -func (s *Storage) IsSmesherExists(parent context.Context, smesher string) bool { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - count, err := s.db.Collection("smeshers").CountDocuments(ctx, bson.D{{Key: "id", Value: smesher}}) - if err != nil { - log.Info("IsSmesherExists: %v", err) - return false - } - return count > 0 -} - -func (s *Storage) GetSmeshers(parent context.Context, query *bson.D, opts ...*options.FindOptions) ([]bson.D, error) { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - cursor, err := s.db.Collection("smeshers").Find(ctx, query, opts...) - if err != nil { - log.Info("GetSmeshers: %v", err) - return nil, err - } - var docs interface{} = []bson.D{} - err = cursor.All(ctx, &docs) - if err != nil { - log.Info("GetSmeshers: %v", err) - return nil, err - } - if len(docs.([]bson.D)) == 0 { - return nil, nil - } - return docs.([]bson.D), nil -} - -func (s *Storage) SaveSmesher(parent context.Context, in *model.Smesher, epoch uint32) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - opts := options.Update().SetUpsert(true) - _, err := s.db.Collection("smeshers").UpdateOne(ctx, bson.D{{Key: "id", Value: in.Id}}, bson.D{ - {Key: "$set", Value: bson.D{ - {Key: "id", Value: in.Id}, - {Key: "cSize", Value: in.CommitmentSize}, - {Key: "coinbase", Value: in.Coinbase}, - {Key: "atxcount", Value: in.AtxCount}, - {Key: "timestamp", Value: in.Timestamp}, - }}, - {Key: "$addToSet", Value: bson.M{"epochs": epoch}}, - }, opts) - if err != nil { - return fmt.Errorf("error save smesher: %w", err) - } - return nil -} - -func (s *Storage) SaveSmesherQuery(in *model.Smesher) *mongo.UpdateOneModel { - filter := bson.D{{Key: "id", Value: in.Id}} - update := bson.D{ - {Key: "$set", Value: bson.D{ - {Key: "id", Value: in.Id}, - {Key: "cSize", Value: in.CommitmentSize}, - {Key: "coinbase", Value: in.Coinbase}, - {Key: "atxcount", Value: in.AtxCount}, - {Key: "timestamp", Value: in.Timestamp}, - }}, - } - - updateModel := mongo.NewUpdateOneModel() - updateModel.Filter = filter - updateModel.Update = update - updateModel.SetUpsert(true) - - return updateModel -} - -func (s *Storage) UpdateSmesher(parent context.Context, in *model.Smesher, epoch uint32) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - - filter := bson.D{{Key: "smesherId", Value: in.Id}} - _, err := s.db.Collection("coinbases").UpdateOne( - ctx, - filter, - bson.D{{Key: "$set", Value: bson.D{{Key: "coinbase", Value: in.Coinbase}}}}, - options.Update().SetUpsert(true)) - if err != nil { - return fmt.Errorf("error insert smesher into `coinbases`: %w", err) - } - - atxCount, err := s.db.Collection("activations").CountDocuments(ctx, &bson.D{{Key: "smesher", Value: in.Id}}) - if err != nil { - log.Info("UpdateSmesher: GetActivationsCount: %v", err) - } - - _, err = s.db.Collection("smeshers").UpdateOne(ctx, bson.D{{Key: "id", Value: in.Id}}, bson.D{ - {Key: "$set", Value: bson.D{ - {Key: "id", Value: in.Id}, - {Key: "cSize", Value: in.CommitmentSize}, - {Key: "coinbase", Value: in.Coinbase}, - {Key: "timestamp", Value: in.Timestamp}, - {Key: "atxcount", Value: atxCount}, - }}, - {Key: "$addToSet", Value: bson.M{"epochs": epoch}}, - }, options.Update().SetUpsert(true)) - if err != nil { - log.Info("UpdateSmesher: %v", err) - } - return err -} - -func (s *Storage) UpdateSmesherQuery(in *model.Smesher, epoch uint32) (*mongo.UpdateOneModel, *mongo.UpdateOneModel) { - coinbaseFilter := bson.D{{Key: "smesherId", Value: in.Id}} - coinbaseUpdate := bson.D{ - {Key: "$set", Value: bson.D{ - {Key: "coinbase", Value: in.Coinbase}, - }}} - coinbaseModel := mongo.NewUpdateOneModel() - coinbaseModel.SetFilter(coinbaseFilter) - coinbaseModel.SetUpdate(coinbaseUpdate) - coinbaseModel.SetUpsert(true) - - atxCount, err := s.db.Collection("activations").CountDocuments(context.TODO(), &bson.D{{Key: "smesher", Value: in.Id}}) - if err != nil { - log.Info("UpdateSmesher: GetActivationsCount: %v", err) - } - - smesherFilter := bson.D{{Key: "id", Value: in.Id}} - smesherUpdate := bson.D{ - {Key: "$set", Value: bson.D{ - {Key: "id", Value: in.Id}, - {Key: "cSize", Value: in.CommitmentSize}, - {Key: "coinbase", Value: in.Coinbase}, - {Key: "timestamp", Value: in.Timestamp}, - {Key: "atxcount", Value: atxCount}, - }}, - {Key: "$addToSet", Value: bson.M{"epochs": epoch}}, - } - - smesherModel := mongo.NewUpdateOneModel() - smesherModel.SetFilter(smesherFilter) - smesherModel.SetUpdate(smesherUpdate) - - return coinbaseModel, smesherModel -} diff --git a/storage/storage.go b/storage/storage.go deleted file mode 100644 index a929856..0000000 --- a/storage/storage.go +++ /dev/null @@ -1,661 +0,0 @@ -package storage - -import ( - "container/list" - "context" - "errors" - "fmt" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/spacemeshos/go-spacemesh/common/types" - "sync" - "time" - - "github.com/spacemeshos/explorer-backend/utils" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" - "github.com/spacemeshos/explorer-backend/model" - "github.com/spacemeshos/go-spacemesh/log" -) - -var ( - metricLastProcessedLayer = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "explorer_last_processed_layer", - Help: "", - }) - - metricLayersQueueLen = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "explorer_layers_queue_length", - Help: "", - }) - - metricNodeSyncedLayer = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "explorer_node_synced_layer", - Help: "", - }) - metricNodeTopLayer = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "explorer_node_top_layer", - Help: "", - }) - metricNodeVerifiedLayer = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "explorer_node_verified_layer", - Help: "", - }) -) - -type AccountUpdaterService interface { - GetAccountState(address string) (uint64, uint64, error) -} - -type Storage struct { - NetworkInfo model.NetworkInfo - postUnitSize uint64 - - client *mongo.Client - db *mongo.Database - - AccountUpdater AccountUpdaterService - - sync.Mutex - changedEpoch int32 - lastEpoch int32 - - layersLock sync.Mutex - layersQueue *list.List - layersReady *sync.Cond - - accountsLock sync.Mutex - accountsQueue map[uint32]map[string]bool - accountsReady *sync.Cond -} - -func New(parent context.Context, dbUrl string, dbName string) (*Storage, error) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - client, err := mongo.Connect(ctx, options.Client().ApplyURI(dbUrl)) - - if err != nil { - return nil, err - } - - err = client.Ping(ctx, nil) - - if err != nil { - return nil, err - } - - s := &Storage{ - client: client, - layersQueue: list.New(), - layersReady: sync.NewCond(&sync.Mutex{}), - accountsQueue: make(map[uint32]map[string]bool), - accountsReady: sync.NewCond(&sync.Mutex{}), - changedEpoch: -1, - } - s.db = client.Database(dbName) - - err = s.InitAccountsStorage(ctx) - if err != nil { - log.Info("Init accounts storage error: %v", err) - } - err = s.InitActivationsStorage(ctx) - if err != nil { - log.Info("Init activations storage error: %v", err) - } - err = s.InitBlocksStorage(ctx) - if err != nil { - log.Info("Init blocks storage error: %v", err) - } - err = s.InitEpochsStorage(ctx) - if err != nil { - log.Info("Init epochs storage error: %v", err) - } - err = s.InitLayersStorage(ctx) - if err != nil { - log.Info("Init layers storage error: %v", err) - } - err = s.InitRewardsStorage(ctx) - if err != nil { - log.Info("Init rewards storage error: %v", err) - } - err = s.InitSmeshersStorage(ctx) - if err != nil { - log.Info("Init smeshers storage error: %v", err) - } - err = s.InitTransactionsStorage(ctx) - if err != nil { - log.Info("Init transactions storage error: %v", err) - } - - go s.updateAccounts() - go s.updateLayers() - - return s, nil -} - -func (s *Storage) Close() { - if s.client != nil { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - s.db = nil - err := s.client.Disconnect(ctx) - if err != nil { - log.Err(fmt.Errorf("error while disconnecting from database: %v", err)) - } - } -} - -func (s *Storage) OnNetworkInfo(genesisId string, genesisTime uint64, epochNumLayers uint32, maxTransactionsPerSecond uint64, layerDuration uint64, postUnitSize uint64) { - s.NetworkInfo.GenesisId = genesisId - s.NetworkInfo.GenesisTime = uint32(genesisTime) - s.NetworkInfo.EpochNumLayers = epochNumLayers - s.NetworkInfo.MaxTransactionsPerSecond = uint32(maxTransactionsPerSecond) - s.NetworkInfo.LayerDuration = uint32(layerDuration) - s.NetworkInfo.PostUnitSize = postUnitSize - s.postUnitSize = postUnitSize - - err := s.SaveOrUpdateNetworkInfo(context.Background(), &s.NetworkInfo) - //TODO: better error handling - if err != nil { - log.Err(fmt.Errorf("OnNetworkInfo: error %v", err)) - } - - log.Info("Network Info: id: %s, genesis: %v, epoch layers: %v, max tx: %v, duration: %v", - s.NetworkInfo.GenesisId, - s.NetworkInfo.GenesisTime, - s.NetworkInfo.EpochNumLayers, - s.NetworkInfo.MaxTransactionsPerSecond, - s.NetworkInfo.LayerDuration, - ) -} - -func (s *Storage) OnNodeStatus(connectedPeers uint64, isSynced bool, syncedLayer uint32, topLayer uint32, verifiedLayer uint32) { - s.NetworkInfo.ConnectedPeers = connectedPeers - s.NetworkInfo.IsSynced = isSynced - s.NetworkInfo.SyncedLayer = syncedLayer - s.NetworkInfo.TopLayer = topLayer - s.NetworkInfo.VerifiedLayer = verifiedLayer - - err := s.SaveOrUpdateNetworkInfo(context.Background(), &s.NetworkInfo) - //TODO: better error handling - if err != nil { - log.Err(fmt.Errorf("OnNodeStatus: error %v", err)) - } - - metricNodeTopLayer.Set(float64(topLayer)) - metricNodeVerifiedLayer.Set(float64(verifiedLayer)) - metricNodeSyncedLayer.Set(float64(syncedLayer)) -} - -func (s *Storage) GetEpochLayers(epoch int32) (uint32, uint32) { - start := uint32(epoch) * s.NetworkInfo.EpochNumLayers - end := start + s.NetworkInfo.EpochNumLayers - 1 - return start, end -} - -func (s *Storage) GetEpochForLayer(layer uint32) uint32 { - if s.NetworkInfo.EpochNumLayers > 0 { - return layer / s.NetworkInfo.EpochNumLayers - } - return 0 -} - -func (s *Storage) GetEpochNumLayers() uint32 { - return s.NetworkInfo.EpochNumLayers -} - -func (s *Storage) OnLayer(in *pb.Layer) { - s.pushLayer(in) - metricLayersQueueLen.Set(float64(s.layersQueue.Len())) -} - -func (s *Storage) OnMalfeasanceProof(in *pb.MalfeasanceProof) { - s.updateMalfeasanceProof(in) -} - -func (s *Storage) OnAccounts(accounts []*types.Account) { - log.Info("OnAccounts") - - var updateOps []mongo.WriteModel - - for _, acc := range accounts { - filter := bson.D{{Key: "address", Value: acc.Address.String()}} - update := bson.D{ - {Key: "$set", Value: bson.D{ - {Key: "balance", Value: acc.Balance}, - {Key: "counter", Value: acc.NextNonce}, - }}, - {Key: "$setOnInsert", Value: bson.D{ - {Key: "created", Value: acc.Layer.Uint32()}, - }}, - } - - updateModel := mongo.NewUpdateOneModel() - updateModel.Filter = filter - updateModel.Update = update - updateModel.SetUpsert(true) - - updateOps = append(updateOps, updateModel) - } - - if len(updateOps) > 0 { - _, err := s.db.Collection("accounts").BulkWrite(context.TODO(), updateOps) - if err != nil { - log.Err(fmt.Errorf("OnAccounts: error accounts write %v", err)) - } - } -} - -func (s *Storage) OnReward(in *pb.Reward) { - log.Info("OnReward(%+v)", in) - reward := model.NewReward(in) - if reward == nil { - return - } - reward.Timestamp = s.getLayerTimestamp(reward.Layer) - - err := s.SaveReward(context.Background(), reward) - //TODO: better error handling - if err != nil { - log.Err(fmt.Errorf("OnReward save: error %v", err)) - } - - err = s.AddAccount(context.Background(), reward.Layer, reward.Coinbase, 0) - //TODO: better error handling - if err != nil { - log.Err(fmt.Errorf("OnReward add account: error %v", err)) - } - - err = s.AddAccountReward(context.Background(), reward.Layer, reward.Coinbase, reward.Total, reward.LayerReward) - //TODO: better error handling - if err != nil { - log.Err(fmt.Errorf("OnReward add account reward: error %v", err)) - } - - s.requestBalanceUpdate(reward.Layer, reward.Coinbase) -} - -func (s *Storage) UpdateEpochStats(layer uint32) { - s.setChangedEpoch(layer) - s.updateEpochs() -} - -func (s *Storage) OnTransactionResult(res *pb.TransactionResult, state *pb.TransactionState) { - log.Info("OnTransactionReceipt(%+v, %+v)", res, state) - tx, err := model.NewTransactionResult(res, state, s.NetworkInfo) - if err != nil { - log.Err(fmt.Errorf("OnTransactionResult: error %v", err)) - return - } - - err = s.SaveTransactionResult(context.Background(), tx) - //TODO: better error handling - if err != nil { - log.Err(fmt.Errorf("OnTransactionResult: error %v", err)) - } -} - -func (s *Storage) pushLayer(layer *pb.Layer) { - s.layersLock.Lock() - s.layersQueue.PushBack(layer) - s.layersLock.Unlock() - s.layersReady.Signal() -} - -func (s *Storage) IsLayerInQueue(layer *pb.Layer) bool { - for l := s.layersQueue.Front(); l != nil; l = l.Next() { - if val, ok := l.Value.(*pb.Layer); ok { - if val.Number.Number == layer.Number.Number { - return true - } - } - } - return false -} - -func (s *Storage) processLayer() *pb.Layer { - s.layersLock.Lock() - defer s.layersLock.Unlock() - l := s.layersQueue.Front() - if l == nil { - return nil - } - - layer := l.Value.(*pb.Layer) - s.updateLayer(layer) - - s.layersQueue.Remove(l) - - metricLastProcessedLayer.Set(float64(layer.Number.Number)) - metricLayersQueueLen.Set(float64(s.layersQueue.Len())) - - return layer -} - -func (s *Storage) requestBalanceUpdate(layer uint32, address string) { - s.accountsLock.Lock() - accounts, ok := s.accountsQueue[layer] - if !ok { - accounts = make(map[string]bool) - s.accountsQueue[layer] = accounts - } - accounts[address] = true - s.accountsLock.Unlock() - s.accountsReady.Signal() -} - -func (s *Storage) getAccountsQueue(accounts map[string]bool) int { - s.accountsLock.Lock() - defer s.accountsLock.Unlock() - for layer, accs := range s.accountsQueue { - if layer <= s.NetworkInfo.LastConfirmedLayer { - for acc := range accs { - accounts[acc] = true - } - } - delete(s.accountsQueue, layer) - } - return len(accounts) -} - -func (s *Storage) getChangedEpoch() int32 { - s.Lock() - defer s.Unlock() - epoch := s.changedEpoch - if s.changedEpoch >= 0 { - s.changedEpoch = -1 - } - return epoch -} - -func (s *Storage) setChangedEpoch(layer uint32) { - s.Lock() - defer s.Unlock() - if s.NetworkInfo.EpochNumLayers > 0 { - epoch := int32(layer / s.NetworkInfo.EpochNumLayers) - if s.changedEpoch < 0 || s.changedEpoch > epoch { - s.changedEpoch = epoch - } - if epoch > s.lastEpoch { - s.lastEpoch = epoch - } - } -} - -func (s *Storage) updateLayer(in *pb.Layer) { - layer, blocks, atxs, txs := model.NewLayer(in, &s.NetworkInfo) - log.Info("updateLayer(%v) -> %v, %v, %v, %v, %v", in.Number.Number, layer.Number, len(blocks), len(atxs), len(txs), utils.BytesToHex(in.Hash)) - s.updateNetworkStatus(layer) - - err := s.SaveOrUpdateBlocks(context.Background(), blocks) - //TODO: better error handling - if err != nil { - log.Err(fmt.Errorf("updateLayer: error %v", err)) - } - - s.updateTransactions(layer, txs) - - err = s.SaveOrUpdateLayer(context.Background(), layer) - //TODO: better error handling - if err != nil { - log.Err(fmt.Errorf("updateLayer: error %v", err)) - } - - s.setChangedEpoch(layer.Number) - s.accountsReady.Signal() - s.updateEpochs() -} - -func (s *Storage) updateNetworkStatus(layer *model.Layer) { - s.NetworkInfo.LastLayer = layer.Number - s.NetworkInfo.LastLayerTimestamp = uint32(time.Now().Unix()) - if layer.Status == int(pb.Layer_LAYER_STATUS_APPROVED) { - s.NetworkInfo.LastApprovedLayer = layer.Number - } else if layer.Status == int(pb.Layer_LAYER_STATUS_CONFIRMED) { - s.NetworkInfo.LastConfirmedLayer = layer.Number - } - - err := s.SaveOrUpdateNetworkInfo(context.Background(), &s.NetworkInfo) - //TODO: better error handling - if err != nil { - log.Err(fmt.Errorf("updateNetworkStatus: error %v", err)) - } -} - -func (s *Storage) OnActivation(atx *types.ActivationTx) { - log.Info("OnActivation(%s)", atx.ShortString()) - - activation := model.NewActivation(atx) - - err := s.SaveOrUpdateActivation(context.Background(), activation) - if err != nil { - log.Err(fmt.Errorf("OnActivation: error %v", err)) - } - - err = s.UpdateSmesher(context.Background(), activation.GetSmesher(s.postUnitSize), activation.TargetEpoch) - if err != nil { - log.Err(fmt.Errorf("OnActivation: update smesher error %v", err)) - } - - epochNumLayers := s.GetEpochNumLayers() - err = s.AddAccount(context.Background(), epochNumLayers*activation.PublishEpoch, activation.Coinbase, 0) - //TODO: better error handling - if err != nil { - log.Err(fmt.Errorf("updateActivations: error %v", err)) - } -} - -func (s *Storage) OnActivations(atxs []*model.Activation) { - log.Info("OnActivations(%d)", len(atxs)) - - err := s.SaveOrUpdateActivations(context.Background(), atxs) - if err != nil { - log.Err(fmt.Errorf("OnActivation: error %v", err)) - } - - epochNumLayers := s.GetEpochNumLayers() - - var coinbaseUpdateOps []mongo.WriteModel - var smesherUpdateOps []mongo.WriteModel - var accountsUpdateOps []mongo.WriteModel - - for _, atx := range atxs { - smesherUpdateOps = append(smesherUpdateOps, s.SaveSmesherQuery(atx.GetSmesher(s.postUnitSize))) - coinbaseOp, smesherOp := s.UpdateSmesherQuery(atx.GetSmesher(s.postUnitSize), atx.TargetEpoch) - coinbaseUpdateOps = append(coinbaseUpdateOps, coinbaseOp) - smesherUpdateOps = append(smesherUpdateOps, smesherOp) - accountsUpdateOps = append(accountsUpdateOps, s.AddAccountQuery(epochNumLayers*atx.PublishEpoch, atx.Coinbase, 0)) - } - - if len(smesherUpdateOps) > 0 { - _, err = s.db.Collection("smeshers").BulkWrite(context.TODO(), smesherUpdateOps) - if err != nil { - log.Err(fmt.Errorf("OnActivations: error smeshers write %v", err)) - } - } - - if len(coinbaseUpdateOps) > 0 { - _, err = s.db.Collection("coinbases").BulkWrite(context.TODO(), coinbaseUpdateOps) - if err != nil { - log.Err(fmt.Errorf("OnActivations: error smeshers write %v", err)) - } - } - - if len(accountsUpdateOps) > 0 { - _, err = s.db.Collection("accounts").BulkWrite(context.TODO(), accountsUpdateOps) - if err != nil { - log.Err(fmt.Errorf("OnActivations: error accounts write %v", err)) - } - } -} - -func (s *Storage) updateTransactions(layer *model.Layer, txs map[string]*model.Transaction) { - log.Info("updateTransactions") - for _, tx := range txs { - err := s.SaveTransaction(context.Background(), tx) - if err != nil { - continue - } - - if tx.Sender != "" { - err := s.AddAccount(context.Background(), layer.Number, tx.Sender, 0) - //TODO: better error handling - if err != nil { - log.Err(fmt.Errorf("updateTransactions: error %v", err)) - } - - err = s.AddAccountSent(context.Background(), layer.Number, tx.Sender, tx.Amount, tx.Fee) - //TODO: better error handling - if err != nil { - log.Err(fmt.Errorf("updateTransactions: error %v", err)) - } - - s.requestBalanceUpdate(layer.Number, tx.Sender) - } - if tx.Receiver != "" { - err := s.AddAccount(context.Background(), layer.Number, tx.Receiver, 0) - //TODO: better error handling - if err != nil { - log.Err(fmt.Errorf("updateTransactions: error %v", err)) - } - - err = s.AddAccountReceived(context.Background(), layer.Number, tx.Receiver, tx.Amount) - //TODO: better error handling - if err != nil { - log.Err(fmt.Errorf("updateTransactions: error %v", err)) - } - s.requestBalanceUpdate(layer.Number, tx.Receiver) - } - } -} - -func (s *Storage) updateEpoch(epochNumber int32, prev *model.Epoch) *model.Epoch { - log.Info("updateEpoch(%v)", epochNumber) - epoch := &model.Epoch{Number: epochNumber} - s.computeStatistics(epoch) - if prev != nil { - epoch.Stats.Cumulative.Capacity = epoch.Stats.Current.Capacity - epoch.Stats.Cumulative.Decentral = prev.Stats.Current.Decentral - // epoch.Stats.Cumulative.Smeshers = prev.Stats.Current.Smeshers - epoch.Stats.Cumulative.Smeshers = epoch.Stats.Current.Smeshers - epoch.Stats.Cumulative.Transactions = prev.Stats.Cumulative.Transactions + epoch.Stats.Current.Transactions - epoch.Stats.Cumulative.Accounts = epoch.Stats.Current.Accounts - epoch.Stats.Cumulative.Rewards = prev.Stats.Cumulative.Rewards + epoch.Stats.Current.Rewards - epoch.Stats.Cumulative.RewardsNumber = prev.Stats.Cumulative.RewardsNumber + epoch.Stats.Current.RewardsNumber - epoch.Stats.Cumulative.Security = prev.Stats.Current.Security - epoch.Stats.Cumulative.TxsAmount = prev.Stats.Cumulative.TxsAmount + epoch.Stats.Current.TxsAmount - epoch.Stats.Current.Circulation = epoch.Stats.Cumulative.Rewards - epoch.Stats.Cumulative.Circulation = epoch.Stats.Current.Circulation - } else { - epoch.Stats.Current.Circulation = epoch.Stats.Current.Rewards - epoch.Stats.Cumulative = epoch.Stats.Current - } - err := s.SaveOrUpdateEpoch(context.Background(), epoch) - //TODO: better error handling - if err != nil { - log.Err(fmt.Errorf("updateEpoch: error %v", err)) - } - - return epoch -} - -func (s *Storage) updateEpochs() { - epochNumber := s.getChangedEpoch() - if epochNumber >= 0 { - var prev *model.Epoch - if epochNumber > 0 { - prev, _ = s.GetEpochByNumber(context.Background(), epochNumber-1) - } - for i := epochNumber; i <= s.lastEpoch; i++ { - prev = s.updateEpoch(i, prev) - } - } -} - -func (s *Storage) updateAccount(address string) { - balance, counter, err := s.AccountUpdater.GetAccountState(address) - if err != nil { - return - } - log.Info("Update account %v: balance %v, counter %v", address, balance, counter) - - err = s.UpdateAccount(context.Background(), address, balance, counter) - //TODO: better error handling - if err != nil { - log.Err(fmt.Errorf("updateEpoch: error %v", err)) - } -} - -func (s *Storage) updateLayers() { - for { - s.layersReady.L.Lock() - s.layersReady.Wait() - s.layersReady.L.Unlock() - - for s.processLayer() != nil { - log.Info("processing layer") - } - } -} - -func (s *Storage) updateAccounts() { - for { - s.accountsReady.L.Lock() - s.accountsReady.Wait() - s.accountsReady.L.Unlock() - - accounts := make(map[string]bool) - if s.getAccountsQueue(accounts) > 0 { - for address := range accounts { - s.updateAccount(address) - } - } - } -} - -func (s *Storage) updateMalfeasanceProof(in *pb.MalfeasanceProof) { - proof := model.NewMalfeasanceProof(in) - if proof == nil { - return - } - - log.Info("updateMalfeasanceProof -> %v, %v, %v", proof.Layer, proof.Smesher, proof.Kind) - - err := s.SaveMalfeasanceProof(context.Background(), proof) - if err != nil { - log.Err(fmt.Errorf("updateMalfeasanceProof: %v", err)) - } -} - -func (s *Storage) GetEpochLayersFilter(epochNumber int32, key string) *bson.D { - layerStart, layerEnd := s.GetEpochLayers(epochNumber) - return &bson.D{{Key: key, Value: bson.D{{Key: "$gte", Value: layerStart}, {Key: "$lte", Value: layerEnd}}}} -} - -func (s *Storage) getLayerTimestamp(layer uint32) uint32 { - if layer == 0 { - return s.NetworkInfo.GenesisTime - } - return s.NetworkInfo.GenesisTime + layer*s.NetworkInfo.LayerDuration -} - -func (s *Storage) Ping() error { - if s.client == nil { - return errors.New("Storage not initialized") - } - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - - return s.client.Ping(ctx, nil) -} - -func (s *Storage) LayersInQueue() int { - s.layersLock.Lock() - defer s.layersLock.Unlock() - return s.layersQueue.Len() -} diff --git a/storage/tx.go b/storage/tx.go deleted file mode 100644 index a110012..0000000 --- a/storage/tx.go +++ /dev/null @@ -1,341 +0,0 @@ -package storage - -import ( - "context" - "errors" - "time" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - - "github.com/spacemeshos/go-spacemesh/log" - - "github.com/spacemeshos/explorer-backend/model" - "github.com/spacemeshos/explorer-backend/utils" -) - -func (s *Storage) InitTransactionsStorage(ctx context.Context) error { - models := []mongo.IndexModel{ - {Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index().SetName("idIndex").SetUnique(true)}, - {Keys: bson.D{{Key: "layer", Value: 1}}, Options: options.Index().SetName("layerIndex").SetUnique(false)}, - {Keys: bson.D{{Key: "block", Value: 1}}, Options: options.Index().SetName("blockIndex").SetUnique(false)}, - {Keys: bson.D{{Key: "sender", Value: 1}}, Options: options.Index().SetName("senderIndex").SetUnique(false)}, - {Keys: bson.D{{Key: "receiver", Value: 1}}, Options: options.Index().SetName("receiverIndex").SetUnique(false)}, - {Keys: bson.D{{Key: "timestamp", Value: -1}}, Options: options.Index().SetName("timestampIndex").SetUnique(false)}, - {Keys: bson.D{{Key: "counter", Value: -1}}, Options: options.Index().SetName("counterIndex").SetUnique(false)}, - } - _, err := s.db.Collection("txs").Indexes().CreateMany(ctx, models, options.CreateIndexes().SetMaxTime(20*time.Second)) - return err -} - -func (s *Storage) GetTransaction(parent context.Context, query *bson.D) (*model.Transaction, error) { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - cursor, err := s.db.Collection("txs").Find(ctx, query) - if err != nil { - log.Info("GetTransaction: %v", err) - return nil, err - } - if !cursor.Next(ctx) { - log.Info("GetTransaction: Empty result") - return nil, errors.New("empty result") - } - - doc := cursor.Current - - var signatures []model.SignaturePart - if doc.Lookup("signatures").Type == bson.TypeArray { - sigsArray, err := doc.Lookup("signatures").Array().Values() - if err != nil { - return nil, err - } - for _, sig := range sigsArray { - signatures = append(signatures, model.SignaturePart{ - Ref: utils.GetAsUInt32(sig.Document().Lookup("ref")), - Signature: utils.GetAsString(sig.Document().Lookup("signature")), - }) - } - } - - tx := &model.Transaction{ - Id: utils.GetAsString(doc.Lookup("id")), - Layer: utils.GetAsUInt32(doc.Lookup("layer")), - Block: utils.GetAsString(doc.Lookup("block")), - BlockIndex: utils.GetAsUInt32(doc.Lookup("blockIndex")), - Index: utils.GetAsUInt32(doc.Lookup("index")), - State: utils.GetAsInt(doc.Lookup("state")), - Timestamp: utils.GetAsUInt32(doc.Lookup("timestamp")), - MaxGas: utils.GetAsUInt64(doc.Lookup("maxGas")), - GasPrice: utils.GetAsUInt64(doc.Lookup("gasPrice")), - GasUsed: utils.GetAsUInt64(doc.Lookup("gasUsed")), - Fee: utils.GetAsUInt64(doc.Lookup("fee")), - Amount: utils.GetAsUInt64(doc.Lookup("amount")), - Counter: utils.GetAsUInt64(doc.Lookup("counter")), - Type: utils.GetAsInt(doc.Lookup("type")), - Signature: utils.GetAsString(doc.Lookup("signature")), - Signatures: signatures, - PublicKey: utils.GetAsString(doc.Lookup("pubKey")), - Sender: utils.GetAsString(doc.Lookup("sender")), - Receiver: utils.GetAsString(doc.Lookup("receiver")), - SvmData: utils.GetAsString(doc.Lookup("svmData")), - Vault: utils.GetAsString(doc.Lookup("vault")), - VaultOwner: utils.GetAsString(doc.Lookup("vaultOwner")), - VaultTotalAmount: utils.GetAsUInt64(doc.Lookup("vaultTotalAmount")), - VaultInitialUnlockAmount: utils.GetAsUInt64(doc.Lookup("vaultInitialUnlockAmount")), - VaultVestingStart: utils.GetAsUInt32(doc.Lookup("vaultVestingStart")), - VaultVestingEnd: utils.GetAsUInt32(doc.Lookup("vaultVestingEnd")), - } - return tx, nil -} - -func (s *Storage) GetTransactionsCount(parent context.Context, query *bson.D, opts ...*options.CountOptions) int64 { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - count, err := s.db.Collection("txs").CountDocuments(ctx, query, opts...) - if err != nil { - log.Info("GetTransactionsCount: %v", err) - return 0 - } - return count -} - -func (s *Storage) GetTransactionsAmount(parent context.Context, query *bson.D) int64 { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - matchStage := bson.D{ - {Key: "$match", Value: query}, - } - groupStage := bson.D{ - {Key: "$group", Value: bson.D{ - {Key: "_id", Value: ""}, - {Key: "amount", Value: bson.D{ - {Key: "$sum", Value: "$amount"}, - }}, - }}, - } - cursor, err := s.db.Collection("txs").Aggregate(ctx, mongo.Pipeline{ - matchStage, - groupStage, - }) - if err != nil { - log.Info("GetTransactionsAmount: %v", err) - return 0 - } - if !cursor.Next(ctx) { - log.Info("GetTransactionsAmount: Empty result") - return 0 - } - doc := cursor.Current - return utils.GetAsInt64(doc.Lookup("amount")) -} - -func (s *Storage) IsTransactionExists(parent context.Context, txId string) bool { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - count, err := s.db.Collection("txs").CountDocuments(ctx, bson.D{{Key: "id", Value: txId}}) - if err != nil { - log.Info("IsTransactionExists: %v", err) - return false - } - return count > 0 -} - -func (s *Storage) GetTransactions(parent context.Context, query *bson.D, opts ...*options.FindOptions) ([]model.Transaction, error) { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - cursor, err := s.db.Collection("txs").Find(ctx, query, opts...) - if err != nil { - log.Info("GetTransactions: %v", err) - return nil, err - } - var txs []model.Transaction - err = cursor.All(ctx, &txs) - if err != nil { - log.Info("GetTransactions: %v", err) - return nil, err - } - if len(txs) == 0 { - return nil, nil - } - return txs, nil -} - -func (s *Storage) SaveTransaction(parent context.Context, in *model.Transaction) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - - transaction, err := s.GetTransaction(ctx, &bson.D{{Key: "id", Value: in.Id}}) - if err != nil && err.Error() != "empty result" { - return err - } - - tx := bson.D{ - { - Key: "$set", - Value: bson.D{ - {Key: "id", Value: in.Id}, - {Key: "layer", Value: in.Layer}, - {Key: "block", Value: in.Block}, - {Key: "blockIndex", Value: in.BlockIndex}, - {Key: "index", Value: in.Index}, - {Key: "state", Value: in.State}, - {Key: "timestamp", Value: in.Timestamp}, - {Key: "maxGas", Value: in.MaxGas}, - {Key: "gasPrice", Value: in.GasPrice}, - {Key: "gasUsed", Value: in.GasUsed}, - {Key: "fee", Value: in.Fee}, - {Key: "amount", Value: in.Amount}, - {Key: "counter", Value: in.Counter}, - {Key: "type", Value: in.Type}, - {Key: "signature", Value: in.Signature}, - {Key: "signatures", Value: in.Signatures}, - {Key: "pubKey", Value: in.PublicKey}, - {Key: "sender", Value: in.Sender}, - {Key: "receiver", Value: in.Receiver}, - {Key: "svmData", Value: in.SvmData}, - {Key: "message", Value: in.Message}, - {Key: "touchedAddresses", Value: in.TouchedAddresses}, - {Key: "vault", Value: in.Vault}, - {Key: "vaultOwner", Value: in.VaultOwner}, - {Key: "vaultTotalAmount", Value: in.VaultTotalAmount}, - {Key: "vaultInitialUnlockAmount", Value: in.VaultInitialUnlockAmount}, - {Key: "vaultVestingStart", Value: in.VaultVestingStart}, - {Key: "vaultVestingEnd", Value: in.VaultVestingEnd}, - }, - }, - } - - if transaction != nil { - tx = bson.D{ - { - Key: "$set", - Value: bson.D{ - {Key: "id", Value: in.Id}, - {Key: "layer", Value: in.Layer}, - {Key: "block", Value: in.Block}, - {Key: "blockIndex", Value: in.BlockIndex}, - {Key: "index", Value: in.Index}, - {Key: "timestamp", Value: in.Timestamp}, - {Key: "maxGas", Value: in.MaxGas}, - {Key: "gasPrice", Value: in.GasPrice}, - {Key: "fee", Value: in.Fee}, - {Key: "amount", Value: in.Amount}, - {Key: "counter", Value: in.Counter}, - {Key: "type", Value: in.Type}, - {Key: "signature", Value: in.Signature}, - {Key: "signatures", Value: in.Signatures}, - {Key: "pubKey", Value: in.PublicKey}, - {Key: "sender", Value: in.Sender}, - {Key: "receiver", Value: in.Receiver}, - {Key: "svmData", Value: in.SvmData}, - {Key: "vault", Value: in.Vault}, - {Key: "vaultOwner", Value: in.VaultOwner}, - {Key: "vaultTotalAmount", Value: in.VaultTotalAmount}, - {Key: "vaultInitialUnlockAmount", Value: in.VaultInitialUnlockAmount}, - {Key: "vaultVestingStart", Value: in.VaultVestingStart}, - {Key: "vaultVestingEnd", Value: in.VaultVestingEnd}, - }, - }, - } - } - - _, err = s.db.Collection("txs").UpdateOne(ctx, - bson.D{{Key: "id", Value: in.Id}}, tx, options.Update().SetUpsert(true)) - if err != nil { - log.Info("SaveTransaction: %v obj: %+v", err, tx) - } - return err -} - -func (s *Storage) SaveTransactionResult(parent context.Context, in *model.Transaction) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - - transaction, err := s.GetTransaction(ctx, &bson.D{{Key: "id", Value: in.Id}}) - if err != nil && err.Error() != "empty result" { - return err - } - - tx := bson.D{ - { - Key: "$set", - Value: bson.D{ - {Key: "id", Value: in.Id}, - {Key: "layer", Value: in.Layer}, - {Key: "block", Value: in.Block}, - {Key: "blockIndex", Value: in.BlockIndex}, - {Key: "index", Value: in.Index}, - {Key: "state", Value: in.State}, - {Key: "timestamp", Value: in.Timestamp}, - {Key: "maxGas", Value: in.MaxGas}, - {Key: "gasPrice", Value: in.GasPrice}, - {Key: "gasUsed", Value: in.GasUsed}, - {Key: "fee", Value: in.Fee}, - {Key: "amount", Value: in.Amount}, - {Key: "counter", Value: in.Counter}, - {Key: "type", Value: in.Type}, - {Key: "signature", Value: in.Signature}, - {Key: "signatures", Value: in.Signatures}, - {Key: "pubKey", Value: in.PublicKey}, - {Key: "sender", Value: in.Sender}, - {Key: "receiver", Value: in.Receiver}, - {Key: "svmData", Value: in.SvmData}, - {Key: "message", Value: in.Message}, - {Key: "touchedAddresses", Value: in.TouchedAddresses}, - {Key: "result", Value: in.Result}, - {Key: "vault", Value: in.Vault}, - {Key: "vaultOwner", Value: in.VaultOwner}, - {Key: "vaultTotalAmount", Value: in.VaultTotalAmount}, - {Key: "vaultInitialUnlockAmount", Value: in.VaultInitialUnlockAmount}, - {Key: "vaultVestingStart", Value: in.VaultVestingStart}, - {Key: "vaultVestingEnd", Value: in.VaultVestingEnd}, - }, - }, - } - - if transaction != nil { - tx = bson.D{ - { - Key: "$set", - Value: bson.D{ - {Key: "id", Value: in.Id}, - {Key: "state", Value: in.State}, - {Key: "gasUsed", Value: in.GasUsed}, - {Key: "fee", Value: in.Fee}, - {Key: "message", Value: in.Message}, - {Key: "touchedAddresses", Value: in.TouchedAddresses}, - {Key: "result", Value: in.Result}, - }, - }, - } - } - - _, err = s.db.Collection("txs").UpdateOne(ctx, - bson.D{{Key: "id", Value: in.Id}}, tx, options.Update().SetUpsert(true)) - if err != nil { - log.Info("SaveTransactionResult: %v obj: %+v", err, tx) - } - return err -} - -func (s *Storage) UpdateTransactionState(parent context.Context, id string, state int32) error { - ctx, cancel := context.WithTimeout(parent, 5*time.Second) - defer cancel() - - tx := bson.D{ - { - Key: "$set", - Value: bson.D{ - {Key: "state", Value: state}, - }, - }, - } - - _, err := s.db.Collection("txs").UpdateOne(ctx, - bson.D{{Key: "id", Value: id}}, tx) - if err != nil { - log.Info("UpdateTransactionState: %v obj: %+v", err, tx) - } - return err -} diff --git a/test/testseed/account.go b/test/testseed/account.go deleted file mode 100644 index 69a948c..0000000 --- a/test/testseed/account.go +++ /dev/null @@ -1,39 +0,0 @@ -package testseed - -import ( - "strings" - - "github.com/spacemeshos/go-spacemesh/signing" - - "github.com/spacemeshos/explorer-backend/model" -) - -// AccountContainer is a container for accounts with transactions and rewards belongs to generated account. -type AccountContainer struct { - layerID uint32 - Account model.Account - Signer *signing.EdSigner - Transactions map[string]*model.Transaction - Rewards map[string]*model.Reward -} - -func (s *SeedGenerator) saveTransactionForAccount(tx *model.Transaction, accountFrom, accountTo string) { - s.Accounts[strings.ToLower(accountFrom)].Transactions[tx.Id] = tx - s.Accounts[strings.ToLower(accountTo)].Transactions[tx.Id] = tx - - tmpContainerFrom := s.Accounts[strings.ToLower(accountFrom)] - tmpContainerFrom.Account.Txs++ - s.Accounts[strings.ToLower(accountFrom)] = tmpContainerFrom - - if accountTo == accountFrom { - return - } - - tmpContainerTo := s.Accounts[strings.ToLower(accountTo)] - tmpContainerTo.Account.Txs++ - s.Accounts[strings.ToLower(accountTo)] = tmpContainerTo -} - -func (s *SeedGenerator) saveReward(reward *model.Reward) { - s.Accounts[strings.ToLower(reward.Coinbase)].Rewards[reward.Smesher] = reward -} diff --git a/test/testseed/db.go b/test/testseed/db.go deleted file mode 100644 index 2223c3c..0000000 --- a/test/testseed/db.go +++ /dev/null @@ -1,220 +0,0 @@ -package testseed - -import ( - "errors" - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" - "github.com/spacemeshos/explorer-backend/utils" - "github.com/spacemeshos/go-spacemesh/common/types" - "github.com/spacemeshos/go-spacemesh/genvm/sdk" - sdkWallet "github.com/spacemeshos/go-spacemesh/genvm/sdk/wallet" - "github.com/spacemeshos/go-spacemesh/genvm/templates/wallet" - "github.com/spacemeshos/go-spacemesh/sql" - "strings" - "time" -) - -type Client struct { - SeedGen *SeedGenerator -} - -const ( - methodSend = 16 -) - -func (c *Client) GetLayer(db *sql.Database, lid types.LayerID, numLayers uint32) (*pb.Layer, error) { - for _, epoch := range c.SeedGen.Epochs { - for _, layerContainer := range epoch.Layers { - if layerContainer.Layer.Number != lid.Uint32() { - continue - } - atx := make([]*pb.Activation, 0, len(layerContainer.Activations)) - for _, atxGenerated := range layerContainer.Activations { - smesherId, _ := utils.StringToBytes(atxGenerated.SmesherId) - atx = append(atx, &pb.Activation{ - Id: &pb.ActivationId{Id: mustParse(atxGenerated.Id)}, - SmesherId: &pb.SmesherId{Id: smesherId}, - Coinbase: &pb.AccountId{Address: atxGenerated.Coinbase}, - PrevAtx: &pb.ActivationId{Id: mustParse(atxGenerated.PrevAtx)}, - Layer: &pb.LayerNumber{Number: atxGenerated.TargetEpoch / numLayers}, - NumUnits: atxGenerated.NumUnits, - }) - } - - blocksRes := make([]*pb.Block, 0) - for _, blockContainer := range layerContainer.Blocks { - tx := make([]*pb.Transaction, 0, len(blockContainer.Transactions)) - for _, txContainer := range blockContainer.Transactions { - receiver, err := types.StringToAddress(txContainer.Receiver) - if err != nil { - panic("invalid receiver address: " + err.Error()) - } - signer := c.SeedGen.Accounts[strings.ToLower(txContainer.Sender)].Signer - tx = append(tx, &pb.Transaction{ - Id: mustParse(txContainer.Id), - Method: methodSend, - Principal: &pb.AccountId{ - Address: txContainer.Sender, - }, - GasPrice: txContainer.GasPrice, - MaxGas: txContainer.MaxGas, - Nonce: &pb.Nonce{ - Counter: txContainer.Counter, - }, - Template: &pb.AccountId{ - Address: wallet.TemplateAddress.String(), - }, - Raw: sdkWallet.Spend(signer.PrivateKey(), receiver, txContainer.Amount, types.Nonce(txContainer.Counter), sdk.WithGasPrice(txContainer.GasPrice)), - }) - } - smesherId, _ := utils.StringToBytes(blockContainer.SmesherID) - blocksRes = append(blocksRes, &pb.Block{ - Id: mustParse(blockContainer.Block.Id), - Transactions: tx, - SmesherId: &pb.SmesherId{ - Id: smesherId, - }, - }) - } - pbLayer := &pb.Layer{ - Number: &pb.LayerNumber{Number: layerContainer.Layer.Number}, - Status: pb.Layer_LayerStatus(layerContainer.Layer.Status), - Hash: mustParse(layerContainer.Layer.Hash), - Blocks: blocksRes, - Activations: atx, - } - - return pbLayer, nil - } - } - - return nil, errors.New("could not find layer") -} - -func (c *Client) GetLayerRewards(db *sql.Database, lid types.LayerID) (rst []*types.Reward, err error) { - for _, epoch := range c.SeedGen.Epochs { - for _, reward := range epoch.Rewards { - if reward.Layer != lid.Uint32() { - continue - } - - coinbase, _ := types.StringToAddress(reward.Coinbase) - smesher, _ := utils.StringToBytes(reward.Smesher) - smesherId := types.BytesToNodeID(smesher) - - r := &types.Reward{ - Layer: types.LayerID(reward.Layer), - TotalReward: reward.Total, - LayerReward: reward.LayerReward, - Coinbase: coinbase, - SmesherID: smesherId, - } - - rst = append(rst, r) - } - } - - return rst, nil -} - -func (c *Client) GetAllRewards(db *sql.Database) (rst []*types.Reward, err error) { - for _, epoch := range c.SeedGen.Epochs { - for _, reward := range epoch.Rewards { - coinbase, _ := types.StringToAddress(reward.Coinbase) - smesher, _ := utils.StringToBytes(reward.Smesher) - smesherId := types.BytesToNodeID(smesher) - - r := &types.Reward{ - Layer: types.LayerID(reward.Layer), - TotalReward: reward.Total, - LayerReward: reward.LayerReward, - Coinbase: coinbase, - SmesherID: smesherId, - } - - rst = append(rst, r) - } - } - - return rst, nil -} - -func (c *Client) AccountsSnapshot(db *sql.Database, lid types.LayerID) (rst []*types.Account, err error) { - for _, accountContainer := range c.SeedGen.Accounts { - if accountContainer.layerID != lid.Uint32() { - continue - } - - accAddr, _ := utils.StringToBytes(accountContainer.Account.Address) - var addr types.Address - copy(addr[:], accAddr) - - rst = append(rst, &types.Account{ - Layer: types.LayerID(accountContainer.layerID), - Address: addr, - Balance: accountContainer.Account.Balance, - }) - } - - return rst, nil -} - -func (c *Client) GetAtxsReceivedAfter(db *sql.Database, ts int64, fn func(tx *types.ActivationTx) bool) error { - for _, generatedAtx := range c.SeedGen.Activations { - smesherIdBytes, _ := utils.StringToBytes(generatedAtx.SmesherId) - var nodeId types.NodeID - copy(nodeId[:], smesherIdBytes) - - addr, err := types.StringToAddress(generatedAtx.Coinbase) - if err != nil { - return err - } - - prevAtxBytes, _ := utils.StringToBytes(generatedAtx.PrevAtx) - var prevAtx types.ATXID - copy(prevAtx[:], prevAtxBytes) - - atxIdBytes, _ := utils.StringToBytes(generatedAtx.Id) - var atxId types.ATXID - copy(atxId[:], atxIdBytes) - - atx := &types.ActivationTx{ - PrevATXID: prevAtx, - PublishEpoch: types.EpochID(generatedAtx.PublishEpoch), - Sequence: 1, - Coinbase: addr, - NumUnits: generatedAtx.NumUnits, - SmesherID: nodeId, - } - - atx.NumUnits = generatedAtx.NumUnits - atx.SetID(atxId) - atx.SetReceived(time.Unix(0, generatedAtx.Received)) - - fn(atx) - } - return nil -} - -func (c *Client) GetAtxsByEpoch(db *sql.Database, epoch int64, fn func(tx *types.ActivationTx) bool) error { - return nil -} - -func (c *Client) CountAtxsByEpoch(db *sql.Database, epoch int64) (int, error) { - return 0, nil -} - -func (c *Client) GetAtxsByEpochPaginated(db *sql.Database, epoch, limit, offset int64, fn func(tx *types.ActivationTx) bool) error { - return nil -} - -func (c *Client) GetAtxById(db *sql.Database, id string) (*types.ActivationTx, error) { - return nil, nil -} - -func mustParse(str string) []byte { - res, err := utils.StringToBytes(str) - if err != nil { - panic("error while parse string to bytes: " + err.Error()) - } - return res -} diff --git a/test/testseed/epoch.go b/test/testseed/epoch.go deleted file mode 100644 index 75f798a..0000000 --- a/test/testseed/epoch.go +++ /dev/null @@ -1,103 +0,0 @@ -package testseed - -import ( - "sort" - - "github.com/spacemeshos/explorer-backend/model" -) - -// TestServerSeed test network config for tests. -type TestServerSeed struct { - GenesisID []byte - EpochNumLayers uint32 - LayersDuration uint64 - MaxTransactionPerSecond uint64 - GenesisTime uint64 - - BitsPerLabel uint32 - LabelsPerUnit uint64 - MinNumUnits uint32 - MaxNumUnits uint32 -} - -// GetPostUnitsSize calcluates size of post units. -func (t *TestServerSeed) GetPostUnitsSize() uint64 { - return (uint64(t.BitsPerLabel) * t.LabelsPerUnit) / 8 -} - -// GetServerSeed generate test network config. -func GetServerSeed() *TestServerSeed { - return &TestServerSeed{ - GenesisTime: 1234567, - GenesisID: []byte("genesisid"), - EpochNumLayers: 10, - LayersDuration: 10, - MaxTransactionPerSecond: 100, - BitsPerLabel: 100, - LabelsPerUnit: 200, - MinNumUnits: 20, - MaxNumUnits: 2000, - } -} - -// SeedEpoch generated epoch for tests. -type SeedEpoch struct { - Epoch model.Epoch - Layers []*LayerContainer - Transactions map[string]*model.Transaction - Rewards map[string]*model.Reward - Blocks map[string]*model.Block - Smeshers map[string]*model.Smesher - SmeshersCommitment map[string]uint64 - Activations map[string]*model.Activation -} - -// SeedEpochs wrapper over generated slice of epochs. -type SeedEpochs []*SeedEpoch - -// GetTransactions extract all transactions from epochs. -func (s SeedEpochs) GetTransactions() map[string]*model.Transaction { - result := make(map[string]*model.Transaction, 0) - for _, epoch := range s { - for _, tx := range epoch.Transactions { - result[tx.Id] = tx - } - } - return result -} - -// GetLayers extract all layers from epochs. -func (s SeedEpochs) GetLayers() []model.Layer { - result := make([]model.Layer, 0) - for _, epoch := range s { - for _, layer := range epoch.Layers { - result = append(result, layer.Layer) - } - } - sort.Slice(result, func(i, j int) bool { - return result[i].Number > result[j].Number - }) - return result -} - -// GetActivations extract all activations from epochs. -func (s SeedEpochs) GetActivations() map[string]*model.Activation { - result := make(map[string]*model.Activation, 0) - for _, epoch := range s { - for _, activation := range epoch.Activations { - result[activation.Id] = activation - } - } - return result -} - -// GetRewards extract all rewards from epochs. -func (s SeedEpochs) GetRewards() map[string]*model.Reward { - result := make(map[string]*model.Reward, 0) - for _, epoch := range s { - for _, reward := range epoch.Rewards { - result[reward.Smesher] = reward - } - } - return result -} diff --git a/test/testseed/generator.go b/test/testseed/generator.go deleted file mode 100644 index 30a0d99..0000000 --- a/test/testseed/generator.go +++ /dev/null @@ -1,406 +0,0 @@ -package testseed - -import ( - "context" - "crypto/sha256" - "fmt" - "github.com/spacemeshos/go-spacemesh/genvm/templates/wallet" - "math/rand" - "strings" - "time" - - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" - "github.com/spacemeshos/go-spacemesh/signing" - - "github.com/spacemeshos/explorer-backend/model" - v0 "github.com/spacemeshos/explorer-backend/pkg/transactionparser/v0" - "github.com/spacemeshos/explorer-backend/storage" - "github.com/spacemeshos/explorer-backend/utils" -) - -// SeedGenerator helper for generate epochs. -type SeedGenerator struct { - Epochs SeedEpochs - Accounts map[string]AccountContainer - Activations map[string]*model.Activation - Blocks map[string]*model.Block - Apps map[string]model.App - Layers map[uint32]*model.Layer - Rewards map[string]*model.Reward - Transactions map[string]*model.Transaction - Smeshers map[string]*model.Smesher - FirstLayerTime time.Time - seed *TestServerSeed -} - -// NewSeedGenerator create object which allow fill database for tests. -func NewSeedGenerator(seed *TestServerSeed) *SeedGenerator { - return &SeedGenerator{ - Epochs: make(SeedEpochs, 0), - Accounts: make(map[string]AccountContainer, 0), - Apps: map[string]model.App{}, - seed: seed, - Activations: map[string]*model.Activation{}, - Blocks: map[string]*model.Block{}, - Layers: map[uint32]*model.Layer{}, - Rewards: map[string]*model.Reward{}, - Smeshers: map[string]*model.Smesher{}, - Transactions: map[string]*model.Transaction{}, - } -} - -// GenerateEpoches generate epochs for test. -func (s *SeedGenerator) GenerateEpoches(count int) error { - now := time.Now() - result := make([]*SeedEpoch, 0, count) - var prevEpoch *model.Epoch - for i := 1; i < count; i++ { - offset := time.Duration(int64(i)*(int64(s.seed.EpochNumLayers)*int64(s.seed.LayersDuration))) * time.Second - layerStartDate := now.Add(-1 * offset) - layersStart := int32(i) * int32(s.seed.EpochNumLayers) - layersEnd := layersStart + int32(s.seed.EpochNumLayers) - 1 - if i == 1 { - s.FirstLayerTime = layerStartDate - } - - seedEpoch := &SeedEpoch{ - Epoch: s.generateEpoch(int32(i)), - Layers: make([]*LayerContainer, 0, s.seed.EpochNumLayers), - Transactions: map[string]*model.Transaction{}, - Rewards: map[string]*model.Reward{}, - Smeshers: map[string]*model.Smesher{}, - SmeshersCommitment: map[string]uint64{}, - Activations: map[string]*model.Activation{}, - Blocks: map[string]*model.Block{}, - } - - result = append(result, seedEpoch) - - layerStart := seedEpoch.Epoch.Number * int32(s.seed.EpochNumLayers) - for j := layerStart; j <= layersEnd; j++ { - if err := s.fillLayer(j, int32(i), seedEpoch); err != nil { - return fmt.Errorf("failed to fill layer: %v", err) - } - seedEpoch.Epoch.Layers++ - } - seedEpoch.Epoch.Stats.Current.Decentral = utils.CalcDecentralCoefficient(seedEpoch.SmeshersCommitment) - duration := float64(s.seed.LayersDuration) * float64(layersEnd-layerStart+1) - seedEpoch.Epoch.Stats.Current.Capacity = utils.CalcEpochCapacity(seedEpoch.Epoch.Stats.Current.Transactions, duration, uint32(s.seed.MaxTransactionPerSecond)) - if prevEpoch != nil { - seedEpoch.Epoch.Stats.Cumulative.Capacity = seedEpoch.Epoch.Stats.Current.Capacity - seedEpoch.Epoch.Stats.Cumulative.Decentral = prevEpoch.Stats.Current.Decentral - seedEpoch.Epoch.Stats.Cumulative.Smeshers = seedEpoch.Epoch.Stats.Current.Smeshers - seedEpoch.Epoch.Stats.Cumulative.Transactions = prevEpoch.Stats.Cumulative.Transactions + seedEpoch.Epoch.Stats.Current.Transactions - seedEpoch.Epoch.Stats.Cumulative.Accounts = seedEpoch.Epoch.Stats.Current.Accounts - seedEpoch.Epoch.Stats.Cumulative.Rewards = prevEpoch.Stats.Cumulative.Rewards + seedEpoch.Epoch.Stats.Current.Rewards - seedEpoch.Epoch.Stats.Cumulative.RewardsNumber = prevEpoch.Stats.Cumulative.RewardsNumber + seedEpoch.Epoch.Stats.Current.RewardsNumber - seedEpoch.Epoch.Stats.Cumulative.Security = prevEpoch.Stats.Current.Security - seedEpoch.Epoch.Stats.Cumulative.TxsAmount = prevEpoch.Stats.Cumulative.TxsAmount + seedEpoch.Epoch.Stats.Current.TxsAmount - - seedEpoch.Epoch.Stats.Current.Circulation = seedEpoch.Epoch.Stats.Cumulative.Rewards - seedEpoch.Epoch.Stats.Cumulative.Circulation = seedEpoch.Epoch.Stats.Current.Circulation - } else { - seedEpoch.Epoch.Stats.Current.Circulation = seedEpoch.Epoch.Stats.Current.Rewards - seedEpoch.Epoch.Stats.Cumulative = seedEpoch.Epoch.Stats.Current - } - prevEpoch = &seedEpoch.Epoch - } - s.Epochs = result - return nil -} - -// SaveEpoches write generated data directly to db. -func (s *SeedGenerator) SaveEpoches(ctx context.Context, db *storage.Storage) error { - for _, epoch := range s.Epochs { - if err := db.SaveEpoch(ctx, &epoch.Epoch); err != nil { - return fmt.Errorf("failed to save epoch: %v", err) - } - for _, layerContainer := range epoch.Layers { - if err := db.SaveLayer(ctx, &layerContainer.Layer); err != nil { - return fmt.Errorf("failed to save layer: %v", err) - } - } - for _, tx := range epoch.Transactions { - if err := db.SaveTransaction(ctx, tx); err != nil { - return fmt.Errorf("failed to save transaction: %v", err) - } - } - for _, reward := range epoch.Rewards { - if err := db.SaveReward(ctx, reward); err != nil { - return fmt.Errorf("failed to save reward: %v", err) - } - } - for _, smesher := range epoch.Smeshers { - if err := db.SaveSmesher(ctx, smesher, uint32(epoch.Epoch.Number)); err != nil { - return fmt.Errorf("failed to save smesher: %v", err) - } - } - for _, atx := range epoch.Activations { - if err := db.SaveActivation(ctx, atx); err != nil { - return fmt.Errorf("failed to save activation: %v", err) - } - } - for _, block := range epoch.Blocks { - if err := db.SaveBlock(ctx, block); err != nil { - return fmt.Errorf("failed to save block: %v", err) - } - } - } - for _, acc := range s.Accounts { - if err := db.SaveAccount(ctx, acc.layerID, &acc.Account); err != nil { - return fmt.Errorf("failed to save account: %s", err) - } - } - return nil -} - -func (s *SeedGenerator) fillLayer(layerID, epochID int32, seedEpoch *SeedEpoch) error { - tmpLayer := s.generateLayer(layerID, epochID) - layerContainer := &LayerContainer{ - Layer: tmpLayer, - Blocks: make([]*BlockContainer, 0), - Activations: map[string]*model.Activation{}, - Smeshers: map[string]*model.Smesher{}, - } - seedEpoch.Layers = append(seedEpoch.Layers, layerContainer) - - for k := 0; k <= rand.Intn(5); k++ { - tmpAcc, tmpAccSigner := s.generateAccount(tmpLayer.Number) - s.Accounts[strings.ToLower(tmpAcc.Address)] = AccountContainer{ - layerID: uint32(layerID), - Account: tmpAcc, - Signer: tmpAccSigner, - Transactions: map[string]*model.Transaction{}, - Rewards: map[string]*model.Reward{}, - } - tmpApp := model.App{ - Address: tmpAcc.Address, - } - s.Apps[tmpApp.Address] = tmpApp - - tmpBl := s.generateBlocks(int32(tmpLayer.Number), seedEpoch.Epoch.Number) - - s.Blocks[tmpBl.Id] = &tmpBl - blockContainer := &BlockContainer{ - Block: &tmpBl, - Transactions: make([]*model.Transaction, 0), - } - layerContainer.Blocks = append(layerContainer.Blocks, blockContainer) - layerContainer.Layer.BlocksNumber++ - - for i := 0; i < rand.Intn(3); i++ { - from := tmpAcc.Address - to := s.getRandomAcc() - tmpTx := generateTransaction(i, &tmpLayer, tmpAccSigner, from, to, &tmpBl) - - seedEpoch.Epoch.Stats.Current.Transactions++ - seedEpoch.Epoch.Stats.Current.TxsAmount += int64(tmpTx.Amount) - - layerContainer.Layer.Txs++ - layerContainer.Layer.TxsAmount += tmpTx.Amount - blockContainer.Block.TxsNumber++ - blockContainer.Block.TxsValue += tmpTx.Amount - s.saveTransactionForAccount(&tmpTx, from, to) - seedEpoch.Transactions[tmpTx.Id] = &tmpTx - s.Transactions[tmpTx.Id] = &tmpTx - blockContainer.Transactions = append(blockContainer.Transactions, &tmpTx) - } - - from := tmpAcc.Address - atxNumUnits := uint32(rand.Intn(1000)) - tmpSm := s.generateSmesher(tmpLayer.Number, from, uint64(atxNumUnits)*s.seed.GetPostUnitsSize()) - layerContainer.Smeshers[tmpSm.Id] = &tmpSm - seedEpoch.Epoch.Stats.Current.Smeshers++ - seedEpoch.SmeshersCommitment[tmpSm.Id] += tmpSm.CommitmentSize - - tmpAtx := s.generateActivation(tmpLayer.Number, atxNumUnits, &tmpSm, s.seed.GetPostUnitsSize(), uint32(epochID)) - seedEpoch.Activations[tmpAtx.Id] = &tmpAtx - layerContainer.Activations[tmpAtx.Id] = &tmpAtx - seedEpoch.Epoch.Stats.Current.Security += int64(tmpAtx.CommitmentSize) - s.Activations[tmpAtx.Id] = &tmpAtx - - seedEpoch.Smeshers[strings.ToLower(tmpSm.Id)] = &tmpSm - blockContainer.SmesherID = tmpSm.Id - - tmpRw := s.generateReward(tmpLayer.Number, &tmpSm) - seedEpoch.Rewards[tmpRw.Smesher] = &tmpRw - s.saveReward(&tmpRw) - seedEpoch.Blocks[tmpBl.Id] = &tmpBl - s.Rewards[strings.ToLower(tmpRw.Smesher)] = &tmpRw - s.Smeshers[strings.ToLower(tmpSm.Id)] = &tmpSm - s.Smeshers[strings.ToLower(tmpSm.Id)].Rewards = int64(tmpRw.Total) - seedEpoch.Epoch.Stats.Current.RewardsNumber++ - seedEpoch.Epoch.Stats.Current.Rewards += int64(tmpRw.Total) - seedEpoch.Epoch.Stats.Current.Capacity += int64(tmpSm.CommitmentSize) - } - s.Layers[tmpLayer.Number] = &layerContainer.Layer - return nil -} - -func (s *SeedGenerator) getRandomAcc() string { - for _, val := range s.Accounts { - return val.Account.Address - } - return "" -} - -func (s *SeedGenerator) generateActivation(layerNum uint32, atxNumUnits uint32, smesher *model.Smesher, postUnitSize uint64, epoch uint32) model.Activation { - tx, _ := utils.CalculateLayerStartEndDate(uint32(s.FirstLayerTime.Unix()), layerNum, uint32(s.seed.LayersDuration)) - return model.Activation{ - Id: strings.ToLower(utils.BytesToHex(randomBytes(32))), - SmesherId: smesher.Id, - Coinbase: smesher.Coinbase, - PrevAtx: strings.ToLower(utils.BytesToHex(randomBytes(32))), - NumUnits: atxNumUnits, - EffectiveNumUnits: atxNumUnits, - CommitmentSize: uint64(atxNumUnits) * postUnitSize, - PublishEpoch: epoch - 1, - TargetEpoch: epoch, - Received: int64(tx), - } -} - -func (s *SeedGenerator) generateEpoch(epochNum int32) model.Epoch { - layersStart := uint32(epochNum) * s.seed.EpochNumLayers - layersEnd := layersStart + s.seed.EpochNumLayers - 1 - - epochStart, _ := utils.CalculateLayerStartEndDate(uint32(s.FirstLayerTime.Unix()), layersStart, uint32(s.seed.LayersDuration)) - _, epochEnd := utils.CalculateLayerStartEndDate(uint32(s.FirstLayerTime.Unix()), layersEnd, uint32(s.seed.LayersDuration)) - return model.Epoch{ - Number: epochNum, - Start: epochStart, - End: epochEnd, - LayerStart: layersStart, - LayerEnd: layersEnd, - Layers: 0, - Stats: model.Stats{ - Current: model.Statistics{ - Capacity: 0, - Decentral: 0, - Smeshers: 0, - Transactions: 0, - Accounts: 0, - Circulation: 0, - Rewards: 0, - RewardsNumber: 0, - Security: 0, - TxsAmount: 0, - }, - Cumulative: model.Statistics{ - Capacity: 0, - Decentral: 0, - Smeshers: 0, - Transactions: 0, - Accounts: 0, - Circulation: 0, - Rewards: 0, - RewardsNumber: 0, - Security: 0, - TxsAmount: 0, - }, - }, - } -} - -func (s *SeedGenerator) generateLayer(layerNum, epochNum int32) model.Layer { - start, end := utils.CalculateLayerStartEndDate(uint32(s.FirstLayerTime.Unix()), uint32(layerNum), uint32(s.seed.LayersDuration)) - return model.Layer{ - Number: uint32(layerNum), - Status: 2, - Txs: 0, - Start: start, - End: end, - TxsAmount: 0, - Rewards: uint64(rand.Intn(1000)), - Epoch: uint32(epochNum), - Hash: strings.ToLower(fmt.Sprintf("%x", sha256.Sum256(randomBytes(32)))), - BlocksNumber: 0, - } -} - -func generateTransaction(index int, layer *model.Layer, senderSigner *signing.EdSigner, sender, receiver string, block *model.Block) model.Transaction { - maxGas := uint64(rand.Intn(1000)) - gasPrice := uint64(rand.Intn(1000)) - return model.Transaction{ - Id: strings.ToLower(utils.BytesToHex(randomBytes(32))), - Layer: layer.Number, - Block: block.Id, - BlockIndex: uint32(index), - Index: 0, - State: int(pb.TransactionState_TRANSACTION_STATE_UNSPECIFIED), - Timestamp: layer.Start, - MaxGas: maxGas, - GasPrice: gasPrice, - GasUsed: 0, - Fee: maxGas * gasPrice, - Amount: uint64(rand.Intn(1000)), - Counter: uint64(rand.Intn(1000)), - Type: 3, - Signature: strings.ToLower(utils.BytesToHex(randomBytes(30))), - PublicKey: senderSigner.PublicKey().String(), - Sender: sender, - Receiver: receiver, - SvmData: "", - } -} - -func (s *SeedGenerator) generateSmesher(layerNum uint32, coinbase string, commitmentSize uint64) model.Smesher { - tx, _ := utils.CalculateLayerStartEndDate(uint32(s.FirstLayerTime.Unix()), layerNum, uint32(s.seed.LayersDuration)) - return model.Smesher{ - Id: utils.BytesToHex(randomBytes(32)), - CommitmentSize: commitmentSize, - Coinbase: coinbase, - AtxCount: 1, - Timestamp: uint64(tx), - } -} - -func (s *SeedGenerator) generateReward(layerNum uint32, smesher *model.Smesher) model.Reward { - tx, _ := utils.CalculateLayerStartEndDate(uint32(s.FirstLayerTime.Unix()), layerNum, uint32(s.seed.LayersDuration)) - return model.Reward{ - Layer: layerNum, - Total: uint64(rand.Intn(1000)), - LayerReward: uint64(rand.Intn(1000)), - LayerComputed: 0, - Coinbase: smesher.Coinbase, - Smesher: smesher.Id, - Timestamp: tx, - } -} - -func (s *SeedGenerator) generateBlocks(layerNum, epochNum int32) model.Block { - blockStart, blockEnd := utils.CalculateLayerStartEndDate(uint32(s.FirstLayerTime.Unix()), uint32(layerNum), uint32(s.seed.LayersDuration)) - return model.Block{ - Id: strings.ToLower(utils.NBytesToHex(randomBytes(20), 20)), - Layer: uint32(layerNum), - Epoch: uint32(epochNum), - Start: blockStart, - End: blockEnd, - TxsNumber: 0, - TxsValue: 0, - } -} - -func (s *SeedGenerator) generateAccount(layerNum uint32) (model.Account, *signing.EdSigner) { - var key [32]byte - signer, _ := signing.NewEdSigner() - copy(key[:], signer.PublicKey().Bytes()) - return model.Account{ - Address: v0.ComputePrincipal(wallet.TemplateAddress, &v0.SpawnArguments{ - PublicKey: key, - }).String(), - Balance: 0, - Counter: 0, - Created: uint64(layerNum), - }, signer -} - -func randomBytes(size int) []byte { - rand.Seed(time.Now().UnixNano()) - b := make([]byte, size) - _, err := rand.Read(b) - if err != nil { - return nil - } - return b -} diff --git a/test/testseed/layers.go b/test/testseed/layers.go deleted file mode 100644 index 5ff09e2..0000000 --- a/test/testseed/layers.go +++ /dev/null @@ -1,30 +0,0 @@ -package testseed - -import "github.com/spacemeshos/explorer-backend/model" - -// BlockContainer container for block and related transactions. -type BlockContainer struct { - Block *model.Block - Transactions []*model.Transaction - SmesherID string -} - -// LayerContainer container for layer and related blocks, activations and smeshers. -type LayerContainer struct { - Layer model.Layer - Blocks []*BlockContainer - Activations map[string]*model.Activation - Smeshers map[string]*model.Smesher -} - -// GetLastLayer return last generated layer. -func (s *SeedGenerator) GetLastLayer() (curLayer, latestLayer, verifiedLayer uint32) { - for _, epoch := range s.Epochs { - for _, layer := range epoch.Layers { - curLayer = layer.Layer.Number - latestLayer = layer.Layer.Number - verifiedLayer = layer.Layer.Number - } - } - return -} diff --git a/test/testserver/fake_node.go b/test/testserver/fake_node.go deleted file mode 100644 index 47905e0..0000000 --- a/test/testserver/fake_node.go +++ /dev/null @@ -1,335 +0,0 @@ -package testserver - -import ( - "context" - "fmt" - "math/rand" - "net" - "strings" - "time" - - "github.com/phayes/freeport" - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" - "github.com/spacemeshos/go-spacemesh/common/types" - "github.com/spacemeshos/go-spacemesh/genvm/sdk" - sdkWallet "github.com/spacemeshos/go-spacemesh/genvm/sdk/wallet" - "github.com/spacemeshos/go-spacemesh/genvm/templates/wallet" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - "github.com/spacemeshos/explorer-backend/test/testseed" - "github.com/spacemeshos/explorer-backend/utils" -) - -const ( - methodSend = 16 -) - -type meshServiceWrapper struct { - startTime time.Time - seed *testseed.TestServerSeed - seedGen *testseed.SeedGenerator - pb.UnimplementedMeshServiceServer -} - -type debugServiceWrapper struct { - seedGen *testseed.SeedGenerator - pb.UnimplementedDebugServiceServer -} - -type smesherServiceWrapper struct { - seedGen *testseed.SeedGenerator - seed *testseed.TestServerSeed - pb.UnimplementedSmesherServiceServer -} - -type globalStateServiceWrapper struct { - seedGen *testseed.SeedGenerator - pb.UnimplementedGlobalStateServiceServer -} - -type nodeServiceWrapper struct { - seedGen *testseed.SeedGenerator - pb.UnimplementedNodeServiceServer -} - -// FakeNode simulate spacemesh node. -type FakeNode struct { - seedGen *testseed.SeedGenerator - NodePort int - InitDone chan struct{} - server *grpc.Server - nodeService *nodeServiceWrapper - meshService *meshServiceWrapper - globalState *globalStateServiceWrapper - debugService *debugServiceWrapper -} - -var stateSynced = make(chan struct{}) - -// CreateFakeSMNode create a fake spacemesh node. -func CreateFakeSMNode(startTime time.Time, seedGen *testseed.SeedGenerator, seedConf *testseed.TestServerSeed) (*FakeNode, error) { - appPort, err := freeport.GetFreePort() - if err != nil { - return nil, fmt.Errorf("failed to get free port: %v", err) - } - return &FakeNode{ - seedGen: seedGen, - NodePort: appPort, - InitDone: make(chan struct{}), - nodeService: &nodeServiceWrapper{seedGen, pb.UnimplementedNodeServiceServer{}}, - meshService: &meshServiceWrapper{startTime, seedConf, seedGen, pb.UnimplementedMeshServiceServer{}}, - globalState: &globalStateServiceWrapper{seedGen, pb.UnimplementedGlobalStateServiceServer{}}, - debugService: &debugServiceWrapper{seedGen, pb.UnimplementedDebugServiceServer{}}, - }, nil -} - -func CreateFakeSMPrivateNode(startTime time.Time, seedGen *testseed.SeedGenerator, seedConf *testseed.TestServerSeed) (*FakePrivateNode, error) { - appPort, err := freeport.GetFreePort() - if err != nil { - return nil, fmt.Errorf("failed to get free port: %v", err) - } - - return &FakePrivateNode{ - seedGen: seedGen, - NodePort: appPort, - InitDone: make(chan struct{}), - smesherService: &smesherServiceWrapper{seedGen, seedConf, pb.UnimplementedSmesherServiceServer{}}, - }, nil -} - -// Start register fake services and start stream generated data. -func (f *FakeNode) Start() error { - lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", f.NodePort)) - if err != nil { - return fmt.Errorf("failed to listen fake node: %v", err) - } - - f.server = grpc.NewServer() - pb.RegisterDebugServiceServer(f.server, f.debugService) - pb.RegisterMeshServiceServer(f.server, f.meshService) - pb.RegisterNodeServiceServer(f.server, f.nodeService) - pb.RegisterGlobalStateServiceServer(f.server, f.globalState) - return f.server.Serve(lis) -} - -// Stop stop fake node. -func (f *FakeNode) Stop() { - f.server.Stop() -} - -func (m meshServiceWrapper) GenesisTime(context.Context, *pb.GenesisTimeRequest) (*pb.GenesisTimeResponse, error) { - return &pb.GenesisTimeResponse{Unixtime: &pb.SimpleInt{Value: uint64(m.startTime.Unix())}}, nil -} - -func (m meshServiceWrapper) GenesisID(context.Context, *pb.GenesisIDRequest) (*pb.GenesisIDResponse, error) { - return &pb.GenesisIDResponse{GenesisId: []byte("genesisid")}, nil -} - -func (m meshServiceWrapper) EpochNumLayers(context.Context, *pb.EpochNumLayersRequest) (*pb.EpochNumLayersResponse, error) { - return &pb.EpochNumLayersResponse{Numlayers: &pb.LayerNumber{Number: m.seed.EpochNumLayers}}, nil -} - -func (m meshServiceWrapper) LayerDuration(context.Context, *pb.LayerDurationRequest) (*pb.LayerDurationResponse, error) { - return &pb.LayerDurationResponse{Duration: &pb.SimpleInt{Value: m.seed.LayersDuration}}, nil -} - -func (m meshServiceWrapper) MaxTransactionsPerSecond(context.Context, *pb.MaxTransactionsPerSecondRequest) (*pb.MaxTransactionsPerSecondResponse, error) { - return &pb.MaxTransactionsPerSecondResponse{MaxTxsPerSecond: &pb.SimpleInt{Value: m.seed.MaxTransactionPerSecond}}, nil -} - -func (d *debugServiceWrapper) Accounts(context.Context, *pb.AccountsRequest) (*pb.AccountsResponse, error) { - accs := make([]*pb.Account, 0, len(d.seedGen.Accounts)) - for _, acc := range d.seedGen.Accounts { - accs = append(accs, &pb.Account{ - AccountId: &pb.AccountId{Address: acc.Account.Address}, - StateProjected: &pb.AccountState{ - Balance: &pb.Amount{Value: acc.Account.Balance}, - Counter: acc.Account.Counter, - }, - }) - } - return &pb.AccountsResponse{AccountWrapper: accs}, nil -} - -func (g *globalStateServiceWrapper) GlobalStateStream(request *pb.GlobalStateStreamRequest, stream pb.GlobalStateService_GlobalStateStreamServer) error { - <-stateSynced - println("global state stream started") - for _, epoch := range g.seedGen.Epochs { - for _, reward := range epoch.Rewards { - smesher, _ := utils.StringToBytes(reward.Smesher) - resp := &pb.GlobalStateStreamResponse{Datum: &pb.GlobalStateData{Datum: &pb.GlobalStateData_Reward{ - Reward: &pb.Reward{ - LayerComputed: &pb.LayerNumber{Number: reward.LayerComputed}, - Layer: &pb.LayerNumber{Number: reward.Layer}, - Total: &pb.Amount{Value: reward.Total}, - LayerReward: &pb.Amount{Value: reward.LayerReward}, - Coinbase: &pb.AccountId{Address: reward.Coinbase}, - Smesher: &pb.SmesherId{Id: smesher}, - }, - }}} - if err := stream.Send(resp); err != nil { - return fmt.Errorf("failed to send global state stream response: %v", err) - } - } - } - ticker := time.NewTicker(1 * time.Second) - defer ticker.Stop() - for { - select { - case <-ticker.C: - println("global state stream sending") - case <-stream.Context().Done(): - return nil - } - } -} - -func (g *globalStateServiceWrapper) Account(_ context.Context, req *pb.AccountRequest) (*pb.AccountResponse, error) { - acc, ok := g.seedGen.Accounts[strings.ToLower(req.AccountId.Address)] - if !ok { - return nil, status.Error(codes.NotFound, "account not found") - } - return &pb.AccountResponse{ - AccountWrapper: &pb.Account{ - AccountId: &pb.AccountId{ - Address: req.AccountId.Address, - }, - StateCurrent: &pb.AccountState{ - Balance: &pb.Amount{Value: acc.Account.Balance}, - Counter: acc.Account.Counter, - }, - }, - }, nil -} - -func (m *meshServiceWrapper) LayerStream(_ *pb.LayerStreamRequest, stream pb.MeshService_LayerStreamServer) error { - if err := m.sendEpoch(stream); err != nil { - return fmt.Errorf("failed to send epoch: %v", err) - } - println("sended all layers") - time.Sleep(1 * time.Second) - close(stateSynced) - ticker := time.NewTicker(1 * time.Second) - defer ticker.Stop() - for { - select { - case <-ticker.C: - println("sending layers") - case <-stream.Context().Done(): - return nil - } - } -} - -func (m *meshServiceWrapper) sendEpoch(stream pb.MeshService_LayerStreamServer) error { - for _, epoch := range m.seedGen.Epochs { - for _, layerContainer := range epoch.Layers { - atx := make([]*pb.Activation, 0, len(layerContainer.Activations)) - for _, atxGenerated := range layerContainer.Activations { - smesherId, _ := utils.StringToBytes(atxGenerated.SmesherId) - atx = append(atx, &pb.Activation{ - Id: &pb.ActivationId{Id: mustParse(atxGenerated.Id)}, - Layer: &pb.LayerNumber{Number: atxGenerated.TargetEpoch * m.seed.EpochNumLayers}, - SmesherId: &pb.SmesherId{Id: smesherId}, - Coinbase: &pb.AccountId{Address: atxGenerated.Coinbase}, - PrevAtx: &pb.ActivationId{Id: mustParse(atxGenerated.PrevAtx)}, - NumUnits: atxGenerated.NumUnits, - }) - } - blocksRes := make([]*pb.Block, 0) - for _, blockContainer := range layerContainer.Blocks { - tx := make([]*pb.Transaction, 0, len(blockContainer.Transactions)) - for _, txContainer := range blockContainer.Transactions { - receiver, err := types.StringToAddress(txContainer.Receiver) - if err != nil { - panic("invalid receiver address: " + err.Error()) - } - signer := m.seedGen.Accounts[strings.ToLower(txContainer.Sender)].Signer - tx = append(tx, &pb.Transaction{ - Id: mustParse(txContainer.Id), - Method: methodSend, - Principal: &pb.AccountId{ - Address: txContainer.Sender, - }, - GasPrice: txContainer.GasPrice, - MaxGas: txContainer.MaxGas, - Nonce: &pb.Nonce{ - Counter: txContainer.Counter, - }, - Template: &pb.AccountId{ - Address: wallet.TemplateAddress.String(), - }, - Raw: sdkWallet.Spend(signer.PrivateKey(), receiver, txContainer.Amount, types.Nonce(txContainer.Counter), sdk.WithGasPrice(txContainer.GasPrice)), - }) - } - smesherId, _ := utils.StringToBytes(blockContainer.SmesherID) - blocksRes = append(blocksRes, &pb.Block{ - Id: mustParse(blockContainer.Block.Id), - Transactions: tx, - SmesherId: &pb.SmesherId{ - Id: smesherId, - }, - }) - } - pbLayer := &pb.Layer{ - Number: &pb.LayerNumber{Number: layerContainer.Layer.Number}, - Status: pb.Layer_LayerStatus(layerContainer.Layer.Status), - Hash: mustParse(layerContainer.Layer.Hash), - Blocks: blocksRes, - Activations: atx, - } - if err := stream.Send(&pb.LayerStreamResponse{Layer: pbLayer}); err != nil { - return fmt.Errorf("send to stream: %w", err) - } - } - } - return nil -} - -func (n *nodeServiceWrapper) StatusStream(req *pb.StatusStreamRequest, stream pb.NodeService_StatusStreamServer) error { - ticker := time.NewTicker(1 * time.Second) - defer ticker.Stop() - for { - select { - case <-ticker.C: - curLayer, latestLayer, verifiedLayer := n.seedGen.GetLastLayer() - resp := &pb.StatusStreamResponse{ - Status: &pb.NodeStatus{ - ConnectedPeers: uint64(rand.Intn(10)) + 1, // number of connected peers - IsSynced: true, // whether the node is synced - SyncedLayer: &pb.LayerNumber{Number: latestLayer}, // latest layer we saw from the network - TopLayer: &pb.LayerNumber{Number: curLayer}, // current layer, based on time - VerifiedLayer: &pb.LayerNumber{Number: verifiedLayer}, // latest verified layer - }, - } - - if err := stream.Send(resp); err != nil { - return fmt.Errorf("send to stream: %w", err) - } - case <-stream.Context().Done(): - return nil - } - } -} - -func (n *nodeServiceWrapper) Status(context.Context, *pb.StatusRequest) (*pb.StatusResponse, error) { - return &pb.StatusResponse{Status: &pb.NodeStatus{SyncedLayer: &pb.LayerNumber{Number: 0}}}, nil -} - -func mustParse(str string) []byte { - res, err := utils.StringToBytes(str) - if err != nil { - panic("error while parse string to bytes: " + err.Error()) - } - return res -} - -func addressToBytes(addr string) []byte { - res, err := types.StringToAddress(addr) - if err != nil { - panic("error while parse string to address: " + err.Error()) - } - return res.Bytes() -} diff --git a/test/testserver/fake_private_node.go b/test/testserver/fake_private_node.go deleted file mode 100644 index ebababa..0000000 --- a/test/testserver/fake_private_node.go +++ /dev/null @@ -1,45 +0,0 @@ -package testserver - -import ( - "context" - "fmt" - "github.com/golang/protobuf/ptypes/empty" - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" - "github.com/spacemeshos/explorer-backend/test/testseed" - "google.golang.org/grpc" - "net" -) - -type FakePrivateNode struct { - seedGen *testseed.SeedGenerator - NodePort int - InitDone chan struct{} - server *grpc.Server - smesherService *smesherServiceWrapper -} - -// Start register fake services and start stream generated data. -func (f *FakePrivateNode) Start() error { - lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", f.NodePort)) - if err != nil { - return fmt.Errorf("failed to listen fake node: %v", err) - } - - f.server = grpc.NewServer() - pb.RegisterSmesherServiceServer(f.server, f.smesherService) - return f.server.Serve(lis) -} - -// Stop stop fake node. -func (f *FakePrivateNode) Stop() { - f.server.Stop() -} - -func (s *smesherServiceWrapper) PostConfig(context.Context, *empty.Empty) (*pb.PostConfigResponse, error) { - return &pb.PostConfigResponse{ - BitsPerLabel: s.seed.BitsPerLabel, - LabelsPerUnit: s.seed.LabelsPerUnit, - MinNumUnits: s.seed.MinNumUnits, - MaxNumUnits: s.seed.MaxNumUnits, - }, nil -} diff --git a/test/testserver/test_api.go b/test/testserver/test_api.go deleted file mode 100644 index 1314d4c..0000000 --- a/test/testserver/test_api.go +++ /dev/null @@ -1,92 +0,0 @@ -package testserver - -import ( - "bytes" - "fmt" - "golang.org/x/net/websocket" - "log" - "net/http" - "strings" - "testing" - "time" - - "github.com/phayes/freeport" - "github.com/stretchr/testify/require" - - apiv2 "github.com/spacemeshos/explorer-backend/internal/api" - service2 "github.com/spacemeshos/explorer-backend/internal/service" - "github.com/spacemeshos/explorer-backend/internal/storage/storagereader" - "github.com/spacemeshos/explorer-backend/storage" - "github.com/spacemeshos/explorer-backend/test/testutils" -) - -// TestAPIService wrapper over fake api service. -type TestAPIService struct { - Storage *storage.Storage - port int -} - -// StartTestAPIServiceV2 start test api service with refacored router. -func StartTestAPIServiceV2(db *storage.Storage, dbReader *storagereader.Reader) (*TestAPIService, error) { - appPort, err := freeport.GetFreePort() - if err != nil { - return nil, fmt.Errorf("failed to get free port: %s", err) - } - println("starting test api service on port", appPort) - - api := apiv2.Init(service2.NewService(dbReader, time.Second), []string{"*"}, false) - go api.Run(fmt.Sprintf(":%d", appPort)) - return &TestAPIService{ - Storage: db, - port: appPort, - }, nil -} - -// Get allow to execute GET request to the fake server. -func (tx *TestAPIService) Get(t *testing.T, path string) *testutils.TestResponse { - t.Helper() - - path = strings.TrimLeft(path, "/") - url := fmt.Sprintf("http://localhost:%d/%s", tx.port, path) - req, err := http.NewRequest(http.MethodGet, url, bytes.NewBuffer([]byte{})) - require.NoError(t, err, "failed to construct new request for url %s: %s", url, err) - if err != nil { - t.Fatal(err) - } - - client := http.Client{ - Timeout: 1 * time.Second, - } - res, err := client.Do(req) - require.NoError(t, err, "failed to make request to %s: %s", url, err) - t.Cleanup(func() { - require.NoError(t, res.Body.Close()) - }) - return &testutils.TestResponse{Res: res} -} - -// GetReadWS allow to execute WS read-only request to the fake server. -func (tx *TestAPIService) GetReadWS(t *testing.T, path string) <-chan []byte { - t.Helper() - - path = strings.TrimLeft(path, "/") - origin := fmt.Sprintf("http://localhost:%d", tx.port) - conn, err := websocket.Dial(fmt.Sprintf("ws://localhost:%d/%s", tx.port, path), "", origin) - if err != nil { - t.Fatal("can't connect to websocket:", err) - } - result := make(chan []byte, 10) - go func() { - defer conn.Close() - for { - data := make([]byte, 1024) - readLen, err := conn.Read(data) - if err != nil { - log.Println(err) - return - } - result <- data[:readLen] - } - }() - return result -} diff --git a/test/testutils/response.go b/test/testutils/response.go deleted file mode 100644 index 11d0ea3..0000000 --- a/test/testutils/response.go +++ /dev/null @@ -1,39 +0,0 @@ -package testutils - -import ( - "encoding/json" - "net/http" - "testing" - - "github.com/stretchr/testify/require" -) - -// TestResponse wrapper for http.Response. -type TestResponse struct { - Res *http.Response -} - -// RequireUnmarshal try unmarshal response body to given struct. -func (r *TestResponse) RequireUnmarshal(t *testing.T, dst interface{}) { - t.Helper() - err := json.NewDecoder(r.Res.Body).Decode(dst) - require.NoError(t, err) -} - -// RequireTooEarly check that response code is 429 Too Early. -func (r *TestResponse) RequireTooEarly(t *testing.T) { - t.Helper() - r.requireStatus(t, http.StatusTooEarly) -} - -// RequireOK check that response code is 200 OK. -func (r *TestResponse) RequireOK(t *testing.T) { - t.Helper() - r.requireStatus(t, http.StatusOK) -} - -func (r *TestResponse) requireStatus(t *testing.T, status int) { - t.Helper() - require.NotNil(t, r.Res, "response is nil") - require.Equal(t, status, r.Res.StatusCode, "invalid response status code") -} diff --git a/utils/bson.go b/utils/bson.go deleted file mode 100644 index ae8fd91..0000000 --- a/utils/bson.go +++ /dev/null @@ -1,90 +0,0 @@ -package utils - -import ( - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/bsontype" -) - -func GetAsBool(rv bson.RawValue) bool { - if rv.Type == bsontype.Int64 { - return 0 != rv.Int64() - } - if rv.Type == bsontype.Int32 { - return 0 != rv.Int32() - } - if rv.Type == bsontype.Boolean { - return rv.Boolean() - } - return false -} - -func GetAsInt64(rv bson.RawValue) int64 { - if rv.Type == bsontype.Int64 { - return rv.Int64() - } - if rv.Type == bsontype.Int32 { - return int64(rv.Int32()) - } - return 0 -} - -func GetAsInt32(rv bson.RawValue) int32 { - if rv.Type == bsontype.Int32 { - return rv.Int32() - } - if rv.Type == bsontype.Int64 { - return int32(rv.Int64()) - } - return 0 -} - -func GetAsInt(rv bson.RawValue) int { - if rv.Type == bsontype.Int32 { - return int(rv.Int32()) - } - if rv.Type == bsontype.Int64 { - return int(rv.Int64()) - } - return 0 -} - -func GetAsUInt64(rv bson.RawValue) uint64 { - if rv.Type == bsontype.Int64 { - return uint64(rv.Int64()) - } - if rv.Type == bsontype.Int32 { - return uint64(rv.Int32()) - } - return 0 -} - -func GetAsUInt32(rv bson.RawValue) uint32 { - if rv.Type == bsontype.Int32 { - return uint32(rv.Int32()) - } - if rv.Type == bsontype.Int64 { - return uint32(rv.Int64()) - } - return 0 -} - -func GetAsString(rv bson.RawValue) string { - str, ok := rv.StringValueOK() - if !ok { - return "" - } - return str -} - -func FindStringValue(obj *bson.D, key string) string { - for _, e := range *obj { - if e.Key == key { - str, ok := e.Value.(string) - if ok { - return str - } - return "" - } - } - return "" -} diff --git a/utils/bytes-to-string.go b/utils/bytes-to-string.go deleted file mode 100644 index ca239da..0000000 --- a/utils/bytes-to-string.go +++ /dev/null @@ -1,9 +0,0 @@ -package utils - -import ( - "encoding/hex" -) - -func BytesToHex(a []byte) string { return "0x" + hex.EncodeToString(a[:]) } - -func NBytesToHex(a []byte, n int) string { return "0x" + hex.EncodeToString(a[:n]) } diff --git a/utils/capacity.go b/utils/capacity.go deleted file mode 100644 index 32b741d..0000000 --- a/utils/capacity.go +++ /dev/null @@ -1,8 +0,0 @@ -package utils - -import "math" - -// CalcEpochCapacity calc capacity for epoch stat. -func CalcEpochCapacity(transactionsNum int64, epochDuration float64, maxTransactionPerSecond uint32) int64 { - return int64(math.Round(((float64(transactionsNum) / epochDuration) / float64(maxTransactionPerSecond)) * 100.0)) -} diff --git a/utils/decentral.go b/utils/decentral.go deleted file mode 100644 index e16e4a9..0000000 --- a/utils/decentral.go +++ /dev/null @@ -1,11 +0,0 @@ -package utils - -import ( - "math" -) - -// CalcDecentralCoefficient calc decentral coefficient for epoch stat. -func CalcDecentralCoefficient(smeshers map[string]uint64) int64 { - a := math.Min(float64(len(smeshers)), 1e4) - return int64(100.0 * (0.5*(a*a)/1e8 + 0.5*(1.0-Gini(smeshers)))) -} diff --git a/utils/layer.go b/utils/layer.go deleted file mode 100644 index 1bf9947..0000000 --- a/utils/layer.go +++ /dev/null @@ -1,12 +0,0 @@ -package utils - -// CalculateLayerStartEndDate ... -func CalculateLayerStartEndDate(genesisTime, layerNum, layerDuration uint32) (layerStartDate, layerEndDate uint32) { - if layerNum == 0 { - layerStartDate = genesisTime - } else { - layerStartDate = genesisTime + layerNum*layerDuration - } - layerEndDate = layerStartDate + layerDuration - 1 - return layerStartDate, layerEndDate -} diff --git a/utils/print.go b/utils/print.go deleted file mode 100644 index a6b30d2..0000000 --- a/utils/print.go +++ /dev/null @@ -1,69 +0,0 @@ -package utils - -import ( - "github.com/spacemeshos/go-spacemesh/log" - pb "github.com/spacemeshos/api/release/go/spacemesh/v1" -) - -func PrintLayer(layer *pb.Layer) { - log.Info("Layer %v, status: %v, blocks: %v, activations: %v", - layer.Number, - layer.Status, - len(layer.Blocks), - len(layer.Activations), - ) - for _, atx := range layer.Activations { - PrintActivation(atx) - } - for _, block := range layer.Blocks { - PrintBlock(block) - } -} - -func PrintBlock(block *pb.Block) { -/* - log.Info("Block ID: %v, txs: %v", - block.Id, - len(block.Transactions), - ) - for _, tx := range block.Transactions { - PrintTransaction(tx) - } -*/ -} - -func PrintTransaction(tx *pb.Transaction) { -/* - log.Info("TX ID: %v, sender: %v, gas: %v, amount: %v, counter: %v", - tx.Id, - tx.Sender, - tx.GasOffered, - tx.Amount, - tx.Counter, - ) -*/ -} - -func PrintActivation(atx *pb.Activation) { - log.Info("ATX ID: %v, layer: %v, smesher: %v, coinbase: %v, prev: %v, size: %v", - atx.Id, - atx.Layer, - atx.SmesherId, - atx.Coinbase, - atx.PrevAtx, - atx.NumUnits, - ) -} - -func PrintReward(reward *pb.Reward) { -/* - log.Info("Reward layer: %v, total: %v, layerReward: %v, computed: %v, coinbase: %v, smesher: %v", - reward.Layer, - reward.Total, - reward.LayerReward, - reward.LayerComputed, - reward.Coinbase, - reward.Smesher, - ) -*/ -} diff --git a/utils/string-to-bytes.go b/utils/string-to-bytes.go deleted file mode 100644 index 7b72476..0000000 --- a/utils/string-to-bytes.go +++ /dev/null @@ -1,17 +0,0 @@ -package utils - -import ( - "encoding/hex" -) - -func StringToBytes(s string) ([]byte, error) { - if len(s) > 1 { - if s[0:2] == "0x" || s[0:2] == "0X" { - s = s[2:] - } - } - if len(s)%2 == 1 { - s = "0" + s - } - return hex.DecodeString(s) -} From dd5e52017cd0af1cb4521cc53081658c53200931 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Thu, 22 Aug 2024 15:26:37 +0200 Subject: [PATCH 20/31] Update ci.yaml --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 57d9f60..f196e7b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -47,6 +47,7 @@ jobs: docker-push: runs-on: ubuntu-latest + needs: golangci steps: - uses: actions/checkout@v2 From 04632009f6ca873a59dc4c8d8c20bf462e500712 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Fri, 23 Aug 2024 13:22:21 +0200 Subject: [PATCH 21/31] test lint ci --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f196e7b..69b4505 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,8 +42,8 @@ jobs: with: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version version: v1.59.0 - only-new-issues: true - args: --timeout=10m + #only-new-issues: true + args: --timeout=10m --config=.golangci.yml --out-format=github-actions --issues-exit-code=0 docker-push: runs-on: ubuntu-latest From 49afc66105501f297b2a659b66d56e8171328529 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Fri, 23 Aug 2024 13:29:26 +0200 Subject: [PATCH 22/31] Update golangci to v1.60.3 --- .github/workflows/ci.yaml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 69b4505..784aa8d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -41,7 +41,7 @@ jobs: uses: golangci/golangci-lint-action@v3 with: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: v1.59.0 + version: v1.60.3 #only-new-issues: true args: --timeout=10m --config=.golangci.yml --out-format=github-actions --issues-exit-code=0 diff --git a/Makefile b/Makefile index 8e3eb92..6feeb8d 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ export GO111MODULE = on BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) -GOLANGCI_LINT_VERSION := v1.59.0 +GOLANGCI_LINT_VERSION := v1.60.3 # Set BRANCH when running make manually ifeq ($(BRANCH),) From 3f094b980469763e3ec3354185e0fcad988513c7 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Fri, 23 Aug 2024 14:34:09 +0200 Subject: [PATCH 23/31] golangci-lint only new issues --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 784aa8d..afccf14 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,7 +42,7 @@ jobs: with: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version version: v1.60.3 - #only-new-issues: true + only-new-issues: true args: --timeout=10m --config=.golangci.yml --out-format=github-actions --issues-exit-code=0 docker-push: From dd2441c4897fdc9d26529e20842fdda91dfcec10 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Thu, 29 Aug 2024 12:19:17 +0200 Subject: [PATCH 24/31] Refresh data in goroutine --- api/handler/circulation.go | 24 ++++++++------- api/handler/epoch.go | 44 ++++++++++++++++------------ api/handler/overview.go | 24 ++++++++------- api/handler/smesher.go | 60 +++++++++++++++++++++----------------- api/router/router.go | 2 +- 5 files changed, 89 insertions(+), 65 deletions(-) diff --git a/api/handler/circulation.go b/api/handler/circulation.go index 3b3f0a6..e4b3c69 100644 --- a/api/handler/circulation.go +++ b/api/handler/circulation.go @@ -34,16 +34,20 @@ func Circulation(c echo.Context) error { func CirculationRefresh(c echo.Context) error { cc := c.(*ApiContext) - circulation, err := cc.StorageClient.GetCirculation(cc.Storage) - if err != nil { - log.Warning("failed to get circulation: %v", err) - return c.NoContent(http.StatusInternalServerError) - } - - if err = cc.Cache.Set(context.Background(), "circulation", circulation); err != nil { - log.Warning("failed to cache circulation: %v", err) - return c.NoContent(http.StatusInternalServerError) - } + go func() { + circulation, err := cc.StorageClient.GetCirculation(cc.Storage) + if err != nil { + log.Warning("failed to get circulation: %v", err) + return + } + + if err = cc.Cache.Set(context.Background(), "circulation", circulation); err != nil { + log.Warning("failed to cache circulation: %v", err) + return + } + + log.Info("circulation refreshed") + }() return c.NoContent(http.StatusOK) } diff --git a/api/handler/epoch.go b/api/handler/epoch.go index b179cbe..4a1c2af 100644 --- a/api/handler/epoch.go +++ b/api/handler/epoch.go @@ -44,16 +44,20 @@ func EpochRefresh(c echo.Context) error { return c.NoContent(http.StatusBadRequest) } - epochStats, err := cc.StorageClient.GetEpochStats(cc.Storage, int64(id), cc.LayersPerEpoch) - if err != nil { - log.Warning("failed to get epoch stats: %v", err) - return c.NoContent(http.StatusInternalServerError) - } + go func() { + epochStats, err := cc.StorageClient.GetEpochStats(cc.Storage, int64(id), cc.LayersPerEpoch) + if err != nil { + log.Warning("failed to get epoch stats: %v", err) + return + } - if err = cc.Cache.Set(context.Background(), "epochStats"+c.Param("id"), epochStats); err != nil { - log.Warning("failed to cache epoch stats: %v", err) - return c.NoContent(http.StatusInternalServerError) - } + if err = cc.Cache.Set(context.Background(), "epochStats"+c.Param("id"), epochStats); err != nil { + log.Warning("failed to cache epoch stats: %v", err) + return + } + + log.Info("epoch %d refreshed", id) + }() return c.NoContent(http.StatusOK) } @@ -91,16 +95,20 @@ func EpochDecentralRefresh(c echo.Context) error { return c.NoContent(http.StatusBadRequest) } - epochStats, err := cc.StorageClient.GetEpochDecentralRatio(cc.Storage, int64(id)) - if err != nil { - log.Warning("failed to get epoch stats: %v", err) - return c.NoContent(http.StatusInternalServerError) - } + go func() { + epochStats, err := cc.StorageClient.GetEpochDecentralRatio(cc.Storage, int64(id)) + if err != nil { + log.Warning("failed to get epoch stats: %v", err) + return + } - if err = cc.Cache.Set(context.Background(), "epochStatsDecentral"+c.Param("id"), epochStats); err != nil { - log.Warning("failed to cache epoch stats: %v", err) - return c.NoContent(http.StatusInternalServerError) - } + if err = cc.Cache.Set(context.Background(), "epochStatsDecentral"+c.Param("id"), epochStats); err != nil { + log.Warning("failed to cache epoch stats: %v", err) + return + } + + log.Info("epoch decentral refreshed") + }() return c.NoContent(http.StatusOK) } diff --git a/api/handler/overview.go b/api/handler/overview.go index fb9ad83..2bb8b2e 100644 --- a/api/handler/overview.go +++ b/api/handler/overview.go @@ -34,16 +34,20 @@ func Overview(c echo.Context) error { func OverviewRefresh(c echo.Context) error { cc := c.(*ApiContext) - overview, err := cc.StorageClient.Overview(cc.Storage) - if err != nil { - log.Warning("failed to get overview: %v", err) - return c.NoContent(http.StatusInternalServerError) - } - - if err = cc.Cache.Set(context.Background(), "overview", overview); err != nil { - log.Warning("failed to cache overview: %v", err) - return c.NoContent(http.StatusInternalServerError) - } + go func() { + overview, err := cc.StorageClient.Overview(cc.Storage) + if err != nil { + log.Warning("failed to get overview: %v", err) + return + } + + if err = cc.Cache.Set(context.Background(), "overview", overview); err != nil { + log.Warning("failed to cache overview: %v", err) + return + } + + log.Info("overview refreshed") + }() return c.NoContent(http.StatusOK) } diff --git a/api/handler/smesher.go b/api/handler/smesher.go index 3683794..644ce5f 100644 --- a/api/handler/smesher.go +++ b/api/handler/smesher.go @@ -41,20 +41,24 @@ func Smeshers(c echo.Context) error { func SmeshersRefresh(c echo.Context) error { cc := c.(*ApiContext) - smeshers, err := cc.StorageClient.GetSmeshers(cc.Storage, 1000, 0) - if err != nil { - log.Warning("failed to get smeshers: %v", err) - return c.NoContent(http.StatusInternalServerError) - } + go func() { + smeshers, err := cc.StorageClient.GetSmeshers(cc.Storage, 1000, 0) + if err != nil { + log.Warning("failed to get smeshers: %v", err) + return + } - for i := 0; i < len(smeshers.Smeshers); i += 20 { - if err = cc.Cache.Set(context.Background(), fmt.Sprintf("smeshers-%d-%d", 20, i), &storage.SmesherList{ - Smeshers: smeshers.Smeshers[i : i+20], - }); err != nil { - log.Warning("failed to cache smeshers: %v", err) - return c.NoContent(http.StatusInternalServerError) + for i := 0; i < len(smeshers.Smeshers); i += 20 { + if err = cc.Cache.Set(context.Background(), fmt.Sprintf("smeshers-%d-%d", 20, i), &storage.SmesherList{ + Smeshers: smeshers.Smeshers[i : i+20], + }); err != nil { + log.Warning("failed to cache smeshers: %v", err) + return + } } - } + + log.Info("smeshers refreshed") + }() return c.NoContent(http.StatusOK) } @@ -97,23 +101,27 @@ func SmeshersByEpochRefresh(c echo.Context) error { return c.NoContent(http.StatusBadRequest) } - smeshers, err := cc.StorageClient.GetSmeshersByEpoch(cc.Storage, 1000, 0, uint64(epochId)) - if err != nil { - log.Warning("failed to get smeshers: %v", err) - return c.NoContent(http.StatusInternalServerError) - } + go func() { + smeshers, err := cc.StorageClient.GetSmeshersByEpoch(cc.Storage, 1000, 0, uint64(epochId)) + if err != nil { + log.Warning("failed to get smeshers: %v", err) + return + } - for i := 0; i < len(smeshers.Smeshers); i += 20 { - if err = cc.Cache.Set(context.Background(), - fmt.Sprintf("smeshers-epoch-%d-%d-%d", epochId, 20, i), &storage.SmesherList{ - Smeshers: smeshers.Smeshers[i : i+20], - }); err != nil { - log.Warning("failed to cache smeshers: %v", err) - return c.NoContent(http.StatusInternalServerError) + for i := 0; i < len(smeshers.Smeshers); i += 20 { + if err = cc.Cache.Set(context.Background(), + fmt.Sprintf("smeshers-epoch-%d-%d-%d", epochId, 20, i), &storage.SmesherList{ + Smeshers: smeshers.Smeshers[i : i+20], + }); err != nil { + log.Warning("failed to cache smeshers: %v", err) + return + } } - } - return c.JSON(http.StatusOK, smeshers) + log.Info("smeshers by epoch %d refreshed", epochId) + }() + + return c.NoContent(http.StatusOK) } func Smesher(c echo.Context) error { diff --git a/api/router/router.go b/api/router/router.go index 919b73c..7eebd48 100644 --- a/api/router/router.go +++ b/api/router/router.go @@ -23,7 +23,7 @@ func RefreshRouter(e *echo.Echo) { e.Use(echoprometheus.NewMiddleware("spacemesh_explorer_stats_api_refresh")) g := e.Group("/refresh") g.GET("/epoch/:id", handler.EpochRefresh) - e.GET("/epoch/:id/decentral", handler.EpochDecentralRefresh) + g.GET("/epoch/:id/decentral", handler.EpochDecentralRefresh) g.GET("/overview", handler.OverviewRefresh) g.GET("/smeshers/:epoch", handler.SmeshersByEpochRefresh) g.GET("/smeshers", handler.SmeshersRefresh) From 332afa5a873f69f541edeb56344517f7efa3245d Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Mon, 2 Sep 2024 15:15:00 +0200 Subject: [PATCH 25/31] Add cache metrics --- api/cache/cache.go | 24 +++++++++++++++++++++--- api/handler/account.go | 1 + api/handler/circulation.go | 3 +++ api/handler/epoch.go | 10 +++++++++- api/handler/layer.go | 2 ++ api/handler/overview.go | 4 ++++ api/handler/smesher.go | 9 +++++++++ 7 files changed, 49 insertions(+), 4 deletions(-) diff --git a/api/cache/cache.go b/api/cache/cache.go index 2873762..6280e1b 100644 --- a/api/cache/cache.go +++ b/api/cache/cache.go @@ -1,6 +1,8 @@ package cache import ( + "github.com/eko/gocache/lib/v4/metrics" + "github.com/prometheus/client_golang/prometheus" "time" "github.com/eko/gocache/lib/v4/cache" @@ -18,21 +20,37 @@ var ( RedisAddress = "" Expiration time.Duration = 0 ShortExpiration = 5 * time.Minute + promMetrics = metrics.NewPrometheus("explorer_cache") + LastUpdated = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "explorer_cache_last_updated", + Help: "The last time the cache was updated, labeled by endpoint and id", + }, + []string{"endpoint"}, + ) ) func New() *marshaler.Marshaler { - var manager *cache.Cache[any] + prometheus.MustRegister(LastUpdated) + var manager *cache.MetricCache[any] if RedisAddress != "" { log.Info("using redis cache") redisStore := redis_store.NewRedis(redis.NewClient(&redis.Options{ Addr: RedisAddress, }), store.WithExpiration(Expiration)) - manager = cache.New[any](redisStore) + manager = cache.NewMetric[any]( + promMetrics, + cache.New[any](redisStore), + ) } else { log.Info("using memory cahe") client := gocache.New(Expiration, 6*time.Hour) s := gocacheStore.NewGoCache(client) - manager = cache.New[any](s) + manager = cache.NewMetric[any]( + promMetrics, + cache.New[any](s), + ) } + return marshaler.New(manager) } diff --git a/api/handler/account.go b/api/handler/account.go index 9cb6afd..7f554ca 100644 --- a/api/handler/account.go +++ b/api/handler/account.go @@ -39,6 +39,7 @@ func Account(c echo.Context) error { log.Warning("failed to cache account stats: %v", err) return c.NoContent(http.StatusInternalServerError) } + cache.LastUpdated.WithLabelValues("/account/" + address).SetToCurrentTime() return c.JSON(http.StatusOK, accountStats) } diff --git a/api/handler/circulation.go b/api/handler/circulation.go index e4b3c69..321cd99 100644 --- a/api/handler/circulation.go +++ b/api/handler/circulation.go @@ -2,6 +2,7 @@ package handler import ( "context" + "github.com/spacemeshos/explorer-backend/api/cache" "net/http" "github.com/labstack/echo/v4" @@ -27,6 +28,7 @@ func Circulation(c echo.Context) error { log.Warning("failed to cache circulation: %v", err) return c.NoContent(http.StatusInternalServerError) } + cache.LastUpdated.WithLabelValues("/circulation").SetToCurrentTime() return c.JSON(http.StatusOK, circulation) } @@ -47,6 +49,7 @@ func CirculationRefresh(c echo.Context) error { } log.Info("circulation refreshed") + cache.LastUpdated.WithLabelValues("/refresh/circulation").SetToCurrentTime() }() return c.NoContent(http.StatusOK) diff --git a/api/handler/epoch.go b/api/handler/epoch.go index 4a1c2af..2436f2d 100644 --- a/api/handler/epoch.go +++ b/api/handler/epoch.go @@ -2,6 +2,8 @@ package handler import ( "context" + "fmt" + "github.com/spacemeshos/explorer-backend/api/cache" "net/http" "strconv" @@ -34,6 +36,8 @@ func Epoch(c echo.Context) error { return c.NoContent(http.StatusInternalServerError) } + cache.LastUpdated.WithLabelValues("/epoch/" + c.Param("id")).SetToCurrentTime() + return c.JSON(http.StatusOK, epochStats) } @@ -57,6 +61,7 @@ func EpochRefresh(c echo.Context) error { } log.Info("epoch %d refreshed", id) + cache.LastUpdated.WithLabelValues("/refresh/epoch/" + c.Param("id")).SetToCurrentTime() }() return c.NoContent(http.StatusOK) @@ -85,6 +90,8 @@ func EpochDecentral(c echo.Context) error { return c.NoContent(http.StatusInternalServerError) } + cache.LastUpdated.WithLabelValues(fmt.Sprintf("/epoch/%s/decentral", c.Param("id"))).SetToCurrentTime() + return c.JSON(http.StatusOK, epochStats) } @@ -107,7 +114,8 @@ func EpochDecentralRefresh(c echo.Context) error { return } - log.Info("epoch decentral refreshed") + log.Info("epoch %d decentral refreshed", id) + cache.LastUpdated.WithLabelValues(fmt.Sprintf("/refresh/epoch/%s/decentral", c.Param("id"))).SetToCurrentTime() }() return c.NoContent(http.StatusOK) diff --git a/api/handler/layer.go b/api/handler/layer.go index 4f1d9d0..88644a7 100644 --- a/api/handler/layer.go +++ b/api/handler/layer.go @@ -37,5 +37,7 @@ func Layer(c echo.Context) error { return c.NoContent(http.StatusInternalServerError) } + cache.LastUpdated.WithLabelValues("/layer/" + c.Param("id")).SetToCurrentTime() + return c.JSON(http.StatusOK, layerStats) } diff --git a/api/handler/overview.go b/api/handler/overview.go index 2bb8b2e..7b4b593 100644 --- a/api/handler/overview.go +++ b/api/handler/overview.go @@ -2,6 +2,7 @@ package handler import ( "context" + "github.com/spacemeshos/explorer-backend/api/cache" "net/http" "github.com/labstack/echo/v4" @@ -28,6 +29,8 @@ func Overview(c echo.Context) error { return c.NoContent(http.StatusInternalServerError) } + cache.LastUpdated.WithLabelValues("/overview").SetToCurrentTime() + return c.JSON(http.StatusOK, overview) } @@ -47,6 +50,7 @@ func OverviewRefresh(c echo.Context) error { } log.Info("overview refreshed") + cache.LastUpdated.WithLabelValues("/refresh/overview").SetToCurrentTime() }() return c.NoContent(http.StatusOK) diff --git a/api/handler/smesher.go b/api/handler/smesher.go index 644ce5f..19b8abb 100644 --- a/api/handler/smesher.go +++ b/api/handler/smesher.go @@ -35,6 +35,8 @@ func Smeshers(c echo.Context) error { return c.NoContent(http.StatusInternalServerError) } + cache.LastUpdated.WithLabelValues(fmt.Sprintf("/smeshers?limit=%d&offset=%d", limit, offset)).SetToCurrentTime() + return c.JSON(http.StatusOK, smeshers) } @@ -58,6 +60,7 @@ func SmeshersRefresh(c echo.Context) error { } log.Info("smeshers refreshed") + cache.LastUpdated.WithLabelValues("/refresh/smeshers").SetToCurrentTime() }() return c.NoContent(http.StatusOK) @@ -90,6 +93,9 @@ func SmeshersByEpoch(c echo.Context) error { return c.NoContent(http.StatusInternalServerError) } + cache.LastUpdated.WithLabelValues( + fmt.Sprintf("/smeshers/%s?limit=%d&offset=%d", c.Param("epoch"), limit, offset)).SetToCurrentTime() + return c.JSON(http.StatusOK, smeshers) } @@ -119,6 +125,7 @@ func SmeshersByEpochRefresh(c echo.Context) error { } log.Info("smeshers by epoch %d refreshed", epochId) + cache.LastUpdated.WithLabelValues("/refresh/smeshers/" + c.Param("epoch")).SetToCurrentTime() }() return c.NoContent(http.StatusOK) @@ -150,5 +157,7 @@ func Smesher(c echo.Context) error { return c.NoContent(http.StatusInternalServerError) } + cache.LastUpdated.WithLabelValues("/smesher/" + smesherId).SetToCurrentTime() + return c.JSON(http.StatusOK, smesher) } From 382f6bb7a7fed124578bcc953dd4b91076d6048b Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Tue, 3 Sep 2024 10:39:39 +0200 Subject: [PATCH 26/31] Fix metrics help text --- api/cache/cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/cache/cache.go b/api/cache/cache.go index 6280e1b..e0cd23e 100644 --- a/api/cache/cache.go +++ b/api/cache/cache.go @@ -24,7 +24,7 @@ var ( LastUpdated = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "explorer_cache_last_updated", - Help: "The last time the cache was updated, labeled by endpoint and id", + Help: "The last time the cache was updated, labeled by endpoint", }, []string{"endpoint"}, ) From 64abe90a7abfc38ad6bf3711421dcfc1f3416a95 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Fri, 6 Sep 2024 09:22:59 +0200 Subject: [PATCH 27/31] Fix pipeline and remove metrics for smesher and account --- .github/workflows/ci.yaml | 2 +- api/handler/account.go | 1 - api/handler/smesher.go | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index afccf14..9259b55 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -43,7 +43,7 @@ jobs: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version version: v1.60.3 only-new-issues: true - args: --timeout=10m --config=.golangci.yml --out-format=github-actions --issues-exit-code=0 + args: --timeout=10m --config=.golangci.yml --out-format=github-actions docker-push: runs-on: ubuntu-latest diff --git a/api/handler/account.go b/api/handler/account.go index 7f554ca..9cb6afd 100644 --- a/api/handler/account.go +++ b/api/handler/account.go @@ -39,7 +39,6 @@ func Account(c echo.Context) error { log.Warning("failed to cache account stats: %v", err) return c.NoContent(http.StatusInternalServerError) } - cache.LastUpdated.WithLabelValues("/account/" + address).SetToCurrentTime() return c.JSON(http.StatusOK, accountStats) } diff --git a/api/handler/smesher.go b/api/handler/smesher.go index 19b8abb..440bdbf 100644 --- a/api/handler/smesher.go +++ b/api/handler/smesher.go @@ -157,7 +157,5 @@ func Smesher(c echo.Context) error { return c.NoContent(http.StatusInternalServerError) } - cache.LastUpdated.WithLabelValues("/smesher/" + smesherId).SetToCurrentTime() - return c.JSON(http.StatusOK, smesher) } From e70b2359a432bcce4a492ba80afe11129308ff6d Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Fri, 6 Sep 2024 09:26:32 +0200 Subject: [PATCH 28/31] Fix lint --- api/cache/cache.go | 4 ++-- api/handler/circulation.go | 2 +- api/handler/epoch.go | 2 +- api/handler/overview.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/cache/cache.go b/api/cache/cache.go index e0cd23e..e1a2558 100644 --- a/api/cache/cache.go +++ b/api/cache/cache.go @@ -1,16 +1,16 @@ package cache import ( - "github.com/eko/gocache/lib/v4/metrics" - "github.com/prometheus/client_golang/prometheus" "time" "github.com/eko/gocache/lib/v4/cache" "github.com/eko/gocache/lib/v4/marshaler" + "github.com/eko/gocache/lib/v4/metrics" "github.com/eko/gocache/lib/v4/store" gocacheStore "github.com/eko/gocache/store/go_cache/v4" redis_store "github.com/eko/gocache/store/redis/v4" gocache "github.com/patrickmn/go-cache" + "github.com/prometheus/client_golang/prometheus" "github.com/redis/go-redis/v9" "github.com/spacemeshos/go-spacemesh/log" diff --git a/api/handler/circulation.go b/api/handler/circulation.go index 321cd99..b3ab6e7 100644 --- a/api/handler/circulation.go +++ b/api/handler/circulation.go @@ -2,10 +2,10 @@ package handler import ( "context" - "github.com/spacemeshos/explorer-backend/api/cache" "net/http" "github.com/labstack/echo/v4" + "github.com/spacemeshos/explorer-backend/api/cache" "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/log" diff --git a/api/handler/epoch.go b/api/handler/epoch.go index 2436f2d..faa9c33 100644 --- a/api/handler/epoch.go +++ b/api/handler/epoch.go @@ -3,11 +3,11 @@ package handler import ( "context" "fmt" - "github.com/spacemeshos/explorer-backend/api/cache" "net/http" "strconv" "github.com/labstack/echo/v4" + "github.com/spacemeshos/explorer-backend/api/cache" "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/log" diff --git a/api/handler/overview.go b/api/handler/overview.go index 7b4b593..ab97aef 100644 --- a/api/handler/overview.go +++ b/api/handler/overview.go @@ -2,10 +2,10 @@ package handler import ( "context" - "github.com/spacemeshos/explorer-backend/api/cache" "net/http" "github.com/labstack/echo/v4" + "github.com/spacemeshos/explorer-backend/api/cache" "github.com/spacemeshos/explorer-backend/api/storage" "github.com/spacemeshos/go-spacemesh/log" From cdd743ea8bb3b6516fca38e959a6933065a9c7ea Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Thu, 12 Sep 2024 13:43:04 +0200 Subject: [PATCH 29/31] Add accounts count and vested amount to epoch stats --- api/storage/epoch.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/api/storage/epoch.go b/api/storage/epoch.go index 975236e..e02c3f4 100644 --- a/api/storage/epoch.go +++ b/api/storage/epoch.go @@ -1,6 +1,7 @@ package storage import ( + "github.com/spacemeshos/economics/constants" "math" "github.com/spacemeshos/explorer-backend/utils" @@ -20,6 +21,8 @@ type EpochStats struct { NumUnits uint64 `json:"num_units,omitempty"` SmeshersCount uint64 `json:"smeshers_count,omitempty"` Decentral uint64 `json:"decentral,omitempty"` + VestedAmount uint64 `json:"vested_amount,omitempty"` + AccountsCount uint64 `json:"accounts_count,omitempty"` } func (c *Client) GetEpochStats(db *sql.Database, epoch, layersPerEpoch int64) (*EpochStats, error) { @@ -32,6 +35,15 @@ func (c *Client) GetEpochStats(db *sql.Database, epoch, layersPerEpoch int64) (* start := epoch * layersPerEpoch end := start + layersPerEpoch - 1 + currentEpoch := c.NodeClock.CurrentLayer().Uint32() / uint32(layersPerEpoch) + + if !c.Testnet && start >= constants.VestStart && start <= constants.VestEnd { + if epoch == int64(currentEpoch) { + stats.VestedAmount = (uint64(c.NodeClock.CurrentLayer().Uint32()) - uint64(start-1)) * constants.VestPerLayer + } else { + stats.VestedAmount = uint64(layersPerEpoch) * constants.VestPerLayer + } + } _, err := db.Exec(`SELECT COUNT(*) FROM ( @@ -104,6 +116,19 @@ FROM ( return true }) + _, err = db.Exec(`SELECT COUNT(DISTINCT address) + FROM transactions_results_addresses + WHERE tid IN ( + SELECT id FROM transactions WHERE layer >= ?1 AND layer <= ?2)`, + func(statement *sql.Statement) { + statement.BindInt64(1, start) + statement.BindInt64(2, end) + }, + func(statement *sql.Statement) bool { + stats.AccountsCount = uint64(statement.ColumnInt64(0)) + return true + }) + return stats, err } From beae0116561af235f7b36b8e0dd2906541c2df55 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Thu, 12 Sep 2024 13:48:50 +0200 Subject: [PATCH 30/31] Fix lint --- api/storage/epoch.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/api/storage/epoch.go b/api/storage/epoch.go index e02c3f4..069cecf 100644 --- a/api/storage/epoch.go +++ b/api/storage/epoch.go @@ -1,9 +1,9 @@ package storage import ( - "github.com/spacemeshos/economics/constants" "math" + "github.com/spacemeshos/economics/constants" "github.com/spacemeshos/explorer-backend/utils" "github.com/spacemeshos/go-spacemesh/common/types" @@ -39,7 +39,8 @@ func (c *Client) GetEpochStats(db *sql.Database, epoch, layersPerEpoch int64) (* if !c.Testnet && start >= constants.VestStart && start <= constants.VestEnd { if epoch == int64(currentEpoch) { - stats.VestedAmount = (uint64(c.NodeClock.CurrentLayer().Uint32()) - uint64(start-1)) * constants.VestPerLayer + stats.VestedAmount = (uint64(c.NodeClock.CurrentLayer().Uint32()) - + uint64(start-1)) * constants.VestPerLayer } else { stats.VestedAmount = uint64(layersPerEpoch) * constants.VestPerLayer } @@ -115,6 +116,9 @@ FROM ( stats.SmeshersCount = uint64(stmt.ColumnInt64(0)) return true }) + if err != nil { + return nil, err + } _, err = db.Exec(`SELECT COUNT(DISTINCT address) FROM transactions_results_addresses From d27c8e7db156876ebc3849449e5102eb8b0a31de Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Fri, 13 Sep 2024 14:48:37 +0200 Subject: [PATCH 31/31] Fix epoch stats vested amount calculation --- api/storage/epoch.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api/storage/epoch.go b/api/storage/epoch.go index 069cecf..c9484c9 100644 --- a/api/storage/epoch.go +++ b/api/storage/epoch.go @@ -37,10 +37,13 @@ func (c *Client) GetEpochStats(db *sql.Database, epoch, layersPerEpoch int64) (* end := start + layersPerEpoch - 1 currentEpoch := c.NodeClock.CurrentLayer().Uint32() / uint32(layersPerEpoch) - if !c.Testnet && start >= constants.VestStart && start <= constants.VestEnd { + if !c.Testnet && end >= constants.VestStart { + vestStartEpoch := constants.VestStart / layersPerEpoch if epoch == int64(currentEpoch) { stats.VestedAmount = (uint64(c.NodeClock.CurrentLayer().Uint32()) - uint64(start-1)) * constants.VestPerLayer + } else if epoch == vestStartEpoch { + stats.VestedAmount = uint64(end-constants.VestStart) * constants.VestPerLayer } else { stats.VestedAmount = uint64(layersPerEpoch) * constants.VestPerLayer }