Add archived-exporter component

.github/workflows/release.yml

@@ -167,3 +167,15 @@ jobs:
             ghcr.io/${{ github.repository }}/cli:${{ github.ref_name }}
             ghcr.io/${{ github.repository }}/cli:${{ github.ref_name }}-${{ steps.timestamp.outputs.now }}
           outputs: type=image,name=ghcr.io/${{ github.repository }}/cli,annotation-index.org.opencontainers.image.description=${{ github.repository }}
+      - name: Build and push exporter container image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile.exporter
+          platforms: amd64
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}/exporter:latest
+            ghcr.io/${{ github.repository }}/exporter:${{ github.ref_name }}
+            ghcr.io/${{ github.repository }}/exporter:${{ github.ref_name }}-${{ steps.timestamp.outputs.now }}
+          outputs: type=image,name=ghcr.io/${{ github.repository }}/exporter,annotation-index.org.opencontainers.image.description=${{ github.repository }}

.github/workflows/verify.yml

@@ -133,3 +133,11 @@ jobs:
           platforms: amd64
           push: false
           outputs: type=image,name=ghcr.io/${{ github.repository }}/cli,annotation-index.org.opencontainers.image.description=${{ github.repository }}
+      - name: Build exporter container image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile.exporter
+          platforms: amd64
+          push: false
+          outputs: type=image,name=ghcr.io/${{ github.repository }}/exporter,annotation-index.org.opencontainers.image.description=${{ github.repository }}

.goreleaser.yaml

@@ -61,6 +61,21 @@ builds:
     goamd64: ["v1", "v2", "v3"]
     goarm: ["7"]
     mod_timestamp: "{{ .CommitTimestamp }}"
+  - id: archived-exporter
+    main: ./cmd/exporter
+    binary: archived-exporter
+    ldflags:
+      - -s -w -X main.appVersion={{.Version}} -X main.buildTimestamp={{.Date}}
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+    goarch:
+      - amd64
+      - arm64
+    goamd64: ["v1", "v2", "v3"]
+    goarm: ["7"]
+    mod_timestamp: "{{ .CommitTimestamp }}"
 archives:
   - format: binary
 checksum:

Dockerfile.exporter

@@ -0,0 +1,11 @@
+FROM alpine:3.20.0 AS certificates
+
+RUN apk add --update --no-cache \
+  ca-certificates=20240705-r0
+
+FROM scratch
+
+COPY --from=certificates /etc/ssl/cert.pem /etc/ssl/cert.pem
+COPY --chmod=0755 --chown=root:root dist/archived-exporter_linux_amd64_v3/archived-exporter /archived-exporter
+
+ENTRYPOINT [ "/archived-exporter" ]

cmd/exporter/main.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"net/http"
+
+	"github.com/kelseyhightower/envconfig"
+	_ "github.com/lib/pq"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
+
+	"github.com/teran/archived/exporter/service"
+	"github.com/teran/archived/repositories/metadata/postgresql"
+)
+
+var (
+	appVersion     = "n/a (dev build)"
+	buildTimestamp = "undefined"
+)
+
+type config struct {
+	Addr string `envconfig:"METRICS_ADDR" default:":8081"`
+
+	LogLevel log.Level `envconfig:"LOG_LEVEL" default:"info"`
+
+	MetadataDSN string `envconfig:"METADATA_DSN" required:"true"`
+}
+
+func main() {
+	var cfg config
+	envconfig.MustProcess("", &cfg)
+
+	log.SetLevel(cfg.LogLevel)
+
+	lf := new(log.TextFormatter)
+	lf.FullTimestamp = true
+	log.SetFormatter(lf)
+
+	log.Infof("Initializing archived-exporter (%s @ %s) ...", appVersion, buildTimestamp)
+
+	db, err := sql.Open("postgres", cfg.MetadataDSN)
+	if err != nil {
+		panic(err)
+	}
+
+	if err := db.Ping(); err != nil {
+		panic(err)
+	}
+
+	postgresqlRepo := postgresql.New(db)
+
+	svc, err := service.New(postgresqlRepo)
+	if err != nil {
+		panic(err)
+	}
+
+	g, ctx := errgroup.WithContext(context.Background())
+
+	g.Go(func() error {
+		return svc.Run(ctx)
+	})
+
+	g.Go(func() error {
+		http.Handle("/metrics", promhttp.Handler())
+		return http.ListenAndServe(cfg.Addr, nil)
+	})
+
+	if err := g.Wait(); err != nil {
+		panic(err)
+	}
+}

exporter/models/models.go

@@ -0,0 +1,30 @@
+package models
+
+type VersionsCount struct {
+	ContainerName string
+	VersionsCount uint64
+	IsPublished   bool
+}
+
+type ObjectsCount struct {
+	ContainerName string
+	VersionName   string
+	IsPublished   bool
+	ObjectsCount  uint64
+}
+
+type BlobsRawSizeBytes struct {
+	ContainerName string
+	VersionName   string
+	IsPublished   bool
+	SizeBytes     uint64
+}
+
+type Stats struct {
+	ContainersCount     uint64
+	VersionsCount       []VersionsCount
+	ObjectsCount        []ObjectsCount
+	BlobsCount          uint64
+	BlobsRawSizeBytes   []BlobsRawSizeBytes
+	BlobsTotalSizeBytes uint64
+}

exporter/service/metrics.go

@@ -0,0 +1 @@
+package service

exporter/service/service.go

@@ -0,0 +1,143 @@
+package service
+
+import (
+	"context"
+	"strconv"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/teran/archived/repositories/metadata"
+)
+
+type Service interface {
+	Run(ctx context.Context) error
+}
+
+type service struct {
+	containersTotal   *prometheus.GaugeVec
+	versionsTotal     *prometheus.GaugeVec
+	objectsTotal      *prometheus.GaugeVec
+	blobsTotal        *prometheus.GaugeVec
+	blobsSize         *prometheus.GaugeVec
+	blobsTotalRawSize *prometheus.GaugeVec
+
+	repo metadata.Repository
+}
+
+func New(repo metadata.Repository) (Service, error) {
+	svc := &service{
+		repo: repo,
+
+		containersTotal: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: "archived",
+				Name:      "containers_total",
+				Help:      "Total amount of containers",
+			},
+			[]string{},
+		),
+
+		versionsTotal: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: "archived",
+				Name:      "versions_amount",
+				Help:      "Total amount of versions",
+			},
+			[]string{"container", "is_published"},
+		),
+
+		objectsTotal: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: "archived",
+				Name:      "objects_amount",
+				Help:      "Total amount of objects",
+			},
+			[]string{"container", "version", "is_published"},
+		),
+
+		blobsTotal: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: "archived",
+				Name:      "blobs_total",
+				Help:      "Total amount of blobs",
+			},
+			[]string{},
+		),
+
+		blobsSize: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: "archived",
+				Name:      "blobs_raw_size_bytes",
+				Help:      "Total raw size of blobs (i.e. before deduplication)",
+			}, []string{"container", "version", "is_published"},
+		),
+
+		blobsTotalRawSize: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: "archived",
+				Name:      "blobs_effective_size_total_bytes",
+				Help:      "Total effective size of blobs (i.e. after deduplication)",
+			}, []string{},
+		),
+	}
+
+	for _, m := range []*prometheus.GaugeVec{
+		svc.containersTotal,
+		svc.versionsTotal,
+		svc.objectsTotal,
+		svc.blobsTotal,
+		svc.blobsSize,
+		svc.blobsTotalRawSize,
+	} {
+		if err := prometheus.Register(m); err != nil {
+			return nil, err
+		}
+	}
+
+	return svc, nil
+}
+
+func (s *service) observe(ctx context.Context) error {
+	stats, err := s.repo.CountStats(ctx)
+	if err != nil {
+		return err
+	}
+
+	s.containersTotal.WithLabelValues().Set(float64(stats.ContainersCount))
+	for _, vt := range stats.VersionsCount {
+		s.versionsTotal.WithLabelValues(
+			vt.ContainerName, strconv.FormatBool(vt.IsPublished),
+		).Set(float64(vt.VersionsCount))
+	}
+
+	for _, ot := range stats.ObjectsCount {
+		s.objectsTotal.WithLabelValues(
+			ot.ContainerName, ot.VersionName, strconv.FormatBool(ot.IsPublished),
+		).Set(float64(ot.ObjectsCount))
+	}
+
+	s.blobsTotal.WithLabelValues().Set(float64(stats.BlobsCount))
+
+	for _, bs := range stats.BlobsRawSizeBytes {
+		s.blobsSize.WithLabelValues(
+			bs.ContainerName, bs.VersionName, strconv.FormatBool(bs.IsPublished),
+		).Set(float64(bs.SizeBytes))
+	}
+
+	s.blobsTotalRawSize.WithLabelValues().Set(float64(stats.BlobsTotalSizeBytes))
+
+	return nil
+}
+
+func (s *service) Run(ctx context.Context) error {
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(30 * time.Second):
+			if err := s.observe(ctx); err != nil {
+				return err
+			}
+		}
+	}
+}

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/teran/go-time v0.0.2
 	golang.org/x/sync v0.7.0
 	google.golang.org/grpc v1.65.0
+	google.golang.org/protobuf v1.34.2
 )
 
 require (
@@ -98,6 +99,5 @@ require (
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

repositories/metadata/metadata.go

@@ -4,6 +4,8 @@ import (
 	"context"
 
 	"github.com/pkg/errors"
+
+	"github.com/teran/archived/exporter/models"
 )
 
 var ErrNotFound = errors.New("not found")
@@ -27,4 +29,6 @@ type Repository interface {
 	CreateBLOB(ctx context.Context, checksum string, size uint64, mimeType string) error
 	GetBlobKeyByObject(ctx context.Context, container, version, key string) (string, error)
 	EnsureBlobKey(ctx context.Context, key string, size uint64) error
+
+	CountStats(ctx context.Context) (*models.Stats, error)
 }

repositories/metadata/mock/mock.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/stretchr/testify/mock"
+	"github.com/teran/archived/exporter/models"
 	"github.com/teran/archived/repositories/metadata"
 )
 
@@ -91,3 +92,8 @@ func (m *Mock) EnsureBlobKey(_ context.Context, key string, size uint64) error {
 	args := m.Called(key, size)
 	return args.Error(0)
 }
+
+func (m *Mock) CountStats(ctx context.Context) (*models.Stats, error) {
+	args := m.Called()
+	return args.Get(0).(*models.Stats), args.Error(1)
+}