close #143: add container pagination (#180)

Closes #143

---------

Co-authored-by: Демин Михаил <mddemin@nic.ru>

cmd/publisher/main.go

@@ -55,8 +55,9 @@ type config struct {
 	HTMLTemplateDir string `envconfig:"HTML_TEMPLATE_DIR" required:"true"`
 	StaticDir       string `envconfig:"STATIC_DIR" required:"true"`
 
-	VersionsPerPage uint64 `envconfig:"VERSIONS_PER_PAGE" default:"50"`
-	ObjectsPerPage  uint64 `envconfig:"OBJECTS_PER_PAGE" default:"50"`
+	VersionsPerPage    uint64 `envconfig:"VERSIONS_PER_PAGE" default:"50"`
+	ObjectsPerPage     uint64 `envconfig:"OBJECTS_PER_PAGE" default:"50"`
+	ContainersPerPage  uint64 `envconfig:"CONTAINERS_PER_PAGE" default:"50"`
 }
 
 func main() {
@@ -119,7 +120,7 @@ func main() {
 
 	blobRepo := awsBlobRepo.New(s3.New(awsSession), cfg.BLOBS3Bucket, cfg.BLOBS3PresignedLinkTTL)
 
-	publisherSvc := service.NewPublisher(repo, blobRepo, cfg.VersionsPerPage, cfg.ObjectsPerPage)
+	publisherSvc := service.NewPublisher(repo, blobRepo, cfg.VersionsPerPage, cfg.ObjectsPerPage, cfg.ContainersPerPage)
 
 	p := htmlPresenter.New(publisherSvc, cfg.HTMLTemplateDir, cfg.StaticDir)
 	p.Register(e)

docker-compose.yaml

@@ -64,7 +64,7 @@ services:
       METADATA_DSN: postgres://postgres:password@postgresql?sslmode=disable
       BLOB_S3_ENDPOINT: http://minio:9000
       BLOB_S3_BUCKET: test-bucket
-      BLOB_S3_CREATE_BUCKET: "true"
+      BLOB_S3_CREATE_BUCKET: "false"
       BLOB_S3_ACCESS_KEY_ID: minioadmin
       BLOB_S3_SECRET_KEY: minioadmin
     ports:

publisher/presenter/html/handlers.go

@@ -61,21 +61,38 @@ func (h *handlers) NamespaceIndex(c echo.Context) error {
 
 func (h *handlers) ContainerIndex(c echo.Context) error {
 	namespace := c.Param("namespace")
-	containers, err := h.svc.ListContainers(c.Request().Context(), namespace)
+
+	pageParam := c.QueryParam("page")
+	var page uint64 = 1
+
+	var err error
+	if pageParam != "" {
+		page, err = strconv.ParseUint(pageParam, 10, 64)
+		if err != nil {
+			log.Warnf("malformed page parameter: `%s`", pageParam)
+			page = 1
+		}
+	}
+
+	pagesCount, containers, err := h.svc.ListContainersByPage(c.Request().Context(), namespace, page)
 	if err != nil {
 		return err
 	}
 
 	type data struct {
-		Title      string
-		Namespace  string
-		Containers []string
+		Title       string
+		CurrentPage uint64
+		PagesCount  uint64
+		Namespace   string
+		Containers  []string
 	}
 
 	return c.Render(http.StatusOK, "container-list.html", &data{
-		Title:      fmt.Sprintf("Container index (%s)", namespace),
-		Namespace:  namespace,
-		Containers: containers,
+		Title:       fmt.Sprintf("Container index (%s)", namespace),
+		CurrentPage: page,
+		PagesCount:  pagesCount,
+		Namespace:   namespace,
+		Containers:  containers,
 	})
 }
 

publisher/presenter/html/handlers_test.go

@@ -23,7 +23,7 @@ func (s *handlersTestSuite) TestNamespaceIndex() {
 }
 
 func (s *handlersTestSuite) TestContainerIndex() {
-	s.serviceMock.On("ListContainers", defaultNamespace).Return([]string{"test-container-1"}, nil).Once()
+	s.serviceMock.On("ListContainersByPage", defaultNamespace, uint64(1)).Return(uint64(100), []string{"test-container-1"}, nil).Once()
 
 	s.compareHTMLResponse(s.srv.URL+"/default/", "testdata/containers.html.sample")
 }

publisher/presenter/html/templates/container-list.html

@@ -6,4 +6,6 @@
 {{- else }}
         <p>No containers found</p>
 {{- end }}
+
+{{ template "pagination" . }}
 {{ template "footer" }}

publisher/presenter/html/testdata/containers.html.sample

@@ -12,6 +12,12 @@
         <a href="..">..</a><br>
         <a href="/default/test-container-1/">test-container-1/</a><br>
 
+
+        <p>
+            <a href="?page=2">&raquo;</a>
+        </p>
+
+
         <hr>
         Powered by <a href="https://github.com/teran/archived" target="_blank" rel="noopener">archived</a>
     </body>

repositories/cache/metadata/memcache/memcache.go

@@ -138,6 +138,55 @@ func (m *memcache) ListContainers(ctx context.Context, namespace string) ([]stri
 	return retrievedValue, nil
 }
 
+
+func (m *memcache) ListContainersByPage(ctx context.Context, namespace string, offset, limit uint64) (uint64, []string, error) {
+	type proxy struct {
+		Total    uint64
+		Containers []string
+	}
+
+	cacheKey := strings.Join([]string{
+		m.keyPrefix,
+		"ListContainersByPage",
+		namespace,
+		strconv.FormatUint(offset, 10),
+		strconv.FormatUint(limit, 10),
+	}, ":")
+
+	item, err := m.cli.Get(cacheKey)
+	if err != nil {
+		if err == memcacheCli.ErrCacheMiss {
+			log.WithFields(log.Fields{
+				"key": cacheKey,
+			}).Tracef("cache miss")
+
+			n, containers, err := m.repo.ListContainersByPage(ctx, namespace, offset, limit)
+			if err != nil {
+				return 0, nil, err
+			}
+
+			if err := store(m, cacheKey, proxy{Total: n, Containers: containers}); err != nil {
+				return 0, nil, err
+			}
+
+			return n, containers, nil
+		}
+
+		return 0, nil, err
+	}
+	log.WithFields(log.Fields{
+		"key": cacheKey,
+	}).Tracef("cache hit")
+
+	var retrievedValue proxy
+	err = json.Unmarshal(item.Value, &retrievedValue)
+	if err != nil {
+		return 0, nil, err
+	}
+
+	return retrievedValue.Total, retrievedValue.Containers, nil
+}
+
 func (m *memcache) DeleteContainer(ctx context.Context, namespace, name string) error {
 	return m.repo.DeleteContainer(ctx, namespace, name)
 }

repositories/cache/metadata/memcache/memcache_test.go

@@ -44,6 +44,20 @@ func (s *memcacheTestSuite) TestListContainersError() {
 	s.Require().Equal("some error", err.Error())
 }
 
+func (s *memcacheTestSuite) TestListContainersByPage() {
+	s.repoMock.On("ListContainersByPage", defaultNamespace, uint64(0), uint64(15)).Return(uint64(500), []string{"container1"}, nil).Once()
+
+	total, containers, err := s.cache.ListContainersByPage(s.ctx, defaultNamespace, 0, 15)
+	s.Require().NoError(err)
+	s.Require().Equal([]string{"container1"}, containers)
+	s.Require().Equal(uint64(500), total)
+
+	total, containers, err = s.cache.ListContainersByPage(s.ctx, defaultNamespace, 0, 15)
+	s.Require().NoError(err)
+	s.Require().Equal([]string{"container1"}, containers)
+	s.Require().Equal(uint64(500), total)
+}
+
 func (s *memcacheTestSuite) TestGetLatestPublishedVersionByContainer() {
 	s.repoMock.On("GetLatestPublishedVersionByContainer", defaultNamespace, "test-container").Return("test-version", nil).Once()
 

repositories/metadata/metadata.go

@@ -23,6 +23,7 @@ type Repository interface {
 	CreateContainer(ctx context.Context, namespace, name string) error
 	RenameContainer(ctx context.Context, namespace, oldName, newNamespace, newName string) error
 	ListContainers(ctx context.Context, namespace string) ([]string, error)
+	ListContainersByPage(ctx context.Context, namespace string, offset, limit uint64) (uint64, []string, error)
 	DeleteContainer(ctx context.Context, namespace, name string) error
 
 	CreateVersion(ctx context.Context, namespace, container string) (string, error)

repositories/metadata/mock/mock.go

@@ -54,6 +54,11 @@ func (m *Mock) ListContainers(_ context.Context, namespace string) ([]string, er
 	return args.Get(0).([]string), args.Error(1)
 }
 
+func (m *Mock) ListContainersByPage(_ context.Context, namespace string, offset, limit uint64) (uint64, []string, error) {
+	args := m.Called(namespace, offset, limit)
+	return args.Get(0).(uint64), args.Get(1).([]string), args.Error(2)
+}
+
 func (m *Mock) DeleteContainer(_ context.Context, namespace, name string) error {
 	args := m.Called(namespace, name)
 	return args.Error(0)

repositories/metadata/postgresql/containers.go

@@ -154,6 +154,63 @@ func (r *repository) ListContainers(ctx context.Context, namespace string) ([]st
 	return result, nil
 }
 
+func (r *repository) ListContainersByPage(ctx context.Context, namespace string, offset, limit uint64) (uint64, []string, error) {
+	row, err := selectQueryRow(ctx, r.db, psql.
+		Select("id").
+		From("namespaces").
+		Where(sq.Eq{"name": namespace}))
+	if err != nil {
+		return 0, nil, mapSQLErrors(err)
+	}
+
+	var namespaceID uint
+	if err := row.Scan(&namespaceID); err != nil {
+		return 0, nil, mapSQLErrors(err)
+	}
+
+	row, err = selectQueryRow(ctx, r.db, psql.
+		Select("COUNT(*)").
+		From("containers").
+		Where(sq.Eq{
+			"namespace_id": namespaceID,
+		}))
+	if err != nil {
+		return 0, nil, mapSQLErrors(err)
+	}
+
+	var containersTotal uint64
+	if err := row.Scan(&containersTotal); err != nil {
+		return 0, nil, mapSQLErrors(err)
+	}
+
+	rows, err := selectQuery(ctx, r.db, psql.
+		Select("name").
+		From("containers").
+		Where(sq.Eq{
+			"namespace_id": namespaceID,
+		}).
+		OrderBy("name").
+		Limit(limit).
+		Offset(offset))
+	if err != nil {
+		return 0, nil, mapSQLErrors(err)
+	}
+	defer rows.Close()
+
+	result := []string{}
+	for rows.Next() {
+		var r string
+		if err := rows.Scan(&r); err != nil {
+			return 0, nil, mapSQLErrors(err)
+		}
+
+		result = append(result, r)
+	}
+
+	return containersTotal, result, nil
+}
+
+
 func (r *repository) DeleteContainer(ctx context.Context, namespace, name string) error {
 	tx, err := r.db.BeginTx(ctx, nil)
 	if err != nil {

repositories/metadata/postgresql/containers_test.go

@@ -58,3 +58,33 @@ func (s *postgreSQLRepositoryTestSuite) TestContainerOperations() {
 		"and-then-there-was-the-one",
 	}, list)
 }
+
+
+func (s *postgreSQLRepositoryTestSuite) TestContainersPagination() {
+	s.tp.On("Now").Return("2024-01-02T01:02:03Z").Times(4)
+
+	err := s.repo.CreateNamespace(s.ctx, "new-namespace")
+	s.Require().NoError(err)
+
+	total, list, err := s.repo.ListContainersByPage(s.ctx, defaultNamespace, 0, 100)
+	s.Require().NoError(err)
+	s.Require().Equal(uint64(0), total)
+	s.Require().Equal([]string{}, list)
+
+	err = s.repo.CreateContainer(s.ctx, defaultNamespace, "test-container1")
+	s.Require().NoError(err)
+
+	err = s.repo.CreateContainer(s.ctx, defaultNamespace, "test-container2")
+	s.Require().NoError(err)
+
+	err = s.repo.CreateContainer(s.ctx, defaultNamespace, "test-container3")
+	s.Require().NoError(err)
+
+	total, list, err = s.repo.ListContainersByPage(s.ctx, defaultNamespace, 0, 2)
+	s.Require().NoError(err)
+	s.Require().Equal(uint64(3), total)
+	s.Require().Equal([]string{
+		"test-container1",
+		"test-container2",
+	}, list)
+}

service/mock.go

@@ -60,6 +60,11 @@ func (m *Mock) ListContainers(_ context.Context, namespace string) ([]string, er
 	return args.Get(0).([]string), args.Error(1)
 }
 
+func (m *Mock) ListContainersByPage(_ context.Context, namespace string, pageNum uint64) (uint64, []string, error) {
+	args := m.Called(namespace, pageNum)
+	return args.Get(0).(uint64), args.Get(1).([]string), args.Error(2)
+}
+
 func (m *Mock) DeleteContainer(_ context.Context, namespace, name string) error {
 	args := m.Called(namespace, name)
 	return args.Error(0)

service/service.go

@@ -41,7 +41,8 @@ type Publisher interface {
 	ListNamespaces(ctx context.Context) ([]string, error)
 
 	ListContainers(ctx context.Context, namespace string) ([]string, error)
-
+	ListContainersByPage(ctx context.Context, namespace string, pageNum uint64) (uint64, []string, error)
+
 	ListPublishedVersions(ctx context.Context, namespace, container string) ([]models.Version, error)
 	ListPublishedVersionsByPage(ctx context.Context, namespace, container string, pageNum uint64) (uint64, []models.Version, error)
 
@@ -54,22 +55,24 @@ type service struct {
 	blobRepo         blob.Repository
 	versionsPageSize uint64
 	objectsPageSize  uint64
+	containersPageSize uint64
 }
 
 func NewManager(mdRepo metadata.Repository, blobRepo blob.Repository) Manager {
-	return newSvc(mdRepo, blobRepo, 50, 50)
+	return newSvc(mdRepo, blobRepo, 50, 50, 50)
 }
 
-func NewPublisher(mdRepo metadata.Repository, blobRepo blob.Repository, versionsPerPage, objectsPerPage uint64) Publisher {
-	return newSvc(mdRepo, blobRepo, versionsPerPage, objectsPerPage)
+func NewPublisher(mdRepo metadata.Repository, blobRepo blob.Repository, versionsPerPage, objectsPerPage, containersPerPage uint64) Publisher {
+	return newSvc(mdRepo, blobRepo, versionsPerPage, objectsPerPage, containersPerPage)
 }
 
-func newSvc(mdRepo metadata.Repository, blobRepo blob.Repository, versionsPerPage, objectsPerPage uint64) *service {
+func newSvc(mdRepo metadata.Repository, blobRepo blob.Repository, versionsPerPage, objectsPerPage, containersPerPage uint64) *service {
 	return &service{
 		mdRepo:           mdRepo,
 		blobRepo:         blobRepo,
 		versionsPageSize: versionsPerPage,
 		objectsPageSize:  objectsPerPage,
+		containersPageSize: containersPerPage,
 	}
 }
 
@@ -128,6 +131,26 @@ func (s *service) ListContainers(ctx context.Context, namespace string) ([]strin
 	return containers, mapMetadataErrors(err)
 }
 
+func (s *service) ListContainersByPage(ctx context.Context, namespace string, pageNum uint64) (uint64, []string, error) {
+	if pageNum < 1 {
+		pageNum = 1
+	}
+
+	offset := (pageNum - 1) * s.containersPageSize
+	limit := s.containersPageSize
+	totalContainers, containers, err := s.mdRepo.ListContainersByPage(ctx, namespace, offset, limit)
+	if err != nil {
+		return 0, nil, mapMetadataErrors(err)
+	}
+
+	totalPages := (totalContainers / s.containersPageSize)
+	if (totalContainers % s.containersPageSize) != 0 {
+		totalPages++
+	}
+
+	return totalPages, containers, mapMetadataErrors(err)
+}
+
 func (s *service) DeleteContainer(ctx context.Context, namespace, name string) error {
 	err := s.mdRepo.DeleteContainer(ctx, namespace, name)
 	return mapMetadataErrors(err)