From 6eafaddd5bf21e207624c4700b68c4b2dc9044bb Mon Sep 17 00:00:00 2001 From: Justin Sherrill Date: Tue, 7 Nov 2023 14:31:13 -0500 Subject: [PATCH] Add command to snapshot a rhel repo --- cmd/external-repos/main.go | 29 ++++++++++++++++++++++++++--- deployments/deployment.yaml | 6 ++++++ pkg/dao/interfaces.go | 2 +- pkg/dao/repository_configs.go | 26 ++++++++++++++++++++------ pkg/dao/repository_configs_mock.go | 20 ++++++++++---------- pkg/dao/repository_configs_test.go | 15 ++++++++++++++- 6 files changed, 77 insertions(+), 21 deletions(-) diff --git a/cmd/external-repos/main.go b/cmd/external-repos/main.go index 45574400b..6db6c49bc 100644 --- a/cmd/external-repos/main.go +++ b/cmd/external-repos/main.go @@ -15,6 +15,7 @@ import ( "github.com/content-services/content-sources-backend/pkg/tasks/queue" _ "github.com/golang-migrate/migrate/v4/source/file" _ "github.com/lib/pq" + "github.com/openlyinc/pointy" "github.com/rs/zerolog/log" "gorm.io/gorm" ) @@ -73,13 +74,27 @@ func main() { log.Panic().Err(errors[i]).Msg("Failed to introspect repository due to fatal errors") } log.Debug().Msgf("Inserted %d packages", count) + } else if args[1] == "snapshot" { + if len(args) < 3 { + log.Error().Msg("Usage: ./external_repos sync URL [URL2]...") + os.Exit(1) + } + var urls []string + for i := 2; i < len(args); i++ { + urls = append(urls, args[i]) + } + + err := enqueueSyncRepos(&urls) + if err != nil { + log.Warn().Msgf("Error enqueuing snapshot tasks: %v", err) + } } else if args[1] == "nightly-jobs" { err = enqueueIntrospectAllRepos() if err != nil { log.Error().Err(err).Msg("error queueing introspection tasks") } if config.Get().Features.Snapshots.Enabled { - err = enqueueSyncAllRepos() + err = enqueueSyncRepos(nil) if err != nil { log.Error().Err(err).Msg("error queueing snapshot tasks") } @@ -161,7 +176,7 @@ func enqueueIntrospectAllRepos() error { return nil } -func enqueueSyncAllRepos() error { +func enqueueSyncRepos(urls *[]string) error { q, err := queue.NewPgQueue(db.GetUrl()) if err != nil { return fmt.Errorf("error getting new task queue: %w", err) @@ -169,7 +184,15 @@ func enqueueSyncAllRepos() error { c := client.NewTaskClient(&q) repoConfigDao := dao.GetRepositoryConfigDao(db.DB) - repoConfigs, err := repoConfigDao.InternalOnly_ListReposToSnapshot() + var filter *dao.ListRepoFilter + if urls != nil { + filter = &dao.ListRepoFilter{ + URLs: urls, + RedhatOnly: pointy.Pointer(true), + } + } + repoConfigs, err := repoConfigDao.InternalOnly_ListReposToSnapshot(filter) + if err != nil { return fmt.Errorf("error getting repository configurations: %w", err) } diff --git a/deployments/deployment.yaml b/deployments/deployment.yaml index 02b1845b7..8fd53757c 100644 --- a/deployments/deployment.yaml +++ b/deployments/deployment.yaml @@ -52,6 +52,12 @@ objects: - introspect - https://cdn.redhat.com/content/dist/layered/rhel8/x86_64/ansible/2/os - https://cdn.redhat.com/content/dist/rhel8/8.8/x86_64/baseos/os + - name: snapshot-single-repo + inheritEnv: true + args: + - /external-repos + - snapshot + - https://cdn.redhat.com/content/dist/layered/rhel8/x86_64/ansible/2/os/ image: ${IMAGE}:${IMAGE_TAG} livenessProbe: failureThreshold: 3 diff --git a/pkg/dao/interfaces.go b/pkg/dao/interfaces.go index 2f1747f69..004167f5b 100644 --- a/pkg/dao/interfaces.go +++ b/pkg/dao/interfaces.go @@ -44,7 +44,7 @@ type RepositoryConfigDao interface { BulkCreate(newRepositories []api.RepositoryRequest) ([]api.RepositoryResponse, []error) Update(orgID, uuid string, repoParams api.RepositoryRequest) (bool, error) Fetch(orgID string, uuid string) (api.RepositoryResponse, error) - InternalOnly_ListReposToSnapshot() ([]models.RepositoryConfiguration, error) + InternalOnly_ListReposToSnapshot(filter *ListRepoFilter) ([]models.RepositoryConfiguration, error) List(orgID string, paginationData api.PaginationData, filterData api.FilterData) (api.RepositoryCollectionResponse, int64, error) Delete(orgID string, uuid string) error SoftDelete(orgID string, uuid string) error diff --git a/pkg/dao/repository_configs.go b/pkg/dao/repository_configs.go index 8ef769f9e..16a79d398 100644 --- a/pkg/dao/repository_configs.go +++ b/pkg/dao/repository_configs.go @@ -220,19 +220,33 @@ func (r repositoryConfigDaoImpl) bulkCreate(tx *gorm.DB, newRepositories []api.R return []api.RepositoryResponse{}, errorList } -func (p repositoryConfigDaoImpl) InternalOnly_ListReposToSnapshot() ([]models.RepositoryConfiguration, error) { +type ListRepoFilter struct { + URLs *[]string + RedhatOnly *bool +} + +func (p repositoryConfigDaoImpl) InternalOnly_ListReposToSnapshot(filter *ListRepoFilter) ([]models.RepositoryConfiguration, error) { var dbRepos []models.RepositoryConfiguration - var result *gorm.DB + var query *gorm.DB interval := fmt.Sprintf("%v hours", config.SnapshotInterval) if config.Get().Options.AlwaysRunCronTasks { - result = p.db.Where("snapshot IS TRUE").Find(&dbRepos) + query = p.db.Where("snapshot IS TRUE") } else { - result = p.db.Where("snapshot IS TRUE").Joins("LEFT JOIN tasks on last_snapshot_task_uuid = tasks.id"). + query = p.db.Where("snapshot IS TRUE").Joins("LEFT JOIN tasks on last_snapshot_task_uuid = tasks.id"). Where(p.db.Where("tasks.queued_at <= (current_date - cast(? as interval))", interval). Or("tasks.status NOT IN ?", []string{config.TaskStatusCompleted, config.TaskStatusPending, config.TaskStatusRunning}). - Or("last_snapshot_task_uuid is NULL")). - Find(&dbRepos) + Or("last_snapshot_task_uuid is NULL")) + } + if filter != nil { + query = query.Joins("INNER JOIN repositories r on r.uuid = repository_configurations.repository_uuid") + if filter.RedhatOnly != nil && *filter.RedhatOnly { + query = query.Where("r.origin = ?", config.OriginRedHat) + } + if filter.URLs != nil { + query = query.Where("r.url in ?", *filter.URLs) + } } + result := query.Find(&dbRepos) if result.Error != nil { return dbRepos, result.Error diff --git a/pkg/dao/repository_configs_mock.go b/pkg/dao/repository_configs_mock.go index 01d6d7516..9fc4891e7 100644 --- a/pkg/dao/repository_configs_mock.go +++ b/pkg/dao/repository_configs_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.33.0. DO NOT EDIT. +// Code generated by mockery v2.36.1. DO NOT EDIT. package dao @@ -177,25 +177,25 @@ func (_m *MockRepositoryConfigDao) InternalOnly_FetchRepoConfigsForRepoUUID(uuid return r0 } -// InternalOnly_ListReposToSnapshot provides a mock function with given fields: -func (_m *MockRepositoryConfigDao) InternalOnly_ListReposToSnapshot() ([]models.RepositoryConfiguration, error) { - ret := _m.Called() +// InternalOnly_ListReposToSnapshot provides a mock function with given fields: filter +func (_m *MockRepositoryConfigDao) InternalOnly_ListReposToSnapshot(filter *ListRepoFilter) ([]models.RepositoryConfiguration, error) { + ret := _m.Called(filter) var r0 []models.RepositoryConfiguration var r1 error - if rf, ok := ret.Get(0).(func() ([]models.RepositoryConfiguration, error)); ok { - return rf() + if rf, ok := ret.Get(0).(func(*ListRepoFilter) ([]models.RepositoryConfiguration, error)); ok { + return rf(filter) } - if rf, ok := ret.Get(0).(func() []models.RepositoryConfiguration); ok { - r0 = rf() + if rf, ok := ret.Get(0).(func(*ListRepoFilter) []models.RepositoryConfiguration); ok { + r0 = rf(filter) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]models.RepositoryConfiguration) } } - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() + if rf, ok := ret.Get(1).(func(*ListRepoFilter) error); ok { + r1 = rf(filter) } else { r1 = ret.Error(1) } diff --git a/pkg/dao/repository_configs_test.go b/pkg/dao/repository_configs_test.go index f260128a4..211f167fd 100644 --- a/pkg/dao/repository_configs_test.go +++ b/pkg/dao/repository_configs_test.go @@ -1695,6 +1695,7 @@ type RepoToSnapshotTest struct { Opts *seeds.TaskSeedOptions Included bool OptionAlwaysRunCronTasks bool + Filter *ListRepoFilter } func (suite *RepositoryConfigSuite) TestListReposToSnapshot() { @@ -1734,6 +1735,18 @@ func (suite *RepositoryConfigSuite) TestListReposToSnapshot() { Opts: &seeds.TaskSeedOptions{RepoConfigUUID: repo.UUID, OrgID: repo.OrgID, Status: config.TaskStatusFailed}, Included: true, }, + { + Name: "Previous Snapshot Failed, and url specified", + Opts: &seeds.TaskSeedOptions{RepoConfigUUID: repo.UUID, OrgID: repo.OrgID, Status: config.TaskStatusFailed}, + Included: true, + Filter: &ListRepoFilter{URLs: &[]string{repo.URL}}, + }, + { + Name: "Previous Snapshot Failed, and url specified", + Opts: &seeds.TaskSeedOptions{RepoConfigUUID: repo.UUID, OrgID: repo.OrgID, Status: config.TaskStatusFailed}, + Included: false, + Filter: &ListRepoFilter{RedhatOnly: pointy.Pointer(true)}, + }, { Name: "Previous Snapshot was successful and recent", Opts: &seeds.TaskSeedOptions{RepoConfigUUID: repo.UUID, OrgID: repo.OrgID, Status: config.TaskStatusCompleted}, @@ -1763,7 +1776,7 @@ func (suite *RepositoryConfigSuite) TestListReposToSnapshot() { config.Get().Options.AlwaysRunCronTasks = testCase.OptionAlwaysRunCronTasks - afterRepos, err := dao.InternalOnly_ListReposToSnapshot() + afterRepos, err := dao.InternalOnly_ListReposToSnapshot(testCase.Filter) assert.NoError(t, err) for i := range afterRepos { if repo.UUID == afterRepos[i].UUID {