diff --git a/api/v1/v1.go b/api/v1/v1.go index e6440ee..af719e5 100644 --- a/api/v1/v1.go +++ b/api/v1/v1.go @@ -58,6 +58,34 @@ func fn(labels ...string) string { return fmt.Sprintf("[%s():%s]", shortname, strings.Join(labels, ":")) } +func getBatchedSlices(batchSize int, unbatched ...string) [][]string { + batchedSlices := make([][]string, 0) + + index := 0 + + for range unbatched { + batchedSlice := make([]string, 0) + + for c := 0; c < batchSize; c++ { + batchedSlice = append(batchedSlice, unbatched[index]) + + index++ + + if index == len(unbatched) { + break + } + } + + batchedSlices = append(batchedSlices, batchedSlice) + + if index == len(unbatched) { + break + } + } + + return batchedSlices +} + // CollectTags collects information on tags present in remote registry and [local] Docker daemon, // makes required comparisons between them and spits organized info back as collection.Collection func (api *API) CollectTags(refs ...string) (*collection.Collection, error) { @@ -65,56 +93,66 @@ func (api *API) CollectTags(refs ...string) (*collection.Collection, error) { return nil, fmt.Errorf("no image references passed") } - log.Debugf("%s references: %+v", fn(), refs) - - repos, err := repository.ParseRefs(refs) + _, err := repository.ParseRefs(refs) if err != nil { return nil, err } - for _, repo := range repos { - log.Debugf("%s repository: %+v", fn(), repo) - } - done := make(chan error, len(repos)) tags := make(map[string][]*tag.Tag) - for _, repo := range repos { - go func(repo *repository.Repository, done chan error) { - log.Infof("ANALYZE %s", repo.Ref()) + batchedSlicesOfRefs := getBatchedSlices(api.config.ConcurrentRequests, refs...) - username, password, _ := api.dockerClient.Config().GetCredentials(repo.Registry()) + for bindex, brefs := range batchedSlicesOfRefs { + log.Infof("BATCH %d of %d", bindex+1, len(batchedSlicesOfRefs)) - remoteTags, err := remote.FetchTags(repo, username, password) - if err != nil { - done <- err - return - } - log.Debugf("%s remote tags: %+v", fn(repo.Ref()), remoteTags) + log.Debugf("%s references: %+v", fn(), brefs) + + repos, _ := repository.ParseRefs(brefs) + for _, repo := range repos { + log.Debugf("%s repository: %+v", fn(), repo) + } - localTags, _ := local.FetchTags(repo, api.dockerClient) + done := make(chan error, len(repos)) - log.Debugf("%s local tags: %+v", fn(repo.Ref()), localTags) + for _, repo := range repos { + go func(repo *repository.Repository, done chan error) { + log.Infof("ANALYZE %s", repo.Ref()) - sortedKeys, tagNames, joinedTags := tag.Join( - remoteTags, - localTags, - repo.Tags(), - ) - log.Debugf("%s joined tags: %+v", fn(repo.Ref()), joinedTags) + username, password, _ := api.dockerClient.Config().GetCredentials(repo.Registry()) - tags[repo.Ref()] = tag.Collect(sortedKeys, tagNames, joinedTags) + remoteTags, err := remote.FetchTags(repo, username, password) + if err != nil { + done <- err + return + } + log.Debugf("%s remote tags: %+v", fn(repo.Ref()), remoteTags) - done <- nil + localTags, _ := local.FetchTags(repo, api.dockerClient) - log.Infof("FETCHED %s", repo.Ref()) + log.Debugf("%s local tags: %+v", fn(repo.Ref()), localTags) - return - }(repo, done) - } + sortedKeys, tagNames, joinedTags := tag.Join( + remoteTags, + localTags, + repo.Tags(), + ) + log.Debugf("%s joined tags: %+v", fn(repo.Ref()), joinedTags) - if err := wait.Until(done); err != nil { - return nil, err + tags[repo.Ref()] = tag.Collect(sortedKeys, tagNames, joinedTags) + + done <- nil + + log.Infof("FETCHED %s", repo.Ref()) + + return + }(repo, done) + } + + if err := wait.Until(done); err != nil { + return nil, err + } } + log.Debugf("%s tags: %+v", fn(), tags) return collection.New(refs, tags) diff --git a/api/v1/v1_test.go b/api/v1/v1_test.go index 749af81..ef8f6bc 100644 --- a/api/v1/v1_test.go +++ b/api/v1/v1_test.go @@ -181,3 +181,40 @@ func TestGetPushPrefix(t *testing.T) { assert.Equal(expected, actual) } } + +func TestGetBatchedSlices(t *testing.T) { + var unbatched = []string{ + "unbatched/repo01", + "unbatched/repo02", + "unbatched/repo03", + "unbatched/repo04", + "unbatched/repo05", + "unbatched/repo06", + "unbatched/repo07", + "unbatched/repo08", + "unbatched/repo09", + "unbatched/repo10", + } + + var testCases = map[int][][]string{ + 1: [][]string{{"unbatched/repo01"}, {"unbatched/repo02"}, {"unbatched/repo03"}, {"unbatched/repo04"}, {"unbatched/repo05"}, {"unbatched/repo06"}, {"unbatched/repo07"}, {"unbatched/repo08"}, {"unbatched/repo09"}, {"unbatched/repo10"}}, + 3: [][]string{{"unbatched/repo01", "unbatched/repo02", "unbatched/repo03"}, {"unbatched/repo04", "unbatched/repo05", "unbatched/repo06"}, {"unbatched/repo07", "unbatched/repo08", "unbatched/repo09"}, {"unbatched/repo10"}}, + 7: [][]string{{"unbatched/repo01", "unbatched/repo02", "unbatched/repo03", "unbatched/repo04", "unbatched/repo05", "unbatched/repo06", "unbatched/repo07"}, {"unbatched/repo08", "unbatched/repo09", "unbatched/repo10"}}, + 10: [][]string{{"unbatched/repo01", "unbatched/repo02", "unbatched/repo03", "unbatched/repo04", "unbatched/repo05", "unbatched/repo06", "unbatched/repo07", "unbatched/repo08", "unbatched/repo09", "unbatched/repo10"}}, + 11: [][]string{{"unbatched/repo01", "unbatched/repo02", "unbatched/repo03", "unbatched/repo04", "unbatched/repo05", "unbatched/repo06", "unbatched/repo07", "unbatched/repo08", "unbatched/repo09", "unbatched/repo10"}}, + 100: [][]string{{"unbatched/repo01", "unbatched/repo02", "unbatched/repo03", "unbatched/repo04", "unbatched/repo05", "unbatched/repo06", "unbatched/repo07", "unbatched/repo08", "unbatched/repo09", "unbatched/repo10"}}, + } + + var assert = assert.New(t) + + for batchSize, expectedBatchedSlices := range testCases { + actualBatchedSlices := getBatchedSlices(batchSize, unbatched...) + + assert.Equalf( + expectedBatchedSlices, + actualBatchedSlices, + "unexpected result for batch size: %d", + batchSize, + ) + } +}