diff --git a/.circleci/config.yml b/.circleci/config.yml index bf24ead3c..fa64c202c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,6 +4,8 @@ jobs: docker: - image: circleci/golang:1.11 working_directory: /go/src/moul.io/depviz + environment: + GO111MODULE: "on" steps: - checkout - run: go get -v -t -d ./... diff --git a/fetch.go b/fetch.go index 8c5d9e1ba..1fb28fb10 100644 --- a/fetch.go +++ b/fetch.go @@ -3,9 +3,10 @@ package main import ( "context" "encoding/json" + "fmt" "io/ioutil" "log" - "strings" + "os" "sync" "github.com/google/go-github/github" @@ -13,6 +14,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" + gitlab "github.com/xanzy/go-gitlab" "go.uber.org/zap" "golang.org/x/oauth2" ) @@ -56,45 +58,95 @@ func newFetchCommand() *cobra.Command { func fetch(opts *fetchOptions) error { logger().Debug("fetch", zap.Stringer("opts", *opts)) - ctx := context.Background() - ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: opts.GithubToken}) - tc := oauth2.NewClient(ctx, ts) - client := github.NewClient(tc) var ( wg sync.WaitGroup - allIssues []*github.Issue - out = make(chan []*github.Issue, 100) + allIssues []*Issue + out = make(chan []*Issue, 100) ) wg.Add(len(opts.Repos)) - for _, repo := range opts.Repos { - parts := strings.Split(repo, "/") - organization := parts[0] - repo := parts[1] + for _, repoURL := range opts.Repos { + repo := NewRepo(repoURL) + switch repo.Provider() { + case GitHubProvider: + ctx := context.Background() + ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: opts.GithubToken}) + tc := oauth2.NewClient(ctx, ts) + client := github.NewClient(tc) - go func(repo string) { - total := 0 - defer wg.Done() - opts := &github.IssueListByRepoOptions{State: "all"} - for { - issues, resp, err := client.Issues.ListByRepo(ctx, organization, repo, opts) - if err != nil { - log.Fatal(err) - return + go func(repo Repo) { + total := 0 + defer wg.Done() + opts := &github.IssueListByRepoOptions{State: "all"} + for { + issues, resp, err := client.Issues.ListByRepo(ctx, repo.Namespace(), repo.Project(), opts) + if err != nil { + log.Fatal(err) + return + } + total += len(issues) + logger().Debug("paginate", + zap.String("provider", "github"), + zap.String("repo", repo.Canonical()), + zap.Int("new-issues", len(issues)), + zap.Int("total-issues", total), + ) + normalizedIssues := []*Issue{} + for _, issue := range issues { + normalizedIssues = append(normalizedIssues, FromGitHubIssue(issue)) + } + out <- normalizedIssues + if resp.NextPage == 0 { + break + } + opts.Page = resp.NextPage } - total += len(issues) - logger().Debug("paginate", - zap.String("repo", repo), - zap.Int("new-issues", len(issues)), - zap.Int("total-issues", total), - ) - out <- issues - if resp.NextPage == 0 { - break + if rateLimits, _, err := client.RateLimits(ctx); err == nil { + logger().Debug("github API rate limiting", zap.Stringer("limit", rateLimits.GetCore())) } - opts.Page = resp.NextPage - } - }(repo) + }(repo) + case GitLabProvider: + go func(repo Repo) { + client := gitlab.NewClient(nil, os.Getenv("GITLAB_TOKEN")) + client.SetBaseURL(fmt.Sprintf("%s/api/v4", repo.SiteURL())) + + //projectID := url.QueryEscape(repo.RepoPath()) + projectID := repo.RepoPath() + total := 0 + defer wg.Done() + opts := &gitlab.ListProjectIssuesOptions{ + ListOptions: gitlab.ListOptions{ + PerPage: 30, + Page: 1, + }, + } + for { + issues, resp, err := client.Issues.ListProjectIssues(projectID, opts) + if err != nil { + logger().Error("failed to fetch issues", zap.Error(err)) + return + } + total += len(issues) + logger().Debug("paginate", + zap.String("provider", "gitlab"), + zap.String("repo", repo.Canonical()), + zap.Int("new-issues", len(issues)), + zap.Int("total-issues", total), + ) + normalizedIssues := []*Issue{} + for _, issue := range issues { + normalizedIssues = append(normalizedIssues, FromGitLabIssue(issue)) + } + out <- normalizedIssues + if resp.NextPage == 0 { + break + } + opts.ListOptions.Page = resp.NextPage + } + }(repo) + default: + panic("should not happen") + } } wg.Wait() close(out) @@ -103,10 +155,5 @@ func fetch(opts *fetchOptions) error { } issuesJson, _ := json.MarshalIndent(allIssues, "", " ") - rateLimits, _, err := client.RateLimits(ctx) - if err != nil { - return err - } - logger().Debug("github API rate limiting", zap.Stringer("limit", rateLimits.GetCore())) - return errors.Wrap(ioutil.WriteFile(opts.DBOpts.Path, issuesJson, 0644), "failed to write db file") + return errors.Wrap(ioutil.WriteFile(opts.DBOpts.Path, issuesJson, 0644), "failed to write db") } diff --git a/go.mod b/go.mod index 1cbe9d933..26227cd7a 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module moul.io/depviz require ( github.com/awalterschulze/gographviz v0.0.0-20180813113015-16ed621cdb51 github.com/fsnotify/fsnotify v1.4.7 // indirect + github.com/gilliek/go-opml v1.0.0 github.com/golang/protobuf v1.2.0 // indirect github.com/google/go-github v17.0.0+incompatible github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect @@ -18,6 +19,7 @@ require ( github.com/spf13/jwalterweatherman v0.0.0-20180814060501-14d3d4c51834 // indirect github.com/spf13/pflag v1.0.2 github.com/spf13/viper v1.1.0 + github.com/xanzy/go-gitlab v0.11.0 go.uber.org/atomic v1.3.2 // indirect go.uber.org/multierr v1.1.0 // indirect go.uber.org/zap v1.9.1 diff --git a/go.sum b/go.sum index 2a326267e..6613cc49c 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ github.com/awalterschulze/gographviz v0.0.0-20180813113015-16ed621cdb51/go.mod h github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gilliek/go-opml v1.0.0 h1:X8xVjtySRXU/x6KvaiXkn7OV3a4DHqxY8Rpv6U/JvCY= +github.com/gilliek/go-opml v1.0.0/go.mod h1:fOxmtlzyBvUjU6bjpdjyxCGlWz+pgtAHrHf/xRZl3lk= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= @@ -33,6 +35,8 @@ github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.1.0 h1:V7OZpY8i3C1x/pDmU0zNNlfVoDz112fSYvtWMjjS3f4= github.com/spf13/viper v1.1.0/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= +github.com/xanzy/go-gitlab v0.11.0 h1:H9exAT9A+VQSkDnoJPxMhC4uJip8kQ8edlVC28gFuOI= +github.com/xanzy/go-gitlab v0.11.0/go.mod h1:CRKHkvFWNU6C3AEfqLWjnCNnAs4nj8Zk95rX2S3X6Mw= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= diff --git a/graphviz.go b/graphviz.go index 19ead9212..55d48f23b 100644 --- a/graphviz.go +++ b/graphviz.go @@ -106,7 +106,7 @@ func roadmapGraph(issues Issues, opts *renderOptions) (string, error) { // issue nodes issueNumbers := []string{} for _, issue := range issues { - issueNumbers = append(issueNumbers, issue.NodeName()) + issueNumbers = append(issueNumbers, issue.URL) } sort.Strings(issueNumbers) for _, id := range issueNumbers { diff --git a/issue.go b/issue.go index b6fb56e00..4c8a8afef 100644 --- a/issue.go +++ b/issue.go @@ -3,6 +3,7 @@ package main import ( "fmt" "html" + "net/url" "regexp" "strconv" "strings" @@ -10,19 +11,98 @@ import ( "github.com/awalterschulze/gographviz" "github.com/google/go-github/github" "github.com/spf13/viper" + gitlab "github.com/xanzy/go-gitlab" +) + +type Provider string + +const ( + UnknownProvider Provider = "unknown" + GitHubProvider = "github" + GitLabProvider = "gitlab" ) type Issue struct { - github.Issue + // proxy + GitHub *github.Issue + GitLab *gitlab.Issue + + // internal + Provider Provider DependsOn IssueSlice Blocks IssueSlice weightMultiplier int BaseWeight int IsOrphan bool Hidden bool - IsDuplicate int + Duplicates []string LinkedWithEpic bool Errors []error + + // mapping + Number int + Title string + State string + Body string + RepoURL string + URL string + Labels []*IssueLabel + Assignees []*Profile +} + +type IssueLabel struct { + Name string + Color string +} + +type Profile struct { + Name string + Username string +} + +func FromGitHubIssue(input *github.Issue) *Issue { + body := "" + if input.Body != nil { + body = *input.Body + } + issue := &Issue{ + Provider: GitHubProvider, + GitHub: input, + Number: *input.Number, + Title: *input.Title, + State: *input.State, + Body: body, + URL: *input.HTMLURL, + RepoURL: *input.RepositoryURL, + Labels: make([]*IssueLabel, 0), + Assignees: make([]*Profile, 0), + } + return issue +} + +func FromGitLabIssue(input *gitlab.Issue) *Issue { + issue := &Issue{ + Provider: GitLabProvider, + GitLab: input, + Number: input.ID, + Title: input.Title, + State: input.State, + URL: input.WebURL, + Body: input.Description, + RepoURL: input.Links.Project, + Labels: make([]*IssueLabel, 0), + Assignees: make([]*Profile, 0), + } + return issue +} + +func (i Issue) Path() string { + u, err := url.Parse(i.URL) + if err != nil { + return "" + } + parts := strings.Split(u.Path, "/") + return strings.Join(parts[:len(parts)-2], "/") } type IssueSlice []*Issue @@ -44,14 +124,19 @@ func (m Issues) ToSlice() IssueSlice { func (s IssueSlice) ToMap() Issues { m := Issues{} for _, issue := range s { - m[issue.NodeName()] = issue + m[issue.URL] = issue } return m } +func (i Issue) ProviderURL() string { + u, _ := url.Parse(i.URL) + return fmt.Sprintf("%s://%s", u.Scheme, u.Host) +} + func (i Issue) IsEpic() bool { for _, label := range i.Labels { - if *label.Name == viper.GetString("epic-label") { + if label.Name == viper.GetString("epic-label") { return true } } @@ -60,7 +145,7 @@ func (i Issue) IsEpic() bool { } func (i Issue) Repo() string { - return strings.Split(*i.RepositoryURL, "/")[5] + return strings.Split(i.URL, "/")[5] } func (i Issue) FullRepo() string { @@ -68,15 +153,18 @@ func (i Issue) FullRepo() string { } func (i Issue) RepoID() string { - return strings.Replace(i.FullRepo(), "/", "", -1) + id := i.FullRepo() + id = strings.Replace(id, "/", "", -1) + id = strings.Replace(id, "-", "", -1) + return id } func (i Issue) Owner() string { - return strings.Split(*i.RepositoryURL, "/")[4] + return strings.Split(i.URL, "/")[4] } func (i Issue) IsClosed() bool { - return *i.State == "closed" + return i.State == "closed" } func (i Issue) IsReady() bool { @@ -84,19 +172,20 @@ func (i Issue) IsReady() bool { } func (i Issue) NodeName() string { - return fmt.Sprintf(`%s#%d`, i.FullRepo(), *i.Number) + return fmt.Sprintf(`%s#%d`, i.FullRepo(), i.Number) } func (i Issue) NodeTitle() string { - title := fmt.Sprintf("%s: %s", i.NodeName(), *i.Title) + title := fmt.Sprintf("%s: %s", i.NodeName(), i.Title) + title = strings.Replace(title, "|", "-", -1) title = strings.Replace(html.EscapeString(wrap(title, 20)), "\n", "
", -1) labels := []string{} for _, label := range i.Labels { - switch *label.Name { + switch label.Name { case "t/step", "t/epic": continue } - labels = append(labels, fmt.Sprintf(`%s`, *label.Color, *label.Name)) + labels = append(labels, fmt.Sprintf(`%s`, label.Color, label.Name)) } labelsText := "" if len(labels) > 0 { @@ -106,7 +195,7 @@ func (i Issue) NodeTitle() string { if len(i.Assignees) > 0 { assignees := []string{} for _, assignee := range i.Assignees { - assignees = append(assignees, *assignee.Login) + assignees = append(assignees, assignee.Username) } assigneeText = fmt.Sprintf(`@%s`, strings.Join(assignees, ", @")) } @@ -121,6 +210,23 @@ func (i Issue) NodeTitle() string { return fmt.Sprintf(`<%s%s%s
%s
>`, title, labelsText, assigneeText, errorsText) } +func (i Issue) GetRelativeIssueURL(target string) string { + if strings.Contains(target, "://") { + return target + } + + u, err := url.Parse(target) + if err != nil { + return "" + } + path := u.Path + if path == "" { + path = i.Path() + } + + return fmt.Sprintf("%s%s/issues/%s", i.ProviderURL(), path, u.Fragment) +} + func (i Issue) BlocksAnEpic() bool { for _, dep := range i.Blocks { if dep.IsEpic() || dep.BlocksAnEpic() { @@ -179,10 +285,10 @@ func (i Issue) AddEdgesToGraph(g *gographviz.Graph) error { attrs["color"] = "orange" attrs["style"] = "dashed" } - //log.Print("edge", i.NodeName(), "->", dependency.NodeName()) + //log.Print("edge", escape(i.URL), "->", escape(dependency.URL)) if err := g.AddEdge( - escape(i.NodeName()), - escape(dependency.NodeName()), + escape(i.URL), + escape(dependency.URL), true, attrs, ); err != nil { @@ -199,7 +305,7 @@ func (i Issue) AddNodeToGraph(g *gographviz.Graph, parent string) error { attrs["shape"] = "record" attrs["style"] = `"rounded,filled"` attrs["color"] = "lightblue" - attrs["href"] = escape(*i.HTMLURL) + attrs["href"] = escape(i.URL) if i.IsEpic() { attrs["shape"] = "oval" @@ -221,19 +327,18 @@ func (i Issue) AddNodeToGraph(g *gographviz.Graph, parent string) error { attrs["color"] = "gray" } - //log.Print("node", i.NodeName(), parent) return g.AddNode( parent, - escape(i.NodeName()), + escape(i.URL), attrs, ) } func (issues Issues) prepare() error { var ( - dependsOnRegex, _ = regexp.Compile(`(?i)(require|requires|blocked by|block by|depend on|depends on|parent of) ([a-z/]*#[0-9]+)`) - blocksRegex, _ = regexp.Compile(`(?i)(blocks|block|address|addresses|part of|child of|fix|fixes) ([a-z/]*#[0-9]+)`) - isDuplicateRegex, _ = regexp.Compile(`(?i)(duplicates|duplicate|dup of|dup|duplicate of) #([0-9]+)`) + dependsOnRegex, _ = regexp.Compile(`(?i)(require|requires|blocked by|block by|depend on|depends on|parent of) ([a-z0-9:/_.-]+|[a-z0-9/_-]*#[0-9]+)`) + blocksRegex, _ = regexp.Compile(`(?i)(blocks|block|address|addresses|part of|child of|fix|fixes) ([a-z0-9:/_.-]+|[a-z0-9/_-]*#[0-9]+)`) + isDuplicateRegex, _ = regexp.Compile(`(?i)(duplicates|duplicate|dup of|dup|duplicate of) ([a-z0-9:/_.-]+|[a-z0-9/_-]*#[0-9]+)`) weightMultiplierRegex, _ = regexp.Compile(`(?i)(depviz.weight_multiplier[:= ]+)([0-9]+)`) baseWeightRegex, _ = regexp.Compile(`(?i)(depviz.base_weight[:= ]+)([0-9]+)`) hideFromRoadmapRegex, _ = regexp.Compile(`(?i)(depviz.hide_from_roadmap)`) // FIXME: use label @@ -247,33 +352,31 @@ func (issues Issues) prepare() error { issue.BaseWeight = 1 } for _, issue := range issues { - if issue.Body == nil { + if issue.Body == "" { continue } - if match := isDuplicateRegex.FindStringSubmatch(*issue.Body); match != nil { - issue.IsDuplicate, _ = strconv.Atoi(match[len(match)-1]) + if match := isDuplicateRegex.FindStringSubmatch(issue.Body); match != nil { + issue.Duplicates = append(issue.Duplicates, issue.GetRelativeIssueURL(match[len(match)-1])) } - if match := weightMultiplierRegex.FindStringSubmatch(*issue.Body); match != nil { + if match := weightMultiplierRegex.FindStringSubmatch(issue.Body); match != nil { issue.weightMultiplier, _ = strconv.Atoi(match[len(match)-1]) } - if match := hideFromRoadmapRegex.FindStringSubmatch(*issue.Body); match != nil { - delete(issues, issue.NodeName()) + if match := hideFromRoadmapRegex.FindStringSubmatch(issue.Body); match != nil { + delete(issues, issue.URL) continue } - if match := baseWeightRegex.FindStringSubmatch(*issue.Body); match != nil { + if match := baseWeightRegex.FindStringSubmatch(issue.Body); match != nil { issue.BaseWeight, _ = strconv.Atoi(match[len(match)-1]) } - for _, match := range dependsOnRegex.FindAllStringSubmatch(*issue.Body, -1) { - num := match[len(match)-1] - if num[0] == '#' { - num = fmt.Sprintf(`%s%s`, issue.FullRepo(), num) - } + for _, match := range dependsOnRegex.FindAllStringSubmatch(issue.Body, -1) { + num := issue.GetRelativeIssueURL(match[len(match)-1]) dep, found := issues[num] + //fmt.Println(issue.URL, num, found, match[len(match)-1]) if !found { issue.Errors = append(issue.Errors, fmt.Errorf("parent %q not found", num)) continue @@ -284,11 +387,8 @@ func (issues Issues) prepare() error { issues[num].IsOrphan = false } - for _, match := range blocksRegex.FindAllStringSubmatch(*issue.Body, -1) { - num := match[len(match)-1] - if num[0] == '#' { - num = fmt.Sprintf(`%s%s`, issue.FullRepo(), num) - } + for _, match := range blocksRegex.FindAllStringSubmatch(issue.Body, -1) { + num := issue.GetRelativeIssueURL(match[len(match)-1]) dep, found := issues[num] if !found { issue.Errors = append(issue.Errors, fmt.Errorf("child %q not found", num)) @@ -301,10 +401,10 @@ func (issues Issues) prepare() error { } } for _, issue := range issues { - if issue.IsDuplicate != 0 { + if len(issue.Duplicates) > 0 { issue.Hidden = true } - if issue.PullRequestLinks != nil { + if issue.IsPR() { issue.Hidden = true } } @@ -312,6 +412,16 @@ func (issues Issues) prepare() error { return nil } +func (i Issue) IsPR() bool { + switch i.Provider { + case GitHubProvider: + return i.GitHub.PullRequestLinks != nil + case GitLabProvider: + return false // only fetching issues for now + } + panic("should not happen") +} + func (issues Issues) processEpicLinks() { for _, issue := range issues { issue.LinkedWithEpic = !issue.Hidden && (issue.IsEpic() || issue.BlocksAnEpic() || issue.DependsOnAnEpic()) diff --git a/repo.go b/repo.go new file mode 100644 index 000000000..03582627b --- /dev/null +++ b/repo.go @@ -0,0 +1,69 @@ +package main + +import ( + "fmt" + "net/url" + "strings" +) + +type Repo string + +func NewRepo(path string) Repo { + parts := strings.Split(path, "/") + if len(parts) < 3 { + return Repo(fmt.Sprintf("https://github.com/%s", path)) + } + if !strings.Contains(path, "://") { + return Repo(fmt.Sprintf("https://%s", path)) + } + return Repo(path) +} + +// FIXME: make something more reliable +func (r Repo) Provider() Provider { + if strings.Contains(string(r), "github.com") { + return GitHubProvider + } + return GitLabProvider +} + +func (r Repo) Namespace() string { + u, err := url.Parse(string(r)) + if err != nil { + return "" + } + parts := strings.Split(u.Path, "/")[1:] + return strings.Join(parts[:len(parts)-1], "/") +} + +func (r Repo) Project() string { + parts := strings.Split(string(r), "/") + return parts[len(parts)-1] +} + +func (r Repo) Canonical() string { + // FIXME: use something smarter (the shortest unique response) + return string(r) +} + +func (r Repo) SiteURL() string { + switch r.Provider() { + case GitHubProvider: + return "https://github.com" + case GitLabProvider: + u, err := url.Parse(string(r)) + if err != nil { + return "" + } + return fmt.Sprintf("%s://%s", u.Scheme, u.Host) + } + panic("should not happen") +} + +func (r Repo) RepoPath() string { + u, err := url.Parse(string(r)) + if err != nil { + panic(err) + } + return u.Path[1:] +}