Skip to content

Commit

Permalink
feat: support gitlab (#32)
Browse files Browse the repository at this point in the history
  • Loading branch information
@moul
moul committed Sep 11, 2018
1 parent e5c77a1 commit 371103a
Show file tree
Hide file tree
Showing 3 changed files with 249 additions and 57 deletions.
123 changes: 85 additions & 38 deletions fetch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,19 @@ package main
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"strings"
"net/url"
"os"
"sync"

"github.com/google/go-github/github"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/spf13/viper"
gitlab "github.com/xanzy/go-gitlab"
"go.uber.org/zap"
"golang.org/x/oauth2"
)
Expand Down Expand Up @@ -56,45 +59,94 @@ func newFetchCommand() *cobra.Command {

func fetch(opts *fetchOptions) error {
logger().Debug("fetch", zap.Stringer("opts", *opts))
ctx := context.Background()
ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: opts.GithubToken})
tc := oauth2.NewClient(ctx, ts)
client := github.NewClient(tc)

var (
wg sync.WaitGroup
allIssues []*github.Issue
out = make(chan []*github.Issue, 100)
allIssues []*Issue
out = make(chan []*Issue, 100)
)
wg.Add(len(opts.Repos))
for _, repo := range opts.Repos {
parts := strings.Split(repo, "/")
organization := parts[0]
repo := parts[1]
for _, repoURL := range opts.Repos {
repo := NewRepo(repoURL)
switch repo.Provider() {
case GitHubProvider:
ctx := context.Background()
ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: opts.GithubToken})
tc := oauth2.NewClient(ctx, ts)
client := github.NewClient(tc)

go func(repo string) {
total := 0
defer wg.Done()
opts := &github.IssueListByRepoOptions{State: "all"}
for {
issues, resp, err := client.Issues.ListByRepo(ctx, organization, repo, opts)
if err != nil {
log.Fatal(err)
return
go func(repo Repo) {
total := 0
defer wg.Done()
opts := &github.IssueListByRepoOptions{State: "all"}
for {
issues, resp, err := client.Issues.ListByRepo(ctx, repo.Namespace(), repo.Project(), opts)
if err != nil {
log.Fatal(err)
return
}
total += len(issues)
logger().Debug("paginate",
zap.String("provider", "github"),
zap.String("repo", repo.Canonical()),
zap.Int("new-issues", len(issues)),
zap.Int("total-issues", total),
)
normalizedIssues := []*Issue{}
for _, issue := range issues {
normalizedIssues = append(normalizedIssues, FromGitHubIssue(issue))
}
out <- normalizedIssues
if resp.NextPage == 0 {
break
}
opts.Page = resp.NextPage
}
total += len(issues)
logger().Debug("paginate",
zap.String("repo", repo),
zap.Int("new-issues", len(issues)),
zap.Int("total-issues", total),
)
out <- issues
if resp.NextPage == 0 {
break
if rateLimits, _, err := client.RateLimits(ctx); err == nil {
logger().Debug("github API rate limiting", zap.Stringer("limit", rateLimits.GetCore()))
}
opts.Page = resp.NextPage
}
}(repo)
}(repo)
case GitLabProvider:
go func(repo Repo) {
client := gitlab.NewClient(nil, os.Getenv("GITLAB_TOKEN"))
client.SetBaseURL(fmt.Sprintf("%s/api/v4/", repo.SiteURL()))

projectID := url.QueryEscape(repo.RepoPath())
total := 0
defer wg.Done()
opts := &gitlab.ListProjectIssuesOptions{
ListOptions: gitlab.ListOptions{
PerPage: 30,
Page: 1,
},
}
for {
issues, resp, err := client.Issues.ListProjectIssues(projectID, opts)
if err != nil {
logger().Error("failed to fetch issues", zap.Error(err))
return
}
total += len(issues)
logger().Debug("paginate",
zap.String("provider", "gitlab"),
zap.String("repo", repo.Canonical()),
zap.Int("new-issues", len(issues)),
zap.Int("total-issues", total),
)
normalizedIssues := []*Issue{}
for _, issue := range issues {
normalizedIssues = append(normalizedIssues, FromGitLabIssue(issue))
}
out <- normalizedIssues
if resp.NextPage == 0 {
break
}
opts.ListOptions.Page = resp.NextPage
}
}(repo)
default:
panic("should not happen")
}
}
wg.Wait()
close(out)
Expand All @@ -103,10 +155,5 @@ func fetch(opts *fetchOptions) error {
}

issuesJson, _ := json.MarshalIndent(allIssues, "", " ")
rateLimits, _, err := client.RateLimits(ctx)
if err != nil {
return err
}
logger().Debug("github API rate limiting", zap.Stringer("limit", rateLimits.GetCore()))
return errors.Wrap(ioutil.WriteFile(opts.DBOpts.Path, issuesJson, 0644), "failed to write db file")
return errors.Wrap(ioutil.WriteFile(opts.DBOpts.Path, issuesJson, 0644), "failed to write db")
}
114 changes: 95 additions & 19 deletions issue.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,24 @@ import (
"github.com/awalterschulze/gographviz"
"github.com/google/go-github/github"
"github.com/spf13/viper"
gitlab "github.com/xanzy/go-gitlab"
)

type Provider string

const (
UnknownProvider Provider = "unknown"
GitHubProvider = "github"
GitLabProvider = "gitlab"
)

type Issue struct {
github.Issue
// proxy
GitHub *github.Issue
GitLab *gitlab.Issue

// internal
Provider Provider
DependsOn IssueSlice
Blocks IssueSlice
weightMultiplier int
Expand All @@ -23,6 +37,58 @@ type Issue struct {
IsDuplicate int
LinkedWithEpic bool
Errors []error

// mapping
Number int
Title string
State string
Body string
RepoURL string
URL string
Labels []*IssueLabel
Assignees []*Profile
}

type IssueLabel struct {
Name string
Color string
}

type Profile struct {
Name string
Username string
}

func FromGitHubIssue(input *github.Issue) *Issue {
issue := &Issue{
Provider: GitHubProvider,
GitHub: input,
Number: *input.Number,
Title: *input.Title,
State: *input.State,
Body: *input.Body,
URL: *input.HTMLURL,
RepoURL: *input.RepositoryURL,
Labels: make([]*IssueLabel, 0),
Assignees: make([]*Profile, 0),
}
return issue
}

func FromGitLabIssue(input *gitlab.Issue) *Issue {
issue := &Issue{
Provider: GitLabProvider,
GitLab: input,
Number: input.ID,
Title: input.Title,
State: input.State,
URL: input.WebURL,
Body: input.Description,
RepoURL: input.Links.Project,
Labels: make([]*IssueLabel, 0),
Assignees: make([]*Profile, 0),
}
return issue
}

type IssueSlice []*Issue
Expand Down Expand Up @@ -51,7 +117,7 @@ func (s IssueSlice) ToMap() Issues {

func (i Issue) IsEpic() bool {
for _, label := range i.Labels {
if *label.Name == viper.GetString("epic-label") {
if label.Name == viper.GetString("epic-label") {
return true
}
}
Expand All @@ -60,7 +126,7 @@ func (i Issue) IsEpic() bool {
}

func (i Issue) Repo() string {
return strings.Split(*i.RepositoryURL, "/")[5]
return strings.Split(i.URL, "/")[5]
}

func (i Issue) FullRepo() string {
Expand All @@ -72,31 +138,31 @@ func (i Issue) RepoID() string {
}

func (i Issue) Owner() string {
return strings.Split(*i.RepositoryURL, "/")[4]
return strings.Split(i.URL, "/")[4]
}

func (i Issue) IsClosed() bool {
return *i.State == "closed"
return i.State == "closed"
}

func (i Issue) IsReady() bool {
return !i.IsOrphan && len(i.DependsOn) == 0
}

func (i Issue) NodeName() string {
return fmt.Sprintf(`%s#%d`, i.FullRepo(), *i.Number)
return fmt.Sprintf(`%s#%d`, i.FullRepo(), i.Number)
}

func (i Issue) NodeTitle() string {
title := fmt.Sprintf("%s: %s", i.NodeName(), *i.Title)
title := fmt.Sprintf("%s: %s", i.NodeName(), i.Title)
title = strings.Replace(html.EscapeString(wrap(title, 20)), "\n", "<br/>", -1)
labels := []string{}
for _, label := range i.Labels {
switch *label.Name {
switch label.Name {
case "t/step", "t/epic":
continue
}
labels = append(labels, fmt.Sprintf(`<td bgcolor="#%s">%s</td>`, *label.Color, *label.Name))
labels = append(labels, fmt.Sprintf(`<td bgcolor="#%s">%s</td>`, label.Color, label.Name))
}
labelsText := ""
if len(labels) > 0 {
Expand All @@ -106,7 +172,7 @@ func (i Issue) NodeTitle() string {
if len(i.Assignees) > 0 {
assignees := []string{}
for _, assignee := range i.Assignees {
assignees = append(assignees, *assignee.Login)
assignees = append(assignees, assignee.Username)
}
assigneeText = fmt.Sprintf(`<tr><td><font color="purple"><i>@%s</i></font></td></tr>`, strings.Join(assignees, ", @"))
}
Expand Down Expand Up @@ -199,7 +265,7 @@ func (i Issue) AddNodeToGraph(g *gographviz.Graph, parent string) error {
attrs["shape"] = "record"
attrs["style"] = `"rounded,filled"`
attrs["color"] = "lightblue"
attrs["href"] = escape(*i.HTMLURL)
attrs["href"] = escape(i.URL)

if i.IsEpic() {
attrs["shape"] = "oval"
Expand Down Expand Up @@ -247,28 +313,28 @@ func (issues Issues) prepare() error {
issue.BaseWeight = 1
}
for _, issue := range issues {
if issue.Body == nil {
if issue.Body == "" {
continue
}

if match := isDuplicateRegex.FindStringSubmatch(*issue.Body); match != nil {
if match := isDuplicateRegex.FindStringSubmatch(issue.Body); match != nil {
issue.IsDuplicate, _ = strconv.Atoi(match[len(match)-1])
}

if match := weightMultiplierRegex.FindStringSubmatch(*issue.Body); match != nil {
if match := weightMultiplierRegex.FindStringSubmatch(issue.Body); match != nil {
issue.weightMultiplier, _ = strconv.Atoi(match[len(match)-1])
}

if match := hideFromRoadmapRegex.FindStringSubmatch(*issue.Body); match != nil {
if match := hideFromRoadmapRegex.FindStringSubmatch(issue.Body); match != nil {
delete(issues, issue.NodeName())
continue
}

if match := baseWeightRegex.FindStringSubmatch(*issue.Body); match != nil {
if match := baseWeightRegex.FindStringSubmatch(issue.Body); match != nil {
issue.BaseWeight, _ = strconv.Atoi(match[len(match)-1])
}

for _, match := range dependsOnRegex.FindAllStringSubmatch(*issue.Body, -1) {
for _, match := range dependsOnRegex.FindAllStringSubmatch(issue.Body, -1) {
num := match[len(match)-1]
if num[0] == '#' {
num = fmt.Sprintf(`%s%s`, issue.FullRepo(), num)
Expand All @@ -284,7 +350,7 @@ func (issues Issues) prepare() error {
issues[num].IsOrphan = false
}

for _, match := range blocksRegex.FindAllStringSubmatch(*issue.Body, -1) {
for _, match := range blocksRegex.FindAllStringSubmatch(issue.Body, -1) {
num := match[len(match)-1]
if num[0] == '#' {
num = fmt.Sprintf(`%s%s`, issue.FullRepo(), num)
Expand All @@ -304,14 +370,24 @@ func (issues Issues) prepare() error {
if issue.IsDuplicate != 0 {
issue.Hidden = true
}
if issue.PullRequestLinks != nil {
if issue.IsPR() {
issue.Hidden = true
}
}
issues.processEpicLinks()
return nil
}

func (i Issue) IsPR() bool {
switch i.Provider {
case GitHubProvider:
return i.GitHub.PullRequestLinks != nil
case GitLabProvider:
return false // only fetching issues for now
}
panic("should not happen")
}

func (issues Issues) processEpicLinks() {
for _, issue := range issues {
issue.LinkedWithEpic = !issue.Hidden && (issue.IsEpic() || issue.BlocksAnEpic() || issue.DependsOnAnEpic())
Expand Down
Loading

0 comments on commit 371103a

Please sign in to comment.