Merge pull request #38 from moul/dev/moul/support-gitlab

Initial GitLab support

.circleci/config.yml

@@ -4,6 +4,8 @@ jobs:
     docker:
       - image: circleci/golang:1.11
     working_directory: /go/src/moul.io/depviz
+    environment:
+      GO111MODULE: "on"
     steps:
       - checkout
       - run: go get -v -t -d ./...

fetch.go

@@ -3,16 +3,18 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"log"
-	"strings"
+	"os"
 	"sync"
 
 	"github.com/google/go-github/github"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
+	gitlab "github.com/xanzy/go-gitlab"
 	"go.uber.org/zap"
 	"golang.org/x/oauth2"
 )
@@ -56,45 +58,95 @@ func newFetchCommand() *cobra.Command {
 
 func fetch(opts *fetchOptions) error {
 	logger().Debug("fetch", zap.Stringer("opts", *opts))
-	ctx := context.Background()
-	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: opts.GithubToken})
-	tc := oauth2.NewClient(ctx, ts)
-	client := github.NewClient(tc)
 
 	var (
 		wg        sync.WaitGroup
-		allIssues []*github.Issue
-		out       = make(chan []*github.Issue, 100)
+		allIssues []*Issue
+		out       = make(chan []*Issue, 100)
 	)
 	wg.Add(len(opts.Repos))
-	for _, repo := range opts.Repos {
-		parts := strings.Split(repo, "/")
-		organization := parts[0]
-		repo := parts[1]
+	for _, repoURL := range opts.Repos {
+		repo := NewRepo(repoURL)
+		switch repo.Provider() {
+		case GitHubProvider:
+			ctx := context.Background()
+			ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: opts.GithubToken})
+			tc := oauth2.NewClient(ctx, ts)
+			client := github.NewClient(tc)
 
-		go func(repo string) {
-			total := 0
-			defer wg.Done()
-			opts := &github.IssueListByRepoOptions{State: "all"}
-			for {
-				issues, resp, err := client.Issues.ListByRepo(ctx, organization, repo, opts)
-				if err != nil {
-					log.Fatal(err)
-					return
+			go func(repo Repo) {
+				total := 0
+				defer wg.Done()
+				opts := &github.IssueListByRepoOptions{State: "all"}
+				for {
+					issues, resp, err := client.Issues.ListByRepo(ctx, repo.Namespace(), repo.Project(), opts)
+					if err != nil {
+						log.Fatal(err)
+						return
+					}
+					total += len(issues)
+					logger().Debug("paginate",
+						zap.String("provider", "github"),
+						zap.String("repo", repo.Canonical()),
+						zap.Int("new-issues", len(issues)),
+						zap.Int("total-issues", total),
+					)
+					normalizedIssues := []*Issue{}
+					for _, issue := range issues {
+						normalizedIssues = append(normalizedIssues, FromGitHubIssue(issue))
+					}
+					out <- normalizedIssues
+					if resp.NextPage == 0 {
+						break
+					}
+					opts.Page = resp.NextPage
 				}
-				total += len(issues)
-				logger().Debug("paginate",
-					zap.String("repo", repo),
-					zap.Int("new-issues", len(issues)),
-					zap.Int("total-issues", total),
-				)
-				out <- issues
-				if resp.NextPage == 0 {
-					break
+				if rateLimits, _, err := client.RateLimits(ctx); err == nil {
+					logger().Debug("github API rate limiting", zap.Stringer("limit", rateLimits.GetCore()))
 				}
-				opts.Page = resp.NextPage
-			}
-		}(repo)
+			}(repo)
+		case GitLabProvider:
+			go func(repo Repo) {
+				client := gitlab.NewClient(nil, os.Getenv("GITLAB_TOKEN"))
+				client.SetBaseURL(fmt.Sprintf("%s/api/v4", repo.SiteURL()))
+
+				//projectID := url.QueryEscape(repo.RepoPath())
+				projectID := repo.RepoPath()
+				total := 0
+				defer wg.Done()
+				opts := &gitlab.ListProjectIssuesOptions{
+					ListOptions: gitlab.ListOptions{
+						PerPage: 30,
+						Page:    1,
+					},
+				}
+				for {
+					issues, resp, err := client.Issues.ListProjectIssues(projectID, opts)
+					if err != nil {
+						logger().Error("failed to fetch issues", zap.Error(err))
+						return
+					}
+					total += len(issues)
+					logger().Debug("paginate",
+						zap.String("provider", "gitlab"),
+						zap.String("repo", repo.Canonical()),
+						zap.Int("new-issues", len(issues)),
+						zap.Int("total-issues", total),
+					)
+					normalizedIssues := []*Issue{}
+					for _, issue := range issues {
+						normalizedIssues = append(normalizedIssues, FromGitLabIssue(issue))
+					}
+					out <- normalizedIssues
+					if resp.NextPage == 0 {
+						break
+					}
+					opts.ListOptions.Page = resp.NextPage
+				}
+			}(repo)
+		default:
+			panic("should not happen")
+		}
 	}
 	wg.Wait()
 	close(out)
@@ -103,10 +155,5 @@ func fetch(opts *fetchOptions) error {
 	}
 
 	issuesJson, _ := json.MarshalIndent(allIssues, "", "  ")
-	rateLimits, _, err := client.RateLimits(ctx)
-	if err != nil {
-		return err
-	}
-	logger().Debug("github API rate limiting", zap.Stringer("limit", rateLimits.GetCore()))
-	return errors.Wrap(ioutil.WriteFile(opts.DBOpts.Path, issuesJson, 0644), "failed to write db file")
+	return errors.Wrap(ioutil.WriteFile(opts.DBOpts.Path, issuesJson, 0644), "failed to write db")
 }

go.mod

@@ -3,6 +3,7 @@ module moul.io/depviz
 require (
 	github.com/awalterschulze/gographviz v0.0.0-20180813113015-16ed621cdb51
 	github.com/fsnotify/fsnotify v1.4.7 // indirect
+	github.com/gilliek/go-opml v1.0.0
 	github.com/golang/protobuf v1.2.0 // indirect
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect
@@ -18,6 +19,7 @@ require (
 	github.com/spf13/jwalterweatherman v0.0.0-20180814060501-14d3d4c51834 // indirect
 	github.com/spf13/pflag v1.0.2
 	github.com/spf13/viper v1.1.0
+	github.com/xanzy/go-gitlab v0.11.0
 	go.uber.org/atomic v1.3.2 // indirect
 	go.uber.org/multierr v1.1.0 // indirect
 	go.uber.org/zap v1.9.1

go.sum

@@ -3,6 +3,8 @@ github.com/awalterschulze/gographviz v0.0.0-20180813113015-16ed621cdb51/go.mod h
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/gilliek/go-opml v1.0.0 h1:X8xVjtySRXU/x6KvaiXkn7OV3a4DHqxY8Rpv6U/JvCY=
+github.com/gilliek/go-opml v1.0.0/go.mod h1:fOxmtlzyBvUjU6bjpdjyxCGlWz+pgtAHrHf/xRZl3lk=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
@@ -33,6 +35,8 @@ github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc=
 github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/viper v1.1.0 h1:V7OZpY8i3C1x/pDmU0zNNlfVoDz112fSYvtWMjjS3f4=
 github.com/spf13/viper v1.1.0/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
+github.com/xanzy/go-gitlab v0.11.0 h1:H9exAT9A+VQSkDnoJPxMhC4uJip8kQ8edlVC28gFuOI=
+github.com/xanzy/go-gitlab v0.11.0/go.mod h1:CRKHkvFWNU6C3AEfqLWjnCNnAs4nj8Zk95rX2S3X6Mw=
 go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=

graphviz.go

@@ -106,7 +106,7 @@ func roadmapGraph(issues Issues, opts *renderOptions) (string, error) {
 	// issue nodes
 	issueNumbers := []string{}
 	for _, issue := range issues {
-		issueNumbers = append(issueNumbers, issue.NodeName())
+		issueNumbers = append(issueNumbers, issue.URL)
 	}
 	sort.Strings(issueNumbers)
 	for _, id := range issueNumbers {