Skip to content

Commit

Permalink
Merge pull request #147 from ahhx/refactor
Browse files Browse the repository at this point in the history
refactor: refactor depviz
  • Loading branch information
moul authored Dec 27, 2018
2 parents bde36ad + 70e2bf1 commit 88a632c
Show file tree
Hide file tree
Showing 31 changed files with 1,675 additions and 1,976 deletions.
5 changes: 3 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
FROM golang:1.11-alpine as build
RUN apk add --update --no-cache git gcc musl-dev
RUN apk add --update --no-cache git gcc musl-dev make
ADD . /go/src/moul.io/depviz
WORKDIR /go/src/moul.io/depviz
RUN GO111MODULE=on go get -v .
RUN make install

FROM alpine
RUN apk add --update --no-cache ca-certificates
COPY --from=build /go/bin/depviz /bin/
ENTRYPOINT ["depviz"]
ENTRYPOINT ["depviz"]
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,9 @@ _inspired by this discussion: [jbenet/random-ideas#37](https://github.com/jbenet

## Install (with Golang)

`go get moul.io/depviz`
```
go get moul.io/depviz
```

## Usage

Expand Down
2 changes: 1 addition & 1 deletion chi_util.go → cli/chi_util.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package main
package cli

import (
"net/http"
Expand Down
249 changes: 249 additions & 0 deletions cli/cmd_airtable.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,249 @@
package cli

import (
"encoding/json"
"fmt"

"github.com/brianloveswords/airtable"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/spf13/viper"
"go.uber.org/zap"
"moul.io/depviz/pkg/airtabledb"
"moul.io/depviz/pkg/issues"
)

type airtableOptions struct {
IssuesTableName string `mapstructure:"airtable-issues-table-name"`
RepositoriesTableName string `mapstructure:"airtable-repositories-table-name"`
LabelsTableName string `mapstructure:"airtable-labels-table-name"`
MilestonesTableName string `mapstructure:"airtable-milestones-table-name"`
ProvidersTableName string `mapstructure:"airtable-providers-table-name"`
AccountsTableName string `mapstructure:"airtable-accounts-table-name"`
BaseID string `mapstructure:"airtable-base-id"`
Token string `mapstructure:"airtable-token"`
DestroyInvalidRecords bool `mapstructure:"airtable-destroy-invalid-records"`
TableNames []string

Targets []issues.Target `mapstructure:"targets"`
}

func (opts airtableOptions) String() string {
out, _ := json.Marshal(opts)
return string(out)
}

type airtableCommand struct {
opts airtableOptions
}

func (cmd *airtableCommand) LoadDefaultOptions() error {
if err := viper.Unmarshal(&cmd.opts); err != nil {
return err
}
return nil
}

func (cmd *airtableCommand) ParseFlags(flags *pflag.FlagSet) {
cmd.opts.TableNames = make([]string, airtabledb.NumTables)

flags.StringVarP(&cmd.opts.IssuesTableName, "airtable-issues-table-name", "", "Issues and PRs", "Airtable issues table name")
cmd.opts.TableNames[airtabledb.IssueIndex] = cmd.opts.IssuesTableName

flags.StringVarP(&cmd.opts.RepositoriesTableName, "airtable-repositories-table-name", "", "Repositories", "Airtable repositories table name")
cmd.opts.TableNames[airtabledb.RepositoryIndex] = cmd.opts.RepositoriesTableName

flags.StringVarP(&cmd.opts.AccountsTableName, "airtable-accounts-table-name", "", "Accounts", "Airtable accounts table name")
cmd.opts.TableNames[airtabledb.AccountIndex] = cmd.opts.AccountsTableName

flags.StringVarP(&cmd.opts.LabelsTableName, "airtable-labels-table-name", "", "Labels", "Airtable labels table name")
cmd.opts.TableNames[airtabledb.LabelIndex] = cmd.opts.LabelsTableName

flags.StringVarP(&cmd.opts.MilestonesTableName, "airtable-milestones-table-name", "", "Milestones", "Airtable milestones table nfame")
cmd.opts.TableNames[airtabledb.MilestoneIndex] = cmd.opts.MilestonesTableName

flags.StringVarP(&cmd.opts.ProvidersTableName, "airtable-providers-table-name", "", "Providers", "Airtable providers table name")
cmd.opts.TableNames[airtabledb.ProviderIndex] = cmd.opts.ProvidersTableName

flags.StringVarP(&cmd.opts.BaseID, "airtable-base-id", "", "", "Airtable base ID")
flags.StringVarP(&cmd.opts.Token, "airtable-token", "", "", "Airtable token")
flags.BoolVarP(&cmd.opts.DestroyInvalidRecords, "airtable-destroy-invalid-records", "", false, "Destroy invalid records")

viper.BindPFlags(flags)
}

func (cmd *airtableCommand) NewCobraCommand(dc map[string]DepvizCommand) *cobra.Command {
cc := &cobra.Command{
Use: "airtable",
Short: "Upload issue info stored in database to airtable spreadsheets",
}
cc.AddCommand(cmd.airtableSyncCommand())
return cc
}

func (cmd *airtableCommand) airtableSyncCommand() *cobra.Command {
cc := &cobra.Command{
Use: "sync",
Short: "Upload issue info stored in database to airtable spreadsheets",
RunE: func(_ *cobra.Command, args []string) error {
opts := cmd.opts
var err error
if opts.Targets, err = issues.ParseTargets(args); err != nil {
return errors.Wrap(err, "invalid targets")
}
return airtableSync(&opts)
},
}
cmd.ParseFlags(cc.Flags())
return cc
}

// airtableSync pushes issue info to the airtable base specified in opts.
// Repository info is loaded from the targets specified in opts.
func airtableSync(opts *airtableOptions) error {
if opts.BaseID == "" || opts.Token == "" {
return fmt.Errorf("missing token or baseid, check '-h'")
}

//
// prepare
//

loadedIssues, err := issues.Load(db, nil)
if err != nil {
return errors.Wrap(err, "failed to load issues")
}
loadedIssues = loadedIssues.FilterByTargets(opts.Targets)
zap.L().Debug("fetch db entries", zap.Int("count", len(loadedIssues)))

issueFeatures := make([]map[string]issues.Feature, airtabledb.NumTables)
for i, _ := range issueFeatures {
issueFeatures[i] = make(map[string]issues.Feature)
}

// Parse the loaded issues into the issueFeature map.
for _, issue := range loadedIssues {
// providers
issueFeatures[airtabledb.ProviderIndex][issue.Repository.Provider.ID] = issue.Repository.Provider

// labels
for _, label := range issue.Labels {
issueFeatures[airtabledb.LabelIndex][label.ID] = label
}

// accounts
if issue.Repository.Owner != nil {
issueFeatures[airtabledb.AccountIndex][issue.Repository.Owner.ID] = issue.Repository.Owner
}

issueFeatures[airtabledb.AccountIndex][issue.Author.ID] = issue.Author
for _, assignee := range issue.Assignees {
issueFeatures[airtabledb.AccountIndex][assignee.ID] = assignee
}
if issue.Milestone != nil && issue.Milestone.Creator != nil {
issueFeatures[airtabledb.AccountIndex][issue.Milestone.Creator.ID] = issue.Milestone.Creator
}

// repositories
issueFeatures[airtabledb.RepositoryIndex][issue.Repository.ID] = issue.Repository
// FIXME: find external repositories based on depends-on links

// milestones
if issue.Milestone != nil {
issueFeatures[airtabledb.MilestoneIndex][issue.Milestone.ID] = issue.Milestone
}

// issue
issueFeatures[airtabledb.IssueIndex][issue.ID] = issue
// FIXME: find external issues based on depends-on links
}

client := airtable.Client{
APIKey: opts.Token,
BaseID: opts.BaseID,
Limiter: airtable.RateLimiter(5),
}

// cache stores issueFeatures inserted into the airtable base.
cache := airtabledb.NewDB()

// Store already existing issueFeatures into the cache.
for tableKind, tableName := range opts.TableNames {
table := client.Table(tableName)
if err := cache.Tables[tableKind].Fetch(table); err != nil {
return err
}
}

// unmatched stores new issueFeatures (exist in the loaded issues but not the airtable base).
unmatched := airtabledb.NewDB()

// Insert new issueFeatures into unmatched and mark altered cache issueFeatures with airtabledb.StateChanged.
for tableKind, featureMap := range issueFeatures {
for _, dbEntry := range featureMap {
matched := false
dbRecord := dbEntry.ToRecord(cache)
for idx := 0; idx < cache.Tables[tableKind].Len(); idx++ {
t := cache.Tables[tableKind]
if t.GetFieldID(idx) == dbEntry.GetID() {
if t.RecordsEqual(idx, dbRecord) {
t.SetState(idx, airtabledb.StateUnchanged)
} else {
t.CopyFields(idx, dbRecord)
t.SetState(idx, airtabledb.StateChanged)
}
matched = true
break
}
}
if !matched {
unmatched.Tables[tableKind].Append(dbRecord)
}
}
}

// Add new issueFeatures from unmatched to cache.
// Then, push new and altered issueFeatures from cache to airtable base.
for tableKind, tableName := range opts.TableNames {
table := client.Table(tableName)
ut := unmatched.Tables[tableKind]
ct := cache.Tables[tableKind]
for i := 0; i < ut.Len(); i++ {
zap.L().Debug("create airtable entry", zap.String("type", tableName), zap.String("entry", ut.StringAt(i)))
if err := table.Create(ut.GetPtr(i)); err != nil {
return err
}
ut.SetState(i, airtabledb.StateNew)
ct.Append(ut.Get(i))
}
for i := 0; i < ct.Len(); i++ {
var err error
switch ct.GetState(i) {
case airtabledb.StateUnknown:
err = table.Delete(ct.GetPtr(i))
zap.L().Debug("delete airtable entry", zap.String("type", tableName), zap.String("entry", ct.StringAt(i)), zap.Error(err))
case airtabledb.StateChanged:
err = table.Update(ct.GetPtr(i))
zap.L().Debug("update airtable entry", zap.String("type", tableName), zap.String("entry", ct.StringAt(i)), zap.Error(err))
case airtabledb.StateUnchanged:
zap.L().Debug("unchanged airtable entry", zap.String("type", tableName), zap.String("entry", ct.StringAt(i)), zap.Error(err))
// do nothing
case airtabledb.StateNew:
zap.L().Debug("new airtable entry", zap.String("type", tableName), zap.String("entry", ct.StringAt(i)), zap.Error(err))
// do nothing
}
}
}

for tableKind, tableName := range opts.TableNames {
fmt.Println("-------", tableName)
ct := cache.Tables[tableKind]
for i := 0; i < ct.Len(); i++ {
fmt.Println(ct.GetID(i), airtabledb.StateString[ct.GetState(i)], ct.GetFieldID(i))
}
}
fmt.Println("-------")

return nil
}
72 changes: 72 additions & 0 deletions cli/cmd_db.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package cli

import (
"encoding/json"
"fmt"

"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/spf13/viper"
"moul.io/depviz/pkg/issues"
)

type dbOptions struct{}

func (opts dbOptions) String() string {
out, _ := json.Marshal(opts)
return string(out)
}

type dbCommand struct {
opts dbOptions
}

func (cmd *dbCommand) LoadDefaultOptions() error {
if err := viper.Unmarshal(&cmd.opts); err != nil {
return err
}
return nil
}

func (cmd *dbCommand) NewCobraCommand(dc map[string]DepvizCommand) *cobra.Command {
cc := &cobra.Command{
Use: "db",
}
cc.AddCommand(cmd.dbDumpCommand())
return cc
}

func (cmd *dbCommand) ParseFlags(flags *pflag.FlagSet) {
viper.BindPFlags(flags)
}

func (cmd *dbCommand) dbDumpCommand() *cobra.Command {
cc := &cobra.Command{
Use: "dump",
Short: "Print all issues stored in the database, formatted as JSON",
RunE: func(_ *cobra.Command, args []string) error {
opts := cmd.opts
return dbDump(&opts)
},
}
cmd.ParseFlags(cc.Flags())
return cc
}

func dbDump(opts *dbOptions) error {
issues := []*issues.Issue{}
if err := db.Find(&issues).Error; err != nil {
return err
}

for _, issue := range issues {
issue.PostLoad()
}

out, err := json.MarshalIndent(issues, "", " ")
if err != nil {
return err
}
fmt.Println(string(out))
return nil
}
Loading

0 comments on commit 88a632c

Please sign in to comment.