diff --git a/Makefile b/Makefile index c5111d219..c7cbcb117 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,10 @@ .PHONY: install install: - GO111MODULE=auto go install -v + GO111MODULE=on go install -v + +.PHONY: test +test: + go test -v ./... .PHONY: update_examples update_examples: diff --git a/cmd_airtable.go b/cmd_airtable.go index bd0f07da3..28cc0da7b 100644 --- a/cmd_airtable.go +++ b/cmd_airtable.go @@ -2,9 +2,7 @@ package main import ( "encoding/json" - "reflect" - "sort" - "time" + "fmt" "github.com/brianloveswords/airtable" "github.com/pkg/errors" @@ -15,21 +13,36 @@ import ( ) type airtableOptions struct { - AirtableTableName string `mapstructure:"airtable-table-name"` - AirtableBaseID string `mapstructure:"airtable-base-id"` - AirtableToken string `mapstructure:"airtable-token"` - Targets []string + IssuesTableName string `mapstructure:"airtable-issues-table-name"` + RepositoriesTableName string `mapstructure:"airtable-repositories-table-name"` + LabelsTableName string `mapstructure:"airtable-labels-table-name"` + MilestonesTableName string `mapstructure:"airtable-milestones-table-name"` + ProvidersTableName string `mapstructure:"airtable-providers-table-name"` + AccountsTableName string `mapstructure:"airtable-accounts-table-name"` + BaseID string `mapstructure:"airtable-base-id"` + Token string `mapstructure:"airtable-token"` + DestroyInvalidRecords bool `mapstructure:"airtable-destroy-invalid-records"` + + Targets []Target `mapstructure:"targets"` } +var globalAirtableOptions airtableOptions + func (opts airtableOptions) String() string { out, _ := json.Marshal(opts) return string(out) } func airtableSetupFlags(flags *pflag.FlagSet, opts *airtableOptions) { - flags.StringVarP(&opts.AirtableTableName, "airtable-table-name", "", "Issues and PRs", "Airtable table name") - flags.StringVarP(&opts.AirtableBaseID, "airtable-base-id", "", "", "Airtable base ID") - flags.StringVarP(&opts.AirtableToken, "airtable-token", "", "", "Airtable token") + flags.StringVarP(&opts.IssuesTableName, "airtable-issues-table-name", "", "Issues and PRs", "Airtable issues table name") + flags.StringVarP(&opts.RepositoriesTableName, "airtable-repositories-table-name", "", "Repositories", "Airtable repositories table name") + flags.StringVarP(&opts.AccountsTableName, "airtable-accounts-table-name", "", "Accounts", "Airtable accounts table name") + flags.StringVarP(&opts.LabelsTableName, "airtable-labels-table-name", "", "Labels", "Airtable labels table name") + flags.StringVarP(&opts.MilestonesTableName, "airtable-milestones-table-name", "", "Milestones", "Airtable milestones table nfame") + flags.StringVarP(&opts.ProvidersTableName, "airtable-providers-table-name", "", "Providers", "Airtable providers table name") + flags.StringVarP(&opts.BaseID, "airtable-base-id", "", "", "Airtable base ID") + flags.StringVarP(&opts.Token, "airtable-token", "", "", "Airtable token") + flags.BoolVarP(&opts.DestroyInvalidRecords, "airtable-destroy-invalid-records", "", false, "Destroy invalid records") viper.BindPFlags(flags) } @@ -42,234 +55,464 @@ func newAirtableCommand() *cobra.Command { } func newAirtableSyncCommand() *cobra.Command { - opts := &airtableOptions{} cmd := &cobra.Command{ Use: "sync", RunE: func(cmd *cobra.Command, args []string) error { - if err := viper.Unmarshal(opts); err != nil { - return err + opts := globalAirtableOptions + var err error + if opts.Targets, err = ParseTargets(args); err != nil { + return errors.Wrap(err, "invalid targets") } - opts.Targets = args - return airtableSync(opts) + return airtableSync(&opts) }, } - airtableSetupFlags(cmd.Flags(), opts) + airtableSetupFlags(cmd.Flags(), &globalAirtableOptions) return cmd } +type AirtableDB struct { + Providers ProviderRecords + Labels LabelRecords + Accounts AccountRecords + Repositories RepositoryRecords + Milestones MilestoneRecords + Issues IssueRecords +} + func airtableSync(opts *airtableOptions) error { - issues, err := loadIssues(db, nil) + if opts.BaseID == "" || opts.Token == "" { + return fmt.Errorf("missing token or baseid, check '-h'") + } + + // + // prepare + // + + // load issues + issues, err := loadIssues(nil) if err != nil { return errors.Wrap(err, "failed to load issues") } - if err := issues.prepare(true); err != nil { - return errors.Wrap(err, "failed to prepare issues") + filtered := issues.FilterByTargets(opts.Targets) + logger().Debug("fetch db entries", zap.Int("count", len(filtered))) + + // unique entries + var ( + providerMap = make(map[string]*Provider) + labelMap = make(map[string]*Label) + accountMap = make(map[string]*Account) + repositoryMap = make(map[string]*Repository) + milestoneMap = make(map[string]*Milestone) + issueMap = make(map[string]*Issue) + ) + for _, issue := range filtered { + // providers + providerMap[issue.Repository.Provider.ID] = issue.Repository.Provider + + // labels + for _, label := range issue.Labels { + labelMap[label.ID] = label + } + + // accounts + if issue.Repository.Owner != nil { + accountMap[issue.Repository.Owner.ID] = issue.Repository.Owner + } + accountMap[issue.Author.ID] = issue.Author + for _, assignee := range issue.Assignees { + accountMap[assignee.ID] = assignee + } + if issue.Milestone != nil && issue.Milestone.Creator != nil { + accountMap[issue.Milestone.Creator.ID] = issue.Milestone.Creator + } + + // repositories + repositoryMap[issue.Repository.ID] = issue.Repository + // FIXME: find external repositories based on depends-on links + + // milestones + if issue.Milestone != nil { + milestoneMap[issue.Milestone.ID] = issue.Milestone + } + + // issue + issueMap[issue.ID] = issue + // FIXME: find external issues based on depends-on links } - issues.filterByTargets(opts.Targets) - logger().Debug("fetch db entries", zap.Int("count", len(issues))) + // init client at := airtable.Client{ - APIKey: opts.AirtableToken, - BaseID: opts.AirtableBaseID, + APIKey: opts.Token, + BaseID: opts.BaseID, Limiter: airtable.RateLimiter(5), } - table := at.Table(opts.AirtableTableName) - alreadyInAirtable := map[string]bool{} - - records := []airtableRecord{} - if err := table.List(&records, &airtable.Options{}); err != nil { + // fetch remote data + cache := AirtableDB{} + table := at.Table(opts.ProvidersTableName) + if err := table.List(&cache.Providers, &airtable.Options{}); err != nil { + return err + } + table = at.Table(opts.LabelsTableName) + if err := table.List(&cache.Labels, &airtable.Options{}); err != nil { + return err + } + table = at.Table(opts.AccountsTableName) + if err := table.List(&cache.Accounts, &airtable.Options{}); err != nil { + return err + } + table = at.Table(opts.RepositoriesTableName) + if err := table.List(&cache.Repositories, &airtable.Options{}); err != nil { + return err + } + table = at.Table(opts.MilestonesTableName) + if err := table.List(&cache.Milestones, &airtable.Options{}); err != nil { + return err + } + table = at.Table(opts.IssuesTableName) + if err := table.List(&cache.Issues, &airtable.Options{}); err != nil { return err } - logger().Debug("fetched airtable records", zap.Int("count", len(records))) - // create new records - for _, record := range records { - alreadyInAirtable[record.Fields.URL] = true + unmatched := AirtableDB{ + Providers: ProviderRecords{}, + Labels: LabelRecords{}, + Accounts: AccountRecords{}, + Repositories: RepositoryRecords{}, + Milestones: MilestoneRecords{}, + Issues: IssueRecords{}, } - for _, issue := range issues { - if issue.Hidden { - continue + + // + // compute fields + // + + // providers + for _, dbEntry := range providerMap { + matched := false + dbRecord := dbEntry.ToRecord(cache) + for idx, atEntry := range cache.Providers { + if atEntry.Fields.ID == dbEntry.ID { + if atEntry.Equals(dbRecord) { + cache.Providers[idx].State = airtableStateUnchanged + } else { + cache.Providers[idx].Fields = dbRecord.Fields + cache.Providers[idx].State = airtableStateChanged + } + matched = true + break + } } - if _, found := alreadyInAirtable[issue.URL]; found { - continue + if !matched { + unmatched.Providers = append(unmatched.Providers, *dbRecord) } - logger().Debug("creating airtable record without slices", zap.String("URL", issue.URL)) - r := minimalAirtableRecord{ - Fields: minimalAirtableIssue{ - URL: issue.URL, - Errors: "initialization", - }, + } + + // labels + for _, dbEntry := range labelMap { + matched := false + dbRecord := dbEntry.ToRecord(cache) + for idx, atEntry := range cache.Labels { + if atEntry.Fields.ID == dbEntry.ID { + if atEntry.Equals(dbRecord) { + cache.Labels[idx].State = airtableStateUnchanged + } else { + cache.Labels[idx].Fields = dbRecord.Fields + cache.Labels[idx].State = airtableStateChanged + } + matched = true + break + } } - if err := table.Create(&r); err != nil { - return err + if !matched { + unmatched.Labels = append(unmatched.Labels, *dbRecord) } - records = append(records, airtableRecord{ - ID: r.ID, - Fields: airtableIssue{ - URL: issue.URL, - }, - }) - } - - // update/destroy existing ones - for _, record := range records { - if issue, found := issues[record.Fields.URL]; !found { - logger().Debug("destroying airtable record", zap.String("URL", record.Fields.URL)) - if err := table.Delete(&record); err != nil { - return errors.Wrap(err, "failed to destroy record") - } - } else { - if issue.Hidden { - continue - } + } - if issue.ToAirtableRecord().Fields.Equals(record.Fields) { - continue + // accounts + for _, dbEntry := range accountMap { + matched := false + dbRecord := dbEntry.ToRecord(cache) + for idx, atEntry := range cache.Accounts { + if atEntry.Fields.ID == dbEntry.ID { + if atEntry.Equals(dbRecord) { + cache.Accounts[idx].State = airtableStateUnchanged + } else { + cache.Accounts[idx].Fields = dbRecord.Fields + cache.Accounts[idx].State = airtableStateChanged + } + matched = true + break } + } + if !matched { + unmatched.Accounts = append(unmatched.Accounts, *dbRecord) + } + } - logger().Debug("updating airtable record", zap.String("URL", issue.URL)) - record.Fields = issue.ToAirtableRecord().Fields - if err := table.Update(&record); err != nil { - logger().Warn("failed to update record, retrying without slices", zap.String("URL", issue.URL), zap.Error(err)) - record := minimalAirtableRecord{ - ID: record.ID, - Fields: minimalAirtableIssue{ - URL: issue.URL, - }, - } - if typedErr, ok := err.(airtable.ErrClientRequest); ok { - record.Fields.Errors = typedErr.Err.Error() + // repositories + for _, dbEntry := range repositoryMap { + matched := false + dbRecord := dbEntry.ToRecord(cache) + for idx, atEntry := range cache.Repositories { + if atEntry.Fields.ID == dbEntry.ID { + if atEntry.Equals(dbRecord) { + cache.Repositories[idx].State = airtableStateUnchanged } else { - record.Fields.Errors = err.Error() + cache.Repositories[idx].Fields = dbRecord.Fields + cache.Repositories[idx].State = airtableStateChanged } - if err := table.Update(&record); err != nil { - logger().Error("failed to update record without slices", zap.String("URL", issue.URL), zap.Error(err)) + matched = true + break + } + } + if !matched { + unmatched.Repositories = append(unmatched.Repositories, *dbRecord) + } + } + + // milestones + for _, dbEntry := range milestoneMap { + matched := false + dbRecord := dbEntry.ToRecord(cache) + for idx, atEntry := range cache.Milestones { + if atEntry.Fields.ID == dbEntry.ID { + if atEntry.Equals(dbRecord) { + cache.Milestones[idx].State = airtableStateUnchanged + } else { + cache.Milestones[idx].Fields = dbRecord.Fields + cache.Milestones[idx].State = airtableStateChanged } + matched = true + break } } + if !matched { + unmatched.Milestones = append(unmatched.Milestones, *dbRecord) + } } - return nil -} + // issues + for _, dbEntry := range issueMap { + matched := false + dbRecord := dbEntry.ToRecord(cache) + for idx, atEntry := range cache.Issues { + if atEntry.Fields.ID == dbEntry.ID { + if atEntry.Equals(dbRecord) { + cache.Issues[idx].State = airtableStateUnchanged + } else { + cache.Issues[idx].Fields = dbRecord.Fields + cache.Issues[idx].State = airtableStateChanged + } + matched = true + break + } + } + if !matched { + unmatched.Issues = append(unmatched.Issues, *dbRecord) + } + } -type airtableRecord struct { - ID string `json:"id,omitempty"` - Fields airtableIssue `json:"fields,omitempty"` -} + // + // update airtable + // -type minimalAirtableRecord struct { - ID string `json:"id,omitempty"` - Fields minimalAirtableIssue `json:"fields,omitempty"` -} + // providers + table = at.Table(opts.ProvidersTableName) + for _, entry := range unmatched.Providers { + logger().Debug("create airtable entry", zap.String("type", "provider"), zap.Stringer("entry", entry)) + if err := table.Create(&entry); err != nil { + return err + } + entry.State = airtableStateNew + cache.Providers = append(cache.Providers, entry) + } + for _, entry := range cache.Providers { + var err error + switch entry.State { + case airtableStateUnknown: + err = table.Delete(&entry) + logger().Debug("delete airtable entry", zap.String("type", "provider"), zap.Stringer("entry", entry), zap.Error(err)) + case airtableStateChanged: + err = table.Update(&entry) + logger().Debug("update airtable entry", zap.String("type", "provider"), zap.Stringer("entry", entry), zap.Error(err)) + case airtableStateUnchanged: + logger().Debug("unchanged airtable entry", zap.String("type", "provider"), zap.Stringer("entry", entry), zap.Error(err)) + // do nothing + case airtableStateNew: + logger().Debug("new airtable entry", zap.String("type", "provider"), zap.Stringer("entry", entry), zap.Error(err)) + // do nothing + } + } -func (ai airtableIssue) String() string { - out, _ := json.Marshal(ai) - return string(out) -} + // labels + table = at.Table(opts.LabelsTableName) + for _, entry := range unmatched.Labels { + logger().Debug("create airtable entry", zap.String("type", "label"), zap.Stringer("entry", entry)) + if err := table.Create(&entry); err != nil { + return err + } + entry.State = airtableStateNew + cache.Labels = append(cache.Labels, entry) + } + for _, entry := range cache.Labels { + var err error + switch entry.State { + case airtableStateUnknown: + err = table.Delete(&entry) + logger().Debug("delete airtable entry", zap.String("type", "label"), zap.Stringer("entry", entry), zap.Error(err)) + case airtableStateChanged: + err = table.Update(&entry) + logger().Debug("update airtable entry", zap.String("type", "label"), zap.Stringer("entry", entry), zap.Error(err)) + case airtableStateUnchanged: + logger().Debug("unchanged airtable entry", zap.String("type", "label"), zap.Stringer("entry", entry), zap.Error(err)) + // do nothing + case airtableStateNew: + logger().Debug("new airtable entry", zap.String("type", "label"), zap.Stringer("entry", entry), zap.Error(err)) + // do nothing + } + } -func (i Issue) ToAirtableRecord() airtableRecord { - typ := "issue" - if i.IsPR { - typ = "pull-request" - } - labels := []string{} - for _, label := range i.Labels { - labels = append(labels, label.ID) - } - assignees := []string{} - for _, assignee := range i.Assignees { - assignees = append(assignees, assignee.ID) - } - - return airtableRecord{ - ID: "", - Fields: airtableIssue{ - URL: i.URL, - Created: i.CreatedAt, - Updated: i.UpdatedAt, - Completed: i.CompletedAt, - Title: i.Title, - Type: typ, - Labels: labels, - Assignees: assignees, - Provider: string(i.Provider), - RepoURL: i.RepoURL, - Body: i.Body, - State: i.State, - Locked: i.Locked, - IsOrphan: i.IsOrphan, - Author: i.AuthorID, - Comments: i.Comments, - Milestone: i.Milestone, - Upvotes: i.Upvotes, - Downvotes: i.Downvotes, - Weight: i.Weight(), - Errors: "", - }, + // accounts + table = at.Table(opts.AccountsTableName) + for _, entry := range unmatched.Accounts { + logger().Debug("create airtable entry", zap.String("type", "account"), zap.Stringer("entry", entry)) + if err := table.Create(&entry); err != nil { + return err + } + entry.State = airtableStateNew + cache.Accounts = append(cache.Accounts, entry) + } + for _, entry := range cache.Accounts { + var err error + switch entry.State { + case airtableStateUnknown: + err = table.Delete(&entry) + logger().Debug("delete airtable entry", zap.String("type", "account"), zap.Stringer("entry", entry), zap.Error(err)) + case airtableStateChanged: + err = table.Update(&entry) + logger().Debug("update airtable entry", zap.String("type", "account"), zap.Stringer("entry", entry), zap.Error(err)) + case airtableStateUnchanged: + logger().Debug("unchanged airtable entry", zap.String("type", "account"), zap.Stringer("entry", entry), zap.Error(err)) + // do nothing + case airtableStateNew: + logger().Debug("new airtable entry", zap.String("type", "account"), zap.Stringer("entry", entry), zap.Error(err)) + // do nothing + } } -} -type airtableIssue struct { - URL string - Created time.Time - Updated time.Time - Completed time.Time - Title string - Provider string - State string - Body string - RepoURL string - Type string - Locked bool - Author string - Comments int - Milestone string - Upvotes int - Downvotes int - IsOrphan bool - Labels []string - Assignees []string - Weight int - Errors string -} + // repositories + table = at.Table(opts.RepositoriesTableName) + for _, entry := range unmatched.Repositories { + logger().Debug("create airtable entry", zap.String("type", "repository"), zap.Stringer("entry", entry)) + if err := table.Create(&entry); err != nil { + return err + } + entry.State = airtableStateNew + cache.Repositories = append(cache.Repositories, entry) + } + for _, entry := range cache.Repositories { + var err error + switch entry.State { + case airtableStateUnknown: + err = table.Delete(&entry) + logger().Debug("delete airtable entry", zap.String("type", "repository"), zap.Stringer("entry", entry), zap.Error(err)) + case airtableStateChanged: + err = table.Update(&entry) + logger().Debug("update airtable entry", zap.String("type", "repository"), zap.Stringer("entry", entry), zap.Error(err)) + case airtableStateUnchanged: + logger().Debug("unchanged airtable entry", zap.String("type", "repository"), zap.Stringer("entry", entry), zap.Error(err)) + // do nothing + case airtableStateNew: + logger().Debug("new airtable entry", zap.String("type", "repository"), zap.Stringer("entry", entry), zap.Error(err)) + // do nothing + } + } -type minimalAirtableIssue struct { - URL string - Errors string -} + // milestones + table = at.Table(opts.MilestonesTableName) + for _, entry := range unmatched.Milestones { + logger().Debug("create airtable entry", zap.String("type", "milestone"), zap.Stringer("entry", entry)) + if err := table.Create(&entry); err != nil { + return err + } + entry.State = airtableStateNew + cache.Milestones = append(cache.Milestones, entry) + } + for _, entry := range cache.Milestones { + var err error + switch entry.State { + case airtableStateUnknown: + err = table.Delete(&entry) + logger().Debug("delete airtable entry", zap.String("type", "milestone"), zap.Stringer("entry", entry), zap.Error(err)) + case airtableStateChanged: + err = table.Update(&entry) + logger().Debug("update airtable entry", zap.String("type", "milestone"), zap.Stringer("entry", entry), zap.Error(err)) + case airtableStateUnchanged: + logger().Debug("unchanged airtable entry", zap.String("type", "milestone"), zap.Stringer("entry", entry), zap.Error(err)) + // do nothing + case airtableStateNew: + logger().Debug("new airtable entry", zap.String("type", "milestone"), zap.Stringer("entry", entry), zap.Error(err)) + // do nothing + } + } -func (ai airtableIssue) Equals(other airtableIssue) bool { - sameSlice := func(a, b []string) bool { - if a == nil { - a = []string{} + // issues + table = at.Table(opts.IssuesTableName) + for _, entry := range unmatched.Issues { + logger().Debug("create airtable entry", zap.String("type", "issue"), zap.Stringer("entry", entry)) + if err := table.Create(&entry); err != nil { + return err } - if b == nil { - b = []string{} + entry.State = airtableStateNew + cache.Issues = append(cache.Issues, entry) + } + for _, entry := range cache.Issues { + var err error + switch entry.State { + case airtableStateUnknown: + err = table.Delete(&entry) + logger().Debug("delete airtable entry", zap.String("type", "issue"), zap.Stringer("entry", entry), zap.Error(err)) + case airtableStateChanged: + err = table.Update(&entry) + logger().Debug("update airtable entry", zap.String("type", "issue"), zap.Stringer("entry", entry), zap.Error(err)) + case airtableStateUnchanged: + logger().Debug("unchanged airtable entry", zap.String("type", "issue"), zap.Stringer("entry", entry), zap.Error(err)) + // do nothing + case airtableStateNew: + logger().Debug("new airtable entry", zap.String("type", "issue"), zap.Stringer("entry", entry), zap.Error(err)) + // do nothing } - sort.Strings(a) - sort.Strings(b) - return reflect.DeepEqual(a, b) - } - return ai.URL == other.URL && - ai.Created.Truncate(time.Millisecond).UTC() == other.Created.Truncate(time.Millisecond).UTC() && - ai.Updated.Truncate(time.Millisecond).UTC() == other.Updated.Truncate(time.Millisecond).UTC() && - ai.Completed.Truncate(time.Millisecond).UTC() == other.Completed.Truncate(time.Millisecond).UTC() && - ai.Title == other.Title && - ai.Provider == other.Provider && - ai.State == other.State && - ai.Body == other.Body && - ai.RepoURL == other.RepoURL && - ai.Type == other.Type && - ai.Locked == other.Locked && - ai.Author == other.Author && - ai.Comments == other.Comments && - ai.Milestone == other.Milestone && - ai.Weight == other.Weight && - ai.IsOrphan == other.IsOrphan && - ai.Upvotes == other.Upvotes && - ai.Downvotes == other.Downvotes && - sameSlice(ai.Labels, other.Labels) && - sameSlice(ai.Assignees, other.Assignees) && - ai.Errors == other.Errors + } + + // + // debug + // + fmt.Println("------- providers") + for _, entry := range cache.Providers { + fmt.Println(entry.ID, airtableStateString[entry.State], entry.Fields.ID) + } + fmt.Println("------- labels") + for _, entry := range cache.Labels { + fmt.Println(entry.ID, airtableStateString[entry.State], entry.Fields.ID) + } + fmt.Println("------- accounts") + for _, entry := range cache.Accounts { + fmt.Println(entry.ID, airtableStateString[entry.State], entry.Fields.ID) + } + fmt.Println("------- repositories") + for _, entry := range cache.Repositories { + fmt.Println(entry.ID, airtableStateString[entry.State], entry.Fields.ID) + } + fmt.Println("------- milestones") + for _, entry := range cache.Milestones { + fmt.Println(entry.ID, airtableStateString[entry.State], entry.Fields.ID) + } + fmt.Println("------- issues") + for _, entry := range cache.Issues { + fmt.Println(entry.ID, airtableStateString[entry.State], entry.Fields.ID) + } + fmt.Println("-------") + + return nil } diff --git a/cmd_db.go b/cmd_db.go index 0b72c76ff..43943281e 100644 --- a/cmd_db.go +++ b/cmd_db.go @@ -4,15 +4,14 @@ import ( "encoding/json" "fmt" - "github.com/jinzhu/gorm" - "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" ) -type dbOptions struct { -} +type dbOptions struct{} + +var globalDBOptions dbOptions func (opts dbOptions) String() string { out, _ := json.Marshal(opts) @@ -32,26 +31,28 @@ func newDBCommand() *cobra.Command { } func newDBDumpCommand() *cobra.Command { - opts := &dbOptions{} cmd := &cobra.Command{ Use: "dump", RunE: func(cmd *cobra.Command, args []string) error { - if err := viper.Unmarshal(opts); err != nil { - return err - } - return dbDump(opts) + opts := globalDBOptions + return dbDump(&opts) }, } - dbSetupFlags(cmd.Flags(), opts) + dbSetupFlags(cmd.Flags(), &globalDBOptions) return cmd } func dbDump(opts *dbOptions) error { - issues, err := loadIssues(db, nil) - if err != nil { - return errors.Wrap(err, "failed to load issues") + issues := []*Issue{} + if err := db.Find(&issues).Error; err != nil { + return err } - out, err := json.MarshalIndent(issues.ToSlice(), "", " ") + + for _, issue := range issues { + issue.PostLoad() + } + + out, err := json.MarshalIndent(issues, "", " ") if err != nil { return err } @@ -59,26 +60,15 @@ func dbDump(opts *dbOptions) error { return nil } -func canonicalTargets(input []string) []string { - output := []string{} - base := Issue{RepoURL: "https://github.com/moul/depviz", URL: "https://github.com/moul/depviz/issues/1"} - for _, target := range input { - output = append(output, base.GetRelativeIssueURL(target)) - } - return output -} - -func loadIssues(db *gorm.DB, targets []string) (Issues, error) { - query := db.Model(Issue{}) +func loadIssues(targets []string) (Issues, error) { + query := db.Model(Issue{}).Order("created_at") if len(targets) > 0 { - query = query.Where("repo_url IN (?)", canonicalTargets(targets)) + return nil, fmt.Errorf("not implemented") + // query = query.Where("repo_url IN (?)", canonicalTargets(targets)) + // OR WHERE parents IN .... + // etc } - /*var count int - if err := query.Count(&count).Error; err != nil { - return nil, err - }*/ - perPage := 100 var issues []*Issue for page := 0; ; page++ { @@ -91,6 +81,12 @@ func loadIssues(db *gorm.DB, targets []string) (Issues, error) { break } } - slice := IssueSlice(issues) - return slice.ToMap(), nil + + for _, issue := range issues { + issue.PostLoad() + } + + return Issues(issues), nil } + +// FIXME: try to use gorm hooks to auto preload/postload items diff --git a/cmd_graph.go b/cmd_graph.go new file mode 100644 index 000000000..0165cc6b2 --- /dev/null +++ b/cmd_graph.go @@ -0,0 +1,461 @@ +package main + +import ( + "encoding/json" + "fmt" + "html" + "io" + "math" + "net/url" + "os" + "sort" + "strings" + + "github.com/awalterschulze/gographviz" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" + "go.uber.org/zap" +) + +type graphOptions struct { + Output string `mapstructure:"output"` + DebugGraph bool `mapstructure:"debug-graph"` + NoCompress bool `mapstructure:"no-compress"` + DarkTheme bool `mapstructure:"dark-theme"` + ShowClosed bool `mapstructure:"show-closed"` + ShowOrphans bool `mapstructure:"show-orphans"` + ShowPRs bool `mapstructure:"show-prs"` + Preview bool `mapstructure:"preview"` + Format string `mapstructure:"format"` + Targets Targets `mapstructure:"targets"` + // FocusMode + // NoExternal +} + +var globalGraphOptions graphOptions + +func (opts graphOptions) String() string { + out, _ := json.Marshal(opts) + return string(out) +} + +func graphSetupFlags(flags *pflag.FlagSet, opts *graphOptions) { + flags.BoolVarP(&opts.ShowClosed, "show-closed", "", false, "show closed issues") + flags.BoolVarP(&opts.DebugGraph, "debug-graph", "", false, "debug graph") + flags.BoolVarP(&opts.ShowOrphans, "show-orphans", "", false, "show issues not linked to an epic") + flags.BoolVarP(&opts.NoCompress, "no-compress", "", false, "do not compress graph (no overlap)") + flags.BoolVarP(&opts.DarkTheme, "dark-theme", "", false, "dark theme") + flags.BoolVarP(&opts.ShowPRs, "show-prs", "", false, "show PRs") + flags.StringVarP(&opts.Output, "output", "o", "-", "output file ('-' for stdout, dot)") + flags.StringVarP(&opts.Format, "format", "f", "", "output file format (if empty, will determine thanks to output extension)") + //flags.BoolVarP(&opts.Preview, "preview", "p", false, "preview result") + viper.BindPFlags(flags) +} + +func newGraphCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "graph", + RunE: func(cmd *cobra.Command, args []string) error { + opts := globalGraphOptions + var err error + if opts.Targets, err = ParseTargets(args); err != nil { + return errors.Wrap(err, "invalid targets") + } + return graph(&opts) + }, + } + graphSetupFlags(cmd.Flags(), &globalGraphOptions) + return cmd +} + +func graph(opts *graphOptions) error { + logger().Debug("graph", zap.Stringer("opts", *opts)) + issues, err := loadIssues(nil) + if err != nil { + return errors.Wrap(err, "failed to load issues") + } + filtered := issues.FilterByTargets(opts.Targets) + + out, err := graphviz(filtered, opts) + if err != nil { + return errors.Wrap(err, "failed to render graph") + } + + switch opts.Format { + case "png", "svg": + return fmt.Errorf("only supporting .dot format for now") + + //case "dot": + default: + } + + var dest io.WriteCloser + switch opts.Output { + case "-", "": + dest = os.Stdout + default: + var err error + dest, err = os.Create(opts.Output) + if err != nil { + return err + } + defer dest.Close() + } + fmt.Fprintln(dest, out) + return nil +} + +func isIssueHidden(issue *Issue, opts *graphOptions) bool { + if issue.IsHidden { + return true + } + if !opts.ShowClosed && issue.IsClosed() { + return true + } + if !opts.ShowOrphans && issue.IsOrphan { + return true + } + if !opts.ShowPRs && issue.IsPR { + return true + } + return false +} + +func graphviz(issues Issues, opts *graphOptions) (string, error) { + for _, issue := range issues { + if isIssueHidden(issue, opts) { + continue + } + } + + var ( + stats = map[string]int{ + "nodes": 0, + "edges": 0, + "hidden": 0, + "subgraphs": 0, + } + invisStyle = map[string]string{"style": "invis", "label": escape("")} + weightMap = map[int]bool{} + weights = []int{} + existingNodes = map[string]bool{} + ) + if opts.DebugGraph { + invisStyle = map[string]string{} + } + for _, issue := range issues { + if isIssueHidden(issue, opts) { + stats["hidden"]++ + continue + } + weightMap[issue.Weight] = true + } + for weight := range weightMap { + weights = append(weights, weight) + } + sort.Ints(weights) + + // initialize graph + g := gographviz.NewGraph() + panicIfErr(g.SetName("G")) + attrs := map[string]string{} + attrs["truecolor"] = "true" + attrs["rankdir"] = "RL" + attrs["constraint"] = "true" + attrs["compound"] = "true" + if !opts.NoCompress { + attrs["center"] = "true" + attrs["ranksep"] = "0.3" + attrs["nodesep"] = "0.1" + attrs["margin"] = "0.2" + attrs["sep"] = "-0.7" + attrs["constraint"] = "false" + attrs["splines"] = "true" + attrs["overlap"] = "compress" + } + if opts.DarkTheme { + attrs["bgcolor"] = "black" + } + + for k, v := range attrs { + panicIfErr(g.AddAttr("G", k, v)) + } + panicIfErr(g.SetDir(true)) + + // issue nodes + issueNumbers := []string{} + for _, issue := range issues { + issueNumbers = append(issueNumbers, issue.URL) + } + sort.Strings(issueNumbers) + + orphansWithoutLinks := 0 + for _, id := range issueNumbers { + issue := issues.Get(id) + if isIssueHidden(issue, opts) { + continue + } + if len(issue.Parents) == 0 && len(issue.Children) == 0 { + orphansWithoutLinks++ + } + } + orphansCols := int(math.Ceil(math.Sqrt(float64(orphansWithoutLinks)) / 2)) + colIndex := 0 + hasOrphansWithLinks := false + for _, id := range issueNumbers { + issue := issues.Get(id) + if isIssueHidden(issue, opts) { + continue + } + parent := fmt.Sprintf("cluster_weight_%d", issue.Weight) + if issue.IsOrphan || !issue.HasEpic { + if len(issue.Children) > 0 || len(issue.Parents) > 0 { + parent = "cluster_orphans_with_links" + hasOrphansWithLinks = true + } else { + parent = fmt.Sprintf("cluster_orphans_without_links_%d", colIndex%orphansCols) + colIndex++ + } + } + + existingNodes[issue.URL] = true + panicIfErr(issue.AddNodeToGraph(g, parent)) + stats["nodes"]++ + } + + // issue relationships + for _, issue := range issues { + panicIfErr(issue.AddEdgesToGraph(g, opts, existingNodes)) + stats["edges"]++ + } + + // orphans cluster and placeholder + if orphansWithoutLinks > 0 { + panicIfErr(g.AddSubGraph( + "G", + "cluster_orphans_without_links", + map[string]string{"label": escape("orphans without links"), "style": "dashed"}, + )) + stats["subgraphs"]++ + + panicIfErr(g.AddSubGraph( + "cluster_orphans_without_links", + "cluster_orphans_without_links_0", + invisStyle, + )) + stats["subgraphs"]++ + for i := 0; i < orphansCols; i++ { + panicIfErr(g.AddNode( + fmt.Sprintf("cluster_orphans_without_links_%d", i), + fmt.Sprintf("placeholder_orphans_without_links_%d", i), + invisStyle, + )) + stats["nodes"]++ + } + + panicIfErr(g.AddEdge( + fmt.Sprintf("placeholder_%d", weights[len(weights)-1]), + "placeholder_orphans_without_links_0", + true, + invisStyle, + )) + stats["edges"]++ + + for i := 1; i < orphansCols; i++ { + panicIfErr(g.AddSubGraph( + "cluster_orphans_without_links", + fmt.Sprintf("cluster_orphans_without_links_%d", i), + invisStyle, + )) + stats["subgraphs"]++ + panicIfErr(g.AddEdge( + fmt.Sprintf("placeholder_orphans_without_links_%d", i-1), + fmt.Sprintf("placeholder_orphans_without_links_%d", i), + true, + invisStyle, + )) + stats["edges"]++ + } + } + if hasOrphansWithLinks { + attrs := map[string]string{} + attrs["label"] = escape("orphans with links") + attrs["style"] = "dashed" + panicIfErr(g.AddSubGraph("G", "cluster_orphans_with_links", attrs)) + stats["subgraphs"]++ + + panicIfErr(g.AddNode("cluster_orphans_with_links", "placeholder_orphans_with_links", invisStyle)) + stats["nodes"]++ + + panicIfErr(g.AddEdge( + "placeholder_orphans_with_links", + fmt.Sprintf("placeholder_%d", weights[0]), + true, + invisStyle, + )) + stats["edges"]++ + } + + // set weights clusters and placeholders + for _, weight := range weights { + clusterName := fmt.Sprintf("cluster_weight_%d", weight) + attrs := invisStyle + attrs["rank"] = "same" + panicIfErr(g.AddSubGraph("G", clusterName, attrs)) + stats["subgraphs"]++ + + attrs = invisStyle + attrs["shape"] = "none" + attrs["label"] = fmt.Sprintf(`"weight=%d"`, weight) + panicIfErr(g.AddNode( + clusterName, + fmt.Sprintf("placeholder_%d", weight), + attrs, + )) + stats["nodes"]++ + } + for i := 0; i < len(weights)-1; i++ { + panicIfErr(g.AddEdge( + fmt.Sprintf("placeholder_%d", weights[i]), + fmt.Sprintf("placeholder_%d", weights[i+1]), + true, + invisStyle, + )) + stats["edges"]++ + } + + logger().Debug("graph stats", zap.Any("stats", stats)) + return g.String(), nil +} + +func (i Issue) AddNodeToGraph(g *gographviz.Graph, parent string) error { + attrs := map[string]string{} + attrs["label"] = i.GraphNodeTitle() + //attrs["xlabel"] = "" + attrs["shape"] = "record" + attrs["style"] = `"rounded,filled"` + attrs["color"] = "lightblue" + attrs["href"] = escape(i.URL) + + if i.IsEpic { + attrs["shape"] = "oval" + } + + switch { + + case i.IsClosed(): + attrs["color"] = `"#cccccc33"` + + case i.IsEpic: + attrs["color"] = "orange" + attrs["style"] = `"rounded,filled,bold"` + + case i.IsReady(): + attrs["color"] = "pink" + + case i.IsOrphan || !i.HasEpic: + attrs["color"] = "gray" + } + + //logger().Debug("add node to graph", zap.String("url", i.URL)) + return g.AddNode( + parent, + escape(i.URL), + attrs, + ) +} + +func (i Issue) AddEdgesToGraph(g *gographviz.Graph, opts *graphOptions, existingNodes map[string]bool) error { + if isIssueHidden(&i, opts) { + return nil + } + for _, dependency := range i.Parents { + if isIssueHidden(dependency, opts) { + continue + } + if _, found := existingNodes[dependency.URL]; !found { + continue + } + attrs := map[string]string{} + attrs["color"] = "lightblue" + //attrs["label"] = "depends on" + //attrs["style"] = "dotted" + attrs["dir"] = "none" + if i.IsClosed() || dependency.IsClosed() { + attrs["color"] = "grey" + attrs["style"] = "dashed" + } else if dependency.IsReady() { + attrs["color"] = "pink" + } + if i.IsEpic { + attrs["color"] = "orange" + attrs["style"] = "dashed" + } + //log.Print("edge", escape(i.URL), "->", escape(dependency.URL)) + //logger().Debug("add edge to graph", zap.String("url", i.URL), zap.String("dep", dependency.URL)) + if err := g.AddEdge( + escape(i.URL), + escape(dependency.URL), + true, + attrs, + ); err != nil { + return err + } + } + return nil +} + +func (i Issue) GraphNodeName() string { + return fmt.Sprintf(`%s#%s`, i.Path()[1:], i.Number()) +} + +func (i Issue) Number() string { + u, err := url.Parse(i.URL) + if err != nil { + return "" + } + parts := strings.Split(u.Path, "/") + return parts[len(parts)-1] +} + +func (i Issue) Path() string { + u, err := url.Parse(i.URL) + if err != nil { + return "" + } + parts := strings.Split(u.Path, "/") + return strings.Join(parts[:len(parts)-2], "/") +} + +func (i Issue) GraphNodeTitle() string { + title := fmt.Sprintf("%s: %s", i.GraphNodeName(), i.Title) + title = strings.Replace(title, "|", "-", -1) + title = strings.Replace(html.EscapeString(wrap(title, 20)), "\n", "
", -1) + labels := []string{} + for _, label := range i.Labels { + switch label.ID { + case "t/step", "t/epic", "epic": + continue + } + labels = append(labels, fmt.Sprintf(`%s`, label.Color, label.Name)) + } + labelsText := "" + if len(labels) > 0 { + labelsText = "" + strings.Join(labels, "") + "
" + } + assigneeText := "" + if len(i.Assignees) > 0 { + assignees := []string{} + for _, assignee := range i.Assignees { + assignees = append(assignees, assignee.ID) + } + assigneeText = fmt.Sprintf(`@%s`, strings.Join(assignees, ", @")) + } + errorsText := "" + if len(i.Errors) > 0 { + errorsText = fmt.Sprintf(`ERR: %s`, strings.Join(i.Errors, ";
ERR: ")) + } + return fmt.Sprintf(`<%s%s%s
%s
>`, title, labelsText, assigneeText, errorsText) +} diff --git a/cmd_pull.go b/cmd_pull.go index a523e24a7..f3c1ad1da 100644 --- a/cmd_pull.go +++ b/cmd_pull.go @@ -1,34 +1,28 @@ package main import ( - "context" "encoding/json" - "fmt" - "log" + "os" "sync" - "github.com/google/go-github/github" + "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" - gitlab "github.com/xanzy/go-gitlab" "go.uber.org/zap" - "golang.org/x/oauth2" ) type pullOptions struct { - // db - DBOpts dbOptions - // pull - Repos []string GithubToken string `mapstructure:"github-token"` GitlabToken string `mapstructure:"gitlab-token"` // includeExternalDeps bool - Targets []string + Targets Targets `mapstructure:"targets"` } +var globalPullOptions pullOptions + func (opts pullOptions) String() string { out, _ := json.Marshal(opts) return string(out) @@ -41,23 +35,35 @@ func pullSetupFlags(flags *pflag.FlagSet, opts *pullOptions) { } func newPullCommand() *cobra.Command { - opts := &pullOptions{} cmd := &cobra.Command{ Use: "pull", RunE: func(cmd *cobra.Command, args []string) error { - if err := viper.Unmarshal(opts); err != nil { - return err + opts := globalPullOptions + var err error + if opts.Targets, err = ParseTargets(args); err != nil { + return errors.Wrap(err, "invalid targets") } - opts.Targets = args - return pull(opts) + return pullAndCompute(&opts) }, } - pullSetupFlags(cmd.Flags(), opts) - dbSetupFlags(cmd.Flags(), &opts.DBOpts) + pullSetupFlags(cmd.Flags(), &globalPullOptions) return cmd } +func pullAndCompute(opts *pullOptions) error { + if os.Getenv("DEPVIZ_NOPULL") != "1" { + if err := pull(opts); err != nil { + return errors.Wrap(err, "failed to pull") + } + } + if err := compute(opts); err != nil { + return errors.Wrap(err, "failed to compute") + } + return nil +} + func pull(opts *pullOptions) error { + // FIXME: handle the special '@me' target logger().Debug("pull", zap.Stringer("opts", *opts)) var ( @@ -66,101 +72,16 @@ func pull(opts *pullOptions) error { out = make(chan []*Issue, 100) ) - repos := getReposFromTargets(opts.Targets) - - wg.Add(len(repos)) - for _, repoURL := range repos { - repo := NewRepo(repoURL) - switch repo.Provider() { - case GitHubProvider: - ctx := context.Background() - ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: opts.GithubToken}) - tc := oauth2.NewClient(ctx, ts) - client := github.NewClient(tc) - - go func(repo Repo) { - - total := 0 - defer wg.Done() - opts := &github.IssueListByRepoOptions{State: "all"} - - var lastEntry Issue - if err := db.Where("repo_url = ?", repo.Canonical()).Order("updated_at desc").First(&lastEntry).Error; err == nil { - opts.Since = lastEntry.UpdatedAt - } - - for { - issues, resp, err := client.Issues.ListByRepo(ctx, repo.Namespace(), repo.Project(), opts) - if err != nil { - log.Fatal(err) - return - } - total += len(issues) - logger().Debug("paginate", - zap.String("provider", "github"), - zap.String("repo", repo.Canonical()), - zap.Int("new-issues", len(issues)), - zap.Int("total-issues", total), - ) - normalizedIssues := []*Issue{} - for _, issue := range issues { - normalizedIssues = append(normalizedIssues, FromGitHubIssue(issue)) - } - out <- normalizedIssues - if resp.NextPage == 0 { - break - } - opts.Page = resp.NextPage - } - if rateLimits, _, err := client.RateLimits(ctx); err == nil { - logger().Debug("github API rate limiting", zap.Stringer("limit", rateLimits.GetCore())) - } - }(repo) - case GitLabProvider: - go func(repo Repo) { - client := gitlab.NewClient(nil, opts.GitlabToken) - client.SetBaseURL(fmt.Sprintf("%s/api/v4", repo.SiteURL())) - - //projectID := url.QueryEscape(repo.RepoPath()) - projectID := repo.RepoPath() - total := 0 - defer wg.Done() - opts := &gitlab.ListProjectIssuesOptions{ - ListOptions: gitlab.ListOptions{ - PerPage: 30, - Page: 1, - }, - } - - var lastEntry Issue - if err := db.Where("repo_url = ?", repo.Canonical()).Order("updated_at desc").First(&lastEntry).Error; err == nil { - opts.UpdatedAfter = &lastEntry.UpdatedAt - } + targets := opts.Targets.UniqueProjects() - for { - issues, resp, err := client.Issues.ListProjectIssues(projectID, opts) - if err != nil { - logger().Error("failed to pull issues", zap.Error(err)) - return - } - total += len(issues) - logger().Debug("paginate", - zap.String("provider", "gitlab"), - zap.String("repo", repo.Canonical()), - zap.Int("new-issues", len(issues)), - zap.Int("total-issues", total), - ) - normalizedIssues := []*Issue{} - for _, issue := range issues { - normalizedIssues = append(normalizedIssues, FromGitLabIssue(issue)) - } - out <- normalizedIssues - if resp.NextPage == 0 { - break - } - opts.ListOptions.Page = resp.NextPage - } - }(repo) + // parallel fetches + wg.Add(len(targets)) + for _, target := range targets { + switch target.Driver() { + case GithubDriver: + go githubPull(target, &wg, opts, out) + case GitlabDriver: + go gitlabPull(target, &wg, opts, out) default: panic("should not happen") } @@ -171,6 +92,7 @@ func pull(opts *pullOptions) error { allIssues = append(allIssues, issues...) } + // save for _, issue := range allIssues { if err := db.Save(issue).Error; err != nil { return err diff --git a/cmd_run.go b/cmd_run.go index 5e74d6bd7..715bbf40d 100644 --- a/cmd_run.go +++ b/cmd_run.go @@ -2,127 +2,68 @@ package main import ( "encoding/json" - "fmt" - "io" - "os" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" - "go.uber.org/zap" ) type runOptions struct { - // pull - PullOpts pullOptions - NoPull bool - ReposToFetch []string - - // db - DBOpts dbOptions - GraphOpts graphOptions - - // run - AdditionalPulls []string - Destination string - - //Preview bool + GraphOptions graphOptions `mapstructure:"graph"` + PullOptions pullOptions `mapstructure:"pull"` + AdditionalPulls []string `mapstructure:"additional-pulls"` + NoPull bool `mapstructure:"no-pull"` } +var globalRunOptions runOptions + func (opts runOptions) String() string { out, _ := json.Marshal(opts) return string(out) } func runSetupFlags(flags *pflag.FlagSet, opts *runOptions) { - flags.BoolVarP(&opts.NoPull, "no-pull", "f", false, "do not pull new issues before runing") - flags.StringVarP(&opts.Destination, "destination", "", "-", "destination ('-' for stdout)") - flags.StringSliceVarP(&opts.AdditionalPulls, "additional-pull", "", []string{}, "additional pull that won't necessarily be displayed on the graph") - //flags.BoolVarP(&opts.Preview, "preview", "p", false, "preview result") + flags.BoolVarP(&opts.NoPull, "no-pull", "", false, "do not pull new issues before running") + flags.StringSliceVarP(&opts.AdditionalPulls, "additional-pulls", "", []string{}, "additional pull that won't necessarily be displayed on the graph") viper.BindPFlags(flags) } func newRunCommand() *cobra.Command { - opts := &runOptions{} cmd := &cobra.Command{ Use: "run", RunE: func(cmd *cobra.Command, args []string) error { - if err := viper.Unmarshal(opts); err != nil { - return err - } - if err := viper.Unmarshal(&opts.PullOpts); err != nil { - return err - } - if err := viper.Unmarshal(&opts.DBOpts); err != nil { - return err + opts := globalRunOptions + opts.GraphOptions = globalGraphOptions + opts.PullOptions = globalPullOptions + + targets, err := ParseTargets(args) + if err != nil { + return errors.Wrap(err, "invalid targets") } - if err := viper.Unmarshal(&opts.GraphOpts); err != nil { - return err + additionalPulls, err := ParseTargets(opts.AdditionalPulls) + if err != nil { + return errors.Wrap(err, "invalid targets") } - opts.PullOpts.DBOpts = opts.DBOpts - opts.PullOpts.Targets = append(args, opts.AdditionalPulls...) - opts.GraphOpts.Targets = args - return run(opts) + opts.PullOptions.Targets = append(targets, additionalPulls...) + opts.GraphOptions.Targets = targets + return run(&opts) }, } - runSetupFlags(cmd.Flags(), opts) - pullSetupFlags(cmd.Flags(), &opts.PullOpts) - dbSetupFlags(cmd.Flags(), &opts.DBOpts) - graphSetupFlags(cmd.Flags(), &opts.GraphOpts) + runSetupFlags(cmd.Flags(), &globalRunOptions) + graphSetupFlags(cmd.Flags(), &globalGraphOptions) + pullSetupFlags(cmd.Flags(), &globalPullOptions) return cmd } -func graphviz(opts *graphOptions) (string, error) { - if opts.Targets == nil || len(opts.Targets) < 1 || opts.Targets[0] == "" { - return "", fmt.Errorf("you need to specify at least one target") - } - issues, err := loadIssues(db, nil) - if err != nil { - return "", errors.Wrap(err, "failed to load issues") - } - - if err := issues.prepare(false); err != nil { - return "", errors.Wrap(err, "failed to prepare issues") - } - - if !opts.ShowClosed { - issues.HideClosed() - } - issues.filterByTargets(opts.Targets) - if opts.ShowOrphans { - logger().Warn("--show-orphans is deprecated and will be removed") - } - - return graphvizRender(issues, opts) -} - func run(opts *runOptions) error { - logger().Debug("run", zap.Stringer("opts", *opts)) if !opts.NoPull { - if err := pull(&opts.PullOpts); err != nil { - return err + if err := pullAndCompute(&opts.PullOptions); err != nil { + return errors.Wrap(err, "failed to pull") } } - - out, err := graphviz(&opts.GraphOpts) - if err != nil { - return err + if err := graph(&opts.GraphOptions); err != nil { + return errors.Wrap(err, "failed to graph") } - - var dest io.WriteCloser - switch opts.Destination { - case "-": - dest = os.Stdout - default: - var err error - dest, err = os.Create(opts.Destination) - if err != nil { - return err - } - defer dest.Close() - } - fmt.Fprintln(dest, out) - return nil } diff --git a/cmd_web.go b/cmd_web.go index cf01fedbb..9d92c30ba 100644 --- a/cmd_web.go +++ b/cmd_web.go @@ -25,11 +25,10 @@ type webOptions struct { // web specific Bind string ShowRoutes bool - - // db - DBOpts dbOptions } +var globalWebOptions webOptions + func (opts webOptions) String() string { out, _ := json.Marshal(opts) return string(out) @@ -42,21 +41,14 @@ func webSetupFlags(flags *pflag.FlagSet, opts *webOptions) { } func newWebCommand() *cobra.Command { - opts := &webOptions{} cmd := &cobra.Command{ Use: "web", RunE: func(cmd *cobra.Command, args []string) error { - if err := viper.Unmarshal(opts); err != nil { - return err - } - if err := viper.Unmarshal(&opts.DBOpts); err != nil { - return err - } - return web(opts) + opts := globalWebOptions + return web(&opts) }, } - webSetupFlags(cmd.Flags(), opts) - dbSetupFlags(cmd.Flags(), &opts.DBOpts) + webSetupFlags(cmd.Flags(), &globalWebOptions) return cmd } @@ -65,23 +57,18 @@ func (i *Issue) Render(w http.ResponseWriter, r *http.Request) error { } func webListIssues(w http.ResponseWriter, r *http.Request) { - issues, err := loadIssues(db, nil) + issues, err := loadIssues(nil) if err != nil { render.Render(w, r, ErrRender(err)) return } - issues.prepare(true) - - targets := strings.Split(r.URL.Query().Get("targets"), ",") - issues.filterByTargets(targets) - list := []render.Renderer{} for _, issue := range issues { - if issue.Hidden { + if issue.IsHidden { continue } - list = append(list, issue.WithJSONFields()) + list = append(list, issue) } if err := render.RenderList(w, r, list); err != nil { @@ -91,11 +78,20 @@ func webListIssues(w http.ResponseWriter, r *http.Request) { } func webGraphviz(r *http.Request) (string, error) { + targets, err := ParseTargets(strings.Split(r.URL.Query().Get("targets"), ",")) + if err != nil { + return "", err + } opts := &graphOptions{ - Targets: strings.Split(r.URL.Query().Get("targets"), ","), + Targets: targets, ShowClosed: r.URL.Query().Get("show-closed") == "1", } - return graphviz(opts) + issues, err := loadIssues(nil) + if err != nil { + return "", err + } + filtered := issues.FilterByTargets(targets) + return graphviz(filtered, opts) } func webDotIssues(w http.ResponseWriter, r *http.Request) { diff --git a/compute.go b/compute.go new file mode 100644 index 000000000..57f3b6a96 --- /dev/null +++ b/compute.go @@ -0,0 +1,160 @@ +package main + +import ( + "fmt" + "regexp" + "sort" + "strconv" + + "go.uber.org/zap" +) + +var ( + childrenRegex, _ = regexp.Compile(`(?i)(require|requires|blocked by|block by|depend on|depends on|parent of) ([a-z0-9:/_.-]+issues/[0-9]+|[a-z0-9:/_.-]+#[0-9]+|[a-z0-9/_-]*#[0-9]+)`) + parentsRegex, _ = regexp.Compile(`(?i)(blocks|block|address|addresses|part of|child of|fix|fixes) ([a-z0-9:/_.-]+issues/[0-9]+|[a-z0-9:/_.-]+#[0-9]+|[a-z0-9/_-]*#[0-9]+)`) + isDuplicateRegex, _ = regexp.Compile(`(?i)(duplicates|duplicate|dup of|dup|duplicate of) ([a-z0-9:/_.-]+issues/[0-9]+|[a-z0-9:/_.-]+#[0-9]+|[a-z0-9/_-]*#[0-9]+)`) + //weightMultiplierRegex, _ = regexp.Compile(`(?i)(depviz.weight_multiplier[:= ]+)([0-9]+)`) + weightRegex, _ = regexp.Compile(`(?i)(depviz.base_weight|depviz.weight)[:= ]+([0-9]+)`) + hideRegex, _ = regexp.Compile(`(?i)(depviz.hide)`) // FIXME: use label +) + +func compute(opts *pullOptions) error { + logger().Debug("compute", zap.Stringer("opts", *opts)) + issues, err := loadIssues(nil) + if err != nil { + return err + } + + for _, issue := range issues { + // reset default values + issue.Errors = []string{} + issue.Parents = []*Issue{} + issue.Children = []*Issue{} + issue.Duplicates = []*Issue{} + issue.Weight = 0 + issue.IsHidden = false + issue.IsEpic = false + issue.HasEpic = false + issue.IsOrphan = true + } + + for _, issue := range issues { + if issue.Body == "" { + continue + } + + // is epic + for _, label := range issue.Labels { + // FIXME: get epic labels dynamically based on a configuration filein the repo + if label.Name == "epic" || label.Name == "t/epic" { + issue.IsEpic = true + } + } + + // hidden + if match := hideRegex.FindStringSubmatch(issue.Body); match != nil { + issue.IsHidden = true + continue + } + + // duplicates + if match := isDuplicateRegex.FindStringSubmatch(issue.Body); match != nil { + canonical := issue.GetRelativeURL(match[len(match)-1]) + rel := issues.Get(canonical) + if rel == nil { + issue.Errors = append(issue.Errors, fmt.Errorf("duplicate %q not found", canonical).Error()) + continue + } + issue.Duplicates = append(issue.Duplicates, rel) + issue.IsHidden = true + continue + } + + // weight + if match := weightRegex.FindStringSubmatch(issue.Body); match != nil { + issue.Weight, _ = strconv.Atoi(match[len(match)-1]) + } + + // children + for _, match := range childrenRegex.FindAllStringSubmatch(issue.Body, -1) { + canonical := issue.GetRelativeURL(match[len(match)-1]) + child := issues.Get(canonical) + if child == nil { + issue.Errors = append(issue.Errors, fmt.Errorf("children %q not found", canonical).Error()) + continue + } + issue.Children = append(issue.Children, child) + issue.IsOrphan = false + child.Parents = append(child.Parents, issue) + child.IsOrphan = false + } + + // parents + for _, match := range parentsRegex.FindAllStringSubmatch(issue.Body, -1) { + canonical := issue.GetRelativeURL(match[len(match)-1]) + parent := issues.Get(canonical) + if parent == nil { + issue.Errors = append(issue.Errors, fmt.Errorf("parent %q not found", canonical).Error()) + continue + } + issue.Parents = append(issue.Parents, parent) + issue.IsOrphan = false + parent.Children = append(parent.Children, issue) + parent.IsOrphan = false + } + } + + for _, issue := range issues { + if issue.IsEpic { + issue.HasEpic = true + continue + } + // has epic + issue.HasEpic, err = issue.computeHasEpic(0) + if err != nil { + issue.Errors = append(issue.Errors, err.Error()) + } + } + + for _, issue := range issues { + issue.PostLoad() + + issue.ParentIDs = uniqueStrings(issue.ParentIDs) + sort.Strings(issue.ParentIDs) + issue.ChildIDs = uniqueStrings(issue.ChildIDs) + sort.Strings(issue.ChildIDs) + issue.DuplicateIDs = uniqueStrings(issue.DuplicateIDs) + sort.Strings(issue.DuplicateIDs) + } + + for _, issue := range issues { + // TODO: add a "if changed" to preserve some CPU and time + if err := db.Set("gorm:association_autoupdate", false).Save(issue).Error; err != nil { + return err + } + } + + return nil +} + +func (i Issue) computeHasEpic(depth int) (bool, error) { + if depth > 100 { + return false, fmt.Errorf("very high blocking depth (>100), do not continue. (issue=%s)", i.URL) + } + if i.IsHidden { + return false, nil + } + for _, parent := range i.Parents { + if parent.IsEpic { + return true, nil + } + parentHasEpic, err := parent.computeHasEpic(depth + 1) + if err != nil { + return false, nil + } + if parentHasEpic { + return true, nil + } + } + return false, nil +} diff --git a/github.go b/github.go new file mode 100644 index 000000000..5252b1d8c --- /dev/null +++ b/github.go @@ -0,0 +1,190 @@ +package main + +import ( + "context" + "log" + "strings" + "sync" + + "github.com/google/go-github/github" + "go.uber.org/zap" + "golang.org/x/oauth2" +) + +func githubPull(target Target, wg *sync.WaitGroup, opts *pullOptions, out chan []*Issue) { + defer wg.Done() + ctx := context.Background() + ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: opts.GithubToken}) + tc := oauth2.NewClient(ctx, ts) + client := github.NewClient(tc) + + total := 0 + callOpts := &github.IssueListByRepoOptions{State: "all"} + + var lastEntry Issue + if err := db.Where("repository_id = ?", target.ProjectURL()).Order("updated_at desc").First(&lastEntry).Error; err == nil { + callOpts.Since = lastEntry.UpdatedAt + } else { + logger().Warn("failed to get last entry", zap.Error(err)) + } + + for { + issues, resp, err := client.Issues.ListByRepo(ctx, target.Namespace(), target.Project(), callOpts) + if err != nil { + log.Fatal(err) + return + } + total += len(issues) + logger().Debug("paginate", + zap.String("provider", "github"), + zap.String("repo", target.ProjectURL()), + zap.Int("new-issues", len(issues)), + zap.Int("total-issues", total), + ) + normalizedIssues := []*Issue{} + for _, issue := range issues { + normalizedIssues = append(normalizedIssues, fromGithubIssue(issue)) + } + out <- normalizedIssues + if resp.NextPage == 0 { + break + } + callOpts.Page = resp.NextPage + } + if rateLimits, _, err := client.RateLimits(ctx); err == nil { + logger().Debug("github API rate limiting", zap.Stringer("limit", rateLimits.GetCore())) + } +} + +func fromGithubUser(input *github.User) *Account { + name := input.GetName() + if name == "" { + name = input.GetLogin() + } + return &Account{ + Base: Base{ + ID: input.GetLogin(), + CreatedAt: input.GetCreatedAt().Time, + UpdatedAt: input.GetUpdatedAt().Time, + }, + Provider: &Provider{ + Base: Base{ + ID: "github", // FIXME: support multiple github instances + }, + Driver: GithubDriver, + }, + URL: input.GetURL(), + Location: input.GetLocation(), + Company: input.GetCompany(), + Blog: input.GetBlog(), + Email: input.GetEmail(), + AvatarURL: input.GetAvatarURL(), + Login: input.GetLogin(), + FullName: name, + } +} + +func fromGithubRepository(input *github.Repository) *Repository { + panic("not implemented") +} + +func fromGithubRepositoryURL(input string) *Repository { + return &Repository{ + Base: Base{ + ID: input, + }, + URL: input, + Provider: &Provider{ + Base: Base{ + ID: "github", // FIXME: support multiple github instances + }, + Driver: GithubDriver, + }, + } +} + +func fromGithubMilestone(input *github.Milestone) *Milestone { + if input == nil { + return nil + } + parts := strings.Split(input.GetHTMLURL(), "/") + return &Milestone{ + Base: Base{ + ID: input.GetURL(), // FIXME: make it smaller + CreatedAt: input.GetCreatedAt(), + UpdatedAt: input.GetUpdatedAt(), + }, + URL: input.GetURL(), + Title: input.GetTitle(), + Description: input.GetDescription(), + ClosedAt: input.GetClosedAt(), + DueOn: input.GetDueOn(), + Creator: fromGithubUser(input.GetCreator()), + Repository: fromGithubRepositoryURL(strings.Join(parts[0:len(parts)-2], "/")), + } +} + +func fromGithubLabel(input *github.Label) *Label { + if input == nil { + return nil + } + return &Label{ + Base: Base{ + ID: input.GetURL(), // FIXME: make it smaller + }, + Name: input.GetName(), + Color: input.GetColor(), + Description: input.GetDescription(), + URL: input.GetURL(), + } +} + +func fromGithubIssue(input *github.Issue) *Issue { + parts := strings.Split(input.GetHTMLURL(), "/") + url := strings.Replace(input.GetHTMLURL(), "/pull/", "/issues/", -1) + + issue := &Issue{ + Base: Base{ + ID: url, + CreatedAt: input.GetCreatedAt(), + UpdatedAt: input.GetUpdatedAt(), + }, + CompletedAt: input.GetClosedAt(), + Repository: fromGithubRepositoryURL(strings.Join(parts[0:len(parts)-2], "/")), + Title: input.GetTitle(), + State: input.GetState(), + Body: input.GetBody(), + IsPR: input.PullRequestLinks != nil, + URL: url, + IsLocked: input.GetLocked(), + Comments: input.GetComments(), + Upvotes: *input.Reactions.PlusOne, + Downvotes: *input.Reactions.MinusOne, + Labels: make([]*Label, 0), + Assignees: make([]*Account, 0), + Author: fromGithubUser(input.User), + Milestone: fromGithubMilestone(input.Milestone), + + /* + IsOrphan bool `json:"is-orphan"` + IsHidden bool `json:"is-hidden"` + BaseWeight int `json:"base-weight"` + Weight int `json:"weight"` + IsEpic bool `json:"is-epic"` + HasEpic bool `json:"has-epic"` + + // internal + Parents []*Issue `json:"-" gorm:"-"` + Children []*Issue `json:"-" gorm:"-"` + Duplicates []*Issue `json:"-" gorm:"-"` + */ + + } + for _, label := range input.Labels { + issue.Labels = append(issue.Labels, fromGithubLabel(&label)) + } + for _, assignee := range input.Assignees { + issue.Assignees = append(issue.Assignees, fromGithubUser(assignee)) + } + return issue +} diff --git a/gitlab.go b/gitlab.go new file mode 100644 index 000000000..a148eee5c --- /dev/null +++ b/gitlab.go @@ -0,0 +1,215 @@ +package main + +import ( + "fmt" + "net/url" + "strings" + "sync" + "time" + + gitlab "github.com/xanzy/go-gitlab" + "go.uber.org/zap" +) + +func gitlabPull(target Target, wg *sync.WaitGroup, opts *pullOptions, out chan []*Issue) { + defer wg.Done() + client := gitlab.NewClient(nil, opts.GitlabToken) + client.SetBaseURL(fmt.Sprintf("%s/api/v4", target.ProviderURL())) + total := 0 + gitlabOpts := &gitlab.ListProjectIssuesOptions{ + ListOptions: gitlab.ListOptions{ + PerPage: 30, + Page: 1, + }, + } + + var lastEntry Issue + if err := db.Where("repository_id = ?", target.ProjectURL()).Order("updated_at desc").First(&lastEntry).Error; err == nil { + gitlabOpts.UpdatedAfter = &lastEntry.UpdatedAt + } + + // FIXME: fetch PRs + + for { + issues, resp, err := client.Issues.ListProjectIssues(target.Path(), gitlabOpts) + if err != nil { + logger().Error("failed to pull issues", zap.Error(err)) + return + } + total += len(issues) + logger().Debug("paginate", + zap.String("provider", "gitlab"), + zap.String("repo", target.ProjectURL()), + zap.Int("new-issues", len(issues)), + zap.Int("total-issues", total), + ) + normalizedIssues := []*Issue{} + for _, issue := range issues { + normalizedIssues = append(normalizedIssues, fromGitlabIssue(issue)) + } + out <- normalizedIssues + if resp.NextPage == 0 { + break + } + gitlabOpts.ListOptions.Page = resp.NextPage + } +} + +func fromGitlabIssue(input *gitlab.Issue) *Issue { + repoURL := input.Links.Project + if repoURL == "" { + repoURL = strings.Replace(input.WebURL, fmt.Sprintf("/issues/%d", input.IID), "", -1) + } + + //out, _ := json.MarshalIndent(input, "", " ") + //fmt.Println(string(out)) + + repo := fromGitlabRepositoryURL(repoURL) + issue := &Issue{ + Base: Base{ + ID: input.WebURL, + CreatedAt: *input.CreatedAt, + UpdatedAt: *input.UpdatedAt, + }, + Repository: repo, + Title: input.Title, + State: input.State, + Body: input.Description, + IsPR: false, + URL: input.WebURL, + IsLocked: false, // not supported on GitLab + Comments: 0, // not supported directly + Upvotes: input.Upvotes, + Downvotes: input.Downvotes, + Labels: make([]*Label, 0), + Assignees: make([]*Account, 0), + Author: fromGitlabFakeUser(repo.Provider, input.Author), + Milestone: fromGitlabMilestone(repo, input.Milestone), + /* + IsOrphan bool `json:"is-orphan"` + IsHidden bool `json:"is-hidden"` + BaseWeight int `json:"base-weight"` + Weight int `json:"weight"` + IsEpic bool `json:"is-epic"` + HasEpic bool `json:"has-epic"` + + // internal + Parents []*Issue `json:"-" gorm:"-"` + Children []*Issue `json:"-" gorm:"-"` + Duplicates []*Issue `json:"-" gorm:"-"` + */ + } + if input.ClosedAt != nil { + issue.CompletedAt = *input.ClosedAt + } + for _, label := range input.Labels { + issue.Labels = append(issue.Labels, fromGitlabLabelname(repo, label)) + } + //issue.Assignees = append(issue.Assignees, fromGitlabFakeUser(input.Assignee)) + for _, assignee := range input.Assignees { + issue.Assignees = append(issue.Assignees, fromGitlabFakeUser(repo.Provider, assignee)) + } + return issue +} + +func fromGitlabLabelname(repository *Repository, name string) *Label { + url := fmt.Sprintf("%s/labels/%s", repository.URL, name) + return &Label{ + Base: Base{ + ID: url, + }, + Name: name, + Color: "aaaacc", + URL: url, + //Description: input.GetDescription(), + } +} + +type gitlabFakeUser struct { + ID int `json:"id"` + Username string `json:"username"` + Email string `json:"email"` + Name string `json:"name"` + State string `json:"state"` + CreatedAt *time.Time `json:"created_at"` +} + +func fromGitlabFakeUser(provider *Provider, input gitlabFakeUser) *Account { + name := input.Name + if name == "" { + name = input.Username + } + url := fmt.Sprintf("%s/%s", provider.URL, input.Username) + account := Account{ + Base: Base{ + ID: url, + //UpdatedAt: input.UpdatedAt.Time, + }, + Provider: &Provider{ + Base: Base{ + ID: "gitlab", // FIXME: support multiple gitlab instances + }, + Driver: GitlabDriver, + }, + Email: input.Email, + FullName: name, + Login: input.Username, + URL: url, + // State: input.State, + + //Location: input.GetLocation(), + //Company: input.GetCompany(), + //Blog: input.GetBlog(), + //AvatarURL: input.GetAvatarURL(), + } + if input.CreatedAt != nil { + account.CreatedAt = *input.CreatedAt + } + + return &account +} + +func fromGitlabRepositoryURL(input string) *Repository { + u, err := url.Parse(input) + if err != nil { + logger().Warn("invalid repository URL", zap.String("URL", input)) + return nil + } + providerURL := fmt.Sprintf("%s://%s", u.Scheme, u.Host) + return &Repository{ + Base: Base{ + ID: input, + }, + URL: input, + Provider: &Provider{ + Base: Base{ + ID: "gitlab", // FIXME: support multiple gitlab instances + }, + URL: providerURL, + Driver: GitlabDriver, + }, + } +} + +func fromGitlabMilestone(repository *Repository, input *gitlab.Milestone) *Milestone { + if input == nil { + return nil + } + url := fmt.Sprintf("%s/milestones/%d", repository.URL, input.ID) + milestone := Milestone{ + Base: Base{ + ID: url, + CreatedAt: *input.CreatedAt, + UpdatedAt: *input.UpdatedAt, + }, + URL: url, + Title: input.Title, + Description: input.Description, + } + if input.DueDate != nil { + milestone.DueOn = time.Time(*input.DueDate) + } + // startdate // FIXME: todo + // state // FIXME: todo + return &milestone +} diff --git a/go.mod b/go.mod index 2c5b8396d..6c6cc8f85 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jinzhu/gorm v1.9.1 github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect github.com/jinzhu/now v0.0.0-20180511015916-ed742868f2ae // indirect - github.com/lib/pq v1.0.0 // indirect + github.com/lib/pq v1.0.0 github.com/mattn/go-sqlite3 v1.9.0 github.com/mitchellh/mapstructure v1.1.2 // indirect github.com/pkg/errors v0.8.0 diff --git a/go.sum b/go.sum index f445a2e77..1e34c04d2 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ cloud.google.com/go v0.29.0 h1:gv/9Wwq5WPVIGaROMQg8tw4jLFFiyacODxEIrlz0wTw= cloud.google.com/go v0.29.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.30.0 h1:xKvyLgk56d0nksWq49J0UyGEeUIicTl4+UBiX1NPX9g= cloud.google.com/go v0.30.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -13,6 +14,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.0.0-20180901172138-1eb28afdf9b6 h1:BZGp1dbKFjqlGmxEpwkDpCWNxVwEYnUPoncIzLiHlPo= github.com/denisenkom/go-mssqldb v0.0.0-20180901172138-1eb28afdf9b6/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= +github.com/denisenkom/go-mssqldb v0.0.0-20181014144952-4e0d7dc8888f h1:WH0w/R4Yoey+04HhFxqZ6VX6I0d7RMyw5aXQ9UTvQPs= github.com/denisenkom/go-mssqldb v0.0.0-20181014144952-4e0d7dc8888f/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= @@ -91,6 +93,7 @@ go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4 h1:Vk3wNqEZwyGyei9yq5ekj7frek2u7HUfffJ1/opblzc= golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e h1:IzypfodbhbnViNUO/MEh0FzCUooG97cIGfdggUrUSyU= golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181003013248-f5e5bdd77824 h1:MkjFNbaZJyH98M67Q3umtwZ+EdVdrNJLqSwZp5vcv60= diff --git a/graphviz.go b/graphviz.go deleted file mode 100644 index afbba1056..000000000 --- a/graphviz.go +++ /dev/null @@ -1,231 +0,0 @@ -package main - -import ( - "fmt" - "math" - "sort" - - "github.com/awalterschulze/gographviz" - "github.com/spf13/pflag" - "github.com/spf13/viper" - "go.uber.org/zap" -) - -type graphOptions struct { - DebugGraph bool `mapstructure:"debug-graph"` - NoCompress bool `mapstructure:"no-compress"` - DarkTheme bool `mapstructure:"dark-theme"` - ShowClosed bool `mapstructure:"show-closed"` - ShowOrphans bool `mapstructure:"show-orphans"` - EpicLabel string `mapstructure:"epic-label"` - Targets []string -} - -func graphSetupFlags(flags *pflag.FlagSet, opts *graphOptions) { - flags.BoolVarP(&opts.ShowClosed, "show-closed", "", false, "show closed issues") - flags.BoolVarP(&opts.DebugGraph, "debug-graph", "", false, "debug graph") - flags.BoolVarP(&opts.ShowOrphans, "show-orphans", "", false, "show issues not linked to an epic") - flags.BoolVarP(&opts.NoCompress, "no-compress", "", false, "do not compress graph (no overlap)") - flags.BoolVarP(&opts.DarkTheme, "dark-theme", "", false, "dark theme") - flags.StringVarP(&opts.EpicLabel, "epic-label", "", "epic", "label used for epics (empty means issues with dependencies but without dependants)") - viper.BindPFlags(flags) -} - -func graphvizRender(issues Issues, opts *graphOptions) (string, error) { - var ( - stats = map[string]int{ - "nodes": 0, - "edges": 0, - "hidden": 0, - "subgraphs": 0, - } - invisStyle = map[string]string{"style": "invis", "label": escape("")} - weightMap = map[int]bool{} - weights = []int{} - ) - if opts.DebugGraph { - invisStyle = map[string]string{} - } - for _, issue := range issues { - if issue.Hidden { - stats["hidden"]++ - continue - } - weightMap[issue.Weight()] = true - } - for weight := range weightMap { - weights = append(weights, weight) - } - sort.Ints(weights) - - // initialize graph - g := gographviz.NewGraph() - panicIfErr(g.SetName("G")) - attrs := map[string]string{} - attrs["truecolor"] = "true" - attrs["rankdir"] = "RL" - attrs["constraint"] = "true" - attrs["compound"] = "true" - if !opts.NoCompress { - attrs["center"] = "true" - attrs["ranksep"] = "0.3" - attrs["nodesep"] = "0.1" - attrs["margin"] = "0.2" - attrs["sep"] = "-0.7" - attrs["constraint"] = "false" - attrs["splines"] = "true" - attrs["overlap"] = "compress" - } - if opts.DarkTheme { - attrs["bgcolor"] = "black" - } - - for k, v := range attrs { - panicIfErr(g.AddAttr("G", k, v)) - } - panicIfErr(g.SetDir(true)) - - // issue nodes - issueNumbers := []string{} - for _, issue := range issues { - issueNumbers = append(issueNumbers, issue.URL) - } - sort.Strings(issueNumbers) - - orphansWithoutLinks := 0 - for _, id := range issueNumbers { - issue := issues[id] - if issue.Hidden { - continue - } - if len(issue.DependsOn) == 0 && len(issue.Blocks) == 0 { - orphansWithoutLinks++ - } - } - orphansCols := int(math.Ceil(math.Sqrt(float64(orphansWithoutLinks)) / 2)) - colIndex := 0 - hasOrphansWithLinks := false - for _, id := range issueNumbers { - issue := issues[id] - if issue.Hidden { - continue - } - parent := fmt.Sprintf("cluster_weight_%d", issue.Weight()) - if issue.IsOrphan || !issue.LinkedWithEpic { - if len(issue.DependsOn) > 0 || len(issue.Blocks) > 0 { - parent = "cluster_orphans_with_links" - hasOrphansWithLinks = true - } else { - parent = fmt.Sprintf("cluster_orphans_without_links_%d", colIndex%orphansCols) - colIndex++ - } - } - - panicIfErr(issue.AddNodeToGraph(g, parent)) - stats["nodes"]++ - } - - // issue relationships - for _, issue := range issues { - panicIfErr(issue.AddEdgesToGraph(g)) - stats["edges"]++ - } - - // orphans cluster and placeholder - if orphansWithoutLinks > 0 { - panicIfErr(g.AddSubGraph( - "G", - "cluster_orphans_without_links", - map[string]string{"label": escape("orphans without links"), "style": "dashed"}, - )) - stats["subgraphs"]++ - - panicIfErr(g.AddSubGraph( - "cluster_orphans_without_links", - "cluster_orphans_without_links_0", - invisStyle, - )) - stats["subgraphs"]++ - for i := 0; i < orphansCols; i++ { - panicIfErr(g.AddNode( - fmt.Sprintf("cluster_orphans_without_links_%d", i), - fmt.Sprintf("placeholder_orphans_without_links_%d", i), - invisStyle, - )) - stats["nodes"]++ - } - - panicIfErr(g.AddEdge( - fmt.Sprintf("placeholder_%d", weights[len(weights)-1]), - "placeholder_orphans_without_links_0", - true, - invisStyle, - )) - stats["edges"]++ - - for i := 1; i < orphansCols; i++ { - panicIfErr(g.AddSubGraph( - "cluster_orphans_without_links", - fmt.Sprintf("cluster_orphans_without_links_%d", i), - invisStyle, - )) - stats["subgraphs"]++ - panicIfErr(g.AddEdge( - fmt.Sprintf("placeholder_orphans_without_links_%d", i-1), - fmt.Sprintf("placeholder_orphans_without_links_%d", i), - true, - invisStyle, - )) - stats["edges"]++ - } - } - if hasOrphansWithLinks { - attrs := map[string]string{} - attrs["label"] = escape("orphans with links") - attrs["style"] = "dashed" - panicIfErr(g.AddSubGraph("G", "cluster_orphans_with_links", attrs)) - stats["subgraphs"]++ - - panicIfErr(g.AddNode("cluster_orphans_with_links", "placeholder_orphans_with_links", invisStyle)) - stats["nodes"]++ - - panicIfErr(g.AddEdge( - "placeholder_orphans_with_links", - fmt.Sprintf("placeholder_%d", weights[0]), - true, - invisStyle, - )) - stats["edges"]++ - } - - // set weights clusters and placeholders - for _, weight := range weights { - clusterName := fmt.Sprintf("cluster_weight_%d", weight) - attrs := invisStyle - attrs["rank"] = "same" - panicIfErr(g.AddSubGraph("G", clusterName, attrs)) - stats["subgraphs"]++ - - attrs = invisStyle - attrs["shape"] = "none" - attrs["label"] = fmt.Sprintf(`"weight=%d"`, weight) - panicIfErr(g.AddNode( - clusterName, - fmt.Sprintf("placeholder_%d", weight), - attrs, - )) - stats["nodes"]++ - } - for i := 0; i < len(weights)-1; i++ { - panicIfErr(g.AddEdge( - fmt.Sprintf("placeholder_%d", weights[i]), - fmt.Sprintf("placeholder_%d", weights[i+1]), - true, - invisStyle, - )) - stats["edges"]++ - } - - logger().Debug("graph stats", zap.Any("stats", stats)) - return g.String(), nil -} diff --git a/issue.go b/issue.go index e3a619c42..92e5b490e 100644 --- a/issue.go +++ b/issue.go @@ -1,326 +1,18 @@ package main import ( - "encoding/json" "fmt" - "html" "log" - "net/url" - "regexp" - "strconv" "strings" - "time" - - "github.com/awalterschulze/gographviz" - "github.com/google/go-github/github" - "github.com/spf13/viper" - gitlab "github.com/xanzy/go-gitlab" -) - -var debugFetch = false - -type Provider string - -const ( - UnknownProvider Provider = "unknown" - GitHubProvider = "github" - GitLabProvider = "gitlab" ) -type Issue struct { - // proxy - GitHub *github.Issue `json:"-" gorm:"-"` - GitLab *gitlab.Issue `json:"-" gorm:"-"` - - // internal - Provider Provider `json:"provider"` - DependsOn IssueSlice `json:"-" gorm:"-"` - Blocks IssueSlice `json:"-" gorm:"-"` - weightMultiplier int `gorm:"-" json:"-"` - BaseWeight int `json:"-" gorm:"-"` - IsOrphan bool `json:"is-orphan" gorm:"-"` - Hidden bool `json:"is-hidden" gorm:"-"` - Duplicates []string `json:"duplicates" gorm:"-"` - LinkedWithEpic bool `json:"is-linked-with-an-epic" gorm:"-"` - Errors []error `json:"errors" gorm:"-"` - - // mapping - CreatedAt time.Time `json:"created-at"` - UpdatedAt time.Time `json:"updated-at"` - CompletedAt time.Time `json:"completed-at"` - Number int `json:"number"` - Title string `json:"title"` - State string `json:"state"` - Body string `json:"body"` - RepoURL string `json:"repo-url"` - URL string `gorm:"primary_key" json:"url"` - Labels []*IssueLabel `gorm:"many2many:issue_labels;" json:"labels"` - Assignees []*Profile `gorm:"many2many:issue_assignees;" json:"assignees"` - IsPR bool `json:"is-pr"` - - // json export fields - JSONChildren []string `gorm:"-" json:"children"` - JSONParents []string `gorm:"-" json:"parents"` - JSONWeight int `gorm:"-" json:"weight"` - - Locked bool `json:"is-locked"` - Author Profile `json:"author"` - AuthorID string `json:"-"` - Comments int `json:"comments"` - Milestone string `json:"milestone"` - Upvotes int `json:"upvotes"` - Downvotes int `json:"downvotes"` -} - -type IssueLabel struct { - ID string `gorm:"primary_key"` - Color string -} - -type Profile struct { - ID string `gorm:"primary_key"` - Name string -} - -func FromGitHubIssue(input *github.Issue) *Issue { - body := "" - if input.Body != nil { - body = *input.Body - } - parts := strings.Split(*input.HTMLURL, "/") - authorName := *input.User.Login - if input.User.Name != nil { - authorName = *input.User.Name - } - issue := &Issue{ - CreatedAt: *input.CreatedAt, - UpdatedAt: *input.UpdatedAt, - CompletedAt: input.GetClosedAt(), - Provider: GitHubProvider, - GitHub: input, - Number: *input.Number, - Title: *input.Title, - State: *input.State, - Body: body, - IsPR: input.PullRequestLinks != nil, - URL: strings.Replace(*input.HTMLURL, "/pull/", "/issues/", -1), - RepoURL: strings.Join(parts[0:len(parts)-2], "/"), - Labels: make([]*IssueLabel, 0), - Assignees: make([]*Profile, 0), - Locked: *input.Locked, - Comments: *input.Comments, - Upvotes: *input.Reactions.PlusOne, - Downvotes: *input.Reactions.MinusOne, - Author: Profile{ - ID: *input.User.Login, - Name: authorName, - }, - } - if input.Milestone != nil { - issue.Milestone = *input.Milestone.Title - } - for _, label := range input.Labels { - issue.Labels = append(issue.Labels, &IssueLabel{ - ID: *label.Name, - Color: *label.Color, - }) - } - for _, assignee := range input.Assignees { - name := *assignee.Login - if assignee.Name != nil { - name = *assignee.Name - } - issue.Assignees = append(issue.Assignees, &Profile{ - ID: *assignee.Login, - Name: name, - }) - } - return issue -} - -func FromGitLabIssue(input *gitlab.Issue) *Issue { - if debugFetch { - out, _ := json.MarshalIndent(input, "", " ") - log.Println(string(out)) - } - repoURL := input.Links.Project - if repoURL == "" { - repoURL = strings.Replace(input.WebURL, fmt.Sprintf("/issues/%d", input.IID), "", -1) - } - issue := &Issue{ - CreatedAt: *input.CreatedAt, - UpdatedAt: *input.UpdatedAt, - Provider: GitLabProvider, - GitLab: input, - Number: input.IID, - Title: input.Title, - State: input.State, - URL: input.WebURL, - Body: input.Description, - RepoURL: repoURL, - Labels: make([]*IssueLabel, 0), - Assignees: make([]*Profile, 0), - } - if issue.State == "opened" { - issue.State = "open" - } - for _, label := range input.Labels { - issue.Labels = append(issue.Labels, &IssueLabel{ - ID: label, - Color: "cccccc", - }) - } - for _, assignee := range input.Assignees { - issue.Assignees = append(issue.Assignees, &Profile{ - Name: assignee.Name, - ID: assignee.Username, - }) - } - return issue -} - -func (i *Issue) WithJSONFields() *Issue { - i.JSONWeight = i.Weight() - if len(i.Blocks) > 0 { - i.JSONParents = []string{} - for _, rel := range i.Blocks { - i.JSONParents = append(i.JSONParents, rel.URL) - } - } - if len(i.DependsOn) > 0 { - i.JSONChildren = []string{} - for _, rel := range i.DependsOn { - i.JSONChildren = append(i.JSONChildren, rel.URL) - } - } - return i -} - -func (i Issue) Path() string { - u, err := url.Parse(i.URL) - if err != nil { - return "" - } - parts := strings.Split(u.Path, "/") - return strings.Join(parts[:len(parts)-2], "/") -} - -type IssueSlice []*Issue - -func (s IssueSlice) Unique() IssueSlice { - return s.ToMap().ToSlice() -} - -type Issues map[string]*Issue - -func (m Issues) ToSlice() IssueSlice { - slice := IssueSlice{} - for _, issue := range m { - slice = append(slice, issue) - } - return slice -} - -func (s IssueSlice) ToMap() Issues { - m := Issues{} - for _, issue := range s { - m[issue.URL] = issue - } - return m -} - -func (i Issue) ProviderURL() string { - u, _ := url.Parse(i.URL) - return fmt.Sprintf("%s://%s", u.Scheme, u.Host) -} - -func (i Issue) IsEpic() bool { - for _, label := range i.Labels { - if label.ID == viper.GetString("epic-label") { - return true - } - } - return false - //return !i.IsOrphan && len(i.Blocks) == 0 -} - -func (i Issue) Repo() string { - return strings.Split(i.URL, "/")[5] -} - -func (i Issue) RepoID() string { - id := i.Path()[1:] - id = strings.Replace(id, "/", "", -1) - id = strings.Replace(id, "-", "", -1) - return id -} - -func (i Issue) Owner() string { - return strings.Split(i.URL, "/")[4] -} - -func (i Issue) IsClosed() bool { - return i.State == "closed" -} - -func (i Issue) IsReady() bool { - return !i.IsOrphan && len(i.DependsOn) == 0 -} - -func (i Issue) NodeName() string { - return fmt.Sprintf(`%s#%d`, i.Path()[1:], i.Number) -} - -func (i Issue) NodeTitle() string { - title := fmt.Sprintf("%s: %s", i.NodeName(), i.Title) - title = strings.Replace(title, "|", "-", -1) - title = strings.Replace(html.EscapeString(wrap(title, 20)), "\n", "
", -1) - labels := []string{} - for _, label := range i.Labels { - switch label.ID { - case "t/step", "t/epic", "epic": - continue - } - labels = append(labels, fmt.Sprintf(`%s`, label.Color, label.ID)) - } - labelsText := "" - if len(labels) > 0 { - labelsText = "" + strings.Join(labels, "") + "
" - } - assigneeText := "" - if len(i.Assignees) > 0 { - assignees := []string{} - for _, assignee := range i.Assignees { - assignees = append(assignees, assignee.ID) - } - assigneeText = fmt.Sprintf(`@%s`, strings.Join(assignees, ", @")) - } - errorsText := "" - if len(i.Errors) > 0 { - errors := []string{} - for _, err := range i.Errors { - errors = append(errors, err.Error()) - } - errorsText = fmt.Sprintf(`ERR: %s`, strings.Join(errors, ";
ERR: ")) - } - return fmt.Sprintf(`<%s%s%s
%s
>`, title, labelsText, assigneeText, errorsText) -} - -func normalizeURL(input string) string { - parts := strings.Split(input, "://") - output := fmt.Sprintf("%s://%s", parts[0], strings.Replace(parts[1], "//", "/", -1)) - output = strings.TrimRight(output, "#") - output = strings.TrimRight(output, "/") - return output -} - -func (i Issue) GetRelativeIssueURL(target string) string { +func (i Issue) GetRelativeURL(target string) string { if strings.Contains(target, "://") { return normalizeURL(target) } if target[0] == '#' { - return fmt.Sprintf("%s/issues/%s", i.RepoURL, target[1:]) + return fmt.Sprintf("%s/issues/%s", i.Repository.URL, target[1:]) } target = strings.Replace(target, "#", "/issues/", -1) @@ -330,293 +22,65 @@ func (i Issue) GetRelativeIssueURL(target string) string { return fmt.Sprintf("https://%s", target) } - return fmt.Sprintf("%s/%s", strings.TrimRight(i.ProviderURL(), "/"), target) -} - -func (i Issue) BlocksAnEpic() bool { - return i.blocksAnEpic(0) -} - -func (i Issue) String() string { - out, _ := json.Marshal(i) - return string(out) + return fmt.Sprintf("%s/%s", strings.TrimRight(i.Repository.Provider.URL, "/"), target) } -func (i Issue) blocksAnEpic(depth int) bool { - if depth > 100 { - log.Printf("very high blocking depth (>100), do not continue. (issue=%s)", i) - return false - } - for _, dep := range i.Blocks { - if dep.IsEpic() || dep.blocksAnEpic(depth+1) { - return true - } - } - return false -} - -func (i Issue) DependsOnAnEpic() bool { - return i.dependsOnAnEpic(0) -} - -func (i Issue) dependsOnAnEpic(depth int) bool { - if depth > 100 { - log.Printf("very high blocking depth (>100), do not continue. (issue=%s)", i) - return false - } - for _, dep := range i.DependsOn { - if dep.IsEpic() || dep.dependsOnAnEpic(depth+1) { - return true - } - } - return false -} - -func (i Issue) Weight() int { - weight := i.BaseWeight - for _, dep := range i.Blocks.Unique() { - weight += dep.Weight() - } - return weight * i.WeightMultiplier() -} - -func (i Issue) WeightMultiplier() int { - multiplier := i.weightMultiplier - for _, dep := range i.Blocks.Unique() { - multiplier *= dep.WeightMultiplier() - } - return multiplier -} - -func (i Issue) AddEdgesToGraph(g *gographviz.Graph) error { - if i.Hidden { - return nil - } - for _, dependency := range i.DependsOn { - if dependency.Hidden { - continue - } - attrs := map[string]string{} - attrs["color"] = "lightblue" - //attrs["label"] = "depends on" - //attrs["style"] = "dotted" - attrs["dir"] = "none" - if i.IsClosed() || dependency.IsClosed() { - attrs["color"] = "grey" - attrs["style"] = "dotted" - } - if dependency.IsReady() { - attrs["color"] = "pink" - } - if i.IsEpic() { - attrs["color"] = "orange" - attrs["style"] = "dashed" - } - //log.Print("edge", escape(i.URL), "->", escape(dependency.URL)) - if err := g.AddEdge( - escape(i.URL), - escape(dependency.URL), - true, - attrs, - ); err != nil { - return err - } - } - return nil -} - -func (i Issue) AddNodeToGraph(g *gographviz.Graph, parent string) error { - attrs := map[string]string{} - attrs["label"] = i.NodeTitle() - //attrs["xlabel"] = "" - attrs["shape"] = "record" - attrs["style"] = `"rounded,filled"` - attrs["color"] = "lightblue" - attrs["href"] = escape(i.URL) - - if i.IsEpic() { - attrs["shape"] = "oval" - } - - switch { - - case i.IsClosed(): - attrs["color"] = `"#cccccc33"` - - case i.IsReady(): - attrs["color"] = "pink" - - case i.IsEpic(): - attrs["color"] = "orange" - attrs["style"] = `"rounded,filled,bold"` - - case i.IsOrphan || !i.LinkedWithEpic: - attrs["color"] = "gray" - } - - return g.AddNode( - parent, - escape(i.URL), - attrs, - ) -} - -func (issues Issues) prepare(includePRs bool) error { - var ( - dependsOnRegex, _ = regexp.Compile(`(?i)(require|requires|blocked by|block by|depend on|depends on|parent of) ([a-z0-9:/_.-]+issues/[0-9]+|[a-z0-9:/_.-]+#[0-9]+|[a-z0-9/_-]*#[0-9]+)`) - blocksRegex, _ = regexp.Compile(`(?i)(blocks|block|address|addresses|part of|child of|fix|fixes) ([a-z0-9:/_.-]+issues/[0-9]+|[a-z0-9:/_.-]+#[0-9]+|[a-z0-9/_-]*#[0-9]+)`) - isDuplicateRegex, _ = regexp.Compile(`(?i)(duplicates|duplicate|dup of|dup|duplicate of) ([a-z0-9:/_.-]+issues/[0-9]+|[a-z0-9:/_.-]+#[0-9]+|[a-z0-9/_-]*#[0-9]+)`) - weightMultiplierRegex, _ = regexp.Compile(`(?i)(depviz.weight_multiplier[:= ]+)([0-9]+)`) - baseWeightRegex, _ = regexp.Compile(`(?i)(depviz.base_weight|depviz.weight)[:= ]+([0-9]+)`) - hideFromRoadmapRegex, _ = regexp.Compile(`(?i)(depviz.hide)`) // FIXME: use label - ) - - for _, issue := range issues { - issue.DependsOn = make([]*Issue, 0) - issue.Blocks = make([]*Issue, 0) - issue.IsOrphan = true - issue.weightMultiplier = 1 - issue.BaseWeight = 1 +func (i *Issue) PostLoad() { + i.ParentIDs = []string{} + i.ChildIDs = []string{} + i.DuplicateIDs = []string{} + for _, rel := range i.Parents { + i.ParentIDs = append(i.ParentIDs, rel.ID) } - for _, issue := range issues { - if issue.Body == "" { - continue - } - - if match := isDuplicateRegex.FindStringSubmatch(issue.Body); match != nil { - issue.Duplicates = append(issue.Duplicates, issue.GetRelativeIssueURL(match[len(match)-1])) - } - - if match := weightMultiplierRegex.FindStringSubmatch(issue.Body); match != nil { - issue.weightMultiplier, _ = strconv.Atoi(match[len(match)-1]) - } - - if match := hideFromRoadmapRegex.FindStringSubmatch(issue.Body); match != nil { - delete(issues, issue.URL) - continue - } - - if match := baseWeightRegex.FindStringSubmatch(issue.Body); match != nil { - issue.BaseWeight, _ = strconv.Atoi(match[len(match)-1]) - } - - for _, match := range dependsOnRegex.FindAllStringSubmatch(issue.Body, -1) { - num := issue.GetRelativeIssueURL(match[len(match)-1]) - dep, found := issues[num] - if !found { - issue.Errors = append(issue.Errors, fmt.Errorf("parent %q not found", num)) - continue - } - issue.DependsOn = append(issue.DependsOn, dep) - issues[num].Blocks = append(dep.Blocks, issue) - issue.IsOrphan = false - issues[num].IsOrphan = false - } - - for _, match := range blocksRegex.FindAllStringSubmatch(issue.Body, -1) { - num := issue.GetRelativeIssueURL(match[len(match)-1]) - dep, found := issues[num] - if !found { - issue.Errors = append(issue.Errors, fmt.Errorf("child %q not found", num)) - continue - } - issues[num].DependsOn = append(dep.DependsOn, issue) - issue.Blocks = append(issue.Blocks, dep) - issue.IsOrphan = false - issues[num].IsOrphan = false - } + for _, rel := range i.Children { + i.ChildIDs = append(i.ChildIDs, rel.ID) } - for _, issue := range issues { - if len(issue.Duplicates) > 0 { - issue.Hidden = true - } - if !includePRs && issue.IsPR { - issue.Hidden = true - } + for _, rel := range i.Duplicates { + i.DuplicateIDs = append(i.DuplicateIDs, rel.ID) } - issues.processEpicLinks() - return nil } -func (issues Issues) processEpicLinks() { - for _, issue := range issues { - issue.LinkedWithEpic = !issue.Hidden && (issue.IsEpic() || issue.BlocksAnEpic() || issue.DependsOnAnEpic()) - - } +func (i Issue) IsClosed() bool { + return i.State == "closed" } -func (issues Issues) filterByTargets(targets []string) { - for _, issue := range issues { - if issue.Hidden { - continue - } - issue.Hidden = !issue.MatchesWithATarget(targets) - } +func (i Issue) IsReady() bool { + return !i.IsOrphan && len(i.Parents) == 0 // FIXME: switch parents with children? } -func (i Issue) MatchesWithATarget(targets []string) bool { +func (i Issue) MatchesWithATarget(targets Targets) bool { return i.matchesWithATarget(targets, 0) } -func (i Issue) matchesWithATarget(targets []string, depth int) bool { +func (i Issue) matchesWithATarget(targets Targets, depth int) bool { if depth > 100 { - log.Printf("very high blocking depth (>100), do not continue. (issue=%s)", i) + log.Printf("circular dependency or too deep graph (>100), skipping this node. (issue=%s)", i) return false } - issueParts := strings.Split(strings.TrimRight(i.URL, "/"), "/") + for _, target := range targets { - fullTarget := i.GetRelativeIssueURL(target) - targetParts := strings.Split(strings.TrimRight(fullTarget, "/"), "/") - if len(issueParts) == len(targetParts) { - if i.URL == fullTarget { + if target.Issue() != "" { // issue-mode + if target.Canonical() == i.URL { return true } - } else { - if len(fullTarget) <= len(i.URL) && i.URL[:len(fullTarget)] == fullTarget { + } else { // project-mode + if i.RepositoryID == target.ProjectURL() { return true } } } - for _, parent := range i.Blocks { + for _, parent := range i.Parents { if parent.matchesWithATarget(targets, depth+1) { return true } } - return false -} - -func (issues Issues) HideClosed() { - for _, issue := range issues { - if issue.IsClosed() { - issue.Hidden = true - } - } -} - -func (issues Issues) HideOrphans() { - for _, issue := range issues { - if issue.IsOrphan || !issue.LinkedWithEpic { - issue.Hidden = true - } - } -} - -func (issues Issues) HasOrphans() bool { - for _, issue := range issues { - if !issue.Hidden && issue.IsOrphan { + for _, child := range i.Children { + if child.matchesWithATarget(targets, depth+1) { return true } } - return false -} -func (issues Issues) HasNonOrphans() bool { - for _, issue := range issues { - if !issue.Hidden && !issue.IsOrphan && issue.LinkedWithEpic { - return true - } - } return false } diff --git a/issue_test.go b/issue_test.go index 556ff4ca4..a2fb2ea63 100644 --- a/issue_test.go +++ b/issue_test.go @@ -2,10 +2,15 @@ package main import "fmt" -func ExampleIssue_GetRelativeIssueURL() { +func ExampleIssue_GetRelativeURL() { issue := Issue{ - URL: "https://github.com/moul/depviz/issues/42", - RepoURL: "https://github.com/moul/depviz", + URL: "https://github.com/moul/depviz/issues/42", + Repository: &Repository{ + URL: "https://github.com/moul/depviz", + Provider: &Provider{ + URL: "https://github.com/", + }, + }, } for _, target := range []string{ "#43", @@ -20,12 +25,17 @@ func ExampleIssue_GetRelativeIssueURL() { "gitlab.com/test2/issues/52", "gitlab.com/test2/test3/test4/issues/53", } { - fmt.Printf("%-42s -> %s\n", target, issue.GetRelativeIssueURL(target)) + fmt.Printf("%-42s -> %s\n", target, issue.GetRelativeURL(target)) } issue = Issue{ - URL: "https://gitlab.com/moul/depviz/issues/42", - RepoURL: "https://gitlab.com/moul/depviz", + URL: "https://gitlab.com/moul/depviz/issues/42", + Repository: &Repository{ + URL: "https://gitlab.com/moul/depviz", + Provider: &Provider{ + URL: "https://gitlab.com/", + }, + }, } for _, target := range []string{ "#43", @@ -40,7 +50,7 @@ func ExampleIssue_GetRelativeIssueURL() { "github.com/test2/issues/52", "github.com/test2/test3/test4/issues/53", } { - fmt.Printf("%-42s -> %s\n", target, issue.GetRelativeIssueURL(target)) + fmt.Printf("%-42s -> %s\n", target, issue.GetRelativeURL(target)) } // Output: diff --git a/issues.go b/issues.go new file mode 100644 index 000000000..a3b03d7d3 --- /dev/null +++ b/issues.go @@ -0,0 +1,24 @@ +package main + +type Issues []*Issue + +func (issues Issues) Get(id string) *Issue { + for _, issue := range issues { + if issue.ID == id { + return issue + } + } + return nil +} + +func (issues Issues) FilterByTargets(targets []Target) Issues { + filtered := Issues{} + + for _, issue := range issues { + if issue.MatchesWithATarget(targets) { + filtered = append(filtered, issue) + } + } + + return filtered +} diff --git a/main.go b/main.go index 02d85df48..53d5c958d 100644 --- a/main.go +++ b/main.go @@ -71,6 +71,26 @@ func newRootCommand() *cobra.Command { } } + // fill global options + if err := viper.Unmarshal(&globalGraphOptions); err != nil { + return err + } + if err := viper.Unmarshal(&globalRunOptions); err != nil { + return err + } + if err := viper.Unmarshal(&globalPullOptions); err != nil { + return err + } + if err := viper.Unmarshal(&globalWebOptions); err != nil { + return err + } + if err := viper.Unmarshal(&globalAirtableOptions); err != nil { + return err + } + if err := viper.Unmarshal(&globalDBOptions); err != nil { + return err + } + // configure sql dbPath = os.ExpandEnv(dbPath) db, err = gorm.Open("sqlite3", dbPath) @@ -89,8 +109,11 @@ func newRootCommand() *cobra.Command { db.LogMode(verbose) if err := db.AutoMigrate( Issue{}, - IssueLabel{}, - Profile{}, + Label{}, + Account{}, + Milestone{}, + Repository{}, + Provider{}, ).Error; err != nil { return err } @@ -99,10 +122,11 @@ func newRootCommand() *cobra.Command { } cmd.AddCommand( newPullCommand(), - newRunCommand(), newDBCommand(), - newWebCommand(), newAirtableCommand(), + newGraphCommand(), + newRunCommand(), + newWebCommand(), ) viper.AutomaticEnv() viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) diff --git a/models.go b/models.go new file mode 100644 index 000000000..4d9837b23 --- /dev/null +++ b/models.go @@ -0,0 +1,172 @@ +package main + +import ( + "encoding/json" + "time" + + "github.com/lib/pq" +) + +// +// Base +// + +type Base struct { + ID string `gorm:"primary_key" json:"id"` + CreatedAt time.Time `json:"created-at,omitempty"` + UpdatedAt time.Time `json:"updated-at,omitempty"` + Errors pq.StringArray `json:"errors,omitempty" gorm:"type:varchar[]"` +} + +// +// Repository +// + +type Repository struct { + Base + + // base fields + URL string `json:"url"` + Title string `json:"name"` + Description string `json:"description"` + Homepage string `json:"homepage"` + PushedAt time.Time `json:"pushed-at"` + IsFork bool `json:"fork"` + + // relationships + Provider *Provider `json:"provider"` + ProviderID string `json:"provider-id"` + Owner *Account `json:"owner"` + OwnerID string `json:"owner-id"` +} + +// +// Provider +// + +type ProviderDriver string + +const ( + UnknownProviderDriver ProviderDriver = "unknown" + GithubDriver = "github" + GitlabDriver = "gitlab" +) + +type Provider struct { + Base + + // base fields + URL string `json:"url"` + Driver string `json:"driver"` // github, gitlab, unknown +} + +// +// Milestone +// + +type Milestone struct { + Base + + // base fields + URL string `json:"url"` + Title string `json:"title"` + Description string `json:"description"` + ClosedAt time.Time `json:"closed-at"` + DueOn time.Time `json:"due-on"` + // State string // FIXME: todo + // StartAt time.Time // FIXME: todo + + // relationships + Creator *Account `json:"creator"` + CreatorID string `json:"creator-id"` + Repository *Repository `json:"repository"` + RepositoryID string `json:"repository-id"` +} + +// +// Issue +// + +type Issue struct { + Base + + // base fields + URL string `json:"url"` + CompletedAt time.Time `json:"completed-at"` + Title string `json:"title"` + State string `json:"state"` + Body string `json:"body"` + IsPR bool `json:"is-pr"` + IsLocked bool `json:"is-locked"` + Comments int `json:"comments"` + Upvotes int `json:"upvotes"` + Downvotes int `json:"downvotes"` + IsOrphan bool `json:"is-orphan"` + IsHidden bool `json:"is-hidden"` + Weight int `json:"weight"` + IsEpic bool `json:"is-epic"` + HasEpic bool `json:"has-epic"` + + // relationships + Repository *Repository `json:"repository"` + RepositoryID string `json:"repository_id"` + Milestone *Milestone `json:"milestone"` + MilestoneID string `json:"milestone_id"` + Author *Account `json:"author"` + AuthorID string `json:"author_id"` + Labels []*Label `gorm:"many2many:issue_labels" json:"labels"` + Assignees []*Account `gorm:"many2many:issue_assignees" json:"assignees"` + Parents []*Issue `json:"-" gorm:"many2many:issue_parents;association_jointable_foreignkey:parent_id"` + Children []*Issue `json:"-" gorm:"many2many:issue_children;association_jointable_foreignkey:child_id"` + Duplicates []*Issue `json:"-" gorm:"many2many:issue_duplicates;association_jointable_foreignkey:duplicate_id"` + + // internal + ChildIDs []string `json:"child_ids" gorm:"-"` + ParentIDs []string `json:"parent_ids" gorm:"-"` + DuplicateIDs []string `json:"duplicate_ids" gorm:"-"` +} + +func (i Issue) String() string { + out, _ := json.Marshal(i) + return string(out) +} + +// +// Label +// + +type Label struct { + Base + + // base fields + URL string `json:"url"` + Name string `json:"name"` + Color string `json:"color"` + Description string `json:"description"` +} + +// +// Account +// + +type Account struct { + Base + + // base fields + URL string `json:"url"` + Login string `json:"login"` + FullName string `json:"fullname"` + Type string `json:"type"` + Bio string `json:"bio"` + Location string `json:"location"` + Company string `json:"company"` + Blog string `json:"blog"` + Email string `json:"email"` + AvatarURL string `json:"avatar-url"` + + // relationships + Provider *Provider `json:"provider"` + ProviderID string `json:"provider-id"` +} + +// FIXME: create a User struct to handle multiple accounts and aliases diff --git a/models_airtable.go b/models_airtable.go new file mode 100644 index 000000000..b87d39cd7 --- /dev/null +++ b/models_airtable.go @@ -0,0 +1,610 @@ +package main + +import ( + "encoding/json" + "strings" + "time" + + "github.com/brianloveswords/airtable" +) + +type AirtableBase struct { + ID string `json:"id"` + CreatedAt time.Time `json:"created-at"` + UpdatedAt time.Time `json:"updated-at"` + Errors string `json:"errors"` +} + +type airtableState int + +type AirtableRecords []interface{} + +type AirtableEntry interface { + ToRecord(cache AirtableDB) interface{} +} + +const ( + airtableStateUnknown airtableState = iota + airtableStateUnchanged + airtableStateChanged + airtableStateNew +) + +var ( + airtableStateString = map[airtableState]string{ + airtableStateUnknown: "unknown", + airtableStateUnchanged: "unchanged", + airtableStateChanged: "changed", + airtableStateNew: "new", + } +) + +// +// provider +// + +type ProviderRecord struct { + State airtableState `json:"-"` // internal + + airtable.Record // provides ID, CreatedTime + Fields struct { + // base + AirtableBase + + // specific + URL string `json:"url"` + Driver string `json:"driver"` + + // relationship + // n/a + } `json:"fields,omitempty"` +} + +func (r ProviderRecord) String() string { + out, _ := json.Marshal(r) + return string(out) +} + +func (p Provider) ToRecord(cache AirtableDB) *ProviderRecord { + record := ProviderRecord{} + + // base + record.Fields.ID = p.ID + record.Fields.CreatedAt = p.CreatedAt + record.Fields.UpdatedAt = p.UpdatedAt + record.Fields.Errors = strings.Join(p.Errors, ", ") + + // specific + record.Fields.URL = p.URL + record.Fields.Driver = p.Driver + + // relationships + // n/a + + return &record +} + +func (r *ProviderRecord) Equals(n *ProviderRecord) bool { + return true && + // base + r.Fields.ID == n.Fields.ID && + isSameAirtableDate(r.Fields.CreatedAt, n.Fields.CreatedAt) && + isSameAirtableDate(r.Fields.UpdatedAt, n.Fields.UpdatedAt) && + r.Fields.Errors == n.Fields.Errors && + + // specific + r.Fields.URL == n.Fields.URL && + r.Fields.Driver == n.Fields.Driver && + + // relationships + // n/a + + true +} + +type ProviderRecords []ProviderRecord + +func (records ProviderRecords) ByID(id string) string { + for _, record := range records { + if record.Fields.ID == id { + return record.ID + } + } + return "" +} + +// +// label +// + +type LabelRecord struct { + State airtableState `json:"-"` // internal + + airtable.Record // provides ID, CreatedTime + Fields struct { + // base + AirtableBase + + // specific + URL string `json:"url"` + Name string `json:"name"` + Color string `json:"color"` + Description string `json:"description"` + + // relationship + // n/a + } `json:"fields,omitempty"` +} + +func (r LabelRecord) String() string { + out, _ := json.Marshal(r) + return string(out) +} + +func (p Label) ToRecord(cache AirtableDB) *LabelRecord { + record := LabelRecord{} + + // base + record.Fields.ID = p.ID + record.Fields.CreatedAt = p.CreatedAt + record.Fields.UpdatedAt = p.UpdatedAt + record.Fields.Errors = strings.Join(p.Errors, ", ") + + // specific + record.Fields.URL = p.URL + record.Fields.Name = p.Name + record.Fields.Color = p.Color + record.Fields.Description = p.Description + + // relationships + // n/a + + return &record +} + +func (r *LabelRecord) Equals(n *LabelRecord) bool { + return true && + // base + r.Fields.ID == n.Fields.ID && + isSameAirtableDate(r.Fields.CreatedAt, n.Fields.CreatedAt) && + isSameAirtableDate(r.Fields.UpdatedAt, n.Fields.UpdatedAt) && + r.Fields.Errors == n.Fields.Errors && + + // specific + r.Fields.URL == n.Fields.URL && + r.Fields.Name == n.Fields.Name && + r.Fields.Color == n.Fields.Color && + r.Fields.Description == n.Fields.Description && + + // relationships + // n/a + + true +} + +type LabelRecords []LabelRecord + +func (records LabelRecords) ByID(id string) string { + for _, record := range records { + if record.Fields.ID == id { + return record.ID + } + } + return "" +} + +// +// account +// + +type AccountRecord struct { + State airtableState `json:"-"` // internal + + airtable.Record // provides ID, CreatedTime + Fields struct { + // base + AirtableBase + + // specific + URL string `json:"url"` + Login string `json:"login"` + FullName string `json:"fullname"` + Type string `json:"type"` + Bio string `json:"bio"` + Location string `json:"location"` + Company string `json:"company"` + Blog string `json:"blog"` + Email string `json:"email"` + AvatarURL string `json:"avatar-url"` + + // relationships + Provider []string `json:"provider"` + } `json:"fields,omitempty"` +} + +func (r AccountRecord) String() string { + out, _ := json.Marshal(r) + return string(out) +} + +func (p Account) ToRecord(cache AirtableDB) *AccountRecord { + record := AccountRecord{} + // base + record.Fields.ID = p.ID + record.Fields.CreatedAt = p.CreatedAt + record.Fields.UpdatedAt = p.UpdatedAt + record.Fields.Errors = strings.Join(p.Errors, ", ") + + // specific + record.Fields.URL = p.URL + record.Fields.Login = p.Login + record.Fields.FullName = p.FullName + record.Fields.Type = p.Type + record.Fields.Bio = p.Bio + record.Fields.Location = p.Location + record.Fields.Company = p.Company + record.Fields.Blog = p.Blog + record.Fields.Email = p.Email + record.Fields.AvatarURL = p.AvatarURL + + // relationships + record.Fields.Provider = []string{cache.Providers.ByID(p.Provider.ID)} + + return &record +} + +func (r *AccountRecord) Equals(n *AccountRecord) bool { + return true && + + // base + r.Fields.ID == n.Fields.ID && + isSameAirtableDate(r.Fields.CreatedAt, n.Fields.CreatedAt) && + isSameAirtableDate(r.Fields.UpdatedAt, n.Fields.UpdatedAt) && + r.Fields.Errors == n.Fields.Errors && + + // specific + r.Fields.URL == n.Fields.URL && + r.Fields.Login == n.Fields.Login && + r.Fields.FullName == n.Fields.FullName && + r.Fields.Type == n.Fields.Type && + r.Fields.Bio == n.Fields.Bio && + r.Fields.Location == n.Fields.Location && + r.Fields.Company == n.Fields.Company && + r.Fields.Blog == n.Fields.Blog && + r.Fields.Email == n.Fields.Email && + r.Fields.AvatarURL == n.Fields.AvatarURL && + + // relationships + isSameStringSlice(r.Fields.Provider, n.Fields.Provider) && + + true +} + +type AccountRecords []AccountRecord + +func (records AccountRecords) ByID(id string) string { + for _, record := range records { + if record.Fields.ID == id { + return record.ID + } + } + return "" +} + +// +// repository +// + +type RepositoryRecord struct { + State airtableState `json:"-"` // internal + + airtable.Record // provides ID, CreatedTime + Fields struct { + // base + AirtableBase + + // specific + URL string `json:"url"` + Title string `json:"title"` + Description string `json:"description"` + Homepage string `json:"homepage"` + PushedAt time.Time `json:"pushed-at"` + IsFork bool `json:"is-fork"` + + // relationships + Provider []string `json:"provider"` + Owner []string `json:"owner"` + } `json:"fields,omitempty"` +} + +func (r RepositoryRecord) String() string { + out, _ := json.Marshal(r) + return string(out) +} + +func (p Repository) ToRecord(cache AirtableDB) *RepositoryRecord { + record := RepositoryRecord{} + + // base + record.Fields.ID = p.ID + record.Fields.CreatedAt = p.CreatedAt + record.Fields.UpdatedAt = p.UpdatedAt + record.Fields.Errors = strings.Join(p.Errors, ", ") + + // specific + record.Fields.URL = p.URL + record.Fields.Title = p.Title + record.Fields.Description = p.Description + record.Fields.Homepage = p.Homepage + record.Fields.PushedAt = p.PushedAt + record.Fields.IsFork = p.IsFork + + // relationships + record.Fields.Provider = []string{cache.Providers.ByID(p.Provider.ID)} + if p.Owner != nil { + record.Fields.Owner = []string{cache.Accounts.ByID(p.Owner.ID)} + } + + return &record +} + +func (r *RepositoryRecord) Equals(n *RepositoryRecord) bool { + return true && + + // base + r.Fields.ID == n.Fields.ID && + isSameAirtableDate(r.Fields.CreatedAt, n.Fields.CreatedAt) && + isSameAirtableDate(r.Fields.UpdatedAt, n.Fields.UpdatedAt) && + r.Fields.Errors == n.Fields.Errors && + + // specific + r.Fields.URL == n.Fields.URL && + r.Fields.Title == n.Fields.Title && + r.Fields.Description == n.Fields.Description && + r.Fields.Homepage == n.Fields.Homepage && + isSameAirtableDate(r.Fields.PushedAt, n.Fields.PushedAt) && + r.Fields.IsFork == n.Fields.IsFork && + + // relationships + isSameStringSlice(r.Fields.Provider, n.Fields.Provider) && + isSameStringSlice(r.Fields.Owner, n.Fields.Owner) && + + true +} + +type RepositoryRecords []RepositoryRecord + +func (records RepositoryRecords) ByID(id string) string { + for _, record := range records { + if record.Fields.ID == id { + return record.ID + } + } + return "" +} + +// +// milestone +// + +type MilestoneRecord struct { + State airtableState `json:"-"` // internal + + airtable.Record // provides ID, CreatedTime + Fields struct { + // base + AirtableBase + + // specific + URL string `json:"url"` + Title string `json:"title"` + Description string `json:"description"` + ClosedAt time.Time `json:"closed-at"` + DueOn time.Time `json:"due-on"` + + // relationships + Creator []string `json:"creator"` + Repository []string `json:"repository"` + } `json:"fields,omitempty"` +} + +func (r MilestoneRecord) String() string { + out, _ := json.Marshal(r) + return string(out) +} + +func (p Milestone) ToRecord(cache AirtableDB) *MilestoneRecord { + record := MilestoneRecord{} + // base + record.Fields.ID = p.ID + record.Fields.CreatedAt = p.CreatedAt + record.Fields.UpdatedAt = p.UpdatedAt + record.Fields.Errors = strings.Join(p.Errors, ", ") + + // specific + record.Fields.URL = p.URL + record.Fields.Title = p.Title + record.Fields.Description = p.Description + record.Fields.ClosedAt = p.ClosedAt + record.Fields.DueOn = p.DueOn + + // relationships + if p.Creator != nil { + record.Fields.Creator = []string{cache.Accounts.ByID(p.Creator.ID)} + } + if p.Repository != nil { + record.Fields.Repository = []string{cache.Repositories.ByID(p.Repository.ID)} + } + + return &record +} + +func (r *MilestoneRecord) Equals(n *MilestoneRecord) bool { + return true && + + // base + r.Fields.ID == n.Fields.ID && + isSameAirtableDate(r.Fields.CreatedAt, n.Fields.CreatedAt) && + isSameAirtableDate(r.Fields.UpdatedAt, n.Fields.UpdatedAt) && + r.Fields.Errors == n.Fields.Errors && + + // specific + r.Fields.URL == n.Fields.URL && + r.Fields.Title == n.Fields.Title && + r.Fields.Description == n.Fields.Description && + isSameAirtableDate(r.Fields.ClosedAt, n.Fields.ClosedAt) && + isSameAirtableDate(r.Fields.DueOn, n.Fields.DueOn) && + + // relationships + isSameStringSlice(r.Fields.Creator, n.Fields.Creator) && + isSameStringSlice(r.Fields.Repository, n.Fields.Repository) && + + true +} + +type MilestoneRecords []MilestoneRecord + +func (records MilestoneRecords) ByID(id string) string { + for _, record := range records { + if record.Fields.ID == id { + return record.ID + } + } + return "" +} + +// +// issue +// + +type IssueRecord struct { + State airtableState `json:"-"` // internal + + airtable.Record // provides ID, CreatedTime + Fields struct { + // base + AirtableBase + + // specific + URL string `json:"url"` + CompletedAt time.Time `json:"completed-at"` + Title string `json:"title"` + State string `json:"state"` + Body string `json:"body"` + IsPR bool `json:"is-pr"` + IsLocked bool `json:"is-locked"` + Comments int `json:"comments"` + Upvotes int `json:"upvotes"` + Downvotes int `json:"downvotes"` + IsOrphan bool `json:"is-orphan"` + IsHidden bool `json:"is-hidden"` + Weight int `json:"weight"` + IsEpic bool `json:"is-epic"` + HasEpic bool `json:"has-epic"` + + // relationships + Repository []string `json:"repository"` + Milestone []string `json:"milestone"` + Author []string `json:"author"` + Labels []string `json:"labels"` + Assignees []string `json:"assignees"` + //Parents []string `json:"-"` + //Children []string `json:"-"` + //Duplicates []string `json:"-"` + } `json:"fields,omitempty"` +} + +func (r IssueRecord) String() string { + out, _ := json.Marshal(r) + return string(out) +} + +func (p Issue) ToRecord(cache AirtableDB) *IssueRecord { + record := IssueRecord{} + // base + record.Fields.ID = p.ID + record.Fields.CreatedAt = p.CreatedAt + record.Fields.UpdatedAt = p.UpdatedAt + record.Fields.Errors = strings.Join(p.Errors, ", ") + + // specific + record.Fields.URL = p.URL + record.Fields.CompletedAt = p.CompletedAt + record.Fields.Title = p.Title + record.Fields.State = p.State + record.Fields.Body = p.Body + record.Fields.IsPR = p.IsPR + record.Fields.IsLocked = p.IsLocked + record.Fields.Comments = p.Comments + record.Fields.Upvotes = p.Upvotes + record.Fields.Downvotes = p.Downvotes + record.Fields.IsOrphan = p.IsOrphan + record.Fields.IsHidden = p.IsHidden + record.Fields.Weight = p.Weight + record.Fields.IsEpic = p.IsEpic + record.Fields.HasEpic = p.HasEpic + + // relationships + record.Fields.Repository = []string{cache.Repositories.ByID(p.Repository.ID)} + if p.Milestone != nil { + record.Fields.Milestone = []string{cache.Milestones.ByID(p.Milestone.ID)} + } + record.Fields.Author = []string{cache.Accounts.ByID(p.Author.ID)} + record.Fields.Labels = []string{} + for _, label := range p.Labels { + record.Fields.Labels = append(record.Fields.Labels, cache.Labels.ByID(label.ID)) + } + record.Fields.Assignees = []string{} + for _, assignee := range p.Assignees { + record.Fields.Assignees = append(record.Fields.Assignees, cache.Accounts.ByID(assignee.ID)) + } + + return &record +} + +func (r *IssueRecord) Equals(n *IssueRecord) bool { + return true && + + // base + r.Fields.ID == n.Fields.ID && + isSameAirtableDate(r.Fields.CreatedAt, n.Fields.CreatedAt) && + isSameAirtableDate(r.Fields.UpdatedAt, n.Fields.UpdatedAt) && + r.Fields.Errors == n.Fields.Errors && + + // specific + r.Fields.URL == n.Fields.URL && + isSameAirtableDate(r.Fields.CompletedAt, n.Fields.CompletedAt) && + r.Fields.Title == n.Fields.Title && + r.Fields.State == n.Fields.State && + r.Fields.Body == n.Fields.Body && + r.Fields.IsPR == n.Fields.IsPR && + r.Fields.IsLocked == n.Fields.IsLocked && + r.Fields.Comments == n.Fields.Comments && + r.Fields.Upvotes == n.Fields.Upvotes && + r.Fields.Downvotes == n.Fields.Downvotes && + r.Fields.IsOrphan == n.Fields.IsOrphan && + r.Fields.IsHidden == n.Fields.IsHidden && + r.Fields.Weight == n.Fields.Weight && + r.Fields.IsEpic == n.Fields.IsEpic && + r.Fields.HasEpic == n.Fields.HasEpic && + + // relationships + isSameStringSlice(r.Fields.Repository, n.Fields.Repository) && + isSameStringSlice(r.Fields.Milestone, n.Fields.Milestone) && + isSameStringSlice(r.Fields.Author, n.Fields.Author) && + isSameStringSlice(r.Fields.Labels, n.Fields.Labels) && + isSameStringSlice(r.Fields.Assignees, n.Fields.Assignees) && + + true +} + +type IssueRecords []IssueRecord + +func (records IssueRecords) ByID(id string) string { + for _, record := range records { + if record.Fields.ID == id { + return record.ID + } + } + return "" +} diff --git a/repo.go b/repo.go deleted file mode 100644 index 03582627b..000000000 --- a/repo.go +++ /dev/null @@ -1,69 +0,0 @@ -package main - -import ( - "fmt" - "net/url" - "strings" -) - -type Repo string - -func NewRepo(path string) Repo { - parts := strings.Split(path, "/") - if len(parts) < 3 { - return Repo(fmt.Sprintf("https://github.com/%s", path)) - } - if !strings.Contains(path, "://") { - return Repo(fmt.Sprintf("https://%s", path)) - } - return Repo(path) -} - -// FIXME: make something more reliable -func (r Repo) Provider() Provider { - if strings.Contains(string(r), "github.com") { - return GitHubProvider - } - return GitLabProvider -} - -func (r Repo) Namespace() string { - u, err := url.Parse(string(r)) - if err != nil { - return "" - } - parts := strings.Split(u.Path, "/")[1:] - return strings.Join(parts[:len(parts)-1], "/") -} - -func (r Repo) Project() string { - parts := strings.Split(string(r), "/") - return parts[len(parts)-1] -} - -func (r Repo) Canonical() string { - // FIXME: use something smarter (the shortest unique response) - return string(r) -} - -func (r Repo) SiteURL() string { - switch r.Provider() { - case GitHubProvider: - return "https://github.com" - case GitLabProvider: - u, err := url.Parse(string(r)) - if err != nil { - return "" - } - return fmt.Sprintf("%s://%s", u.Scheme, u.Host) - } - panic("should not happen") -} - -func (r Repo) RepoPath() string { - u, err := url.Parse(string(r)) - if err != nil { - panic(err) - } - return u.Path[1:] -} diff --git a/target.go b/target.go new file mode 100644 index 000000000..242757dd7 --- /dev/null +++ b/target.go @@ -0,0 +1,156 @@ +package main + +import ( + "fmt" + "net/url" + "os" + "sort" + "strconv" + "strings" +) + +type Target string + +func ParseTargets(inputs []string) (Targets, error) { + targetMap := map[string]string{} + for _, input := range inputs { + // check if input is a local path + if _, err := os.Stat(input); err == nil { + return nil, fmt.Errorf("filesystem target are not yet supported") + } + + // parse issue + str := input + issue := "" + parts := strings.Split(str, "/issues/") + switch len(parts) { + case 1: + case 2: + str = parts[0] + issue = parts[1] + default: + return nil, fmt.Errorf("invalid target: %q", input) + } + parts = strings.Split(str, "#") + switch len(parts) { + case 1: + case 2: + str = parts[0] + issue = parts[1] + default: + return nil, fmt.Errorf("invalid target: %q", input) + } + + // parse scheme + parts = strings.Split(str, "/") + if len(parts) < 3 { + str = fmt.Sprintf("https://github.com/%s", str) + } + + if !strings.Contains(str, "://") { + str = fmt.Sprintf("https://%s", str) + } + + // append issue + if issue != "" { + _, err := strconv.Atoi(issue) + if err != nil { + return nil, fmt.Errorf("invalid target (issue): %q", input) + } + str = str + "/issues/" + issue + } + + target := string(str) + targetMap[target] = target + } + targets := []string{} + for _, target := range targetMap { + targets = append(targets, target) + } + targets = uniqueStrings(targets) + sort.Strings(targets) + + typed := Targets{} + for _, target := range targets { + typed = append(typed, Target(target)) + } + return typed, nil +} + +func (t Target) Issue() string { + parts := strings.Split(string(t), "/issues/") + switch len(parts) { + case 1: + return "" + case 2: + return parts[1] + default: + panic("invalid target") + } +} + +func (t Target) ProjectURL() string { + return strings.Split(string(t), "/issues/")[0] +} + +func (t Target) Namespace() string { + u, err := url.Parse(t.ProjectURL()) + if err != nil { + return "" + } + parts := strings.Split(u.Path, "/")[1:] + return strings.Join(parts[:len(parts)-1], "/") +} + +func (t Target) Project() string { + parts := strings.Split(t.ProjectURL(), "/") + return parts[len(parts)-1] +} + +func (t Target) Path() string { + return fmt.Sprintf("%s/%s", t.Namespace(), t.Project()) +} + +func (t Target) Canonical() string { return string(t) } + +func (t Target) Driver() ProviderDriver { + if strings.Contains(string(t), "github.com") { + return GithubDriver + } + return GitlabDriver +} + +func (t Target) ProviderURL() string { + switch t.Driver() { + case GithubDriver: + return "https://github.com" + case GitlabDriver: + u, err := url.Parse(string(t)) + if err != nil { + return "" + } + return fmt.Sprintf("%s://%s", u.Scheme, u.Host) + } + panic("should not happen") +} + +type Targets []Target + +func (t Targets) UniqueProjects() Targets { + urlMap := map[string]bool{} + for _, target := range t { + urlMap[target.ProjectURL()] = true + } + + urls := []string{} + for url := range urlMap { + urls = append(urls, url) + } + sort.Strings(urls) + + filtered := Targets{} + for _, url := range urls { + filtered = append(filtered, Target(url)) + } + return filtered +} diff --git a/target_test.go b/target_test.go new file mode 100644 index 000000000..3219f1091 --- /dev/null +++ b/target_test.go @@ -0,0 +1,57 @@ +package main + +import "fmt" + +func ExampleParseTargets() { + targets, _ := ParseTargets([]string{ + "moul/project1", + "moul/project2#42", + "github.com/moul/project3", + "https://github.com/moul/project4/issues/42", + "https://gitlab.com/moul/project5#42", + "gitlab.com/moul/project6", + }) + for _, target := range targets { + fmt.Printf( + "target=%q canonical=%q project=%q namespace=%q providerurl=%q driver=%q path=%q projecturl=%q issue=%q\n", + target, + target.Canonical(), + target.Project(), + target.Namespace(), + target.ProviderURL(), + target.Driver(), + target.Path(), + target.ProjectURL(), + target.Issue(), + ) + } + // Output: + // target="https://github.com/moul/project1" canonical="https://github.com/moul/project1" project="project1" namespace="moul" providerurl="https://github.com" driver="github" path="moul/project1" projecturl="https://github.com/moul/project1" issue="" + // target="https://github.com/moul/project2/issues/42" canonical="https://github.com/moul/project2/issues/42" project="project2" namespace="moul" providerurl="https://github.com" driver="github" path="moul/project2" projecturl="https://github.com/moul/project2" issue="42" + // target="https://github.com/moul/project3" canonical="https://github.com/moul/project3" project="project3" namespace="moul" providerurl="https://github.com" driver="github" path="moul/project3" projecturl="https://github.com/moul/project3" issue="" + // target="https://github.com/moul/project4/issues/42" canonical="https://github.com/moul/project4/issues/42" project="project4" namespace="moul" providerurl="https://github.com" driver="github" path="moul/project4" projecturl="https://github.com/moul/project4" issue="42" + // target="https://gitlab.com/moul/project5/issues/42" canonical="https://gitlab.com/moul/project5/issues/42" project="project5" namespace="moul" providerurl="https://gitlab.com" driver="gitlab" path="moul/project5" projecturl="https://gitlab.com/moul/project5" issue="42" + // target="https://gitlab.com/moul/project6" canonical="https://gitlab.com/moul/project6" project="project6" namespace="moul" providerurl="https://gitlab.com" driver="gitlab" path="moul/project6" projecturl="https://gitlab.com/moul/project6" issue="" +} + +func ExampleTargets_UniqueProjects() { + targets, _ := ParseTargets([]string{ + "moul/project1", + "moul/project2#42", + "moul/project2", + "github.com/moul/project1", + "https://github.com/moul/project2/issues/42", + "https://gitlab.com/moul/project1#42", + "https://gitlab.com/moul/project1", + "https://gitlab.com/moul/project2", + "gitlab.com/moul/project1", + }) + for _, target := range targets.UniqueProjects() { + fmt.Println(target.Canonical()) + } + // Output: + // https://github.com/moul/project1 + // https://github.com/moul/project2 + // https://gitlab.com/moul/project1 + // https://gitlab.com/moul/project2 +} diff --git a/util.go b/util.go index 84a04c261..f9dd6c595 100644 --- a/util.go +++ b/util.go @@ -2,9 +2,11 @@ package main import ( "fmt" - "os" + "reflect" "regexp" + "sort" "strings" + "time" ) func wrap(text string, lineWidth int) string { @@ -37,24 +39,6 @@ func panicIfErr(err error) { } } -func getReposFromTargets(targets []string) []string { - reposMap := map[string]bool{} - - for _, target := range targets { - if _, err := os.Stat(target); err == nil { - logger().Fatal("filesystem target are not yet supported") - } - repo := strings.Split(target, "/issues")[0] - repo = strings.Split(target, "#")[0] - reposMap[repo] = true - } - repos := []string{} - for repo := range reposMap { - repos = append(repos, repo) - } - return uniqueStrings(repos) -} - func uniqueStrings(input []string) []string { u := make([]string, 0, len(input)) m := make(map[string]bool) @@ -69,8 +53,32 @@ func uniqueStrings(input []string) []string { return u } +func normalizeURL(input string) string { + parts := strings.Split(input, "://") + output := fmt.Sprintf("%s://%s", parts[0], strings.Replace(parts[1], "//", "/", -1)) + output = strings.TrimRight(output, "#") + output = strings.TrimRight(output, "/") + return output +} + var rxDNSName = regexp.MustCompile(`^([a-zA-Z0-9_]{1}[a-zA-Z0-9_-]{0,62}){1}(\.[a-zA-Z0-9_]{1}[a-zA-Z0-9_-]{0,62})*[\._]?$`) func isDNSName(input string) bool { return rxDNSName.MatchString(input) } + +func isSameStringSlice(a, b []string) bool { + if a == nil { + a = []string{} + } + if b == nil { + b = []string{} + } + sort.Strings(a) + sort.Strings(b) + return reflect.DeepEqual(a, b) +} + +func isSameAirtableDate(a, b time.Time) bool { + return a.Truncate(time.Millisecond).UTC() == b.Truncate(time.Millisecond).UTC() +}