Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: avoid updating identiqual airtable record (using another golang… #99

Merged
merged 1 commit into from
Oct 7, 2018
Merged

feat: avoid updating identiqual airtable record (using another golang… #99

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 37 additions & 32 deletions cmd_airtable.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@ package main
import (
"encoding/json"
"reflect"
"sort"
"time"

airtable "github.com/fabioberger/airtable-go"
"github.com/brianloveswords/airtable"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
Expand Down Expand Up @@ -67,15 +68,17 @@ func airtableSync(opts *airtableOptions) error {
issues.filterByTargets(opts.Targets)
logger().Debug("fetch db entries", zap.Int("count", len(issues)))

at, err := airtable.New(opts.AirtableToken, opts.AirtableBaseID)
if err != nil {
return err
at := airtable.Client{
APIKey: opts.AirtableToken,
BaseID: opts.AirtableBaseID,
Limiter: airtable.RateLimiter(5),
}
table := at.Table(opts.AirtableTableName)

alreadyInAirtable := map[string]bool{}

records := []airtableRecord{}
if err := at.ListRecords(opts.AirtableTableName, &records); err != nil {
if err := table.List(&records, &airtable.Options{}); err != nil {
return err
}
logger().Debug("fetched airtable records", zap.Int("count", len(records)))
Expand All @@ -95,16 +98,17 @@ func airtableSync(opts *airtableOptions) error {
r := issue.ToAirtableRecord()
r.Fields.Labels = []string{}
r.Fields.Assignees = []string{}
if err := at.CreateRecord(opts.AirtableTableName, r); err != nil {
if err := table.Create(&r); err != nil {
return err
}
records = append(records, r)
}

// update/destroy existing ones
for _, record := range records {
if issue, found := issues[record.Fields.URL]; !found {
logger().Debug("destroying airtable record", zap.String("URL", record.Fields.URL))
if err := at.DestroyRecord(opts.AirtableTableName, record.ID); err != nil {
if err := table.Delete(&record); err != nil {
return errors.Wrap(err, "failed to destroy record")
}
} else {
Expand All @@ -117,13 +121,17 @@ func airtableSync(opts *airtableOptions) error {
}

logger().Debug("updating airtable record", zap.String("URL", issue.URL))
m := issue.ToAirtableRecord().Fields.Map()
if err := at.UpdateRecord(opts.AirtableTableName, record.ID, m, &record); err != nil {
record.Fields = issue.ToAirtableRecord().Fields
if err := table.Update(&record); err != nil {
logger().Warn("failed to update record, retrying without slices", zap.String("URL", issue.URL), zap.Error(err))
m["Labels"] = []string{}
m["Assignees"] = []string{}
m["Errors"] = err.Error()
if err := at.UpdateRecord(opts.AirtableTableName, record.ID, m, &record); err != nil {
record.Fields.Labels = []string{}
record.Fields.Assignees = []string{}
if typedErr, ok := err.(airtable.ErrClientRequest); ok {
record.Fields.Errors = typedErr.Err.Error()
} else {
record.Fields.Errors = err.Error()
}
if err := table.Update(&record); err != nil {
logger().Error("failed to update record without slices", zap.String("URL", issue.URL), zap.Error(err))
}
}
Expand Down Expand Up @@ -171,6 +179,7 @@ func (i Issue) ToAirtableRecord() airtableRecord {
RepoURL: i.RepoURL,
Body: i.Body,
State: i.State,
Errors: "",
},
}
}
Expand All @@ -187,9 +196,21 @@ type airtableIssue struct {
Type string
Labels []string
Assignees []string
Errors string
}

func (ai airtableIssue) Equals(other airtableIssue) bool {
sameSlice := func(a, b []string) bool {
if a == nil {
a = []string{}
}
if b == nil {
b = []string{}
}
sort.Strings(a)
sort.Strings(b)
return reflect.DeepEqual(a, b)
}
return ai.URL == other.URL &&
ai.Created.Truncate(time.Millisecond).UTC() == other.Created.Truncate(time.Millisecond).UTC() &&
ai.Updated.Truncate(time.Millisecond).UTC() == other.Updated.Truncate(time.Millisecond).UTC() &&
Expand All @@ -199,23 +220,7 @@ func (ai airtableIssue) Equals(other airtableIssue) bool {
ai.Body == other.Body &&
ai.RepoURL == other.RepoURL &&
ai.Type == other.Type &&
reflect.DeepEqual(ai.Labels, other.Labels) &&
reflect.DeepEqual(ai.Assignees, other.Assignees)
}

func (a airtableIssue) Map() map[string]interface{} {
return map[string]interface{}{
"URL": a.URL,
"Created": a.Created,
"Updated": a.Updated,
"Title": a.Title,
"Provider": a.Provider,
"State": a.State,
"Body": a.Body,
"RepoURL": a.RepoURL,
"Type": a.Type,
"Labels": a.Labels,
"Assignees": a.Assignees,
"Errors": "",
}
sameSlice(ai.Labels, other.Labels) &&
sameSlice(ai.Assignees, other.Assignees) &&
ai.Errors == other.Errors
}
6 changes: 4 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@ require (
cloud.google.com/go v0.29.0 // indirect
github.com/BurntSushi/toml v0.3.1 // indirect
github.com/awalterschulze/gographviz v0.0.0-20180927133620-e69668a01397
github.com/brianloveswords/airtable v0.0.0-20180329193050-a39294038dd9
github.com/denisenkom/go-mssqldb v0.0.0-20180901172138-1eb28afdf9b6 // indirect
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 // indirect
github.com/fabioberger/airtable-go v3.1.0+incompatible // indirect
github.com/go-chi/chi v3.3.3+incompatible
github.com/go-chi/docgen v1.0.2 // indirect
github.com/go-chi/docgen v1.0.2
github.com/go-chi/render v1.0.1
github.com/go-sql-driver/mysql v1.4.0 // indirect
github.com/google/go-cmp v0.2.0 // indirect
Expand All @@ -24,7 +24,9 @@ require (
github.com/spf13/cobra v0.0.3
github.com/spf13/pflag v1.0.3
github.com/spf13/viper v1.2.1
github.com/uber-go/atomic v1.3.2 // indirect
github.com/xanzy/go-gitlab v0.11.1
go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 // indirect
go.uber.org/zap v1.9.1
golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4 // indirect
golang.org/x/net v0.0.0-20181003013248-f5e5bdd77824 // indirect
Expand Down
8 changes: 6 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/awalterschulze/gographviz v0.0.0-20180927133620-e69668a01397 h1:SzTF/aqwdSBijZUpXvw1ditdkwfU+ONMyocC2CyRtrM=
github.com/awalterschulze/gographviz v0.0.0-20180927133620-e69668a01397/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs=
github.com/brianloveswords/airtable v0.0.0-20180329193050-a39294038dd9 h1:2oAZVcirE7xEkkA3lQqEB6ElATe5r60zwQoeTQ64WNI=
github.com/brianloveswords/airtable v0.0.0-20180329193050-a39294038dd9/go.mod h1:jXij3SzY3HghKUmTjQYGRIXyCRBkFK+0wztOaI6a9Bk=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/denisenkom/go-mssqldb v0.0.0-20180901172138-1eb28afdf9b6 h1:BZGp1dbKFjqlGmxEpwkDpCWNxVwEYnUPoncIzLiHlPo=
github.com/denisenkom/go-mssqldb v0.0.0-20180901172138-1eb28afdf9b6/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
github.com/fabioberger/airtable-go v3.1.0+incompatible h1:n5dw+HWBc+hytrVL75xe94EGt7FtNFGDII1tNoWTCAE=
github.com/fabioberger/airtable-go v3.1.0+incompatible/go.mod h1:EoKuSh7EefzhMCyVr6iXPlgFzDgHyZCZ3E5Sg8Cy9GM=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/go-chi/chi v3.3.3+incompatible h1:KHkmBEMNkwKuK4FdQL7N2wOeB9jnIx7jR5wsuSBEFI8=
Expand Down Expand Up @@ -69,12 +69,16 @@ github.com/spf13/viper v1.2.1 h1:bIcUwXqLseLF3BDAZduuNfekWG87ibtFxi59Bq+oI9M=
github.com/spf13/viper v1.2.1/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
github.com/xanzy/go-gitlab v0.11.1 h1:kgVxG9YFerbzMnLuwcFio5nwdv/5bCDHgm7WviT+EEE=
github.com/xanzy/go-gitlab v0.11.1/go.mod h1:CRKHkvFWNU6C3AEfqLWjnCNnAs4nj8Zk95rX2S3X6Mw=
go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 h1:d9qaMM+ODpCq+9We41//fu/sHsTnXcrqd1en3x+GKy4=
go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o=
go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4 h1:Vk3wNqEZwyGyei9yq5ekj7frek2u7HUfffJ1/opblzc=
Expand Down