Skip to content

Commit

Permalink
Improve reduxer calls
Browse files Browse the repository at this point in the history
  • Loading branch information
waybackarchiver committed Sep 28, 2022
1 parent f97f212 commit 14034f8
Show file tree
Hide file tree
Showing 4 changed files with 19 additions and 12 deletions.
7 changes: 6 additions & 1 deletion cmd/wayback/wayback.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"github.com/spf13/cobra"
"github.com/wabarc/wayback"
"github.com/wabarc/wayback/errors"
"github.com/wabarc/wayback/reduxer"
"golang.org/x/sync/errgroup"
)
Expand All @@ -31,7 +32,11 @@ func assets(art reduxer.Artifact) []reduxer.Asset {
func archive(cmd *cobra.Command, args []string) {
archiving := func(ctx context.Context, urls []*url.URL) error {
g, ctx := errgroup.WithContext(ctx)
cols, rdx, err := wayback.Wayback(ctx, urls...)
rdx, err := reduxer.Do(ctx, urls...)
if err != nil {
return errors.Wrap(err, "reduxer unexpected")
}
cols, err := wayback.Wayback(ctx, rdx, urls...)
if err != nil {
return err
}
Expand Down
3 changes: 2 additions & 1 deletion service/httpd/httpd_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/wabarc/wayback"
"github.com/wabarc/wayback/config"
"github.com/wabarc/wayback/pooling"
"github.com/wabarc/wayback/reduxer"
"github.com/wabarc/wayback/service"
)

Expand All @@ -33,7 +34,7 @@ func TestTransform(t *testing.T) {

text := "some text https://example.com"
urls := service.MatchURL(text)
col, _, _ := wayback.Wayback(context.Background(), urls...)
col, _ := wayback.Wayback(context.Background(), reduxer.BundleExample(), urls...)
collector := transform(col)

bytes, err := json.Marshal(collector)
Expand Down
10 changes: 8 additions & 2 deletions service/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,16 @@ func Wayback(ctx context.Context, urls []*url.URL, do doFunc) error {
var done = make(chan error, 1)
var cols []wayback.Collect
var rdx reduxer.Reduxer
var err error

go func() {
var err error
cols, rdx, err = wayback.Wayback(ctx, urls...)
rdx, err = reduxer.Do(ctx, urls...)
if err != nil {
done <- errors.Wrap(err, "reduxer unexpected")
return
}

cols, err = wayback.Wayback(ctx, rdx, urls...)
if err != nil {
done <- errors.Wrap(err, "wayback failed")
return
Expand Down
11 changes: 3 additions & 8 deletions wayback.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ func wayback(w Waybacker, r reduxer.Reduxer) string {
}

// Wayback returns URLs archived to the time capsules of given URLs.
func Wayback(ctx context.Context, urls ...*url.URL) ([]Collect, reduxer.Reduxer, error) {
func Wayback(ctx context.Context, rdx reduxer.Reduxer, urls ...*url.URL) ([]Collect, error) {
logger.Debug("start...")

if _, ok := ctx.Deadline(); !ok {
Expand All @@ -169,11 +169,6 @@ func Wayback(ctx context.Context, urls ...*url.URL) ([]Collect, reduxer.Reduxer,
defer cancel()
}

rdx, err := reduxer.Do(ctx, urls...)
if err != nil {
logger.Warn("reduxer error: %v", err)
}

mu := sync.Mutex{}
cols := []Collect{}
g, ctx := errgroup.WithContext(ctx)
Expand Down Expand Up @@ -214,10 +209,10 @@ func Wayback(ctx context.Context, urls ...*url.URL) ([]Collect, reduxer.Reduxer,
}

if len(cols) == 0 {
return cols, rdx, errors.New("archiving failed: no cols")
return cols, errors.New("archiving failed: no cols")
}

return cols, rdx, nil
return cols, nil
}

// Playback returns URLs archived from the time capsules.
Expand Down

0 comments on commit 14034f8

Please sign in to comment.