Request final URI before wayback

wabarc · Feb 20, 2022 · 8f910c4 · 8f910c4
1 parent f9ee2d3
commit 8f910c4
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 18 deletions.
diff --git a/go.mod b/go.mod
@@ -30,7 +30,7 @@ require (
 	github.com/wabarc/archive.org v1.2.1-0.20210708220121-cb9b83ff9896
 	github.com/wabarc/go-anonfile v0.1.0
 	github.com/wabarc/go-catbox v0.1.0
-	github.com/wabarc/helper v0.0.0-20211225065210-3d35291efe54
+	github.com/wabarc/helper v0.0.0-20220220012023-0697c095968c
 	github.com/wabarc/ipfs-pinner v1.1.1-0.20220126131044-16299c0dd43d
 	github.com/wabarc/logger v0.0.0-20210730133522-86bd3f31e792
 	github.com/wabarc/playback v0.0.0-20210718054702-cab6c6004933

diff --git a/go.sum b/go.sum
@@ -693,8 +693,9 @@ github.com/wabarc/helper v0.0.0-20210407153720-1bfe98b427fe/go.mod h1:TuTZtoiOu9
 github.com/wabarc/helper v0.0.0-20210614160629-1a5ba5e551eb/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E=
 github.com/wabarc/helper v0.0.0-20210701193643-e0fe0a807cb9/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E=
 github.com/wabarc/helper v0.0.0-20210718171053-59c70d0b20c2/go.mod h1:uS6mimKlWkGvEZXkJ6JoW7LYnnB2JP6dLU9q7pgDaWQ=
-github.com/wabarc/helper v0.0.0-20211225065210-3d35291efe54 h1:g0hdl4gCuS61IGANiCa/vs5M4D2rM2p0f6VnCPbVQAk=
 github.com/wabarc/helper v0.0.0-20211225065210-3d35291efe54/go.mod h1:uS6mimKlWkGvEZXkJ6JoW7LYnnB2JP6dLU9q7pgDaWQ=
+github.com/wabarc/helper v0.0.0-20220220012023-0697c095968c h1:9YU9X7efQJcZxpDqpe0oHWspIgzHNHtlBprRHNz8cBQ=
+github.com/wabarc/helper v0.0.0-20220220012023-0697c095968c/go.mod h1:uS6mimKlWkGvEZXkJ6JoW7LYnnB2JP6dLU9q7pgDaWQ=
 github.com/wabarc/imgbb v1.0.0 h1:vNud9qY4dRGp1nikb+GHIrc/sE3WOdNq5gfT/vhWEUQ=
 github.com/wabarc/imgbb v1.0.0/go.mod h1:piz6QXkVfFpQbaQHx70/rPDwaICkW5Ew3lShTzbnVt0=
 github.com/wabarc/ipfs-pinner v1.1.1-0.20220126131044-16299c0dd43d h1:rSjZNYtr/5ep62nmcAF31v8zUGb0scD3bov/vAzYUpc=

diff --git a/service/twitter/twitter.go b/service/twitter/twitter.go
@@ -6,13 +6,10 @@ package twitter // import "github.com/wabarc/wayback/service/twitter"
 
 import (
 	"context"
-	"net/url"
 	"sync"
 	"time"
 
-	twitter "github.com/dghubble/go-twitter/twitter"
 	"github.com/dghubble/oauth1"
-	"github.com/wabarc/helper"
 	"github.com/wabarc/logger"
 	"github.com/wabarc/wayback"
 	"github.com/wabarc/wayback/config"
@@ -24,6 +21,8 @@ import (
 	"github.com/wabarc/wayback/service"
 	"github.com/wabarc/wayback/storage"
 	"github.com/wabarc/wayback/template/render"
+
+	twitter "github.com/dghubble/go-twitter/twitter"
 )
 
 // ErrServiceClosed is returned by the Service's Serve method after a call to Shutdown.
@@ -161,23 +160,13 @@ func (t *Twitter) process(event twitter.DirectMessageEvent) error {
 	}()
 
 	urls := service.MatchURL(text)
-	var realURLs []*url.URL
-	for _, uri := range urls {
-		u, err := url.Parse(helper.RealURI(uri.String()))
-		if err != nil {
-			continue
-		}
-		realURLs = append(realURLs, u)
-	}
-	logger.Debug("real urls: %v", realURLs)
-
-	if len(realURLs) == 0 {
+	if len(urls) == 0 {
 		logger.Warn("archives failure, URL no found.")
 		return errors.New("Twitter: URL no found")
 	}
 
 	var bundles reduxer.Bundles
-	cols, err := wayback.Wayback(context.TODO(), &bundles, realURLs...)
+	cols, err := wayback.Wayback(t.ctx, &bundles, urls...)
 	if err != nil {
 		logger.Error("archives failure, ", err)
 		return err

diff --git a/service/utils.go b/service/utils.go
@@ -31,7 +31,7 @@ func MatchURL(s string) (urls []*url.URL) {
 
 	for i := range matches {
 		u, _ := url.Parse(matches[i])
-		urls = append(urls, u)
+		urls = append(urls, helper.RealURI(u))
 	}
 
 	return removeDuplicates(urls)