Auto fallback to Google cache if URI is missing

wabarc · May 12, 2021 · 071b3fc · 071b3fc
1 parent 3d82ff1
commit 071b3fc
Show file tree

Hide file tree

Showing 15 changed files with 24 additions and 18 deletions.
diff --git a/go.mod b/go.mod
@@ -27,7 +27,7 @@ require (
 	github.com/thoj/go-ircevent v0.0.0-20190807115034-8e7ce4b5a1eb
 	github.com/wabarc/archive.is v1.2.4-0.20210505135936-034a6c963560
 	github.com/wabarc/archive.org v1.1.2
-	github.com/wabarc/helper v0.0.0-20210420080053-35532f1f6a79
+	github.com/wabarc/helper v0.0.0-20210511232523-02eb9f3480da
 	github.com/wabarc/logger v0.0.0-20210417045349-d0d82e8e99ee
 	github.com/wabarc/playback v0.0.0-20210418074547-4bf9d94a794d
 	github.com/wabarc/screenshot v1.1.1 // indirect

diff --git a/go.sum b/go.sum
@@ -508,8 +508,9 @@ github.com/wabarc/archive.org v1.1.2 h1:XCFprQh5rt8bLBOWLEBO108LqVBZ34Y9ItJR3JfW
 github.com/wabarc/archive.org v1.1.2/go.mod h1:f+VqacB+XUJKQ7MmGgZ9CDB1x1ypwkMKtPjEfRD/qUs=
 github.com/wabarc/helper v0.0.0-20210127120855-10af37cc2616/go.mod h1:N9P4r7Rn46p4nkWtXV6ztN3p5ACVnp++bgfwjTqSxQ8=
 github.com/wabarc/helper v0.0.0-20210407153720-1bfe98b427fe/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E=
-github.com/wabarc/helper v0.0.0-20210420080053-35532f1f6a79 h1:J9YG66PKaYbCpOoE9nh5aNHZ8uFXoVGT29WPR2iDjTs=
 github.com/wabarc/helper v0.0.0-20210420080053-35532f1f6a79/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E=
+github.com/wabarc/helper v0.0.0-20210511232523-02eb9f3480da h1:ieCdQ0CRkzRtCSoLarNIo35M48vNTtkTwVNPw+rM5Qk=
+github.com/wabarc/helper v0.0.0-20210511232523-02eb9f3480da/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E=
 github.com/wabarc/imgbb v1.0.0 h1:vNud9qY4dRGp1nikb+GHIrc/sE3WOdNq5gfT/vhWEUQ=
 github.com/wabarc/imgbb v1.0.0/go.mod h1:piz6QXkVfFpQbaQHx70/rPDwaICkW5Ew3lShTzbnVt0=
 github.com/wabarc/ipfs-pinner v1.0.1 h1:PaWBDn1X556IyiOVd143XO9PVRjbohppA+JMJQwNBHI=

diff --git a/publish/github.go b/publish/github.go
@@ -76,8 +76,8 @@ func (gh *GitHub) Render(vars []*wayback.Collect) string {
 
 	const tmpl = `{{range $ := .}}**[{{ $.Arc }}]({{ $.Ext }})**:
 {{ range $src, $dst := $.Dst -}}
-> origin: [{{ $src | unescape}}]({{ $src }})
-> archived: {{ if $dst | isURL }}[{{ $dst | unescape}}]({{ $dst }}){{ else }}{{ $dst }}{{ end }}
+> origin: [{{ $src | unescape | revert }}]({{ $src | revert }})
+> archived: {{ if $dst | isURL }}[{{ $dst | unescape }}]({{ $dst }}){{ else }}{{ $dst }}{{ end }}
 {{end}}
 {{end}}`
 

diff --git a/publish/github_test.go b/publish/github_test.go
@@ -64,8 +64,8 @@ func TestRenderForGitHubFlawed(t *testing.T) {
 > archived: Archive failed.
 
 **[Telegraph](https://telegra.ph/)**:
-> origin: [https://example.com/](https://example.com/)
-> archived: Screenshots failed.
+> origin: [https://example.com/404](https://example.com/404)
+> archived: [https://web.archive.org/*/https://webcache.googleusercontent.com/search?q=cache:https://example.com/404](https://web.archive.org/*/https://webcache.googleusercontent.com/search?q=cache:https://example.com/404)
 `
 
 	gh := NewGitHub(&http.Client{})

diff --git a/publish/matrix.go b/publish/matrix.go
@@ -79,11 +79,11 @@ func (m *Matrix) Render(vars []*wayback.Collect) string {
 
 	const tmpl = `{{range $ := .}}<b><a href='{{ $.Ext }}'>{{ $.Arc }}</a></b>:<br>
 {{ range $src, $dst := $.Dst -}}
-• <a href="{{ $src }}">origin</a> - {{ $dst }}<br>
+• <a href="{{ $src | revert }}">origin</a> - {{ $dst }}<br>
 {{end}}<br>
 {{end}}`
 
-	tpl, err := template.New("message").Parse(tmpl)
+	tpl, err := template.New("message").Funcs(funcMap()).Parse(tmpl)
 	if err != nil {
 		logger.Debug("[publish] parse Mastodon template failed, %v", err)
 		return ""

diff --git a/publish/publish.go b/publish/publish.go
@@ -7,6 +7,7 @@ package publish // import "github.com/wabarc/wayback/publish"
 import (
 	"context"
 	"net/url"
+	"strings"
 	"text/template"
 
 	"github.com/dghubble/go-twitter/twitter"
@@ -121,6 +122,7 @@ func To(ctx context.Context, col []*wayback.Collect, args ...string) {
 }
 
 func funcMap() template.FuncMap {
+	cache := "https://webcache.googleusercontent.com/search?q=cache:"
 	return template.FuncMap{
 		"unescape": func(link string) string {
 			unescaped, err := url.QueryUnescape(link)
@@ -130,5 +132,8 @@ func funcMap() template.FuncMap {
 			return unescaped
 		},
 		"isURL": helper.IsURL,
+		"revert": func(link string) string {
+			return strings.Replace(link, cache, "", 1)
+		},
 	}
 }
diff --git a/publish/publish_test.go b/publish/publish_test.go
@@ -77,7 +77,7 @@ var flawed = []*wayback.Collect{
 	{
 		Arc: config.SlotName(config.SLOT_PH),
 		Dst: map[string]string{
-			"https://example.com/": "Screenshots failed.",
+			"https://example.com/404": "https://web.archive.org/*/https://webcache.googleusercontent.com/search?q=cache:https://example.com/404",
 		},
 		Ext: config.SlotExtra(config.SLOT_PH),
 	},

diff --git a/publish/telegram.go b/publish/telegram.go
@@ -78,7 +78,7 @@ func (t *Telegram) Render(vars []*wayback.Collect) string {
 
 	const tmpl = `{{range $ := .}}<b><a href='{{ $.Ext }}'>{{ $.Arc }}</a></b>:
 {{ range $src, $dst := $.Dst -}}
-• <a href="{{ $src }}">origin</a> - {{ if $dst | isURL }}<a href="{{ $dst }}">{{ $dst }}</a>{{ else }}{{ $dst }}{{ end }}
+• <a href="{{ $src | revert }}">origin</a> - {{ if $dst | isURL }}<a href="{{ $dst }}">{{ $dst }}</a>{{ else }}{{ $dst }}{{ end }}
 {{end}}
 {{end}}`
 

diff --git a/publish/telegram_test.go b/publish/telegram_test.go
@@ -85,7 +85,7 @@ func TestRenderForTelegramFlawed(t *testing.T) {
 • <a href="https://example.com/">origin</a> - Archive failed.
 
 <b><a href='https://telegra.ph/'>Telegraph</a></b>:
-• <a href="https://example.com/">origin</a> - Screenshots failed.
+• <a href="https://example.com/404">origin</a> - <a href="https://web.archive.org/*/https://webcache.googleusercontent.com/search?q=cache:https://example.com/404">https://web.archive.org/*/https://webcache.googleusercontent.com/search?q=cache:https://example.com/404</a>
 `
 	tel := &Telegram{}
 	got := tel.Render(flawed)

diff --git a/service/anonymity/web.go b/service/anonymity/web.go
@@ -198,7 +198,7 @@ func (web *web) process(w http.ResponseWriter, r *http.Request) {
 	}
 	logger.Debug("[web] text: %s", text)
 
-	urls := helper.MatchURL(text)
+	urls := helper.MatchURLFallback(text)
 	if len(urls) == 0 {
 		logger.Info("[web] url no found.")
 	}

diff --git a/service/mastodon/mastodon.go b/service/mastodon/mastodon.go
@@ -148,7 +148,7 @@ func (m *Mastodon) process(ctx context.Context, conv *mastodon.Conversation) err
 		delete(m.archiving, conv.ID)
 	}()
 
-	urls := helper.MatchURL(text)
+	urls := helper.MatchURLFallback(text)
 	pub := publish.NewMastodon(m.client)
 	if len(urls) == 0 {
 		logger.Info("[mastodon] archives failure, URL no found.")

diff --git a/service/matrix/matrix.go b/service/matrix/matrix.go
@@ -136,7 +136,7 @@ func (m *Matrix) process(ctx context.Context, ev *event.Event) error {
 	text := ev.Content.AsMessage().Body
 	logger.Debug("[matrix] from: %s message: %s", ev.Sender, text)
 
-	urls := helper.MatchURL(text)
+	urls := helper.MatchURLFallback(text)
 	if len(urls) == 0 {
 		logger.Info("[matrix] archives failure, URL no found.")
 		// Redact message

diff --git a/service/relaychat/relaychat.go b/service/relaychat/relaychat.go
@@ -98,7 +98,7 @@ func (i *IRC) process(ctx context.Context, ev *irc.Event) error {
 	text := ev.MessageWithoutFormat()
 	logger.Debug("[irc] from: %s message: %s", ev.Nick, text)
 
-	urls := helper.MatchURL(text)
+	urls := helper.MatchURLFallback(text)
 	if len(urls) == 0 {
 		logger.Info("[irc] archives failure, URL no found.")
 		return errors.New("IRC: URL no found")

diff --git a/service/telegram/telegram.go b/service/telegram/telegram.go
@@ -104,7 +104,7 @@ func (t *Telegram) Serve(ctx context.Context) (err error) {
 				return false
 			}
 
-			go t.archive(ctx, callback.Message, helper.MatchURL(string(data)))
+			go t.archive(ctx, callback.Message, helper.MatchURLFallback(string(data)))
 		case update.Message != nil:
 			logger.Debug("[telegram] message: %#v", update.Message)
 
@@ -137,7 +137,7 @@ func (t *Telegram) process(ctx context.Context, update *telegram.Update) error {
 	if message.IsForwarded() && content == "" {
 		return nil
 	}
-	urls := helper.MatchURL(content)
+	urls := helper.MatchURLFallback(content)
 
 	// Set command as playback if receive a playback command without URLs, and
 	// required user reply a message with URLs.

diff --git a/service/twitter/twitter.go b/service/twitter/twitter.go
@@ -134,7 +134,7 @@ func (t *Twitter) process(ctx context.Context, event twitter.DirectMessageEvent)
 		delete(t.archiving, event.ID)
 	}()
 
-	urls := helper.MatchURL(text)
+	urls := helper.MatchURLFallback(text)
 	pub := publish.NewTwitter(t.client)
 	var realURLs []string
 	for _, url := range urls {