Skip to content

Commit

Permalink
Add PWA support
Browse files Browse the repository at this point in the history
Required Go version 1.16
  • Loading branch information
web-flow committed Apr 23, 2021
1 parent 0594970 commit 09e3e04
Show file tree
Hide file tree
Showing 21 changed files with 608 additions and 117 deletions.
5 changes: 3 additions & 2 deletions .github/workflows/testing.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
fail-fast: false
matrix:
os: [ ubuntu-latest, macos-latest, windows-latest ]
go: [ "1.13", "1.14", "1.15", "1.16" ]
#go: [ "1.13", "1.14", "1.15", "1.16" ]
env:
SENDER_UID: ${{ secrets.MATRIX_SENDER_UID }}
SENDER_PWD: ${{ secrets.MATRIX_SENDER_PWD }}
Expand All @@ -39,7 +39,8 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go }}
#go-version: ${{ matrix.go }}
go-version: "1.16"

- name: Set up IPFS
uses: ibnesayeed/setup-ipfs@master
Expand Down
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/wabarc/wayback

go 1.13
go 1.16

require (
github.com/btcsuite/btcd v0.21.0-beta // indirect
Expand All @@ -11,6 +11,7 @@ require (
github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.0.0-rc1.0.20210311030851-d0e1dfd8c604
github.com/gogo/protobuf v1.3.2 // indirect
github.com/google/go-github/v33 v33.0.0
github.com/gorilla/mux v1.8.0
github.com/klauspost/cpuid/v2 v2.0.6 // indirect
// github.com/ipsn/go-libtor v1.0.329
github.com/libp2p/go-libp2p-core v0.8.5 // indirect
Expand Down
4 changes: 1 addition & 3 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
Expand Down Expand Up @@ -212,10 +213,8 @@ github.com/klauspost/cpuid/v2 v2.0.6 h1:dQ5ueTiftKxp0gyjKSx5+8BtPWkyQbd95m8Gys/R
github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs=
Expand Down Expand Up @@ -590,7 +589,6 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
Expand Down
36 changes: 13 additions & 23 deletions service/anonymity/tor.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,53 +88,47 @@ func (t *Tor) Serve(ctx context.Context) error {
logger.Info("[web] please open a Tor capable browser and navigate to http://%v.onion", onion.ID)

go func() {
http.HandleFunc("/", home)
http.HandleFunc("/w", func(w http.ResponseWriter, r *http.Request) { t.process(w, r, ctx) })
http.Serve(onion, nil)
http.Serve(onion, newWeb().handle())
}()

stop := make(chan os.Signal)
signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
signal.Notify(stop, os.Interrupt, syscall.SIGTERM, syscall.SIGINT)
<-stop

return errors.New("done")
}

func home(w http.ResponseWriter, r *http.Request) {
tmpl := template.Collector{}
if html, ok := tmpl.Render(); ok {
w.Write(html)
} else {
logger.Error("[web] render template for home request failed")
http.Error(w, "Internal Server Error", 500)
}
}

func (t *Tor) process(w http.ResponseWriter, r *http.Request, ctx context.Context) {
func (web *web) process(w http.ResponseWriter, r *http.Request) {
logger.Debug("[web] process request start...")
if r.Method != http.MethodPost {
logger.Info("[web] request method no specific.")
http.Redirect(w, r, "/", 405)
http.Redirect(w, r, "/", http.StatusNotModified)
return
}

if err := r.ParseForm(); err != nil {
logger.Error("[web] parse form error, %v", err)
http.Redirect(w, r, "/", 400)
http.Redirect(w, r, "/", http.StatusNotModified)
return
}

text := r.PostFormValue("text")
if len(strings.TrimSpace(text)) == 0 {
logger.Info("[web] post form value empty.")
http.Redirect(w, r, "/", 411)
http.Redirect(w, r, "/", http.StatusFound)
return
}
logger.Debug("[web] text: %s", text)

urls := helper.MatchURL(text)
if len(urls) == 0 {
logger.Info("[web] url no found.")
http.Redirect(w, r, "/", http.StatusFound)
return
}
col, _ := wayback.Wayback(urls)
collector := transform(col)
ctx := context.Background()
switch r.PostFormValue("data-type") {
case "json":
w.Header().Set("Content-Type", "application/json")
Expand All @@ -145,19 +139,15 @@ func (t *Tor) process(w http.ResponseWriter, r *http.Request, ctx context.Contex
go publish.To(ctx, col, "web")
w.Write(data)
}

return
default:
w.Header().Set("Content-Type", "text/html; charset=utf-8")

if html, ok := collector.Render(); ok {
if html, ok := web.template.Render("layout", collector); ok {
go publish.To(ctx, col, "web")
w.Write(html)
} else {
logger.Error("[web] render template for response failed")
}

return
}
}

Expand Down
160 changes: 160 additions & 0 deletions service/anonymity/web.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
// Copyright 2020 Wayback Archiver. All rights reserved.
// Use of this source code is governed by the GNU GPL v3
// license that can be found in the LICENSE file.

package anonymity // import "github.com/wabarc/wayback/service/anonymity"

import (
"encoding/json"
"net/http"

"github.com/gorilla/mux"
"github.com/wabarc/logger"
"github.com/wabarc/wayback/template"
)

type web struct {
router *mux.Router
template *template.Template
}

func newWeb() *web {
router := mux.NewRouter()
web := &web{
router: router,
template: template.New(router),
}
if err := web.template.ParseTemplates(); err != nil {
logger.Fatal("[web] unable to parse templates: %v", err)
}
if err := template.GenerateJavascriptBundles(); err != nil {
logger.Fatal("[web] unable to generate JavaScript bundles: %v", err)
}
return web
}

func (web *web) handle() http.Handler {
web.router.HandleFunc("/", web.home)
web.router.HandleFunc("/{name}.js", web.showJavascript).Name("javascript").Methods(http.MethodGet)
web.router.HandleFunc("/favicon.ico", web.showFavicon).Name("favicon").Methods(http.MethodGet)
web.router.HandleFunc("/icon/{filename}", web.showAppIcon).Name("icon").Methods(http.MethodGet)
web.router.HandleFunc("/manifest.json", web.showWebManifest).Name("manifest").Methods(http.MethodGet)
web.router.HandleFunc("/offline.html", web.showOfflinePage).Methods(http.MethodGet)

web.router.HandleFunc("/w", func(w http.ResponseWriter, r *http.Request) { web.process(w, r) }).Methods(http.MethodPost)

web.router.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
w.Write([]byte("User-agent: *\nDisallow: /"))
})

return web.router
}

func (web *web) home(w http.ResponseWriter, r *http.Request) {
logger.Debug("[web] access home")
w.Header().Set("Cache-Control", "max-age=2592000")
if html, ok := web.template.Render("layout", nil); ok {
w.Write(html)
} else {
logger.Error("[web] render template for home request failed")
http.Error(w, "Internal Server Error", 500)
}
}

func (web *web) showOfflinePage(w http.ResponseWriter, r *http.Request) {
logger.Debug("[web] access offline page")
// if f, ok := w.(http.Flusher); ok {
// f.Flush()
// }
if html, ok := web.template.Render("offline", nil); ok {
w.Write(html)
} else {
logger.Error("[web] render template for offline request failed")
http.Error(w, "Internal Server Error", 500)
}
}

func (web *web) showWebManifest(w http.ResponseWriter, r *http.Request) {
logger.Debug("[web] access manifest")
type webManifestIcon struct {
Source string `json:"src"`
Sizes string `json:"sizes"`
Type string `json:"type"`
}

type webManifest struct {
Name string `json:"name"`
Description string `json:"description"`
ShortName string `json:"short_name"`
StartURL string `json:"start_url"`
Icons []webManifestIcon `json:"icons"`
Display string `json:"display"`
ThemeColor string `json:"theme_color"`
}

manifest := &webManifest{
Name: "Wayback Archiver",
ShortName: "Wayback",
Description: "A toolkit for snapshot webpages",
Display: "standalone",
ThemeColor: "#f7f7f7",
StartURL: "/",
Icons: []webManifestIcon{
{Source: template.Path(web.router, "icon", "filename", "icon-120.png"), Sizes: "120x120", Type: "image/png"},
{Source: template.Path(web.router, "icon", "filename", "icon-192.png"), Sizes: "192x192", Type: "image/png"},
{Source: template.Path(web.router, "icon", "filename", "icon-512.png"), Sizes: "512x512", Type: "image/png"},
},
}

w.Header().Set("Cache-Control", "max-age=259200")
w.Header().Set("Content-Type", "application/manifest+json")
if data, err := json.Marshal(manifest); err != nil {
logger.Error("[web] encode for response failed, %v", err)
} else {
w.Write(data)
}
}

func (web *web) showFavicon(w http.ResponseWriter, r *http.Request) {
logger.Debug("[web] access favicon")

blob, err := template.LoadImageFile("favicon.ico")
if err != nil {
return
}
w.Header().Set("Content-Type", "image/x-icon")
w.Write(blob)
}

func (web *web) showAppIcon(w http.ResponseWriter, r *http.Request) {
logger.Debug("[web] access application icon")

filename := routeParam(r, "filename")
blob, err := template.LoadImageFile(filename)
if err != nil {
return
}
w.Header().Set("Cache-Control", "max-age=2592000")
w.Header().Set("Content-Type", "image/png")
w.Write(blob)
}

func (web *web) showJavascript(w http.ResponseWriter, r *http.Request) {
filename := routeParam(r, "name")
logger.Debug("[web] access javascript %s", filename)
_, found := template.JavascriptBundleChecksums[filename]
if !found {
return
}
contents := template.JavascriptBundles[filename]

w.Header().Set("Cache-Control", "max-age=2592000")
w.Header().Set("Content-Type", "text/javascript; charset=utf-8")
w.Write(contents)
}

func routeParam(r *http.Request, param string) string {
vars := mux.Vars(r)
return vars[param]
}
13 changes: 6 additions & 7 deletions service/anonymity/web_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
package anonymity // import "github.com/wabarc/wayback/service/anonymity"

import (
"context"
"encoding/json"
"io/ioutil"
"net/http"
Expand Down Expand Up @@ -43,11 +42,10 @@ func TestTransform(t *testing.T) {
}

func TestProcessRespStatus(t *testing.T) {
tor := New()
httpClient, mux, server := helper.MockServer()
defer server.Close()
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
tor.process(w, r, context.Background())
newWeb().process(w, r)
})

var tests = []struct {
Expand All @@ -57,12 +55,12 @@ func TestProcessRespStatus(t *testing.T) {
}{
{
method: http.MethodGet,
status: http.StatusMethodNotAllowed,
status: http.StatusNotModified,
data: `{"text":"", "data-type":"json"}`,
},
{
method: http.MethodPost,
status: http.StatusLengthRequired,
status: http.StatusNotModified,
data: `{"text":"foo bar", "data-type":"json"}`,
},
}
Expand Down Expand Up @@ -94,11 +92,12 @@ func TestProcessContentType(t *testing.T) {
t.Fatalf("Parse enviroment variables or flags failed, error: %v", err)
}

tor := New()
web := newWeb()
web.handle()
httpClient, mux, server := helper.MockServer()
defer server.Close()
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
tor.process(w, r, context.Background())
web.process(w, r)
})

var tests = []struct {
Expand Down
Binary file added template/assets/image/favicon-16.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added template/assets/image/favicon-32.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added template/assets/image/favicon.ico
Binary file not shown.
Binary file added template/assets/image/icon-120.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added template/assets/image/icon-128.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added template/assets/image/icon-152.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added template/assets/image/icon-167.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added template/assets/image/icon-180.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added template/assets/image/icon-192.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added template/assets/image/icon-512.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading

0 comments on commit 09e3e04

Please sign in to comment.