Store playback data locally

.gitignore

@@ -22,3 +22,4 @@
 # Ignore Tor embedded process data directory
 **/tor-data*
 **/data-dir*
+wayback.db

Makefile

@@ -206,6 +206,7 @@ clean:
 	rm -rf data-dir*
 	rm -rf coverage*
 	rm -rf *.out
+	rm -rf wayback.db
 
 fmt:
 	@echo "-> Running go fmt"

cmd/wayback/serve.go

@@ -16,16 +16,23 @@ import (
 	"github.com/wabarc/wayback/service/relaychat"
 	"github.com/wabarc/wayback/service/telegram"
 	"github.com/wabarc/wayback/service/twitter"
+	"github.com/wabarc/wayback/storage"
 )
 
 type service struct {
 	errCh chan error
 }
 
 func serve(_ *cobra.Command, args []string) {
+	store, err := storage.Open("")
+	if err != nil {
+		logger.Fatal("open storage failed: %v", err)
+	}
+	defer store.Close()
+
 	ctx, cancel := context.WithCancel(context.Background())
 	srv := &service{}
-	ran := srv.run(ctx)
+	ran := srv.run(ctx, store)
 
 	go srv.stop(cancel)
 	defer close(srv.errCh)
@@ -39,37 +46,37 @@ func serve(_ *cobra.Command, args []string) {
 	}
 }
 
-func (srv *service) run(ctx context.Context) *service {
+func (srv *service) run(ctx context.Context, store *storage.Storage) *service {
 	srv.errCh = make(chan error, len(daemon))
 	for _, s := range daemon {
 		switch s {
 		case "irc":
-			irc := relaychat.New()
+			irc := relaychat.New(store)
 			go func(errCh chan error) {
 				errCh <- irc.Serve(ctx)
 			}(srv.errCh)
 		case "mastodon", "mstdn":
-			mastodon := mastodon.New()
+			mastodon := mastodon.New(store)
 			go func(errCh chan error) {
 				errCh <- mastodon.Serve(ctx)
 			}(srv.errCh)
 		case "telegram":
-			telegram := telegram.New()
+			telegram := telegram.New(store)
 			go func(errCh chan error) {
 				errCh <- telegram.Serve(ctx)
 			}(srv.errCh)
 		case "twitter":
-			twitter := twitter.New()
+			twitter := twitter.New(store)
 			go func(errCh chan error) {
 				errCh <- twitter.Serve(ctx)
 			}(srv.errCh)
 		case "matrix":
-			matrix := matrix.New()
+			matrix := matrix.New(store)
 			go func(errCh chan error) {
 				errCh <- matrix.Serve(ctx)
 			}(srv.errCh)
 		case "web":
-			tor := anonymity.New()
+			tor := anonymity.New(store)
 			go func(errCh chan error) {
 				errCh <- tor.Serve(ctx)
 			}(srv.errCh)

config/options.go

@@ -55,6 +55,7 @@ const (
 
 	defChromeRemoteAddr    = ""
 	defEnabledChromeRemote = false
+	defBoltFilename        = "wayback.db"
 )
 
 var (
@@ -453,3 +454,8 @@ func (o *Options) EnabledChromeRemote() bool {
 func (o *Options) ChromeRemoteAddr() string {
 	return o.chromeRemoteAddr
 }
+
+// BoltFilename returns filename of bolt database
+func (o *Options) BoltFilename() string {
+	return defBoltFilename
+}

entity/doc.go

@@ -0,0 +1,9 @@
+// Copyright 2021 Wayback Archiver. All rights reserved.
+// Use of this source code is governed by the GNU GPL v3
+// license that can be found in the LICENSE file.
+
+/*
+Package entity contains all data structures used by the application.
+*/
+
+package entity // import "github.com/wabarc/entity"

entity/entity.go

@@ -0,0 +1,5 @@
+// Copyright 2021 Wayback Archiver. All rights reserved.
+// Use of this source code is governed by the GNU GPL v3
+// license that can be found in the LICENSE file.
+
+package entity // import "github.com/wabarc/entity"

entity/playback.go

@@ -0,0 +1,13 @@
+// Copyright 2021 Wayback Archiver. All rights reserved.
+// Use of this source code is governed by the GNU GPL v3
+// license that can be found in the LICENSE file.
+
+package entity // import "github.com/wabarc/entity"
+
+const EntityPlayback = "playback"
+
+// Playback represents a Playback in the application.
+type Playback struct {
+	ID     int
+	Source string
+}

go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/wabarc/telegra.ph v0.0.0-20210505140622-220623b0de58
 	github.com/wabarc/wbipfs v0.1.3
 	github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b // indirect
+	go.etcd.io/bbolt v1.3.5
 	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/crypto v0.0.0-20210505212654-3497b51f5e64 // indirect
 	golang.org/x/net v0.0.0-20210505214959-0714010a04ed

go.sum

@@ -537,6 +537,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
@@ -662,6 +664,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

service/anonymity/tor.go

@@ -20,14 +20,19 @@ import (
 	"github.com/wabarc/logger"
 	"github.com/wabarc/wayback/config"
 	"github.com/wabarc/wayback/errors"
+	"github.com/wabarc/wayback/storage"
 )
 
 type Tor struct {
+	store *storage.Storage
 }
 
 // New tor struct.
-func New() *Tor {
-	return &Tor{}
+func New(store *storage.Storage) *Tor {
+	if store == nil {
+		logger.Fatal("[web] must initialize storage")
+	}
+	return &Tor{store: store}
 }
 
 // Serve accepts incoming HTTP requests over Tor network, or open

service/mastodon/mastodon.go

@@ -19,22 +19,27 @@ import (
 	"github.com/wabarc/wayback/errors"
 	"github.com/wabarc/wayback/metrics"
 	"github.com/wabarc/wayback/publish"
+	"github.com/wabarc/wayback/storage"
 	"golang.org/x/net/html"
 )
 
 type Mastodon struct {
 	sync.RWMutex
 
 	client *mastodon.Client
+	store  *storage.Storage
 
 	archiving map[mastodon.ID]bool
 }
 
 // New mastodon struct.
-func New() *Mastodon {
+func New(store *storage.Storage) *Mastodon {
 	if !config.Opts.PublishToMastodon() {
 		logger.Fatal("[mastodon] missing required environment variable")
 	}
+	if store == nil {
+		logger.Fatal("[mastodon] must initialize storage")
+	}
 
 	client := mastodon.NewClient(&mastodon.Config{
 		Server:       config.Opts.MastodonServer(),
@@ -44,6 +49,7 @@ func New() *Mastodon {
 	})
 	return &Mastodon{
 		client: client,
+		store:  store,
 	}
 }
 

service/mastodon/mastodon_test.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/wabarc/helper"
 	"github.com/wabarc/wayback/config"
+	"github.com/wabarc/wayback/storage"
 )
 
 func TestProcess(t *testing.T) {
@@ -45,7 +46,7 @@ func TestProcess(t *testing.T) {
 		t.Fatalf("Parse enviroment variables or flags failed, error: %v", err)
 	}
 
-	m := New()
+	m := New(&storage.Storage{})
 	ctx := context.Background()
 	convs, err := m.client.GetConversations(ctx, nil)
 	if err != nil {

service/matrix/matrix.go

@@ -15,6 +15,7 @@ import (
 	"github.com/wabarc/wayback/errors"
 	"github.com/wabarc/wayback/metrics"
 	"github.com/wabarc/wayback/publish"
+	"github.com/wabarc/wayback/storage"
 	matrix "maunium.net/go/mautrix"
 	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/id"
@@ -24,12 +25,16 @@ type Matrix struct {
 	sync.RWMutex
 
 	client *matrix.Client
+	store  *storage.Storage
 }
 
 // New Matrix struct.
-func New() *Matrix {
+func New(store *storage.Storage) *Matrix {
 	if config.Opts.MatrixUserID() == "" || config.Opts.MatrixPassword() == "" || config.Opts.MatrixHomeserver() == "" {
-		logger.Fatal("Missing required environment variable")
+		logger.Fatal("[matrix] missing required environment variable")
+	}
+	if store == nil {
+		logger.Fatal("[matrix] must initialize storage")
 	}
 
 	client, err := matrix.NewClient(config.Opts.MatrixHomeserver(), "", "")
@@ -48,6 +53,7 @@ func New() *Matrix {
 
 	return &Matrix{
 		client: client,
+		store:  store,
 	}
 }
 

service/matrix/matrix_test.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/wabarc/helper"
 	"github.com/wabarc/wayback/config"
+	"github.com/wabarc/wayback/storage"
 	matrix "maunium.net/go/mautrix"
 	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/id"
@@ -89,7 +90,7 @@ func senderClient(t *testing.T) *Matrix {
 	if config.Opts, err = parser.ParseEnvironmentVariables(); err != nil {
 		t.Fatalf("Parse enviroment variables or flags failed, error: %v", err)
 	}
-	return New()
+	return New(&storage.Storage{})
 }
 
 func recverClient(t *testing.T) *Matrix {
@@ -99,7 +100,7 @@ func recverClient(t *testing.T) *Matrix {
 	if config.Opts, err = parser.ParseEnvironmentVariables(); err != nil {
 		t.Fatalf("Parse enviroment variables or flags failed, error: %v", err)
 	}
-	return New()
+	return New(&storage.Storage{})
 }
 
 func TestProcess(t *testing.T) {

service/relaychat/relaychat.go

@@ -17,18 +17,23 @@ import (
 	"github.com/wabarc/wayback/errors"
 	"github.com/wabarc/wayback/metrics"
 	"github.com/wabarc/wayback/publish"
+	"github.com/wabarc/wayback/storage"
 )
 
 type IRC struct {
 	sync.RWMutex
 
-	conn *irc.Connection
+	conn  *irc.Connection
+	store *storage.Storage
 }
 
 // New IRC struct.
-func New() *IRC {
+func New(store *storage.Storage) *IRC {
 	if config.Opts.IRCNick() == "" {
-		logger.Fatal("Missing required environment variable")
+		logger.Fatal("[irc] missing required environment variable")
+	}
+	if store == nil {
+		logger.Fatal("[irc] must initialize storage")
 	}
 
 	// TODO: support SASL authenticate
@@ -40,7 +45,8 @@ func New() *IRC {
 	conn.TLSConfig = &tls.Config{InsecureSkipVerify: false}
 
 	return &IRC{
-		conn: conn,
+		conn:  conn,
+		store: store,
 	}
 }
 

service/relaychat/relaychat_test.go

@@ -17,6 +17,7 @@ import (
 	irc "github.com/thoj/go-ircevent"
 	"github.com/wabarc/helper"
 	"github.com/wabarc/wayback/config"
+	"github.com/wabarc/wayback/storage"
 )
 
 var (
@@ -86,7 +87,7 @@ func TestProcess(t *testing.T) {
 	recvConn.AddCallback("PRIVMSG", func(ev *irc.Event) {
 		if ev.Nick == sender {
 			done <- true
-			i := New()
+			i := New(&storage.Storage{})
 			// Replace IRC connection to receive connection
 			i.conn = recvConn
 			if err = i.process(context.Background(), ev); err != nil {
@@ -178,7 +179,7 @@ func TestToIRCChannel(t *testing.T) {
 	recvConn.AddCallback("PRIVMSG", func(ev *irc.Event) {
 		if ev.Nick == sender {
 			done <- true
-			i := New()
+			i := New(&storage.Storage{})
 			// Replace IRC connection to receive connection
 			i.conn = recvConn
 			if err = i.process(context.Background(), ev); err != nil {