Skip to content

Commit

Permalink
Merge pull request #328 from wabarc/improvement
Browse files Browse the repository at this point in the history
  • Loading branch information
@waybackarchiver
waybackarchiver authored Mar 7, 2023
2 parents 1ffabac + 7209038 commit 08cb62c
Show file tree
Hide file tree
Showing 91 changed files with 2,367 additions and 1,214 deletions.
15 changes: 8 additions & 7 deletions .github/workflows/testing.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ jobs:
uses: ibnesayeed/setup-ipfs@3e5452e358dfa43a0fb92496e96c82cf756be94a
with:
run_daemon: true
ipfs_version: ${{ vars.IPFS_VERSION }}

- name: Set up Tor
uses: tor-actions/setup-tor@462b4eef22eecc86cc4abdedafc95a38667e2557
Expand All @@ -81,9 +82,9 @@ jobs:
if: matrix.os == 'ubuntu-latest'
uses: moy2010/meilisearch-github-action@fcc5ef714af0596633665032d459bfb279d3c730 # 0.1.4
with:
meilisearch-version: v0.28.0
meilisearch-port: 7700
meilisearch-api-key: foobar
meilisearch-version: ${{ vars.MEILISEARCH_VERSION }}
meilisearch-port: ${{ vars.MEILISEARCH_PORT }}
meilisearch-api-key: ${{ vars.MEILISEARCH_APIKEY }}

- name: Set up Chocolatey
if: matrix.os == 'windows-latest'
Expand Down Expand Up @@ -141,10 +142,10 @@ jobs:
shell: bash
run: |
# Set env to enable reduxer
echo "WAYBACK_MEILI_ENDPOINT=http://localhost:7700" >> $GITHUB_ENV
echo "PLAYBACK_MEILI_ENDPOINT=http://localhost:7700" >> $GITHUB_ENV
echo "WAYBACK_MEILI_APIKEY=foobar" >> $GITHUB_ENV
echo "PLAYBACK_MEILI_APIKEY=foobar" >> $GITHUB_ENV
echo "WAYBACK_MEILI_ENDPOINT=${{ vars.WAYBACK_MEILI_ENDPOINT }}" >> $GITHUB_ENV
echo "PLAYBACK_MEILI_ENDPOINT=${{ vars.WAYBACK_MEILI_ENDPOINT }}" >> $GITHUB_ENV
echo "WAYBACK_MEILI_APIKEY=${{ vars.MEILISEARCH_APIKEY }}" >> $GITHUB_ENV
echo "PLAYBACK_MEILI_APIKEY=${{ vars.MEILISEARCH_APIKEY }}" >> $GITHUB_ENV
- name: Check out code base
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
Expand Down
13 changes: 10 additions & 3 deletions cmd/playback/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,26 +10,33 @@ import (
"os"

"github.com/spf13/cobra"
"github.com/wabarc/logger"
"github.com/wabarc/playback"
"github.com/wabarc/wayback"
"github.com/wabarc/wayback/config"
)

func main() {
opts, err := config.NewParser().ParseEnvironmentVariables()
if err != nil {
logger.Fatal("Parse environment variables or flags failed, error: %v", err)
}

var rootCmd = &cobra.Command{
Use: "playback",
Short: "A toolkit to playback archived webpage from time capsules.",
Example: ` playback https://example.com https://example.org`,
Version: playback.Version,
Run: func(cmd *cobra.Command, args []string) {
handle(cmd, args)
handle(cmd, opts, args)
},
}

// nolint:errcheck
rootCmd.Execute()
}

func handle(cmd *cobra.Command, args []string) {
func handle(cmd *cobra.Command, opts *config.Options, args []string) {
if len(args) < 1 {
// nolint:errcheck
cmd.Usage()
Expand All @@ -42,7 +49,7 @@ func handle(cmd *cobra.Command, args []string) {
os.Exit(1)
}

collects, err := wayback.Playback(context.TODO(), urls...)
collects, err := wayback.Playback(context.TODO(), opts, urls...)
if err != nil {
cmd.Println(err)
os.Exit(1)
Expand Down
19 changes: 10 additions & 9 deletions cmd/wayback/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,28 +163,29 @@ func handle(cmd *cobra.Command, args []string) {
setToEnv(cmd)
parser := config.NewParser()

var opts *config.Options
if len(daemon) > 0 {
logger.Info("Run wayback using configuration file")
if config.Opts, err = parser.ParseFile(configFile); err != nil {
if _, err = parser.ParseFile(configFile); err != nil {
logger.Fatal("Parse configuration file failed, error: %v", err)
}
}

if config.Opts, err = parser.ParseEnvironmentVariables(); err != nil {
if opts, err = parser.ParseEnvironmentVariables(); err != nil {
logger.Fatal("Parse environment variables or flags failed, error: %v", err)
}

if !config.Opts.LogTime() {
if !opts.LogTime() {
logger.DisableTime()
}

logger.SetLogLevel(config.Opts.LogLevel())
if debug || config.Opts.HasDebugMode() {
logger.SetLogLevel(opts.LogLevel())
if debug || opts.HasDebugMode() {
profiling()
logger.EnableDebug()
}

if config.Opts.EnabledMetrics() {
if opts.EnabledMetrics() {
metrics.Gather = metrics.NewCollector()
}

Expand All @@ -194,7 +195,7 @@ func handle(cmd *cobra.Command, args []string) {
}

if print {
cmd.Println(spew.Sdump(config.Opts))
cmd.Println(spew.Sdump(opts))
return
}

Expand All @@ -204,9 +205,9 @@ func handle(cmd *cobra.Command, args []string) {
hasArgs := len(args) > 0
switch {
case hasDaemon:
serve(cmd, args)
serve(cmd, opts, args)
case hasArgs:
archive(cmd, args)
archive(cmd, opts, args)
default:
// nolint:errcheck
cmd.Help()
Expand Down
75 changes: 43 additions & 32 deletions cmd/wayback/serve.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/wabarc/logger"
"github.com/wabarc/wayback/config"
"github.com/wabarc/wayback/pooling"
"github.com/wabarc/wayback/publish"
"github.com/wabarc/wayback/service"
"github.com/wabarc/wayback/service/discord"
"github.com/wabarc/wayback/service/httpd"
Expand All @@ -24,6 +25,8 @@ import (
"github.com/wabarc/wayback/service/twitter"
"github.com/wabarc/wayback/storage"
"github.com/wabarc/wayback/systemd"

_ "github.com/wabarc/wayback/publish/register"
)

// Create channel to listen for signals.
Expand All @@ -38,30 +41,35 @@ type services struct {
targets []target
}

func serve(_ *cobra.Command, _ []string) {
store, err := storage.Open("")
func serve(_ *cobra.Command, opts *config.Options, _ []string) {
store, err := storage.Open(opts, "")
if err != nil {
logger.Fatal("open storage failed: %v", err)
}
defer store.Close()

cfg := []pooling.Option{
pooling.Capacity(opts.PoolingSize()),
pooling.Timeout(opts.WaybackTimeout()),
pooling.MaxRetries(opts.WaybackMaxRetries()),
}
ctx, cancel := context.WithCancel(context.Background())
pool := pooling.New(ctx, config.Opts.PoolingSize())
pool := pooling.New(ctx, cfg...)
go pool.Roll()

if config.Opts.EnabledMeilisearch() {
endpoint := config.Opts.WaybackMeiliEndpoint()
indexing := config.Opts.WaybackMeiliIndexing()
apikey := config.Opts.WaybackMeiliApikey()
meili := service.NewMeili(endpoint, apikey, indexing)
if err := meili.Setup(); err != nil {
logger.Error("setup meilisearch failed: %v", err)
}
logger.Debug("setup meilisearch success")
pub := publish.New(ctx, opts)
go pub.Start()

opt := []service.Option{
service.Config(opts),
service.Storage(store),
service.Pool(pool),
service.Publish(pub),
}
options := service.ParseOptions(opt...)

srv := &services{}
_ = srv.run(ctx, store, pool)
_ = srv.run(ctx, options)

if systemd.HasNotifySocket() {
logger.Info("sending readiness notification to Systemd")
Expand All @@ -71,100 +79,101 @@ func serve(_ *cobra.Command, _ []string) {
}
}

go srv.stop(pool, cancel)
go srv.daemon(pool, pub, cancel)
<-ctx.Done()

logger.Info("wayback service stopped.")
}

// nolint:gocyclo
func (srv *services) run(ctx context.Context, store *storage.Storage, pool *pooling.Pool) *services {
func (srv *services) run(ctx context.Context, opts service.Options) *services {
size := len(daemon)
srv.targets = make([]target, 0, size)
for _, s := range daemon {
s := s
switch s {
case "irc":
irc := relaychat.New(ctx, store, pool)
irc := relaychat.New(ctx, opts)
go func() {
if err := irc.Serve(); err != relaychat.ErrServiceClosed {
logger.Error("%v", err)
logger.Error("start %s service failed: %v", s, err)
}
}()
srv.targets = append(srv.targets, target{
call: func() { irc.Shutdown() }, // nolint:errcheck
name: s,
})
case "slack":
sl := slack.New(ctx, store, pool)
sl := slack.New(ctx, opts)
go func() {
if err := sl.Serve(); err != slack.ErrServiceClosed {
logger.Error("%v", err)
logger.Error("start %s service failed: %v", s, err)
}
}()
srv.targets = append(srv.targets, target{
call: func() { sl.Shutdown() }, // nolint:errcheck
name: s,
})
case "discord":
d := discord.New(ctx, store, pool)
d := discord.New(ctx, opts)
go func() {
if err := d.Serve(); err != discord.ErrServiceClosed {
logger.Error("%v", err)
logger.Error("start %s service failed: %v", s, err)
}
}()
srv.targets = append(srv.targets, target{
call: func() { d.Shutdown() }, // nolint:errcheck
name: s,
})
case "mastodon", "mstdn":
m := mastodon.New(ctx, store, pool)
m := mastodon.New(ctx, opts)
go func() {
if err := m.Serve(); err != mastodon.ErrServiceClosed {
logger.Error("%v", err)
logger.Error("start %s service failed: %v", s, err)
}
}()
srv.targets = append(srv.targets, target{
call: func() { m.Shutdown() }, // nolint:errcheck
name: s,
})
case "telegram":
t := telegram.New(ctx, store, pool)
t := telegram.New(ctx, opts)
go func() {
if err := t.Serve(); err != telegram.ErrServiceClosed {
logger.Error("%v", err)
logger.Error("start %s service failed: %v", s, err)
}
}()
srv.targets = append(srv.targets, target{
call: func() { t.Shutdown() }, // nolint:errcheck
name: s,
})
case "twitter":
t := twitter.New(ctx, store, pool)
t := twitter.New(ctx, opts)
go func() {
if err := t.Serve(); err != twitter.ErrServiceClosed {
logger.Error("%v", err)
logger.Error("start %s service failed: %v", s, err)
}
}()
srv.targets = append(srv.targets, target{
call: func() { t.Shutdown() }, // nolint:errcheck
name: s,
})
case "matrix":
m := matrix.New(ctx, store, pool)
m := matrix.New(ctx, opts)
go func() {
if err := m.Serve(); err != matrix.ErrServiceClosed {
logger.Error("%v", err)
logger.Error("start %s service failed: %v", s, err)
}
}()
srv.targets = append(srv.targets, target{
call: func() { m.Shutdown() }, // nolint:errcheck
name: s,
})
case "web", "httpd":
h := httpd.New(ctx, store, pool)
h := httpd.New(ctx, opts)
go func() {
if err := h.Serve(); err != httpd.ErrServiceClosed {
logger.Error("%v", err)
logger.Error("start %s service failed: %v", s, err)
}
}()
srv.targets = append(srv.targets, target{
Expand All @@ -179,7 +188,7 @@ func (srv *services) run(ctx context.Context, store *storage.Storage, pool *pool
return srv
}

func (srv *services) stop(pool *pooling.Pool, cancel context.CancelFunc) {
func (srv *services) daemon(pool *pooling.Pool, pub *publish.Publish, cancel context.CancelFunc) {
// SIGINT handles Ctrl+C locally.
// SIGTERM handles termination signal from cloud service.
signal.Notify(
Expand All @@ -199,6 +208,8 @@ func (srv *services) stop(pool *pooling.Pool, cancel context.CancelFunc) {
srv.shutdown()
// Gracefully closes the worker pool
pool.Close()
// Stop publish service
pub.Stop()
cancel()
}

Expand Down
7 changes: 4 additions & 3 deletions cmd/wayback/wayback.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/spf13/cobra"
"github.com/wabarc/helper"
"github.com/wabarc/wayback"
"github.com/wabarc/wayback/config"
"github.com/wabarc/wayback/errors"
"github.com/wabarc/wayback/reduxer"
"golang.org/x/sync/errgroup"
Expand All @@ -34,15 +35,15 @@ func assets(art reduxer.Artifact) []reduxer.Asset {
}
}

func archive(cmd *cobra.Command, args []string) {
func archive(cmd *cobra.Command, opts *config.Options, args []string) {
// TODO: clean the auto-created temporary directory.
archiving := func(ctx context.Context, urls []*url.URL) error {
g, ctx := errgroup.WithContext(ctx)
rdx, err := reduxer.Do(ctx, urls...)
rdx, err := reduxer.Do(ctx, opts, urls...)
if err != nil {
return errors.Wrap(err, "reduxer unexpected")
}
cols, err := wayback.Wayback(ctx, rdx, urls...)
cols, err := wayback.Wayback(ctx, rdx, opts, urls...)
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
package config // import "github.com/wabarc/wayback/config"

// Opts holds parsed configuration options.
var Opts *Options
// var Opts *Options

// nolint:stylecheck
const (
Expand Down
1 change: 1 addition & 0 deletions metrics/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ const (
PublishMatrix = "matrix"
PublishSlack = "slack"
PublishNostr = "nostr"
PublishMeili = "meili"

StatusRequest = "request"
StatusSuccess = "success"
Expand Down
Loading

0 comments on commit 08cb62c

Please sign in to comment.