Add support for XMPP (#380)

wabarc · Apr 30, 2023 · da72fdf · da72fdf
1 parent 4bbf9ea
commit da72fdf
Show file tree

Hide file tree

Showing 21 changed files with 905 additions and 10 deletions.
diff --git a/README.md b/README.md
@@ -21,12 +21,13 @@ Supported Golang version: See [.github/workflows/testing.yml](./.github/workflow
 ## Features
 
 - Free and open-source
+- Expose prometheus metrics
 - Cross-platform compatibility
 - Batch wayback URLs for faster archiving
 - Built-in CLI (`wayback`) for convenient use
 - Serve as a Tor Hidden Service or local web entry for added privacy and accessibility
 - Easier wayback to Internet Archive, archive.today, IPFS and Telegraph integration
-- Interactive with IRC, Matrix, Telegram bot, Discord bot, Mastodon, and Twitter as a daemon service for convenient use
+- Interactive with IRC, Matrix, Telegram bot, Discord bot, Mastodon, Twitter, and XMPP as a daemon service for convenient use
 - Supports publishing wayback results to Telegram channel, Mastodon, and GitHub Issues for sharing
 - Supports storing archived files to disk for offline use
 - Download streaming media (requires [FFmpeg](https://ffmpeg.org/)) for convenient media archiving.
@@ -114,7 +115,7 @@ Examples:
 Flags:
       --chatid string      Telegram channel id
   -c, --config string      Configuration file path, defaults: ./wayback.conf, ~/wayback.conf, /etc/wayback.conf
-  -d, --daemon strings     Run as daemon service, supported services are telegram, web, mastodon, twitter, discord, slack, irc
+  -d, --daemon strings     Run as daemon service, supported services are telegram, web, mastodon, twitter, discord, slack, irc, xmpp
       --debug              Enable debug mode (default mode is false)
   -h, --help               help for wayback
       --ia                 Wayback webpages to Internet Archive
@@ -283,6 +284,10 @@ You can also specify configuration options either via command flags or via envir
 | -                   | `WAYBACK_SLACK_HELPTEXT`          | -                          | The help text for Slack slash command                        |
 | -                   | `WAYBACK_NOSTR_RELAY_URL`         | `wss://nostr.developer.li` | Nostr relay server url, multiple separated by comma          |
 | -                   | `WAYBACK_NOSTR_PRIVATE_KEY`       | -                          | The private key of a Nostr account                           |
+| -                   | `WAYBACK_XMPP_JID`                | -                          | The JID of a XMPP account                                    |
+| -                   | `WAYBACK_XMPP_PASSWORD`           | -                          | The password of a XMPP account                               |
+| -                   | `WAYBACK_XMPP_NOTLS`              | -                          | Connect to XMPP server without TLS                           |
+| -                   | `WAYBACK_XMPP_HELPTEXT`           | -                          | The help text for XMPP command                               |
 | `--tor`             | `WAYBACK_USE_TOR`                 | `false`                    | Snapshot webpage via Tor anonymity network                   |
 | `--tor-key`         | `WAYBACK_ONION_PRIVKEY`           | -                          | The private key for Tor Hidden Service                       |
 | -                   | `WAYBACK_ONION_LOCAL_PORT`        | `8964`                     | Local port for Tor Hidden Service, also support for a **reverse proxy**. This is ignored if `WAYBACK_LISTEN_ADDR` is set. |

diff --git a/cmd/wayback/serve.go b/cmd/wayback/serve.go
@@ -7,6 +7,7 @@ import (
 	"context"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 
 	"github.com/spf13/cobra"
@@ -23,6 +24,7 @@ import (
 	"github.com/wabarc/wayback/service/slack"
 	"github.com/wabarc/wayback/service/telegram"
 	"github.com/wabarc/wayback/service/twitter"
+	"github.com/wabarc/wayback/service/xmpp"
 	"github.com/wabarc/wayback/storage"
 	"github.com/wabarc/wayback/systemd"
 
@@ -91,7 +93,7 @@ func (srv *services) run(ctx context.Context, opts service.Options) *services {
 	srv.targets = make([]target, 0, size)
 	for _, s := range daemon {
 		s := s
-		switch s {
+		switch strings.ToLower(s) {
 		case "irc":
 			irc := relaychat.New(ctx, opts)
 			go func() {
@@ -180,6 +182,17 @@ func (srv *services) run(ctx context.Context, opts service.Options) *services {
 				call: func() { h.Shutdown() }, // nolint:errcheck
 				name: s,
 			})
+		case "jabber", "xmpp":
+			h := xmpp.New(ctx, opts)
+			go func() {
+				if err := h.Serve(); err != xmpp.ErrServiceClosed {
+					logger.Error("start %s service failed: %v", s, err)
+				}
+			}()
+			srv.targets = append(srv.targets, target{
+				call: func() { h.Shutdown() }, // nolint:errcheck
+				name: s,
+			})
 		default:
 			logger.Fatal("unrecognize %s in `--daemon`", s)
 		}

diff --git a/config/config_test.go b/config/config_test.go
@@ -1295,6 +1295,75 @@ func TestPublishToSlackChannel(t *testing.T) {
 	}
 }
 
+func TestXMPPUsername(t *testing.T) {
+	expected := "foo@example.com"
+
+	os.Clearenv()
+	os.Setenv("WAYBACK_XMPP_USERNAME", expected)
+
+	opts, err := NewParser().ParseEnvironmentVariables()
+	if err != nil {
+		t.Fatalf(`Parsing environment variables failed: %v`, err)
+	}
+
+	got := opts.XMPPUsername()
+	if got != expected {
+		t.Fatalf(`Unexpected XMPP username got %v instead of %v`, got, expected)
+	}
+}
+
+func TestXMPPPassword(t *testing.T) {
+	expected := "bar"
+
+	os.Clearenv()
+	os.Setenv("WAYBACK_XMPP_PASSWORD", expected)
+
+	opts, err := NewParser().ParseEnvironmentVariables()
+	if err != nil {
+		t.Fatalf(`Parsing environment variables failed: %v`, err)
+	}
+
+	got := opts.XMPPPassword()
+	if got != expected {
+		t.Fatalf(`Unexpected XMPP password got %v instead of %v`, got, expected)
+	}
+}
+
+func TestXMPPNoTLS(t *testing.T) {
+	expected := true
+
+	os.Clearenv()
+	os.Setenv("WAYBACK_XMPP_NOTLS", strconv.FormatBool(expected))
+
+	opts, err := NewParser().ParseEnvironmentVariables()
+	if err != nil {
+		t.Fatalf(`Parsing environment variables failed: %v`, err)
+	}
+
+	got := opts.XMPPNoTLS()
+	if got != expected {
+		t.Fatalf(`Unexpected disable XMPP TLS got %v instead of %v`, got, expected)
+	}
+}
+
+func TestXMPPHelptext(t *testing.T) {
+	expected := "some text"
+
+	os.Clearenv()
+	os.Setenv("WAYBACK_XMPP_HELPTEXT", expected)
+
+	parser := NewParser()
+	opts, err := parser.ParseEnvironmentVariables()
+	if err != nil {
+		t.Fatalf(`Parsing environment variables failed: %v`, err)
+	}
+
+	got := opts.XMPPHelptext()
+	if got != expected {
+		t.Fatalf(`Unexpected XMPP help text got %v instead of %v`, got, expected)
+	}
+}
+
 func TestNostrRelayURL(t *testing.T) {
 	var tests = []struct {
 		url string

diff --git a/config/options.go b/config/options.go
@@ -64,6 +64,11 @@ const (
 	defIRCChannel  = ""
 	defIRCServer   = "irc.libera.chat:6697"
 
+	defXMPPUsername = ""
+	defXMPPPassword = ""
+	defXMPPNoTLS    = false
+	defXMPPHelptext = "Hi there."
+
 	defMatrixHomeserver = "https://matrix.org"
 	defMatrixUserID     = ""
 	defMatrixRoomID     = ""
@@ -125,6 +130,7 @@ type Options struct {
 	nostr    *nostr
 	irc      *irc
 	onion    *onion
+	xmpp     *xmpp
 
 	listenAddr          string
 	chromeRemoteAddr    string
@@ -226,6 +232,13 @@ type onion struct {
 	disabled bool
 }
 
+type xmpp struct {
+	username string
+	password string
+	noTLS    bool
+	helptext string
+}
+
 // NewOptions returns Options with default values.
 func NewOptions() *Options {
 	opts := &Options{
@@ -320,6 +333,12 @@ func NewOptions() *Options {
 			localPort:   defOnionLocalPort,
 			remotePorts: defOnionRemotePorts,
 		},
+		xmpp: &xmpp{
+			username: defXMPPUsername,
+			password: defXMPPPassword,
+			noTLS:    defXMPPNoTLS,
+			helptext: defXMPPHelptext,
+		},
 	}
 
 	return opts
@@ -636,6 +655,26 @@ func (o *Options) PublishToSlackChannel() bool {
 	return o.SlackBotToken() != "" && o.SlackChannel() != ""
 }
 
+// XMPPUsername returns the XMPP username (JID).
+func (o *Options) XMPPUsername() string {
+	return o.xmpp.username
+}
+
+// XMPPPassword returns the XMPP password.
+func (o *Options) XMPPPassword() string {
+	return o.xmpp.password
+}
+
+// XMPPNoTLS returns whether disable TLS.
+func (o *Options) XMPPNoTLS() bool {
+	return o.xmpp.noTLS
+}
+
+// XMPPHelptext returns the help text for XMPP.
+func (o *Options) XMPPHelptext() string {
+	return breakLine(o.xmpp.helptext)
+}
+
 // NotionToken returns the Notion integration token.
 func (o *Options) NotionToken() string {
 	return o.notion.token

diff --git a/config/parser.go b/config/parser.go
@@ -177,6 +177,14 @@ func (p *Parser) parseLines(lines []string) (err error) {
 			p.opts.slack.channel = parseString(val, defSlackChannel)
 		case "WAYBACK_SLACK_HELPTEXT":
 			p.opts.slack.helptext = parseString(val, defSlackHelptext)
+		case "WAYBACK_XMPP_JID", "WAYBACK_XMPP_USERNAME":
+			p.opts.xmpp.username = parseString(val, defXMPPUsername)
+		case "WAYBACK_XMPP_PASSWORD":
+			p.opts.xmpp.password = parseString(val, defXMPPPassword)
+		case "WAYBACK_XMPP_NOTLS":
+			p.opts.xmpp.noTLS = parseBool(val, defXMPPNoTLS)
+		case "WAYBACK_XMPP_HELPTEXT":
+			p.opts.xmpp.helptext = parseString(val, defXMPPHelptext)
 		case "WAYBACK_NOSTR_RELAY_URL":
 			p.opts.nostr.url = parseString(val, defNostrRelayURL)
 		case "WAYBACK_NOSTR_PRIVATE_KEY":

diff --git a/docs/index.md b/docs/index.md
@@ -7,12 +7,13 @@ Whether you need to archive a single web page or a large collection of websites,
 ## Features
 
 - Free and open-source
+- Expose prometheus metrics
 - Cross-platform compatibility
 - Batch wayback URLs for faster archiving
 - Built-in CLI (`wayback`) for convenient use
 - Serve as a Tor Hidden Service or local web entry for added privacy and accessibility
 - Easier wayback to Internet Archive, archive.today, IPFS and Telegraph integration
-- Interactive with IRC, Matrix, Telegram bot, Discord bot, Mastodon, and Twitter as a daemon service for convenient use
+- Interactive with IRC, Matrix, Telegram bot, Discord bot, Mastodon, Twitter, and XMPP as a daemon service for convenient use
 - Supports publishing wayback results to Telegram channel, Mastodon, and GitHub Issues for sharing
 - Supports storing archived files to disk for offline use
 - Download streaming media (requires [FFmpeg](https://ffmpeg.org/)) for convenient media archiving.

diff --git a/docs/index.zh.md b/docs/index.zh.md
@@ -12,11 +12,12 @@ Wayback是用Go编写的开源网络存档应用程序。具有模块化和可
 
 - 完全开源
 - 跨平台兼容
+- 输出Prometheus度量指标
 - 批量存档URL以加快存档速度
 - 内置CLI工具（`wayback`）以便于使用
 - 可作为Tor隐藏服务或本地Web入口，增加隐私和可访问性
 - 更容易地集成到Internet Archive、archive.today、IPFS和Telegraph中
-- 可与IRC、Matrix、Telegram机器人、Discord机器人、Mastodon和Twitter进行交互，作为守护进程服务，以便于使用
+- 可与IRC、Matrix、Telegram机器人、Discord机器人、Mastodon、Twitter和XMPP进行交互，作为守护进程服务，以便于使用
 - 支持将存档结果发布到Telegram频道、Mastodon和GitHub Issues中进行共享
 - 支持将存档文件存储到磁盘中以供离线使用
 - 下载流媒体（需要[FFmpeg](https://ffmpeg.org/)）以便于媒体存档。

diff --git a/docs/integrations/xmpp.md b/docs/integrations/xmpp.md
@@ -1 +1,24 @@
-WIP
+---
+title: Interactive with XMPP
+---
+
+## How to build a XMPP Service
+
+To create an XMPP account, you need to find a client and a public server. Here are some recommended collections of XMPP servers to help you get started.
+
+- [Directory 404](https://xmpp.404.city/)
+- [Public XMPP servers](https://list.jabber.at/)
+- [Tracking the progress of OMEMO integration in XMPP clients](https://omemo.top)
+
+## Configuration
+
+To use the XMPP service, you will need to set the following environment variables or configuration file:
+
+- `WAYBACK_IRC_JID`: The JID for the XMPP client (required).
+- `WAYBACK_IRC_PASSWORD`: The password for the XMPP client (required).
+- `WAYBACK_IRC_NOTLS`: Connect to XMPP server without TLS (optional).
+- `WAYBACK_IRC_HELPTEXT`: The help text for XMPP command (optional).
+
+## Further reading
+- [XMPP | The universal messaging standard](https://xmpp.org/)
+
diff --git a/docs/integrations/xmpp.zh.md b/docs/integrations/xmpp.zh.md
@@ -1 +1,25 @@
-WIP
+---
+title: XMPP
+---
+
+## 如何构建XMPP服务
+
+创建 XMPP 账户，您需要找到一个客户端和一个公共服务器。以下是一些推荐的 XMPP 服务器集合，可帮助您入门。
+
+- [Directory 404](https://xmpp.404.city/)
+- [Public XMPP servers](https://list.jabber.at/)
+- [Tracking the progress of OMEMO integration in XMPP clients](https://omemo.top)
+
+## 配置
+
+为了使用XMPP服务，您需要设置以下环境变量或配置文件：
+
+- `WAYBACK_IRC_JID`: XMPP客户端的JID（必填）。
+- `WAYBACK_IRC_PASSWORD`: XMPP客户端的密码（必填）。
+- `WAYBACK_IRC_NOTLS`: 连接到XMPP服务器时不使用TLS加密（可选）。
+- `WAYBACK_IRC_HELPTEXT`: XMPP命令的帮助文本（可选）。
+
+## 相关资料
+
+- [XMPP | The universal messaging standard](https://xmpp.org/)
+
diff --git a/go.mod b/go.mod
@@ -52,6 +52,9 @@ require (
 	golang.org/x/sync v0.1.0
 	gopkg.in/telebot.v3 v3.0.0-20220130115853-f0291132d3c3
 	maunium.net/go/mautrix v0.12.0
+	mellium.im/sasl v0.3.1
+	mellium.im/xmlstream v0.15.4
+	mellium.im/xmpp v0.21.4
 )
 
 require (
@@ -146,13 +149,16 @@ require (
 	github.com/wabarc/memento v0.0.0-20210703205719-adc2f8ab8bae // indirect
 	github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b // indirect
 	github.com/ybbus/httpretry v1.0.1 // indirect
-	golang.org/x/crypto v0.4.0 // indirect
+	golang.org/x/crypto v0.5.0 // indirect
 	golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect
+	golang.org/x/mod v0.8.0 // indirect
 	golang.org/x/sys v0.7.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/tools v0.6.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/sourcemap.v1 v1.0.5 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	lukechampine.com/blake3 v1.1.7 // indirect
+	mellium.im/reader v0.1.0 // indirect
 	mvdan.cc/xurls/v2 v2.4.0 // indirect
 )
diff --git a/go.sum b/go.sum
@@ -681,8 +681,8 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5
 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
-golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
+golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
+golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -716,6 +716,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -919,6 +921,8 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1040,6 +1044,14 @@ lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0=
 lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=
 maunium.net/go/mautrix v0.12.0 h1:jyT1TkJBIRJ7+OW7NhmMHmnEEBLsQe9ml+FYwSLhlaU=
 maunium.net/go/mautrix v0.12.0/go.mod h1:hHvNi5iKVAiI2MAdAeXHtP4g9BvNEX2rsQpSF/x6Kx4=
+mellium.im/reader v0.1.0 h1:UUEMev16gdvaxxZC7fC08j7IzuDKh310nB6BlwnxTww=
+mellium.im/reader v0.1.0/go.mod h1:F+X5HXpkIfJ9EE1zHQG9lM/hO946iYAmU7xjg5dsQHI=
+mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo=
+mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw=
+mellium.im/xmlstream v0.15.4 h1:gLKxcWl4rLMUpKgtzrTBvr4OexPeO/edYus+uK3F6ZI=
+mellium.im/xmlstream v0.15.4/go.mod h1:yXaCW2++fmVO4L9piKVkyLDqnCmictVYF7FDQW8prb4=
+mellium.im/xmpp v0.21.4 h1:hhAGFC/mGt2Bbmx46vPn+kQT0pJec7uaq+9xckkr9uI=
+mellium.im/xmpp v0.21.4/go.mod h1:Emo7bXXyEEgH2hdTO9zp9eGJoc9yK5dAlG0/YVJlh+U=
 mvdan.cc/xurls/v2 v2.2.0/go.mod h1:EV1RMtya9D6G5DMYPGD8zTQzaHet6Jh8gFlRgGRJeO8=
 mvdan.cc/xurls/v2 v2.4.0 h1:tzxjVAj+wSBmDcF6zBB7/myTy3gX9xvi8Tyr28AuQgc=
 mvdan.cc/xurls/v2 v2.4.0/go.mod h1:+GEjq9uNjqs8LQfM9nVnM8rff0OQ5Iash5rzX+N1CSg=