Add wayback user agent

README.md

@@ -211,6 +211,7 @@ You can also specify configuration options either via command flags or via envir
 | -                   | `WAYBACK_STORAGE_DIR`             | -                       | Directory to store binary file, e.g. PDF, html file          |
 | -                   | `WAYBACK_MAX_MEDIA_SIZE`          | `512MB`                 | Max size to limit download stream media                      |
 | -                   | `WAYBACK_TIMEOUT`                 | `300`                   | Timeout for single wayback request, defaults to 300 second   |
+| -                   | `WAYBACK_USERAGENT`               | `WaybackArchiver/1.0`   | User-Agent for a wayback request                             |
 | `-d`, `--daemon`    | -                                 | -                       | Run as daemon service, e.g. `telegram`, `web`, `mastodon`, `twitter`, `discord` |
 | `--ia`              | `WAYBACK_ENABLE_IA`               | `true`                  | Wayback webpages to **Internet Archive**                     |
 | `--is`              | `WAYBACK_ENABLE_IS`               | `true`                  | Wayback webpages to **Archive Today**                        |

config/config_test.go

@@ -1247,3 +1247,39 @@ func TestWaybackTimeout(t *testing.T) {
 		})
 	}
 }
+
+func TestWaybackUserAgent(t *testing.T) {
+	t.Parallel()
+
+	var tests = []struct {
+		userAgent string
+		expected  string
+	}{
+		{
+			userAgent: "",
+			expected:  defWaybackUserAgent,
+		},
+		{
+			userAgent: "foo bar",
+			expected:  "foo bar",
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			os.Clearenv()
+			os.Setenv("WAYBACK_USERAGENT", test.userAgent)
+
+			parser := NewParser()
+			opts, err := parser.ParseEnvironmentVariables()
+			if err != nil {
+				t.Fatalf(`Parsing environment variables failed: %v`, err)
+			}
+
+			got := opts.WaybackUserAgent()
+			if got != test.expected {
+				t.Fatalf(`Unexpected set wayback user agent got %s instead of %s`, got, test.expected)
+			}
+		})
+	}
+}

config/options.go

@@ -73,6 +73,7 @@ const (
 	defStorageDir          = ""
 	defMaxMediaSize        = "512MB"
 	defWaybackTimeout      = 300
+	defWaybackUserAgent    = "WaybackArchiver/1.0"
 )
 
 var (
@@ -107,6 +108,7 @@ type Options struct {
 	storageDir          string
 	maxMediaSize        string
 	waybackTimeout      int
+	waybackUserAgent    string
 }
 
 type ipfs struct {
@@ -193,6 +195,7 @@ func NewOptions() *Options {
 		storageDir:          defStorageDir,
 		maxMediaSize:        defMaxMediaSize,
 		waybackTimeout:      defWaybackTimeout,
+		waybackUserAgent:    defWaybackUserAgent,
 		ipfs: &ipfs{
 			host: defIPFSHost,
 			port: defIPFSPort,
@@ -630,3 +633,8 @@ func (o *Options) MaxAttachSize(scope string) int64 {
 func (o *Options) WaybackTimeout() time.Duration {
 	return time.Duration(o.waybackTimeout) * time.Second
 }
+
+// WaybackUserAgent returns User-Agent for a wayback request.
+func (o *Options) WaybackUserAgent() string {
+	return o.waybackUserAgent
+}

config/parser.go

@@ -185,6 +185,8 @@ func (p *Parser) parseLines(lines []string) (err error) {
 			p.opts.maxMediaSize = parseString(val, defMaxMediaSize)
 		case "WAYBACK_TIMEOUT":
 			p.opts.waybackTimeout = parseInt(val, defWaybackTimeout)
+		case "WAYBACK_USERAGENT":
+			p.opts.waybackUserAgent = parseString(val, defWaybackUserAgent)
 		default:
 			if os.Getenv(key) == "" && val != "" {
 				os.Setenv(key, val)

wayback.1

@@ -156,6 +156,9 @@ Number of worker pool for wayback at once. default 3\&.
 .B WAYBACK_TIMEOUT
 Timeout for single wayback request, default 300\&.
 .TP
+.B WAYBACK_USERAGENT
+User-Agent for a wayback request, default WaybackArchiver/1.0\&.
+.TP
 .B WAYBACK_BOLT_PATH
 File path of bolt database. default ./wayback.db\&.
 .TP

wayback.conf

@@ -49,6 +49,7 @@ WAYBACK_POOLING_SIZE=3
 WAYBACK_STORAGE_DIR=
 WAYBACK_MAX_MEDIA_SIZE=512MB
 WAYBACK_TIMEOUT=300
+WAYBACK_USERAGENT=WaybackArchiver/1.0
 
 # ipfs slot: infura, pinata
 # doc: https://github.com/wabarc/ipfs-pinner#supported-pinning-services