Add user-agent in http request header

README.md

@@ -6,7 +6,7 @@
 [![GitHub license](https://img.shields.io/badge/license-Apache%202-blue.svg)](https://raw.githubusercontent.com/goark/ml/master/LICENSE)
 [![GitHub release](http://img.shields.io/github/release/goark/ml.svg)](https://github.com/goark/ml/releases/latest)
 
-This package is required Go 1.18 or later.
+This package is required Go 1.16 or later.
 
 **Migrated repository to [github.com/goark/ml][ml]**
 
@@ -28,12 +28,13 @@ Usage:
   ml [flags] [URL [URL]...]
 
 Flags:
-      --debug          for debug
-  -h, --help           help for ml
-  -i, --interactive    interactive mode
-  -l, --log int        history log size
-  -s, --style string   link style [markdown|wiki|html|csv|json] (default "markdown")
-  -v, --version        output version of ml
+      --debug               for debug
+  -h, --help                help for ml
+  -i, --interactive         interactive mode
+  -l, --log int             history log size
+  -s, --style string        link style [markdown|wiki|html|csv|json] (default "markdown")
+  -a, --user-agent string   User-Agent string
+  -v, --version             output version of ml
 ```
 
 ```
@@ -80,7 +81,7 @@ import (
 )
 
 func main() {
-    lnk, err := makelink.New(context.Background(), "https://git.io/vFR5M")
+    lnk, err := makelink.New(context.Background(), "https://git.io/vFR5M", "")
     if err != nil {
         fmt.Fprintln(os.Stderr, err)
         return

Taskfile.yml

@@ -18,7 +18,7 @@ tasks:
     cmds:
       - go mod verify
       - go test -shuffle on ./...
-      - docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.45.0 golangci-lint run --enable gosec --timeout 3m0s ./...
+      - docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.45.2 golangci-lint run --enable gosec --timeout 3m0s ./...
     sources:
       - ./go.mod
       - '**/*.go'

facade/facade.go

@@ -47,6 +47,10 @@ func newRootCmd(ui *rwi.RWI, args []string) *cobra.Command {
 			if err != nil {
 				return debugPrint(ui, err)
 			}
+			userAgent, err := cmd.Flags().GetString("user-agent")
+			if err != nil {
+				return debugPrint(ui, err)
+			}
 
 			//history log
 			log, err := cmd.Flags().GetInt("log") //log size
@@ -68,7 +72,7 @@ func newRootCmd(ui *rwi.RWI, args []string) *cobra.Command {
 			}()
 
 			//set options
-			opts := options.New(style, hist)
+			opts := options.New(style, hist, userAgent)
 			if interactiveFlag {
 				//interactive mode
 				if err := interactive.Do(opts); err != nil {
@@ -126,6 +130,7 @@ func newRootCmd(ui *rwi.RWI, args []string) *cobra.Command {
 	rootCmd.Flags().BoolVarP(&interactiveFlag, "interactive", "i", false, "interactive mode")
 	rootCmd.Flags().BoolVarP(&debugFlag, "debug", "", false, "for debug")
 	rootCmd.Flags().StringP("style", "s", makelink.StyleMarkdown.String(), "link style ["+makelink.StyleList()+"]")
+	rootCmd.Flags().StringP("user-agent", "a", "", "User-Agent string")
 	rootCmd.Flags().IntP("log", "l", 0, "history log size")
 
 	return rootCmd

facade/options/options.go

@@ -14,14 +14,15 @@ import (
 type Options struct {
 	linkStyle makelink.Style
 	hist      *history.HistoryFile
+	userAgent string
 }
 
 //New returns new Options instance
-func New(s makelink.Style, hist *history.HistoryFile) *Options {
+func New(s makelink.Style, hist *history.HistoryFile, userAgent string) *Options {
 	if hist == nil {
 		hist = history.NewFile(0, "")
 	}
-	return &Options{linkStyle: s, hist: hist}
+	return &Options{linkStyle: s, hist: hist, userAgent: userAgent}
 }
 
 //History method returns history.HistoryFile instance.
@@ -33,7 +34,7 @@ func (c *Options) MakeLink(ctx context.Context, urlStr string) (io.Reader, error
 		return nil, errs.Wrap(ecode.ErrNullPointer)
 	}
 	c.History().Add(urlStr)
-	lnk, err := makelink.New(ctx, urlStr)
+	lnk, err := makelink.New(ctx, urlStr, c.userAgent)
 	if err != nil {
 		return nil, errs.Wrap(err)
 	}

facade/options/options_test.go

@@ -15,7 +15,7 @@ import (
 
 func TestMakeLink(t *testing.T) {
 	urlStr := "https://git.io/vFR5M"
-	opt := options.New(makelink.StyleMarkdown, history.NewFile(1, ""))
+	opt := options.New(makelink.StyleMarkdown, history.NewFile(1, ""), "")
 	rRes, err := opt.MakeLink(context.Background(), urlStr)
 	if err != nil {
 		t.Errorf("Error in Context.MakeLink(): %+v", err)
@@ -37,7 +37,7 @@ func TestMakeLink(t *testing.T) {
 }
 
 func TestMakeLinkNil(t *testing.T) {
-	rRes, err := options.New(makelink.StyleMarkdown, nil).MakeLink(context.Background(), "https://git.io/vFR5M")
+	rRes, err := options.New(makelink.StyleMarkdown, nil, "").MakeLink(context.Background(), "https://git.io/vFR5M")
 	if err != nil {
 		t.Errorf("Error in Context.MakeLink(): %+v", err)
 	}
@@ -54,7 +54,7 @@ func TestMakeLinkNil(t *testing.T) {
 }
 
 func TestMakeLinkErr(t *testing.T) {
-	_, err := options.New(makelink.StyleMarkdown, nil).MakeLink(context.Background(), "https://foo.bar")
+	_, err := options.New(makelink.StyleMarkdown, nil, "").MakeLink(context.Background(), "https://foo.bar")
 	if err == nil {
 		t.Error("Context.MakeLink() = nil error, not want nil error.")
 	} else {

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/mattn/go-encoding v0.0.2
 	github.com/nyaosorg/go-readline-ny v0.7.0
 	github.com/spf13/cobra v1.4.0
-	golang.org/x/net v0.0.0-20220225172249-27dd8689420f
+	golang.org/x/net v0.0.0-20220325170049-de3da57026de
 )
 
 require (

go.sum

@@ -31,8 +31,8 @@ github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220325170049-de3da57026de h1:pZB1TWnKi+o4bENlbzAgLrEbY4RMYmUIRobMcSmfeYc=
+golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

makelink/makelink.go

@@ -27,13 +27,20 @@ type Link struct {
 }
 
 //New returns new Link instance
-func New(ctx context.Context, urlStr string) (*Link, error) {
+func New(ctx context.Context, urlStr, userAgent string) (*Link, error) {
 	link := &Link{URL: urlStr}
 	u, err := fetch.URL(urlStr)
 	if err != nil {
 		return link, errs.Wrap(err, errs.WithContext("url", urlStr))
 	}
-	resp, err := fetch.New(fetch.WithHTTPClient(&http.Client{})).Get(u, fetch.WithContext(ctx))
+	if len(userAgent) == 0 {
+		userAgent = "goark/ml (+https://github.com/goark/ml)" //dummy user-agent string
+	}
+	resp, err := fetch.New(fetch.WithHTTPClient(&http.Client{})).Get(
+		u,
+		fetch.WithContext(ctx),
+		fetch.WithRequestHeaderSet("User-Agent", userAgent),
+	)
 	if err != nil {
 		return link, errs.Wrap(err, errs.WithContext("url", urlStr))
 	}

makelink/makelink_test.go

@@ -66,7 +66,7 @@ func TestString(t *testing.T) {
 }
 
 func TestNewErr(t *testing.T) {
-	_, err := New(context.Background(), "https://foo.bar")
+	_, err := New(context.Background(), "https://foo.bar", "")
 	if err == nil {
 		t.Error("New()  = nil error, not want nil error.")
 	} else {
@@ -75,7 +75,7 @@ func TestNewErr(t *testing.T) {
 }
 
 func ExampleNew() {
-	link, err := New(context.Background(), "https://git.io/vFR5M")
+	link, err := New(context.Background(), "https://git.io/vFR5M", "")
 	if err != nil {
 		fmt.Println(err)
 		return

makelink/sample/sample.go

@@ -13,7 +13,7 @@ import (
 )
 
 func main() {
-	lnk, err := makelink.New(context.Background(), "https://git.io/vFR5M")
+	lnk, err := makelink.New(context.Background(), "https://git.io/vFR5M", "")
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err)
 		return