Few things

staticaland · Jul 29, 2022 · 5d88d6d · 5d88d6d
1 parent 27c1308
commit 5d88d6d
Show file tree

Hide file tree

Showing 5 changed files with 38 additions and 8 deletions.
diff --git a/.goreleaser.yaml b/.goreleaser.yaml
@@ -1,6 +1,8 @@
 brews:
   -
     name: whaturl
+    description: "A CLI for creating titled markup language links out of text containing raw URLs"
+    homepage: "https://github.com/staticaland/go-whaturl"
 
     goarm: 6
 
@@ -11,3 +13,6 @@ brews:
       name: go-whaturl
 
     folder: Formula
+
+    install: |
+      bin.install "whaturl"
diff --git a/README.org b/README.org
@@ -14,3 +14,19 @@ A Homebrew formula is included at [[./Formula/whaturl.rb][Formula/whaturl.rb]].
 brew tap staticaland/go-whaturl https://github.com/staticaland/go-whaturl
 brew install whaturl
 #+end_src
+
+* Build and run
+
+#+begin_src sh
+go build whaturl.go
+#+end_src
+
+#+begin_src sh
+go run whaturl.go
+#+end_src
+
+* Test
+
+#+begin_src sh
+go test
+#+end_src
diff --git a/go.mod b/go.mod
@@ -4,7 +4,10 @@ go 1.18
 
 require (
 	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 // indirect
+	golang.org/x/text v0.3.6 // indirect
 )
diff --git a/go.sum b/go.sum
@@ -1,5 +1,9 @@
 github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
 github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
@@ -9,5 +13,6 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
diff --git a/whaturl.go b/whaturl.go
@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
+	"github.com/PuerkitoBio/purell"
 	"github.com/google/uuid"
 )
 
@@ -29,7 +30,7 @@ func GetTitle(url string) string {
 		log.Fatalln(err)
 	}
 
-	req.Header.Set("User-Agent", "Golang_Whaturl/1.0")
+	req.Header.Set("User-Agent", "")
 
 	resp, err := client.Do(req)
 	if err != nil {
@@ -39,7 +40,7 @@ func GetTitle(url string) string {
 	defer resp.Body.Close()
 
 	if resp.StatusCode != 200 {
-		return "Error fetching link"
+		return url
 	}
 
 	doc, err := goquery.NewDocumentFromReader(resp.Body)
@@ -48,7 +49,7 @@ func GetTitle(url string) string {
 		log.Fatal(err)
 	}
 
-	title := doc.Find("title").Text()
+	title := doc.Find("title").First().Text()
 
 	return title
 
@@ -74,9 +75,8 @@ func main() {
 	var linkFormat string
 	flag.StringVar(&linkFormat, "format", "markdown", "Specify link format")
 	flag.Parse()
-	fmt.Println(linkFormat)
 
-	re := regexp.MustCompile(`(?i)(?:\[(?P<title>[^\]]*)\]\(|\b)(?P<url>(?:(?:https?)://|(?:www)\.)[-A-Z0-9+&@/%?=~_|$!:,.;]*[A-Z0-9+&@#/%=~_|$])\)?`)
+	re := regexp.MustCompile(`(?i)\b(?:[a-z][\w.+-]+:(?:/{1,3}|[?+]?[a-z0-9%]))(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s\x60!()\[\]{};:'".,<>?«»“”‘’])`)
 
 	scanner := bufio.NewScanner(os.Stdin)
 
@@ -90,17 +90,18 @@ func main() {
 
 		for _, url := range matches {
 
-			id := uuid.New()
+			id := uuid.New().String()
 
-			l := link{url: url, id: id.String()}
+			l := link{url: url, id: id}
 
 			matchesEnriched = append(matchesEnriched, l)
 
 		}
 
 		for _, link := range matchesEnriched {
 			title := GetTitle(link.url)
-			s = strings.Replace(s, link.url, CreateLink(link.url, title, linkFormat), -1)
+			normalized_url, _ := purell.NormalizeURLString(link.url, purell.FlagsUsuallySafeGreedy|purell.FlagRemoveDuplicateSlashes|purell.FlagRemoveFragment)
+			s = strings.Replace(s, link.url, CreateLink(normalized_url, title, linkFormat), -1)
 		}
 
 		fmt.Println(s)