Implement all text extractor

main.go

@@ -13,6 +13,7 @@ import (
 
 func main() {
 	splitPtr := flag.String("split", "word", "a string")
+	extractorPtr := flag.String("extractor", "shallow", "a string")
 	flag.Parse()
 
 	file, err := os.Open("sample/" + os.Args[len(os.Args)-1])
@@ -21,12 +22,17 @@ func main() {
 	}
 
 	var ex boilertext.Extractor
-	if *splitPtr == "word" {
-		ex = extractor.NewShallowTextExtractor(bufio.ScanWords)
-	} else if *splitPtr == "rune" {
-		ex = extractor.NewShallowTextExtractor(bufio.ScanRunes)
+	if *extractorPtr == "shallow" {
+		if *splitPtr == "word" {
+			ex = extractor.NewShallowTextExtractor(bufio.ScanWords)
+		} else if *splitPtr == "rune" {
+			ex = extractor.NewShallowTextExtractor(bufio.ScanRunes)
+		} else {
+			log.Fatal("Missing split argument")
+		}
 	} else {
-		log.Fatal("Missing split argument")
+		// Returns all text
+		ex = &extractor.AllTextExtractor{}
 	}
 	res, err := ex.Process(file)
 	if err != nil {

pkg/extractor/alltext.go

@@ -0,0 +1,27 @@
+package extractor
+
+import (
+	"bufio"
+	"io"
+	"strings"
+
+	boilertext "github.com/PageDash/boilertext/pkg"
+)
+
+// AllTextExtractor returns all text in the document.
+type AllTextExtractor struct{}
+
+// Process takes raw HTML as an input and returns all text within that HTML.
+func (a *AllTextExtractor) Process(reader io.Reader) (string, error) {
+	blocks, err := boilertext.GenerateTextBlocks(reader, bufio.ScanWords)
+	if err != nil {
+		return "", err
+	}
+
+	var contentText string
+	for _, block := range blocks {
+		contentText += strings.TrimSpace(block.Content) + " "
+	}
+
+	return contentText, nil
+}