Refactor scanner into a package (#24)

* Refactor into a package

* More tweaks

e2e_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/unmango/go-make"
 	"github.com/unmango/go-make/printer"
+	"github.com/unmango/go-make/scanner"
 	"github.com/unmango/go-make/token"
 	"github.com/unmango/go-make/writer"
 )
@@ -26,7 +27,7 @@ var _ = Describe("E2E", func() {
 		fi, err := f.Stat()
 		Expect(err).NotTo(HaveOccurred())
 		file := token.NewFileSet().AddFile(f.Name(), 1, int(fi.Size()))
-		s := make.NewScanner(f, file)
+		s := scanner.New(f, file)
 
 		// By tweaking the duration and interval we can approximate the number of tokens
 		// scanned and pick values that should cover the entire Makefile. This approach

make.go

@@ -1,125 +1,10 @@
 package make
 
-const (
-	DefineDirective      = "define"
-	EndefDirective       = "endef"
-	UndefineDirective    = "undefine"
-	IfdefDirective       = "ifdef"
-	IfndefDirective      = "ifndef"
-	IfeqDirective        = "ifeq"
-	IfneqDirective       = "ifneq"
-	ElseDirective        = "else"
-	EndifDirective       = "endif"
-	IncludeDirective     = "include"
-	DashIncludeDirective = "-include"
-	SincludeDirective    = "sinclude"
-	OverrideDirective    = "override"
-	ExportDirective      = "export"
-	UnexportDirective    = "unexport"
-	PrivateDirective     = "private"
-	VpathDirective       = "vpath"
-)
-
-var Directives = []string{
-	DefineDirective,
-	EndefDirective,
-	UndefineDirective,
-	IfdefDirective,
-	IfndefDirective,
-	IfeqDirective,
-	IfneqDirective,
-	ElseDirective,
-	EndifDirective,
-	IncludeDirective,
-	DashIncludeDirective,
-	SincludeDirective,
-	OverrideDirective,
-	ExportDirective,
-	UndefineDirective,
-	PrivateDirective,
-	VpathDirective,
-}
-
-const (
-	SubstFunction      = "subst"
-	PatsubstFunction   = "patsubst"
-	StripFunction      = "strip"
-	FindstringFunction = "findstring"
-	FilterFunction     = "filter"
-	FilterOutFunction  = "filter-out"
-	SortFunction       = "sort"
-	WordFunction       = "word"
-	WordsFunction      = "words"
-	WordlistFunction   = "wordlist"
-	FirstwordFunction  = "firstword"
-	LastwordFunction   = "lastword"
-	DirFunction        = "dir"
-	NotdirFunction     = "notdir"
-	SuffixFunction     = "suffix"
-	BasenameFunction   = "basename"
-	AddsuffixFunction  = "addsuffix"
-	AddprefixFunction  = "addprefix"
-	JoinFunction       = "join"
-	WildcardFunction   = "wildcard"
-	RealpathFunction   = "realpath"
-	AbspathFunction    = "abspath"
-	ErrorFunction      = "error"
-	WarningFunction    = "warning"
-	ShellFunction      = "shell"
-	OriginFunction     = "origin"
-	FlavorFunction     = "flavor"
-	LetFunction        = "let"
-	ForeachFunction    = "foreach"
-	IfFunction         = "if"
-	OrFunction         = "or"
-	AndFunction        = "and"
-	IntcmpFunction     = "intcmp"
-	CallFunction       = "call"
-	EvalFunction       = "eval"
-	FileFunction       = "file"
-	ValueFunction      = "value"
+import (
+	"github.com/unmango/go-make/scanner"
+	"github.com/unmango/go-make/writer"
 )
 
-var BuiltinFunctions = []string{
-	SubstFunction,
-	PatsubstFunction,
-	StripFunction,
-	FindstringFunction,
-	FilterFunction,
-	FilterOutFunction,
-	SortFunction,
-	WordFunction,
-	WordsFunction,
-	WordlistFunction,
-	FirstwordFunction,
-	LastwordFunction,
-	DirFunction,
-	NotdirFunction,
-	SuffixFunction,
-	BasenameFunction,
-	AddsuffixFunction,
-	AddprefixFunction,
-	JoinFunction,
-	WildcardFunction,
-	RealpathFunction,
-	AbspathFunction,
-	ErrorFunction,
-	WarningFunction,
-	ShellFunction,
-	OriginFunction,
-	FlavorFunction,
-	LetFunction,
-	ForeachFunction,
-	IfFunction,
-	OrFunction,
-	AndFunction,
-	IntcmpFunction,
-	CallFunction,
-	EvalFunction,
-	FileFunction,
-	ValueFunction,
-}
-
 const (
 	MakefilesVariable    = "MAKEFILES"
 	VpathVariable        = "VPATH"
@@ -192,18 +77,7 @@ var BuiltinTargets = []string{
 	PosixTarget,
 }
 
-type (
-	Target string
-	PreReq string
-	Recipe string
+var (
+	NewScanner = scanner.New
+	NewWriter  = writer.New
 )
-
-type Rule struct {
-	Target  []string
-	PreReqs []string
-	Recipe  []string
-}
-
-type Makefile struct {
-	Rules []Rule
-}

parser.go

@@ -1,17 +1,17 @@
 package make
 
 import (
-	"go/scanner"
 	"io"
 	"math"
 	"strings"
 
 	"github.com/unmango/go-make/ast"
+	"github.com/unmango/go-make/scanner"
 	"github.com/unmango/go-make/token"
 )
 
 type Parser struct {
-	s      *Scanner
+	s      *scanner.Scanner
 	file   *token.File
 	errors scanner.ErrorList
 
@@ -28,7 +28,7 @@ func NewParser(r io.Reader, file *token.File) *Parser {
 	}
 
 	p := &Parser{
-		s:    NewScanner(r, file),
+		s:    scanner.New(r, file),
 		file: file,
 
 		recipePrefix: token.TAB,

scan.go → scanner/scan.go

@@ -1,4 +1,4 @@
-package make
+package scanner
 
 import (
 	"bytes"

scan_test.go → scanner/scan_test.go

@@ -1,4 +1,4 @@
-package make_test
+package scanner_test
 
 import (
 	"bufio"
@@ -7,7 +7,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
-	"github.com/unmango/go-make"
+	"github.com/unmango/go-make/scanner"
 )
 
 var _ = Describe("Scan", func() {
@@ -95,7 +95,7 @@ var _ = Describe("Scan", func() {
 			func(text string, expected []string) {
 				buf := bytes.NewBufferString(text)
 				s := bufio.NewScanner(buf)
-				s.Split(make.ScanTokens)
+				s.Split(scanner.ScanTokens)
 
 				tokens := []string{}
 				for s.Scan() {

scanner.go → scanner/scanner.go

@@ -1,14 +1,17 @@
-package make
+package scanner
 
 import (
 	"bufio"
 	"bytes"
+	"go/scanner"
 	"io"
 	"math"
 
 	"github.com/unmango/go-make/token"
 )
 
+type ErrorList = scanner.ErrorList
+
 type Scanner struct {
 	file *token.File
 	s    *bufio.Scanner
@@ -21,7 +24,7 @@ type Scanner struct {
 	done bool
 }
 
-func NewScanner(r io.Reader, file *token.File) *Scanner {
+func New(r io.Reader, file *token.File) *Scanner {
 	scanner := bufio.NewScanner(r)
 	scanner.Split(ScanTokens)
 

scanner/scanner_suite_test.go

@@ -0,0 +1,13 @@
+package scanner_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestScanner(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Scanner Suite")
+}

scanner_test.go → scanner/scanner_test.go

@@ -1,4 +1,4 @@
-package make_test
+package scanner_test
 
 import (
 	"bytes"
@@ -10,8 +10,8 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
-	"github.com/unmango/go-make"
 	"github.com/unmango/go-make/internal/testing"
+	"github.com/unmango/go-make/scanner"
 	"github.com/unmango/go-make/token"
 )
 
@@ -24,7 +24,7 @@ var _ = Describe("Scanner", func() {
 
 	Describe("Position", func() {
 		It("should be equivalent to calling file.PositionFor(p, false)", func() {
-			s := make.NewScanner(&bytes.Buffer{}, file)
+			s := scanner.New(&bytes.Buffer{}, file)
 
 			err := quick.Check(func(p int) bool {
 				pos := token.Pos(p)
@@ -51,7 +51,7 @@ var _ = Describe("Scanner", func() {
 		Entry(nil, "foo_"),
 		func(input string) {
 			buf := bytes.NewBufferString(input)
-			s := make.NewScanner(buf, file)
+			s := scanner.New(buf, file)
 
 			pos, tok, lit := s.Scan()
 			Expect(tok).To(Equal(token.TEXT))
@@ -76,7 +76,7 @@ var _ = Describe("Scanner", func() {
 		Entry(nil, "foo_\n"),
 		func(input string) {
 			buf := bytes.NewBufferString(input)
-			s := make.NewScanner(buf, file)
+			s := scanner.New(buf, file)
 
 			pos, tok, lit := s.Scan()
 			Expect(tok).To(Equal(token.TEXT))
@@ -119,7 +119,7 @@ var _ = Describe("Scanner", func() {
 		Entry(nil, "ident ,"),
 		func(input string) {
 			buf := bytes.NewBufferString(input)
-			s := make.NewScanner(buf, file)
+			s := scanner.New(buf, file)
 
 			pos, tok, lit := s.Scan()
 			Expect(tok).To(Equal(token.TEXT))
@@ -153,7 +153,7 @@ var _ = Describe("Scanner", func() {
 		Entry(nil, "\n\n", token.NEWLINE),
 		func(input string, expected token.Token) {
 			buf := bytes.NewBufferString(input)
-			s := make.NewScanner(buf, file)
+			s := scanner.New(buf, file)
 
 			pos, tok, _ := s.Scan()
 			Expect(tok).To(Equal(expected))
@@ -168,7 +168,7 @@ var _ = Describe("Scanner", func() {
 		Entry(nil, "#", token.COMMENT),
 		func(input string, expected token.Token) {
 			buf := bytes.NewBufferString(input)
-			s := make.NewScanner(buf, file)
+			s := scanner.New(buf, file)
 
 			pos, tok, _ := s.Scan()
 
@@ -197,7 +197,7 @@ var _ = Describe("Scanner", func() {
 		Entry(nil, "\n)"),
 		func(input string) {
 			buf := bytes.NewBufferString(input)
-			s := make.NewScanner(buf, file)
+			s := scanner.New(buf, file)
 
 			pos, tok, _ := s.Scan()
 			Expect(tok).To(Equal(token.NEWLINE))
@@ -257,7 +257,7 @@ var _ = Describe("Scanner", func() {
 		Entry(nil, "identifier foo bar", 12),
 		func(input string, expected int) {
 			buf := bytes.NewBufferString(input)
-			s := make.NewScanner(buf, file)
+			s := scanner.New(buf, file)
 
 			_, _, _ = s.Scan()
 			pos, tok, lit := s.Scan()
@@ -293,7 +293,7 @@ var _ = Describe("Scanner", func() {
 		Entry(nil, "identifier\nfoo", 11),
 		func(input string, nlPos int) {
 			buf := bytes.NewBufferString(input)
-			s := make.NewScanner(buf, file)
+			s := scanner.New(buf, file)
 
 			_, _, _ = s.Scan()
 			pos, tok, _ := s.Scan()
@@ -321,15 +321,15 @@ var _ = Describe("Scanner", func() {
 
 	It("should return IO errors", func() {
 		r := testing.ErrReader("io error")
-		s := make.NewScanner(r, file)
+		s := scanner.New(r, file)
 
 		_, _, _ = s.Scan()
 		Expect(s.Err()).To(MatchError("io error"))
 	})
 
 	It("should support a nil *token.File value", func() {
 		buf := bytes.NewBufferString("target:")
-		s := make.NewScanner(buf, nil)
+		s := scanner.New(buf, nil)
 
 		pos, tok, lit := s.Scan()
 		Expect(tok).To(Equal(token.TEXT))