common_token_factory.go

// Copyright (c) 2012-2022 The ANTLR Project. All rights reserved.
// Use of this file is governed by the BSD 3-clause license that
// can be found in the LICENSE.txt file in the project root.

package antlr

// TokenFactory creates CommonToken objects.
type TokenFactory interface {
	Create(source *TokenSourceCharStreamPair, ttype int, text string, channel, start, stop, line, column int) Token
}

// CommonTokenFactory is the default TokenFactory implementation.
type CommonTokenFactory struct {
	// copyText indicates whether CommonToken.setText should be called after
	// constructing tokens to explicitly set the text. This is useful for cases
	// where the input stream might not be able to provide arbitrary substrings of
	// text from the input after the lexer creates a token (e.g. the
	// implementation of CharStream.GetText in UnbufferedCharStream panics an
	// UnsupportedOperationException). Explicitly setting the token text allows
	// Token.GetText to be called at any time regardless of the input stream
	// implementation.
	//
	// The default value is false to avoid the performance and memory overhead of
	// copying text for every token unless explicitly requested.
	copyText bool
}

func NewCommonTokenFactory(copyText bool) *CommonTokenFactory {
	return &CommonTokenFactory{copyText: copyText}
}

// CommonTokenFactoryDEFAULT is the default CommonTokenFactory. It does not
// explicitly copy token text when constructing tokens.
var CommonTokenFactoryDEFAULT = NewCommonTokenFactory(false)

func (c *CommonTokenFactory) Create(source *TokenSourceCharStreamPair, ttype int, text string, channel, start, stop, line, column int) Token {
	t := NewCommonToken(source, ttype, channel, start, stop)

	t.line = line
	t.column = column

	if text != "" {
		t.SetText(text)
	} else if c.copyText && source.charStream != nil {
		t.SetText(source.charStream.GetTextFromInterval(NewInterval(start, stop)))
	}

	return t
}

func (c *CommonTokenFactory) createThin(ttype int, text string) Token {
	t := NewCommonToken(nil, ttype, TokenDefaultChannel, -1, -1)
	t.SetText(text)

	return t
}