-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #23 from alllex/scannerless-parsing
Scannerless parsing
- Loading branch information
Showing
17 changed files
with
436 additions
and
96 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
41 changes: 41 additions & 0 deletions
41
src/commonMain/kotlin/me/alllex/parsus/tokenizer/AbstractTokenizer.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| package me.alllex.parsus.tokenizer | ||
|
|
||
| import me.alllex.parsus.annotations.ExperimentalParsusApi | ||
| import me.alllex.parsus.token.Token | ||
| import me.alllex.parsus.token.TokenMatch | ||
| import me.alllex.parsus.trace.TokenMatchingEvent | ||
| import me.alllex.parsus.trace.TokenMatchingTrace | ||
|
|
||
| @OptIn(ExperimentalParsusApi::class) | ||
| internal abstract class AbstractTokenizer( | ||
| override val input: String, | ||
| protected val tokens: List<Token>, | ||
| traceTokenMatching: Boolean = false, | ||
| ) : Tokenizer { | ||
|
|
||
| private val traceEvents: MutableList<TokenMatchingEvent>? = if (traceTokenMatching) mutableListOf() else null | ||
|
|
||
| override fun getTokenMatchingTrace(): TokenMatchingTrace? { | ||
| return traceEvents?.let { TokenMatchingTrace(input, it) } | ||
| } | ||
|
|
||
| protected fun matchImpl(fromIndex: Int, token: Token): TokenMatch? { | ||
| val length = token.match(input, fromIndex) | ||
| if (length == 0) { | ||
| traceMismatch(token, fromIndex) | ||
| return null | ||
| } | ||
|
|
||
| val match = TokenMatch(token, fromIndex, length) | ||
| traceMatch(token, match) | ||
| return match | ||
| } | ||
|
|
||
| private fun traceMismatch(token: Token, offset: Int) { | ||
| traceEvents?.add(TokenMatchingEvent(token, offset, null)) | ||
| } | ||
|
|
||
| private fun traceMatch(token: Token, match: TokenMatch) { | ||
| traceEvents?.add(TokenMatchingEvent(token, match.offset, match)) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
92 changes: 92 additions & 0 deletions
92
src/commonMain/kotlin/me/alllex/parsus/tokenizer/ScannerlessTokenizer.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| package me.alllex.parsus.tokenizer | ||
|
|
||
| import me.alllex.parsus.token.Token | ||
| import me.alllex.parsus.token.TokenMatch | ||
|
|
||
| /** | ||
| * Scannerless tokenizer tries to parse the target token at the give position. | ||
| * | ||
| * It treats the target token as having higher priority than all other tokens. | ||
| */ | ||
| internal class ScannerlessTokenizer( | ||
| input: String, | ||
| tokens: List<Token>, | ||
| traceTokenMatching: Boolean = false, | ||
| ) : AbstractTokenizer(input, tokens, traceTokenMatching) { | ||
|
|
||
| private val ignoredTokens = tokens.filter { it.ignored } | ||
|
|
||
| // We cache one mismatch and one match of ignored tokens. | ||
| // This is for the frequent case, when there is exactly one ignored token before the target token. | ||
| // Example: | ||
| // parser = t1 or t2 or t3, ws = ignored whitespace | ||
| // input = " t3" | ||
| // In this example, t1 will fail to match at 0, but ws will match at 0, so we cache the match. | ||
| // Then t1 will try to match at 1, but it will fail again, so we try ignored tokens again, | ||
| // but this time we get a mismatch, which we cache separately. This fails the t1 branch of the parser. | ||
| // Now, we backtrack and try t2 at 0, which fails. | ||
| // But we can avoid rematching ws at 0, because we cached this match. | ||
| // Then we try t2 at position 1, which fails. But we don't retry ws, because we cached the mismatch. | ||
| // In the last t3 branch, we try t3 at 0, which fails, but then we skip rematching ws at 0, | ||
| // because it is still cached. Then t3 succeeds at 0, and parsing succeeds. | ||
| private var cacheIgnoredMismatchFromIndex = -1 | ||
| private var cachedIgnoredFromIndex: Int = -1 | ||
| private var cachedIgnoredTokenMatch: TokenMatch? = null | ||
|
|
||
| override fun findContextFreeMatch(fromIndex: Int): TokenMatch? = null | ||
|
|
||
| override fun findMatchOf(fromIndex: Int, targetToken: Token): TokenMatch? { | ||
| var pos = fromIndex | ||
| while (true) { | ||
| matchTarget(pos, targetToken)?.let { return it } | ||
|
|
||
| val ignoredMatch = matchIgnored(pos) | ||
| @Suppress("LiftReturnOrAssignment") | ||
| if (ignoredMatch != null) { | ||
| val posAfterIgnored = ignoredMatch.offset + ignoredMatch.length | ||
| if (posAfterIgnored > pos) { | ||
| pos = posAfterIgnored | ||
| continue | ||
| } else { | ||
| // An ignored token matched, but it did not advance the position. | ||
| // This should not happen normally, but this is a safeguard. | ||
| return null | ||
| } | ||
| } else { | ||
| // No ignored tokens matched at the current position either, | ||
| // so it is a mismatch overall | ||
| return null | ||
| } | ||
| } | ||
| // The loop will exit via a mismatch, because no tokens can match "after the end of input" | ||
| } | ||
|
|
||
| private fun matchIgnored(fromIndex: Int): TokenMatch? { | ||
| require(fromIndex >= 0) { "fromIndex must be non-negative, but was $fromIndex" } | ||
|
|
||
| if (fromIndex == cacheIgnoredMismatchFromIndex) { | ||
| return null | ||
| } | ||
| if (fromIndex == cachedIgnoredFromIndex) { | ||
| return cachedIgnoredTokenMatch | ||
| } | ||
|
|
||
| var match: TokenMatch? = null | ||
| for (ignoredToken in ignoredTokens) { | ||
| match = matchImpl(fromIndex, ignoredToken) | ||
| if (match != null) { | ||
| break | ||
| } | ||
| } | ||
|
|
||
| if (match == null) { | ||
| cacheIgnoredMismatchFromIndex = fromIndex | ||
| } else { | ||
| cachedIgnoredFromIndex = fromIndex | ||
| cachedIgnoredTokenMatch = match | ||
| } | ||
| return match | ||
| } | ||
|
|
||
| private fun matchTarget(pos: Int, targetToken: Token) = matchImpl(pos, targetToken) | ||
| } |
Oops, something went wrong.