Skip to content

Commit

Permalink
Rename tokenizers that are actually parsers (#298)
Browse files Browse the repository at this point in the history
  • Loading branch information
@atifaziz
atifaziz authored Oct 29, 2024
1 parent 58f7da6 commit 088cf99
Show file tree
Hide file tree
Showing 6 changed files with 40 additions and 40 deletions.
14 changes: 7 additions & 7 deletions src/CSnakes.SourceGeneration/Parser/PythonParser.Args.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,26 @@
namespace CSnakes.Parser;
public static partial class PythonParser
{
public static TokenListParser<PythonToken, PythonParameterType> PythonStarArgTokenizer { get; } =
public static TokenListParser<PythonToken, PythonParameterType> PythonStarArgParser { get; } =
(from star in Token.EqualTo(PythonToken.Asterisk)
from name in Token.EqualTo(PythonToken.Identifier).Optional()
select new PythonParameterType(name.HasValue ? name.Value.ToStringValue() : "args", PythonFunctionParameterType.Star))
.Named("Star Arg");

public static TokenListParser<PythonToken, PythonParameterType> PythonDoubleStarArgTokenizer { get; } =
public static TokenListParser<PythonToken, PythonParameterType> PythonDoubleStarArgParser { get; } =
(from star in Token.EqualTo(PythonToken.DoubleAsterisk)
from name in Token.EqualTo(PythonToken.Identifier)
select new PythonParameterType(name.ToStringValue(), PythonFunctionParameterType.DoubleStar))
.Named("Double Star Arg");

public static TokenListParser<PythonToken, PythonParameterType> PythonNormalArgTokenizer { get; } =
public static TokenListParser<PythonToken, PythonParameterType> PythonNormalArgParser { get; } =
(from name in Token.EqualTo(PythonToken.Identifier)
select new PythonParameterType(name.ToStringValue(), PythonFunctionParameterType.Normal))
.Named("Normal Arg");

public static TokenListParser<PythonToken, PythonParameterType?> PythonArgTokenizer { get; } =
PythonStarArgTokenizer.AsNullable()
.Or(PythonDoubleStarArgTokenizer.AsNullable())
.Or(PythonNormalArgTokenizer.AsNullable())
public static TokenListParser<PythonToken, PythonParameterType?> PythonArgParser { get; } =
PythonStarArgParser.AsNullable()
.Or(PythonDoubleStarArgParser.AsNullable())
.Or(PythonNormalArgParser.AsNullable())
.Named("Arg");
}
8 changes: 4 additions & 4 deletions src/CSnakes.SourceGeneration/Parser/PythonParser.Function.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@
namespace CSnakes.Parser;
public static partial class PythonParser
{
public static TokenListParser<PythonToken, PythonFunctionDefinition> PythonFunctionDefinitionTokenizer { get; } =
public static TokenListParser<PythonToken, PythonFunctionDefinition> PythonFunctionDefinitionParser { get; } =
(from def in Token.EqualTo(PythonToken.Def)
from name in Token.EqualTo(PythonToken.Identifier)
from parameters in PythonParameterListTokenizer.AssumeNotNull()
from arrow in Token.EqualTo(PythonToken.Arrow).Optional().Then(returnType => PythonTypeDefinitionTokenizer.AssumeNotNull().OptionalOrDefault())
from parameters in PythonParameterListParser.AssumeNotNull()
from arrow in Token.EqualTo(PythonToken.Arrow).Optional().Then(returnType => PythonTypeDefinitionParser.AssumeNotNull().OptionalOrDefault())
from colon in Token.EqualTo(PythonToken.Colon)
select new PythonFunctionDefinition(name.ToStringValue(), arrow, parameters))
.Named("Function Definition");
Expand Down Expand Up @@ -90,7 +90,7 @@ public static bool TryParseFunctionDefinitions(SourceText source, out PythonFunc
foreach ((IEnumerable<TextLine> currentLines, ParsedTokens tokens) in functionLines)
{
TokenListParserResult<PythonToken, PythonFunctionDefinition> functionDefinition =
PythonFunctionDefinitionTokenizer.TryParse(tokens);
PythonFunctionDefinitionParser.TryParse(tokens);
if (functionDefinition.HasValue)
{
functionDefinitions.Add(functionDefinition.Value);
Expand Down
14 changes: 7 additions & 7 deletions src/CSnakes.SourceGeneration/Parser/PythonParser.Parameters.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,25 +10,25 @@ public static partial class PythonParser
.Select(d => new PythonFunctionParameter("/", null, null, PythonFunctionParameterType.Slash))
.Named("Positional Only Signal");

public static TokenListParser<PythonToken, PythonFunctionParameter> PythonParameterTokenizer { get; } =
(from arg in PythonArgTokenizer
public static TokenListParser<PythonToken, PythonFunctionParameter> PythonParameterParser { get; } =
(from arg in PythonArgParser
from type in Token.EqualTo(PythonToken.Colon).Optional().Then(
_ => PythonTypeDefinitionTokenizer.AssumeNotNull().OptionalOrDefault()
_ => PythonTypeDefinitionParser.AssumeNotNull().OptionalOrDefault()
)
from defaultValue in Token.EqualTo(PythonToken.Equal).Optional().Then(
_ => ConstantValueTokenizer.AssumeNotNull().OptionalOrDefault()
)
select new PythonFunctionParameter(arg.Name, type, defaultValue, arg.ParameterType))
.Named("Parameter");

public static TokenListParser<PythonToken, PythonFunctionParameter?> ParameterOrSlash { get; } =
public static TokenListParser<PythonToken, PythonFunctionParameter?> ParameterOrSlashParser { get; } =
PositionalOnlyParameterParser.AsNullable()
.Or(PythonParameterTokenizer.AsNullable())
.Or(PythonParameterParser.AsNullable())
.Named("Parameter");

public static TokenListParser<PythonToken, PythonFunctionParameter[]> PythonParameterListTokenizer { get; } =
public static TokenListParser<PythonToken, PythonFunctionParameter[]> PythonParameterListParser { get; } =
(from openParen in Token.EqualTo(PythonToken.OpenParenthesis)
from parameters in ParameterOrSlash.ManyDelimitedBy(Token.EqualTo(PythonToken.Comma))
from parameters in ParameterOrSlashParser.ManyDelimitedBy(Token.EqualTo(PythonToken.Comma))
from closeParen in Token.EqualTo(PythonToken.CloseParenthesis)
select parameters)
.Named("Parameter List");
Expand Down
4 changes: 2 additions & 2 deletions src/CSnakes.SourceGeneration/Parser/PythonParser.TypeDef.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ public static partial class PythonParser
.AtLeastOnceDelimitedBy(Character.EqualTo('.'))
);

public static TokenListParser<PythonToken, PythonTypeSpec?> PythonTypeDefinitionTokenizer { get; } =
public static TokenListParser<PythonToken, PythonTypeSpec?> PythonTypeDefinitionParser { get; } =
(from name in Token.EqualTo(PythonToken.Identifier).Or(Token.EqualTo(PythonToken.None)).OptionalOrDefault()
#pragma warning disable CS8620
from openBracket in Token.EqualTo(PythonToken.OpenBracket)
.Then(_ =>
PythonTypeDefinitionTokenizer
PythonTypeDefinitionParser
.AssumeNotNull()
.ManyDelimitedBy(
Token.EqualTo(PythonToken.Comma),
Expand Down
38 changes: 19 additions & 19 deletions src/CSnakes.Tests/TokenizerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ public void TokenizeWithDefaultValue()
public void ParseFunctionParameter(string code, string expectedName, string expectedType)
{
var tokens = PythonTokenizer.Instance.Tokenize(code);
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal(expectedName, result.Value.Name);
Assert.Equal(expectedType, result.Value.Type.ToString());
Expand All @@ -100,7 +100,7 @@ public void ParseFunctionParameter(string code, string expectedName, string expe
public void ParseFunctionParameterNoType()
{
var tokens = PythonTokenizer.Instance.Tokenize("a");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.False(result.Value.HasTypeAnnotation());
Expand All @@ -110,7 +110,7 @@ public void ParseFunctionParameterNoType()
public void ParseFunctionParameterDefault()
{
var tokens = PythonTokenizer.Instance.Tokenize("a = 1");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal("1", result.Value.DefaultValue?.ToString());
Expand All @@ -125,7 +125,7 @@ public void ParseFunctionParameterDefault()
public void ParseFunctionParameterDefaultValuesNoType(string value)
{
var tokens = PythonTokenizer.Instance.Tokenize($"a = {value}");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal(value, result.Value.DefaultValue?.ToString());
Expand All @@ -136,7 +136,7 @@ public void ParseFunctionParameterDefaultValuesNoType(string value)
public void ParseFunctionParameterDefaultSingleQuotedString()
{
var tokens = PythonTokenizer.Instance.Tokenize($"a = 'hello'");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal("hello", result.Value.DefaultValue?.ToString());
Expand All @@ -147,7 +147,7 @@ public void ParseFunctionParameterDefaultSingleQuotedString()
public void ParseFunctionParameterDefaultDoubleQuotedString()
{
var tokens = PythonTokenizer.Instance.Tokenize($"a = \"hello\"");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal("hello", result.Value.DefaultValue?.ToString());
Expand All @@ -158,7 +158,7 @@ public void ParseFunctionParameterDefaultDoubleQuotedString()
public void ParseFunctionParameterDefaultDouble()
{
var tokens = PythonTokenizer.Instance.Tokenize($"a: float = -1.1");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal("-1.1", result.Value.DefaultValue?.ToString());
Expand All @@ -169,7 +169,7 @@ public void ParseFunctionParameterDefaultDouble()
public void ParseFunctionParameterDefaultInt()
{
var tokens = PythonTokenizer.Instance.Tokenize($"a: int = 1234");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal("1234", result.Value.DefaultValue?.ToString());
Expand All @@ -180,7 +180,7 @@ public void ParseFunctionParameterDefaultInt()
public void ParseFunctionParameterDefaultBoolTrue()
{
var tokens = PythonTokenizer.Instance.Tokenize($"a: bool = True");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal("True", result.Value.DefaultValue?.ToString());
Expand All @@ -191,7 +191,7 @@ public void ParseFunctionParameterDefaultBoolTrue()
public void ParseFunctionParameterDefaultBoolFalse()
{
var tokens = PythonTokenizer.Instance.Tokenize($"a: bool = False");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal("False", result.Value.DefaultValue?.ToString());
Expand All @@ -202,7 +202,7 @@ public void ParseFunctionParameterDefaultBoolFalse()
public void ParseFunctionParameterDefaultNone()
{
var tokens = PythonTokenizer.Instance.Tokenize($"a: bool = None");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal("None", result.Value.DefaultValue?.ToString());
Expand All @@ -214,7 +214,7 @@ public void ParseFunctionParameterListSingleGeneric()
{
var code = "(a: list[int])";
var tokens = PythonTokenizer.Instance.Tokenize(code);
var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterListParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value[0].Name);
Assert.Equal("list[int]", result.Value[0].Type.ToString());
Expand All @@ -225,7 +225,7 @@ public void ParseFunctionParameterListDoubleGeneric()
{
var code = "(a: list[int], b)";
var tokens = PythonTokenizer.Instance.Tokenize(code);
var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterListParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value[0].Name);
Assert.Equal("list[int]", result.Value[0].Type.ToString());
Expand All @@ -236,7 +236,7 @@ public void ParseFunctionParameterListQualifiedGenericType()
{
var code = "(a: typing.List[int], b)";
var tokens = PythonTokenizer.Instance.Tokenize(code);
var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterListParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value[0].Name);
Assert.Equal("typing.List[int]", result.Value[0].Type.ToString());
Expand All @@ -247,7 +247,7 @@ public void ParseFunctionParameterListQualifiedBasicType()
{
var code = "(a: np.ndarray, b)";
var tokens = PythonTokenizer.Instance.Tokenize(code);
var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterListParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value[0].Name);
Assert.Equal("np.ndarray", result.Value[0].Type.ToString());
Expand All @@ -258,7 +258,7 @@ public void ParseFunctionParameterListEasy()
{
var code = "(a: int, b: float, c: str)";
var tokens = PythonTokenizer.Instance.Tokenize(code);
var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterListParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value[0].Name);
Assert.Equal("int", result.Value[0].Type.Name);
Expand All @@ -273,7 +273,7 @@ public void ParseFunctionParameterListUntyped()
{
var code = "(a, b, c)";
var tokens = PythonTokenizer.Instance.Tokenize(code);
var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterListParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value[0].Name);
Assert.Equal("Any", result.Value[0].Type.Name);
Expand All @@ -287,7 +287,7 @@ public void ParseFunctionParameterListUntyped()
public void ParseFunctionDefinition()
{
var tokens = PythonTokenizer.Instance.Tokenize("def foo(a: int, b: str) -> None:");
var result = PythonParser.PythonFunctionDefinitionTokenizer.TryParse(tokens);
var result = PythonParser.PythonFunctionDefinitionParser.TryParse(tokens);

Assert.True(result.HasValue);
Assert.Equal("foo", result.Value.Name);
Expand All @@ -313,7 +313,7 @@ public void ParseFunctionDefinitionComplex(string code)
{
var tokens = PythonTokenizer.Instance.Tokenize(code);
Assert.True(tokens.IsAtEnd == false, "Tokenize failed");
var result = PythonParser.PythonFunctionDefinitionTokenizer.TryParse(tokens);
var result = PythonParser.PythonFunctionDefinitionParser.TryParse(tokens);
Assert.True(result.HasValue, result.ToString());
}

Expand Down
2 changes: 1 addition & 1 deletion src/CSnakes.Tests/TypeReflectionTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ public void TupleParsingTest(string pythonType, string expectedType) =>
private static void ParsingTestInternal(string pythonType, string expectedType)
{
var tokens = PythonTokenizer.Instance.Tokenize(pythonType);
var result = PythonParser.PythonTypeDefinitionTokenizer.TryParse(tokens);
var result = PythonParser.PythonTypeDefinitionParser.TryParse(tokens);
Assert.True(result.HasValue, result.ToString());
Assert.NotNull(result.Value);
var reflectedType = TypeReflection.AsPredefinedType(result.Value, TypeReflection.ConversionDirection.FromPython);
Expand Down

0 comments on commit 088cf99

Please sign in to comment.