Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rename tokenizers that are actually parsers #298

Merged
merged 1 commit into from
Oct 29, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions src/CSnakes.SourceGeneration/Parser/PythonParser.Args.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,26 @@
namespace CSnakes.Parser;
public static partial class PythonParser
{
public static TokenListParser<PythonToken, PythonParameterType> PythonStarArgTokenizer { get; } =
public static TokenListParser<PythonToken, PythonParameterType> PythonStarArgParser { get; } =
(from star in Token.EqualTo(PythonToken.Asterisk)
from name in Token.EqualTo(PythonToken.Identifier).Optional()
select new PythonParameterType(name.HasValue ? name.Value.ToStringValue() : "args", PythonFunctionParameterType.Star))
.Named("Star Arg");

public static TokenListParser<PythonToken, PythonParameterType> PythonDoubleStarArgTokenizer { get; } =
public static TokenListParser<PythonToken, PythonParameterType> PythonDoubleStarArgParser { get; } =
(from star in Token.EqualTo(PythonToken.DoubleAsterisk)
from name in Token.EqualTo(PythonToken.Identifier)
select new PythonParameterType(name.ToStringValue(), PythonFunctionParameterType.DoubleStar))
.Named("Double Star Arg");

public static TokenListParser<PythonToken, PythonParameterType> PythonNormalArgTokenizer { get; } =
public static TokenListParser<PythonToken, PythonParameterType> PythonNormalArgParser { get; } =
(from name in Token.EqualTo(PythonToken.Identifier)
select new PythonParameterType(name.ToStringValue(), PythonFunctionParameterType.Normal))
.Named("Normal Arg");

public static TokenListParser<PythonToken, PythonParameterType?> PythonArgTokenizer { get; } =
PythonStarArgTokenizer.AsNullable()
.Or(PythonDoubleStarArgTokenizer.AsNullable())
.Or(PythonNormalArgTokenizer.AsNullable())
public static TokenListParser<PythonToken, PythonParameterType?> PythonArgParser { get; } =
PythonStarArgParser.AsNullable()
.Or(PythonDoubleStarArgParser.AsNullable())
.Or(PythonNormalArgParser.AsNullable())
.Named("Arg");
}
8 changes: 4 additions & 4 deletions src/CSnakes.SourceGeneration/Parser/PythonParser.Function.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@
namespace CSnakes.Parser;
public static partial class PythonParser
{
public static TokenListParser<PythonToken, PythonFunctionDefinition> PythonFunctionDefinitionTokenizer { get; } =
public static TokenListParser<PythonToken, PythonFunctionDefinition> PythonFunctionDefinitionParser { get; } =
(from def in Token.EqualTo(PythonToken.Def)
from name in Token.EqualTo(PythonToken.Identifier)
from parameters in PythonParameterListTokenizer.AssumeNotNull()
from arrow in Token.EqualTo(PythonToken.Arrow).Optional().Then(returnType => PythonTypeDefinitionTokenizer.AssumeNotNull().OptionalOrDefault())
from parameters in PythonParameterListParser.AssumeNotNull()
from arrow in Token.EqualTo(PythonToken.Arrow).Optional().Then(returnType => PythonTypeDefinitionParser.AssumeNotNull().OptionalOrDefault())
from colon in Token.EqualTo(PythonToken.Colon)
select new PythonFunctionDefinition(name.ToStringValue(), arrow, parameters))
.Named("Function Definition");
Expand Down Expand Up @@ -90,7 +90,7 @@ public static bool TryParseFunctionDefinitions(SourceText source, out PythonFunc
foreach ((IEnumerable<TextLine> currentLines, ParsedTokens tokens) in functionLines)
{
TokenListParserResult<PythonToken, PythonFunctionDefinition> functionDefinition =
PythonFunctionDefinitionTokenizer.TryParse(tokens);
PythonFunctionDefinitionParser.TryParse(tokens);
if (functionDefinition.HasValue)
{
functionDefinitions.Add(functionDefinition.Value);
Expand Down
14 changes: 7 additions & 7 deletions src/CSnakes.SourceGeneration/Parser/PythonParser.Parameters.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,25 +10,25 @@ public static partial class PythonParser
.Select(d => new PythonFunctionParameter("/", null, null, PythonFunctionParameterType.Slash))
.Named("Positional Only Signal");

public static TokenListParser<PythonToken, PythonFunctionParameter> PythonParameterTokenizer { get; } =
(from arg in PythonArgTokenizer
public static TokenListParser<PythonToken, PythonFunctionParameter> PythonParameterParser { get; } =
(from arg in PythonArgParser
from type in Token.EqualTo(PythonToken.Colon).Optional().Then(
_ => PythonTypeDefinitionTokenizer.AssumeNotNull().OptionalOrDefault()
_ => PythonTypeDefinitionParser.AssumeNotNull().OptionalOrDefault()
)
from defaultValue in Token.EqualTo(PythonToken.Equal).Optional().Then(
_ => ConstantValueTokenizer.AssumeNotNull().OptionalOrDefault()
)
select new PythonFunctionParameter(arg.Name, type, defaultValue, arg.ParameterType))
.Named("Parameter");

public static TokenListParser<PythonToken, PythonFunctionParameter?> ParameterOrSlash { get; } =
public static TokenListParser<PythonToken, PythonFunctionParameter?> ParameterOrSlashParser { get; } =
PositionalOnlyParameterParser.AsNullable()
.Or(PythonParameterTokenizer.AsNullable())
.Or(PythonParameterParser.AsNullable())
.Named("Parameter");

public static TokenListParser<PythonToken, PythonFunctionParameter[]> PythonParameterListTokenizer { get; } =
public static TokenListParser<PythonToken, PythonFunctionParameter[]> PythonParameterListParser { get; } =
(from openParen in Token.EqualTo(PythonToken.OpenParenthesis)
from parameters in ParameterOrSlash.ManyDelimitedBy(Token.EqualTo(PythonToken.Comma))
from parameters in ParameterOrSlashParser.ManyDelimitedBy(Token.EqualTo(PythonToken.Comma))
from closeParen in Token.EqualTo(PythonToken.CloseParenthesis)
select parameters)
.Named("Parameter List");
Expand Down
4 changes: 2 additions & 2 deletions src/CSnakes.SourceGeneration/Parser/PythonParser.TypeDef.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ public static partial class PythonParser
.AtLeastOnceDelimitedBy(Character.EqualTo('.'))
);

public static TokenListParser<PythonToken, PythonTypeSpec?> PythonTypeDefinitionTokenizer { get; } =
public static TokenListParser<PythonToken, PythonTypeSpec?> PythonTypeDefinitionParser { get; } =
(from name in Token.EqualTo(PythonToken.Identifier).Or(Token.EqualTo(PythonToken.None)).OptionalOrDefault()
#pragma warning disable CS8620
from openBracket in Token.EqualTo(PythonToken.OpenBracket)
.Then(_ =>
PythonTypeDefinitionTokenizer
PythonTypeDefinitionParser
.AssumeNotNull()
.ManyDelimitedBy(
Token.EqualTo(PythonToken.Comma),
Expand Down
38 changes: 19 additions & 19 deletions src/CSnakes.Tests/TokenizerTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ public void TokenizeWithDefaultValue()
public void ParseFunctionParameter(string code, string expectedName, string expectedType)
{
var tokens = PythonTokenizer.Instance.Tokenize(code);
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal(expectedName, result.Value.Name);
Assert.Equal(expectedType, result.Value.Type.ToString());
Expand All @@ -100,7 +100,7 @@ public void ParseFunctionParameter(string code, string expectedName, string expe
public void ParseFunctionParameterNoType()
{
var tokens = PythonTokenizer.Instance.Tokenize("a");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.False(result.Value.HasTypeAnnotation());
Expand All @@ -110,7 +110,7 @@ public void ParseFunctionParameterNoType()
public void ParseFunctionParameterDefault()
{
var tokens = PythonTokenizer.Instance.Tokenize("a = 1");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal("1", result.Value.DefaultValue?.ToString());
Expand All @@ -125,7 +125,7 @@ public void ParseFunctionParameterDefault()
public void ParseFunctionParameterDefaultValuesNoType(string value)
{
var tokens = PythonTokenizer.Instance.Tokenize($"a = {value}");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal(value, result.Value.DefaultValue?.ToString());
Expand All @@ -136,7 +136,7 @@ public void ParseFunctionParameterDefaultValuesNoType(string value)
public void ParseFunctionParameterDefaultSingleQuotedString()
{
var tokens = PythonTokenizer.Instance.Tokenize($"a = 'hello'");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal("hello", result.Value.DefaultValue?.ToString());
Expand All @@ -147,7 +147,7 @@ public void ParseFunctionParameterDefaultSingleQuotedString()
public void ParseFunctionParameterDefaultDoubleQuotedString()
{
var tokens = PythonTokenizer.Instance.Tokenize($"a = \"hello\"");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal("hello", result.Value.DefaultValue?.ToString());
Expand All @@ -158,7 +158,7 @@ public void ParseFunctionParameterDefaultDoubleQuotedString()
public void ParseFunctionParameterDefaultDouble()
{
var tokens = PythonTokenizer.Instance.Tokenize($"a: float = -1.1");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal("-1.1", result.Value.DefaultValue?.ToString());
Expand All @@ -169,7 +169,7 @@ public void ParseFunctionParameterDefaultDouble()
public void ParseFunctionParameterDefaultInt()
{
var tokens = PythonTokenizer.Instance.Tokenize($"a: int = 1234");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal("1234", result.Value.DefaultValue?.ToString());
Expand All @@ -180,7 +180,7 @@ public void ParseFunctionParameterDefaultInt()
public void ParseFunctionParameterDefaultBoolTrue()
{
var tokens = PythonTokenizer.Instance.Tokenize($"a: bool = True");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal("True", result.Value.DefaultValue?.ToString());
Expand All @@ -191,7 +191,7 @@ public void ParseFunctionParameterDefaultBoolTrue()
public void ParseFunctionParameterDefaultBoolFalse()
{
var tokens = PythonTokenizer.Instance.Tokenize($"a: bool = False");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal("False", result.Value.DefaultValue?.ToString());
Expand All @@ -202,7 +202,7 @@ public void ParseFunctionParameterDefaultBoolFalse()
public void ParseFunctionParameterDefaultNone()
{
var tokens = PythonTokenizer.Instance.Tokenize($"a: bool = None");
var result = PythonParser.PythonParameterTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value.Name);
Assert.Equal("None", result.Value.DefaultValue?.ToString());
Expand All @@ -214,7 +214,7 @@ public void ParseFunctionParameterListSingleGeneric()
{
var code = "(a: list[int])";
var tokens = PythonTokenizer.Instance.Tokenize(code);
var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterListParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value[0].Name);
Assert.Equal("list[int]", result.Value[0].Type.ToString());
Expand All @@ -225,7 +225,7 @@ public void ParseFunctionParameterListDoubleGeneric()
{
var code = "(a: list[int], b)";
var tokens = PythonTokenizer.Instance.Tokenize(code);
var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterListParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value[0].Name);
Assert.Equal("list[int]", result.Value[0].Type.ToString());
Expand All @@ -236,7 +236,7 @@ public void ParseFunctionParameterListQualifiedGenericType()
{
var code = "(a: typing.List[int], b)";
var tokens = PythonTokenizer.Instance.Tokenize(code);
var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterListParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value[0].Name);
Assert.Equal("typing.List[int]", result.Value[0].Type.ToString());
Expand All @@ -247,7 +247,7 @@ public void ParseFunctionParameterListQualifiedBasicType()
{
var code = "(a: np.ndarray, b)";
var tokens = PythonTokenizer.Instance.Tokenize(code);
var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterListParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value[0].Name);
Assert.Equal("np.ndarray", result.Value[0].Type.ToString());
Expand All @@ -258,7 +258,7 @@ public void ParseFunctionParameterListEasy()
{
var code = "(a: int, b: float, c: str)";
var tokens = PythonTokenizer.Instance.Tokenize(code);
var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterListParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value[0].Name);
Assert.Equal("int", result.Value[0].Type.Name);
Expand All @@ -273,7 +273,7 @@ public void ParseFunctionParameterListUntyped()
{
var code = "(a, b, c)";
var tokens = PythonTokenizer.Instance.Tokenize(code);
var result = PythonParser.PythonParameterListTokenizer.TryParse(tokens);
var result = PythonParser.PythonParameterListParser.TryParse(tokens);
Assert.True(result.HasValue);
Assert.Equal("a", result.Value[0].Name);
Assert.Equal("Any", result.Value[0].Type.Name);
Expand All @@ -287,7 +287,7 @@ public void ParseFunctionParameterListUntyped()
public void ParseFunctionDefinition()
{
var tokens = PythonTokenizer.Instance.Tokenize("def foo(a: int, b: str) -> None:");
var result = PythonParser.PythonFunctionDefinitionTokenizer.TryParse(tokens);
var result = PythonParser.PythonFunctionDefinitionParser.TryParse(tokens);

Assert.True(result.HasValue);
Assert.Equal("foo", result.Value.Name);
Expand All @@ -313,7 +313,7 @@ public void ParseFunctionDefinitionComplex(string code)
{
var tokens = PythonTokenizer.Instance.Tokenize(code);
Assert.True(tokens.IsAtEnd == false, "Tokenize failed");
var result = PythonParser.PythonFunctionDefinitionTokenizer.TryParse(tokens);
var result = PythonParser.PythonFunctionDefinitionParser.TryParse(tokens);
Assert.True(result.HasValue, result.ToString());
}

Expand Down
2 changes: 1 addition & 1 deletion src/CSnakes.Tests/TypeReflectionTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ public void TupleParsingTest(string pythonType, string expectedType) =>
private static void ParsingTestInternal(string pythonType, string expectedType)
{
var tokens = PythonTokenizer.Instance.Tokenize(pythonType);
var result = PythonParser.PythonTypeDefinitionTokenizer.TryParse(tokens);
var result = PythonParser.PythonTypeDefinitionParser.TryParse(tokens);
Assert.True(result.HasValue, result.ToString());
Assert.NotNull(result.Value);
var reflectedType = TypeReflection.AsPredefinedType(result.Value, TypeReflection.ConversionDirection.FromPython);
Expand Down
Loading