Skip to content

Commit

Permalink
Fix bug in identifier lexer module and add regression tests.
Browse files Browse the repository at this point in the history
  • Loading branch information
@GGG-KILLER
GGG-KILLER committed Nov 5, 2020
1 parent b3d345b commit 811b518
Show file tree
Hide file tree
Showing 2 changed files with 56 additions and 34 deletions.
88 changes: 55 additions & 33 deletions Loretta.Tests/Lexing/Modules/IdentifierLexerModuleTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,58 +11,80 @@ namespace Loretta.Tests.Lexing.Modules
[TestClass]
public class IdentifierLexerModuleTests
{
private readonly IdentifierLexerModule lexerModule;

public IdentifierLexerModuleTests ( )
private static IdentifierLexerModule GetLexerModule ( Boolean useLuaJitIdentifierRules )
{
this.lexerModule = new IdentifierLexerModule ( LuaOptions.All, new[]
{
var identifierLexerModule = new IdentifierLexerModule ( useLuaJitIdentifierRules ? LuaOptions.LuaJIT : LuaOptions.Lua51, new[]
{
"do",
"end"
}, new[]
{
{
"and",
"or"
} );
identifierLexerModule.AddLiteral ( "nil", LuaTokenType.Nil, null );
identifierLexerModule.AddLiteral ( "true", LuaTokenType.Boolean, true );
identifierLexerModule.AddLiteral ( "false", LuaTokenType.Boolean, false );
return identifierLexerModule;
}

private static readonly IdentifierLexerModule moduleWithoutLuajitRules = GetLexerModule ( false );
private static readonly IdentifierLexerModule moduleWithLuajitRules = GetLexerModule ( true );

[DataTestMethod]
[DataRow ( false, "123" )]
[DataRow ( false, " " )]
[DataRow ( false, "[" )]
[DataRow ( false, "]" )]
[DataRow ( true, "_" )]
[DataRow ( true, "🅱" )]
[DataRow ( true, "\ufeff" /* ZERO WIDTH NO-BREAK SPACE */ )]
[DataRow ( true, "\u206b" /* ACTIVATE SYMMETRIC SWAPPING */ )]
[DataRow ( true, "\u202a" /* LEFT-TO-RIGHT EMBEDDING */ )]
[DataRow ( true, "\u206a" /* INHIBIT SYMMETRIC SWAPPING */ )]
[DataRow ( true, "\ufeff" /* ZERO WIDTH NO-BREAK SPACE */ )]
[DataRow ( true, "\u206a" /* INHIBIT SYMMETRIC SWAPPING */ )]
[DataRow ( true, "\u200e" /* LEFT-TO-RIGHT MARK */ )]
[DataRow ( true, "\u200c" /* ZERO WIDTH NON-JOINER */ )]
[DataRow ( true, "\u200e" /* LEFT-TO-RIGHT MARK */ )]
public void CanConsumeWorks ( Boolean expected, String str ) =>
Assert.AreEqual ( expected, this.lexerModule.CanConsumeNext ( new StringCodeReader ( str ) ) );
[DataRow ( true, false, "123" )]
[DataRow ( true, false, " " )]
[DataRow ( true, false, "[" )]
[DataRow ( true, false, "]" )]
[DataRow ( true, true, "_" )]
[DataRow ( false, true, "🅱" )]
[DataRow ( false, true, "\ufeff" /* ZERO WIDTH NO-BREAK SPACE */ )]
[DataRow ( false, true, "\u206b" /* ACTIVATE SYMMETRIC SWAPPING */ )]
[DataRow ( false, true, "\u202a" /* LEFT-TO-RIGHT EMBEDDING */ )]
[DataRow ( false, true, "\u206a" /* INHIBIT SYMMETRIC SWAPPING */ )]
[DataRow ( false, true, "\ufeff" /* ZERO WIDTH NO-BREAK SPACE */ )]
[DataRow ( false, true, "\u206a" /* INHIBIT SYMMETRIC SWAPPING */ )]
[DataRow ( false, true, "\u200e" /* LEFT-TO-RIGHT MARK */ )]
[DataRow ( false, true, "\u200c" /* ZERO WIDTH NON-JOINER */ )]
[DataRow ( false, true, "\u200e" /* LEFT-TO-RIGHT MARK */ )]
public void CanConsumeWorks ( Boolean shouldWorkWithoutLuajitIdentifierRules, Boolean expected, String str )
{
Assert.AreEqual ( expected, moduleWithLuajitRules.CanConsumeNext ( new StringCodeReader ( str ) ) );
if ( shouldWorkWithoutLuajitIdentifierRules )
Assert.AreEqual ( expected, moduleWithoutLuajitRules.CanConsumeNext ( new StringCodeReader ( str ) ) );
}

[DataTestMethod]
[DataRow ( LuaTokenType.Keyword, "do", "do", "do return end" )]
[DataRow ( LuaTokenType.Keyword, "end", "end", "end" )]
[DataRow ( LuaTokenType.Operator, "and", "and", "and true" )]
[DataRow ( LuaTokenType.Identifier, "\ufeff\u206b\u202a\u206a\ufeff\u206a\u200e\u200c\u200e", "\ufeff\u206b\u202a\u206a\ufeff\u206a\u200e\u200c\u200e", "\ufeff\u206b\u202a\u206a\ufeff\u206a\u200e\u200c\u200e = _G" )]
[DataRow ( LuaTokenType.Identifier, "_G", "_G", "_G = nil" )]
[DataRow ( LuaTokenType.Nil, "nil", null, "nil" )]
[DataRow ( LuaTokenType.Boolean, "true", true, "true and false" )]
[DataRow ( LuaTokenType.Boolean, "false", false, "false and true" )]
public void ConsumeNextWorks ( LuaTokenType expectedType, String expectedRaw, Object expectedValue, String input )
[DataRow ( true, LuaTokenType.Keyword, "do", "do", "do return end" )]
[DataRow ( true, LuaTokenType.Keyword, "end", "end", "end" )]
[DataRow ( true, LuaTokenType.Operator, "and", "and", "and true" )]
[DataRow ( false, LuaTokenType.Identifier, "\ufeff\u206b\u202a\u206a\ufeff\u206a\u200e\u200c\u200e", "\ufeff\u206b\u202a\u206a\ufeff\u206a\u200e\u200c\u200e", "\ufeff\u206b\u202a\u206a\ufeff\u206a\u200e\u200c\u200e = _G" )]
[DataRow ( true, LuaTokenType.Identifier, "_G", "_G", "_G = nil" )]
[DataRow ( true, LuaTokenType.Identifier, "test0", "test0", "test0 = 1" )]
[DataRow ( true, LuaTokenType.Identifier, "te0st", "te0st", "te0st = 1" )]
[DataRow ( true, LuaTokenType.Nil, "nil", null, "nil" )]
[DataRow ( true, LuaTokenType.Boolean, "true", true, "true and false" )]
[DataRow ( true, LuaTokenType.Boolean, "false", false, "false and true" )]
public void ConsumeNextWorks ( Boolean shouldWorkWithoutLuajitIdentifierRules, LuaTokenType expectedType, String expectedRaw, Object expectedValue, String input )
{
var diagnostics = new DiagnosticList ( );
Token<LuaTokenType> token = this.lexerModule.ConsumeNext ( new StringCodeReader ( input ), diagnostics );
Token<LuaTokenType> token = moduleWithLuajitRules.ConsumeNext ( new StringCodeReader ( input ), diagnostics );

TestUtils.AssertDiagnosticsEmpty ( diagnostics, input );
Assert.AreEqual ( expectedType, token.Type );
Assert.AreEqual ( expectedRaw, token.Raw );
Assert.AreEqual ( expectedValue, token.Value );

if ( shouldWorkWithoutLuajitIdentifierRules )
{
diagnostics = new DiagnosticList ( );
token = moduleWithoutLuajitRules.ConsumeNext ( new StringCodeReader ( input ), diagnostics );

TestUtils.AssertDiagnosticsEmpty ( diagnostics, input );
Assert.AreEqual ( expectedType, token.Type );
Assert.AreEqual ( expectedRaw, token.Raw );
Assert.AreEqual ( expectedValue, token.Value );
}
}
}
}
2 changes: 1 addition & 1 deletion Loretta/Lexing/Modules/IdentifierLexerModule.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ public Token<LuaTokenType> ConsumeNext ( ICodeReader reader, IProgress<Diagnosti
SourceLocation start = reader.Location;
var identifier = this.LuaOptions.UseLuaJitIdentifierRules
? reader.ReadStringWhile ( ch => CharUtils.IsValidTrailingIdentifierChar ( true, ch ) )
: reader.ReadStringWhile ( ch => CharUtils.IsValidFirstIdentifierChar ( false, ch ) );
: reader.ReadStringWhile ( ch => CharUtils.IsValidTrailingIdentifierChar ( false, ch ) );
SourceRange range = start.To ( reader.Location );

if ( this.Keywords.Contains ( identifier ) )
Expand Down

0 comments on commit 811b518

Please sign in to comment.