Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix locations reporting #2093

Merged
merged 6 commits into from
Nov 17, 2024
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions DMCompiler/Compiler/DM/AST/DMAST.Expression.cs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,13 @@ public virtual IEnumerable<DMASTExpression> Leaves() {
public virtual DMASTExpression GetUnwrapped() {
return this;
}

public override string ToStringNoLocation() {
var leaves = Leaves().ToList();
if (leaves.Count == 0)
return $"{GetType().Name}";
return $"{GetType().Name}({string.Join(", ", Leaves().Select(l => l.ToString(Location)))})";
}
}

/// <summary>
Expand Down
15 changes: 15 additions & 0 deletions DMCompiler/Compiler/DM/AST/DMAST.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,21 @@ namespace DMCompiler.Compiler.DM.AST;

public abstract class DMASTNode(Location location) {
public readonly Location Location = location;

public override string ToString() {
return $"{ToString(null)}";
}

public string ToString(Location? loc, bool hideLoc = false) {
wixoaGit marked this conversation as resolved.
Show resolved Hide resolved
if (hideLoc || (loc is not null && Location.SourceFile == loc.Value.SourceFile && Location.Line == loc.Value.Line))
return $"{ToStringNoLocation()}";
wixoaGit marked this conversation as resolved.
Show resolved Hide resolved
return $"{ToStringNoLocation()} [{Location}]";
}

public virtual string ToStringNoLocation()
{
wixoaGit marked this conversation as resolved.
Show resolved Hide resolved
return $"{GetType().Name}";
}
}

public sealed class DMASTFile(Location location, DMASTBlockInner blockInner) : DMASTNode(location) {
Expand Down
27 changes: 13 additions & 14 deletions DMCompiler/Compiler/DM/DMLexer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ protected override Token ParseNextToken() {
token = preprocToken;
}
} else {
var firstTokenLocation = CurrentLocation;
switch (preprocToken.Type) {
case TokenType.DM_Preproc_Whitespace: Advance(); token = CreateToken(TokenType.DM_Whitespace, preprocToken.Text); break;
case TokenType.DM_Preproc_Punctuator_LeftParenthesis: BracketNesting++; Advance(); token = CreateToken(TokenType.DM_LeftParenthesis, preprocToken.Text); break;
Expand All @@ -125,19 +126,19 @@ protected override Token ParseNextToken() {
case TokenType.DM_Preproc_Punctuator_Question:
switch (Advance().Type) {
case TokenType.DM_Preproc_Punctuator_Period:
token = CreateToken(TokenType.DM_QuestionPeriod, "?.");
Advance();
token = CreateToken(TokenType.DM_QuestionPeriod, "?.", firstTokenLocation);
break;

case TokenType.DM_Preproc_Punctuator_Colon:
token = CreateToken(TokenType.DM_QuestionColon, "?:");
Advance();
token = CreateToken(TokenType.DM_QuestionColon, "?:", firstTokenLocation);
break;

case TokenType.DM_Preproc_Punctuator_LeftBracket:
token = CreateToken(TokenType.DM_QuestionLeftBracket, "?[");
BracketNesting++;
Advance();
token = CreateToken(TokenType.DM_QuestionLeftBracket, "?[", firstTokenLocation);
BracketNesting++;
break;

default:
Expand All @@ -149,11 +150,10 @@ protected override Token ParseNextToken() {
switch (Advance().Type) {
case TokenType.DM_Preproc_Punctuator_Period:
if (Advance().Type == TokenType.DM_Preproc_Punctuator_Period) {
token = CreateToken(TokenType.DM_IndeterminateArgs, "...");

Advance();
token = CreateToken(TokenType.DM_IndeterminateArgs, "...", firstTokenLocation);
} else {
token = CreateToken(TokenType.DM_SuperProc, "..");
token = CreateToken(TokenType.DM_SuperProc, "..", firstTokenLocation);
}

break;
Expand Down Expand Up @@ -231,6 +231,7 @@ protected override Token ParseNextToken() {
break;
}
case TokenType.DM_Preproc_ConstantString: {
Advance();
string tokenText = preprocToken.Text;
switch (preprocToken.Text[0]) {
case '"':
Expand All @@ -239,21 +240,19 @@ protected override Token ParseNextToken() {
case '@': token = CreateToken(TokenType.DM_RawString, tokenText, preprocToken.Value); break;
default: token = CreateToken(TokenType.Error, tokenText, "Invalid string"); break;
}

Advance();
break;
}
case TokenType.DM_Preproc_StringBegin:
token = CreateToken(TokenType.DM_StringBegin, preprocToken.Text, preprocToken.Value);
Advance();
token = CreateToken(TokenType.DM_StringBegin, preprocToken.Text, preprocToken.Value);
break;
case TokenType.DM_Preproc_StringMiddle:
token = CreateToken(TokenType.DM_StringMiddle, preprocToken.Text, preprocToken.Value);
Advance();
token = CreateToken(TokenType.DM_StringMiddle, preprocToken.Text, preprocToken.Value);
break;
case TokenType.DM_Preproc_StringEnd:
token = CreateToken(TokenType.DM_StringEnd, preprocToken.Text, preprocToken.Value);
Advance();
token = CreateToken(TokenType.DM_StringEnd, preprocToken.Text, preprocToken.Value);
break;
case TokenType.DM_Preproc_Identifier: {
TokenTextBuilder.Clear();
Expand All @@ -267,7 +266,7 @@ protected override Token ParseNextToken() {
var identifierText = TokenTextBuilder.ToString();
var tokenType = Keywords.GetValueOrDefault(identifierText, TokenType.DM_Identifier);

token = CreateToken(tokenType, identifierText);
token = CreateToken(tokenType, identifierText, firstTokenLocation);
break;
}
case TokenType.DM_Preproc_Number: {
Expand All @@ -290,7 +289,7 @@ protected override Token ParseNextToken() {

break;
}
case TokenType.EndOfFile: token = preprocToken; Advance(); break;
case TokenType.EndOfFile: Advance(); token = preprocToken; break;
default: token = CreateToken(TokenType.Error, preprocToken.Text, "Invalid token"); break;
}
}
Expand Down
6 changes: 3 additions & 3 deletions DMCompiler/Compiler/DM/DMParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@
}

if (path.IsOperator) {
List<DMASTProcStatement> procStatements = procBlock.Statements.ToList();

Check warning on line 257 in DMCompiler/Compiler/DM/DMParser.cs

View workflow job for this annotation

GitHub Actions / build (ubuntu-latest)

Dereference of a possibly null reference.
Location tokenLoc = procBlock.Location;
//add ". = src" as the first expression in the operator
DMASTProcStatementExpression assignEqSrc = new DMASTProcStatementExpression(tokenLoc,
Expand Down Expand Up @@ -337,7 +337,7 @@
}

//Empty object definition
Compiler.VerbosePrint($"Parsed object {CurrentPath}");
Compiler.VerbosePrint($"Parsed object {CurrentPath} - empty");
return new DMASTObjectDefinition(loc, CurrentPath, null);
}

Expand Down Expand Up @@ -517,7 +517,7 @@
Newline();
Consume(TokenType.DM_RightCurlyBracket, "Expected '}'");

return new DMASTBlockInner(loc, blockInner.ToArray());

Check warning on line 520 in DMCompiler/Compiler/DM/DMParser.cs

View workflow job for this annotation

GitHub Actions / build (ubuntu-latest)

Dereference of a possibly null reference.
}

return null;
Expand Down Expand Up @@ -1787,10 +1787,10 @@
}

private DMASTExpression? ExpressionIn() {
var loc = Current().Location; // Don't check this inside, as Check() will advance and point at next token instead
DMASTExpression? value = ExpressionAssign();

while (value != null && Check(TokenType.DM_In)) {
var loc = Current().Location;

Whitespace();
DMASTExpression? list = ExpressionAssign();
Expand Down Expand Up @@ -2191,7 +2191,7 @@
if (inner is null) {
inner = new DMASTVoid(loc);
} else {
inner = new DMASTExpressionWrapped(inner.Location, inner);
inner = new DMASTExpressionWrapped(loc, inner);
}

return inner;
Expand Down
4 changes: 2 additions & 2 deletions DMCompiler/Compiler/DMPreprocessor/DMPreprocessor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -638,7 +638,7 @@ private void HandlePragmaDirective() {

Token warningTypeToken = GetNextToken(true);
if (warningTypeToken.Type != TokenType.DM_Preproc_Identifier) {
compiler.Emit(WarningCode.BadDirective, warningNameToken.Location, $"Warnings can only be set to disabled, notice, warning, or error");
compiler.Emit(WarningCode.BadDirective, warningTypeToken.Location, "Warnings can only be set to disabled, notice, warning, or error");
return;
}
switch(warningTypeToken.Text.ToLower()) {
Expand All @@ -660,7 +660,7 @@ private void HandlePragmaDirective() {
compiler.SetPragma(warningCode, ErrorLevel.Error);
break;
default:
compiler.Emit(WarningCode.BadDirective, warningNameToken.Location, $"Warnings can only be set to disabled, notice, warning, or error");
compiler.Emit(WarningCode.BadDirective, warningTypeToken.Location, "Warnings can only be set to disabled, notice, warning, or error");
return;
}
}
Expand Down
16 changes: 10 additions & 6 deletions DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ internal sealed class DMPreprocessorLexer {
private readonly bool _isDMStandard;
private char _current;
private int _currentLine = 1, _currentColumn;
private int _previousLine = 1, _previousColumn;
private readonly Queue<Token> _pendingTokenQueue = new(); // TODO: Possible to remove this?

public DMPreprocessorLexer(DMCompiler compiler, string includeDirectory, string file, string source) {
Expand Down Expand Up @@ -61,6 +62,9 @@ public Token NextToken(bool ignoreWhitespace = false) {

char c = GetCurrent();

_previousLine = _currentLine;
_previousColumn = _currentColumn;

switch (c) {
case '\0':
return CreateToken(TokenType.EndOfFile, c);
Expand Down Expand Up @@ -367,7 +371,7 @@ public Token NextToken(bool ignoreWhitespace = false) {
LexString(true) :
CreateToken(TokenType.DM_Preproc_Punctuator, c);
case '#': {
bool isConcat = (Advance() == '#');
bool isConcat = Advance() == '#';
if (isConcat) Advance();

// Whitespace after '#' is ignored
Expand Down Expand Up @@ -444,7 +448,7 @@ public Token NextToken(bool ignoreWhitespace = false) {
}

Advance();
return CreateToken(TokenType.Error, string.Empty, $"Unknown character: {c.ToString()}");
return CreateToken(TokenType.Error, string.Empty, $"Unknown character: {c}");
}
}
}
Expand Down Expand Up @@ -619,7 +623,7 @@ private bool HandleLineEnd() {
goto case '\n';
case '\n':
_currentLine++;
_currentColumn = 1;
_currentColumn = 0; // Because Advance will bump this to 1 and any position reads will happen next NextToken() call

if (c == '\n') // This line could have ended with only \r
Advance();
Expand All @@ -641,7 +645,7 @@ private char Advance() {

if (value == -1) {
_current = '\0';
} else {
} else {
_currentColumn++;
_current = (char)value;
}
Expand All @@ -656,11 +660,11 @@ private bool AtEndOfSource() {

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private Token CreateToken(TokenType type, string text, object? value = null) {
return new Token(type, text, new Location(File, _currentLine, _currentColumn, _isDMStandard), value);
return new Token(type, text, new Location(File, _previousLine, _previousColumn, _isDMStandard), value);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private Token CreateToken(TokenType type, char text, object? value = null) {
return new Token(type, text.ToString(), new Location(File, _currentLine, _currentColumn, _isDMStandard), value);
return new Token(type, text.ToString(), new Location(File, _previousLine, _previousColumn, _isDMStandard), value);
}
}
18 changes: 15 additions & 3 deletions DMCompiler/Compiler/Lexer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,20 @@

internal class Lexer<TSourceType> {
public Location CurrentLocation { get; protected set; }
public IEnumerable<TSourceType> Source { get; }
public bool AtEndOfSource { get; private set; }
public Location PreviousLocation { get; protected set; }
public string SourceName { get; protected set; }
public IEnumerable<TSourceType> Source { get; protected set; }
public bool AtEndOfSource { get; protected set; } = false;
github-advanced-security[bot] marked this conversation as resolved.
Fixed
Show resolved Hide resolved
wixoaGit marked this conversation as resolved.
Show resolved Hide resolved

protected Queue<Token> _pendingTokenQueue = new();

private readonly IEnumerator<TSourceType> _sourceEnumerator;
private TSourceType _current;

protected Lexer(string sourceName, IEnumerable<TSourceType> source) {

Check warning on line 17 in DMCompiler/Compiler/Lexer.cs

View workflow job for this annotation

GitHub Actions / build (ubuntu-latest)

Non-nullable field '_current' must contain a non-null value when exiting constructor. Consider declaring the field as nullable.
CurrentLocation = new Location(sourceName, 1, 0);
PreviousLocation = CurrentLocation;
SourceName = sourceName;
Source = source;
if (source == null)
throw new FileNotFoundException("Source file could not be read: " + sourceName);
Expand All @@ -39,8 +43,13 @@
return CreateToken(TokenType.Unknown, GetCurrent()?.ToString() ?? string.Empty);
}

protected Token CreateToken(TokenType type, string text, Location location, object? value = null) {
var token = new Token(type, text, location, value);
return token;
}

protected Token CreateToken(TokenType type, string text, object? value = null) {
return new Token(type, text, CurrentLocation, value);
return CreateToken(type, text, PreviousLocation, value);
}

protected Token CreateToken(TokenType type, char text, object? value = null) {
Expand All @@ -51,7 +60,10 @@
return _current;
}

/// <remarks>Call before CreateToken to make sure the location is correct</remarks>
protected virtual TSourceType Advance() {
PreviousLocation = CurrentLocation;

if (_sourceEnumerator.MoveNext()) {
_current = _sourceEnumerator.Current;
} else {
Expand Down
Loading