Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Record skipped tokens as leading trivia on the fabricated token #176

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Record skipped tokens as leading trivia on the fabricated token #176

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 13 additions & 8 deletions src/Minsk.Tests/CodeAnalysis/EvaluationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,12 +175,11 @@ public void Evaluator_InvokeFunctionArguments_Exceeding()
public void Evaluator_InvokeFunctionArguments_NoInfiniteLoop()
{
var text = @"
print(""Hi""[[=]][)]
print(""Hi""[=][)]
";

var diagnostics = @"
Unexpected token <EqualsToken>, expected <CloseParenthesisToken>.
Unexpected token <EqualsToken>, expected <IdentifierToken>.
Unexpected token <CloseParenthesisToken>, expected <IdentifierToken>.
";

Expand All @@ -191,17 +190,15 @@ public void Evaluator_InvokeFunctionArguments_NoInfiniteLoop()
public void Evaluator_FunctionParameters_NoInfiniteLoop()
{
var text = @"
function hi(name: string[[[=]]][)]
function hi(name: string[=][)]
{
print(""Hi "" + name + ""!"" )
}[]
";

var diagnostics = @"
Unexpected token <EqualsToken>, expected <CloseParenthesisToken>.
Unexpected token <EqualsToken>, expected <OpenBraceToken>.
Unexpected token <EqualsToken>, expected <IdentifierToken>.
Unexpected token <CloseParenthesisToken>, expected <IdentifierToken>.
Unexpected token <CloseParenthesisToken>, expected <OpenBraceToken>.
Unexpected token <EndOfFileToken>, expected <CloseBraceToken>.
";

Expand Down Expand Up @@ -691,14 +688,22 @@ A parameter with the name 'a' already exists.
public void Evaluator_Function_Must_Have_Name()
{
var text = @"
function [(]a: int, b: int): int
function [(][a][:] [int][,] [b]: int[)][:] int
{
return a + b
}
}[]
";

var diagnostics = @"
Unexpected token <OpenParenthesisToken>, expected <IdentifierToken>.
Unexpected token <IdentifierToken>, expected <OpenParenthesisToken>.
Unexpected token <ColonToken>, expected <IdentifierToken>.
Unexpected token <IdentifierToken>, expected <ColonToken>.
Unexpected token <CommaToken>, expected <IdentifierToken>.
Unexpected token <IdentifierToken>, expected <CloseParenthesisToken>.
Unexpected token <CloseParenthesisToken>, expected <OpenBraceToken>.
Unexpected token <ColonToken>, expected <IdentifierToken>.
Unexpected token <EndOfFileToken>, expected <CloseBraceToken>.
";

AssertDiagnostics(text, diagnostics);
Expand Down
40 changes: 27 additions & 13 deletions src/Minsk/CodeAnalysis/Syntax/Parser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,33 @@ private SyntaxToken MatchToken(SyntaxKind kind)
return NextToken();

_diagnostics.ReportUnexpectedToken(Current.Location, Current.Kind, kind);
return new SyntaxToken(_syntaxTree, kind, Current.Position, null, null, ImmutableArray<SyntaxTrivia>.Empty, ImmutableArray<SyntaxTrivia>.Empty);
return FabricateToken(kind);
}

private SyntaxToken FabricateToken(SyntaxKind expectedKind, bool eatCurrent = true)
{
var leadingTrivia = ImmutableArray<SyntaxTrivia>.Empty;

if (eatCurrent)
{
var badToken = NextToken();

var triviaBuilder = leadingTrivia.ToBuilder();
var index = 0;

foreach (var lt in badToken.LeadingTrivia)
triviaBuilder.Insert(index++, lt);

var trivia = new SyntaxTrivia(badToken.SyntaxTree, SyntaxKind.SkippedTextTrivia, badToken.Position, badToken.Text);
triviaBuilder.Insert(index++, trivia);

foreach (var tt in badToken.TrailingTrivia)
triviaBuilder.Insert(index++, tt);

leadingTrivia = triviaBuilder.ToImmutableArray();
}

return new SyntaxToken(_syntaxTree, expectedKind, Current.Position, null, null, leadingTrivia, ImmutableArray<SyntaxTrivia>.Empty);
}

public CompilationUnitSyntax ParseCompilationUnit()
Expand All @@ -103,20 +129,8 @@ private ImmutableArray<MemberSyntax> ParseMembers()

while (Current.Kind != SyntaxKind.EndOfFileToken)
{
var startToken = Current;

var member = ParseMember();
members.Add(member);

// If ParseMember() did not consume any tokens,
// we need to skip the current token and continue
// in order to avoid an infinite loop.
//
// We don't need to report an error, because we'll
// already tried to parse an expression statement
// and reported one.
if (Current == startToken)
NextToken();
}

return members.ToImmutable();
Expand Down