diff --git a/src/main/java/com/merkrafter/lexing/IdentToken.java b/src/main/java/com/merkrafter/lexing/IdentToken.java index 49696c97..b4b519e0 100644 --- a/src/main/java/com/merkrafter/lexing/IdentToken.java +++ b/src/main/java/com/merkrafter/lexing/IdentToken.java @@ -30,7 +30,7 @@ public IdentToken(final String ident, final String filename, final long line, //============================================================== /** - * @return the keyword this token stands for + * @return the identifier this token stands for */ String getIdent() { return ident; @@ -57,7 +57,7 @@ public boolean equals(final Object obj) { } /** - * Creates a String representation of this KeywordToken in the following format: + * Creates a String representation of this IdentToken in the following format: * FILENAME(LINE,POSITION): TYPE(IDENT) * * @return a String representation of this IdentToken diff --git a/src/main/java/com/merkrafter/lexing/NumberToken.java b/src/main/java/com/merkrafter/lexing/NumberToken.java new file mode 100644 index 00000000..75dfe992 --- /dev/null +++ b/src/main/java/com/merkrafter/lexing/NumberToken.java @@ -0,0 +1,69 @@ +package com.merkrafter.lexing; + +/**** + * This class serves as a token and stores the (integer) number found. + * + * @version v0.2.0 + * @author merkrafter + ***************************************************************/ +public class NumberToken extends Token { + // ATTRIBUTES + //============================================================== + /** + * the number this token stands for + */ + private final long number; + + // CONSTRUCTORS + //============================================================== + + /**** + * Creates a new NumberToken from a number and position data. + ***************************************************************/ + public NumberToken(final long number, final String filename, final long line, + final int position) { + super(TokenType.NUMBER, filename, line, position); + this.number = number; + } + + // GETTER + //============================================================== + + /** + * @return the number this token stands for + */ + long getNumber() { + return number; + } + + // METHODS + //============================================================== + // public methods + //-------------------------------------------------------------- + + /** + * Two NumberTokens are equal if both have the type NumberToken and their numbers, line + * numbers, positions and filenames are equal. + * + * @param obj ideally a NumberToken to compare this with + * @return whether this is equal to obj + */ + @Override + public boolean equals(final Object obj) { + if (!super.equals(obj)) { + return false; + } + return obj instanceof NumberToken && ((NumberToken) obj).number == number; + } + + /** + * Creates a String representation of this NumberToken in the following format: + * FILENAME(LINE,POSITION): TYPE(NUMBER) + * + * @return a String representation of this NumberToken + */ + @Override + public String toString() { + return super.toString() + String.format("(%d)", number); + } +} diff --git a/src/main/java/com/merkrafter/lexing/OtherToken.java b/src/main/java/com/merkrafter/lexing/OtherToken.java new file mode 100644 index 00000000..9ef8b53d --- /dev/null +++ b/src/main/java/com/merkrafter/lexing/OtherToken.java @@ -0,0 +1,69 @@ +package com.merkrafter.lexing; + +/**** + * This class serves as a token and stores a string that could not be recognized as another token. + * + * @version v0.2.0 + * @author merkrafter + ***************************************************************/ +public class OtherToken extends Token { + // ATTRIBUTES + //============================================================== + /** + * the string that could not be recognized as another token + */ + private final String string; + + // CONSTRUCTORS + //============================================================== + + /**** + * Creates a new OtherToken from a string and position data. + ***************************************************************/ + public OtherToken(final String string, final String filename, final long line, + final int position) { + super(TokenType.OTHER, filename, line, position); + this.string = string; + } + + // GETTER + //============================================================== + + /** + * @return the string that could not be recognized as another token + */ + String getString() { + return string; + } + + // METHODS + //============================================================== + // public methods + //-------------------------------------------------------------- + + /** + * Two OtherTokens are equal if both have the type OtherToken and their strings, line + * numbers, positions and filenames are equal. + * + * @param obj ideally a OtherToken to compare this with + * @return whether this is equal to obj + */ + @Override + public boolean equals(final Object obj) { + if (!super.equals(obj)) { + return false; + } + return obj instanceof OtherToken && ((OtherToken) obj).string.equals(string); + } + + /** + * Creates a String representation of this OtherToken in the following format: + * FILENAME(LINE,POSITION): TYPE(STRING) + * + * @return a String representation of this OtherToken + */ + @Override + public String toString() { + return super.toString() + String.format("(%s)", string); + } +} diff --git a/src/main/java/com/merkrafter/lexing/Scanner.java b/src/main/java/com/merkrafter/lexing/Scanner.java index 09161613..c9950bc0 100644 --- a/src/main/java/com/merkrafter/lexing/Scanner.java +++ b/src/main/java/com/merkrafter/lexing/Scanner.java @@ -133,9 +133,11 @@ public void processToken() { do { num += ch; if (!this.loadNextCharSuccessfully()) { + setNumber(); // parse the num attribute to a NumberToken return; } } while (ch >= '0' && ch <= '9'); + setNumber(); // parse the num attribute to a NumberToke break; case 'a': case 'b': @@ -343,7 +345,7 @@ public void processToken() { } break; default: - sym = new Token(TokenType.OTHER, filename, line, position); + sym = new OtherToken(Character.toString(ch), filename, line, position); this.loadNextCharSuccessfully(); } } @@ -411,4 +413,19 @@ private void setIdentOrKeyword() { } } + /** + * Tests whether num currently holds a number. If that's the case, sym is changed + * to a NumberToken. Else, a OTHER TokenType is emitted in order to indicate an error. + */ + private void setNumber() { + try { + final long number = Long.parseLong(num); + // if this actually is a number: + sym = new NumberToken(number, sym.getFilename(), sym.getLine(), sym.getPosition()); + } catch (NumberFormatException ignored) { + // id is not a number + sym = new Token(TokenType.OTHER, sym.getFilename(), sym.getLine(), sym.getPosition()); + } + } + } diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 759b3fc4..804507b2 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -384,9 +384,9 @@ boolean parseActualParameters() { return false; } } - } else { - return true; // it is okay if no expression comes here } + // it is okay if no expression comes here + // but it is still necessary to check for the right paren } else { return false; } @@ -444,6 +444,10 @@ boolean parseTerm() { boolean parseFactor() { if (parseIdentifier()) { + // check whether this actually is a intern procedure call + if (parseActualParameters()) { + return true; + } return true; } else if (parseNumber()) { return true; @@ -457,7 +461,7 @@ boolean parseFactor() { } return success; // whether the above parseExpression() was successful } - return parseInternProcedureCall(); + return false; } /** diff --git a/src/test/java/com/merkrafter/ConfigTest.java b/src/test/java/com/merkrafter/ConfigTest.java index 42a31bfd..2af68a65 100644 --- a/src/test/java/com/merkrafter/ConfigTest.java +++ b/src/test/java/com/merkrafter/ConfigTest.java @@ -10,8 +10,19 @@ import static com.merkrafter.config.Config.fromString; import static org.junit.jupiter.api.Assertions.*; +/** + * The test cases of this class verify that the conversion from command line arguments to program + * configuration object works correctly. Therefore, the Config::fromArgs static method is tested + * intensely. + */ class ConfigTest { + /** + * The fromArgs method should be able to extract the input file name correctly. + * It should not set the verbosity nor the output file name. + * + * @throws ArgumentParserException if the arguments can not be parsed; should not happen + */ @Test void parseOnlyInputFile() throws ArgumentParserException { final String[] args = fromString("Test.java"); @@ -26,121 +37,64 @@ void parseOnlyInputFile() throws ArgumentParserException { assertEquals(expectedVerbosity, actualConfig.isVerbose()); } - @Test - void parseInputFileWithVerbosityShortFirst() throws ArgumentParserException { - final String[] args = fromString("-v Test.java"); - final Config actualConfig = Config.fromArgs(args); - - final String expectedInputFilename = "Test.java"; - final String expectedOutputFilename = null; - final boolean expectedVerbosity = true; - - assertEquals(expectedInputFilename, actualConfig.getInputFile()); - assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); - assertEquals(expectedVerbosity, actualConfig.isVerbose()); - } - - @Test - void parseInputFileWithVerbosityShortAfter() throws ArgumentParserException { - final String[] args = fromString("Test.java -v"); - final Config actualConfig = Config.fromArgs(args); - - final String expectedInputFilename = "Test.java"; - final String expectedOutputFilename = null; - final boolean expectedVerbosity = true; - - assertEquals(expectedInputFilename, actualConfig.getInputFile()); - assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); - assertEquals(expectedVerbosity, actualConfig.isVerbose()); - } - - @Test - void parseInputFileWithVerbosityLongFirst() throws ArgumentParserException { - final String[] args = fromString("--verbose Test.java"); - final Config actualConfig = Config.fromArgs(args); - - final String expectedInputFilename = "Test.java"; - final String expectedOutputFilename = null; - final boolean expectedVerbosity = true; - - assertEquals(expectedInputFilename, actualConfig.getInputFile()); - assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); - assertEquals(expectedVerbosity, actualConfig.isVerbose()); - } - - @Test - void parseInputFileWithVerbosityLongAfter() throws ArgumentParserException { - final String[] args = fromString("Test.java --verbose"); + /** + * The fromArgs method should be able to detect the verbosity flag being set, independent of + * whether the long or short argument was used or whether it was specified before or after + * the input file. + * + * @throws ArgumentParserException if the arguments can not be parsed; should not happen + */ + @ParameterizedTest + // {short, long} x {before input file, after input file} + @ValueSource(strings = { + "-v Test.java", "--verbose Test.java", "Test.java -v", "Test.java --verbose"}) + void parseInputFileWithVerbosity(final String string) throws ArgumentParserException { + final String[] args = fromString(string); final Config actualConfig = Config.fromArgs(args); final String expectedInputFilename = "Test.java"; - final String expectedOutputFilename = null; final boolean expectedVerbosity = true; assertEquals(expectedInputFilename, actualConfig.getInputFile()); - assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); assertEquals(expectedVerbosity, actualConfig.isVerbose()); } - @Test - void parseInputFileAndShortOutputFileFirst() throws ArgumentParserException { - final String[] args = fromString("-o OtherTest.class Test.java"); - final Config actualConfig = Config.fromArgs(args); - - final String expectedInputFilename = "Test.java"; - final String expectedOutputFilename = "OtherTest.class"; - final boolean expectedVerbosity = false; - - assertEquals(expectedInputFilename, actualConfig.getInputFile()); - assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); - assertEquals(expectedVerbosity, actualConfig.isVerbose()); - } - - @Test - void parseInputFileAndShortOutputFileAfter() throws ArgumentParserException { - final String[] args = fromString("Test.java -o OtherTest.class"); - final Config actualConfig = Config.fromArgs(args); - - final String expectedInputFilename = "Test.java"; - final String expectedOutputFilename = "OtherTest.class"; - final boolean expectedVerbosity = false; - - assertEquals(expectedInputFilename, actualConfig.getInputFile()); - assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); - assertEquals(expectedVerbosity, actualConfig.isVerbose()); - } - - @Test - void parseInputFileAndLongOutputFileFirst() throws ArgumentParserException { - final String[] args = fromString("--output OtherTest.class Test.java"); - final Config actualConfig = Config.fromArgs(args); - - final String expectedInputFilename = "Test.java"; - final String expectedOutputFilename = "OtherTest.class"; - final boolean expectedVerbosity = false; - - assertEquals(expectedInputFilename, actualConfig.getInputFile()); - assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); - assertEquals(expectedVerbosity, actualConfig.isVerbose()); - } - - @Test - void parseInputFileAndLongOutputFileAfter() throws ArgumentParserException { - final String[] args = fromString("Test.java --output OtherTest.class"); + /** + * The fromArgs method should be able to detect the output file being specified, independent of + * whether the long or short argument was used, it was assigned using = or not or whether + * it was specified before or after the input file. + * + * @throws ArgumentParserException if the arguments can not be parsed; should not happen + */ + @ParameterizedTest + // {short, long} x {before input file, after input file} + @ValueSource(strings = { + "-o=OtherTest.class Test.java", + "--output=OtherTest.class Test.java", + "-o OtherTest.class Test.java", + "--output OtherTest.class Test.java", + "Test.java -o OtherTest.class", + "Test.java --output OtherTest.class"}) + void parseInputFileAndOutputFile(final String string) throws ArgumentParserException { + final String[] args = fromString(string); final Config actualConfig = Config.fromArgs(args); final String expectedInputFilename = "Test.java"; final String expectedOutputFilename = "OtherTest.class"; - final boolean expectedVerbosity = false; assertEquals(expectedInputFilename, actualConfig.getInputFile()); assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); - assertEquals(expectedVerbosity, actualConfig.isVerbose()); } + /** + * The fromArgs method should be able to extract the output file and the verbosity flag if both + * are given at the same time. + * + * @throws ArgumentParserException if the arguments can not be parsed; should not happen + */ @Test - void parseInputFileAndOutputFileAndVerbosity() throws ArgumentParserException { - final String[] args = fromString("--verbose Test.java --output OtherTest.class"); + void parseInputFileAndOutputFileWithVerbosity() throws ArgumentParserException { + final String[] args = fromString("-v -o OtherTest.class Test.java"); final Config actualConfig = Config.fromArgs(args); final String expectedInputFilename = "Test.java"; @@ -152,6 +106,12 @@ void parseInputFileAndOutputFileAndVerbosity() throws ArgumentParserException { assertEquals(expectedVerbosity, actualConfig.isVerbose()); } + /** + * The fromArgs method should be able to extract the compiler stage "scanning" while ignoring + * the case. + * + * @throws ArgumentParserException if the arguments can not be parsed; should not happen + */ @ParameterizedTest @ValueSource(strings = {"scanning", "SCANNING", "sCanning", "sCaNnInG"}) void skipAfterScanning(final String spelling) throws ArgumentParserException { @@ -163,6 +123,12 @@ void skipAfterScanning(final String spelling) throws ArgumentParserException { assertEquals(expectedStage, actualConfig.getStage()); } + /** + * The fromArgs method should be able to extract the compiler stage "parsing" while ignoring + * the case. + * + * @throws ArgumentParserException if the arguments can not be parsed; should not happen + */ @ParameterizedTest @ValueSource(strings = {"parsing", "PARSING", "pArsing", "pArSinG"}) void skipAfterParsing(final String spelling) throws ArgumentParserException { @@ -174,12 +140,18 @@ void skipAfterParsing(final String spelling) throws ArgumentParserException { assertEquals(expectedStage, actualConfig.getStage()); } + /** + * The fromArgs method should set the latest compiler stage correctly when it is not specified + * explicitly. + * + * @throws ArgumentParserException if the arguments can not be parsed; should not happen + */ @Test void defaultStage() throws ArgumentParserException { final String[] args = fromString("Test.java"); final Config actualConfig = Config.fromArgs(args); - final CompilerStage expectedStage = CompilerStage.PARSING; + final CompilerStage expectedStage = CompilerStage.latest(); assertEquals(expectedStage, actualConfig.getStage()); } diff --git a/src/test/java/com/merkrafter/MerkompilerRunTest.java b/src/test/java/com/merkrafter/MerkompilerRunTest.java index dfd9b4e6..4d09f39e 100644 --- a/src/test/java/com/merkrafter/MerkompilerRunTest.java +++ b/src/test/java/com/merkrafter/MerkompilerRunTest.java @@ -49,7 +49,7 @@ class MerkompilerRunTest { * @throws IOException if there is a read/write error in one of the files */ @ParameterizedTest - @ValueSource(strings = "EmptyClass") + @ValueSource(strings = {"EmptyClass", "SmokeClass"}) void scanWithOutputCreatesFile(final String baseFileName) throws ArgumentParserException, IOException { // java source file to read @@ -83,7 +83,7 @@ void scanWithOutputCreatesFile(final String baseFileName) * @throws IOException if there is a read/write error in one of the files */ @ParameterizedTest - @ValueSource(strings = "EmptyClass") + @ValueSource(strings = {"EmptyClass", "SmokeClass"}) void scanWithoutOutput(final String baseFileName) throws ArgumentParserException, IOException { final PrintStream originalOut = System.out; try { // will reset System.out in case of errors diff --git a/src/test/java/com/merkrafter/lexing/ScannerTest.java b/src/test/java/com/merkrafter/lexing/ScannerTest.java index c4a9f381..b7a832f9 100644 --- a/src/test/java/com/merkrafter/lexing/ScannerTest.java +++ b/src/test/java/com/merkrafter/lexing/ScannerTest.java @@ -3,6 +3,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; import java.util.Iterator; import java.util.LinkedList; @@ -38,6 +39,48 @@ void setUp() { scanner = new Scanner(stringIterator); } + /** + * The scanner should be able to detect "other" tokens, i.e. special characters that are not + * part of the language. + */ + @ParameterizedTest + @ValueSource(strings = {"$", "\"", "@", "_", "!", "ยง", "%", "&", "|", "^", "\\", "?", "~", "#"}) + void scanOtherTokens(final String string) { + final String programCode = string; + final Token[] expectedTokenList = { + new OtherToken(string, null, 1, 1), new Token(EOF, null, 1, string.length())}; + shouldScan(programCode, expectedTokenList); + } + + /** + * The scanner should be able to detect number arguments. + */ + @ParameterizedTest + // edge cases 0 and MAX_VALUE, one, two and three digit numbers + @ValueSource(longs = {0, 1, 10, 123, Long.MAX_VALUE}) + void scanNormalNumbers(final long number) { + final String programCode = Long.toString(number); + final Token[] expectedTokenList = { + new NumberToken(number, null, 1, 1), + new Token(EOF, null, 1, Long.toString(number).length())}; + shouldScan(programCode, expectedTokenList); + } + + /** + * The scanner should be able to detect special number arguments, i.e. with leading zeros. + */ + @ParameterizedTest + // all values should be decimal 8's, because in JavaSST there are no octal numbers hence these + // value source numbers will cause an error when trying to evaluate them as octal. + @ValueSource(strings = {"08", "008"}) + void scanSpecialNumbers(final String number) { + final long expectedNumber = 8; + final Token[] expectedTokenList = { + new NumberToken(expectedNumber, null, 1, 1), + new Token(EOF, null, 1, number.length())}; + shouldScan(number, expectedTokenList); + } + /** * The scanner should be able to detect keyword arguments. */ @@ -101,9 +144,23 @@ void scanSingleIdentifierWithMixedCase() { @org.junit.jupiter.api.Test void scanAssignmentWithSpaces() { final String programCode = "int result = a + ( b - c ) * d / e;"; - final TokenType[] expectedTokenList = - {KEYWORD, IDENT, ASSIGN, IDENT, PLUS, L_PAREN, IDENT, MINUS, IDENT, R_PAREN, TIMES, - IDENT, DIVIDE, IDENT, SEMICOLON, EOF}; + final TokenType[] expectedTokenList = { + KEYWORD, + IDENT, + ASSIGN, + IDENT, + PLUS, + L_PAREN, + IDENT, + MINUS, + IDENT, + R_PAREN, + TIMES, + IDENT, + DIVIDE, + IDENT, + SEMICOLON, + EOF}; shouldScan(programCode, expectedTokenList); } @@ -126,9 +183,23 @@ void scanSimpleAssignmentWithWhitespace() { @org.junit.jupiter.api.Test void scanAssignmentWithoutWhitespace() { final String programCode = "int result=a+(b-c)*d/e;"; - final TokenType[] expectedTokenList = - {KEYWORD, IDENT, ASSIGN, IDENT, PLUS, L_PAREN, IDENT, MINUS, IDENT, R_PAREN, TIMES, - IDENT, DIVIDE, IDENT, SEMICOLON, EOF}; + final TokenType[] expectedTokenList = { + KEYWORD, + IDENT, + ASSIGN, + IDENT, + PLUS, + L_PAREN, + IDENT, + MINUS, + IDENT, + R_PAREN, + TIMES, + IDENT, + DIVIDE, + IDENT, + SEMICOLON, + EOF}; shouldScan(programCode, expectedTokenList); } @@ -202,8 +273,7 @@ void scanNoBlockComment() { */ @org.junit.jupiter.api.Test void scanAndIgnoreBlockCommentsMultiline() { - final String programCode = - "/*\nThis is a description of the method\n*/public void draw();"; + final String programCode = "/*\nThis is a description of the method\n*/public void draw();"; final TokenType[] expectedTokenList = {KEYWORD, KEYWORD, IDENT, L_PAREN, R_PAREN, SEMICOLON, EOF}; shouldScan(programCode, expectedTokenList); @@ -249,9 +319,19 @@ void scanAndIgnoreAsterisksInComments() { @org.junit.jupiter.api.Test void scanMainFunction() { final String programCode = "public void main(String[] args) {}"; - final TokenType[] expectedTokenList = - {KEYWORD, KEYWORD, IDENT, L_PAREN, IDENT, L_SQ_BRACKET, R_SQ_BRACKET, IDENT, - R_PAREN, L_BRACE, R_BRACE, EOF}; + final TokenType[] expectedTokenList = { + KEYWORD, + KEYWORD, + IDENT, + L_PAREN, + IDENT, + L_SQ_BRACKET, + R_SQ_BRACKET, + IDENT, + R_PAREN, + L_BRACE, + R_BRACE, + EOF}; shouldScan(programCode, expectedTokenList); } @@ -316,9 +396,19 @@ void scanMethodCallWithTwoArguments() { @org.junit.jupiter.api.Test void scanMethodCallWithMultipleArguments() { final String programCode = "int sum = add(a,b,5)"; - final TokenType[] expectedTokenList = - {KEYWORD, IDENT, ASSIGN, IDENT, L_PAREN, IDENT, COMMA, IDENT, COMMA, NUMBER, R_PAREN, - EOF}; + final TokenType[] expectedTokenList = { + KEYWORD, + IDENT, + ASSIGN, + IDENT, + L_PAREN, + IDENT, + COMMA, + IDENT, + COMMA, + NUMBER, + R_PAREN, + EOF}; shouldScan(programCode, expectedTokenList); } diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index 03aa460b..1a742455 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -4,6 +4,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; import java.util.Arrays; @@ -162,193 +163,27 @@ void parseLocalDeclaration() { } /** - * The parser should accept an assignment and a return statement as a statement sequence. - */ - @Test - void parseStatementSequence() { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.ASSIGN, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0), - new KeywordToken(Keyword.RETURN, "", 0, 0), - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatementSequence()); - } - - /** - * The parser should accept an assignment of the result of a binary operation to a variable - * as a statement sequence. + * The parser should be able to parse single statements as statement sequences. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#statements()} */ @ParameterizedTest - @EnumSource(value = TokenType.class, names = {"PLUS", "MINUS", "TIMES", "DIVIDE"}) - void parseAssignmentWithBinOpAsStatementSequence(final TokenType binOp) { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.ASSIGN, "", 0, 0), - new Token(TokenType.IDENT, "", 0, 0), - new Token(binOp, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatementSequence()); - } - - /** - * The parser should accept a simple assignment of a number as a statement sequence. - */ - @Test - void parseAssignmentAsStatementSequence() { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.ASSIGN, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatementSequence()); - } - - /** - * The parser should accept a simple procedure call as a statement sequence. - */ - @Test - void parseProcedureCallAsStatementSequence() { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.L_PAREN, "", 0, 0), - new IdentToken("a", "", 0, 0), - new Token(TokenType.R_PAREN, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#statements") + void parseStatementSequence(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseStatementSequence()); } /** - * The parser should accept a single return keyword as a statement sequence. - */ - @Test - void parseStandaloneReturnAsStatementSequence() { - final Scanner scanner = new TestScanner(new Token[]{ - new KeywordToken(Keyword.RETURN, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1)}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatementSequence()); - } - - /** - * The parser should accept an assignment of the result of a binary operation to a variable - * as a statement. + * The parser should be able to parse statements. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#statements()} */ @ParameterizedTest - @EnumSource(value = TokenType.class, names = {"PLUS", "MINUS", "TIMES", "DIVIDE"}) - void parseAssignmentWithBinOpAsStatement(final TokenType binOp) { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.ASSIGN, "", 0, 0), - new Token(TokenType.IDENT, "", 0, 0), - new Token(binOp, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatement()); - } - - /** - * The parser should accept a simple assignment of a number as a statement. - */ - @Test - void parseAssignmentAsStatement() { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.ASSIGN, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatement()); - } - - /** - * The parser should accept a simple procedure call as a statement. - */ - @Test - void parseProcedureCallAsStatement() { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.L_PAREN, "", 0, 0), - new IdentToken("a", "", 0, 0), - new Token(TokenType.R_PAREN, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatement()); - } - - /** - * The parser should accept a simple if-else construct as a statement, that is an "if" keyword, - * a comparison between an identifier and a number as the condition and blocks with single - * assignments for if and else. - */ - @Test - void parseSimpleIfAsStatement() { - final Scanner scanner = new TestScanner(new Token[]{ - new KeywordToken(Keyword.IF, null, 1, 1), - new Token(TokenType.L_PAREN, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.EQUAL, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.R_PAREN, null, 1, 1), - - new Token(TokenType.L_BRACE, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.ASSIGN, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1), - new Token(TokenType.R_BRACE, null, 1, 1), - - new KeywordToken(Keyword.ELSE, null, 1, 1), - new Token(TokenType.L_BRACE, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.ASSIGN, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1), - new Token(TokenType.R_BRACE, null, 1, 1),}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatement()); - } - - /** - * The parser should accept a simple while loop as a statement, that is a "while" keyword, a - * comparison between an identifier and a number as the condition and a block that has only an - * assignment inside it. - */ - @Test - void parseSimpleWhileAsStatement() { - final Scanner scanner = new TestScanner(new Token[]{ - new KeywordToken(Keyword.WHILE, null, 1, 1), - new Token(TokenType.L_PAREN, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.EQUAL, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.R_PAREN, null, 1, 1), - new Token(TokenType.L_BRACE, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.ASSIGN, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1), - new Token(TokenType.R_BRACE, null, 1, 1),}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatement()); - } - - /** - * The parser should accept a single return keyword as a statement. - */ - @Test - void parseStandaloneReturnAsStatement() { - final Scanner scanner = new TestScanner(new Token[]{ - new KeywordToken(Keyword.RETURN, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1)}); + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#statements") + void parseStatement(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseStatement()); } @@ -365,118 +200,92 @@ void parseType() { } /** - * The parser should accept an assignment of the result of a binary operation to a variable, - * as "a = a*5;". + * The parser should be able to parse assignments. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#assignments()} */ @ParameterizedTest - @EnumSource(value = TokenType.class, names = {"PLUS", "MINUS", "TIMES", "DIVIDE"}) - void parseAssignmentWithBinOp(final TokenType binOp) { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.ASSIGN, "", 0, 0), - new Token(TokenType.IDENT, "", 0, 0), - new Token(binOp, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#assignments") + void parseAssignment(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseAssignment()); } /** - * The parser should accept a direct assignment of a number to a variable ident, as "a = 5;". + * The parser should be able to detect syntactically wrong assignments. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#assignmentsWithoutSemicolon()} */ - @Test - void parseDirectAssignmentOfNumber() { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.ASSIGN, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); + @ParameterizedTest + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#assignmentsWithoutSemicolon") + void parseFaultyAssignment(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); - assertTrue(parser.parseAssignment()); + assertFalse(parser.parseAssignment()); } /** - * The parser should accept a simple procedure call, as "parse();" + * The parser should be able to parse procedure calls. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#procedureCalls()} */ - @Test - void parseProcedureCall() { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.L_PAREN, "", 0, 0), - new IdentToken("a", "", 0, 0), - new Token(TokenType.R_PAREN, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); + @ParameterizedTest + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#procedureCalls") + void parseProcedureCall(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseProcedureCall()); } - /** - * The parser should accept a simple if-else construct, that is an "if" keyword, a comparison - * between an identifier and a number as the condition and blocks with single assignments for - * if and else. + * The parser should be able to parse intern procedure calls. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#internProcedureCalls()} */ - @Test - void parseSimpleIfStatement() { - final Scanner scanner = new TestScanner(new Token[]{ - new KeywordToken(Keyword.IF, null, 1, 1), - new Token(TokenType.L_PAREN, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.EQUAL, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.R_PAREN, null, 1, 1), - - new Token(TokenType.L_BRACE, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.ASSIGN, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1), - new Token(TokenType.R_BRACE, null, 1, 1), + @ParameterizedTest + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#internProcedureCalls") + void parseInternProcedureCall(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseInternProcedureCall()); + } - new KeywordToken(Keyword.ELSE, null, 1, 1), - new Token(TokenType.L_BRACE, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.ASSIGN, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1), - new Token(TokenType.R_BRACE, null, 1, 1),}); + /** + * The parser should be able to parse simple if constructs. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#ifConstructs()} + */ + @ParameterizedTest + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#ifConstructs") + void parseIfStatement(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseIfStatement()); } /** - * The parser should accept a simple while loop, that is a "while" keyword, a comparison - * between an identifier and a number as the condition and a block that has only an assignment - * inside it. + * The parser should be able to parse simple while loops. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#whileLoops()} */ - @Test - void parseSimpleWhileStatement() { - final Scanner scanner = new TestScanner(new Token[]{ - new KeywordToken(Keyword.WHILE, null, 1, 1), - new Token(TokenType.L_PAREN, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.EQUAL, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.R_PAREN, null, 1, 1), - new Token(TokenType.L_BRACE, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.ASSIGN, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1), - new Token(TokenType.R_BRACE, null, 1, 1),}); + @ParameterizedTest + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#whileLoops") + void parseWhileStatement(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseWhileStatement()); } /** - * The parser should accept a single return statement. + * The parser should be able to parse return statements. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#returnStatements()} */ - @Test - void parseStandaloneReturnStatement() { - final Scanner scanner = new TestScanner(new Token[]{ - new KeywordToken(Keyword.RETURN, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1)}); + @ParameterizedTest + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#returnStatements") + void parseReturnStatement(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseReturnStatement()); } @@ -504,30 +313,27 @@ void parseEmptyActualParameters() { } /** - * The parser should accept a single comparison between an ident and a number as an expression. + * The parser should be able to parse expressions. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#expressions()} */ @ParameterizedTest - @EnumSource(value = TokenType.class, names = { - "LOWER", "LOWER_EQUAL", "EQUAL", "GREATER_EQUAL", "GREATER"}) - void parseSingleComparisonAsExpression(final TokenType comparisonType) { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(comparisonType, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0)}); + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#expressions") + void parseExpression(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseExpression()); } /** - * The parser should accept a single addition/subtraction as a simple expression. + * The parser should be able to parse simple expressions. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#simpleExpressions()} */ @ParameterizedTest - @EnumSource(value = TokenType.class, names = {"PLUS", "MINUS"}) - void parseSimpleExpression(final TokenType tokenType) { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(tokenType, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0)}); + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#simpleExpressions") + void parseSimpleExpression(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseSimpleExpression()); } diff --git a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java new file mode 100644 index 00000000..fa37df36 --- /dev/null +++ b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java @@ -0,0 +1,453 @@ +package com.merkrafter.parsing; + +import com.merkrafter.lexing.*; + +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Stream; + +/**** + * This class serves as a test data provider for ParserTest. + * All non-private methods of this class are static and return a stream + * of TokenWrappers (essentially lists of Tokens) that satisfy certain syntax criteria. + *

+ * The methods are organized hierarchically which means that {@link #statements()} joins the tokens + * from {@link #assignments()}, {@link #returnStatements()} and some other, for instance. + *

+ * This file also defines some static methods that allow the fast creation of + * tokens without the need to specify the filename, line and position numbers + * as they are not relevant for the syntax analysis tests. + * + * @version v0.2.0 + * @author merkrafter + ***************************************************************/ + +class ParserTestDataProvider { + // METHODS + //============================================================== + // public methods + //-------------------------------------------------------------- + + /** + * This method generates a stream of TokenWrappers that are valid statements. + * They are a union of assignments, procedure calls, if, while and return statements. + * + * @return a stream of TokenWrappers that define the test data + */ + static Stream statements() { + return Stream.of(assignments(), + procedureCalls(), + ifConstructs(), + whileLoops(), + returnStatements()).flatMap(i -> i); + } + + /** + * This method generates a stream of TokenWrappers that are valid assignments EXCEPT they're + * lacking the ending semicolon. + * + * @return a stream of TokenWrappers that define the test data + */ + static Stream assignmentsWithoutSemicolon() { + return simpleExpressions().map( + // add all simple expressions at the end of "a = " + expression -> new TokenWrapper().add(tokenFrom("a")) + .add(tokenFrom(TokenType.ASSIGN)) + .add(expression)); + } + + /** + * This method generates a stream of TokenWrappers that are valid assignments. + * This method returns the same TokenWrappers as + * the {@link #assignmentsWithoutSemicolon() assignmentsWithoutSemicolon} method does, but with + * semicolons appended. + * + * @return a stream of TokenWrappers that define the test data + */ + static Stream assignments() { + return assignmentsWithoutSemicolon().map(tokenWrapper -> tokenWrapper.add(tokenFrom( + TokenType.SEMICOLON))); + } + + /** + * This method generates a stream of TokenWrappers that are valid procedure calls. + * This method returns the same TokenWrappers as the {@link #internProcedureCalls()}} method + * does, but with semicolons appended. + * + * @return a stream of TokenWrappers that define the test data + */ + static Stream procedureCalls() { + return internProcedureCalls().map(tokenWrapper -> tokenWrapper.add(tokenFrom(TokenType.SEMICOLON))); + } + + /** + * This method generates a stream of TokenWrappers that are valid intern procedure calls. + * These include calls with empty argument lists, one and two element arguments lists, and + * expressions as arguments. + * + * @return a stream of TokenWrappers that define the test data + */ + static Stream internProcedureCalls() { + return Stream.of( + // a call of an intern procedure without arguments + new TokenWrapper().add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom(TokenType.R_PAREN)), + + // a call of an intern procedure with a single identifier as its argument + new TokenWrapper().add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.R_PAREN)), + + // a call of an intern procedure with a single number as its argument + new TokenWrapper().add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom(TokenType.NUMBER)) + .add(tokenFrom(TokenType.R_PAREN)), + + // a call of an intern procedure with two identifiers as arguments + new TokenWrapper().add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.COMMA)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.R_PAREN)), + + // a call of an intern procedure with an expression like a*b/2 as argument + new TokenWrapper().add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.PLUS)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.DIVIDE)) + .add(tokenFrom(TokenType.NUMBER)) + .add(tokenFrom(TokenType.R_PAREN))); + } + + /** + * This method generates a stream of TokenWrappers that are valid if constructs. + * In particular, the if and else bodies are single assignments, while the comparison is being + * made between an identifier and a number. + * + * @return a stream of TokenWrappers that define the test data + */ + static Stream ifConstructs() { + return Stream.of( + // if constructs with all comparison operators between an ident and a number + // the if and else bodies are simple assignments + Stream.of(TokenType.LOWER_EQUAL, + TokenType.LOWER, + TokenType.EQUAL, + TokenType.GREATER, + TokenType.GREATER_EQUAL) + .map(cmpOp -> new TokenWrapper().add(tokenFrom(Keyword.IF)) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(cmpOp)) + .add(tokenFrom(TokenType.NUMBER)) + .add(tokenFrom(TokenType.R_PAREN)) + + .add(tokenFrom(TokenType.L_BRACE)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.ASSIGN)) + .add(tokenFrom(TokenType.NUMBER)) + .add(tokenFrom(TokenType.SEMICOLON)) + .add(tokenFrom(TokenType.R_BRACE)) + + .add(tokenFrom(Keyword.ELSE)) + .add(tokenFrom(TokenType.L_BRACE)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.ASSIGN)) + .add(tokenFrom(TokenType.NUMBER)) + .add(tokenFrom(TokenType.SEMICOLON)) + .add(tokenFrom(TokenType.R_BRACE)))) + // merge all the above (outer) streams + .flatMap(i -> i); + } + + /** + * This method generates a stream of TokenWrappers that are valid while loops. + * In particular, the body is a single assignment, while the comparison is being + * made between an identifier and a number. + * + * @return a stream of TokenWrappers that define the test data + */ + static Stream whileLoops() { + return Stream.of( + // while loops with all comparison operators between an ident and a number + // the body is a simple assignment + Stream.of(TokenType.LOWER_EQUAL, + TokenType.LOWER, + TokenType.EQUAL, + TokenType.GREATER, + TokenType.GREATER_EQUAL) + .map(cmpOp -> new TokenWrapper().add(tokenFrom(Keyword.WHILE)) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(cmpOp)) + .add(tokenFrom(TokenType.NUMBER)) + .add(tokenFrom(TokenType.R_PAREN)) + + .add(tokenFrom(TokenType.L_BRACE)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.ASSIGN)) + .add(tokenFrom(TokenType.NUMBER)) + .add(tokenFrom(TokenType.SEMICOLON)) + .add(tokenFrom(TokenType.R_BRACE)))) + // merge all the above (outer) streams + .flatMap(i -> i); + } + + /** + * This method generates a stream of TokenWrappers that are valid return statements. + * These include a single return statement without return value as well as returning + * {@link #simpleExpressions()}. + * + * @return a stream of TokenWrappers that define the test data + */ + static Stream returnStatements() { + return Stream.of( + /* + unparameterized data + */ + Stream.of( + // return keyword without an value to return + new TokenWrapper().add(tokenFrom(Keyword.RETURN)) + .add(tokenFrom(TokenType.SEMICOLON))), + /* + parameterized data + */ + simpleExpressions().map( + // return keyword with simple expressions as return values + tokenWrapper -> new TokenWrapper().add(tokenFrom(Keyword.RETURN)) + .add(tokenWrapper) + .add(tokenFrom(TokenType.SEMICOLON)))) + // merge all the above (outer) streams + .flatMap(i -> i); + } + + /** + * This method generates a stream of TokenWrappers that are valid simple expressions. + * These include pretty basic expressions as single identifiers and numbers, as well as more + * complex expressions that include multiple operators and procedure calls. + * + * @return a stream of TokenWrappers that define the test data + */ + static Stream simpleExpressions() { + return Stream.of( + /* + These TokenWrappers are independent from any operators + */ + Stream.of( + // an identifier with a single letter + new TokenWrapper().add(tokenFrom("a")), + + // an identifier with two letters + new TokenWrapper().add(tokenFrom("ab")), + + // a single number + new TokenWrapper().add(tokenFrom(5)), + + // complex expression including intern procedure calls + // 2*fib(n-1) + fib(n-2) + new TokenWrapper().add(tokenFrom(2)) + .add(tokenFrom(TokenType.TIMES)) + .add(tokenFrom("fib")) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom("n")) + .add(tokenFrom(TokenType.MINUS)) + .add(tokenFrom(1)) + .add(tokenFrom(TokenType.R_PAREN)) + .add(tokenFrom(TokenType.PLUS)) + .add(tokenFrom("fib")) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom("n")) + .add(tokenFrom(TokenType.MINUS)) + .add(tokenFrom(2)) + .add(tokenFrom(TokenType.R_PAREN)), + + // complex expression with multiplication, addition and subtraction + // " a*a + b*b - c*c + new TokenWrapper().add(tokenFrom("a")) + .add(tokenFrom(TokenType.TIMES)) + .add(tokenFrom("a")) + .add(tokenFrom(TokenType.PLUS)) + .add(tokenFrom("b")) + .add(tokenFrom(TokenType.TIMES)) + .add(tokenFrom("b")) + .add(tokenFrom(TokenType.MINUS)) + .add(tokenFrom("c")) + .add(tokenFrom(TokenType.TIMES)) + .add(tokenFrom("c"))), + /* + These TokenWrappers are multiplied by using the 4 elementary arithmetic operations + */ + Stream.of(TokenType.PLUS, TokenType.MINUS, TokenType.TIMES, TokenType.DIVIDE) + .flatMap(operator -> Stream.of( + // simple offset of an ident and a number against each other + // with the ident being the first argument + new TokenWrapper().add(tokenFrom("a")) + .add(tokenFrom(operator)) + .add(tokenFrom(5)), + + // simple offset of two idents against each other + new TokenWrapper().add(tokenFrom("a")) + .add(tokenFrom(operator)) + .add(tokenFrom("b")), + + // simple offset of a number and an ident against each other + // with the ident being the second argument + new TokenWrapper().add(tokenFrom(3)) + .add(tokenFrom(operator)) + .add(tokenFrom("b")), + + // simple offset of two numbers against each other + new TokenWrapper().add(tokenFrom(3)) + .add(tokenFrom(operator)) + .add(tokenFrom(5)), + + // chain of 4 idents and 3 operators + new TokenWrapper().add(tokenFrom("a")) + .add(tokenFrom(operator)) + .add(tokenFrom("b")) + .add(tokenFrom(operator)) + .add(tokenFrom("c")) + .add(tokenFrom(operator)) + .add(tokenFrom("d"))))) + + // merge all the above (outer) streams + .flatMap(i -> i); + } + + /** + * This method generates a stream of TokenWrappers that are valid expressions. + * These include {@link #simpleExpressions()} as well as comparisons between those and idents. + * + * @return a stream of TokenWrappers that define the test data + */ + static Stream expressions() { + return Stream.of( + /* + Every simple expression is an expression as well + */ + simpleExpressions(), + + /* + Comparisons of all simple expressions with an identifier + */ + Stream.of(TokenType.LOWER, + TokenType.LOWER_EQUAL, + TokenType.EQUAL, + TokenType.GREATER_EQUAL, + TokenType.GREATER).flatMap( + + cmpOp -> + // simple expression first, then comparison operator, then ident + simpleExpressions().map( + + tokenWrapper -> tokenWrapper.add(tokenFrom(cmpOp)) + .add(tokenFrom("a")))) + + ).flatMap(i -> i); + } + + + // support methods + //-------------------------------------------------------------- + + /** + * Creates a new Token from a TokenType by setting file name, line and position number to some + * default values in order to make increase the readability of test cases. + * + * @return a basic Token with the given TokenType set + */ + static Token tokenFrom(final TokenType type) { + return new Token(type, null, 1, 1); + } + + /** + * Creates a new Token from a Keyword by setting file name, line and position number to some + * default values in order to make increase the readability of test cases. + * + * @return a KeywordToken with the given Keyword set + */ + static Token tokenFrom(final Keyword keyword) { + return new KeywordToken(keyword, null, 1, 1); + } + + /** + * Creates a new Token from an identifier string by setting file name, line and position number + * to some default values in order to make increase the readability of test cases. + * + * @return an IdentToken with the given identifier set + */ + static Token tokenFrom(final String identifier) { + return new IdentToken(identifier, null, 1, 1); + } + + /** + * Creates a new Token from a number string by setting file name, line and position number to + * some default values in order to make increase the readability of test cases. + * + * @return a NumberToken with the given number set + */ + static Token tokenFrom(final long number) { + return new NumberToken(number, null, 1, 1); + } + + + // inner classes + //-------------------------------------------------------------- + + /** + * This class serves as a wrapper around a list of tokens as directly passing around lists/arrays + * of tokens from the provider to the test methods does not work, as they're merged into one big + * stream of Tokens. + */ + static class TokenWrapper { + private List tokenList; + + TokenWrapper() { + tokenList = new LinkedList<>(); + } + + /** + * Adds the given token at the end of the token list and returns this TokenWrapper instance. + * + * @param token a token to append to this token wrapper + * @return itself in order to allow chaining + */ + TokenWrapper add(final Token token) { + tokenList.add(token); + return this; + } + + /** + * Adds all tokens of the given token wrapper at the end of this wrapper's token list and + * returns this TokenWrapper instance. + * + * @param tokenWrapper a TokenWrapper to append at the end of this wrapper + * @return itself in order to allow chaining + */ + TokenWrapper add(final TokenWrapper tokenWrapper) { + tokenList.addAll(tokenWrapper.tokenList); + return this; + } + + /** + * @return the stored tokens as an array + */ + Token[] getTokens() { + return tokenList.toArray(new Token[0]); + } + + /** + * @return the string representation of the underlying token list + */ + @Override + public String toString() { + return tokenList.toString(); + } + } +} diff --git a/src/test/resources/SmokeClass.expected b/src/test/resources/SmokeClass.expected new file mode 100644 index 00000000..f3d0a37a --- /dev/null +++ b/src/test/resources/SmokeClass.expected @@ -0,0 +1,174 @@ +SmokeClass.java(2,1): KEYWORD(class) +SmokeClass.java(2,7): IDENT(SmokeClass) +SmokeClass.java(2,18): L_BRACE +SmokeClass.java(5,5): KEYWORD(public) +SmokeClass.java(5,12): KEYWORD(void) +SmokeClass.java(5,17): IDENT(main) +SmokeClass.java(5,21): L_PAREN +SmokeClass.java(5,22): R_PAREN +SmokeClass.java(5,24): L_BRACE +SmokeClass.java(6,9): KEYWORD(int) +SmokeClass.java(6,13): IDENT(a) +SmokeClass.java(6,14): SEMICOLON +SmokeClass.java(7,9): KEYWORD(int) +SmokeClass.java(7,13): IDENT(b) +SmokeClass.java(7,14): SEMICOLON +SmokeClass.java(8,9): KEYWORD(int) +SmokeClass.java(8,13): IDENT(c) +SmokeClass.java(8,14): SEMICOLON +SmokeClass.java(9,9): KEYWORD(int) +SmokeClass.java(9,13): IDENT(d) +SmokeClass.java(9,14): SEMICOLON +SmokeClass.java(10,9): KEYWORD(int) +SmokeClass.java(10,13): IDENT(e) +SmokeClass.java(10,14): SEMICOLON +SmokeClass.java(11,9): KEYWORD(int) +SmokeClass.java(11,13): IDENT(result) +SmokeClass.java(11,19): SEMICOLON +SmokeClass.java(12,9): IDENT(a) +SmokeClass.java(12,11): ASSIGN +SmokeClass.java(12,13): NUMBER(1) +SmokeClass.java(12,14): SEMICOLON +SmokeClass.java(13,9): IDENT(b) +SmokeClass.java(13,11): ASSIGN +SmokeClass.java(13,13): NUMBER(2) +SmokeClass.java(13,14): SEMICOLON +SmokeClass.java(14,9): IDENT(c) +SmokeClass.java(14,11): ASSIGN +SmokeClass.java(14,13): NUMBER(3) +SmokeClass.java(14,14): SEMICOLON +SmokeClass.java(15,9): IDENT(d) +SmokeClass.java(15,11): ASSIGN +SmokeClass.java(15,13): NUMBER(4) +SmokeClass.java(15,14): SEMICOLON +SmokeClass.java(16,9): IDENT(e) +SmokeClass.java(16,11): ASSIGN +SmokeClass.java(16,13): NUMBER(5) +SmokeClass.java(16,14): SEMICOLON +SmokeClass.java(17,9): IDENT(result) +SmokeClass.java(17,16): ASSIGN +SmokeClass.java(17,18): IDENT(a) +SmokeClass.java(17,19): PLUS +SmokeClass.java(17,20): L_PAREN +SmokeClass.java(17,21): IDENT(b) +SmokeClass.java(17,22): MINUS +SmokeClass.java(17,23): IDENT(c) +SmokeClass.java(17,24): R_PAREN +SmokeClass.java(17,25): TIMES +SmokeClass.java(17,26): IDENT(d) +SmokeClass.java(17,27): DIVIDE +SmokeClass.java(17,28): IDENT(e) +SmokeClass.java(17,29): SEMICOLON +SmokeClass.java(18,9): KEYWORD(if) +SmokeClass.java(18,12): L_PAREN +SmokeClass.java(18,13): IDENT(a) +SmokeClass.java(18,16): LOWER_EQUAL +SmokeClass.java(18,18): IDENT(b) +SmokeClass.java(18,19): R_PAREN +SmokeClass.java(18,21): L_BRACE +SmokeClass.java(18,22): IDENT(println) +SmokeClass.java(18,29): L_PAREN +SmokeClass.java(18,30): IDENT(a) +SmokeClass.java(18,31): R_PAREN +SmokeClass.java(18,32): SEMICOLON +SmokeClass.java(18,33): R_BRACE +SmokeClass.java(18,35): KEYWORD(else) +SmokeClass.java(18,40): L_BRACE +SmokeClass.java(18,41): IDENT(println) +SmokeClass.java(18,48): L_PAREN +SmokeClass.java(18,49): IDENT(b) +SmokeClass.java(18,50): R_PAREN +SmokeClass.java(18,51): SEMICOLON +SmokeClass.java(18,52): R_BRACE +SmokeClass.java(19,9): KEYWORD(if) +SmokeClass.java(19,12): L_PAREN +SmokeClass.java(19,13): IDENT(a) +SmokeClass.java(19,15): LOWER +SmokeClass.java(19,18): IDENT(b) +SmokeClass.java(19,19): R_PAREN +SmokeClass.java(19,21): L_BRACE +SmokeClass.java(19,22): IDENT(println) +SmokeClass.java(19,29): L_PAREN +SmokeClass.java(19,30): IDENT(a) +SmokeClass.java(19,31): R_PAREN +SmokeClass.java(19,32): SEMICOLON +SmokeClass.java(19,33): R_BRACE +SmokeClass.java(19,35): KEYWORD(else) +SmokeClass.java(19,40): L_BRACE +SmokeClass.java(19,41): IDENT(println) +SmokeClass.java(19,48): L_PAREN +SmokeClass.java(19,49): IDENT(b) +SmokeClass.java(19,50): R_PAREN +SmokeClass.java(19,51): SEMICOLON +SmokeClass.java(19,52): R_BRACE +SmokeClass.java(20,9): KEYWORD(if) +SmokeClass.java(20,12): L_PAREN +SmokeClass.java(20,13): IDENT(a) +SmokeClass.java(20,16): EQUAL +SmokeClass.java(20,18): IDENT(b) +SmokeClass.java(20,19): R_PAREN +SmokeClass.java(20,21): L_BRACE +SmokeClass.java(20,22): IDENT(println) +SmokeClass.java(20,29): L_PAREN +SmokeClass.java(20,30): IDENT(a) +SmokeClass.java(20,31): R_PAREN +SmokeClass.java(20,32): SEMICOLON +SmokeClass.java(20,33): R_BRACE +SmokeClass.java(20,35): KEYWORD(else) +SmokeClass.java(20,40): L_BRACE +SmokeClass.java(20,41): IDENT(println) +SmokeClass.java(20,48): L_PAREN +SmokeClass.java(20,49): IDENT(b) +SmokeClass.java(20,50): R_PAREN +SmokeClass.java(20,51): SEMICOLON +SmokeClass.java(20,52): R_BRACE +SmokeClass.java(21,9): KEYWORD(if) +SmokeClass.java(21,12): L_PAREN +SmokeClass.java(21,13): IDENT(a) +SmokeClass.java(21,16): GREATER +SmokeClass.java(21,18): IDENT(b) +SmokeClass.java(21,19): R_PAREN +SmokeClass.java(21,21): L_BRACE +SmokeClass.java(21,22): IDENT(println) +SmokeClass.java(21,29): L_PAREN +SmokeClass.java(21,30): IDENT(a) +SmokeClass.java(21,31): R_PAREN +SmokeClass.java(21,32): SEMICOLON +SmokeClass.java(21,33): R_BRACE +SmokeClass.java(21,35): KEYWORD(else) +SmokeClass.java(21,40): L_BRACE +SmokeClass.java(21,41): IDENT(println) +SmokeClass.java(21,48): L_PAREN +SmokeClass.java(21,49): IDENT(b) +SmokeClass.java(21,50): R_PAREN +SmokeClass.java(21,51): SEMICOLON +SmokeClass.java(21,52): R_BRACE +SmokeClass.java(22,9): KEYWORD(if) +SmokeClass.java(22,12): L_PAREN +SmokeClass.java(22,13): IDENT(a) +SmokeClass.java(22,16): GREATER_EQUAL +SmokeClass.java(22,18): IDENT(b) +SmokeClass.java(22,19): R_PAREN +SmokeClass.java(22,21): L_BRACE +SmokeClass.java(22,22): IDENT(println) +SmokeClass.java(22,29): L_PAREN +SmokeClass.java(22,30): IDENT(a) +SmokeClass.java(22,31): R_PAREN +SmokeClass.java(22,32): SEMICOLON +SmokeClass.java(22,33): R_BRACE +SmokeClass.java(22,35): KEYWORD(else) +SmokeClass.java(22,40): L_BRACE +SmokeClass.java(22,41): IDENT(println) +SmokeClass.java(22,48): L_PAREN +SmokeClass.java(22,49): IDENT(b) +SmokeClass.java(22,50): R_PAREN +SmokeClass.java(22,51): SEMICOLON +SmokeClass.java(22,52): R_BRACE +SmokeClass.java(23,9): IDENT(println) +SmokeClass.java(23,16): L_PAREN +SmokeClass.java(23,17): IDENT(result) +SmokeClass.java(23,23): R_PAREN +SmokeClass.java(23,24): SEMICOLON +SmokeClass.java(24,5): R_BRACE +SmokeClass.java(25,1): R_BRACE +SmokeClass.java(26,0): EOF