From b5c9c57bb0acb8f157ac7dbb9cf444fab06c5fe1 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Tue, 19 Nov 2019 08:00:20 +0100 Subject: [PATCH 001/135] Start version 0.2.0 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 72dcf5a9..81854ab7 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ Merkompiler Merkompiler - 0.1.0-SNAPSHOT + 0.2.0-SNAPSHOT org.junit.jupiter @@ -78,4 +78,4 @@ - \ No newline at end of file + From 386bc1d28de47407957373b2997905d4fd436e18 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 24 Nov 2019 02:54:35 +0100 Subject: [PATCH 002/135] Add COMMA to TokenType enum --- src/main/java/com/merkrafter/lexing/TokenType.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/merkrafter/lexing/TokenType.java b/src/main/java/com/merkrafter/lexing/TokenType.java index ce9a6d66..a1e31701 100644 --- a/src/main/java/com/merkrafter/lexing/TokenType.java +++ b/src/main/java/com/merkrafter/lexing/TokenType.java @@ -19,6 +19,7 @@ public enum TokenType { R_BRACE, L_SQ_BRACKET, R_SQ_BRACKET, + COMMA, SEMICOLON, EQUAL, LOWER_EQUAL, From 8f041180063e7759ac2f35f698b482af7c86c1bd Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 24 Nov 2019 03:06:27 +0100 Subject: [PATCH 003/135] Add test cases for comma detection --- .../com/merkrafter/lexing/ScannerTest.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/test/java/com/merkrafter/lexing/ScannerTest.java b/src/test/java/com/merkrafter/lexing/ScannerTest.java index be01fc38..19759498 100644 --- a/src/test/java/com/merkrafter/lexing/ScannerTest.java +++ b/src/test/java/com/merkrafter/lexing/ScannerTest.java @@ -283,6 +283,30 @@ void scanEmptyClass() { shouldScan(programCode, expectedTokenList); } + /** + * The scanner should be able to tokenize a method call with two comma separated arguments. + */ + @org.junit.jupiter.api.Test + void scanMethodCallWithTwoArguments() { + final String programCode = "int sum = add(a,b)"; + final TokenType[] expectedTokenList = + {IDENT, IDENT, ASSIGN, IDENT, L_PAREN, IDENT, COMMA, IDENT, R_PAREN, EOF}; + shouldScan(programCode, expectedTokenList); + } + + /** + * The scanner should be able to tokenize a method call with three comma separated arguments. + * Two of them are identifiers and the third is a number constant. + */ + @org.junit.jupiter.api.Test + void scanMethodCallWithMultipleArguments() { + final String programCode = "int sum = add(a,b,5)"; + final TokenType[] expectedTokenList = + {IDENT, IDENT, ASSIGN, IDENT, L_PAREN, IDENT, COMMA, IDENT, COMMA, NUMBER, R_PAREN, + EOF}; + shouldScan(programCode, expectedTokenList); + } + /** * Collects all tokens emitted by this scanner. * From ba472e467a96f4cb90cebe238291f854cd2c7190 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 24 Nov 2019 03:06:57 +0100 Subject: [PATCH 004/135] Make scanner detect commas --- src/main/java/com/merkrafter/lexing/Scanner.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/merkrafter/lexing/Scanner.java b/src/main/java/com/merkrafter/lexing/Scanner.java index 44562e71..02c8dae6 100644 --- a/src/main/java/com/merkrafter/lexing/Scanner.java +++ b/src/main/java/com/merkrafter/lexing/Scanner.java @@ -325,6 +325,12 @@ public void processToken() { charBuffer = Optional.of(ch); } break; + case ',': + sym = new Token(TokenType.COMMA, filename, line, position); + if (!this.loadNextCharSuccessfully()) { + return; + } + break; case ';': sym = new Token(TokenType.SEMICOLON, filename, line, position); if (!this.loadNextCharSuccessfully()) { From 86e6619059fa9170613d7e43cc31a5b17768ce5b Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 24 Nov 2019 03:54:56 +0100 Subject: [PATCH 005/135] Begin Parser implementation --- .../java/com/merkrafter/parsing/Parser.java | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 src/main/java/com/merkrafter/parsing/Parser.java diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java new file mode 100644 index 00000000..814cd063 --- /dev/null +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -0,0 +1,78 @@ +package com.merkrafter.parsing; + +import com.merkrafter.lexing.Scanner; + +import static com.merkrafter.lexing.TokenType.*; + +/**** + * This class can be used to parse tokens. + * To start parsing, call the parse() method which triggers a recursive descent. + * + * @since v0.2.0 + * @author merkrafter + ***************************************************************/ +public class Parser { + // ATTRIBUTES + //============================================================== + /** + * The scanner that provides the tokens for this parser + */ + private final Scanner scanner; + + // CONSTRUCTORS + //============================================================== + + /**** + * Creates a new Parser based on a scanner that provides tokens. + ***************************************************************/ + public Parser(final Scanner scanner) { + this.scanner = scanner; + this.scanner.processToken(); + } + + // METHODS + //============================================================== + // public methods + //-------------------------------------------------------------- + + /** + * Checks the underlying token iterator for a single number. + * + * @return whether a single NUMBER token comes next + */ + boolean parseNumber() { + if (scanner.getSym().getType() == NUMBER) { + scanner.processToken(); + return true; + } else { + error("Expected number but found " + scanner.getSym().getType().name()); + return false; + } + } + + /** + * Checks the underlying token iterator for a single identifier. + * + * @return whether a single IDENT token comes next + */ + boolean parseIdentifier() { + if (scanner.getSym().getType() == IDENT) { + scanner.processToken(); + return true; + } else { + error("Expected identifier but found " + scanner.getSym().getType().name()); + return false; + } + } + + /** + * Prints the given message to stderr. + * Change soon, as only the main class should be printing. + * + * @param msg the message to print + */ + private static void error(final String msg) { + System.err.println(msg); + } + +} From 40189a8fdb91d191ab89abfdad5717934c22ba37 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 24 Nov 2019 03:57:57 +0100 Subject: [PATCH 006/135] Begin writing a test class for Parser --- .../com/merkrafter/parsing/ParserTest.java | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 src/test/java/com/merkrafter/parsing/ParserTest.java diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java new file mode 100644 index 00000000..5fcc40bc --- /dev/null +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -0,0 +1,85 @@ +package com.merkrafter.parsing; + +import com.merkrafter.lexing.Scanner; +import com.merkrafter.lexing.Token; +import com.merkrafter.lexing.TokenType; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.*; + +class ParserTest { + + /** + * The parser should accept a single number. + */ + @Test + void parseNumber() { + final Scanner scanner = new TestScanner(new Token[]{new Token(TokenType.NUMBER, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseNumber()); + } + + /** + * The parser should not accept an identifier when expecting a number. + */ + @Test + void tryParseNoNumber() { + final Scanner scanner = new TestScanner(new Token[]{new Token(TokenType.IDENT, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertFalse(parser.parseNumber()); + } + + /** + * The parser should accept a single identifier. + */ + @Test + void parseIdentifier() { + final Scanner scanner = new TestScanner(new Token[]{new Token(TokenType.IDENT, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseIdentifier()); + } + + /** + * The parser should not accept a number when expecting an identifier. + */ + @Test + void tryParseNoIdentifier() { + final Scanner scanner = new TestScanner(new Token[]{new Token(TokenType.NUMBER, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertFalse(parser.parseIdentifier()); + } + + /** + * This class serves as a mock for scanner and will likely be removed later on when Parsers can + * accept token iterators. + */ + private static class TestScanner extends Scanner { + private final Token[] tokens; + private int index; + + TestScanner(final Token[] tokens) { + // the input that is not needed here + super(Arrays.asList(new Character[]{' '}).iterator()); + this.tokens = tokens; + index = -1; // one step before first token + } + + /** + * Simply advances the index inside the token array. + */ + @Override + public void processToken() { + index++; + } + + /** + * @return the token at the current index + */ + @Override + public Token getSym() { + return tokens[index]; + } + } +} \ No newline at end of file From fb013a3309b729a5013e7c5315051a25d8123438 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 24 Nov 2019 03:57:57 +0100 Subject: [PATCH 007/135] Begin writing a test class for Parser --- .../com/merkrafter/parsing/ParserTest.java | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 src/test/java/com/merkrafter/parsing/ParserTest.java diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java new file mode 100644 index 00000000..068d1c67 --- /dev/null +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -0,0 +1,89 @@ +package com.merkrafter.parsing; + +import com.merkrafter.lexing.Scanner; +import com.merkrafter.lexing.Token; +import com.merkrafter.lexing.TokenType; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.*; + +class ParserTest { + + /** + * The parser should accept a single number. + */ + @Test + void parseNumber() { + final Scanner scanner = new TestScanner(new Token[]{new Token(TokenType.NUMBER, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseNumber()); + } + + /** + * The parser should not accept another token when expecting a number. + */ + @ParameterizedTest + @EnumSource(value = TokenType.class, names = {"NUMBER"}, mode = EnumSource.Mode.EXCLUDE) + void tryParseNoNumber(final TokenType tokenType) { + final Scanner scanner = new TestScanner(new Token[]{new Token(tokenType, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertFalse(parser.parseNumber()); + } + + /** + * The parser should accept a single identifier. + */ + @Test + void parseIdentifier() { + final Scanner scanner = new TestScanner(new Token[]{new Token(TokenType.IDENT, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseIdentifier()); + } + + /** + * The parser should not accept another token when expecting an identifier. + */ + @ParameterizedTest + @EnumSource(value = TokenType.class, names = {"IDENT"}, mode = EnumSource.Mode.EXCLUDE) + void tryParseNoIdentifier(final TokenType tokenType) { + final Scanner scanner = new TestScanner(new Token[]{new Token(tokenType, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertFalse(parser.parseIdentifier()); + } + + /** + * This class serves as a mock for scanner and will likely be removed later on when Parsers can + * accept token iterators. + */ + private static class TestScanner extends Scanner { + private final Token[] tokens; + private int index; + + TestScanner(final Token[] tokens) { + // the input that is not needed here + super(Arrays.asList(new Character[]{' '}).iterator()); + this.tokens = tokens; + index = -1; // one step before first token + } + + /** + * Simply advances the index inside the token array. + */ + @Override + public void processToken() { + index++; + } + + /** + * @return the token at the current index + */ + @Override + public Token getSym() { + return tokens[index]; + } + } +} \ No newline at end of file From d1fe561ede293eb0a4d4777ef4c45bcb2b29f7cc Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 24 Nov 2019 16:26:02 +0100 Subject: [PATCH 008/135] Write empty parsing methods --- .../java/com/merkrafter/parsing/Parser.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 814cd063..bf9945fd 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -35,6 +35,37 @@ public Parser(final Scanner scanner) { // public methods //-------------------------------------------------------------- + /** + * Parses the tokens given by the underlying token iterator. + */ + public boolean parse() { + return false; + } + + boolean parseInternProcedureCall() { + return false; + } + + boolean parseActualParameters() { + return false; + } + + boolean parseExpression() { + return false; + } + + boolean parseSimpleExpression() { + return false; + } + + boolean parseTerm() { + return false; + } + + boolean parseFactor() { + return false; + } + /** * Checks the underlying token iterator for a single number. * From bb69777330e2b05011792203775dd2216db05a74 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 24 Nov 2019 16:26:52 +0100 Subject: [PATCH 009/135] Add happy path test cases for parser methods --- .../com/merkrafter/parsing/ParserTest.java | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index 068d1c67..e653f49e 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -13,6 +13,149 @@ class ParserTest { + /** + * The parser should accept a single pair of parentheses as actual parameters. + */ + @Test + void parseEmptyActualParameters() { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.L_PAREN, "", 0, 0), new Token(TokenType.R_PAREN, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseActualParameters()); + } + + /** + * The parser should accept a single comparison between an ident and a number as an expression. + */ + @ParameterizedTest + @EnumSource(value = TokenType.class, names = { + "LOWER", "LOWER_EQUAL", "EQUAL", "GREATER_EQUAL", "GREATER"}) + void parseSingleComparisonAsExpression(final TokenType comparisonType) { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.IDENT, "", 0, 0), + new Token(comparisonType, "", 0, 0), + new Token(TokenType.NUMBER, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseExpression()); + } + + /** + * The parser should accept a single addition/subtraction as a simple expression. + */ + @ParameterizedTest + @EnumSource(value = TokenType.class, names = {"PLUS", "MINUS"}) + void parseSimpleExpression(final TokenType tokenType) { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.IDENT, "", 0, 0), + new Token(tokenType, "", 0, 0), + new Token(TokenType.NUMBER, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseSimpleExpression()); + } + + /** + * The parser should accept a single multiplication/division as a term. + */ + @ParameterizedTest + @EnumSource(value = TokenType.class, names = {"TIMES", "DIVIDE"}) + void parseTerm(final TokenType tokenType) { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.IDENT, "", 0, 0), + new Token(tokenType, "", 0, 0), + new Token(TokenType.NUMBER, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseTerm()); + } + + /** + * The parser should accept an intern procedure call without arguments as a factor. + */ + @Test + void parseInternProcedureCallWithoutArgsAsFactor() { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.IDENT, "", 0, 0), + new Token(TokenType.L_PAREN, "", 0, 0), + new Token(TokenType.R_PAREN, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseFactor()); + } + + /** + * The parser should accept an intern procedure call with only one argument (number or ident) + * as a factor. + */ + @ParameterizedTest + @EnumSource(value = TokenType.class, names = {"NUMBER", "IDENT"}) + void parseInternProcedureCallWithOneArgAsFactor(final TokenType tokenType) { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.IDENT, "", 0, 0), + new Token(TokenType.L_PAREN, "", 0, 0), + new Token(tokenType, "", 0, 0), + new Token(TokenType.R_PAREN, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseFactor()); + } + + /** + * The parser should accept an intern procedure call with two comma-separated arguments (number + * and an identifier) as a factor. + */ + @Test + void parseInternProcedureCallWithTwoArgsAsFactor() { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.IDENT, "", 0, 0), + new Token(TokenType.L_PAREN, "", 0, 0), + new Token(TokenType.NUMBER, "", 0, 0), + new Token(TokenType.COMMA, "", 0, 0), + new Token(TokenType.IDENT, "", 0, 0), + new Token(TokenType.R_PAREN, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseFactor()); + } + + /** + * The parser should accept a simple expression representing a binary operation between a + * number and an identifier as a factor. + */ + @ParameterizedTest + @EnumSource(value = TokenType.class, names = {"PLUS", "MINUS", "TIMES", "DIVIDE"}) + void parseBinOpExpressionAsFactor(final TokenType binOp) { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.L_PAREN, "", 0, 0), + new Token(TokenType.NUMBER, "", 0, 0), + new Token(binOp, "", 0, 0), + new Token(TokenType.IDENT, "", 0, 0), + new Token(TokenType.R_PAREN, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseFactor()); + } + + /** + * The parser should accept a simple expression (consisting only of parentheses and + * identifier/number in between) as a factor. + */ + @ParameterizedTest + @EnumSource(value = TokenType.class, names = {"IDENT", "NUMBER"}) + void parseSimpleExpressionAsFactor(final TokenType tokenType) { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.L_PAREN, "", 0, 0), + new Token(tokenType, "", 0, 0), + new Token(TokenType.R_PAREN, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseFactor()); + } + + /** + * The parser should accept a single identifier or number as a factor. + */ + @ParameterizedTest + @EnumSource(value = TokenType.class, names = {"IDENT", "NUMBER"}) + void parseIdentifierOrNumberAsFactor(final TokenType tokenType) { + final Scanner scanner = new TestScanner(new Token[]{new Token(tokenType, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseFactor()); + } + /** * The parser should accept a single number. */ From 1bfc6dbf66d077fe4d62643171b315ab55b93986 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 24 Nov 2019 17:45:22 +0100 Subject: [PATCH 010/135] Make the test scanner return an EOF token when index out of bounds --- src/test/java/com/merkrafter/parsing/ParserTest.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index e653f49e..b2ff65d5 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -222,11 +222,15 @@ public void processToken() { } /** - * @return the token at the current index + * @return the token at the current index or TokenType.EOF if index is out of bounds */ @Override public Token getSym() { - return tokens[index]; + if (index == tokens.length) { + return new Token(TokenType.EOF, "", 0, 0); + } else { + return tokens[index]; + } } } } \ No newline at end of file From 11021e3cfa422e97db7288e74c3e7e1d6ced0fcb Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 24 Nov 2019 17:53:51 +0100 Subject: [PATCH 011/135] Implement the different parsing methods for Parser --- .../java/com/merkrafter/parsing/Parser.java | 84 +++++++++++++++++-- 1 file changed, 79 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index bf9945fd..d51bc05c 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -43,27 +43,101 @@ public boolean parse() { } boolean parseInternProcedureCall() { + if (parseIdentifier()) { + scanner.processToken(); + return parseActualParameters(); + } return false; } boolean parseActualParameters() { - return false; + if (scanner.getSym().getType() == L_PAREN) { + scanner.processToken(); + if (parseExpression()) { + while (scanner.getSym().getType() == COMMA) { + scanner.processToken(); + if (!parseExpression()) { + error("Expected expression after comma in actual parameters"); + return false; + } + } + } else { + return true; // it is okay if no expression comes here + } + } else { + error("Expected left parenthesis in actual parameters"); + return false; + } + if (scanner.getSym().getType() == R_PAREN) { + return true; + } else { + error("Wrong use of parenthesis"); + return false; + } } boolean parseExpression() { - return false; + if (parseSimpleExpression()) { + switch (scanner.getSym().getType()) { + case EQUAL: + case LOWER: + case LOWER_EQUAL: + case GREATER: + case GREATER_EQUAL: + scanner.processToken(); + return parseSimpleExpression(); + default: + return true; + } + } else { + error("Error parsing expression"); + return false; + } } boolean parseSimpleExpression() { - return false; + boolean success = parseTerm(); + while (success) { + if (scanner.getSym().getType() == PLUS || scanner.getSym().getType() == MINUS) { + scanner.processToken(); + success = parseTerm(); + } else { + break; + } + } + return success; } boolean parseTerm() { - return false; + boolean success = parseFactor(); + while (success) { + if (scanner.getSym().getType() == TIMES || scanner.getSym().getType() == DIVIDE) { + scanner.processToken(); + success = parseFactor(); + } else { + break; + } + } + return success; } boolean parseFactor() { - return false; + if (parseIdentifier()) { + return true; + } else if (parseNumber()) { + return true; + } else if (scanner.getSym().getType() == L_PAREN) { + scanner.processToken(); + final boolean success = parseExpression(); + if (scanner.getSym().getType() == R_PAREN) { + scanner.processToken(); + } else { + error("Wrong use of parenthesis"); + return false; + } + return success; // whether the above parseExpression() was successful + } + return parseInternProcedureCall(); } /** From 3ad24ae6b21d3bb265c84bdc5d070dab7e1ad1a8 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 24 Nov 2019 18:42:47 +0100 Subject: [PATCH 012/135] Add parser prototypes for type, assignment, procedurecall and return --- src/main/java/com/merkrafter/parsing/Parser.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index d51bc05c..9bd1ebe8 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -42,6 +42,18 @@ public boolean parse() { return false; } + boolean parseType() { + return false; + } + + boolean parseAssignment() { + return false; + } + + boolean parseProcedureCall() { + return false; + } + boolean parseInternProcedureCall() { if (parseIdentifier()) { scanner.processToken(); @@ -50,6 +62,10 @@ boolean parseInternProcedureCall() { return false; } + boolean parseReturnStatement() { + return false; + } + boolean parseActualParameters() { if (scanner.getSym().getType() == L_PAREN) { scanner.processToken(); From e35aed74fa6231655be4c59a2656fb5de5262777 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 24 Nov 2019 19:27:22 +0100 Subject: [PATCH 013/135] Add happy path test cases for new parser methods --- .../com/merkrafter/parsing/ParserTest.java | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index b2ff65d5..21f355c9 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -13,6 +13,74 @@ class ParserTest { + /** + * The parser should accept a single "int" as a type. + */ + @Test + void parseType() { + // TODO implement this when int token is available + final Scanner scanner = new TestScanner(new Token[]{}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseType()); + } + + /** + * The parser should accept an assignment of the result of a binary operation to a variable, + * as "a = a*5;". + */ + @ParameterizedTest + @EnumSource(value = TokenType.class, names = {"PLUS", "MINUS", "TIMES", "DIVIDE"}) + void parseAssignmentWithBinOp(final TokenType binOp) { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.IDENT, "", 0, 0), + new Token(TokenType.ASSIGN, "", 0, 0), + new Token(TokenType.IDENT, "", 0, 0), + new Token(binOp, "", 0, 0), + new Token(TokenType.NUMBER, "", 0, 0), + new Token(TokenType.SEMICOLON, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseAssignment()); + } + + /** + * The parser should accept a direct assignment of a number to a variable ident, as "a = 5;". + */ + @Test + void parseDirectAssignmentOfNumber() { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.IDENT, "", 0, 0), + new Token(TokenType.ASSIGN, "", 0, 0), + new Token(TokenType.NUMBER, "", 0, 0), + new Token(TokenType.SEMICOLON, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseAssignment()); + } + + /** + * The parser should accept a simple procedure call, as "parse();" + */ + @Test + void parseProcedureCall() { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.IDENT, "", 0, 0), + new Token(TokenType.L_PAREN, "", 0, 0), + new Token(TokenType.R_PAREN, "", 0, 0), + new Token(TokenType.SEMICOLON, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseProcedureCall()); + } + + /** + * The parser should accept a single return statement. + */ + @Test + void parseStandaloneReturnStatement() { + // TODO implement this when return token is available + final Scanner scanner = new TestScanner(new Token[]{}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseReturnStatement()); + } + /** * The parser should accept a single pair of parentheses as actual parameters. */ From 798a6d6326a5461f034d4ee93add45561d70e8d5 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 24 Nov 2019 19:53:14 +0100 Subject: [PATCH 014/135] Use correct junit-jupiter-params version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 81854ab7..d0effd91 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.junit.jupiter junit-jupiter-params - 5.5.2 + 5.4.2 test From aea8126ab01f4959ae2931430646cea18ff503e7 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 24 Nov 2019 20:01:08 +0100 Subject: [PATCH 015/135] Add Keywords enum --- .../java/com/merkrafter/lexing/Keyword.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/main/java/com/merkrafter/lexing/Keyword.java diff --git a/src/main/java/com/merkrafter/lexing/Keyword.java b/src/main/java/com/merkrafter/lexing/Keyword.java new file mode 100644 index 00000000..a5211fda --- /dev/null +++ b/src/main/java/com/merkrafter/lexing/Keyword.java @@ -0,0 +1,19 @@ +package com.merkrafter.lexing; + +/**** + * This enum lists all keywords that can be encountered in JavaSST files. + * + * @since v0.2.0 + * @author merkrafter + ***************************************************************/ +public enum Keyword { + CLASS, + ELSE, + FINAL, + IF, + INT, + PUBLIC, + RETURN, + VOID, + WHILE, +} From 90d369df4a888c68e4ce884e128f9b63630e17ba Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 24 Nov 2019 20:07:46 +0100 Subject: [PATCH 016/135] Add KEYWORD to TokenType --- src/main/java/com/merkrafter/lexing/TokenType.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/merkrafter/lexing/TokenType.java b/src/main/java/com/merkrafter/lexing/TokenType.java index a1e31701..c3a950bf 100644 --- a/src/main/java/com/merkrafter/lexing/TokenType.java +++ b/src/main/java/com/merkrafter/lexing/TokenType.java @@ -6,6 +6,7 @@ * @author merkrafter ***************************************************************/ public enum TokenType { + KEYWORD, IDENT, NUMBER, PLUS, From 912fbe6bf39b5ebac3b911043ae0803fd98dabd8 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 24 Nov 2019 20:09:01 +0100 Subject: [PATCH 017/135] Add basic scanner test case for keywords --- .../java/com/merkrafter/lexing/ScannerTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/test/java/com/merkrafter/lexing/ScannerTest.java b/src/test/java/com/merkrafter/lexing/ScannerTest.java index 19759498..6a296388 100644 --- a/src/test/java/com/merkrafter/lexing/ScannerTest.java +++ b/src/test/java/com/merkrafter/lexing/ScannerTest.java @@ -1,6 +1,8 @@ package com.merkrafter.lexing; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.util.Iterator; import java.util.LinkedList; @@ -36,6 +38,17 @@ void setUp() { scanner = new Scanner(stringIterator); } + /** + * The scanner should be able to detect keyword arguments. + */ + @ParameterizedTest + @EnumSource(Keyword.class) + void scanKeyword(final Keyword keyword) { + final String programCode = keyword.name().toLowerCase(); + final TokenType[] expectedTokenList = {KEYWORD, EOF}; + shouldScan(programCode, expectedTokenList); + } + /** * The scanner should be able to handle an empty string by returning the EOF token. */ From 47d501ed935f61efbbc7702174e0e37eb788ff25 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 14:00:38 +0100 Subject: [PATCH 018/135] Prepare other ScannerTest test cases for keywords --- .../com/merkrafter/lexing/ScannerTest.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/test/java/com/merkrafter/lexing/ScannerTest.java b/src/test/java/com/merkrafter/lexing/ScannerTest.java index 6a296388..e2ac5982 100644 --- a/src/test/java/com/merkrafter/lexing/ScannerTest.java +++ b/src/test/java/com/merkrafter/lexing/ScannerTest.java @@ -100,7 +100,7 @@ void scanSingleIdentifierWithMixedCase() { void scanAssignmentWithSpaces() { final String programCode = "int result = a + ( b - c ) * d / e;"; final TokenType[] expectedTokenList = - {IDENT, IDENT, ASSIGN, IDENT, PLUS, L_PAREN, IDENT, MINUS, IDENT, R_PAREN, TIMES, + {KEYWORD, IDENT, ASSIGN, IDENT, PLUS, L_PAREN, IDENT, MINUS, IDENT, R_PAREN, TIMES, IDENT, DIVIDE, IDENT, SEMICOLON, EOF}; shouldScan(programCode, expectedTokenList); } @@ -112,7 +112,7 @@ void scanAssignmentWithSpaces() { @org.junit.jupiter.api.Test void scanSimpleAssignmentWithWhitespace() { final String programCode = "int\n\t a \n=\n5\t\t\t ;"; - final TokenType[] expectedTokenList = {IDENT, IDENT, ASSIGN, NUMBER, SEMICOLON, EOF}; + final TokenType[] expectedTokenList = {KEYWORD, IDENT, ASSIGN, NUMBER, SEMICOLON, EOF}; shouldScan(programCode, expectedTokenList); } @@ -125,7 +125,7 @@ void scanSimpleAssignmentWithWhitespace() { void scanAssignmentWithoutWhitespace() { final String programCode = "int result=a+(b-c)*d/e;"; final TokenType[] expectedTokenList = - {IDENT, IDENT, ASSIGN, IDENT, PLUS, L_PAREN, IDENT, MINUS, IDENT, R_PAREN, TIMES, + {KEYWORD, IDENT, ASSIGN, IDENT, PLUS, L_PAREN, IDENT, MINUS, IDENT, R_PAREN, TIMES, IDENT, DIVIDE, IDENT, SEMICOLON, EOF}; shouldScan(programCode, expectedTokenList); } @@ -137,7 +137,7 @@ void scanAssignmentWithoutWhitespace() { @org.junit.jupiter.api.Test void scanAndIgnoreStandaloneLineComments() { final String programCode = " //in mph\nint velocity;"; - final TokenType[] expectedTokenList = {IDENT, IDENT, SEMICOLON, EOF}; + final TokenType[] expectedTokenList = {KEYWORD, IDENT, SEMICOLON, EOF}; shouldScan(programCode, expectedTokenList); } @@ -149,7 +149,7 @@ void scanAndIgnoreStandaloneLineComments() { void scanAndIgnoreAppendedLineComments() { final String programCode = "int velocity; //in mph\nint acceleration;"; final TokenType[] expectedTokenList = - {IDENT, IDENT, SEMICOLON, IDENT, IDENT, SEMICOLON, EOF}; + {KEYWORD, IDENT, SEMICOLON, KEYWORD, IDENT, SEMICOLON, EOF}; shouldScan(programCode, expectedTokenList); } @@ -201,9 +201,9 @@ void scanNoBlockComment() { @org.junit.jupiter.api.Test void scanAndIgnoreBlockCommentsMultiline() { final String programCode = - "/*\nThis is a description of the method\n*/public abstract void draw();"; + "/*\nThis is a description of the method\n*/public void draw();"; final TokenType[] expectedTokenList = - {IDENT, IDENT, IDENT, IDENT, L_PAREN, R_PAREN, SEMICOLON, EOF}; + {KEYWORD, KEYWORD, IDENT, L_PAREN, R_PAREN, SEMICOLON, EOF}; shouldScan(programCode, expectedTokenList); } @@ -214,7 +214,7 @@ void scanAndIgnoreBlockCommentsMultiline() { @org.junit.jupiter.api.Test void scanAndIgnoreBlockCommentsInline() { final String programCode = "int a /*a really important variable*/ = 5;"; - final TokenType[] expectedTokenList = {IDENT, IDENT, ASSIGN, NUMBER, SEMICOLON, EOF}; + final TokenType[] expectedTokenList = {KEYWORD, IDENT, ASSIGN, NUMBER, SEMICOLON, EOF}; shouldScan(programCode, expectedTokenList); } @@ -225,7 +225,7 @@ void scanAndIgnoreBlockCommentsInline() { @org.junit.jupiter.api.Test void scanAndIgnoreBlockCommentsAtEndOfLine() { final String programCode = "int a = 5;/*a really important variable*/"; - final TokenType[] expectedTokenList = {IDENT, IDENT, ASSIGN, NUMBER, SEMICOLON, EOF}; + final TokenType[] expectedTokenList = {KEYWORD, IDENT, ASSIGN, NUMBER, SEMICOLON, EOF}; shouldScan(programCode, expectedTokenList); } @@ -246,9 +246,9 @@ void scanAndIgnoreAsterisksInComments() { */ @org.junit.jupiter.api.Test void scanMainFunction() { - final String programCode = "public static void main(String[] args) {}"; + final String programCode = "public void main(String[] args) {}"; final TokenType[] expectedTokenList = - {IDENT, IDENT, IDENT, IDENT, L_PAREN, IDENT, L_SQ_BRACKET, R_SQ_BRACKET, IDENT, + {KEYWORD, KEYWORD, IDENT, L_PAREN, IDENT, L_SQ_BRACKET, R_SQ_BRACKET, IDENT, R_PAREN, L_BRACE, R_BRACE, EOF}; shouldScan(programCode, expectedTokenList); } @@ -292,7 +292,7 @@ void recognizeNewlines() { @org.junit.jupiter.api.Test void scanEmptyClass() { final String programCode = "public class Test{}"; - final TokenType[] expectedTokenList = {IDENT, IDENT, IDENT, L_BRACE, R_BRACE, EOF}; + final TokenType[] expectedTokenList = {KEYWORD, KEYWORD, IDENT, L_BRACE, R_BRACE, EOF}; shouldScan(programCode, expectedTokenList); } @@ -303,7 +303,7 @@ void scanEmptyClass() { void scanMethodCallWithTwoArguments() { final String programCode = "int sum = add(a,b)"; final TokenType[] expectedTokenList = - {IDENT, IDENT, ASSIGN, IDENT, L_PAREN, IDENT, COMMA, IDENT, R_PAREN, EOF}; + {KEYWORD, IDENT, ASSIGN, IDENT, L_PAREN, IDENT, COMMA, IDENT, R_PAREN, EOF}; shouldScan(programCode, expectedTokenList); } @@ -315,7 +315,7 @@ void scanMethodCallWithTwoArguments() { void scanMethodCallWithMultipleArguments() { final String programCode = "int sum = add(a,b,5)"; final TokenType[] expectedTokenList = - {IDENT, IDENT, ASSIGN, IDENT, L_PAREN, IDENT, COMMA, IDENT, COMMA, NUMBER, R_PAREN, + {KEYWORD, IDENT, ASSIGN, IDENT, L_PAREN, IDENT, COMMA, IDENT, COMMA, NUMBER, R_PAREN, EOF}; shouldScan(programCode, expectedTokenList); } From ee2f25da791207387a0668881e9d957154cb69ce Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 14:02:09 +0100 Subject: [PATCH 019/135] Adjust expected output from the EmptyClass.java test --- src/test/resources/EmptyClass.expected | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/resources/EmptyClass.expected b/src/test/resources/EmptyClass.expected index bfeb5dcc..775ca33a 100644 --- a/src/test/resources/EmptyClass.expected +++ b/src/test/resources/EmptyClass.expected @@ -1,5 +1,5 @@ -EmptyClass.java(1,1): IDENT -EmptyClass.java(1,8): IDENT +EmptyClass.java(1,1): KEYWORD +EmptyClass.java(1,8): KEYWORD EmptyClass.java(1,14): IDENT EmptyClass.java(1,25): L_BRACE EmptyClass.java(1,26): R_BRACE From 39fbd5e4a67851d60dfe0b8d9f1fcf6c7f20ab80 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 14:21:34 +0100 Subject: [PATCH 020/135] Add KeywordToken class that joins keywords and tokens --- .../com/merkrafter/lexing/KeywordToken.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/main/java/com/merkrafter/lexing/KeywordToken.java diff --git a/src/main/java/com/merkrafter/lexing/KeywordToken.java b/src/main/java/com/merkrafter/lexing/KeywordToken.java new file mode 100644 index 00000000..d823866e --- /dev/null +++ b/src/main/java/com/merkrafter/lexing/KeywordToken.java @@ -0,0 +1,38 @@ +package com.merkrafter.lexing; + +/**** + * This class serves as a token and stores a keyword. + * + * @version v0.2.0 + * @author merkrafter + ***************************************************************/ +public class KeywordToken extends Token { + // ATTRIBUTES + //============================================================== + /** + * the keyword this token stands for + */ + private final Keyword keyword; + + // CONSTRUCTORS + //============================================================== + + /**** + * Creates a new KeywordToken from a keyword and position data. + ***************************************************************/ + public KeywordToken(final Keyword keyword, final String filename, final long line, + final int position) { + super(TokenType.KEYWORD, filename, line, position); + this.keyword = keyword; + } + + // GETTER + //============================================================== + + /** + * @return the keyword this token stands for + */ + Keyword getKeyword() { + return keyword; + } +} From 8ad357341dbf08235fef383a104fb35f92b224af Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 14:59:41 +0100 Subject: [PATCH 021/135] Overload ScannerTest::shouldScan for Token --- .../java/com/merkrafter/lexing/ScannerTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/test/java/com/merkrafter/lexing/ScannerTest.java b/src/test/java/com/merkrafter/lexing/ScannerTest.java index e2ac5982..42b1c40d 100644 --- a/src/test/java/com/merkrafter/lexing/ScannerTest.java +++ b/src/test/java/com/merkrafter/lexing/ScannerTest.java @@ -360,6 +360,20 @@ private void shouldScan(final String programCode, final TokenType[] expectedToke assertArrayEquals(expectedTokenList, actualTokenList.toArray(), actualTokenList.toString()); } + /** + * Conveniently wraps the assertions of equal tokens between expected and the ones emitted by a scanner. + * It therefore sets the programCode as an input to this class's stringIterator and reads from + * its scanner. Both must be initialized before this method can be called safely. + * + * @param programCode the string to tokenize by this class's scanner + * @param expectedTokenList an array with all expected tokens in the right order + */ + private void shouldScan(final String programCode, final Token[] expectedTokenList) { + stringIterator.setString(programCode); + final List actualTokenList = getTokenList(scanner); + assertArrayEquals(expectedTokenList, actualTokenList.toArray(), actualTokenList.toString()); + } + /** * This class provides the possibility to iterate over strings. * It can be used as a tool to mock a file as the input to a Scanner. From 43b668a0d581c280ecbda95b264c5fc44e131d5f Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 15:00:05 +0100 Subject: [PATCH 022/135] Adjust keyword test case to KeywordToken --- src/test/java/com/merkrafter/lexing/ScannerTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/merkrafter/lexing/ScannerTest.java b/src/test/java/com/merkrafter/lexing/ScannerTest.java index 42b1c40d..c4a9f381 100644 --- a/src/test/java/com/merkrafter/lexing/ScannerTest.java +++ b/src/test/java/com/merkrafter/lexing/ScannerTest.java @@ -45,7 +45,9 @@ void setUp() { @EnumSource(Keyword.class) void scanKeyword(final Keyword keyword) { final String programCode = keyword.name().toLowerCase(); - final TokenType[] expectedTokenList = {KEYWORD, EOF}; + final Token[] expectedTokenList = { + new KeywordToken(keyword, null, 1, 1), + new Token(EOF, null, 1, keyword.name().length())}; shouldScan(programCode, expectedTokenList); } From 0bfdf29c28cf1c970cabb2c01252c2a0d82196e5 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 15:08:01 +0100 Subject: [PATCH 023/135] Implement toString() and equals() for KeywordToken --- .../com/merkrafter/lexing/KeywordToken.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/main/java/com/merkrafter/lexing/KeywordToken.java b/src/main/java/com/merkrafter/lexing/KeywordToken.java index d823866e..f34c054c 100644 --- a/src/main/java/com/merkrafter/lexing/KeywordToken.java +++ b/src/main/java/com/merkrafter/lexing/KeywordToken.java @@ -35,4 +35,35 @@ public KeywordToken(final Keyword keyword, final String filename, final long lin Keyword getKeyword() { return keyword; } + + // METHODS + //============================================================== + // public methods + //-------------------------------------------------------------- + + /** + * Two KeywordTokens are equal if both have the type KeywordToken and their keywords, line + * numbers, positions and filenames are equal. + * + * @param obj ideally a KeywordToken to compare this with + * @return whether this is equal to obj + */ + @Override + public boolean equals(final Object obj) { + if (!super.equals(obj)) { + return false; + } + return obj instanceof KeywordToken && ((KeywordToken) obj).keyword == keyword; + } + + /** + * Creates a String representation of this KeywordToken in the following format: + * FILENAME(LINE,POSITION): TYPE(KEYWORD) + * + * @return a String representation of this KeywordToken + */ + @Override + public String toString() { + return super.toString() + String.format("(%s)", keyword.name().toLowerCase()); + } } From e164506d179bf9c2a00e73f22c691be66581d601 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 19:44:12 +0100 Subject: [PATCH 024/135] Make the Scanner distinguish keywords --- .../java/com/merkrafter/lexing/Scanner.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/merkrafter/lexing/Scanner.java b/src/main/java/com/merkrafter/lexing/Scanner.java index 02c8dae6..1434b50f 100644 --- a/src/main/java/com/merkrafter/lexing/Scanner.java +++ b/src/main/java/com/merkrafter/lexing/Scanner.java @@ -190,13 +190,17 @@ public void processToken() { case 'Y': case 'Z': sym = new Token(TokenType.IDENT, filename, line, position); + // TODO change this if it turns out to be a keyword id = ""; do { id += ch; if (!this.loadNextCharSuccessfully()) { + checkAndSetKeyword(); return; } - } while (ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' || ch >= '0' && ch <= '9'); + } while (ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' + || ch >= '0' && ch <= '9'); + checkAndSetKeyword(); break; case '(': sym = new Token(TokenType.L_PAREN, filename, line, position); @@ -390,4 +394,18 @@ private void processNewline() { line++; position = 0; } + + /** + * Tests whether id currently holds a keyword. If that's the case, sym is changed + * accordingly. + */ + private void checkAndSetKeyword() { + try { + final Keyword keyword = Keyword.valueOf(id.toUpperCase()); + // if this actually is a keyword: + sym = new KeywordToken(keyword, sym.getFilename(), sym.getLine(), sym.getPosition()); + } catch (IllegalArgumentException ignored) { // this is thrown if `id` is not a keyword + } + } + } From 7099be55053dd7ffb2c90f30f03c946517befe28 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 20:10:37 +0100 Subject: [PATCH 025/135] Adjust EmptyClass.java test to new KeywordToken::toString() method --- src/test/resources/EmptyClass.expected | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/resources/EmptyClass.expected b/src/test/resources/EmptyClass.expected index 775ca33a..06d3fbc5 100644 --- a/src/test/resources/EmptyClass.expected +++ b/src/test/resources/EmptyClass.expected @@ -1,5 +1,5 @@ -EmptyClass.java(1,1): KEYWORD -EmptyClass.java(1,8): KEYWORD +EmptyClass.java(1,1): KEYWORD(public) +EmptyClass.java(1,8): KEYWORD(class) EmptyClass.java(1,14): IDENT EmptyClass.java(1,25): L_BRACE EmptyClass.java(1,26): R_BRACE From e84f2a3df475f6739f48b4543fe62581c11a08c2 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 20:22:32 +0100 Subject: [PATCH 026/135] Add IdentToken class that joins identifiers and tokens --- .../com/merkrafter/lexing/IdentToken.java | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 src/main/java/com/merkrafter/lexing/IdentToken.java diff --git a/src/main/java/com/merkrafter/lexing/IdentToken.java b/src/main/java/com/merkrafter/lexing/IdentToken.java new file mode 100644 index 00000000..49696c97 --- /dev/null +++ b/src/main/java/com/merkrafter/lexing/IdentToken.java @@ -0,0 +1,69 @@ +package com.merkrafter.lexing; + +/**** + * This class serves as a token and stores the identifier found. + * + * @version v0.2.0 + * @author merkrafter + ***************************************************************/ +public class IdentToken extends Token { + // ATTRIBUTES + //============================================================== + /** + * the identifier this token stands for + */ + private final String ident; + + // CONSTRUCTORS + //============================================================== + + /**** + * Creates a new IdentToken from an identifier and position data. + ***************************************************************/ + public IdentToken(final String ident, final String filename, final long line, + final int position) { + super(TokenType.IDENT, filename, line, position); + this.ident = ident; + } + + // GETTER + //============================================================== + + /** + * @return the keyword this token stands for + */ + String getIdent() { + return ident; + } + + // METHODS + //============================================================== + // public methods + //-------------------------------------------------------------- + + /** + * Two IdentTokens are equal if both have the type IdentToken and their identifiers, line + * numbers, positions and filenames are equal. + * + * @param obj ideally a IdentToken to compare this with + * @return whether this is equal to obj + */ + @Override + public boolean equals(final Object obj) { + if (!super.equals(obj)) { + return false; + } + return obj instanceof IdentToken && ((IdentToken) obj).ident.equals(ident); + } + + /** + * Creates a String representation of this KeywordToken in the following format: + * FILENAME(LINE,POSITION): TYPE(IDENT) + * + * @return a String representation of this IdentToken + */ + @Override + public String toString() { + return super.toString() + String.format("(%s)", ident); + } +} From 5d167dc8b1cdab45f5a44587582f3176a38fc2aa Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 20:55:19 +0100 Subject: [PATCH 027/135] Make Scanner lex identifiers as IdentToken --- src/main/java/com/merkrafter/lexing/Scanner.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/merkrafter/lexing/Scanner.java b/src/main/java/com/merkrafter/lexing/Scanner.java index 1434b50f..09161613 100644 --- a/src/main/java/com/merkrafter/lexing/Scanner.java +++ b/src/main/java/com/merkrafter/lexing/Scanner.java @@ -189,18 +189,19 @@ public void processToken() { case 'X': case 'Y': case 'Z': + // will be replaced in `setIdentOrKeyword` in a few lines, but this token is needed + // in order to store the starting position of this token sym = new Token(TokenType.IDENT, filename, line, position); - // TODO change this if it turns out to be a keyword id = ""; do { id += ch; if (!this.loadNextCharSuccessfully()) { - checkAndSetKeyword(); + setIdentOrKeyword(); return; } } while (ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' || ch >= '0' && ch <= '9'); - checkAndSetKeyword(); + setIdentOrKeyword(); break; case '(': sym = new Token(TokenType.L_PAREN, filename, line, position); @@ -399,12 +400,14 @@ private void processNewline() { * Tests whether id currently holds a keyword. If that's the case, sym is changed * accordingly. */ - private void checkAndSetKeyword() { + private void setIdentOrKeyword() { try { final Keyword keyword = Keyword.valueOf(id.toUpperCase()); // if this actually is a keyword: sym = new KeywordToken(keyword, sym.getFilename(), sym.getLine(), sym.getPosition()); - } catch (IllegalArgumentException ignored) { // this is thrown if `id` is not a keyword + } catch (IllegalArgumentException ignored) { + // id is not a keyword + sym = new IdentToken(id, sym.getFilename(), sym.getLine(), sym.getPosition()); } } From dda2de73d82635b6052364e47900b06a436df046 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 20:56:59 +0100 Subject: [PATCH 028/135] Adjust EmptyClass test case to new IdentToken --- src/test/resources/EmptyClass.expected | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/resources/EmptyClass.expected b/src/test/resources/EmptyClass.expected index 06d3fbc5..3a57221f 100644 --- a/src/test/resources/EmptyClass.expected +++ b/src/test/resources/EmptyClass.expected @@ -1,6 +1,6 @@ EmptyClass.java(1,1): KEYWORD(public) EmptyClass.java(1,8): KEYWORD(class) -EmptyClass.java(1,14): IDENT +EmptyClass.java(1,14): IDENT(EmptyClass) EmptyClass.java(1,25): L_BRACE EmptyClass.java(1,26): R_BRACE EmptyClass.java(1,26): EOF From 58b29bd831eb6546338538fa6747c8d2ab978ce6 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 21:17:24 +0100 Subject: [PATCH 029/135] Make KeywordToken::getKeyword public --- src/main/java/com/merkrafter/lexing/KeywordToken.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/merkrafter/lexing/KeywordToken.java b/src/main/java/com/merkrafter/lexing/KeywordToken.java index f34c054c..b92e1def 100644 --- a/src/main/java/com/merkrafter/lexing/KeywordToken.java +++ b/src/main/java/com/merkrafter/lexing/KeywordToken.java @@ -32,7 +32,7 @@ public KeywordToken(final Keyword keyword, final String filename, final long lin /** * @return the keyword this token stands for */ - Keyword getKeyword() { + public Keyword getKeyword() { return keyword; } From ad2b827dee86dda918fd016f49bb3da04ca74c2b Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 22:48:04 +0100 Subject: [PATCH 030/135] Fix ParserTest test cases --- .../java/com/merkrafter/parsing/ParserTest.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index 21f355c9..316b7ba8 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -1,8 +1,6 @@ package com.merkrafter.parsing; -import com.merkrafter.lexing.Scanner; -import com.merkrafter.lexing.Token; -import com.merkrafter.lexing.TokenType; +import com.merkrafter.lexing.*; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; @@ -18,8 +16,8 @@ class ParserTest { */ @Test void parseType() { - // TODO implement this when int token is available - final Scanner scanner = new TestScanner(new Token[]{}); + final Scanner scanner = new TestScanner(new Token[]{ + new KeywordToken(Keyword.INT, null, 1, 1)}); final Parser parser = new Parser(scanner); assertTrue(parser.parseType()); } @@ -64,19 +62,22 @@ void parseProcedureCall() { final Scanner scanner = new TestScanner(new Token[]{ new Token(TokenType.IDENT, "", 0, 0), new Token(TokenType.L_PAREN, "", 0, 0), + new IdentToken("a", "", 0, 0), new Token(TokenType.R_PAREN, "", 0, 0), new Token(TokenType.SEMICOLON, "", 0, 0)}); final Parser parser = new Parser(scanner); assertTrue(parser.parseProcedureCall()); } + /** * The parser should accept a single return statement. */ @Test void parseStandaloneReturnStatement() { - // TODO implement this when return token is available - final Scanner scanner = new TestScanner(new Token[]{}); + final Scanner scanner = new TestScanner(new Token[]{ + new KeywordToken(Keyword.RETURN, null, 1, 1), + new Token(TokenType.SEMICOLON, null, 1, 1)}); final Parser parser = new Parser(scanner); assertTrue(parser.parseReturnStatement()); } From 590dccec0b6bc610731367e4ef5bbb3a4058d757 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 22:48:28 +0100 Subject: [PATCH 031/135] Add another return statement test case --- src/test/java/com/merkrafter/parsing/ParserTest.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index 316b7ba8..b7e51ca8 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -82,6 +82,17 @@ void parseStandaloneReturnStatement() { assertTrue(parser.parseReturnStatement()); } + /** + * The parser must not accept a single return statement (without semicolon). + */ + @Test + void parseStandaloneReturnStatementWithoutSemicolon() { + final Scanner scanner = new TestScanner(new Token[]{ + new KeywordToken(Keyword.RETURN, null, 1, 1)}); + final Parser parser = new Parser(scanner); + assertFalse(parser.parseReturnStatement()); + } + /** * The parser should accept a single pair of parentheses as actual parameters. */ From 8970a8a37be6aefe630c0e2541fdfe26964006b4 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 22:50:10 +0100 Subject: [PATCH 032/135] Implement parsing types --- src/main/java/com/merkrafter/parsing/Parser.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 9bd1ebe8..b3f87984 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -1,5 +1,7 @@ package com.merkrafter.parsing; +import com.merkrafter.lexing.Keyword; +import com.merkrafter.lexing.KeywordToken; import com.merkrafter.lexing.Scanner; import static com.merkrafter.lexing.TokenType.*; @@ -42,7 +44,15 @@ public boolean parse() { return false; } + /** + * @return whether the current symbol is a KeywordToken and represents an "int" + */ boolean parseType() { + if (scanner.getSym() instanceof KeywordToken + && ((KeywordToken) scanner.getSym()).getKeyword() == Keyword.INT) { + scanner.processToken(); + return true; + } return false; } From a195d60ba78a4f3a944c6d72c44edfee5ff8d056 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 22:51:08 +0100 Subject: [PATCH 033/135] Implement parsing procedure calls --- src/main/java/com/merkrafter/parsing/Parser.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index b3f87984..33d2264b 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -61,6 +61,10 @@ boolean parseAssignment() { } boolean parseProcedureCall() { + if (parseInternProcedureCall() && scanner.getSym().getType() == SEMICOLON) { + scanner.processToken(); + return true; + } return false; } From 8e4072641f2378762132074ab4dd1d102b1fdacb Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 22:53:07 +0100 Subject: [PATCH 034/135] Fix parsing intern procedure calls --- src/main/java/com/merkrafter/parsing/Parser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 33d2264b..b0a116fe 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -70,7 +70,6 @@ boolean parseProcedureCall() { boolean parseInternProcedureCall() { if (parseIdentifier()) { - scanner.processToken(); return parseActualParameters(); } return false; @@ -99,6 +98,7 @@ boolean parseActualParameters() { return false; } if (scanner.getSym().getType() == R_PAREN) { + scanner.processToken(); return true; } else { error("Wrong use of parenthesis"); From 58a3327bd978a3e65f4775a92e9ac3ab1a9a2a40 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 22:53:52 +0100 Subject: [PATCH 035/135] Implement parsing return statements --- .../java/com/merkrafter/parsing/Parser.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index b0a116fe..a17070e5 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -76,6 +76,25 @@ boolean parseInternProcedureCall() { } boolean parseReturnStatement() { + if (scanner.getSym() instanceof KeywordToken + && ((KeywordToken) scanner.getSym()).getKeyword() == Keyword.RETURN) { + + scanner.processToken(); + if (scanner.getSym().getType() == SEMICOLON) { + // there is no simple expression in between + scanner.processToken(); + return true; + } else if (parseSimpleExpression()) { + if (scanner.getSym().getType() == SEMICOLON) { + scanner.processToken(); + return true; + } else { + return false; + } + } else { // neither a semicolon nor a simple expression + return false; + } + } return false; } From 07650a0638131f8b3a058aba8a9430d0e693bf89 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 23:03:01 +0100 Subject: [PATCH 036/135] Implement parsing assignments --- src/main/java/com/merkrafter/parsing/Parser.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index a17070e5..4124b784 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -57,6 +57,13 @@ boolean parseType() { } boolean parseAssignment() { + if (parseIdentifier() && scanner.getSym().getType() == ASSIGN) { + scanner.processToken(); + if (parseExpression() && scanner.getSym().getType() == SEMICOLON) { + scanner.processToken(); + return true; + } + } return false; } From 1e22dc4d01cd6598f36bb55a2e91552bd01da12b Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 23:16:40 +0100 Subject: [PATCH 037/135] Add parser method stubs for method type, formal params, fp sections and local decl.s --- src/main/java/com/merkrafter/parsing/Parser.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 4124b784..dcf2ae9b 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -44,6 +44,22 @@ public boolean parse() { return false; } + boolean parseMethodType() { + return false; + } + + boolean parseFormalParameters() { + return false; + } + + boolean parseFpSection() { + return false; + } + + boolean parseLocalDeclaration() { + return false; + } + /** * @return whether the current symbol is a KeywordToken and represents an "int" */ From 48d35bd2a9e8ef137cc7ab435e2c28022923296f Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 25 Nov 2019 23:49:08 +0100 Subject: [PATCH 038/135] Add happy path test cases for new parser methods --- .../com/merkrafter/parsing/ParserTest.java | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index b7e51ca8..7897cbcd 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -11,6 +11,56 @@ class ParserTest { + /** + * The parser should accept "void" and "int" as method types. + */ + @ParameterizedTest + @EnumSource(value = Keyword.class, names = {"VOID", "INT"}) + void parseMethodType(final Keyword keyword) { + final Scanner scanner = new TestScanner(new Token[]{ + new KeywordToken(keyword, null, 1, 1)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseMethodType()); + } + + /** + * The parser should accept a single "(int a)" as formal parameters. + */ + @Test + void parseFormalParameters() { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.L_PAREN, null, 1, 1), + new KeywordToken(Keyword.INT, null, 1, 1), + new IdentToken("a", null, 1, 1), + new Token(TokenType.R_PAREN, null, 1, 1)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseFormalParameters()); + } + + /** + * The parser should accept a single "int a" as a fp_section. + */ + @Test + void parseFpSection() { + final Scanner scanner = new TestScanner(new Token[]{ + new KeywordToken(Keyword.INT, null, 1, 1), new IdentToken("a", null, 1, 1)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseFpSection()); + } + + /** + * The parser should accept a single "int a;" as a local declaration. + */ + @Test + void parseLocalDeclaration() { + final Scanner scanner = new TestScanner(new Token[]{ + new KeywordToken(Keyword.INT, null, 1, 1), + new IdentToken("a", null, 1, 1), + new Token(TokenType.SEMICOLON, null, 1, 1)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseLocalDeclaration()); + } + /** * The parser should accept a single "int" as a type. */ From d7890149b33210f46837cfefa6998fced6082e00 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Tue, 26 Nov 2019 10:53:16 +0100 Subject: [PATCH 039/135] Implement parsing method types --- src/main/java/com/merkrafter/parsing/Parser.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index dcf2ae9b..900a5393 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -45,6 +45,15 @@ public boolean parse() { } boolean parseMethodType() { + // expecting a keyword + if (scanner.getSym() instanceof KeywordToken && ( + // check whether this is "void" + ((KeywordToken) scanner.getSym()).getKeyword() == Keyword.VOID + //check whether this is "int" + || ((KeywordToken) scanner.getSym()).getKeyword() == Keyword.INT)) { + scanner.processToken(); + return true; + } return false; } From 85f746535729768adbc0ef1ecfb01ac3e91fc1f4 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Tue, 26 Nov 2019 11:04:22 +0100 Subject: [PATCH 040/135] Implement parsing fp sections --- src/main/java/com/merkrafter/parsing/Parser.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 900a5393..bf75d09b 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -62,6 +62,9 @@ boolean parseFormalParameters() { } boolean parseFpSection() { + if (parseType()) { + return parseIdentifier(); // already reads the next token + } return false; } From 5fcbb1967d57d46f742c3ea35c62ad00f89cc38f Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Tue, 26 Nov 2019 11:11:40 +0100 Subject: [PATCH 041/135] Implement parsing local declarations --- src/main/java/com/merkrafter/parsing/Parser.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index bf75d09b..c9ef99ca 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -69,6 +69,14 @@ boolean parseFpSection() { } boolean parseLocalDeclaration() { + if (parseType()) { + if (parseIdentifier()) { + if (scanner.getSym().getType() == SEMICOLON) { + scanner.processToken(); + return true; + } + } + } return false; } From 41d8f91410a544ed36c5fe79663126d9e172bc4b Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Tue, 26 Nov 2019 11:22:57 +0100 Subject: [PATCH 042/135] Implement parsing formal parameters --- src/main/java/com/merkrafter/parsing/Parser.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index c9ef99ca..82fb4e8b 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -58,6 +58,22 @@ boolean parseMethodType() { } boolean parseFormalParameters() { + if (scanner.getSym().getType() == L_PAREN) { + scanner.processToken(); + if (parseFpSection()) { + while (scanner.getSym().getType() == COMMA) { + scanner.processToken(); + if (!parseFpSection()) { + return false; + } + } + } + if (scanner.getSym().getType() == R_PAREN) { + scanner.processToken(); + return true; + } + + } return false; } From bc89fc2f370d4822745948e2ff860cefa8e2e2a3 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Tue, 26 Nov 2019 11:46:45 +0100 Subject: [PATCH 043/135] Add Parser::parseMethodHead --- src/main/java/com/merkrafter/parsing/Parser.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 82fb4e8b..bfa91088 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -44,6 +44,10 @@ public boolean parse() { return false; } + boolean parseMethodHead() { + return false; + } + boolean parseMethodType() { // expecting a keyword if (scanner.getSym() instanceof KeywordToken && ( From b7706098b99facc5e8a5c80f0a729a4888ca23f9 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Tue, 26 Nov 2019 11:59:26 +0100 Subject: [PATCH 044/135] Add happy path test case for method head parsing method --- .../java/com/merkrafter/parsing/ParserTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index 7897cbcd..f86aa2ef 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -11,6 +11,22 @@ class ParserTest { + /** + * The parser should accept simple method heads without any formal parameters. + */ + @ParameterizedTest + @EnumSource(value = Keyword.class, names = {"VOID", "INT"}) + void parseMethodHead(final Keyword methodType) { + final Scanner scanner = new TestScanner(new Token[]{ + new KeywordToken(Keyword.PUBLIC, null, 1, 1), + new KeywordToken(methodType, null, 1, 1), + new IdentToken("foo", null, 1, 1), + new Token(TokenType.L_PAREN, null, 1, 1), + new Token(TokenType.R_PAREN, null, 1, 1)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseMethodHead()); + } + /** * The parser should accept "void" and "int" as method types. */ From 474fef80841f40cff206e84d7d95d1da07514327 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Tue, 26 Nov 2019 12:08:28 +0100 Subject: [PATCH 045/135] Implement parsing method heads --- src/main/java/com/merkrafter/parsing/Parser.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index bfa91088..57d00e09 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -45,6 +45,15 @@ public boolean parse() { } boolean parseMethodHead() { + if (scanner.getSym() instanceof KeywordToken + && ((KeywordToken) scanner.getSym()).getKeyword() == Keyword.PUBLIC) { + scanner.processToken(); + if (parseMethodType()) { + if (parseIdentifier()) { + return parseFormalParameters(); + } + } + } return false; } From 8eb6c0ad99d4d5882c6eb7e4232a9329e892ac88 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Tue, 26 Nov 2019 12:11:24 +0100 Subject: [PATCH 046/135] Add method stubs for statements and statement sequences --- src/main/java/com/merkrafter/parsing/Parser.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 57d00e09..cd0541ed 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -109,6 +109,14 @@ boolean parseLocalDeclaration() { return false; } + boolean parseStatementSequence() { + return false; + } + + boolean parseStatement() { + return false; + } + /** * @return whether the current symbol is a KeywordToken and represents an "int" */ From 44e8aa914e3e582794eab65fbe6b4b8be598d3f6 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Tue, 26 Nov 2019 13:05:39 +0100 Subject: [PATCH 047/135] Add happy path test cases for statement and statement sequence parsing methods --- .../com/merkrafter/parsing/ParserTest.java | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index f86aa2ef..7f7c8ed8 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -77,6 +77,141 @@ void parseLocalDeclaration() { assertTrue(parser.parseLocalDeclaration()); } + /** + * The parser should accept an assignment and a return statement as a statement sequence. + */ + @Test + void parseStatementSequence() { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.IDENT, "", 0, 0), + new Token(TokenType.ASSIGN, "", 0, 0), + new Token(TokenType.NUMBER, "", 0, 0), + new Token(TokenType.SEMICOLON, "", 0, 0), + new KeywordToken(Keyword.RETURN, "", 0, 0), + new Token(TokenType.IDENT, "", 0, 0), + new Token(TokenType.SEMICOLON, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseStatementSequence()); + } + + /** + * The parser should accept an assignment of the result of a binary operation to a variable + * as a statement sequence. + */ + @ParameterizedTest + @EnumSource(value = TokenType.class, names = {"PLUS", "MINUS", "TIMES", "DIVIDE"}) + void parseAssignmentWithBinOpAsStatementSequence(final TokenType binOp) { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.IDENT, "", 0, 0), + new Token(TokenType.ASSIGN, "", 0, 0), + new Token(TokenType.IDENT, "", 0, 0), + new Token(binOp, "", 0, 0), + new Token(TokenType.NUMBER, "", 0, 0), + new Token(TokenType.SEMICOLON, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseStatementSequence()); + } + + /** + * The parser should accept a simple assignment of a number as a statement sequence. + */ + @Test + void parseAssignmentAsStatementSequence() { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.IDENT, "", 0, 0), + new Token(TokenType.ASSIGN, "", 0, 0), + new Token(TokenType.NUMBER, "", 0, 0), + new Token(TokenType.SEMICOLON, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseStatementSequence()); + } + + /** + * The parser should accept a simple procedure call as a statement sequence. + */ + @Test + void parseProcedureCallAsStatementSequence() { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.IDENT, "", 0, 0), + new Token(TokenType.L_PAREN, "", 0, 0), + new IdentToken("a", "", 0, 0), + new Token(TokenType.R_PAREN, "", 0, 0), + new Token(TokenType.SEMICOLON, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseStatementSequence()); + } + + /** + * The parser should accept a single return keyword as a statement sequence. + */ + @Test + void parseStandaloneReturnAsStatementSequence() { + final Scanner scanner = new TestScanner(new Token[]{ + new KeywordToken(Keyword.RETURN, null, 1, 1), + new Token(TokenType.SEMICOLON, null, 1, 1)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseStatementSequence()); + } + + /** + * The parser should accept an assignment of the result of a binary operation to a variable + * as a statement. + */ + @ParameterizedTest + @EnumSource(value = TokenType.class, names = {"PLUS", "MINUS", "TIMES", "DIVIDE"}) + void parseAssignmentWithBinOpAsStatement(final TokenType binOp) { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.IDENT, "", 0, 0), + new Token(TokenType.ASSIGN, "", 0, 0), + new Token(TokenType.IDENT, "", 0, 0), + new Token(binOp, "", 0, 0), + new Token(TokenType.NUMBER, "", 0, 0), + new Token(TokenType.SEMICOLON, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseStatement()); + } + + /** + * The parser should accept a simple assignment of a number as a statement. + */ + @Test + void parseAssignmentAsStatement() { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.IDENT, "", 0, 0), + new Token(TokenType.ASSIGN, "", 0, 0), + new Token(TokenType.NUMBER, "", 0, 0), + new Token(TokenType.SEMICOLON, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseStatement()); + } + + /** + * The parser should accept a simple procedure call as a statement. + */ + @Test + void parseProcedureCallAsStatement() { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.IDENT, "", 0, 0), + new Token(TokenType.L_PAREN, "", 0, 0), + new IdentToken("a", "", 0, 0), + new Token(TokenType.R_PAREN, "", 0, 0), + new Token(TokenType.SEMICOLON, "", 0, 0)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseStatement()); + } + + /** + * The parser should accept a single return keyword as a statement. + */ + @Test + void parseStandaloneReturnAsStatement() { + final Scanner scanner = new TestScanner(new Token[]{ + new KeywordToken(Keyword.RETURN, null, 1, 1), + new Token(TokenType.SEMICOLON, null, 1, 1)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseStatement()); + } + /** * The parser should accept a single "int" as a type. */ From 88cc3caf163613b4b14d64a8a6d241c67cfee948 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Tue, 26 Nov 2019 13:46:34 +0100 Subject: [PATCH 048/135] Pull apart parsing assignments to allow factoring in parsing statements --- .../java/com/merkrafter/parsing/Parser.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index cd0541ed..1b337e9f 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -130,7 +130,22 @@ boolean parseType() { } boolean parseAssignment() { - if (parseIdentifier() && scanner.getSym().getType() == ASSIGN) { + if (parseIdentifier()) { + return parseAssignmentWithoutIdent(); + } + return false; + } + + /** + * This method is a helper for parsing assignments that assumes that the token stream was + * already checked for an identifier. + * It is needed to distinguish assignments and intern procedure calls who both start with + * an IDENT. + * + * @return whether tokens after an ident match the grammar of assignments + */ + private boolean parseAssignmentWithoutIdent() { + if (scanner.getSym().getType() == ASSIGN) { scanner.processToken(); if (parseExpression() && scanner.getSym().getType() == SEMICOLON) { scanner.processToken(); From 21a5c9dfa547a766d9812106d43512620071c00b Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Tue, 26 Nov 2019 13:57:00 +0100 Subject: [PATCH 049/135] Implement parsing (assignement, procedure call and return) statements --- .../java/com/merkrafter/parsing/Parser.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 1b337e9f..53bd5326 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -114,6 +114,28 @@ boolean parseStatementSequence() { } boolean parseStatement() { + // factoring of + // statement = ident '=' expression ';' | ident actual_parameters ';' + // ^ assignment ^ procedure call + if (parseStatementForAssignmentOrProcedureCall()) { + return true; + } + if (parseReturnStatement()) { + return true; + } + return false; + } + + private boolean parseStatementForAssignmentOrProcedureCall() { + if (parseIdentifier()) { + if (parseAssignmentWithoutIdent()) { + return true; + } else if (parseActualParameters() && scanner.getSym().getType() == SEMICOLON) { + // this actually is a procedure call + scanner.processToken(); + return true; + } + } return false; } From 93f8a2247974ca3a0f93b73d965fb9ca1ef1ba63 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Tue, 26 Nov 2019 14:00:22 +0100 Subject: [PATCH 050/135] Implement parsing statement sequences --- src/main/java/com/merkrafter/parsing/Parser.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 53bd5326..67c51f73 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -110,6 +110,10 @@ boolean parseLocalDeclaration() { } boolean parseStatementSequence() { + if (parseStatement()) { + while(parseStatement()); // just read all statements for now; creating AST follows + return true; + } return false; } From acb94429e85ba4b59e0d5eb9943069e6a141face Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Tue, 26 Nov 2019 18:13:33 +0100 Subject: [PATCH 051/135] Add method stubs for parsing if and while statements --- src/main/java/com/merkrafter/parsing/Parser.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 67c51f73..f5985eec 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -196,6 +196,14 @@ boolean parseInternProcedureCall() { return false; } + boolean parseIfStatement() { + return false; + } + + boolean parseWhileStatement() { + return false; + } + boolean parseReturnStatement() { if (scanner.getSym() instanceof KeywordToken && ((KeywordToken) scanner.getSym()).getKeyword() == Keyword.RETURN) { From 35e071dc9a5e9351a4aa4cff47b8a26a35617679 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Thu, 28 Nov 2019 15:48:28 +0100 Subject: [PATCH 052/135] Add happy path test cases for if and while construct parsing methods --- .../com/merkrafter/parsing/ParserTest.java | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index 7f7c8ed8..772ef693 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -1,6 +1,7 @@ package com.merkrafter.parsing; import com.merkrafter.lexing.*; +import jdk.nashorn.internal.codegen.types.Type; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; @@ -271,6 +272,63 @@ void parseProcedureCall() { } + /** + * The parser should accept a simple if-else construct, that is an "if" keyword, a comparison + * between an identifier and a number as the condition and blocks with single assignments for + * if and else. + */ + @Test + void parseSimpleIfStatement() { + final Scanner scanner = new TestScanner(new Token[]{ + new KeywordToken(Keyword.IF, null, 1, 1), + new Token(TokenType.L_PAREN, null, 1, 1), + new Token(TokenType.IDENT, null, 1, 1), + new Token(TokenType.EQUAL, null, 1, 1), + new Token(TokenType.NUMBER, null, 1, 1), + new Token(TokenType.R_PAREN, null, 1, 1), + + new Token(TokenType.L_BRACE, null, 1, 1), + new Token(TokenType.IDENT, null, 1, 1), + new Token(TokenType.ASSIGN, null, 1, 1), + new Token(TokenType.NUMBER, null, 1, 1), + new Token(TokenType.SEMICOLON, null, 1, 1), + new Token(TokenType.R_BRACE, null, 1, 1), + + new KeywordToken(Keyword.ELSE, null, 1, 1), + new Token(TokenType.L_BRACE, null, 1, 1), + new Token(TokenType.IDENT, null, 1, 1), + new Token(TokenType.ASSIGN, null, 1, 1), + new Token(TokenType.NUMBER, null, 1, 1), + new Token(TokenType.SEMICOLON, null, 1, 1), + new Token(TokenType.R_BRACE, null, 1, 1),}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseIfStatement()); + } + + /** + * The parser should accept a simple while loop, that is a "while" keyword, a comparison + * between an identifier and a number as the condition and a block that has only an assignment + * inside it. + */ + @Test + void parseSimpleWhileStatement() { + final Scanner scanner = new TestScanner(new Token[]{ + new KeywordToken(Keyword.WHILE, null, 1, 1), + new Token(TokenType.L_PAREN, null, 1, 1), + new Token(TokenType.IDENT, null, 1, 1), + new Token(TokenType.EQUAL, null, 1, 1), + new Token(TokenType.NUMBER, null, 1, 1), + new Token(TokenType.R_PAREN, null, 1, 1), + new Token(TokenType.L_BRACE, null, 1, 1), + new Token(TokenType.IDENT, null, 1, 1), + new Token(TokenType.ASSIGN, null, 1, 1), + new Token(TokenType.NUMBER, null, 1, 1), + new Token(TokenType.SEMICOLON, null, 1, 1), + new Token(TokenType.R_BRACE, null, 1, 1),}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseWhileStatement()); + } + /** * The parser should accept a single return statement. */ From d612ff139f0d4f5151a6bd007ca178963825975e Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Thu, 28 Nov 2019 15:50:25 +0100 Subject: [PATCH 053/135] Implement parsing while constructs --- .../java/com/merkrafter/parsing/Parser.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index f5985eec..06f7bc01 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -201,6 +201,29 @@ boolean parseIfStatement() { } boolean parseWhileStatement() { + if (scanner.getSym() instanceof KeywordToken + && ((KeywordToken) scanner.getSym()).getKeyword() == Keyword.WHILE) { + scanner.processToken(); + // condition: + if (scanner.getSym().getType() == L_PAREN) { + scanner.processToken(); + if (parseExpression()) { + if (scanner.getSym().getType() == R_PAREN) { + scanner.processToken(); + // associated block: + if (scanner.getSym().getType() == L_BRACE) { + scanner.processToken(); + if (parseStatementSequence()) { + if (scanner.getSym().getType() == R_BRACE) { + scanner.processToken(); + return true; + } + } + } + } + } + } + } return false; } From 15c4db334d8811542fa6213279dc8667a2bcd07c Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Thu, 28 Nov 2019 16:02:39 +0100 Subject: [PATCH 054/135] Implement parsing if-else constructs --- .../java/com/merkrafter/parsing/Parser.java | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 06f7bc01..5801c486 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -197,6 +197,43 @@ boolean parseInternProcedureCall() { } boolean parseIfStatement() { + if (scanner.getSym() instanceof KeywordToken + && ((KeywordToken) scanner.getSym()).getKeyword() == Keyword.IF) { + scanner.processToken(); + // condition: + if (scanner.getSym().getType() == L_PAREN) { + scanner.processToken(); + if (parseExpression()) { + if (scanner.getSym().getType() == R_PAREN) { + scanner.processToken(); + // if-associated block: + if (scanner.getSym().getType() == L_BRACE) { + scanner.processToken(); + if (parseStatementSequence()) { + if (scanner.getSym().getType() == R_BRACE) { + scanner.processToken(); + if (scanner.getSym() instanceof KeywordToken + && ((KeywordToken) scanner.getSym()).getKeyword() + == Keyword.ELSE) { + scanner.processToken(); + // else-associated block + if (scanner.getSym().getType() == L_BRACE) { + scanner.processToken(); + if (parseStatementSequence()) { + if (scanner.getSym().getType() == R_BRACE) { + scanner.processToken(); + return true; + } + } + } + } + } + } + } + } + } + } + } return false; } From d43200075c68ef8f8af5a39b46f4c838b8bc0e7f Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Thu, 28 Nov 2019 16:07:30 +0100 Subject: [PATCH 055/135] Remove printing error messages (will be implemented centrally later on) --- .../java/com/merkrafter/parsing/Parser.java | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 5801c486..9bcf4759 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -111,7 +111,7 @@ boolean parseLocalDeclaration() { boolean parseStatementSequence() { if (parseStatement()) { - while(parseStatement()); // just read all statements for now; creating AST follows + while (parseStatement()) ; // just read all statements for now; creating AST follows return true; } return false; @@ -294,7 +294,6 @@ boolean parseActualParameters() { while (scanner.getSym().getType() == COMMA) { scanner.processToken(); if (!parseExpression()) { - error("Expected expression after comma in actual parameters"); return false; } } @@ -302,14 +301,12 @@ boolean parseActualParameters() { return true; // it is okay if no expression comes here } } else { - error("Expected left parenthesis in actual parameters"); return false; } if (scanner.getSym().getType() == R_PAREN) { scanner.processToken(); return true; } else { - error("Wrong use of parenthesis"); return false; } } @@ -328,7 +325,6 @@ boolean parseExpression() { return true; } } else { - error("Error parsing expression"); return false; } } @@ -370,7 +366,6 @@ boolean parseFactor() { if (scanner.getSym().getType() == R_PAREN) { scanner.processToken(); } else { - error("Wrong use of parenthesis"); return false; } return success; // whether the above parseExpression() was successful @@ -388,7 +383,6 @@ boolean parseNumber() { scanner.processToken(); return true; } else { - error("Expected number but found " + scanner.getSym().getType().name()); return false; } } @@ -403,19 +397,8 @@ boolean parseIdentifier() { scanner.processToken(); return true; } else { - error("Expected identifier but found " + scanner.getSym().getType().name()); return false; } } - /** - * Prints the given message to stderr. - * Change soon, as only the main class should be printing. - * - * @param msg the message to print - */ - private static void error(final String msg) { - System.err.println(msg); - } - } From 99060a8da59c7c9bffb523853c6dcd1c06aa94be Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Thu, 28 Nov 2019 22:07:20 +0100 Subject: [PATCH 056/135] Add method stubs for parsing method declarations and bodies --- src/main/java/com/merkrafter/parsing/Parser.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 9bcf4759..685bfb54 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -44,6 +44,10 @@ public boolean parse() { return false; } + boolean parseMethodDeclaration() { + return false; + } + boolean parseMethodHead() { if (scanner.getSym() instanceof KeywordToken && ((KeywordToken) scanner.getSym()).getKeyword() == Keyword.PUBLIC) { @@ -97,6 +101,10 @@ boolean parseFpSection() { return false; } + boolean parseMethodBody() { + return false; + } + boolean parseLocalDeclaration() { if (parseType()) { if (parseIdentifier()) { From 276903ea75e71a7e72766e7ce215208e94c94d84 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Thu, 28 Nov 2019 22:18:46 +0100 Subject: [PATCH 057/135] Add happy path test cases for method body and declaration parsing methods --- .../com/merkrafter/parsing/ParserTest.java | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index 772ef693..3d3022cb 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -12,6 +12,30 @@ class ParserTest { + /** + * The parser should accept the declaration of a simple public void method without parameters + * that only returns a constant number. + */ + @ParameterizedTest + @EnumSource(value = Keyword.class, names = {"VOID", "INT"}) + void parseMethodDeclaration(final Keyword methodType) { + final Scanner scanner = new TestScanner(new Token[]{ + // method head + new KeywordToken(Keyword.PUBLIC, null, 1, 1), + new KeywordToken(methodType, null, 1, 1), + new IdentToken("foo", null, 1, 1), + new Token(TokenType.L_PAREN, null, 1, 1), + new Token(TokenType.R_PAREN, null, 1, 1), + // method body + new Token(TokenType.L_BRACE, null, 1, 1), + new KeywordToken(Keyword.RETURN, null, 1, 1), + new Token(TokenType.NUMBER, null, 1, 1), + new Token(TokenType.SEMICOLON, null, 1, 1), + new Token(TokenType.R_BRACE, null, 1, 1)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseMethodDeclaration()); + } + /** * The parser should accept simple method heads without any formal parameters. */ @@ -65,6 +89,21 @@ void parseFpSection() { assertTrue(parser.parseFpSection()); } + /** + * The parser should accept a method body with only one return statement. + */ + @Test + void parseMethodBody() { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.L_BRACE, null, 1, 1), + new KeywordToken(Keyword.RETURN, null, 1, 1), + new Token(TokenType.NUMBER, null, 1, 1), + new Token(TokenType.SEMICOLON, null, 1, 1), + new Token(TokenType.R_BRACE, null, 1, 1)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseMethodBody()); + } + /** * The parser should accept a single "int a;" as a local declaration. */ From 6a84eb64821b3dc1bb53523bed3de9b5333a1714 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Thu, 28 Nov 2019 22:24:11 +0100 Subject: [PATCH 058/135] Implement parsing method bodies --- src/main/java/com/merkrafter/parsing/Parser.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 685bfb54..4cecc101 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -102,6 +102,16 @@ boolean parseFpSection() { } boolean parseMethodBody() { + if (scanner.getSym().getType() == L_BRACE) { + scanner.processToken(); + while (parseLocalDeclaration()) ; // only iterate through them for now + if (parseStatementSequence()) { + if (scanner.getSym().getType() == R_BRACE) { + scanner.processToken(); + return true; + } + } + } return false; } From eac2e710dbef8ccdb667575461b2cfc2da04b627 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Thu, 28 Nov 2019 22:24:32 +0100 Subject: [PATCH 059/135] Implement parsing method declarations --- src/main/java/com/merkrafter/parsing/Parser.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 4cecc101..2143a810 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -45,6 +45,11 @@ public boolean parse() { } boolean parseMethodDeclaration() { + if (parseMethodHead()) { + if (parseMethodBody()) { + return true; + } + } return false; } From e75d18b23f1e54c336927f6008e6d53ee93330ee Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Thu, 28 Nov 2019 22:27:15 +0100 Subject: [PATCH 060/135] Add method stubs for parsing classes, class bodies and declarations --- src/main/java/com/merkrafter/parsing/Parser.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 2143a810..bed0461a 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -44,6 +44,18 @@ public boolean parse() { return false; } + boolean parseClass() { + return false; + } + + boolean parseClassBody() { + return false; + } + + boolean parseDeclarations() { + return false; + } + boolean parseMethodDeclaration() { if (parseMethodHead()) { if (parseMethodBody()) { From abd2bf41b67e0617e7c04a62c33898c9c1ee3977 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Thu, 28 Nov 2019 22:44:48 +0100 Subject: [PATCH 061/135] Add happy path test cases for class, class bodies and declarations parsing methods --- .../com/merkrafter/parsing/ParserTest.java | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index 3d3022cb..637f5c54 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -12,6 +12,51 @@ class ParserTest { + /** + * The parser should accept "class MyClass {int a;}" as a class. + */ + @Test + void parseClass() { + final Scanner scanner = new TestScanner(new Token[]{ + new KeywordToken(Keyword.CLASS, null, 1, 1), + new IdentToken("MyClass", null, 1, 1), + new Token(TokenType.L_BRACE, null, 1, 1), + new KeywordToken(Keyword.INT, null, 1, 1), + new IdentToken("a", null, 1, 1), + new Token(TokenType.SEMICOLON, null, 1, 1), + new Token(TokenType.R_BRACE, null, 1, 1)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseClass()); + } + + /** + * The parser should accept a single "{int a;}" as a class body. + */ + @Test + void parseClassBody() { + final Scanner scanner = new TestScanner(new Token[]{ + new Token(TokenType.L_BRACE, null, 1, 1), + new KeywordToken(Keyword.INT, null, 1, 1), + new IdentToken("a", null, 1, 1), + new Token(TokenType.SEMICOLON, null, 1, 1), + new Token(TokenType.R_BRACE, null, 1, 1)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseClassBody()); + } + + /** + * The parser should accept a single "int a;" as a declaration. + */ + @Test + void parseDeclarations() { + final Scanner scanner = new TestScanner(new Token[]{ + new KeywordToken(Keyword.INT, null, 1, 1), + new IdentToken("a", null, 1, 1), + new Token(TokenType.SEMICOLON, null, 1, 1)}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseDeclarations()); + } + /** * The parser should accept the declaration of a simple public void method without parameters * that only returns a constant number. From 7796185117fd84b659a2a39752006c5c02e57ba2 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Thu, 28 Nov 2019 22:45:40 +0100 Subject: [PATCH 062/135] Implement parsing declarations --- .../java/com/merkrafter/parsing/Parser.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index bed0461a..f92887d8 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -53,6 +53,33 @@ boolean parseClassBody() { } boolean parseDeclarations() { + // final declaration + while (parseFinalDeclaration()) ; + // type declaration + while (parseLocalDeclaration()) ; + // method declaration + while (parseMethodDeclaration()) ; + return true; + } + + private boolean parseFinalDeclaration() { + if (scanner.getSym() instanceof KeywordToken + && ((KeywordToken) scanner.getSym()).getKeyword() == Keyword.FINAL) { + scanner.processToken(); + if (parseType()) { + if (parseIdentifier()) { + if (scanner.getSym().getType() == ASSIGN) { + scanner.processToken(); + if (parseExpression()) { + if (scanner.getSym().getType() == SEMICOLON) { + scanner.processToken(); + return true; + } + } + } + } + } + } return false; } From 97b04ed79a591490773690b2cf6dc15f67cb7964 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Thu, 28 Nov 2019 22:46:42 +0100 Subject: [PATCH 063/135] Implement parsing class bodies --- src/main/java/com/merkrafter/parsing/Parser.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index f92887d8..f90de8a2 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -49,6 +49,16 @@ boolean parseClass() { } boolean parseClassBody() { + if (scanner.getSym().getType() == L_BRACE) { + scanner.processToken(); + if (parseDeclarations()) { + if (scanner.getSym().getType() == R_BRACE) { + scanner.processToken(); + return true; + } + } + + } return false; } From ac0dd5340e711f57f6684d8fb2b84af2ee90a661 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Thu, 28 Nov 2019 22:47:01 +0100 Subject: [PATCH 064/135] Implement parsing classes --- src/main/java/com/merkrafter/parsing/Parser.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index f90de8a2..9b6f4198 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -45,6 +45,15 @@ public boolean parse() { } boolean parseClass() { + if (scanner.getSym() instanceof KeywordToken + && ((KeywordToken) scanner.getSym()).getKeyword() == Keyword.CLASS) { + scanner.processToken(); + if (parseIdentifier()) { + if (parseClassBody()) { + return true; + } + } + } return false; } From ac59df800c49d218c768cc86949b28744c1fb303 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Thu, 28 Nov 2019 22:57:32 +0100 Subject: [PATCH 065/135] Stop tracking .iml file --- merkompiler.iml | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 merkompiler.iml diff --git a/merkompiler.iml b/merkompiler.iml deleted file mode 100644 index 9f89fe87..00000000 --- a/merkompiler.iml +++ /dev/null @@ -1,35 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From e98f675f5d8e9744b9e983768ee143dbd05c8f6d Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Thu, 28 Nov 2019 22:58:30 +0100 Subject: [PATCH 066/135] Remove accidental import in ParserTest --- src/test/java/com/merkrafter/parsing/ParserTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index 637f5c54..38f2a22d 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -1,7 +1,6 @@ package com.merkrafter.parsing; import com.merkrafter.lexing.*; -import jdk.nashorn.internal.codegen.types.Type; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; From 085b2a2d72eb91a6e686700a2096e67350445676 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Thu, 28 Nov 2019 23:04:03 +0100 Subject: [PATCH 067/135] Add maven-model dependency to retrieve data from pom.xml --- pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pom.xml b/pom.xml index d0effd91..afa8fec9 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,11 @@ 5.4.2 test + + org.apache.maven + maven-model + 3.6.2 + ${project.artifactId} From 857d34e798094a57d43d9fbb7cd1e7d9a122eee9 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Fri, 29 Nov 2019 00:15:04 +0100 Subject: [PATCH 068/135] Reformat Config.java --- .../java/com/merkrafter/config/Config.java | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/merkrafter/config/Config.java b/src/main/java/com/merkrafter/config/Config.java index 64c08009..9e263924 100644 --- a/src/main/java/com/merkrafter/config/Config.java +++ b/src/main/java/com/merkrafter/config/Config.java @@ -42,15 +42,20 @@ public static Config fromArgs(final String args) throws ArgumentParserException public static Config fromArgs(final String[] args) throws ArgumentParserException { // define the parser - final ArgumentParser parser = - ArgumentParsers.newFor("Merkompiler").build().defaultHelp(true) - .description("Compiles JavaSST files"); - - parser.addArgument("INPUT").required(true).type(String.class) + final ArgumentParser parser = ArgumentParsers.newFor("Merkompiler") + .build() + .defaultHelp(true) + .description("Compiles JavaSST files"); + parser.addArgument("INPUT") + .required(true) + .type(String.class) .help("JavaSST source code file"); - parser.addArgument("-v", "--verbose").action(Arguments.storeTrue()) + parser.addArgument("-v", "--verbose") + .action(Arguments.storeTrue()) .help("print more information (absolute paths instead of simple file names in error messages, for instance"); - parser.addArgument("-o", "--output").type(String.class).metavar("OUTPUT") + parser.addArgument("-o", "--output") + .type(String.class) + .metavar("OUTPUT") .help("output target; default is stdout"); @@ -86,7 +91,10 @@ public static String[] fromString(final String argsAsString) { */ @Override public String toString() { - return String - .format("Config(INPUT=%s, OUTPUT=%s, verbose=%b)", inputFile, outputFile, verbose); + return String.format("Config(INPUT=%s, OUTPUT=%s, verbose=%b)", + inputFile, + outputFile, + verbose); + } } } From 5f0ba78cf00e2e538b7532156c90e44c4a633a3b Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Fri, 29 Nov 2019 00:15:28 +0100 Subject: [PATCH 069/135] Implement --version argument #6 --- .../java/com/merkrafter/config/Config.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/main/java/com/merkrafter/config/Config.java b/src/main/java/com/merkrafter/config/Config.java index 9e263924..cf51670e 100644 --- a/src/main/java/com/merkrafter/config/Config.java +++ b/src/main/java/com/merkrafter/config/Config.java @@ -5,6 +5,13 @@ import net.sourceforge.argparse4j.inf.ArgumentParser; import net.sourceforge.argparse4j.inf.ArgumentParserException; import net.sourceforge.argparse4j.inf.Namespace; +import org.apache.maven.model.Model; +import org.apache.maven.model.io.xpp3.MavenXpp3Reader; +import org.codehaus.plexus.util.xml.pull.XmlPullParserException; + +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; /** * This class holds configuration data for this program. @@ -46,6 +53,11 @@ public static Config fromArgs(final String[] args) throws ArgumentParserExceptio .build() .defaultHelp(true) .description("Compiles JavaSST files"); + try { + parser.version("${prog} " + getVersion()); + } catch (XmlPullParserException | IOException ignored) { + parser.version("No version information available."); + } parser.addArgument("INPUT") .required(true) .type(String.class) @@ -53,6 +65,9 @@ public static Config fromArgs(final String[] args) throws ArgumentParserExceptio parser.addArgument("-v", "--verbose") .action(Arguments.storeTrue()) .help("print more information (absolute paths instead of simple file names in error messages, for instance"); + parser.addArgument("-V", "--version") + .action(Arguments.version()) + .help("print version information and exit"); parser.addArgument("-o", "--output") .type(String.class) .metavar("OUTPUT") @@ -96,5 +111,15 @@ public String toString() { outputFile, verbose); } + + /** + * Retrieve version information from the pom.xml file. + * + * @return a String containing the software version + */ + private static String getVersion() throws IOException, XmlPullParserException { + MavenXpp3Reader reader = new MavenXpp3Reader(); + Model model = reader.read(new FileReader("pom.xml")); + return model.getVersion(); } } From 6a56deeb32a167f501d9ce231472e594d32b7fcb Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 15:34:33 +0100 Subject: [PATCH 070/135] Implement a NumberToken #13 --- .../com/merkrafter/lexing/NumberToken.java | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 src/main/java/com/merkrafter/lexing/NumberToken.java diff --git a/src/main/java/com/merkrafter/lexing/NumberToken.java b/src/main/java/com/merkrafter/lexing/NumberToken.java new file mode 100644 index 00000000..75dfe992 --- /dev/null +++ b/src/main/java/com/merkrafter/lexing/NumberToken.java @@ -0,0 +1,69 @@ +package com.merkrafter.lexing; + +/**** + * This class serves as a token and stores the (integer) number found. + * + * @version v0.2.0 + * @author merkrafter + ***************************************************************/ +public class NumberToken extends Token { + // ATTRIBUTES + //============================================================== + /** + * the number this token stands for + */ + private final long number; + + // CONSTRUCTORS + //============================================================== + + /**** + * Creates a new NumberToken from a number and position data. + ***************************************************************/ + public NumberToken(final long number, final String filename, final long line, + final int position) { + super(TokenType.NUMBER, filename, line, position); + this.number = number; + } + + // GETTER + //============================================================== + + /** + * @return the number this token stands for + */ + long getNumber() { + return number; + } + + // METHODS + //============================================================== + // public methods + //-------------------------------------------------------------- + + /** + * Two NumberTokens are equal if both have the type NumberToken and their numbers, line + * numbers, positions and filenames are equal. + * + * @param obj ideally a NumberToken to compare this with + * @return whether this is equal to obj + */ + @Override + public boolean equals(final Object obj) { + if (!super.equals(obj)) { + return false; + } + return obj instanceof NumberToken && ((NumberToken) obj).number == number; + } + + /** + * Creates a String representation of this NumberToken in the following format: + * FILENAME(LINE,POSITION): TYPE(NUMBER) + * + * @return a String representation of this NumberToken + */ + @Override + public String toString() { + return super.toString() + String.format("(%d)", number); + } +} From ba0ba45c5beed6311c2b87edee85c2c77e2979c8 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 15:55:58 +0100 Subject: [PATCH 071/135] Add happy path test case for NumberTokens --- .../java/com/merkrafter/lexing/ScannerTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/test/java/com/merkrafter/lexing/ScannerTest.java b/src/test/java/com/merkrafter/lexing/ScannerTest.java index c4a9f381..9e025d59 100644 --- a/src/test/java/com/merkrafter/lexing/ScannerTest.java +++ b/src/test/java/com/merkrafter/lexing/ScannerTest.java @@ -3,6 +3,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; import java.util.Iterator; import java.util.LinkedList; @@ -38,6 +39,20 @@ void setUp() { scanner = new Scanner(stringIterator); } + /** + * The scanner should be able to detect number arguments. + */ + @ParameterizedTest + // edge cases 0 and MAX_VALUE, one, two and three digit numbers + @ValueSource(longs = {0, 1, 10, 123, Long.MAX_VALUE}) + void scanNumber(final long number) { + final String programCode = Long.toString(number); + final Token[] expectedTokenList = { + new NumberToken(number, null, 1, 1), + new Token(EOF, null, 1, Long.toString(number).length())}; + shouldScan(programCode, expectedTokenList); + } + /** * The scanner should be able to detect keyword arguments. */ From 868f06caecfefd8dd480bbac0aa7dc06a414f306 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 15:57:29 +0100 Subject: [PATCH 072/135] Implement scanning numbers to NumberTokens --- .../java/com/merkrafter/lexing/Scanner.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/main/java/com/merkrafter/lexing/Scanner.java b/src/main/java/com/merkrafter/lexing/Scanner.java index 09161613..8a4eb3b4 100644 --- a/src/main/java/com/merkrafter/lexing/Scanner.java +++ b/src/main/java/com/merkrafter/lexing/Scanner.java @@ -133,9 +133,11 @@ public void processToken() { do { num += ch; if (!this.loadNextCharSuccessfully()) { + setNumber(); // parse the num attribute to a NumberToken return; } } while (ch >= '0' && ch <= '9'); + setNumber(); // parse the num attribute to a NumberToke break; case 'a': case 'b': @@ -411,4 +413,19 @@ private void setIdentOrKeyword() { } } + /** + * Tests whether num currently holds a number. If that's the case, sym is changed + * to a NumberToken. Else, a OTHER TokenType is emitted in order to indicate an error. + */ + private void setNumber() { + try { + final long number = Long.parseLong(num); + // if this actually is a number: + sym = new NumberToken(number, sym.getFilename(), sym.getLine(), sym.getPosition()); + } catch (NumberFormatException ignored) { + // id is not a number + sym = new Token(TokenType.OTHER, sym.getFilename(), sym.getLine(), sym.getPosition()); + } + } + } From 792fcf7c2e2bb43f09ce4e7fc1824db1bd59506e Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 15:58:05 +0100 Subject: [PATCH 073/135] Reformat ScannerTest test cases --- .../com/merkrafter/lexing/ScannerTest.java | 75 +++++++++++++++---- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/src/test/java/com/merkrafter/lexing/ScannerTest.java b/src/test/java/com/merkrafter/lexing/ScannerTest.java index 9e025d59..cf8c0add 100644 --- a/src/test/java/com/merkrafter/lexing/ScannerTest.java +++ b/src/test/java/com/merkrafter/lexing/ScannerTest.java @@ -116,9 +116,23 @@ void scanSingleIdentifierWithMixedCase() { @org.junit.jupiter.api.Test void scanAssignmentWithSpaces() { final String programCode = "int result = a + ( b - c ) * d / e;"; - final TokenType[] expectedTokenList = - {KEYWORD, IDENT, ASSIGN, IDENT, PLUS, L_PAREN, IDENT, MINUS, IDENT, R_PAREN, TIMES, - IDENT, DIVIDE, IDENT, SEMICOLON, EOF}; + final TokenType[] expectedTokenList = { + KEYWORD, + IDENT, + ASSIGN, + IDENT, + PLUS, + L_PAREN, + IDENT, + MINUS, + IDENT, + R_PAREN, + TIMES, + IDENT, + DIVIDE, + IDENT, + SEMICOLON, + EOF}; shouldScan(programCode, expectedTokenList); } @@ -141,9 +155,23 @@ void scanSimpleAssignmentWithWhitespace() { @org.junit.jupiter.api.Test void scanAssignmentWithoutWhitespace() { final String programCode = "int result=a+(b-c)*d/e;"; - final TokenType[] expectedTokenList = - {KEYWORD, IDENT, ASSIGN, IDENT, PLUS, L_PAREN, IDENT, MINUS, IDENT, R_PAREN, TIMES, - IDENT, DIVIDE, IDENT, SEMICOLON, EOF}; + final TokenType[] expectedTokenList = { + KEYWORD, + IDENT, + ASSIGN, + IDENT, + PLUS, + L_PAREN, + IDENT, + MINUS, + IDENT, + R_PAREN, + TIMES, + IDENT, + DIVIDE, + IDENT, + SEMICOLON, + EOF}; shouldScan(programCode, expectedTokenList); } @@ -217,8 +245,7 @@ void scanNoBlockComment() { */ @org.junit.jupiter.api.Test void scanAndIgnoreBlockCommentsMultiline() { - final String programCode = - "/*\nThis is a description of the method\n*/public void draw();"; + final String programCode = "/*\nThis is a description of the method\n*/public void draw();"; final TokenType[] expectedTokenList = {KEYWORD, KEYWORD, IDENT, L_PAREN, R_PAREN, SEMICOLON, EOF}; shouldScan(programCode, expectedTokenList); @@ -264,9 +291,19 @@ void scanAndIgnoreAsterisksInComments() { @org.junit.jupiter.api.Test void scanMainFunction() { final String programCode = "public void main(String[] args) {}"; - final TokenType[] expectedTokenList = - {KEYWORD, KEYWORD, IDENT, L_PAREN, IDENT, L_SQ_BRACKET, R_SQ_BRACKET, IDENT, - R_PAREN, L_BRACE, R_BRACE, EOF}; + final TokenType[] expectedTokenList = { + KEYWORD, + KEYWORD, + IDENT, + L_PAREN, + IDENT, + L_SQ_BRACKET, + R_SQ_BRACKET, + IDENT, + R_PAREN, + L_BRACE, + R_BRACE, + EOF}; shouldScan(programCode, expectedTokenList); } @@ -331,9 +368,19 @@ void scanMethodCallWithTwoArguments() { @org.junit.jupiter.api.Test void scanMethodCallWithMultipleArguments() { final String programCode = "int sum = add(a,b,5)"; - final TokenType[] expectedTokenList = - {KEYWORD, IDENT, ASSIGN, IDENT, L_PAREN, IDENT, COMMA, IDENT, COMMA, NUMBER, R_PAREN, - EOF}; + final TokenType[] expectedTokenList = { + KEYWORD, + IDENT, + ASSIGN, + IDENT, + L_PAREN, + IDENT, + COMMA, + IDENT, + COMMA, + NUMBER, + R_PAREN, + EOF}; shouldScan(programCode, expectedTokenList); } From e9df583ee6be403a113ef89f0f00924b0d127900 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 16:22:51 +0100 Subject: [PATCH 074/135] Add scanner test case for special (0-preceded) numbers --- .../java/com/merkrafter/lexing/ScannerTest.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/merkrafter/lexing/ScannerTest.java b/src/test/java/com/merkrafter/lexing/ScannerTest.java index cf8c0add..65922482 100644 --- a/src/test/java/com/merkrafter/lexing/ScannerTest.java +++ b/src/test/java/com/merkrafter/lexing/ScannerTest.java @@ -45,7 +45,7 @@ void setUp() { @ParameterizedTest // edge cases 0 and MAX_VALUE, one, two and three digit numbers @ValueSource(longs = {0, 1, 10, 123, Long.MAX_VALUE}) - void scanNumber(final long number) { + void scanNormalNumbers(final long number) { final String programCode = Long.toString(number); final Token[] expectedTokenList = { new NumberToken(number, null, 1, 1), @@ -53,6 +53,21 @@ void scanNumber(final long number) { shouldScan(programCode, expectedTokenList); } + /** + * The scanner should be able to detect special number arguments, i.e. with leading zeros. + */ + @ParameterizedTest + // all values should be decimal 8's, because in JavaSST there are no octal numbers hence these + // value source numbers will cause an error when trying to evaluate them as octal. + @ValueSource(strings = {"08", "008"}) + void scanSpecialNumbers(final String number) { + final long expectedNumber = 8; + final Token[] expectedTokenList = { + new NumberToken(expectedNumber, null, 1, 1), + new Token(EOF, null, 1, number.length())}; + shouldScan(number, expectedTokenList); + } + /** * The scanner should be able to detect keyword arguments. */ From 916deaa9af5134f91a5de97eb2b78a91c0d9b9f9 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 16:27:19 +0100 Subject: [PATCH 075/135] Fix the use of "KeywordToken" in "IdentToken" #18 --- src/main/java/com/merkrafter/lexing/IdentToken.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/merkrafter/lexing/IdentToken.java b/src/main/java/com/merkrafter/lexing/IdentToken.java index 49696c97..b4b519e0 100644 --- a/src/main/java/com/merkrafter/lexing/IdentToken.java +++ b/src/main/java/com/merkrafter/lexing/IdentToken.java @@ -30,7 +30,7 @@ public IdentToken(final String ident, final String filename, final long line, //============================================================== /** - * @return the keyword this token stands for + * @return the identifier this token stands for */ String getIdent() { return ident; @@ -57,7 +57,7 @@ public boolean equals(final Object obj) { } /** - * Creates a String representation of this KeywordToken in the following format: + * Creates a String representation of this IdentToken in the following format: * FILENAME(LINE,POSITION): TYPE(IDENT) * * @return a String representation of this IdentToken From db52aaf639465f41a1897ea6cd4aab34c4bd7687 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 17:05:39 +0100 Subject: [PATCH 076/135] Implement an OtherToken #13 --- .../com/merkrafter/lexing/OtherToken.java | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 src/main/java/com/merkrafter/lexing/OtherToken.java diff --git a/src/main/java/com/merkrafter/lexing/OtherToken.java b/src/main/java/com/merkrafter/lexing/OtherToken.java new file mode 100644 index 00000000..9ef8b53d --- /dev/null +++ b/src/main/java/com/merkrafter/lexing/OtherToken.java @@ -0,0 +1,69 @@ +package com.merkrafter.lexing; + +/**** + * This class serves as a token and stores a string that could not be recognized as another token. + * + * @version v0.2.0 + * @author merkrafter + ***************************************************************/ +public class OtherToken extends Token { + // ATTRIBUTES + //============================================================== + /** + * the string that could not be recognized as another token + */ + private final String string; + + // CONSTRUCTORS + //============================================================== + + /**** + * Creates a new OtherToken from a string and position data. + ***************************************************************/ + public OtherToken(final String string, final String filename, final long line, + final int position) { + super(TokenType.OTHER, filename, line, position); + this.string = string; + } + + // GETTER + //============================================================== + + /** + * @return the string that could not be recognized as another token + */ + String getString() { + return string; + } + + // METHODS + //============================================================== + // public methods + //-------------------------------------------------------------- + + /** + * Two OtherTokens are equal if both have the type OtherToken and their strings, line + * numbers, positions and filenames are equal. + * + * @param obj ideally a OtherToken to compare this with + * @return whether this is equal to obj + */ + @Override + public boolean equals(final Object obj) { + if (!super.equals(obj)) { + return false; + } + return obj instanceof OtherToken && ((OtherToken) obj).string.equals(string); + } + + /** + * Creates a String representation of this OtherToken in the following format: + * FILENAME(LINE,POSITION): TYPE(STRING) + * + * @return a String representation of this OtherToken + */ + @Override + public String toString() { + return super.toString() + String.format("(%s)", string); + } +} From 89c79164ca117f993462a0d4564e6f7e3708f66d Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 17:06:31 +0100 Subject: [PATCH 077/135] Add happy path test case for OtherTokens --- .../java/com/merkrafter/lexing/ScannerTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/test/java/com/merkrafter/lexing/ScannerTest.java b/src/test/java/com/merkrafter/lexing/ScannerTest.java index 65922482..b7a832f9 100644 --- a/src/test/java/com/merkrafter/lexing/ScannerTest.java +++ b/src/test/java/com/merkrafter/lexing/ScannerTest.java @@ -39,6 +39,19 @@ void setUp() { scanner = new Scanner(stringIterator); } + /** + * The scanner should be able to detect "other" tokens, i.e. special characters that are not + * part of the language. + */ + @ParameterizedTest + @ValueSource(strings = {"$", "\"", "@", "_", "!", "§", "%", "&", "|", "^", "\\", "?", "~", "#"}) + void scanOtherTokens(final String string) { + final String programCode = string; + final Token[] expectedTokenList = { + new OtherToken(string, null, 1, 1), new Token(EOF, null, 1, string.length())}; + shouldScan(programCode, expectedTokenList); + } + /** * The scanner should be able to detect number arguments. */ From 4baaa84f6954ccc244c44cf19d4c7aa34a27f1bb Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 17:12:47 +0100 Subject: [PATCH 078/135] Implement scanning unknown characters to OtherTokens --- src/main/java/com/merkrafter/lexing/Scanner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/merkrafter/lexing/Scanner.java b/src/main/java/com/merkrafter/lexing/Scanner.java index 8a4eb3b4..c9950bc0 100644 --- a/src/main/java/com/merkrafter/lexing/Scanner.java +++ b/src/main/java/com/merkrafter/lexing/Scanner.java @@ -345,7 +345,7 @@ public void processToken() { } break; default: - sym = new Token(TokenType.OTHER, filename, line, position); + sym = new OtherToken(Character.toString(ch), filename, line, position); this.loadNextCharSuccessfully(); } } From 41c8a76464fecb32836cf43773e39f140db328fb Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 18:18:31 +0100 Subject: [PATCH 079/135] Add CompilerStage enum --- .../com/merkrafter/config/CompilerStage.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/main/java/com/merkrafter/config/CompilerStage.java diff --git a/src/main/java/com/merkrafter/config/CompilerStage.java b/src/main/java/com/merkrafter/config/CompilerStage.java new file mode 100644 index 00000000..966ef848 --- /dev/null +++ b/src/main/java/com/merkrafter/config/CompilerStage.java @@ -0,0 +1,21 @@ +package com.merkrafter.config; + +/**** + * This enum represents the steps the compiler goes through in order to convert a JavaSST source + * code file into actual byte code. + * + * @version v0.2.0 + * @author merkrafter + ***************************************************************/ +public enum CompilerStage { + // CONSTANTS + //============================================================== + /** + * Only scan the input and output tokens. + */ + SCANNING, + /** + * Scan and parse the input and output whether this was successful. + */ + PARSING; +} From d187f32b335713708712b14ad67d71e86cb1bd33 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 18:38:39 +0100 Subject: [PATCH 080/135] Add --skip-after CLI argument #11 --- src/main/java/com/merkrafter/config/Config.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/merkrafter/config/Config.java b/src/main/java/com/merkrafter/config/Config.java index cf51670e..bc4bddd5 100644 --- a/src/main/java/com/merkrafter/config/Config.java +++ b/src/main/java/com/merkrafter/config/Config.java @@ -72,6 +72,9 @@ public static Config fromArgs(final String[] args) throws ArgumentParserExceptio .type(String.class) .metavar("OUTPUT") .help("output target; default is stdout"); + parser.addArgument("--skip-after") + .type(Arguments.caseInsensitiveEnumType(CompilerStage.class)) + .help("only process the input file up to the given stage (including)"); // parse the arguments From 83a56664146c8540cfd7d9bb785b1bc12b706447 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 19:01:09 +0100 Subject: [PATCH 081/135] Overwrite CompilerStage::toString --- src/main/java/com/merkrafter/config/CompilerStage.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/com/merkrafter/config/CompilerStage.java b/src/main/java/com/merkrafter/config/CompilerStage.java index 966ef848..f9cbac97 100644 --- a/src/main/java/com/merkrafter/config/CompilerStage.java +++ b/src/main/java/com/merkrafter/config/CompilerStage.java @@ -18,4 +18,13 @@ public enum CompilerStage { * Scan and parse the input and output whether this was successful. */ PARSING; + + + /** + * @return the lowercase name of this enum item + */ + @Override + public String toString() { + return name().toLowerCase(); + } } From 801c81e49158bb6451398066ac20d59c399ec10f Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 19:01:48 +0100 Subject: [PATCH 082/135] Add CompilerStage::latest method --- src/main/java/com/merkrafter/config/CompilerStage.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/com/merkrafter/config/CompilerStage.java b/src/main/java/com/merkrafter/config/CompilerStage.java index f9cbac97..60dc1213 100644 --- a/src/main/java/com/merkrafter/config/CompilerStage.java +++ b/src/main/java/com/merkrafter/config/CompilerStage.java @@ -27,4 +27,13 @@ public enum CompilerStage { public String toString() { return name().toLowerCase(); } + + /** + * Returns the latest stage this enum currently offers in terms of processing data. + * + * @return the latest available compiler stage + */ + public static CompilerStage latest() { + return PARSING; + } } From 20d057adbdbec1cea1c577b8fda9b84f74d7a871 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 19:02:12 +0100 Subject: [PATCH 083/135] Implement actually evaluating the skip--after argument --- .../java/com/merkrafter/config/Config.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/merkrafter/config/Config.java b/src/main/java/com/merkrafter/config/Config.java index bc4bddd5..025b9939 100644 --- a/src/main/java/com/merkrafter/config/Config.java +++ b/src/main/java/com/merkrafter/config/Config.java @@ -25,10 +25,14 @@ public class Config { private final boolean verbose; - private Config(final String inputFile, final String outputFile, boolean verbose) { + private final CompilerStage stage; + + private Config(final String inputFile, final String outputFile, boolean verbose, + final CompilerStage stage) { this.inputFile = inputFile; this.outputFile = outputFile; this.verbose = verbose; + this.stage = stage; } public String getInputFile() { @@ -74,6 +78,8 @@ public static Config fromArgs(final String[] args) throws ArgumentParserExceptio .help("output target; default is stdout"); parser.addArgument("--skip-after") .type(Arguments.caseInsensitiveEnumType(CompilerStage.class)) + .dest("compilerStage") + .setDefault(CompilerStage.latest()) .help("only process the input file up to the given stage (including)"); @@ -85,13 +91,16 @@ public static Config fromArgs(final String[] args) throws ArgumentParserExceptio String inputFileName = null; String outputFileName = null; boolean verbose = false; + CompilerStage stage = CompilerStage.latest(); if (namespace != null) { inputFileName = namespace.getString("INPUT"); outputFileName = namespace.getString("output"); verbose = namespace.getBoolean("verbose"); + stage = namespace.get("compilerStage"); } - return new Config(inputFileName, outputFileName, verbose); + + return new Config(inputFileName, outputFileName, verbose, stage); } /** @@ -109,10 +118,11 @@ public static String[] fromString(final String argsAsString) { */ @Override public String toString() { - return String.format("Config(INPUT=%s, OUTPUT=%s, verbose=%b)", + return String.format("Config(INPUT=%s, OUTPUT=%s, verbose=%b, stage=%s)", inputFile, outputFile, - verbose); + verbose, + stage); } /** From 1b581ca0e6af28552a118e89094df9ea568c11c9 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 19:17:46 +0100 Subject: [PATCH 084/135] Implement Config::getStage --- src/main/java/com/merkrafter/config/Config.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/merkrafter/config/Config.java b/src/main/java/com/merkrafter/config/Config.java index 025b9939..0e3b671f 100644 --- a/src/main/java/com/merkrafter/config/Config.java +++ b/src/main/java/com/merkrafter/config/Config.java @@ -47,6 +47,10 @@ public boolean isVerbose() { return verbose; } + public CompilerStage getStage() { + return stage; + } + public static Config fromArgs(final String args) throws ArgumentParserException { return fromArgs(fromString(args)); } From 562c9734334d546f298c46bcff636b510078dcbf Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 19:18:09 +0100 Subject: [PATCH 085/135] Add test case for --skip-after scanning --- src/test/java/com/merkrafter/ConfigTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/test/java/com/merkrafter/ConfigTest.java b/src/test/java/com/merkrafter/ConfigTest.java index 0ca90998..2ae748cf 100644 --- a/src/test/java/com/merkrafter/ConfigTest.java +++ b/src/test/java/com/merkrafter/ConfigTest.java @@ -1,8 +1,11 @@ package com.merkrafter; +import com.merkrafter.config.CompilerStage; import com.merkrafter.config.Config; import net.sourceforge.argparse4j.inf.ArgumentParserException; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import static com.merkrafter.config.Config.fromString; import static org.junit.jupiter.api.Assertions.*; @@ -149,4 +152,14 @@ void parseInputFileAndOutputFileAndVerbosity() throws ArgumentParserException { assertEquals(expectedVerbosity, actualConfig.isVerbose()); } + @ParameterizedTest + @ValueSource(strings = {"scanning", "SCANNING", "sCanning", "sCaNnInG"}) + void skipAfterScanning(final String spelling) throws ArgumentParserException { + final String[] args = fromString(String.format("--skip-after %s Test.java", spelling)); + final Config actualConfig = Config.fromArgs(args); + + final CompilerStage expectedStage = CompilerStage.SCANNING; + + assertEquals(expectedStage, actualConfig.getStage()); + } } \ No newline at end of file From e3229ebfe9dd7a1b4d7d072a35a74ecfa6abb257 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 19:19:29 +0100 Subject: [PATCH 086/135] Add test case for --skip-after parsing --- src/test/java/com/merkrafter/ConfigTest.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/test/java/com/merkrafter/ConfigTest.java b/src/test/java/com/merkrafter/ConfigTest.java index 2ae748cf..387d27b0 100644 --- a/src/test/java/com/merkrafter/ConfigTest.java +++ b/src/test/java/com/merkrafter/ConfigTest.java @@ -162,4 +162,15 @@ void skipAfterScanning(final String spelling) throws ArgumentParserException { assertEquals(expectedStage, actualConfig.getStage()); } + + @ParameterizedTest + @ValueSource(strings = {"parsing", "PARSING", "pArsing", "pArSinG"}) + void skipAfterParsing(final String spelling) throws ArgumentParserException { + final String[] args = fromString(String.format("--skip-after %s Test.java", spelling)); + final Config actualConfig = Config.fromArgs(args); + + final CompilerStage expectedStage = CompilerStage.PARSING; + + assertEquals(expectedStage, actualConfig.getStage()); + } } \ No newline at end of file From d42b4eb1168e39aee8e1627a5a534fbff3234ddc Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 19:22:10 +0100 Subject: [PATCH 087/135] Add test case for default --skip-after argument --- src/test/java/com/merkrafter/ConfigTest.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/test/java/com/merkrafter/ConfigTest.java b/src/test/java/com/merkrafter/ConfigTest.java index 387d27b0..42a31bfd 100644 --- a/src/test/java/com/merkrafter/ConfigTest.java +++ b/src/test/java/com/merkrafter/ConfigTest.java @@ -173,4 +173,14 @@ void skipAfterParsing(final String spelling) throws ArgumentParserException { assertEquals(expectedStage, actualConfig.getStage()); } + + @Test + void defaultStage() throws ArgumentParserException { + final String[] args = fromString("Test.java"); + final Config actualConfig = Config.fromArgs(args); + + final CompilerStage expectedStage = CompilerStage.PARSING; + + assertEquals(expectedStage, actualConfig.getStage()); + } } \ No newline at end of file From 93fb6248d8dc6780e204c6cc4cd8490d92c072e2 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 19:27:56 +0100 Subject: [PATCH 088/135] Adapt Merkompiler test cases to the --skip-after argument --- src/test/java/com/merkrafter/MerkompilerRunTest.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/merkrafter/MerkompilerRunTest.java b/src/test/java/com/merkrafter/MerkompilerRunTest.java index b1c876c6..cb3a2645 100644 --- a/src/test/java/com/merkrafter/MerkompilerRunTest.java +++ b/src/test/java/com/merkrafter/MerkompilerRunTest.java @@ -1,5 +1,6 @@ package com.merkrafter; +import com.merkrafter.config.CompilerStage; import com.merkrafter.config.Config; import net.sourceforge.argparse4j.inf.ArgumentParserException; import org.junit.jupiter.api.io.TempDir; @@ -49,7 +50,7 @@ class MerkompilerRunTest { */ @ParameterizedTest @ValueSource(strings = "EmptyClass") - void runWithOutputCreatesFile(final String baseFileName) + void scanWithOutputCreatesFile(final String baseFileName) throws ArgumentParserException, IOException { // java source file to read final File inputFile = getFileFromResource(baseFileName + INPUT_FILE_SUFFIX); @@ -58,7 +59,8 @@ void runWithOutputCreatesFile(final String baseFileName) // file where the program output is written to final File outputFile = tempDir.resolve(baseFileName + OUTPUT_FILE_SUFFIX).toFile(); - final Config config = Config.fromArgs(String.format("%s --output %s", + final Config config = Config.fromArgs(String.format("--skip-after %s %s --output %s", + CompilerStage.SCANNING.toString(), inputFile.getAbsolutePath(), outputFile.getAbsolutePath())); Merkompiler.run(config); @@ -82,7 +84,7 @@ void runWithOutputCreatesFile(final String baseFileName) */ @ParameterizedTest @ValueSource(strings = "EmptyClass") - void runWithoutOutput(final String baseFileName) throws ArgumentParserException, IOException { + void scanWithoutOutput(final String baseFileName) throws ArgumentParserException, IOException { final PrintStream originalOut = System.out; try { // will reset System.out in case of errors final ByteArrayOutputStream output = new ByteArrayOutputStream(); @@ -95,7 +97,9 @@ void runWithoutOutput(final String baseFileName) throws ArgumentParserException, System.setOut(new PrintStream(output)); // run main program without specifying output - final Config config = Config.fromArgs(inputFile.getAbsolutePath()); + final Config config = Config.fromArgs(String.format("--skip-after %s %s", + CompilerStage.SCANNING.toString(), + inputFile.getAbsolutePath())); Merkompiler.run(config); assertEquals(toString(expectedFile), output.toString().trim()); From fb511d687451e80eff82171387b19236ccbeb7cb Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 19:57:42 +0100 Subject: [PATCH 089/135] Fix typo in MerkompilerTest --- src/test/java/com/merkrafter/MerkompilerRunTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/merkrafter/MerkompilerRunTest.java b/src/test/java/com/merkrafter/MerkompilerRunTest.java index cb3a2645..152f9ab2 100644 --- a/src/test/java/com/merkrafter/MerkompilerRunTest.java +++ b/src/test/java/com/merkrafter/MerkompilerRunTest.java @@ -44,7 +44,7 @@ class MerkompilerRunTest { * The output file for this experiment will be named baseFileName{@value OUTPUT_FILE_SUFFIX} * and is created as a temporary file by JUnit. * - * @param baseFileName if used to find the source file name and expected file name and create the output file + * @param baseFileName is used to find the source file name and expected file name and create the output file * @throws ArgumentParserException if the arguments in the test case are misconfigured (should not happen) * @throws IOException if there is a read/write error in one of the files */ @@ -78,7 +78,7 @@ void scanWithOutputCreatesFile(final String baseFileName) * This method resets System.out in order to test the output written to it. * If this method throws an exception, System.out might still be unavailable. * - * @param baseFileName if used to find the source file name and expected file name + * @param baseFileName is used to find the source file name and expected file name * @throws ArgumentParserException if the arguments in the test case are misconfigured (should not happen) * @throws IOException if there is a read/write error in one of the files */ From 58dd30e0fe0aa7e6849864af89e903fcec59ac03 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 20:00:12 +0100 Subject: [PATCH 090/135] Add happy path test case for Merkompiler's parsing run --- .../com/merkrafter/MerkompilerRunTest.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/test/java/com/merkrafter/MerkompilerRunTest.java b/src/test/java/com/merkrafter/MerkompilerRunTest.java index 152f9ab2..a862e172 100644 --- a/src/test/java/com/merkrafter/MerkompilerRunTest.java +++ b/src/test/java/com/merkrafter/MerkompilerRunTest.java @@ -15,6 +15,7 @@ import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeTrue; /** @@ -108,6 +109,45 @@ void scanWithoutOutput(final String baseFileName) throws ArgumentParserException } } + /** + * This test case runs the lexer and parser on the file(s) given by ValueSource. + * If there is no syntax error, the program should not write anything to stdout. + * This test assumes that there is baseFileName{@value INPUT_FILE_SUFFIX} present under + * src/test/resources in the project. + * The output for this experiment is not specified by a CLI argument, hence it tests writing to + * stdout. + *

+ * This method resets System.out in order to test the output written to it. + * If this method throws an exception, System.out might still be unavailable. + * + * @param baseFileName is used to find the source file name + * @throws ArgumentParserException if the arguments in the test case are misconfigured (should not happen) + * @throws IOException if there is a read/write error in the input file + */ + @ParameterizedTest + @ValueSource(strings = "EmptyClass") + void parseWithoutOutput(final String baseFileName) throws ArgumentParserException, IOException { + final PrintStream originalOut = System.out; + try { // will reset System.out in case of errors + final ByteArrayOutputStream output = new ByteArrayOutputStream(); + + // java source file to read + final File inputFile = getFileFromResource(baseFileName + INPUT_FILE_SUFFIX); + // set stdout to testable output stream + System.setOut(new PrintStream(output)); + + // run main program without specifying output + final Config config = Config.fromArgs(String.format("--skip-after %s %s", + CompilerStage.PARSING.toString(), + inputFile.getAbsolutePath())); + Merkompiler.run(config); + + assertTrue(output.toString().trim().isEmpty()); + } finally { + System.setOut(originalOut); // reset System.out even in case of errors + } + } + /** * Reads the given file using this class's class loader, checks for its existence and finally * returns it. From ab44dce7dce191883b57f0408b0f51e9ec03933f Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 20:14:13 +0100 Subject: [PATCH 091/135] Make the Merkompiler actually call the Parser --- src/main/java/com/merkrafter/Merkompiler.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/merkrafter/Merkompiler.java b/src/main/java/com/merkrafter/Merkompiler.java index de8e9b31..69388ae1 100644 --- a/src/main/java/com/merkrafter/Merkompiler.java +++ b/src/main/java/com/merkrafter/Merkompiler.java @@ -1,9 +1,11 @@ package com.merkrafter; +import com.merkrafter.config.CompilerStage; import com.merkrafter.config.Config; import com.merkrafter.config.ErrorCode; import com.merkrafter.lexing.Scanner; import com.merkrafter.lexing.TokenType; +import com.merkrafter.parsing.Parser; import net.sourceforge.argparse4j.inf.ArgumentParserException; import java.io.File; @@ -58,9 +60,15 @@ static void run(final Config config) throws FileNotFoundException { out = new PrintStream(config.getOutputFile()); } - do { - scanner.processToken(); - out.println(scanner.getSym()); - } while (scanner.getSym().getType() != TokenType.EOF); + if (config.getStage() == CompilerStage.SCANNING) { + // only print the tokens if the processing should stop after scanning + do { + scanner.processToken(); + out.println(scanner.getSym()); + } while (scanner.getSym().getType() != TokenType.EOF); + } else if (config.getStage() == CompilerStage.PARSING) { + final Parser parser = new Parser(scanner); + out.println(parser.parse()); + } } } \ No newline at end of file From 5bf836fd5393b414ce436b8f00dadfa555f08773 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 20:48:57 +0100 Subject: [PATCH 092/135] Check for EOF after finish parsing --- src/main/java/com/merkrafter/parsing/Parser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 9b6f4198..1e98ff93 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -41,7 +41,7 @@ public Parser(final Scanner scanner) { * Parses the tokens given by the underlying token iterator. */ public boolean parse() { - return false; + return parseClass() && scanner.getSym().getType() == EOF; } boolean parseClass() { From 6e6fca5bc6e8bfb276a586dab62ee9fa2ddfdd25 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 21:38:39 +0100 Subject: [PATCH 093/135] Only print in Merkompiler's parsing part if there are errors --- src/main/java/com/merkrafter/Merkompiler.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/merkrafter/Merkompiler.java b/src/main/java/com/merkrafter/Merkompiler.java index 69388ae1..6bd7c8b3 100644 --- a/src/main/java/com/merkrafter/Merkompiler.java +++ b/src/main/java/com/merkrafter/Merkompiler.java @@ -68,7 +68,9 @@ static void run(final Config config) throws FileNotFoundException { } while (scanner.getSym().getType() != TokenType.EOF); } else if (config.getStage() == CompilerStage.PARSING) { final Parser parser = new Parser(scanner); - out.println(parser.parse()); + if (!parser.parse()) { + out.println("Parsing error!"); + } } } } \ No newline at end of file From 8a9224a6b71af355c30fe7dfdb44ff0beed2f3cf Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 21:53:22 +0100 Subject: [PATCH 094/135] Add test cases to border the bug --- .../com/merkrafter/parsing/ParserTest.java | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index 38f2a22d..03aa460b 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -284,6 +284,63 @@ void parseProcedureCallAsStatement() { assertTrue(parser.parseStatement()); } + /** + * The parser should accept a simple if-else construct as a statement, that is an "if" keyword, + * a comparison between an identifier and a number as the condition and blocks with single + * assignments for if and else. + */ + @Test + void parseSimpleIfAsStatement() { + final Scanner scanner = new TestScanner(new Token[]{ + new KeywordToken(Keyword.IF, null, 1, 1), + new Token(TokenType.L_PAREN, null, 1, 1), + new Token(TokenType.IDENT, null, 1, 1), + new Token(TokenType.EQUAL, null, 1, 1), + new Token(TokenType.NUMBER, null, 1, 1), + new Token(TokenType.R_PAREN, null, 1, 1), + + new Token(TokenType.L_BRACE, null, 1, 1), + new Token(TokenType.IDENT, null, 1, 1), + new Token(TokenType.ASSIGN, null, 1, 1), + new Token(TokenType.NUMBER, null, 1, 1), + new Token(TokenType.SEMICOLON, null, 1, 1), + new Token(TokenType.R_BRACE, null, 1, 1), + + new KeywordToken(Keyword.ELSE, null, 1, 1), + new Token(TokenType.L_BRACE, null, 1, 1), + new Token(TokenType.IDENT, null, 1, 1), + new Token(TokenType.ASSIGN, null, 1, 1), + new Token(TokenType.NUMBER, null, 1, 1), + new Token(TokenType.SEMICOLON, null, 1, 1), + new Token(TokenType.R_BRACE, null, 1, 1),}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseStatement()); + } + + /** + * The parser should accept a simple while loop as a statement, that is a "while" keyword, a + * comparison between an identifier and a number as the condition and a block that has only an + * assignment inside it. + */ + @Test + void parseSimpleWhileAsStatement() { + final Scanner scanner = new TestScanner(new Token[]{ + new KeywordToken(Keyword.WHILE, null, 1, 1), + new Token(TokenType.L_PAREN, null, 1, 1), + new Token(TokenType.IDENT, null, 1, 1), + new Token(TokenType.EQUAL, null, 1, 1), + new Token(TokenType.NUMBER, null, 1, 1), + new Token(TokenType.R_PAREN, null, 1, 1), + new Token(TokenType.L_BRACE, null, 1, 1), + new Token(TokenType.IDENT, null, 1, 1), + new Token(TokenType.ASSIGN, null, 1, 1), + new Token(TokenType.NUMBER, null, 1, 1), + new Token(TokenType.SEMICOLON, null, 1, 1), + new Token(TokenType.R_BRACE, null, 1, 1),}); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseStatement()); + } + /** * The parser should accept a single return keyword as a statement. */ From 4a5dc668b72b48a2dd99f0807f5dc41f1308e5ff Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 21:58:34 +0100 Subject: [PATCH 095/135] Add the missing handling of if and while in Parser::parseStatement --- src/main/java/com/merkrafter/parsing/Parser.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 9b6f4198..d337fba6 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -205,6 +205,12 @@ boolean parseStatement() { if (parseStatementForAssignmentOrProcedureCall()) { return true; } + if (parseIfStatement()) { + return true; + } + if (parseWhileStatement()) { + return true; + } if (parseReturnStatement()) { return true; } From abe9a8b6b44dce5986998d2694a808d557f3794d Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 22:25:18 +0100 Subject: [PATCH 096/135] Add SmokeClass.java test file --- src/test/resources/SmokeClass.java | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 src/test/resources/SmokeClass.java diff --git a/src/test/resources/SmokeClass.java b/src/test/resources/SmokeClass.java new file mode 100644 index 00000000..310993be --- /dev/null +++ b/src/test/resources/SmokeClass.java @@ -0,0 +1,25 @@ +// no public classes allowed +class SmokeClass { + // no 'static' keyword available + // no 'String' type available + public void main() { + int a; + int b; + int c; + int d; + int e; + int result; + a = 1; + b = 2; + c = 3; + d = 4; + e = 5; + result = a+(b-c)*d/e; + if (a <= b) {println(a);} else {println(b);} + if (a < b) {println(a);} else {println(b);} + if (a == b) {println(a);} else {println(b);} + if (a > b) {println(a);} else {println(b);} + if (a >= b) {println(a);} else {println(b);} + println(result); + } +} From 15407842fa7e1d69a09fc74ccdeb5118fb21c917 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 22:27:51 +0100 Subject: [PATCH 097/135] Use SmokeClass.java to test parsing --- src/test/java/com/merkrafter/MerkompilerRunTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/merkrafter/MerkompilerRunTest.java b/src/test/java/com/merkrafter/MerkompilerRunTest.java index a862e172..aef609b5 100644 --- a/src/test/java/com/merkrafter/MerkompilerRunTest.java +++ b/src/test/java/com/merkrafter/MerkompilerRunTest.java @@ -125,7 +125,7 @@ void scanWithoutOutput(final String baseFileName) throws ArgumentParserException * @throws IOException if there is a read/write error in the input file */ @ParameterizedTest - @ValueSource(strings = "EmptyClass") + @ValueSource(strings = "SmokeClass") void parseWithoutOutput(final String baseFileName) throws ArgumentParserException, IOException { final PrintStream originalOut = System.out; try { // will reset System.out in case of errors From d4a691caa8cc93b430ab9dd21afb9802f3cc20c5 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 22:36:34 +0100 Subject: [PATCH 098/135] Add a Merkompiler test case for files with syntax errors --- .../com/merkrafter/MerkompilerRunTest.java | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/merkrafter/MerkompilerRunTest.java b/src/test/java/com/merkrafter/MerkompilerRunTest.java index aef609b5..56b2c5df 100644 --- a/src/test/java/com/merkrafter/MerkompilerRunTest.java +++ b/src/test/java/com/merkrafter/MerkompilerRunTest.java @@ -14,8 +14,7 @@ import java.nio.file.Files; import java.nio.file.Path; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.Assumptions.assumeTrue; /** @@ -148,6 +147,44 @@ void parseWithoutOutput(final String baseFileName) throws ArgumentParserExceptio } } + /** + * This test case runs the lexer and parser on the faulty file(s) given by ValueSource. + * Since there are syntax errors, the program should write some error message(s) to stderr. + * This test assumes that there is baseFileName{@value INPUT_FILE_SUFFIX} present + * under src/test/resources in the project. + *

+ * This method resets System.err in order to test the output written to it. + * If this method throws an exception, System.err might still be unavailable. + * + * @param baseFileName is used to find the source file name + * @throws ArgumentParserException if the arguments in the test case are misconfigured (should not happen) + * @throws IOException if there is a read/write error in the input file + */ + @ParameterizedTest + @ValueSource(strings = "EmptyClass") + void parseFaultyFile(final String baseFileName) throws ArgumentParserException, IOException { + final PrintStream originalErr = System.err; + try { // will reset System.err in case of crashes + final ByteArrayOutputStream output = new ByteArrayOutputStream(); + + // java source file to read + final File inputFile = getFileFromResource(baseFileName + INPUT_FILE_SUFFIX); + // set stderr to testable output stream + System.setErr(new PrintStream(output)); + + // run main program without specifying output + final Config config = Config.fromArgs(String.format("--skip-after %s %s", + CompilerStage.PARSING.toString(), + inputFile.getAbsolutePath())); + Merkompiler.run(config); + + // the compiler should output some message + assertFalse(output.toString().trim().isEmpty()); + } finally { + System.setErr(originalErr); // reset System.err even in case of crashes + } + } + /** * Reads the given file using this class's class loader, checks for its existence and finally * returns it. From 7d6628b689bc0cb4d16807f2121091c86558d6c0 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 22:42:26 +0100 Subject: [PATCH 099/135] Test for parsing errors using stderr instead of stdout --- .../com/merkrafter/MerkompilerRunTest.java | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/test/java/com/merkrafter/MerkompilerRunTest.java b/src/test/java/com/merkrafter/MerkompilerRunTest.java index 56b2c5df..dfd9b4e6 100644 --- a/src/test/java/com/merkrafter/MerkompilerRunTest.java +++ b/src/test/java/com/merkrafter/MerkompilerRunTest.java @@ -110,14 +110,12 @@ void scanWithoutOutput(final String baseFileName) throws ArgumentParserException /** * This test case runs the lexer and parser on the file(s) given by ValueSource. - * If there is no syntax error, the program should not write anything to stdout. + * If there is no syntax error, the program should not write anything to stderr. * This test assumes that there is baseFileName{@value INPUT_FILE_SUFFIX} present under * src/test/resources in the project. - * The output for this experiment is not specified by a CLI argument, hence it tests writing to - * stdout. *

- * This method resets System.out in order to test the output written to it. - * If this method throws an exception, System.out might still be unavailable. + * This method resets System.err in order to test the output written to it. + * If this method throws an exception, System.err might still be unavailable. * * @param baseFileName is used to find the source file name * @throws ArgumentParserException if the arguments in the test case are misconfigured (should not happen) @@ -125,15 +123,15 @@ void scanWithoutOutput(final String baseFileName) throws ArgumentParserException */ @ParameterizedTest @ValueSource(strings = "SmokeClass") - void parseWithoutOutput(final String baseFileName) throws ArgumentParserException, IOException { - final PrintStream originalOut = System.out; - try { // will reset System.out in case of errors + void parseCorrectClass(final String baseFileName) throws ArgumentParserException, IOException { + final PrintStream originalErr = System.err; + try { // will reset System.err in case of crashes final ByteArrayOutputStream output = new ByteArrayOutputStream(); // java source file to read final File inputFile = getFileFromResource(baseFileName + INPUT_FILE_SUFFIX); - // set stdout to testable output stream - System.setOut(new PrintStream(output)); + // set stderr to testable output stream + System.setErr(new PrintStream(output)); // run main program without specifying output final Config config = Config.fromArgs(String.format("--skip-after %s %s", @@ -143,7 +141,7 @@ void parseWithoutOutput(final String baseFileName) throws ArgumentParserExceptio assertTrue(output.toString().trim().isEmpty()); } finally { - System.setOut(originalOut); // reset System.out even in case of errors + System.setErr(originalErr); // reset System.err even in case of crashes } } From 3b6b33f8d728142c0f1c0a746846ba0aeb67d31c Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 22:43:24 +0100 Subject: [PATCH 100/135] Write parser errors to stderr instead of stdout or file --- src/main/java/com/merkrafter/Merkompiler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/merkrafter/Merkompiler.java b/src/main/java/com/merkrafter/Merkompiler.java index 6bd7c8b3..38511c44 100644 --- a/src/main/java/com/merkrafter/Merkompiler.java +++ b/src/main/java/com/merkrafter/Merkompiler.java @@ -69,7 +69,7 @@ static void run(final Config config) throws FileNotFoundException { } else if (config.getStage() == CompilerStage.PARSING) { final Parser parser = new Parser(scanner); if (!parser.parse()) { - out.println("Parsing error!"); + System.err.println("Parsing error!"); } } } From 3fad32df2dc2f27a48d1d9c481ce0f7158f95f1e Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 22:49:41 +0100 Subject: [PATCH 101/135] Add scanner expectation for SmokeClass.java --- src/test/resources/SmokeClass.expected | 174 +++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 src/test/resources/SmokeClass.expected diff --git a/src/test/resources/SmokeClass.expected b/src/test/resources/SmokeClass.expected new file mode 100644 index 00000000..f4ce1344 --- /dev/null +++ b/src/test/resources/SmokeClass.expected @@ -0,0 +1,174 @@ +SmokeClass.java(2,1): KEYWORD(class) +SmokeClass.java(2,7): IDENT(SmokeClass) +SmokeClass.java(2,18): L_BRACE +SmokeClass.java(5,5): KEYWORD(public) +SmokeClass.java(5,12): KEYWORD(void) +SmokeClass.java(5,17): IDENT(main) +SmokeClass.java(5,21): L_PAREN +SmokeClass.java(5,22): R_PAREN +SmokeClass.java(5,24): L_BRACE +SmokeClass.java(6,9): KEYWORD(int) +SmokeClass.java(6,13): IDENT(a) +SmokeClass.java(6,14): SEMICOLON +SmokeClass.java(7,9): KEYWORD(int) +SmokeClass.java(7,13): IDENT(b) +SmokeClass.java(7,14): SEMICOLON +SmokeClass.java(8,9): KEYWORD(int) +SmokeClass.java(8,13): IDENT(c) +SmokeClass.java(8,14): SEMICOLON +SmokeClass.java(9,9): KEYWORD(int) +SmokeClass.java(9,13): IDENT(d) +SmokeClass.java(9,14): SEMICOLON +SmokeClass.java(10,9): KEYWORD(int) +SmokeClass.java(10,13): IDENT(e) +SmokeClass.java(10,14): SEMICOLON +SmokeClass.java(11,9): KEYWORD(int) +SmokeClass.java(11,13): IDENT(result) +SmokeClass.java(11,19): SEMICOLON +SmokeClass.java(12,9): IDENT(a) +SmokeClass.java(12,11): ASSIGN +SmokeClass.java(12,13): NUMBER +SmokeClass.java(12,14): SEMICOLON +SmokeClass.java(13,9): IDENT(b) +SmokeClass.java(13,11): ASSIGN +SmokeClass.java(13,13): NUMBER +SmokeClass.java(13,14): SEMICOLON +SmokeClass.java(14,9): IDENT(c) +SmokeClass.java(14,11): ASSIGN +SmokeClass.java(14,13): NUMBER +SmokeClass.java(14,14): SEMICOLON +SmokeClass.java(15,9): IDENT(d) +SmokeClass.java(15,11): ASSIGN +SmokeClass.java(15,13): NUMBER +SmokeClass.java(15,14): SEMICOLON +SmokeClass.java(16,9): IDENT(e) +SmokeClass.java(16,11): ASSIGN +SmokeClass.java(16,13): NUMBER +SmokeClass.java(16,14): SEMICOLON +SmokeClass.java(17,9): IDENT(result) +SmokeClass.java(17,16): ASSIGN +SmokeClass.java(17,18): IDENT(a) +SmokeClass.java(17,19): PLUS +SmokeClass.java(17,20): L_PAREN +SmokeClass.java(17,21): IDENT(b) +SmokeClass.java(17,22): MINUS +SmokeClass.java(17,23): IDENT(c) +SmokeClass.java(17,24): R_PAREN +SmokeClass.java(17,25): TIMES +SmokeClass.java(17,26): IDENT(d) +SmokeClass.java(17,27): DIVIDE +SmokeClass.java(17,28): IDENT(e) +SmokeClass.java(17,29): SEMICOLON +SmokeClass.java(18,9): KEYWORD(if) +SmokeClass.java(18,12): L_PAREN +SmokeClass.java(18,13): IDENT(a) +SmokeClass.java(18,16): LOWER_EQUAL +SmokeClass.java(18,18): IDENT(b) +SmokeClass.java(18,19): R_PAREN +SmokeClass.java(18,21): L_BRACE +SmokeClass.java(18,22): IDENT(println) +SmokeClass.java(18,29): L_PAREN +SmokeClass.java(18,30): IDENT(a) +SmokeClass.java(18,31): R_PAREN +SmokeClass.java(18,32): SEMICOLON +SmokeClass.java(18,33): R_BRACE +SmokeClass.java(18,35): KEYWORD(else) +SmokeClass.java(18,40): L_BRACE +SmokeClass.java(18,41): IDENT(println) +SmokeClass.java(18,48): L_PAREN +SmokeClass.java(18,49): IDENT(b) +SmokeClass.java(18,50): R_PAREN +SmokeClass.java(18,51): SEMICOLON +SmokeClass.java(18,52): R_BRACE +SmokeClass.java(19,9): KEYWORD(if) +SmokeClass.java(19,12): L_PAREN +SmokeClass.java(19,13): IDENT(a) +SmokeClass.java(19,15): LOWER +SmokeClass.java(19,18): IDENT(b) +SmokeClass.java(19,19): R_PAREN +SmokeClass.java(19,21): L_BRACE +SmokeClass.java(19,22): IDENT(println) +SmokeClass.java(19,29): L_PAREN +SmokeClass.java(19,30): IDENT(a) +SmokeClass.java(19,31): R_PAREN +SmokeClass.java(19,32): SEMICOLON +SmokeClass.java(19,33): R_BRACE +SmokeClass.java(19,35): KEYWORD(else) +SmokeClass.java(19,40): L_BRACE +SmokeClass.java(19,41): IDENT(println) +SmokeClass.java(19,48): L_PAREN +SmokeClass.java(19,49): IDENT(b) +SmokeClass.java(19,50): R_PAREN +SmokeClass.java(19,51): SEMICOLON +SmokeClass.java(19,52): R_BRACE +SmokeClass.java(20,9): KEYWORD(if) +SmokeClass.java(20,12): L_PAREN +SmokeClass.java(20,13): IDENT(a) +SmokeClass.java(20,16): EQUAL +SmokeClass.java(20,18): IDENT(b) +SmokeClass.java(20,19): R_PAREN +SmokeClass.java(20,21): L_BRACE +SmokeClass.java(20,22): IDENT(println) +SmokeClass.java(20,29): L_PAREN +SmokeClass.java(20,30): IDENT(a) +SmokeClass.java(20,31): R_PAREN +SmokeClass.java(20,32): SEMICOLON +SmokeClass.java(20,33): R_BRACE +SmokeClass.java(20,35): KEYWORD(else) +SmokeClass.java(20,40): L_BRACE +SmokeClass.java(20,41): IDENT(println) +SmokeClass.java(20,48): L_PAREN +SmokeClass.java(20,49): IDENT(b) +SmokeClass.java(20,50): R_PAREN +SmokeClass.java(20,51): SEMICOLON +SmokeClass.java(20,52): R_BRACE +SmokeClass.java(21,9): KEYWORD(if) +SmokeClass.java(21,12): L_PAREN +SmokeClass.java(21,13): IDENT(a) +SmokeClass.java(21,16): GREATER +SmokeClass.java(21,18): IDENT(b) +SmokeClass.java(21,19): R_PAREN +SmokeClass.java(21,21): L_BRACE +SmokeClass.java(21,22): IDENT(println) +SmokeClass.java(21,29): L_PAREN +SmokeClass.java(21,30): IDENT(a) +SmokeClass.java(21,31): R_PAREN +SmokeClass.java(21,32): SEMICOLON +SmokeClass.java(21,33): R_BRACE +SmokeClass.java(21,35): KEYWORD(else) +SmokeClass.java(21,40): L_BRACE +SmokeClass.java(21,41): IDENT(println) +SmokeClass.java(21,48): L_PAREN +SmokeClass.java(21,49): IDENT(b) +SmokeClass.java(21,50): R_PAREN +SmokeClass.java(21,51): SEMICOLON +SmokeClass.java(21,52): R_BRACE +SmokeClass.java(22,9): KEYWORD(if) +SmokeClass.java(22,12): L_PAREN +SmokeClass.java(22,13): IDENT(a) +SmokeClass.java(22,16): GREATER_EQUAL +SmokeClass.java(22,18): IDENT(b) +SmokeClass.java(22,19): R_PAREN +SmokeClass.java(22,21): L_BRACE +SmokeClass.java(22,22): IDENT(println) +SmokeClass.java(22,29): L_PAREN +SmokeClass.java(22,30): IDENT(a) +SmokeClass.java(22,31): R_PAREN +SmokeClass.java(22,32): SEMICOLON +SmokeClass.java(22,33): R_BRACE +SmokeClass.java(22,35): KEYWORD(else) +SmokeClass.java(22,40): L_BRACE +SmokeClass.java(22,41): IDENT(println) +SmokeClass.java(22,48): L_PAREN +SmokeClass.java(22,49): IDENT(b) +SmokeClass.java(22,50): R_PAREN +SmokeClass.java(22,51): SEMICOLON +SmokeClass.java(22,52): R_BRACE +SmokeClass.java(23,9): IDENT(println) +SmokeClass.java(23,16): L_PAREN +SmokeClass.java(23,17): IDENT(result) +SmokeClass.java(23,23): R_PAREN +SmokeClass.java(23,24): SEMICOLON +SmokeClass.java(24,5): R_BRACE +SmokeClass.java(25,1): R_BRACE +SmokeClass.java(26,0): EOF From dd67bbe5b7a9c4901896d9a8caebfebd61f198d8 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 22:51:12 +0100 Subject: [PATCH 102/135] Test scanning SmokeClass.java --- src/test/java/com/merkrafter/MerkompilerRunTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/merkrafter/MerkompilerRunTest.java b/src/test/java/com/merkrafter/MerkompilerRunTest.java index dfd9b4e6..4d09f39e 100644 --- a/src/test/java/com/merkrafter/MerkompilerRunTest.java +++ b/src/test/java/com/merkrafter/MerkompilerRunTest.java @@ -49,7 +49,7 @@ class MerkompilerRunTest { * @throws IOException if there is a read/write error in one of the files */ @ParameterizedTest - @ValueSource(strings = "EmptyClass") + @ValueSource(strings = {"EmptyClass", "SmokeClass"}) void scanWithOutputCreatesFile(final String baseFileName) throws ArgumentParserException, IOException { // java source file to read @@ -83,7 +83,7 @@ void scanWithOutputCreatesFile(final String baseFileName) * @throws IOException if there is a read/write error in one of the files */ @ParameterizedTest - @ValueSource(strings = "EmptyClass") + @ValueSource(strings = {"EmptyClass", "SmokeClass"}) void scanWithoutOutput(final String baseFileName) throws ArgumentParserException, IOException { final PrintStream originalOut = System.out; try { // will reset System.out in case of errors From cc156cce2d213f46ce40f7793bac81be386f2ba1 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 22:54:11 +0100 Subject: [PATCH 103/135] Decorrelate ConfigTest cases #20 --- src/test/java/com/merkrafter/ConfigTest.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/test/java/com/merkrafter/ConfigTest.java b/src/test/java/com/merkrafter/ConfigTest.java index 42a31bfd..2fdd9208 100644 --- a/src/test/java/com/merkrafter/ConfigTest.java +++ b/src/test/java/com/merkrafter/ConfigTest.java @@ -32,11 +32,9 @@ void parseInputFileWithVerbosityShortFirst() throws ArgumentParserException { final Config actualConfig = Config.fromArgs(args); final String expectedInputFilename = "Test.java"; - final String expectedOutputFilename = null; final boolean expectedVerbosity = true; assertEquals(expectedInputFilename, actualConfig.getInputFile()); - assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); assertEquals(expectedVerbosity, actualConfig.isVerbose()); } @@ -46,11 +44,9 @@ void parseInputFileWithVerbosityShortAfter() throws ArgumentParserException { final Config actualConfig = Config.fromArgs(args); final String expectedInputFilename = "Test.java"; - final String expectedOutputFilename = null; final boolean expectedVerbosity = true; assertEquals(expectedInputFilename, actualConfig.getInputFile()); - assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); assertEquals(expectedVerbosity, actualConfig.isVerbose()); } @@ -60,11 +56,9 @@ void parseInputFileWithVerbosityLongFirst() throws ArgumentParserException { final Config actualConfig = Config.fromArgs(args); final String expectedInputFilename = "Test.java"; - final String expectedOutputFilename = null; final boolean expectedVerbosity = true; assertEquals(expectedInputFilename, actualConfig.getInputFile()); - assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); assertEquals(expectedVerbosity, actualConfig.isVerbose()); } @@ -74,11 +68,9 @@ void parseInputFileWithVerbosityLongAfter() throws ArgumentParserException { final Config actualConfig = Config.fromArgs(args); final String expectedInputFilename = "Test.java"; - final String expectedOutputFilename = null; final boolean expectedVerbosity = true; assertEquals(expectedInputFilename, actualConfig.getInputFile()); - assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); assertEquals(expectedVerbosity, actualConfig.isVerbose()); } @@ -89,11 +81,9 @@ void parseInputFileAndShortOutputFileFirst() throws ArgumentParserException { final String expectedInputFilename = "Test.java"; final String expectedOutputFilename = "OtherTest.class"; - final boolean expectedVerbosity = false; assertEquals(expectedInputFilename, actualConfig.getInputFile()); assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); - assertEquals(expectedVerbosity, actualConfig.isVerbose()); } @Test @@ -103,11 +93,9 @@ void parseInputFileAndShortOutputFileAfter() throws ArgumentParserException { final String expectedInputFilename = "Test.java"; final String expectedOutputFilename = "OtherTest.class"; - final boolean expectedVerbosity = false; assertEquals(expectedInputFilename, actualConfig.getInputFile()); assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); - assertEquals(expectedVerbosity, actualConfig.isVerbose()); } @Test @@ -117,11 +105,9 @@ void parseInputFileAndLongOutputFileFirst() throws ArgumentParserException { final String expectedInputFilename = "Test.java"; final String expectedOutputFilename = "OtherTest.class"; - final boolean expectedVerbosity = false; assertEquals(expectedInputFilename, actualConfig.getInputFile()); assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); - assertEquals(expectedVerbosity, actualConfig.isVerbose()); } @Test @@ -131,11 +117,9 @@ void parseInputFileAndLongOutputFileAfter() throws ArgumentParserException { final String expectedInputFilename = "Test.java"; final String expectedOutputFilename = "OtherTest.class"; - final boolean expectedVerbosity = false; assertEquals(expectedInputFilename, actualConfig.getInputFile()); assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); - assertEquals(expectedVerbosity, actualConfig.isVerbose()); } @Test From 6b2c3ee0bda140bef227c734ebfe63012ce82854 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 23:04:19 +0100 Subject: [PATCH 104/135] Merge ConfigTest cases using ParameterizedTests --- src/test/java/com/merkrafter/ConfigTest.java | 93 ++++---------------- 1 file changed, 15 insertions(+), 78 deletions(-) diff --git a/src/test/java/com/merkrafter/ConfigTest.java b/src/test/java/com/merkrafter/ConfigTest.java index 2fdd9208..dad35afa 100644 --- a/src/test/java/com/merkrafter/ConfigTest.java +++ b/src/test/java/com/merkrafter/ConfigTest.java @@ -26,45 +26,12 @@ void parseOnlyInputFile() throws ArgumentParserException { assertEquals(expectedVerbosity, actualConfig.isVerbose()); } - @Test - void parseInputFileWithVerbosityShortFirst() throws ArgumentParserException { - final String[] args = fromString("-v Test.java"); - final Config actualConfig = Config.fromArgs(args); - - final String expectedInputFilename = "Test.java"; - final boolean expectedVerbosity = true; - - assertEquals(expectedInputFilename, actualConfig.getInputFile()); - assertEquals(expectedVerbosity, actualConfig.isVerbose()); - } - - @Test - void parseInputFileWithVerbosityShortAfter() throws ArgumentParserException { - final String[] args = fromString("Test.java -v"); - final Config actualConfig = Config.fromArgs(args); - - final String expectedInputFilename = "Test.java"; - final boolean expectedVerbosity = true; - - assertEquals(expectedInputFilename, actualConfig.getInputFile()); - assertEquals(expectedVerbosity, actualConfig.isVerbose()); - } - - @Test - void parseInputFileWithVerbosityLongFirst() throws ArgumentParserException { - final String[] args = fromString("--verbose Test.java"); - final Config actualConfig = Config.fromArgs(args); - - final String expectedInputFilename = "Test.java"; - final boolean expectedVerbosity = true; - - assertEquals(expectedInputFilename, actualConfig.getInputFile()); - assertEquals(expectedVerbosity, actualConfig.isVerbose()); - } - - @Test - void parseInputFileWithVerbosityLongAfter() throws ArgumentParserException { - final String[] args = fromString("Test.java --verbose"); + @ParameterizedTest + // {short, long} x {before input file, after input file} + @ValueSource(strings = { + "-v Test.java", "--verbose Test.java", "Test.java -v", "Test.java --verbose"}) + void parseInputFileWithVerbosity(final String string) throws ArgumentParserException { + final String[] args = fromString(string); final Config actualConfig = Config.fromArgs(args); final String expectedInputFilename = "Test.java"; @@ -74,45 +41,15 @@ void parseInputFileWithVerbosityLongAfter() throws ArgumentParserException { assertEquals(expectedVerbosity, actualConfig.isVerbose()); } - @Test - void parseInputFileAndShortOutputFileFirst() throws ArgumentParserException { - final String[] args = fromString("-o OtherTest.class Test.java"); - final Config actualConfig = Config.fromArgs(args); - - final String expectedInputFilename = "Test.java"; - final String expectedOutputFilename = "OtherTest.class"; - - assertEquals(expectedInputFilename, actualConfig.getInputFile()); - assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); - } - - @Test - void parseInputFileAndShortOutputFileAfter() throws ArgumentParserException { - final String[] args = fromString("Test.java -o OtherTest.class"); - final Config actualConfig = Config.fromArgs(args); - - final String expectedInputFilename = "Test.java"; - final String expectedOutputFilename = "OtherTest.class"; - - assertEquals(expectedInputFilename, actualConfig.getInputFile()); - assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); - } - - @Test - void parseInputFileAndLongOutputFileFirst() throws ArgumentParserException { - final String[] args = fromString("--output OtherTest.class Test.java"); - final Config actualConfig = Config.fromArgs(args); - - final String expectedInputFilename = "Test.java"; - final String expectedOutputFilename = "OtherTest.class"; - - assertEquals(expectedInputFilename, actualConfig.getInputFile()); - assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); - } - - @Test - void parseInputFileAndLongOutputFileAfter() throws ArgumentParserException { - final String[] args = fromString("Test.java --output OtherTest.class"); + @ParameterizedTest + // {short, long} x {before input file, after input file} + @ValueSource(strings = { + "-o OtherTest.class Test.java", + "--output OtherTest.class Test.java", + "Test.java -o OtherTest.class", + "Test.java --output OtherTest.class"}) + void parseInputFileAndOutputFile(final String string) throws ArgumentParserException { + final String[] args = fromString(string); final Config actualConfig = Config.fromArgs(args); final String expectedInputFilename = "Test.java"; From bef8e256dbf0d390a51168a46833fa49589ecdb9 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 23:05:13 +0100 Subject: [PATCH 105/135] Add ConfigTest case for verbosity and output file --- src/test/java/com/merkrafter/ConfigTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/merkrafter/ConfigTest.java b/src/test/java/com/merkrafter/ConfigTest.java index dad35afa..c2bf9b65 100644 --- a/src/test/java/com/merkrafter/ConfigTest.java +++ b/src/test/java/com/merkrafter/ConfigTest.java @@ -44,6 +44,8 @@ void parseInputFileWithVerbosity(final String string) throws ArgumentParserExcep @ParameterizedTest // {short, long} x {before input file, after input file} @ValueSource(strings = { + "-o=OtherTest.class Test.java", + "--output=OtherTest.class Test.java", "-o OtherTest.class Test.java", "--output OtherTest.class Test.java", "Test.java -o OtherTest.class", @@ -60,8 +62,8 @@ void parseInputFileAndOutputFile(final String string) throws ArgumentParserExcep } @Test - void parseInputFileAndOutputFileAndVerbosity() throws ArgumentParserException { - final String[] args = fromString("--verbose Test.java --output OtherTest.class"); + void parseInputFileAndOutputFileWithVerbosity() throws ArgumentParserException { + final String[] args = fromString("-v -o OtherTest.class Test.java"); final Config actualConfig = Config.fromArgs(args); final String expectedInputFilename = "Test.java"; From acd9007bcd4853711a64af45d5023f08c9bd2c5c Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 23:18:55 +0100 Subject: [PATCH 106/135] Add comments to ConfigTest cases --- src/test/java/com/merkrafter/ConfigTest.java | 49 ++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/test/java/com/merkrafter/ConfigTest.java b/src/test/java/com/merkrafter/ConfigTest.java index c2bf9b65..4806d08f 100644 --- a/src/test/java/com/merkrafter/ConfigTest.java +++ b/src/test/java/com/merkrafter/ConfigTest.java @@ -10,8 +10,19 @@ import static com.merkrafter.config.Config.fromString; import static org.junit.jupiter.api.Assertions.*; +/** + * The test cases of this class verify that the conversion from command line arguments to program + * configuration object works correctly. Therefore, the Config::fromArgs static method is tested + * intensely. + */ class ConfigTest { + /** + * The fromArgs method should be able to extract the input file name correctly. + * It should not set the verbosity nor the output file name. + * + * @throws ArgumentParserException if the arguments can not be parsed; should not happen + */ @Test void parseOnlyInputFile() throws ArgumentParserException { final String[] args = fromString("Test.java"); @@ -26,6 +37,13 @@ void parseOnlyInputFile() throws ArgumentParserException { assertEquals(expectedVerbosity, actualConfig.isVerbose()); } + /** + * The fromArgs method should be able to detect the verbosity flag being set, independent of + * whether the long or short argument was used or whether it was specified before or after + * the input file. + * + * @throws ArgumentParserException if the arguments can not be parsed; should not happen + */ @ParameterizedTest // {short, long} x {before input file, after input file} @ValueSource(strings = { @@ -41,6 +59,13 @@ void parseInputFileWithVerbosity(final String string) throws ArgumentParserExcep assertEquals(expectedVerbosity, actualConfig.isVerbose()); } + /** + * The fromArgs method should be able to detect the output file being specified, independent of + * whether the long or short argument was used, it was assigned using = or not or whether + * it was specified before or after the input file. + * + * @throws ArgumentParserException if the arguments can not be parsed; should not happen + */ @ParameterizedTest // {short, long} x {before input file, after input file} @ValueSource(strings = { @@ -61,6 +86,12 @@ void parseInputFileAndOutputFile(final String string) throws ArgumentParserExcep assertEquals(expectedOutputFilename, actualConfig.getOutputFile()); } + /** + * The fromArgs method should be able to extract the output file and the verbosity flag if both + * are given at the same time. + * + * @throws ArgumentParserException if the arguments can not be parsed; should not happen + */ @Test void parseInputFileAndOutputFileWithVerbosity() throws ArgumentParserException { final String[] args = fromString("-v -o OtherTest.class Test.java"); @@ -75,6 +106,12 @@ void parseInputFileAndOutputFileWithVerbosity() throws ArgumentParserException { assertEquals(expectedVerbosity, actualConfig.isVerbose()); } + /** + * The fromArgs method should be able to extract the compiler stage "scanning" while ignoring + * the case. + * + * @throws ArgumentParserException if the arguments can not be parsed; should not happen + */ @ParameterizedTest @ValueSource(strings = {"scanning", "SCANNING", "sCanning", "sCaNnInG"}) void skipAfterScanning(final String spelling) throws ArgumentParserException { @@ -86,6 +123,12 @@ void skipAfterScanning(final String spelling) throws ArgumentParserException { assertEquals(expectedStage, actualConfig.getStage()); } + /** + * The fromArgs method should be able to extract the compiler stage "parsing" while ignoring + * the case. + * + * @throws ArgumentParserException if the arguments can not be parsed; should not happen + */ @ParameterizedTest @ValueSource(strings = {"parsing", "PARSING", "pArsing", "pArSinG"}) void skipAfterParsing(final String spelling) throws ArgumentParserException { @@ -97,6 +140,12 @@ void skipAfterParsing(final String spelling) throws ArgumentParserException { assertEquals(expectedStage, actualConfig.getStage()); } + /** + * The fromArgs method should set the latest compiler stage correctly when it is not specified + * explicitly. + * + * @throws ArgumentParserException if the arguments can not be parsed; should not happen + */ @Test void defaultStage() throws ArgumentParserException { final String[] args = fromString("Test.java"); From 6749baf927c320c22f2302aaaaea76044584e186 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 23:19:37 +0100 Subject: [PATCH 107/135] Testing for the latest CompilerStage in ConfigTest instead of explicit stage --- src/test/java/com/merkrafter/ConfigTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/merkrafter/ConfigTest.java b/src/test/java/com/merkrafter/ConfigTest.java index 4806d08f..2af68a65 100644 --- a/src/test/java/com/merkrafter/ConfigTest.java +++ b/src/test/java/com/merkrafter/ConfigTest.java @@ -151,7 +151,7 @@ void defaultStage() throws ArgumentParserException { final String[] args = fromString("Test.java"); final Config actualConfig = Config.fromArgs(args); - final CompilerStage expectedStage = CompilerStage.PARSING; + final CompilerStage expectedStage = CompilerStage.latest(); assertEquals(expectedStage, actualConfig.getStage()); } From 8484caeba4f8f192489fb37efb74e3d06d15bdce Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sat, 30 Nov 2019 23:48:14 +0100 Subject: [PATCH 108/135] Add ParserTestDataProvider class --- .../parsing/ParserTestDataProvider.java | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java diff --git a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java new file mode 100644 index 00000000..d18e577b --- /dev/null +++ b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java @@ -0,0 +1,58 @@ +package com.merkrafter.parsing; + +import com.merkrafter.lexing.*; + +import java.util.stream.Stream; + +/**** + * This class serves as a test data provider for ParserTest. + * + * @version v0.2.0 + * @author merkrafter + ***************************************************************/ + +public class ParserTestDataProvider { + // METHODS + //============================================================== + // public methods + //-------------------------------------------------------------- + /** + * Creates a new Token from a TokenType by setting file name, line and position number to some + * default values in order to make increase the readability of test cases. + * + * @return a basic Token with the given TokenType set + */ + static Token tokenFrom(final TokenType type) { + return new Token(type, null, 1, 1); + } + + /** + * Creates a new Token from a Keyword by setting file name, line and position number to some + * default values in order to make increase the readability of test cases. + * + * @return a KeywordToken with the given Keyword set + */ + static Token tokenFrom(final Keyword keyword) { + return new KeywordToken(keyword, null, 1, 1); + } + + /** + * Creates a new Token from an identifier string by setting file name, line and position number + * to some default values in order to make increase the readability of test cases. + * + * @return an IdentToken with the given identifier set + */ + static Token tokenFrom(final String identifier) { + return new IdentToken(identifier, null, 1, 1); + } + + /** + * Creates a new Token from a number string by setting file name, line and position number to + * some default values in order to make increase the readability of test cases. + * + * @return a NumberToken with the given number set + */ + static Token tokenFrom(final long number) { + return new NumberToken(number, null, 1, 1); + } +} From 4425a9e51986092c38f89403d68b79bd906653ad Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 00:25:35 +0100 Subject: [PATCH 109/135] Implement a nested class that wraps Token lists --- .../parsing/ParserTestDataProvider.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java index d18e577b..d2691475 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java +++ b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java @@ -2,6 +2,8 @@ import com.merkrafter.lexing.*; +import java.util.LinkedList; +import java.util.List; import java.util.stream.Stream; /**** @@ -55,4 +57,43 @@ static Token tokenFrom(final String identifier) { static Token tokenFrom(final long number) { return new NumberToken(number, null, 1, 1); } + + /** + * This class serves as a wrapper around a list of tokens as directly passing around lists/arrays + * of tokens from the provider to the test methods does not work, as they're merged into one big + * stream of Tokens. + */ + static class TokenWrapper { + private List tokenList; + + TokenWrapper() { + tokenList = new LinkedList<>(); + } + + /** + * Adds the given token at the end of the token list and returns this TokenWrapper instance. + * + * @param token a token to append to this token wrapper + * @return itself in order to allow chaining + */ + TokenWrapper add(final Token token) { + tokenList.add(token); + return this; + } + + /** + * @return the stored tokens as an array + */ + Token[] getTokens() { + return tokenList.toArray(new Token[0]); + } + + /** + * @return the string representation of the underlying token list + */ + @Override + public String toString() { + return tokenList.toString(); + } + } } From d6ad36900e38be02d3eea40b159f325da971e34a Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 00:45:17 +0100 Subject: [PATCH 110/135] Define a data provider method for assignments --- .../parsing/ParserTestDataProvider.java | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java index d2691475..7ddc68da 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java +++ b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java @@ -18,6 +18,50 @@ public class ParserTestDataProvider { //============================================================== // public methods //-------------------------------------------------------------- + + /** + * This method generates a stream of TokenWrappers that are valid assignments EXCEPT they're + * lacking the ending semicolon. + * + * @return a stream of TokenWrappers that define the test data + */ + public static Stream assignmentsWithoutSemicolon() { + return Stream.of( + // non-parameterized TokenWrappers + Stream.of( + // direct assignment of a number to an identifier + // actual number value does not matter + new TokenWrapper().add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.ASSIGN)) + .add(tokenFrom(TokenType.NUMBER)), + + // direct assignment of an identifier to an identifier + new TokenWrapper().add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.ASSIGN)) + .add(tokenFrom(TokenType.IDENT))), + // parameterized TokenWrappers + Stream.of(TokenType.PLUS, TokenType.MINUS, TokenType.TIMES, TokenType.DIVIDE) + .map(binOp -> new TokenWrapper().add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.ASSIGN)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(binOp)) + .add(tokenFrom(TokenType.NUMBER)))) + // merge all the above (outer) streams + .flatMap(i -> i); + } + + /** + * This method returns the same TokenWrappers as + * the {@link #assignmentsWithoutSemicolon() assignmentsWithoutSemicolon} method does but with + * semicolons appended. It therefore returns only valid assignments. + * + * @return a stream of TokenWrappers that define the test data + */ + public static Stream assignments() { + return assignmentsWithoutSemicolon().map(tokenWrapper -> tokenWrapper.add(tokenFrom( + TokenType.SEMICOLON))); + } + /** * Creates a new Token from a TokenType by setting file name, line and position number to some * default values in order to make increase the readability of test cases. From 0f6ab642cca2cfda74f731410693a23974d2cc15 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 00:54:02 +0100 Subject: [PATCH 111/135] Merge ParserTest assignment test cases using the data provider --- .../com/merkrafter/parsing/ParserTest.java | 35 ++++++++----------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index 03aa460b..7d60bc1a 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -4,6 +4,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; import java.util.Arrays; @@ -365,35 +366,29 @@ void parseType() { } /** - * The parser should accept an assignment of the result of a binary operation to a variable, - * as "a = a*5;". + * The parser should be able to parse assignments. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#assignments()} */ @ParameterizedTest - @EnumSource(value = TokenType.class, names = {"PLUS", "MINUS", "TIMES", "DIVIDE"}) - void parseAssignmentWithBinOp(final TokenType binOp) { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.ASSIGN, "", 0, 0), - new Token(TokenType.IDENT, "", 0, 0), - new Token(binOp, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#assignments") + void parseAssignment(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseAssignment()); } /** - * The parser should accept a direct assignment of a number to a variable ident, as "a = 5;". + * The parser should be able to detect syntactically wrong assignments. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#assignmentsWithoutSemicolon()} */ - @Test - void parseDirectAssignmentOfNumber() { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.ASSIGN, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); + @ParameterizedTest + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#assignmentsWithoutSemicolon") + void parseFaultyAssignment(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); - assertTrue(parser.parseAssignment()); + assertFalse(parser.parseAssignment()); } /** From 848fbce3b32ece44fce32d9eb4f37d8279154df6 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 01:09:59 +0100 Subject: [PATCH 112/135] Add ParserTest cases for intern procedure calls --- .../com/merkrafter/parsing/ParserTest.java | 12 +++++ .../parsing/ParserTestDataProvider.java | 44 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index 7d60bc1a..e59fb382 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -406,6 +406,18 @@ void parseProcedureCall() { assertTrue(parser.parseProcedureCall()); } + /** + * The parser should be able to parse intern procedure calls. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#internProcedureCalls()} + */ + @ParameterizedTest + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#internProcedureCalls") + void parseInternProcedureCall(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); + final Parser parser = new Parser(scanner); + assertTrue(parser.parseInternProcedureCall()); + } /** * The parser should accept a simple if-else construct, that is an "if" keyword, a comparison diff --git a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java index 7ddc68da..fc6d7750 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java +++ b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java @@ -62,6 +62,50 @@ public static Stream assignments() { TokenType.SEMICOLON))); } + /** + * This method generates a stream of TokenWrappers that are valid intern procedure calls. + * + * @return a stream of TokenWrappers that define the test data + */ + public static Stream internProcedureCalls() { + return Stream.of( + // a call of an intern procedure without arguments + new TokenWrapper().add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom(TokenType.R_PAREN)), + + // a call of an intern procedure with a single identifier as its argument + new TokenWrapper().add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.R_PAREN)), + + // a call of an intern procedure with a single number as its argument + new TokenWrapper().add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom(TokenType.NUMBER)) + .add(tokenFrom(TokenType.R_PAREN)), + + // a call of an intern procedure with two identifiers as arguments + new TokenWrapper().add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.COMMA)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.R_PAREN)), + + // a call of an intern procedure with an expression like a*b/2 as argument + new TokenWrapper().add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.PLUS)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.DIVIDE)) + .add(tokenFrom(TokenType.NUMBER)) + .add(tokenFrom(TokenType.R_PAREN))); + } + + /** * Creates a new Token from a TokenType by setting file name, line and position number to some * default values in order to make increase the readability of test cases. From e6ba24962b753bc098df24c817b2dfd255e412f4 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 01:20:36 +0100 Subject: [PATCH 113/135] Add test data provider for procedure calls --- .../com/merkrafter/parsing/ParserTestDataProvider.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java index fc6d7750..aa61fceb 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java +++ b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java @@ -62,6 +62,15 @@ public static Stream assignments() { TokenType.SEMICOLON))); } + /** + * This method generates a stream of TokenWrappers that are valid procedure calls. + * + * @return a stream of TokenWrappers that define the test data + */ + public static Stream procedureCalls() { + return internProcedureCalls().map(tokenWrapper -> tokenWrapper.add(tokenFrom(TokenType.SEMICOLON))); + } + /** * This method generates a stream of TokenWrappers that are valid intern procedure calls. * From da8545cd174f7db2836c34a5fead4189275dfffc Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 01:23:17 +0100 Subject: [PATCH 114/135] Augment ParserTest cases for procedure calls with data provider --- .../java/com/merkrafter/parsing/ParserTest.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index e59fb382..02f09612 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -392,16 +392,14 @@ void parseFaultyAssignment(final ParserTestDataProvider.TokenWrapper inputTokens } /** - * The parser should accept a simple procedure call, as "parse();" + * The parser should be able to parse procedure calls. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#procedureCalls()} */ - @Test - void parseProcedureCall() { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.L_PAREN, "", 0, 0), - new IdentToken("a", "", 0, 0), - new Token(TokenType.R_PAREN, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); + @ParameterizedTest + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#procedureCalls") + void parseProcedureCall(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseProcedureCall()); } From 81d7eecb51997cc12de42176c914d03ce3f145a0 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 01:41:43 +0100 Subject: [PATCH 115/135] Adapt SmokeClass.expected to NumberTokens --- src/test/resources/SmokeClass.expected | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/resources/SmokeClass.expected b/src/test/resources/SmokeClass.expected index f4ce1344..f3d0a37a 100644 --- a/src/test/resources/SmokeClass.expected +++ b/src/test/resources/SmokeClass.expected @@ -27,23 +27,23 @@ SmokeClass.java(11,13): IDENT(result) SmokeClass.java(11,19): SEMICOLON SmokeClass.java(12,9): IDENT(a) SmokeClass.java(12,11): ASSIGN -SmokeClass.java(12,13): NUMBER +SmokeClass.java(12,13): NUMBER(1) SmokeClass.java(12,14): SEMICOLON SmokeClass.java(13,9): IDENT(b) SmokeClass.java(13,11): ASSIGN -SmokeClass.java(13,13): NUMBER +SmokeClass.java(13,13): NUMBER(2) SmokeClass.java(13,14): SEMICOLON SmokeClass.java(14,9): IDENT(c) SmokeClass.java(14,11): ASSIGN -SmokeClass.java(14,13): NUMBER +SmokeClass.java(14,13): NUMBER(3) SmokeClass.java(14,14): SEMICOLON SmokeClass.java(15,9): IDENT(d) SmokeClass.java(15,11): ASSIGN -SmokeClass.java(15,13): NUMBER +SmokeClass.java(15,13): NUMBER(4) SmokeClass.java(15,14): SEMICOLON SmokeClass.java(16,9): IDENT(e) SmokeClass.java(16,11): ASSIGN -SmokeClass.java(16,13): NUMBER +SmokeClass.java(16,13): NUMBER(5) SmokeClass.java(16,14): SEMICOLON SmokeClass.java(17,9): IDENT(result) SmokeClass.java(17,16): ASSIGN From 528a6d709d380dd3c6c57c92bc784dd661d443ac Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 01:42:30 +0100 Subject: [PATCH 116/135] Fix the bug in Parser::parseActualParameters that accepted empty paren too early #26 --- src/main/java/com/merkrafter/parsing/Parser.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index 759b3fc4..f3b3b6ae 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -384,9 +384,9 @@ boolean parseActualParameters() { return false; } } - } else { - return true; // it is okay if no expression comes here } + // it is okay if no expression comes here + // but it is still necessary to check for the right paren } else { return false; } From 2c1dd1581f07faf77d81d8f1ca5cab47fed64ed7 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 02:04:28 +0100 Subject: [PATCH 117/135] Add test data provider for if constructs --- .../parsing/ParserTestDataProvider.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java index aa61fceb..26008194 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java +++ b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java @@ -114,6 +114,44 @@ public static Stream internProcedureCalls() { .add(tokenFrom(TokenType.R_PAREN))); } + /** + * This method generates a stream of TokenWrappers that are valid if constructs. + * + * @return a stream of TokenWrappers that define the test data + */ + public static Stream ifConstructs() { + return Stream.of( + // if constructs with all comparison operators between an ident and a number + // the if and else bodies are simple assignments + Stream.of(TokenType.LOWER_EQUAL, + TokenType.LOWER, + TokenType.EQUAL, + TokenType.GREATER, + TokenType.GREATER_EQUAL) + .map(cmpOp -> new TokenWrapper().add(tokenFrom(Keyword.IF)) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(cmpOp)) + .add(tokenFrom(TokenType.NUMBER)) + .add(tokenFrom(TokenType.R_PAREN)) + + .add(tokenFrom(TokenType.L_BRACE)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.ASSIGN)) + .add(tokenFrom(TokenType.NUMBER)) + .add(tokenFrom(TokenType.SEMICOLON)) + .add(tokenFrom(TokenType.R_BRACE)) + + .add(tokenFrom(Keyword.ELSE)) + .add(tokenFrom(TokenType.L_BRACE)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.ASSIGN)) + .add(tokenFrom(TokenType.NUMBER)) + .add(tokenFrom(TokenType.SEMICOLON)) + .add(tokenFrom(TokenType.R_BRACE)))) + // merge all the above (outer) streams + .flatMap(i -> i); + } /** * Creates a new Token from a TokenType by setting file name, line and position number to some From a1ceed18f9fa8673f0c1ad35c681f8437443407c Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 02:06:11 +0100 Subject: [PATCH 118/135] Replace ParserTest case for if constructs with the data provider approach --- .../com/merkrafter/parsing/ParserTest.java | 34 ++++--------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index 02f09612..8b48e617 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -418,34 +418,14 @@ void parseInternProcedureCall(final ParserTestDataProvider.TokenWrapper inputTok } /** - * The parser should accept a simple if-else construct, that is an "if" keyword, a comparison - * between an identifier and a number as the condition and blocks with single assignments for - * if and else. + * The parser should be able to parse simple if constructs. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#ifConstructs()} */ - @Test - void parseSimpleIfStatement() { - final Scanner scanner = new TestScanner(new Token[]{ - new KeywordToken(Keyword.IF, null, 1, 1), - new Token(TokenType.L_PAREN, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.EQUAL, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.R_PAREN, null, 1, 1), - - new Token(TokenType.L_BRACE, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.ASSIGN, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1), - new Token(TokenType.R_BRACE, null, 1, 1), - - new KeywordToken(Keyword.ELSE, null, 1, 1), - new Token(TokenType.L_BRACE, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.ASSIGN, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1), - new Token(TokenType.R_BRACE, null, 1, 1),}); + @ParameterizedTest + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#ifConstructs") + void parseIfStatement(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseIfStatement()); } From 208a820c2fb1263fbf396ce958932feeebc70534 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 02:13:13 +0100 Subject: [PATCH 119/135] Add test data provider for while loops --- .../parsing/ParserTestDataProvider.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java index 26008194..35d15e5b 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java +++ b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java @@ -153,6 +153,37 @@ public static Stream ifConstructs() { .flatMap(i -> i); } + /** + * This method generates a stream of TokenWrappers that are valid while loops. + * + * @return a stream of TokenWrappers that define the test data + */ + public static Stream whileLoops() { + return Stream.of( + // while loops with all comparison operators between an ident and a number + // the body is a simple assignment + Stream.of(TokenType.LOWER_EQUAL, + TokenType.LOWER, + TokenType.EQUAL, + TokenType.GREATER, + TokenType.GREATER_EQUAL) + .map(cmpOp -> new TokenWrapper().add(tokenFrom(Keyword.WHILE)) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(cmpOp)) + .add(tokenFrom(TokenType.NUMBER)) + .add(tokenFrom(TokenType.R_PAREN)) + + .add(tokenFrom(TokenType.L_BRACE)) + .add(tokenFrom(TokenType.IDENT)) + .add(tokenFrom(TokenType.ASSIGN)) + .add(tokenFrom(TokenType.NUMBER)) + .add(tokenFrom(TokenType.SEMICOLON)) + .add(tokenFrom(TokenType.R_BRACE)))) + // merge all the above (outer) streams + .flatMap(i -> i); + } + /** * Creates a new Token from a TokenType by setting file name, line and position number to some * default values in order to make increase the readability of test cases. From 241e0911160f225d427246e624d5927f63655457 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 02:14:48 +0100 Subject: [PATCH 120/135] Replace ParserTest case for while loops with the data provider approach --- .../com/merkrafter/parsing/ParserTest.java | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index 8b48e617..5c39c8e2 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -431,25 +431,14 @@ void parseIfStatement(final ParserTestDataProvider.TokenWrapper inputTokens) { } /** - * The parser should accept a simple while loop, that is a "while" keyword, a comparison - * between an identifier and a number as the condition and a block that has only an assignment - * inside it. + * The parser should be able to parse simple while loops. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#whileLoops()} */ - @Test - void parseSimpleWhileStatement() { - final Scanner scanner = new TestScanner(new Token[]{ - new KeywordToken(Keyword.WHILE, null, 1, 1), - new Token(TokenType.L_PAREN, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.EQUAL, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.R_PAREN, null, 1, 1), - new Token(TokenType.L_BRACE, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.ASSIGN, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1), - new Token(TokenType.R_BRACE, null, 1, 1),}); + @ParameterizedTest + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#whileLoops") + void parseWhileStatement(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseWhileStatement()); } From 2975e093a870a1c891c6e7df29874a51d5f42f14 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 17:53:20 +0100 Subject: [PATCH 121/135] Add test data provider for simple expressions --- .../parsing/ParserTestDataProvider.java | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java index 35d15e5b..3c256990 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java +++ b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java @@ -184,6 +184,79 @@ public static Stream whileLoops() { .flatMap(i -> i); } + /** + * This method generates a stream of TokenWrappers that are valid simple expressions. + * + * @return a stream of TokenWrappers that define the test data + */ + public static Stream simpleExpressions() { + return Stream.of( + /* + These TokenWrappers are independent from any operators + */ + Stream.of( + // an identifier with a single letter + new TokenWrapper().add(tokenFrom("a")), + + // an identifier with two letters + new TokenWrapper().add(tokenFrom("ab")), + + // a single number + new TokenWrapper().add(tokenFrom(5)), + + // complex expression with multiplication, addition and subtraction + // " a*a + b*b - c*c + new TokenWrapper().add(tokenFrom("a")) + .add(tokenFrom(TokenType.TIMES)) + .add(tokenFrom("a")) + .add(tokenFrom(TokenType.PLUS)) + .add(tokenFrom("b")) + .add(tokenFrom(TokenType.TIMES)) + .add(tokenFrom("b")) + .add(tokenFrom(TokenType.MINUS)) + .add(tokenFrom("c")) + .add(tokenFrom(TokenType.TIMES)) + .add(tokenFrom("c"))), + /* + These TokenWrappers are multiplied by using the 4 elementary arithmetic operations + */ + Stream.of(TokenType.PLUS, TokenType.MINUS, TokenType.TIMES, TokenType.DIVIDE) + .flatMap(operator -> Stream.of( + // simple offset of an ident and a number against each other + // with the ident being the first argument + new TokenWrapper().add(tokenFrom("a")) + .add(tokenFrom(operator)) + .add(tokenFrom(5)), + + // simple offset of two idents against each other + new TokenWrapper().add(tokenFrom("a")) + .add(tokenFrom(operator)) + .add(tokenFrom("b")), + + // simple offset of a number and an ident against each other + // with the ident being the second argument + new TokenWrapper().add(tokenFrom(3)) + .add(tokenFrom(operator)) + .add(tokenFrom("b")), + + // simple offset of two numbers against each other + new TokenWrapper().add(tokenFrom(3)) + .add(tokenFrom(operator)) + .add(tokenFrom(5)), + + // chain of 4 idents and 3 operators + new TokenWrapper().add(tokenFrom("a")) + .add(tokenFrom(operator)) + .add(tokenFrom("b")) + .add(tokenFrom(operator)) + .add(tokenFrom("c")) + .add(tokenFrom(operator)) + .add(tokenFrom("d"))))) + + // merge all the above (outer) streams + .flatMap(i -> i); + } + /** * Creates a new Token from a TokenType by setting file name, line and position number to some * default values in order to make increase the readability of test cases. From bcf5fa7a48464ec02829e78e8e1a05e2df72089a Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 17:53:50 +0100 Subject: [PATCH 122/135] Replace ParserTest case for simple expressions with the data provider approach --- .../java/com/merkrafter/parsing/ParserTest.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index 5c39c8e2..6933891e 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -493,15 +493,14 @@ void parseSingleComparisonAsExpression(final TokenType comparisonType) { } /** - * The parser should accept a single addition/subtraction as a simple expression. + * The parser should be able to parse simple expressions. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#simpleExpressions()} */ @ParameterizedTest - @EnumSource(value = TokenType.class, names = {"PLUS", "MINUS"}) - void parseSimpleExpression(final TokenType tokenType) { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(tokenType, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0)}); + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#simpleExpressions") + void parseSimpleExpression(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseSimpleExpression()); } From 34cde921c461a72185ea9a504de8422599e43369 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 18:21:31 +0100 Subject: [PATCH 123/135] Add test data provider for expressions --- .../parsing/ParserTestDataProvider.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java index 3c256990..1bbdad74 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java +++ b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java @@ -257,6 +257,37 @@ public static Stream simpleExpressions() { .flatMap(i -> i); } + /** + * This method generates a stream of TokenWrappers that are valid expressions. + * + * @return a stream of TokenWrappers that define the test data + */ + public static Stream expressions() { + return Stream.of( + /* + Every simple expression is an expression as well + */ + simpleExpressions(), + + /* + Comparisons of all simple expressions with an identifier + */ + Stream.of(TokenType.LOWER, + TokenType.LOWER_EQUAL, + TokenType.EQUAL, + TokenType.GREATER_EQUAL, + TokenType.GREATER).flatMap( + + cmpOp -> + // simple expression first, then comparison operator, then ident + simpleExpressions().map( + + tokenWrapper -> tokenWrapper.add(tokenFrom(cmpOp)) + .add(tokenFrom("a")))) + + ).flatMap(i -> i); + } + /** * Creates a new Token from a TokenType by setting file name, line and position number to some * default values in order to make increase the readability of test cases. From ca1bac738467f686c18814b77b6f86ed425fa12f Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 18:22:37 +0100 Subject: [PATCH 124/135] Replace ParserTest case for expressions with the data provider approach --- .../java/com/merkrafter/parsing/ParserTest.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index 6933891e..a3f7b0ca 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -478,16 +478,14 @@ void parseEmptyActualParameters() { } /** - * The parser should accept a single comparison between an ident and a number as an expression. + * The parser should be able to parse expressions. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#expressions()} */ @ParameterizedTest - @EnumSource(value = TokenType.class, names = { - "LOWER", "LOWER_EQUAL", "EQUAL", "GREATER_EQUAL", "GREATER"}) - void parseSingleComparisonAsExpression(final TokenType comparisonType) { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(comparisonType, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0)}); + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#expressions") + void parseExpression(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseExpression()); } From f07fb7aa8214ce9993611b338e7804047f740ceb Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 19:35:19 +0100 Subject: [PATCH 125/135] Add more complex simple_expression that includes procedure calls --- .../parsing/ParserTestDataProvider.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java index 1bbdad74..5e56eb2f 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java +++ b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java @@ -204,6 +204,24 @@ public static Stream simpleExpressions() { // a single number new TokenWrapper().add(tokenFrom(5)), + // complex expression including intern procedure calls + // 2*fib(n-1) + fib(n-2) + new TokenWrapper().add(tokenFrom(2)) + .add(tokenFrom(TokenType.TIMES)) + .add(tokenFrom("fib")) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom("n")) + .add(tokenFrom(TokenType.MINUS)) + .add(tokenFrom(1)) + .add(tokenFrom(TokenType.R_PAREN)) + .add(tokenFrom(TokenType.PLUS)) + .add(tokenFrom("fib")) + .add(tokenFrom(TokenType.L_PAREN)) + .add(tokenFrom("n")) + .add(tokenFrom(TokenType.MINUS)) + .add(tokenFrom(2)) + .add(tokenFrom(TokenType.R_PAREN)), + // complex expression with multiplication, addition and subtraction // " a*a + b*b - c*c new TokenWrapper().add(tokenFrom("a")) From 5c6f1ae145d6ec448d5d3793f2cbc6ebb8eada17 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 19:51:25 +0100 Subject: [PATCH 126/135] Add method to append TokenWrappers to each other --- .../merkrafter/parsing/ParserTestDataProvider.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java index 5e56eb2f..0a989032 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java +++ b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java @@ -369,6 +369,18 @@ TokenWrapper add(final Token token) { return this; } + /** + * Adds all tokens of the given token wrapper at the end of this wrapper's token list and + * returns this TokenWrapper instance. + * + * @param tokenWrapper a TokenWrapper to append at the end of this wrapper + * @return itself in order to allow chaining + */ + TokenWrapper add(final TokenWrapper tokenWrapper) { + tokenList.addAll(tokenWrapper.tokenList); + return this; + } + /** * @return the stored tokens as an array */ From bff6388dc0245ac273e63e2d312ffc8af44bb6d7 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 20:28:36 +0100 Subject: [PATCH 127/135] Simplify assignment test data provider method --- .../parsing/ParserTestDataProvider.java | 27 ++++--------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java index 0a989032..7f0d4232 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java +++ b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java @@ -26,28 +26,11 @@ public class ParserTestDataProvider { * @return a stream of TokenWrappers that define the test data */ public static Stream assignmentsWithoutSemicolon() { - return Stream.of( - // non-parameterized TokenWrappers - Stream.of( - // direct assignment of a number to an identifier - // actual number value does not matter - new TokenWrapper().add(tokenFrom(TokenType.IDENT)) - .add(tokenFrom(TokenType.ASSIGN)) - .add(tokenFrom(TokenType.NUMBER)), - - // direct assignment of an identifier to an identifier - new TokenWrapper().add(tokenFrom(TokenType.IDENT)) - .add(tokenFrom(TokenType.ASSIGN)) - .add(tokenFrom(TokenType.IDENT))), - // parameterized TokenWrappers - Stream.of(TokenType.PLUS, TokenType.MINUS, TokenType.TIMES, TokenType.DIVIDE) - .map(binOp -> new TokenWrapper().add(tokenFrom(TokenType.IDENT)) - .add(tokenFrom(TokenType.ASSIGN)) - .add(tokenFrom(TokenType.IDENT)) - .add(tokenFrom(binOp)) - .add(tokenFrom(TokenType.NUMBER)))) - // merge all the above (outer) streams - .flatMap(i -> i); + return simpleExpressions().map( + // add all simple expressions at the end of "a = " + expression -> new TokenWrapper().add(tokenFrom("a")) + .add(tokenFrom(TokenType.ASSIGN)) + .add(expression)); } /** From ab895f6509012117743a6d1a788b7782d85e95e2 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 20:51:25 +0100 Subject: [PATCH 128/135] Fix a bug that made Parser::parseFactor ignore intern procedure calls #27 --- src/main/java/com/merkrafter/parsing/Parser.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/merkrafter/parsing/Parser.java b/src/main/java/com/merkrafter/parsing/Parser.java index f3b3b6ae..804507b2 100644 --- a/src/main/java/com/merkrafter/parsing/Parser.java +++ b/src/main/java/com/merkrafter/parsing/Parser.java @@ -444,6 +444,10 @@ boolean parseTerm() { boolean parseFactor() { if (parseIdentifier()) { + // check whether this actually is a intern procedure call + if (parseActualParameters()) { + return true; + } return true; } else if (parseNumber()) { return true; @@ -457,7 +461,7 @@ boolean parseFactor() { } return success; // whether the above parseExpression() was successful } - return parseInternProcedureCall(); + return false; } /** From dcd315b66ebea524ec96504f18687d30dc92b406 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 22:55:45 +0100 Subject: [PATCH 129/135] Add test data provider for return statements --- .../parsing/ParserTestDataProvider.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java index 7f0d4232..f3aff063 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java +++ b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java @@ -167,6 +167,32 @@ public static Stream whileLoops() { .flatMap(i -> i); } + /** + * This method generates a stream of TokenWrappers that are valid return statements. + * + * @return a stream of TokenWrappers that define the test data + */ + public static Stream returnStatements() { + return Stream.of( + /* + unparameterized data + */ + Stream.of( + // return keyword without an value to return + new TokenWrapper().add(tokenFrom(Keyword.RETURN)) + .add(tokenFrom(TokenType.SEMICOLON))), + /* + parameterized data + */ + simpleExpressions().map( + // return keyword with simple expressions as return values + tokenWrapper -> new TokenWrapper().add(tokenFrom(Keyword.RETURN)) + .add(tokenWrapper) + .add(tokenFrom(TokenType.SEMICOLON)))) + // merge all the above (outer) streams + .flatMap(i -> i); + } + /** * This method generates a stream of TokenWrappers that are valid simple expressions. * From 38bb4f153ab328219af36c873ab6f9aff5c74826 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 22:56:17 +0100 Subject: [PATCH 130/135] Replace ParserTest case for return statements with the data provider approach --- .../java/com/merkrafter/parsing/ParserTest.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index a3f7b0ca..efb967aa 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -444,13 +444,14 @@ void parseWhileStatement(final ParserTestDataProvider.TokenWrapper inputTokens) } /** - * The parser should accept a single return statement. + * The parser should be able to parse return statements. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#returnStatements()} */ - @Test - void parseStandaloneReturnStatement() { - final Scanner scanner = new TestScanner(new Token[]{ - new KeywordToken(Keyword.RETURN, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1)}); + @ParameterizedTest + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#returnStatements") + void parseReturnStatement(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseReturnStatement()); } From 3be9be8eab0c0deee3f1b295683e8b7a00165af0 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 23:01:54 +0100 Subject: [PATCH 131/135] Add test data provider for statements --- .../merkrafter/parsing/ParserTestDataProvider.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java index f3aff063..69f17c9b 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java +++ b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java @@ -19,6 +19,19 @@ public class ParserTestDataProvider { // public methods //-------------------------------------------------------------- + /** + * This method generates a stream of TokenWrappers that are valid statements. + * + * @return a stream of TokenWrappers that define the test data + */ + public static Stream statements() { + return Stream.of(assignments(), + procedureCalls(), + ifConstructs(), + whileLoops(), + returnStatements()).flatMap(i -> i); + } + /** * This method generates a stream of TokenWrappers that are valid assignments EXCEPT they're * lacking the ending semicolon. From cb3e76f733fdff0e280f9f314d629e2bbb5fe9a4 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 23:16:26 +0100 Subject: [PATCH 132/135] Replace ParserTest cases for statements with the data provider approach --- .../com/merkrafter/parsing/ParserTest.java | 114 +----------------- 1 file changed, 6 insertions(+), 108 deletions(-) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index efb967aa..bf925387 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -239,120 +239,18 @@ void parseStandaloneReturnAsStatementSequence() { } /** - * The parser should accept an assignment of the result of a binary operation to a variable - * as a statement. + * The parser should be able to parse statements. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#statements()} */ @ParameterizedTest - @EnumSource(value = TokenType.class, names = {"PLUS", "MINUS", "TIMES", "DIVIDE"}) - void parseAssignmentWithBinOpAsStatement(final TokenType binOp) { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.ASSIGN, "", 0, 0), - new Token(TokenType.IDENT, "", 0, 0), - new Token(binOp, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatement()); - } - - /** - * The parser should accept a simple assignment of a number as a statement. - */ - @Test - void parseAssignmentAsStatement() { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.ASSIGN, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatement()); - } - - /** - * The parser should accept a simple procedure call as a statement. - */ - @Test - void parseProcedureCallAsStatement() { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.L_PAREN, "", 0, 0), - new IdentToken("a", "", 0, 0), - new Token(TokenType.R_PAREN, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatement()); - } - - /** - * The parser should accept a simple if-else construct as a statement, that is an "if" keyword, - * a comparison between an identifier and a number as the condition and blocks with single - * assignments for if and else. - */ - @Test - void parseSimpleIfAsStatement() { - final Scanner scanner = new TestScanner(new Token[]{ - new KeywordToken(Keyword.IF, null, 1, 1), - new Token(TokenType.L_PAREN, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.EQUAL, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.R_PAREN, null, 1, 1), - - new Token(TokenType.L_BRACE, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.ASSIGN, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1), - new Token(TokenType.R_BRACE, null, 1, 1), - - new KeywordToken(Keyword.ELSE, null, 1, 1), - new Token(TokenType.L_BRACE, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.ASSIGN, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1), - new Token(TokenType.R_BRACE, null, 1, 1),}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatement()); - } - - /** - * The parser should accept a simple while loop as a statement, that is a "while" keyword, a - * comparison between an identifier and a number as the condition and a block that has only an - * assignment inside it. - */ - @Test - void parseSimpleWhileAsStatement() { - final Scanner scanner = new TestScanner(new Token[]{ - new KeywordToken(Keyword.WHILE, null, 1, 1), - new Token(TokenType.L_PAREN, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.EQUAL, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.R_PAREN, null, 1, 1), - new Token(TokenType.L_BRACE, null, 1, 1), - new Token(TokenType.IDENT, null, 1, 1), - new Token(TokenType.ASSIGN, null, 1, 1), - new Token(TokenType.NUMBER, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1), - new Token(TokenType.R_BRACE, null, 1, 1),}); + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#statements") + void parseStatement(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseStatement()); } - /** - * The parser should accept a single return keyword as a statement. - */ - @Test - void parseStandaloneReturnAsStatement() { - final Scanner scanner = new TestScanner(new Token[]{ - new KeywordToken(Keyword.RETURN, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1)}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatement()); - } /** * The parser should accept a single "int" as a type. From f3d1b25c6512ff67b6fb089e9120d0827a0a845c Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 23:21:10 +0100 Subject: [PATCH 133/135] Replace ParserTest cases for statement sequences with the data provider approach --- .../com/merkrafter/parsing/ParserTest.java | 76 ++----------------- 1 file changed, 6 insertions(+), 70 deletions(-) diff --git a/src/test/java/com/merkrafter/parsing/ParserTest.java b/src/test/java/com/merkrafter/parsing/ParserTest.java index bf925387..1a742455 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTest.java +++ b/src/test/java/com/merkrafter/parsing/ParserTest.java @@ -163,77 +163,14 @@ void parseLocalDeclaration() { } /** - * The parser should accept an assignment and a return statement as a statement sequence. - */ - @Test - void parseStatementSequence() { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.ASSIGN, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0), - new KeywordToken(Keyword.RETURN, "", 0, 0), - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatementSequence()); - } - - /** - * The parser should accept an assignment of the result of a binary operation to a variable - * as a statement sequence. + * The parser should be able to parse single statements as statement sequences. + * + * @param inputTokens token lists provided by {@link ParserTestDataProvider#statements()} */ @ParameterizedTest - @EnumSource(value = TokenType.class, names = {"PLUS", "MINUS", "TIMES", "DIVIDE"}) - void parseAssignmentWithBinOpAsStatementSequence(final TokenType binOp) { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.ASSIGN, "", 0, 0), - new Token(TokenType.IDENT, "", 0, 0), - new Token(binOp, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatementSequence()); - } - - /** - * The parser should accept a simple assignment of a number as a statement sequence. - */ - @Test - void parseAssignmentAsStatementSequence() { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.ASSIGN, "", 0, 0), - new Token(TokenType.NUMBER, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatementSequence()); - } - - /** - * The parser should accept a simple procedure call as a statement sequence. - */ - @Test - void parseProcedureCallAsStatementSequence() { - final Scanner scanner = new TestScanner(new Token[]{ - new Token(TokenType.IDENT, "", 0, 0), - new Token(TokenType.L_PAREN, "", 0, 0), - new IdentToken("a", "", 0, 0), - new Token(TokenType.R_PAREN, "", 0, 0), - new Token(TokenType.SEMICOLON, "", 0, 0)}); - final Parser parser = new Parser(scanner); - assertTrue(parser.parseStatementSequence()); - } - - /** - * The parser should accept a single return keyword as a statement sequence. - */ - @Test - void parseStandaloneReturnAsStatementSequence() { - final Scanner scanner = new TestScanner(new Token[]{ - new KeywordToken(Keyword.RETURN, null, 1, 1), - new Token(TokenType.SEMICOLON, null, 1, 1)}); + @MethodSource("com.merkrafter.parsing.ParserTestDataProvider#statements") + void parseStatementSequence(final ParserTestDataProvider.TokenWrapper inputTokens) { + final Scanner scanner = new TestScanner(inputTokens.getTokens()); final Parser parser = new Parser(scanner); assertTrue(parser.parseStatementSequence()); } @@ -251,7 +188,6 @@ void parseStatement(final ParserTestDataProvider.TokenWrapper inputTokens) { assertTrue(parser.parseStatement()); } - /** * The parser should accept a single "int" as a type. */ From ca67ea18b5c34f9b66061a6e0d62a339d1099cb3 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Sun, 1 Dec 2019 23:58:32 +0100 Subject: [PATCH 134/135] Improve documentation on ParserTestDataProvider --- .../parsing/ParserTestDataProvider.java | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java index 69f17c9b..31e84af2 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java +++ b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java @@ -8,6 +8,15 @@ /**** * This class serves as a test data provider for ParserTest. + * All non-private methods of this class are static and return a stream + * of TokenWrappers (essentially lists of Tokens) that satisfy certain syntax criteria. + *

+ * The methods are organized hierarchically which means that {@link #statements()} joins the tokens + * from {@link #assignments()}, {@link #returnStatements()} and some other, for instance. + *

+ * This file also defines some static methods that allow the fast creation of + * tokens without the need to specify the filename, line and position numbers + * as they are not relevant for the syntax analysis tests. * * @version v0.2.0 * @author merkrafter @@ -21,6 +30,7 @@ public class ParserTestDataProvider { /** * This method generates a stream of TokenWrappers that are valid statements. + * They are a union of assignments, procedure calls, if, while and return statements. * * @return a stream of TokenWrappers that define the test data */ @@ -47,9 +57,10 @@ public static Stream assignmentsWithoutSemicolon() { } /** + * This method generates a stream of TokenWrappers that are valid assignments. * This method returns the same TokenWrappers as - * the {@link #assignmentsWithoutSemicolon() assignmentsWithoutSemicolon} method does but with - * semicolons appended. It therefore returns only valid assignments. + * the {@link #assignmentsWithoutSemicolon() assignmentsWithoutSemicolon} method does, but with + * semicolons appended. * * @return a stream of TokenWrappers that define the test data */ @@ -60,6 +71,8 @@ public static Stream assignments() { /** * This method generates a stream of TokenWrappers that are valid procedure calls. + * This method returns the same TokenWrappers as the {@link #internProcedureCalls()}} method + * does, but with semicolons appended. * * @return a stream of TokenWrappers that define the test data */ @@ -69,6 +82,8 @@ public static Stream procedureCalls() { /** * This method generates a stream of TokenWrappers that are valid intern procedure calls. + * These include calls with empty argument lists, one and two element arguments lists, and + * expressions as arguments. * * @return a stream of TokenWrappers that define the test data */ @@ -112,6 +127,8 @@ public static Stream internProcedureCalls() { /** * This method generates a stream of TokenWrappers that are valid if constructs. + * In particular, the if and else bodies are single assignments, while the comparison is being + * made between an identifier and a number. * * @return a stream of TokenWrappers that define the test data */ @@ -151,6 +168,8 @@ public static Stream ifConstructs() { /** * This method generates a stream of TokenWrappers that are valid while loops. + * In particular, the body is a single assignment, while the comparison is being + * made between an identifier and a number. * * @return a stream of TokenWrappers that define the test data */ @@ -182,6 +201,8 @@ public static Stream whileLoops() { /** * This method generates a stream of TokenWrappers that are valid return statements. + * These include a single return statement without return value as well as returning + * {@link #simpleExpressions()}. * * @return a stream of TokenWrappers that define the test data */ @@ -208,6 +229,8 @@ public static Stream returnStatements() { /** * This method generates a stream of TokenWrappers that are valid simple expressions. + * These include pretty basic expressions as single identifiers and numbers, as well as more + * complex expressions that include multiple operators and procedure calls. * * @return a stream of TokenWrappers that define the test data */ @@ -299,6 +322,7 @@ public static Stream simpleExpressions() { /** * This method generates a stream of TokenWrappers that are valid expressions. + * These include {@link #simpleExpressions()} as well as comparisons between those and idents. * * @return a stream of TokenWrappers that define the test data */ @@ -328,6 +352,10 @@ public static Stream expressions() { ).flatMap(i -> i); } + + // support methods + //-------------------------------------------------------------- + /** * Creates a new Token from a TokenType by setting file name, line and position number to some * default values in order to make increase the readability of test cases. @@ -368,6 +396,10 @@ static Token tokenFrom(final long number) { return new NumberToken(number, null, 1, 1); } + + // inner classes + //-------------------------------------------------------------- + /** * This class serves as a wrapper around a list of tokens as directly passing around lists/arrays * of tokens from the provider to the test methods does not work, as they're merged into one big From 384f774b969d74ae244de23b4c18b616daca0063 Mon Sep 17 00:00:00 2001 From: Mark Umnus Date: Mon, 2 Dec 2019 00:01:38 +0100 Subject: [PATCH 135/135] Restrict access modifiers to members of ParserTestDataProvider --- .../parsing/ParserTestDataProvider.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java index 31e84af2..fa37df36 100644 --- a/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java +++ b/src/test/java/com/merkrafter/parsing/ParserTestDataProvider.java @@ -22,7 +22,7 @@ * @author merkrafter ***************************************************************/ -public class ParserTestDataProvider { +class ParserTestDataProvider { // METHODS //============================================================== // public methods @@ -34,7 +34,7 @@ public class ParserTestDataProvider { * * @return a stream of TokenWrappers that define the test data */ - public static Stream statements() { + static Stream statements() { return Stream.of(assignments(), procedureCalls(), ifConstructs(), @@ -48,7 +48,7 @@ public static Stream statements() { * * @return a stream of TokenWrappers that define the test data */ - public static Stream assignmentsWithoutSemicolon() { + static Stream assignmentsWithoutSemicolon() { return simpleExpressions().map( // add all simple expressions at the end of "a = " expression -> new TokenWrapper().add(tokenFrom("a")) @@ -64,7 +64,7 @@ public static Stream assignmentsWithoutSemicolon() { * * @return a stream of TokenWrappers that define the test data */ - public static Stream assignments() { + static Stream assignments() { return assignmentsWithoutSemicolon().map(tokenWrapper -> tokenWrapper.add(tokenFrom( TokenType.SEMICOLON))); } @@ -76,7 +76,7 @@ public static Stream assignments() { * * @return a stream of TokenWrappers that define the test data */ - public static Stream procedureCalls() { + static Stream procedureCalls() { return internProcedureCalls().map(tokenWrapper -> tokenWrapper.add(tokenFrom(TokenType.SEMICOLON))); } @@ -87,7 +87,7 @@ public static Stream procedureCalls() { * * @return a stream of TokenWrappers that define the test data */ - public static Stream internProcedureCalls() { + static Stream internProcedureCalls() { return Stream.of( // a call of an intern procedure without arguments new TokenWrapper().add(tokenFrom(TokenType.IDENT)) @@ -132,7 +132,7 @@ public static Stream internProcedureCalls() { * * @return a stream of TokenWrappers that define the test data */ - public static Stream ifConstructs() { + static Stream ifConstructs() { return Stream.of( // if constructs with all comparison operators between an ident and a number // the if and else bodies are simple assignments @@ -173,7 +173,7 @@ public static Stream ifConstructs() { * * @return a stream of TokenWrappers that define the test data */ - public static Stream whileLoops() { + static Stream whileLoops() { return Stream.of( // while loops with all comparison operators between an ident and a number // the body is a simple assignment @@ -206,7 +206,7 @@ public static Stream whileLoops() { * * @return a stream of TokenWrappers that define the test data */ - public static Stream returnStatements() { + static Stream returnStatements() { return Stream.of( /* unparameterized data @@ -234,7 +234,7 @@ public static Stream returnStatements() { * * @return a stream of TokenWrappers that define the test data */ - public static Stream simpleExpressions() { + static Stream simpleExpressions() { return Stream.of( /* These TokenWrappers are independent from any operators @@ -326,7 +326,7 @@ public static Stream simpleExpressions() { * * @return a stream of TokenWrappers that define the test data */ - public static Stream expressions() { + static Stream expressions() { return Stream.of( /* Every simple expression is an expression as well