From e1480a02af1ab35eac93d2b74507a3225336cf81 Mon Sep 17 00:00:00 2001 From: EvilBeaver Date: Tue, 10 Oct 2023 16:53:46 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9F=D0=BE=D0=B4=D0=B4=D0=B5=D1=80=D0=B6?= =?UTF-8?q?=D0=BA=D0=B0=20=D0=BF=D0=B0=D1=80=D1=81=D0=B5=D1=80=D0=BE=D0=BC?= =?UTF-8?q?=20=D0=90=D1=81=D0=B8=D0=BD=D1=85=20=D0=B8=20=D0=96=D0=B4=D0=B0?= =?UTF-8?q?=D1=82=D1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/OneScript.Language/LanguageDef.cs | 32 +-- src/OneScript.Language/LexemTrie.cs | 213 ------------------ .../LexicalAnalysis/Token.cs | 2 + .../SyntaxAnalysis/AstNodes/MethodNode.cs | 2 + .../AstNodes/UnaryOperationNode.cs | 2 +- .../SyntaxAnalysis/BslSyntaxWalker.cs | 10 + .../SyntaxAnalysis/DefaultBslParser.cs | 62 ++++- .../SyntaxAnalysis/LocalizedErrors.cs | 5 + .../SyntaxAnalysis/NodeKind.cs | 3 +- .../SyntaxAnalysis/RegionDirectiveHandler.cs | 4 +- .../OneScript.Language.Tests/ParserTests.cs | 67 ++++++ .../OneScript.Language.Tests/TestAstNode.cs | 7 +- .../TreeValidatorExtensions.cs | 35 +++ .../OneScript.Language.Tests/TrieTests.cs | 6 +- 14 files changed, 198 insertions(+), 252 deletions(-) delete mode 100644 src/OneScript.Language/LexemTrie.cs diff --git a/src/OneScript.Language/LanguageDef.cs b/src/OneScript.Language/LanguageDef.cs index fadd827f3..25cf484cc 100644 --- a/src/OneScript.Language/LanguageDef.cs +++ b/src/OneScript.Language/LanguageDef.cs @@ -17,16 +17,13 @@ public static class LanguageDef static readonly Dictionary _priority = new Dictionary(); public const int MAX_OPERATION_PRIORITY = 8; - private static readonly LexemTrie _stringToToken = new LexemTrie(); + private static readonly IdentifiersTrie _stringToToken = new IdentifiersTrie(); - private static readonly LexemTrie _undefined = new LexemTrie(); - private static readonly LexemTrie _booleans = new LexemTrie(); - private static readonly LexemTrie _logicalOp = new LexemTrie(); + private static readonly IdentifiersTrie _undefined = new IdentifiersTrie(); + private static readonly IdentifiersTrie _booleans = new IdentifiersTrie(); + private static readonly IdentifiersTrie _logicalOp = new IdentifiersTrie(); - private static readonly LexemTrie _preprocRegion = new LexemTrie(); - private static readonly LexemTrie _preprocEndRegion = new LexemTrie(); - - private static readonly LexemTrie _preprocImport = new LexemTrie(); + private static readonly IdentifiersTrie _preprocImport = new IdentifiersTrie(); const int BUILTINS_INDEX = (int)Token.ByValParam; @@ -134,6 +131,8 @@ static LanguageDef() AddToken(Token.Equal, "="); AddToken(Token.Semicolon, ";"); AddToken(Token.Question, "?"); + AddToken(Token.Tilde, "~"); + AddToken(Token.Colon, ":"); #endregion @@ -216,11 +215,6 @@ static LanguageDef() #endregion - _preprocRegion.Add("Область",true); - _preprocRegion.Add("Region", true); - _preprocEndRegion.Add("КонецОбласти", true); - _preprocEndRegion.Add("EndRegion", true); - _preprocImport.Add("Использовать", true); _preprocImport.Add("Use", true); } @@ -410,18 +404,6 @@ public static bool IsLogicalOperatorString(string content) return _logicalOp.TryGetValue(content, out var nodeIsFilled) && nodeIsFilled; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool IsPreprocRegion(string value) - { - return _preprocRegion.TryGetValue(value, out var nodeIsFilled) && nodeIsFilled; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool IsPreprocEndRegion(string value) - { - return _preprocEndRegion.TryGetValue(value, out var nodeIsFilled) && nodeIsFilled; - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsImportDirective(string value) { diff --git a/src/OneScript.Language/LexemTrie.cs b/src/OneScript.Language/LexemTrie.cs deleted file mode 100644 index 5263b4e15..000000000 --- a/src/OneScript.Language/LexemTrie.cs +++ /dev/null @@ -1,213 +0,0 @@ -/*---------------------------------------------------------- -This Source Code Form is subject to the terms of the -Mozilla Public License, v.2.0. If a copy of the MPL -was not distributed with this file, You can obtain one -at http://mozilla.org/MPL/2.0/. -----------------------------------------------------------*/ - -using System.Collections.Generic; -using System.Diagnostics; - -namespace OneScript.Language -{ - public class LexemTrie - { - private class TrieNode - { - public char UCase; - public char LCase; - - public TrieNode next; - public TrieNode sibling; - public T value; - } - - private static int _alphabetLength; - private TrieNode[] _alphabet; - private int _count; - - static LexemTrie() - { - var ru = "абвгдеёжзийклмнопрстуфхцчшщьыъэюя"; - var en = "abcdefghijklmnopqrstuvwxyz"; - var symbols = @"+-*/\()[].,<>=;?%0123456789"; - - var all = symbols + - ru + - ru.ToUpper() + - en + - en.ToUpper(); - - _alphabetLength = all.Length; - } - - public LexemTrie() - { - _alphabet = new TrieNode[_alphabetLength]; - } - - public int Count => _count; - - private static int GetIndex(char c) - { - var code = (int) c; - - if (code >= 1040 && code <= 1103) - { - return code - 960; - } - - if (code >= 40 && code <= 57) - { - return code - 39; - } - - if (code >= 59 && code <= 63) - { - return code - 40; - } - - if (code >= 65 && code <= 93) - { - return code - 41; - } - - if (code >= 97 && code <= 122) - { - return code - 44; - } - - switch (c) - { - case '%': - return 0; - case 'Ё': - return 79; - case 'ё': - return 144; - } - - return -1; - } - - private TrieNode GetValueNode(string key) - { - var index = GetIndex(key[0]); - if (index == -1) - return null; - - var node = _alphabet[index]; - if (node == null) - { - node = new TrieNode(); - node.LCase = char.ToLower(key[0]); - node.UCase = char.ToUpper(key[0]); - _alphabet[GetIndex(node.LCase)] = node; - _alphabet[GetIndex(node.UCase)] = node; - } - - for (int i = 1; i < key.Length; i++) - { - var current = node; - node = node.next; - if (node == null) - { - var newNode = new TrieNode(); - newNode.LCase = char.ToLower(key[i]); - newNode.UCase = char.ToUpper(key[i]); - current.next = newNode; - node = newNode; - } - else if (node.LCase != key[i] && node.UCase != key[i]) - { - var insert = node.sibling; - while (insert != null) - { - if (insert.LCase == key[i] || insert.UCase == key[i]) - { - node = insert; - break; - } - - node = insert; - insert = insert.sibling; - } - - if (insert == null) - { - var newNode = new TrieNode(); - newNode.LCase = char.ToLower(key[i]); - newNode.UCase = char.ToUpper(key[i]); - node.sibling = newNode; - node = newNode; - } - } - } - - return node; - } - - public void Add(string key, T value) - { - var node = GetValueNode(key); - Debug.Assert(node != null); - - node.value = value; - ++_count; - } - - public T Get(string key) - { - var node = FindNode(key); - if (node == null) - { - throw new KeyNotFoundException(); - } - - return node.value; - } - - private TrieNode FindNode(string key) - { - var index = GetIndex(key[0]); - if (index == -1 || _alphabet[index] == null) - return null; - - var node = _alphabet[index]; - for (int i = 1; i < key.Length; i++) - { - node = node.next; - if (node == null) - return null; - - while(node.LCase != key[i] && node.UCase != key[i]) - { - node = node.sibling; - if(node == null) - return null; - } - } - - return node; - } - - public bool TryGetValue(string key, out T value) - { - var node = FindNode(key); - if (node == null) - { - value = default(T); - return false; - } - - value = node.value; - return true; - } - - public T this[string key] - { - get => Get(key); - set => Add(key, value); - } - } -} diff --git a/src/OneScript.Language/LexicalAnalysis/Token.cs b/src/OneScript.Language/LexicalAnalysis/Token.cs index fc3fd7824..0a78f9df1 100644 --- a/src/OneScript.Language/LexicalAnalysis/Token.cs +++ b/src/OneScript.Language/LexicalAnalysis/Token.cs @@ -43,6 +43,8 @@ public enum Token RemoveHandler, Async, Await, + Tilde, + Colon, // operators Plus, diff --git a/src/OneScript.Language/SyntaxAnalysis/AstNodes/MethodNode.cs b/src/OneScript.Language/SyntaxAnalysis/AstNodes/MethodNode.cs index d7de41a27..07aa269a3 100644 --- a/src/OneScript.Language/SyntaxAnalysis/AstNodes/MethodNode.cs +++ b/src/OneScript.Language/SyntaxAnalysis/AstNodes/MethodNode.cs @@ -17,6 +17,8 @@ public MethodNode() : base(NodeKind.Method) { } + public bool IsAsync { get; set; } + public MethodSignatureNode Signature { get; private set; } public BslSyntaxNode MethodBody { get; private set; } diff --git a/src/OneScript.Language/SyntaxAnalysis/AstNodes/UnaryOperationNode.cs b/src/OneScript.Language/SyntaxAnalysis/AstNodes/UnaryOperationNode.cs index dddcaf793..05e9b80cd 100644 --- a/src/OneScript.Language/SyntaxAnalysis/AstNodes/UnaryOperationNode.cs +++ b/src/OneScript.Language/SyntaxAnalysis/AstNodes/UnaryOperationNode.cs @@ -13,7 +13,7 @@ public class UnaryOperationNode : NonTerminalNode { public Token Operation { get; } - public UnaryOperationNode(Lexem operation) : base(NodeKind.UnaryOperation) + public UnaryOperationNode(Lexem operation) : base(NodeKind.UnaryOperation, operation) { Operation = operation.Token; } diff --git a/src/OneScript.Language/SyntaxAnalysis/BslSyntaxWalker.cs b/src/OneScript.Language/SyntaxAnalysis/BslSyntaxWalker.cs index fbeae0c73..855ead38a 100644 --- a/src/OneScript.Language/SyntaxAnalysis/BslSyntaxWalker.cs +++ b/src/OneScript.Language/SyntaxAnalysis/BslSyntaxWalker.cs @@ -158,10 +158,20 @@ protected virtual void VisitStatement(BslSyntaxNode statement) VisitGlobalProcedureCall(statement as CallNode); else if (statement.Kind == NodeKind.DereferenceOperation) VisitProcedureDereference(statement); + else if (statement.Kind == NodeKind.UnaryOperation && statement is UnaryOperationNode + { + Operation: Token.Await + } unaryOp) + VisitGlobalAwaitCall(unaryOp); else DefaultVisit(statement); } + private void VisitGlobalAwaitCall(UnaryOperationNode awaitStatement) + { + VisitStatement(awaitStatement.Children[0]); + } + protected virtual void VisitAssignment(BslSyntaxNode assignment) { var left = assignment.Children[0]; diff --git a/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs b/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs index 87e99f83f..5ab6a8f3e 100644 --- a/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs +++ b/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs @@ -27,6 +27,7 @@ public class DefaultBslParser private bool _isMethodsDefined; private bool _isStatementsDefined; private bool _isInFunctionScope; + private bool _isInAsyncMethod; private bool _lastDereferenceIsWritable; private readonly Stack _tokenStack = new Stack(); @@ -286,7 +287,8 @@ private void BuildMethodsSection() { if (_lastExtractedLexem.Type != LexemType.Annotation && _lastExtractedLexem.Token != Token.Procedure - && _lastExtractedLexem.Token != Token.Function) + && _lastExtractedLexem.Token != Token.Function + && _lastExtractedLexem.Token != Token.Async) { return; } @@ -301,7 +303,7 @@ private void BuildMethodsSection() while (true) { BuildAnnotations(); - if (_lastExtractedLexem.Token == Token.Procedure || _lastExtractedLexem.Token == Token.Function) + if (IsStartOfMethod(_lastExtractedLexem)) { if (!sectionExist) { @@ -324,14 +326,26 @@ private void BuildMethodsSection() } } + private static bool IsStartOfMethod(in Lexem lex) + { + return lex.Token == Token.Async || lex.Token == Token.Procedure || lex.Token == Token.Function; + } + private void BuildMethod() { - Debug.Assert(_lastExtractedLexem.Token == Token.Procedure || _lastExtractedLexem.Token == Token.Function); + Debug.Assert(IsStartOfMethod(_lastExtractedLexem)); var method = _nodeContext.AddChild(new MethodNode()); ApplyAnnotations(method); PushContext(method); + if (_lastExtractedLexem.Token == Token.Async) + { + method.IsAsync = true; + _isInAsyncMethod = true; + NextLexem(); + } + try { BuildMethodSignature(); @@ -345,6 +359,7 @@ private void BuildMethod() _isInFunctionScope = false; _inMethodScope = false; _isStatementsDefined = false; + _isInAsyncMethod = false; PopContext(); } } @@ -382,9 +397,8 @@ private void BuildMethodBody() private void BuildMethodSignature() { - var isFunction = _lastExtractedLexem.Token == Token.Function; - var signature = _nodeContext.AddChild(new MethodSignatureNode(_lastExtractedLexem)); + var isFunction = _lastExtractedLexem.Token == Token.Function; CreateChild(signature, isFunction? NodeKind.Function : NodeKind.Procedure, _lastExtractedLexem); _isInFunctionScope = isFunction; NextLexem(); @@ -676,7 +690,10 @@ private void BuildComplexStructureStatement() case Token.AddHandler: case Token.RemoveHandler: BuildEventHandlerOperation(_lastExtractedLexem.Token); - break; + break; + case Token.Await: + BuildGlobalCallAwaitOperator(); + break; default: var expected = _tokenStack.Peek(); AddError(LocalizedErrors.TokenExpected(expected)); @@ -684,6 +701,35 @@ private void BuildComplexStructureStatement() } } + private void BuildGlobalCallAwaitOperator() + { + Debug.Assert(_lastExtractedLexem.Token == Token.Await); + + _nodeContext.AddChild(TerminalNode()); + } + + + private BslSyntaxNode BuildExpressionAwaitOperator(in Lexem lexem) + { + Debug.Assert(_lastExtractedLexem.Token == Token.Await); + CheckAsyncMethod(); + var awaitOperator = new UnaryOperationNode(lexem); + NextLexem(); + + var call = BuildGlobalCall(_lastExtractedLexem); + awaitOperator.AddChild(call); + + return awaitOperator; + } + + private void CheckAsyncMethod() + { + if (!_isInAsyncMethod) + { + AddError(LocalizedErrors.AwaitMustBeInAsyncMethod(), false); + } + } + private void BuildIfStatement() { var condition = _nodeContext.AddChild(new ConditionNode(_lastExtractedLexem)); @@ -1330,6 +1376,10 @@ private BslSyntaxNode TerminalNode() { node = BuildGlobalCall(_lastExtractedLexem); } + else if (_lastExtractedLexem.Token == Token.Await) + { + node = BuildExpressionAwaitOperator(_lastExtractedLexem); + } else { AddError(LocalizedErrors.ExpressionSyntax()); diff --git a/src/OneScript.Language/SyntaxAnalysis/LocalizedErrors.cs b/src/OneScript.Language/SyntaxAnalysis/LocalizedErrors.cs index b2f8b73fe..b069f1656 100644 --- a/src/OneScript.Language/SyntaxAnalysis/LocalizedErrors.cs +++ b/src/OneScript.Language/SyntaxAnalysis/LocalizedErrors.cs @@ -60,6 +60,11 @@ public static CodeError ExportedLocalVar(string varName) $"Local variable can't be exported ({varName})"); } + public static CodeError AwaitMustBeInAsyncMethod() => Create( + "Оператор Ждать (Await) может употребляться только в асинхронных процедурах или функциях", + "Operator Await can be used only in async procedures or functions" + ); + public static CodeError LiteralExpected() => Create("Ожидается константа", "Constant expected"); public static CodeError NumberExpected() => Create("Ожидается числовая константа", "Numeric constant expected"); diff --git a/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs b/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs index bc76bca70..855402c39 100644 --- a/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs +++ b/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs @@ -61,6 +61,7 @@ public enum NodeKind RemoveHandler, Preprocessor, Import, - TopLevelExpression + TopLevelExpression, + Async } } \ No newline at end of file diff --git a/src/OneScript.Language/SyntaxAnalysis/RegionDirectiveHandler.cs b/src/OneScript.Language/SyntaxAnalysis/RegionDirectiveHandler.cs index 90a05ad03..d659ce4da 100644 --- a/src/OneScript.Language/SyntaxAnalysis/RegionDirectiveHandler.cs +++ b/src/OneScript.Language/SyntaxAnalysis/RegionDirectiveHandler.cs @@ -12,8 +12,8 @@ namespace OneScript.Language.SyntaxAnalysis { public class RegionDirectiveHandler : DirectiveHandlerBase { - private readonly LexemTrie _preprocRegion = new LexemTrie(); - private readonly LexemTrie _preprocEndRegion = new LexemTrie(); + private readonly IdentifiersTrie _preprocRegion = new IdentifiersTrie(); + private readonly IdentifiersTrie _preprocEndRegion = new IdentifiersTrie(); private int _regionsNesting = 0; diff --git a/src/Tests/OneScript.Language.Tests/ParserTests.cs b/src/Tests/OneScript.Language.Tests/ParserTests.cs index ec6144928..afd6cdf04 100644 --- a/src/Tests/OneScript.Language.Tests/ParserTests.cs +++ b/src/Tests/OneScript.Language.Tests/ParserTests.cs @@ -1168,6 +1168,73 @@ public void TestLocalExportVar() }); } + [Fact] + public void TestAsyncProcedure() + { + var code = + @"Асинх Процедура Проц1() + Перем Переменная; + КонецПроцедуры"; + + var node = ParseModuleAndGetValidator(code); + node.Is(NodeKind.MethodsSection); + + var methodNode = node.NextChild().Is(NodeKind.Method); + + methodNode.CurrentNode.RealNode.As().IsAsync.Should().BeTrue(); + } + + [Fact] + public void TestAsyncFunction() + { + var code = + @"Асинх Функция Ф1() + Перем Переменная; + КонецФункции"; + + var node = ParseModuleAndGetValidator(code); + node.Is(NodeKind.MethodsSection); + + var methodNode = node.NextChild().Is(NodeKind.Method); + + methodNode.CurrentNode.RealNode.As().IsAsync.Should().BeTrue(); + } + + [Fact] + public void TestAwaitPriority() + { + var code = + @"Асинх Процедура Проц1() + А = Ждать Вызов().Поле; + КонецПроцедуры"; + + var node = ParseModuleAndGetValidator(code); + node.Is(NodeKind.MethodsSection); + + var method = node.NextChild().Is(NodeKind.Method); + var expression = method.DownTo(NodeKind.Assignment) + .NextChildIs(NodeKind.Identifier) + .NextChild(); + + expression.CurrentNode.RealNode.As().Operation.Should().Be(Token.Await); + expression + .NextChildIs(NodeKind.DereferenceOperation) + .NoMoreChildren(); + } + + [Fact] + public void TestAwaitMustBeInAsyncOnly() + { + var code = + @"Процедура Проц1() + Ждать Операция(); + КонецПроцедуры"; + + CatchParsingError(code, errors => + { + errors.Single().Description.Should().Contain("Await"); + }); + } private static void CatchParsingError(string code) { diff --git a/src/Tests/OneScript.Language.Tests/TestAstNode.cs b/src/Tests/OneScript.Language.Tests/TestAstNode.cs index 613371016..e909e7952 100644 --- a/src/Tests/OneScript.Language.Tests/TestAstNode.cs +++ b/src/Tests/OneScript.Language.Tests/TestAstNode.cs @@ -31,7 +31,7 @@ public TestAstNode(BslSyntaxNode node) BinaryOperationNode binary => binary.Operation.ToString(), UnaryOperationNode unary => unary.Operation.ToString(), PreprocessorDirectiveNode preproc => preproc.DirectiveName, - _ => Value + _ => nonTerm.ToString() }; } else if(node is TerminalNode term) @@ -39,6 +39,11 @@ public TestAstNode(BslSyntaxNode node) _childrenLazy = new Lazy>(new TestAstNode[0]); Value = term.Lexem.Content; } + else + { + _childrenLazy = new Lazy>(new TestAstNode[0]); + Value = node.ToString(); + } } public string Value { get; set; } diff --git a/src/Tests/OneScript.Language.Tests/TreeValidatorExtensions.cs b/src/Tests/OneScript.Language.Tests/TreeValidatorExtensions.cs index 93f319b0d..1b246c6cb 100644 --- a/src/Tests/OneScript.Language.Tests/TreeValidatorExtensions.cs +++ b/src/Tests/OneScript.Language.Tests/TreeValidatorExtensions.cs @@ -27,6 +27,41 @@ public static SyntaxTreeValidator Is(this SyntaxTreeValidator validator, NodeKin validator.CurrentNode.Kind.Should().Be(type); return validator; } + + public static SyntaxTreeValidator DownTo(this SyntaxTreeValidator validator, NodeKind type) + { + var node = DownTo(validator.CurrentNode, type); + if (node == null) + { + throw new Exception("No such child: " + type); + } + + return new SyntaxTreeValidator(node); + } + + private static TestAstNode DownTo(this TestAstNode node, NodeKind type) + { + if (node.Kind == type) + return node; + + if (node.Children.Count == 0) + return null; + + foreach (var childNode in node.ChildrenList) + { + if (childNode.Kind == type) + return childNode; + + if (childNode.ChildrenList.Count == 0) + continue; + + var result = DownTo(childNode, type); + if (result != null) + return result; + } + + return null; + } public static void Is(this TestAstNode node, NodeKind type) { diff --git a/src/Tests/OneScript.Language.Tests/TrieTests.cs b/src/Tests/OneScript.Language.Tests/TrieTests.cs index f6c137daa..0c476e7d0 100644 --- a/src/Tests/OneScript.Language.Tests/TrieTests.cs +++ b/src/Tests/OneScript.Language.Tests/TrieTests.cs @@ -12,9 +12,9 @@ namespace OneScript.Language.Tests public class TrieTests { [Fact] - public void LexemTrieAdd() + public void IdentifiersTrieAdd() { - var t = new LexemTrie(); + var t = new IdentifiersTrie(); t.Add("Иван", 0); t.Add("Иволга", 1); @@ -28,7 +28,7 @@ public void LexemTrieAdd() [Fact] public void Tokens() { - var t = new LexemTrie(); + var t = new IdentifiersTrie(); t.Add("иначе", 1); t.Add("иначеесли", 2);