Merge pull request chipsalliance#998 from hzeller/verlog-ls-1

Add baseline language server.

BUILD

@@ -86,6 +86,7 @@ compilation_database(
         "//verilog/tools/preprocessor:verible-verilog-preprocessor",
         "//verilog/tools/project:verible-verilog-project",
         "//verilog/tools/syntax:verible-verilog-syntax",
+        "//verilog/tools/ls:verible-verilog-ls",
         "//common/lsp:dummy-ls",
     ],
 

verilog/tools/ls/BUILD

@@ -0,0 +1,51 @@
+# This package contains a SystemVerilog language server [1] implementation.
+#
+# [1]: https://microsoft.github.io/language-server-protocol/specification
+
+load("//bazel:sh_test_with_runfiles_lib.bzl", "sh_test_with_runfiles_lib")
+
+licenses(["notice"])
+
+package(
+    default_visibility = [
+        "//:__subpackages__",
+    ],
+)
+
+cc_library(
+    name = "lsp-parse-buffer",
+    srcs = [ "lsp-parse-buffer.cc"],
+    hdrs = [ "lsp-parse-buffer.h" ],
+    deps = [
+      "//verilog/analysis:verilog_analyzer",
+      "//verilog/analysis:verilog_linter",
+      "//common/lsp:lsp-text-buffer",
+    ]
+)
+
+cc_binary(
+    name = "verible-verilog-ls",
+    srcs = ["verilog_ls.cc", "lsp-parse-buffer.h"],
+    deps = [
+      ":lsp-parse-buffer",
+      "//common/lsp:lsp-protocol",
+      "//common/lsp:json-rpc-dispatcher",
+      "//common/lsp:message-stream-splitter",
+      "//common/util:init_command_line",
+    ],
+)
+
+sh_test_with_runfiles_lib(
+    name = "verible-verilog-ls_test",
+    size = "small",
+    srcs = ["verible-verilog-ls_test.sh"],
+    args = [
+      "$(location :verible-verilog-ls)",
+      "$(location //common/lsp:json-rpc-expect)",
+    ],
+    data = [
+     ":verible-verilog-ls",
+     "//common/lsp:json-rpc-expect",
+    ],
+    deps = [],
+)

verilog/tools/ls/lsp-parse-buffer.cc

@@ -0,0 +1,86 @@
+// Copyright 2021 The Verible Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "verilog/tools/ls/lsp-parse-buffer.h"
+
+#include "absl/status/status.h"
+
+namespace verilog {
+static absl::StatusOr<std::vector<verible::LintRuleStatus>> RunLinter(
+    absl::string_view filename, const verilog::VerilogAnalyzer &parser) {
+  const auto &text_structure = parser.Data();
+  verilog::LinterConfiguration config;  // TODO: read from project context
+  verilog::RuleBundle bundle;
+  auto status = config.ConfigureFromOptions(verilog::LinterOptions{
+      .ruleset = verilog::RuleSet::kAll,
+      .rules = bundle,
+  });
+  if (!status.ok()) {
+    std::cerr << "Got an issue with the lint configuration" << std::endl;
+    return status;
+  }
+  return VerilogLintTextStructure(filename, config, text_structure);
+}
+
+ParsedBuffer::ParsedBuffer(int64_t version, absl::string_view uri,
+                           absl::string_view content)
+    : version_(version),
+      parser_(verilog::VerilogAnalyzer::AnalyzeAutomaticMode(content, uri)) {
+  std::cerr << "Analyzed " << uri << " lex:" << parser_->LexStatus()
+            << "; parser:" << parser_->ParseStatus() << std::endl;
+  // TODO(hzeller): we should use a filename not URI; strip prefix.
+  if (auto lint_result = RunLinter(uri, *parser_); lint_result.ok()) {
+    lint_statuses_ = std::move(lint_result.value());
+  }
+}
+
+void BufferTracker::Update(const std::string &filename,
+                           const verible::lsp::EditTextBuffer &txt) {
+  if (current_ && current_->version() == txt.last_global_version())
+    return;  // Nothing to do (we don't really expect this to happen)
+  txt.RequestContent([&txt, &filename, this](absl::string_view content) {
+    current_ = std::make_shared<ParsedBuffer>(txt.last_global_version(),
+                                              filename, content);
+  });
+  if (current_->parsed_successfully()) {
+    last_good_ = current_;
+  }
+}
+
+verible::lsp::BufferCollection::UriBufferCallback
+BufferTrackerContainer::GetSubscriptionCallback() {
+  return
+      [this](const std::string &uri, const verible::lsp::EditTextBuffer *txt) {
+        if (txt) {
+          const BufferTracker *tracker = Update(uri, *txt);
+          // Now inform our listeners.
+          if (change_listener_) change_listener_(uri, *tracker);
+        } else {
+          Remove(uri);
+        }
+      };
+}
+
+BufferTracker *BufferTrackerContainer::Update(
+    const std::string &uri, const verible::lsp::EditTextBuffer &txt) {
+  auto inserted = buffers_.insert({uri, nullptr});
+  if (inserted.second) {
+    inserted.first->second.reset(new BufferTracker());
+  }
+  inserted.first->second->Update(uri, txt);
+  return inserted.first->second.get();
+}
+
+}  // namespace verilog

verilog/tools/ls/lsp-parse-buffer.h

@@ -0,0 +1,106 @@
+// Copyright 2021 The Verible Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef VERILOG_TOOLS_LS_LSP_PARSE_BUFFER_H
+#define VERILOG_TOOLS_LS_LSP_PARSE_BUFFER_H
+
+#include <cstdint>
+
+#include "common/lsp/lsp-text-buffer.h"
+#include "verilog/analysis/verilog_analyzer.h"
+#include "verilog/analysis/verilog_linter.h"
+
+// ParseBuffer and BufferTrackerContainer are tracking fully parsed content
+// and are corresponding to verible::lsp::EditTextBuffer and
+// verible::lsp::BufferCollection that are responsible for tracking the
+// bare editor text.
+
+namespace verilog {
+// A parsed buffer collects all the artifacts generated from a text buffer
+// from parsing or running the linter.
+//
+// Right now, the ParsedBuffer is synchronously filling its internal structure
+// on construction, but plan is to do that on-demand and possibly with
+// std::future<>s evaluated in separate threads.
+class ParsedBuffer {
+ public:
+  ParsedBuffer(int64_t version, absl::string_view uri,
+               absl::string_view content);
+
+  bool parsed_successfully() const {
+    return parser_->LexStatus().ok() && parser_->ParseStatus().ok();
+  }
+
+  const verilog::VerilogAnalyzer &parser() const { return *parser_; }
+  const std::vector<verible::LintRuleStatus> &lint_result() const {
+    return lint_statuses_;
+  }
+
+  int64_t version() const { return version_; }
+
+ private:
+  const int64_t version_;
+  std::unique_ptr<verilog::VerilogAnalyzer> parser_;
+  std::vector<verible::LintRuleStatus> lint_statuses_;
+};
+
+// A buffer tracker tracks the EditTextBuffer content and keeps up to
+// two versions of ParsedBuffers - the latest, that might have parse errors,
+// and the last known good that parsed without errors (if available).
+class BufferTracker {
+ public:
+  void Update(const std::string &filename,
+              const verible::lsp::EditTextBuffer &txt);
+
+  const ParsedBuffer *current() const { return current_.get(); }
+  const ParsedBuffer *last_good() const { return last_good_.get(); }
+
+ private:
+  // The same ParsedBuffer can in both, current and last_good, or last_good can
+  // be an older version. Use shared_ptr to keep track of the reference count.
+  std::shared_ptr<ParsedBuffer> current_;
+  std::shared_ptr<ParsedBuffer> last_good_;
+};
+
+// Container holding all buffer trackers keyed by file uri.
+// This is the correspondent to verible::lsp::BufferCollection that
+// internally stores
+class BufferTrackerContainer {
+ public:
+  // Return a callback that allows to subscribe to an lsp::BufferCollection
+  // to update our internal state whenever the editor state changes.
+  // (internally, they exercise Update() and Remove())
+  verible::lsp::BufferCollection::UriBufferCallback GetSubscriptionCallback();
+
+  // Add a change listener for clients of ours interested in updated fresly
+  // parsed content.
+  using ChangeCallback =
+      std::function<void(const std::string &uri, const BufferTracker &tracker)>;
+  void SetChangeListener(const ChangeCallback &cb) { change_listener_ = cb; }
+
+ private:
+  // Update internal state of the given "uri" with the content of the text
+  // buffer. Return the buffer tracker.
+  BufferTracker *Update(const std::string &uri,
+                        const verible::lsp::EditTextBuffer &txt);
+
+  // Remove the buffer tracker for the given "uri".
+  void Remove(const std::string &uri) { buffers_.erase(uri); }
+
+  ChangeCallback change_listener_ = nullptr;
+  std::unordered_map<std::string, std::unique_ptr<BufferTracker>> buffers_;
+};
+}  // namespace verilog
+#endif  // VERILOG_TOOLS_LS_LSP_PARSE_BUFFER_H

verilog/tools/ls/verible-verilog-ls_test.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+# Copyright 2021 The Verible Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[[ "$#" == 2 ]] || {
+  echo "Expecting 2 positional arguments: lsp-server json-rpc-expect"
+  exit 1
+}
+LSP_SERVER="$(rlocation ${TEST_WORKSPACE}/$1)"
+JSON_RPC_EXPECT="$(rlocation ${TEST_WORKSPACE}/$2)"
+
+TMP_IN=${TEST_TMPDIR:-/tmp/}/test-lsp-in.txt
+JSON_EXPECTED=${TEST_TMPDIR:-/tmp/}/test-lsp-json-expect.txt
+
+MSG_OUT=${TEST_TMPDIR:-/tmp/}/test-lsp-out-msg.txt
+
+# One message per line, converted by the awk script to header/body.
+
+# Starting up server, sending two files, a file with a parse error and
+# a file that parses, but has a EOF newline linting diagnostic.
+#
+# TODO: maybe this awk-script should be replaced with something that allows
+# multi-line input with comment.
+awk '{printf("Content-Length: %d\r\n\r\n%s", length($0), $0)}' > ${TMP_IN} <<EOF
+{"jsonrpc":"2.0", "id":1, "method":"initialize","params":null}
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"file://syntaxerror.sv","text":"brokenfile\n"}}}
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"file://mini.sv","text":"module mini();\nendmodule"}}}
+{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"textDocument":{"uri":"file://mini.sv"},"contentChanges":[{"range":{"start":{"character":9,"line":1},"end":{"character":9,"line":1}},"text":"\n"}]}}
+{"jsonrpc":"2.0","method":"textDocument/didClose","params":{"textDocument":{"uri":"file://mini.sv"}}}
+{"jsonrpc":"2.0", "id":100, "method":"shutdown","params":{}}
+EOF
+
+# TODO: change json rpc expect to allow comments in the input.
+cat > "${JSON_EXPECTED}" <<EOF
+[
+  {
+    "json_contains": {
+        "id":1,
+        "result": {
+          "serverInfo": {"name" : "Verible Verilog language server."}
+        }
+    }
+  },
+  {
+    "json_contains": { "id":100 }
+  }
+]
+EOF
+
+"${LSP_SERVER}" < ${TMP_IN} 2> "${MSG_OUT}" \
+  | ${JSON_RPC_EXPECT} ${JSON_EXPECTED}
+
+JSON_RPC_EXIT=$?
+
+if [ $JSON_RPC_EXIT -ne 0 ]; then
+  # json-rpc-expect outputs the entry, where the mismatch occured, in exit code
+  echo "Exit code of json rpc expect; first error at $JSON_RPC_EXIT"
+  exit 1
+fi
+
+echo "-- stderr messages --"
+cat ${MSG_OUT}
+
+grep "shutdown request" "${MSG_OUT}" > /dev/null
+if [ $? -ne 0 ]; then
+  echo "Didn't get shutdown feedback"
+  exit 1
+fi