Standalone command-line tool for analyzing whole projects

First tool is a symbol-table inspector:
* verible-verilog-project symbol-table-defs args...
* verible-verilog-project symbol-table-refs args...

PiperOrigin-RevId: 348548097

BUILD

@@ -24,6 +24,7 @@ installer(
         "//verilog/tools/lint:verible-verilog-lint",
         "//verilog/tools/obfuscator:verible-verilog-obfuscate",
         "//verilog/tools/preprocessor:verible-verilog-preprocessor",
+        "//verilog/tools/project:verible-verilog-project",
         "//verilog/tools/syntax:verible-verilog-syntax",
         "//verilog/tools/kythe:verible-verilog-kythe-extractor",
 

verilog/analysis/BUILD

@@ -390,6 +390,7 @@ cc_test(
         ":symbol_table",
         ":verilog_project",
         "//common/text:tree_utils",
+        "//common/util:file_util",
         "//common/util:logging",
         "//common/util:range",
         "@com_google_absl//absl/base:core_headers",

verilog/analysis/symbol_table.cc

@@ -978,6 +978,7 @@ std::ostream& SymbolTable::PrintSymbolReferences(std::ostream& stream) const {
 
 std::vector<absl::Status> BuildSymbolTable(const VerilogSourceFile& source,
                                            SymbolTable* symbol_table) {
+  VLOG(1) << __FUNCTION__;
   const auto* text_structure = source.GetTextStructure();
   if (text_structure == nullptr) return std::vector<absl::Status>();
   const auto& syntax_tree = text_structure->SyntaxTree();

verilog/analysis/symbol_table_test.cc

@@ -26,6 +26,7 @@
 #include "absl/status/status.h"
 #include "absl/strings/string_view.h"
 #include "common/text/tree_utils.h"
+#include "common/util/file_util.h"
 #include "common/util/logging.h"
 #include "common/util/range.h"
 #include "verilog/analysis/verilog_project.h"
@@ -42,7 +43,9 @@ class SymbolTable::Tester : public SymbolTable {
 
 namespace {
 
+using testing::ElementsAreArray;
 using testing::HasSubstr;
+using verible::file::testing::ScopedTestFile;
 
 // An in-memory source file that doesn't require file-system access,
 // nor create temporary files.
@@ -1968,10 +1971,43 @@ TEST(BuildSymbolTableTest, ClassDeclarationWithParameter) {
   }
 }
 
-// TODO:
-// expressions in ranges of dimensions.
-// parameters package/module/class.
-// generally, more testing unresolved symbols.
+struct FileListTestCase {
+  absl::string_view contents;
+  std::vector<absl::string_view> expected_files;
+};
+
+TEST(ParseSourceFileListFromFileTest, FileNotFound) {
+  const auto files_or_status(ParseSourceFileListFromFile("/no/such/file.txt"));
+  EXPECT_FALSE(files_or_status.ok());
+}
+
+TEST(ParseSourceFileListFromFileTest, VariousValidFiles) {
+  const FileListTestCase kTestCases[] = {
+      {"", {}},                // empty
+      {"\n\n", {}},            // blank lines
+      {"foo.sv", {"foo.sv"}},  // missing terminating newline, but still works
+      {"foo.sv\n", {"foo.sv"}},
+      {"file name contains space.sv\n", {"file name contains space.sv"}},
+      {"foo/bar.sv\n", {"foo/bar.sv"}},  // with path separator
+      {" foo.sv\n", {"foo.sv"}},         // remove leading whitespace
+      {"foo.sv \n", {"foo.sv"}},         // remove trailing whitespace
+      {"#foo.sv\n", {}},                 // commented out
+      {"# foo.sv\n", {}},                // commented out
+      {"foo.sv\nbar/bar.sv\n", {"foo.sv", "bar/bar.sv"}},
+      {"/foo/bar.sv\n"
+       "### ignore this one\n"
+       "bar/baz.txt\n",
+       {"/foo/bar.sv", "bar/baz.txt"}},
+  };
+  for (const auto& test : kTestCases) {
+    const ScopedTestFile test_file(testing::TempDir(), test.contents);
+    const auto files_or_status(
+        ParseSourceFileListFromFile(test_file.filename()));
+    ASSERT_TRUE(files_or_status.ok()) << files_or_status.status().message();
+    EXPECT_THAT(*files_or_status, ElementsAreArray(test.expected_files))
+        << "input: " << test.contents;
+  }
+}
 
 }  // namespace
 }  // namespace verilog

verilog/analysis/verilog_project.cc

@@ -14,8 +14,14 @@
 
 #include "verilog/analysis/verilog_project.h"
 
+#include <iostream>
+#include <string>
+#include <vector>
+
 #include "absl/memory/memory.h"
 #include "absl/status/status.h"
+#include "absl/strings/ascii.h"
+#include "absl/strings/match.h"
 #include "absl/strings/str_join.h"
 #include "common/text/text_structure.h"
 #include "common/util/file_util.h"
@@ -181,4 +187,23 @@ std::vector<absl::Status> VerilogProject::GetErrorStatuses() const {
   return statuses;
 }
 
+absl::StatusOr<std::vector<std::string>> ParseSourceFileListFromFile(
+    absl::string_view file_list_file) {
+  std::string content;
+  const auto read_status = verible::file::GetContents(file_list_file, &content);
+  if (!read_status.ok()) return read_status;
+
+  std::vector<std::string> files_names;
+  std::string filename;
+  std::istringstream stream(content);
+  while (std::getline(stream, filename)) {
+    // Ignore blank lines and "# ..." comments
+    if (filename.empty()) continue;
+    if (filename.front() == '#') continue;
+    absl::RemoveExtraAsciiWhitespace(&filename);
+    files_names.push_back(filename);
+  }
+  return files_names;
+}
+
 }  // namespace verilog

verilog/analysis/verilog_project.h

@@ -249,6 +249,10 @@ class VerilogProject {
       buffer_to_analyzer_map_;
 };
 
+// Reads in a list of files line-by-line from 'file_list_file'.
+absl::StatusOr<std::vector<std::string>> ParseSourceFileListFromFile(
+    absl::string_view file_list_file);
+
 }  // namespace verilog
 
 #endif  // VERIBLE_VERILOG_ANALYSIS_VERILOG_PROJECT_H_

verilog/tools/kythe/verilog_kythe_extractor.cc

@@ -206,24 +206,15 @@ Output: Produces Indexing Facts for kythe (http://kythe.io).
   }
   const std::string file_list_root = absl::GetFlag(FLAGS_file_list_root);
 
-  // TODO(fangism): Push the following block into a VerilogProject method for
-  // consuming a file-list.
-  std::vector<std::string> files_names;
-  {
-    std::string content;
-    if (!verible::file::GetContents(file_list_path, &content).ok()) {
-      LOG(ERROR) << "Error while reading file list at: " << file_list_path;
-      return 1;
-    }
-
-    std::string filename;
-    std::istringstream stream(content);
-    while (stream >> filename) {
-      // Ignore blank lines and "# ..." comments
-      if (filename.empty() || filename[0] == '#') continue;
-      files_names.push_back(filename);
-    }
+  // Load file list.
+  const auto files_names_or_status(
+      verilog::ParseSourceFileListFromFile(file_list_path));
+  if (!files_names_or_status.ok()) {
+    LOG(ERROR) << "Error while reading file list: "
+               << files_names_or_status.status().message();
+    return 1;
   }
+  const std::vector<std::string>& files_names(*files_names_or_status);
 
   verilog::VerilogProject project(file_list_root, include_dir_paths);
 

verilog/tools/project/BUILD

@@ -0,0 +1,30 @@
+"""This package contains multi-tool for working with SystemVerilog projects.
+"""
+
+licenses(["notice"])
+
+cc_binary(
+    name = "verible-verilog-project",
+    srcs = ["project_tool.cc"],
+    visibility = ["//:__subpackages__"],
+    deps = [
+        "//common/util:file_util",
+        "//common/util:init_command_line",
+        "//common/util:logging",
+        "//common/util:subcommand",
+        "//verilog/analysis:symbol_table",
+        "//verilog/analysis:verilog_project",
+        "@com_google_absl//absl/flags:usage",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+sh_test(
+    name = "project_tool_test",
+    size = "small",
+    srcs = ["project_tool_test.sh"],
+    args = ["$(location :verible-verilog-project)"],
+    data = [":verible-verilog-project"],
+    deps = [],
+)

verilog/tools/project/README.md

@@ -0,0 +1,50 @@
+# SystemVerilog Project Multi-tool
+
+<!--*
+freshness: { owner: 'fangism' reviewed: '2020-12-21' }
+*-->
+
+`verible-verilog-project` is a multi-tool that operates on whole Verilog
+projects, consisting of a file list and related configurations. This serves as a
+diagnostic tool for analyzing (and potentially transforming) project-level
+sources.
+
+## Usage
+
+```
+verible-verilog-project COMMAND [options...]
+
+available commands:
+  help
+  symbol-table-defs
+  symbol-table-refs
+
+  Flags from verilog/tools/project/project_tool.cc:
+    --file_list_path (The path to the file list which contains the names of
+      SystemVerilog files.
+      The files should be ordered by definition dependencies.); default: "";
+    --file_list_root (The absolute location which we prepend to the files in the
+      file list (where listed files are relative to).); default: ".";
+    --include_dir_paths (Comma separated paths of the directories used to look
+      for included files.
+      Note: The order of the files here is important.
+      File search will stop at the the first found among the listed directories.
+      e.g --include_dir_paths directory1,directory2
+      if "A.sv" exists in both "directory1" and "directory2" the one in
+      "directory1" is the one we will use.
+      ); default: ;
+```
+
+## Commands
+
+### `symbol-table-defs`
+
+Builds a unified symbol table over all project files, and prints a
+human-readable representation. This does not show nor attempt to resolve symbol
+references.
+
+### `symbol-table-refs`
+
+Builds a unified symbol table over all project files, attempts to resolve all
+symbol references to definitions, and prints a human-readable representation of
+the references.