PR #245: Implement basic file-based waivers

This is an initial implementation of what could serve as an answer to #105.

The waiver file can be specified with a flag. The syntax of the file is:
```
waive <rule name> <line_begin> [line_end]
```

I am going to continue working on this to make it more sophisticated (e.g. add filename matching, maybe matching lines with regex as suggested in #105), but as usually I am opening this early in case there are some comments that are fundamentally against what I am doing here.

GitHub PR #245

Copybara import of the project:

  - 5b8c258 Implement basic file-based waivers by Tomasz Gorochowik <[email protected]>

Closes #245
issues #105

PiperOrigin-RevId: 310496171

README.md

@@ -372,6 +372,16 @@ All lines in between will be waived for rule-X
 // verilog_lint: waive-stop rule-X
 ```
 
+Another option is to use a configuration file with the `--waivers_file` flag.
+The format of this file is as follows: `waive --rule=rule-name-1 --line=10 waive
+--rule=rule-name-2 --line=5:10 waive --rule=rule-name-3 --regex="^\s*abc$"`
+
+The `--line` flag can be used to specify a single line to apply the waiver to or
+a line range (separated with the `:` character). Additionally the `--regex` flag
+can be used to dynamically match lines on which a given rule has to be waived.
+This is especially useful for projects where some of the files are
+auto-generated.
+
 The name of the rule to waive is at the end of each diagnostic message in `[]`.
 
 Syntax errors cannot be waived. A common source of syntax errors is if the file

common/analysis/BUILD

@@ -1,5 +1,7 @@
 # This package contains functions and classes for analyzing text structures.
 
+load("//bazel:flex.bzl", "genlex")
+
 licenses(["notice"])
 
 package(
@@ -43,16 +45,52 @@ cc_library(
     ],
 )
 
+genlex(
+    name = "config_file_lex",
+    src = "config_file.lex",
+    out = "config_file.yy.cc",
+)
+
+cc_library(
+    name = "config_file_lexer",
+    srcs = [
+        "config_file.yy.cc",
+        "config_file_lexer.cc",
+    ],
+    hdrs = [
+        "config_file_lexer.h",
+        "lint_waiver.h",
+    ],
+    copts = ["-Wno-implicit-fallthrough"],
+    deps = [
+        "//bazel:flex",
+        "//common/lexer:flex_lexer_adapter",
+        "//common/lexer:token_stream_adapter",
+        "//common/text:text_structure",
+        "//common/text:token_info",
+        "//common/text:token_stream_view",
+        "//common/util:container_util",
+        "//common/util:interval_set",
+        "//common/util:logging",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
 cc_library(
     name = "lint_waiver",
     srcs = ["lint_waiver.cc"],
     hdrs = ["lint_waiver.h"],
+    copts = ["-fexceptions"],
+    features = ["-use_header_modules"],  # precompiled headers incompatible with -fexceptions.
     deps = [
+        ":config_file_lexer",
         "//common/strings:comment_utils",
         "//common/text:text_structure",
         "//common/text:token_info",
         "//common/text:token_stream_view",
+        "//common/util:container_iterator_range",
         "//common/util:container_util",
+        "//common/util:file_util",
         "//common/util:interval_set",
         "//common/util:iterator_range",
         "//common/util:logging",

common/analysis/config_file.lex

@@ -0,0 +1,134 @@
+/* Copyright 2017-2020 The Verible Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+%{
+#define _WAIVER_FLEXLEXER_H_
+#include "common/analysis/config_file_lexer.h"
+
+#define yy_set_top_state(state)  { yy_pop_state(); yy_push_state(state); }
+%}
+
+%option nodefault
+%option noyywrap
+%option prefix="verible"
+%option c++
+%option yylineno
+%option yyclass="verible::ConfigFileLexer"
+
+LineTerminator \r|\n|\r\n
+InputCharacter [^\r\n\0]
+Space [ \t\f\b]
+
+DecimalDigits [0-9]+
+Alpha [a-zA-Z]
+
+AnyChar ({Alpha}|{DecimalDigits}|-|_|:)
+
+Name {Alpha}({Alpha}|-|_)*
+Value ({AnyChar})*
+QuotedValue ([^\\\"\n]|\\.)*
+
+Quote \"
+
+Command {Name}
+
+ParamPrefix --
+
+Flag {ParamPrefix}{Name}
+FlagWithArg {ParamPrefix}{Name}=
+Arg {Value}
+Param {Value}
+
+
+%x COMMAND
+%x ARG
+%x QUOTED_ARG
+
+%%
+
+{Command} {
+  UpdateLocation();
+  yy_push_state(COMMAND);
+  return CFG_TK_COMMAND;
+}
+
+<COMMAND>{Param} {
+  UpdateLocation();
+  return CFG_TK_PARAM;
+}
+
+<COMMAND>{Flag} {
+  UpdateLocation();
+  return CFG_TK_FLAG;
+}
+
+<COMMAND>{FlagWithArg} {
+  UpdateLocation();
+  yy_push_state(ARG);
+  return CFG_TK_FLAG_WITH_ARG;
+}
+
+<ARG>{Quote} {
+  UpdateLocation();
+  yy_set_top_state(QUOTED_ARG);
+}
+
+<QUOTED_ARG>{QuotedValue} {
+  UpdateLocation();
+  return CFG_TK_ARG;
+}
+
+<QUOTED_ARG>{Quote} {
+  UpdateLocation();
+  yy_pop_state();
+}
+
+<ARG>{Value} {
+  UpdateLocation();
+  yy_pop_state();
+  return CFG_TK_ARG;
+}
+
+<COMMAND>{LineTerminator} {
+  UpdateLocation();
+  yy_pop_state();
+  return CFG_TK_NEWLINE;
+}
+
+<COMMAND><<EOF>> {
+  UpdateLocation();
+  yy_pop_state();
+  return CFG_TK_NEWLINE;
+}
+
+<INITIAL>{LineTerminator} {
+  UpdateLocation();
+}
+
+<*>{LineTerminator} {
+  UpdateLocation();
+  return CFG_TK_ERROR;
+}
+
+<*>{Space} {
+  UpdateLocation();
+}
+
+<*>. {
+  UpdateLocation();
+  return CFG_TK_ERROR;
+}
+
+%%

common/analysis/config_file_lexer.cc

@@ -0,0 +1,77 @@
+// Copyright 2017-2020 The Verible Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "common/analysis/config_file_lexer.h"
+
+#include "absl/strings/string_view.h"
+#include "common/analysis/lint_waiver.h"
+#include "common/lexer/token_stream_adapter.h"
+#include "common/text/token_info.h"
+#include "common/util/logging.h"
+
+namespace verible {
+
+ConfigFileLexer::ConfigFileLexer(absl::string_view config)
+    : parent_lexer_type(config) {
+  const auto lex_status = MakeTokenSequence(
+      this, config, &tokens_, [&](const TokenInfo& error_token) {
+        LOG(ERROR) << "erroneous token: " << error_token;
+      });
+
+  // Pre-process all tokens where its needed
+  for (auto& t : tokens_) {
+    switch (t.token_enum) {
+      case CFG_TK_FLAG:
+        // Skip -- prefix
+        t.text = t.text.substr(2, t.text.length() - 2);
+        break;
+      case CFG_TK_FLAG_WITH_ARG:
+        // Skip -- prefix and = suffix
+        t.text = t.text.substr(2, t.text.length() - 3);
+        break;
+    }
+  }
+}
+
+bool ConfigFileLexer::TokenIsError(const verible::TokenInfo& token) const {
+  return false;
+}
+
+std::vector<TokenRange> ConfigFileLexer::GetCommandsTokenRanges() {
+  std::vector<TokenRange> commands;
+
+  auto i = tokens_.cbegin();
+  auto j = i;
+
+  while (1) {
+    // Note that empty lines or lines with whitespace only are skipped
+    // by the lexer and do not have to be handled here
+    j = std::find_if(i + 1, tokens_.cend(), [](TokenInfo t) {
+      return t.token_enum == CFG_TK_NEWLINE;
+    });
+
+    if (j == tokens_.cend()) {
+      break;
+    }
+
+    // Use const iterators, increment j to include last element
+    commands.push_back(make_range(i, ++j));
+
+    i = j;
+  }
+
+  return commands;
+}
+
+}  // namespace verible

common/analysis/config_file_lexer.h

@@ -0,0 +1,75 @@
+// Copyright 2017-2020 The Verible Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef VERIBLE_CONFIG_FILE_LEXER_H__
+#define VERIBLE_CONFIG_FILE_LEXER_H__
+
+// lint_waiver_config.lex has "%prefix=verible", meaning the class flex
+// creates is veribleFlexLexer. Unfortunately, FlexLexer.h doesn't have proper
+// ifdefs around its inclusion, so we have to put a bar around it here.
+#include "common/analysis/lint_waiver.h"
+#include "common/lexer/flex_lexer_adapter.h"
+#include "common/text/token_info.h"
+#ifndef _WAIVER_FLEXLEXER_H_
+#undef yyFlexLexer  // this is how FlexLexer.h says to do things
+#define yyFlexLexer veribleFlexLexer
+#include <FlexLexer.h>
+#endif
+
+#include "absl/strings/string_view.h"
+
+namespace verible {
+
+// Acceptable syntax:
+//
+// CFG_TK_COMMAND [--CFG_TK_FLAG] [--CFG_TK_FLAG_WITH_ARG=CFG_TK_ARG]
+// [CFG_TK_PARAM]
+enum ConfigTokenEnum {
+  CFG_TK_COMMAND = 1,
+  CFG_TK_FLAG,
+  CFG_TK_FLAG_WITH_ARG,
+  CFG_TK_ARG,
+  CFG_TK_PARAM,
+  CFG_TK_NEWLINE,
+  CFG_TK_ERROR,
+};
+
+enum ConfigParserStateEnum {
+  PARSER_INIT,
+  PARSER_COMMAND,
+};
+
+class ConfigFileLexer : public verible::FlexLexerAdapter<veribleFlexLexer> {
+  using parent_lexer_type = verible::FlexLexerAdapter<veribleFlexLexer>;
+  using parent_lexer_type::Restart;
+
+ public:
+  explicit ConfigFileLexer(absl::string_view config);
+
+  // Main lexing function. Will be defined by Flex.
+  int yylex() override;
+
+  // Returns true if token is invalid.
+  bool TokenIsError(const verible::TokenInfo&) const override;
+
+  // Runs the Lexer and attached command handlers
+  std::vector<TokenRange> GetCommandsTokenRanges();
+
+ private:
+  TokenSequence tokens_;
+};
+
+}  // namespace verible
+
+#endif  // VERIBLE_CONFIG_FILE_LEXER_H__