feat: create lint tests (#15)

* feat: create lint tests

Fixes #11

* chore: docs

* fix todo

* fmt

* code review comments

CONTRIBUTING.md

@@ -43,6 +43,7 @@ This means that any usage of `@rules_eslint` on your system will point to this f
 1. Watch the automation run on GitHub actions
 
 ## Recording a demo
+
 Install from https://asciinema.org/
 Then cd example and start recording, pasting these commands:
 

docs/BUILD.bazel

@@ -2,6 +2,11 @@
 # so that the dependency on stardoc doesn't leak to them.
 load("@aspect_bazel_lib//lib:docs.bzl", "stardoc_with_diff_test", "update_docs")
 
+stardoc_with_diff_test(
+    name = "lint_test",
+    bzl_library_target = "//lint:lint_test",
+)
+
 stardoc_with_diff_test(
     name = "buf",
     bzl_library_target = "//lint:buf",

example/src/BUILD.bazel

@@ -1,5 +1,7 @@
 load("@aspect_rules_ts//ts:defs.bzl", "ts_project")
 
+package(default_visibility = ["//visibility:public"])
+
 ts_project(
     name = "ts",
     srcs = ["file.ts"],
@@ -8,13 +10,11 @@ ts_project(
 proto_library(
     name = "unused",
     srcs = ["unused.proto"],
-    visibility = ["//visibility:public"],
 )
 
 proto_library(
     name = "foo_proto",
     srcs = ["file.proto"],
-    visibility = ["//visibility:public"],
     deps = [":unused"],
 )
 

example/test/BUILD.bazel

@@ -0,0 +1,19 @@
+"Demonstrates how to enforce zero-lint-tolerance policy with tests"
+
+load("//:lint.bzl", "flake8_test", "pmd_test")
+
+flake8_test(
+    name = "flake8",
+    srcs = ["//src:unused_import"],
+    # Expected to fail based on current content of the file.
+    # Normally you'd fix the file instead of tagging this test.
+    tags = ["manual"],
+)
+
+pmd_test(
+    name = "pmd",
+    srcs = ["//src:foo"],
+    # Expected to fail based on current content of the file.
+    # Normally you'd fix the file instead of tagging this test.
+    tags = ["manual"],
+)

example/tools/lint.bzl

@@ -3,6 +3,7 @@
 load("@aspect_rules_lint//lint:buf.bzl", "buf_lint_aspect")
 load("@aspect_rules_lint//lint:eslint.bzl", "eslint_aspect")
 load("@aspect_rules_lint//lint:flake8.bzl", "flake8_aspect")
+load("@aspect_rules_lint//lint:lint_test.bzl", "make_lint_test")
 load("@aspect_rules_lint//lint:pmd.bzl", "pmd_aspect")
 
 buf = buf_lint_aspect(
@@ -19,7 +20,11 @@ flake8 = flake8_aspect(
     config = "@@//:.flake8",
 )
 
+flake8_test = make_lint_test(aspect = flake8)
+
 pmd = pmd_aspect(
     binary = "@@//:pmd",
     rulesets = ["@@//:pmd.xml"],
 )
+
+pmd_test = make_lint_test(aspect = pmd)

lint/BUILD.bazel

@@ -1,6 +1,6 @@
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
 
-exports_files(glob(["*.bzl"]))
+exports_files(glob(["*.bzl"]) + ["lint_test.sh"])
 
 bzl_library(
     name = "buf",
@@ -19,6 +19,13 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "lint_test",
+    srcs = ["lint_test.bzl"],
+    visibility = ["//visibility:public"],
+    deps = ["@aspect_bazel_lib//lib:paths"],
+)
+
 bzl_library(
     name = "flake8",
     srcs = ["flake8.bzl"],

lint/lint_test.bzl

@@ -0,0 +1,70 @@
+"""Factory function to make lint test rules.
+
+The test will fail when the linter reports any non-empty lint results.
+
+To use this, in your `lint.bzl` where you define the aspect, just create a test that references it.
+
+For example, with `flake8`:
+
+```starlark
+load("@aspect_rules_lint//lint:lint_test.bzl", "make_lint_test")
+load("@aspect_rules_lint//lint:flake8.bzl", "flake8_aspect")
+
+flake8 = flake8_aspect(
+    binary = "@@//:flake8",
+    config = "@@//:.flake8",
+)
+
+flake8_test = make_lint_test(aspect = flake8)
+```
+
+Now in your BUILD files you can add a test:
+
+```starlark
+load("//tools:lint.bzl", "flake8_test")
+
+py_library(
+    name = "unused_import",
+    srcs = ["unused_import.py"],
+)
+
+flake8_test(
+    name = "flake8",
+    srcs = [":unused_import"],
+)
+```
+"""
+
+load("@aspect_bazel_lib//lib:paths.bzl", "to_rlocation_path")
+
+def _test_impl(ctx):
+    reports = []
+    for src in ctx.attr.srcs:
+        for report in src[OutputGroupInfo].report.to_list():
+            reports.append(report)
+
+    bin = ctx.actions.declare_file("lint_test.sh")
+    ctx.actions.expand_template(
+        template = ctx.file._bin,
+        output = bin,
+        substitutions = {"{{reports}}": " ".join([to_rlocation_path(ctx, r) for r in reports])},
+        is_executable = True,
+    )
+    return [DefaultInfo(
+        executable = bin,
+        runfiles = ctx.runfiles(reports + [ctx.file._runfiles_lib]),
+    )]
+
+def make_lint_test(aspect):
+    return rule(
+        implementation = _test_impl,
+        attrs = {
+            "srcs": attr.label_list(doc = "*_library targets", aspects = [aspect]),
+            # Note, we don't use this in the test, but the user passes an aspect that has this aspect_attribute,
+            # and that requires that we list it here as well.
+            "fail_on_violation": attr.bool(),
+            "_bin": attr.label(default = ":lint_test.sh", allow_single_file = True, executable = True, cfg = "exec"),
+            "_runfiles_lib": attr.label(default = "@bazel_tools//tools/bash/runfiles", allow_single_file = True),
+        },
+        test = True,
+    )

lint/lint_test.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Asserts that all lint reports are empty.
+
+# --- begin runfiles.bash initialization v3 ---
+# Copy-pasted from the Bazel Bash runfiles library v3.
+set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash
+source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$0.runfiles/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e
+# --- end runfiles.bash initialization v3 ---
+
+for report in {{reports}}; do
+  report_file=$(rlocation "$report")
+  if [ -s "${report_file}" ]; then      
+        cat ${report_file}
+        exit 1
+  fi
+done