Add FileCheck for unit testing

Update SETUP.md including brief instructions on FileCheck.
chipsalliance · Sep 29, 2023 · e5c9813 · e5c9813
1 parent 4186303
commit e5c9813
Show file tree

Hide file tree

Showing 6 changed files with 145 additions and 31 deletions.
diff --git a/.github/workflows/install-filecheck/action.yml b/.github/workflows/install-filecheck/action.yml
@@ -0,0 +1,27 @@
+name: Install FileCheck
+
+inputs:
+  version:
+    description: 'version to install'
+    required: false
+    default: 'FileCheck-16.0.6-test'
+
+runs:
+  using: composite
+  steps:
+    - id: cache-filecheck
+      uses: actions/cache@v3
+      with:
+        path: FileCheck
+        key: filecheck-${{ runner.os }}-${{ inputs.version }}
+
+    - shell: bash
+      if: steps.cache-filecheck.outputs.cache-hit != 'true'
+      run: |
+        mkdir -p filecheck/bin
+        wget -q https://github.com/jackkoenig/FileCheck/releases/download/${{ inputs.version }}/FileCheck-linux-x64
+        chmod +x FileCheck-linux-x64
+        mv FileCheck-linux-x64 filecheck/bin/FileCheck
+
+    - shell: bash
+      run: echo "$(pwd)/filecheck/bin" >> $GITHUB_PATH
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -49,6 +49,11 @@ jobs:
         uses: ./.github/workflows/install-espresso
         with:
           version: ${{ inputs.espresso }}
+      - name: Install FileCheck
+        run: |
+          sudo apt-get update
+          sudo apt-get install llvm-12-tools
+          echo "/usr/lib/llvm-12/bin" >> $GITHUB_PATH
       - name: Setup Scala
         uses: actions/setup-java@v3
         with:

diff --git a/SETUP.md b/SETUP.md
@@ -7,6 +7,10 @@ For a minimal setup, you only need to install [SBT (the Scala Build Tool)](http:
 
 [Verilator](https://www.veripool.org/wiki/verilator) is installation is required to simulate your Verilog designs.
 
+[FileCheck](https://llvm.org/docs/CommandGuide/FileCheck.html) is used by many of the unit tests.
+Please see the FileCheck link for documentation about FileCheck's syntax with examples.
+Alternatively you can search the tests for examples by searching for `CHECK:` which is used to tell FileCheck to attempt to match the following text in the generated output.
+
 ## Ubuntu Linux
 
 1.  Install the JVM
@@ -20,20 +24,20 @@ For a minimal setup, you only need to install [SBT (the Scala Build Tool)](http:
 
     Choose whatever version is being [used in continuous integration](.github/workflows/install-circt/action.yml)
     ```bash
-    wget -q -O - https://github.com/llvm/circt/releases/download/firtool-1.38.0/firrtl-bin-ubuntu-20.04.tar.gz | tar -zx
+    wget -q -O - https://github.com/llvm/circt/releases/download/firtool-1.56.1/circt-full-shared-linux-x64.tar.gz | tar -zx
     ```
-    This will give you a directory called `firtool-1.38.0` containing the firtool binary, add this to your PATH as appropriate.
+    This will give you a directory called `firtool-1.56.1` containing the firtool binary, add this to your PATH as appropriate.
     ```bash
-    export PATH=$PATH:$PWD/firtool-1.38.0/bin
+    export PATH=$PATH:$PWD/firtool-1.56.1/bin
     ```
     Alternatively, you can install the binary to a standard location by simply moving the binary (if you have root access).
     ```bash
-    mv firtool-1.38.0/bin/firtool /usr/local/bin/
+    mv firtool-1.56.1/bin/firtool /usr/local/bin/
     ```
 
 
-2.  Install Verilator.
-    We currently recommend Verilator version v4.226.
+1.  Install Verilator.
+    We recommend relatively recent verilator
     Follow these instructions to compile it from source.
 
     1.  Install prerequisites (if not installed already):
@@ -49,7 +53,7 @@ For a minimal setup, you only need to install [SBT (the Scala Build Tool)](http:
     3.  In the Verilator repository directory, check out a known good version:
         ```bash
         git pull
-        git checkout v4.226
+        git checkout v5.004
         ```
 
     4.  In the Verilator repository directory, build and install:
@@ -61,6 +65,27 @@ For a minimal setup, you only need to install [SBT (the Scala Build Tool)](http:
         sudo make install
         ```
 
+1.  Install FileCheck.
+    FileCheck can usually be found in llvm-\*-tools packages, eg.
+    ```
+    sudo apt-get install llvm-12-tools
+    export PATH=$PATH:/usr/lib/llvm-12/bin
+    ```
+
+    You can alternatively download a statically-ish linked binary from https://github.com/jackkoenig/FileCheck
+    ```
+    mkdir filecheck
+    cd filecheck
+    wget -q https://github.com/jackkoenig/FileCheck/releases/download/FileCheck-16.0.6/FileCheck-linux-x64
+    mv FileCheck-linux-x64 FileCheck
+    chmod +x FileCheck
+    export PATH=$PATH:$PWD
+    ```
+    Similarly to firtool, you can install the binary to a more standard location by moving it.
+    ```
+    mv FileCheck /usr/local/bin
+    ```
+
 ## Arch Linux
 1.  Install Verilator and SBT
     ```bash
@@ -88,13 +113,28 @@ There are no issues with generating Verilog from Chisel, which can be pushed to
 1.  Install firtool
 
     ```bash
-    wget -q -O - https://github.com/llvm/circt/releases/download/firtool-1.38.0/firrtl-bin-macos-11.tar.gz | tar -zx
+    wget -q -O - https://github.com/llvm/circt/releases/download/firtool-1.56.1/circt-full-shared-macos-x64.tar.gz | tar -zx
     ```
-    This will give you a directory called `firtool-1.38.0` containing the firtool binary, add this to your PATH as appropriate.
+    This will give you a directory called `firtool-1.56.1` containing the firtool binary, add this to your PATH as appropriate.
     ```bash
-    export PATH=$PATH:$PWD/firtool-1.38.0/bin
+    export PATH=$PATH:$PWD/firtool-1.56.1/bin
     ```
     Alternatively, you can install the binary to a standard location by simply moving the binary.
     ```bash
-    mv firtool-1.38.0/bin/firtool /usr/local/bin/
+    mv firtool-1.56.1/bin/firtool /usr/local/bin/
+    ```
+
+1.  Install FileCheck.
+    You can download a statically-ish linked binary from https://github.com/jackkoenig/FileCheck
+    ```
+    mkdir filecheck
+    cd filecheck
+    wget -q https://github.com/jackkoenig/FileCheck/releases/download/FileCheck-16.0.6/FileCheck-macos-x64
+    mv FileCheck-macos-x64 FileCheck
+    chmod +x FileCheck
+    export PATH=$PATH:$PWD
+    ```
+    Similarly to firtool, you can install the binary to a more standard location by moving it.
+    ```
+    mv FileCheck /usr/local/bin
     ```
diff --git a/src/test/scala/chiselTests/ChiselEnum.scala b/src/test/scala/chiselTests/ChiselEnum.scala
@@ -361,7 +361,7 @@ class IsOneOfTester extends BasicTester {
   stop()
 }
 
-class ChiselEnumSpec extends ChiselFlatSpec with Utils {
+class ChiselEnumSpec extends ChiselFlatSpec with Utils with FileCheck {
 
   behavior.of("ChiselEnum")
 
@@ -482,9 +482,11 @@ class ChiselEnumSpec extends ChiselFlatSpec with Utils {
       val out = IO(Output(MyEnum()))
       out := MyEnum(in)
     }
-    val (log, _) = grabLog(ChiselStage.emitCHIRRTL(new MyModule))
-    log should include("warn")
-    log should include("Casting non-literal UInt")
+    elaborateAndFileCheckOutAndErr(new MyModule)(
+      """| CHECK:      [W001] Casting non-literal UInt to [[enum:[a-zA-Z0-9_$.]+]].
+         | CHECK-SAME: You can use [[enum]].safe to cast without this warning.
+         |""".stripMargin
+    )
   }
 
   it should "NOT warn if the Enum is total" in {

diff --git a/src/test/scala/chiselTests/ChiselSpec.scala b/src/test/scala/chiselTests/ChiselSpec.scala
@@ -195,6 +195,50 @@ trait ChiselRunners extends Assertions {
   }
 }
 
+trait FileCheck extends BeforeAndAfterEachTestData { this: Suite =>
+  import scala.Console.{withErr, withOut}
+
+  private def sanitize(n: String): String = n.replaceAll(" ", "_").replaceAll("\\W+", "")
+
+  private val testRunDir: os.Path = os.pwd / os.RelPath(BackendCompilationUtilities.TestDirectory)
+  private val suiteDir:   os.Path = testRunDir / sanitize(suiteName)
+  private var checkFile:  Option[os.Path] = None
+
+  override def beforeEach(testData: TestData): Unit = {
+    // TODO check that these are always available
+    val nameDir = suiteDir / sanitize(testData.name)
+    os.makeDir.all(nameDir)
+    checkFile = Some(nameDir / s"${sanitize(testData.text)}.check")
+    super.beforeEach(testData) // To be stackable, must call super.beforeEach
+  }
+
+  override def afterEach(testData: TestData): Unit = {
+    checkFile = None
+    super.afterEach(testData) // To be stackable, must call super.beforeEach
+  }
+
+  /** Elaborate a Module to FIRRTL and check the FIRRTL with FileCheck */
+  def generateFirrtlAndFileCheck(t: => RawModule, fileCheckArgs: String*)(check: String): Unit = {
+    // Filecheck needs the thing to check in a file
+    os.write.over(checkFile.get, check)
+    val result = ChiselStage.emitCHIRRTL(t)
+    val extraArgs = os.Shellable(fileCheckArgs)
+    os.proc("FileCheck", checkFile.get, extraArgs).call(stdin = result)
+  }
+
+  /** Elaborate a Module, capture the stdout and stderr, check stdout and stderr with FileCheck */
+  def elaborateAndFileCheckOutAndErr(t: => RawModule, fileCheckArgs: String*)(check: String): Unit = {
+    // Filecheck needs the thing to check in a file
+    os.write.over(checkFile.get, check)
+    val outStream = new ByteArrayOutputStream()
+    // Discard CHIRRTL
+    withOut(outStream)(withErr(outStream)(ChiselStage.emitCHIRRTL(t)))
+    val result = outStream.toString
+    val extraArgs = os.Shellable(fileCheckArgs)
+    os.proc("FileCheck", checkFile.get, extraArgs).call(stdin = result)
+  }
+}
+
 /** Spec base class for BDD-style testers. */
 abstract class ChiselFlatSpec extends AnyFlatSpec with ChiselRunners with Matchers
 

diff --git a/src/test/scala/chiselTests/DecoupledSpec.scala b/src/test/scala/chiselTests/DecoupledSpec.scala
@@ -6,7 +6,7 @@ import chisel3._
 import circt.stage.ChiselStage
 import chisel3.util.Decoupled
 
-class DecoupledSpec extends ChiselFlatSpec {
+class DecoupledSpec extends ChiselFlatSpec with FileCheck {
   "Decoupled() and Decoupled.empty" should "give DecoupledIO with empty payloads" in {
     ChiselStage.emitCHIRRTL(new Module {
       val io = IO(new Bundle {
@@ -19,21 +19,17 @@ class DecoupledSpec extends ChiselFlatSpec {
   }
 
   "Decoupled.map" should "apply a function to a wrapped Data" in {
-    val chirrtl = ChiselStage
-      .emitCHIRRTL(new Module {
-        val enq = IO(Flipped(Decoupled(UInt(8.W))))
-        val deq = IO(Decoupled(UInt(8.W)))
-        deq <> enq.map(_ + 1.U)
-      })
-
-    // Check for data assignment
-    chirrtl should include("""node _deq_map_bits_T = add(enq.bits, UInt<1>(0h1)""")
-    chirrtl should include("""node _deq_map_bits = tail(_deq_map_bits_T, 1)""")
-    chirrtl should include("""connect _deq_map.bits, _deq_map_bits""")
-    chirrtl should include("""connect deq, _deq_map""")
-
-    // Check for back-pressure (ready signal is driven in the opposite direction of bits + valid)
-    chirrtl should include("""connect enq.ready, _deq_map.ready""")
+    generateFirrtlAndFileCheck(new Module {
+      val enq = IO(Flipped(Decoupled(UInt(8.W))))
+      val deq = IO(Decoupled(UInt(8.W)))
+      deq <> enq.map(_ + 1.U)
+    })("""|CHECK: node [[node1:[a-zA-Z0-9_]+]] = add(enq.bits, UInt<1>(0h1))
+          |CHECK: node [[node2:[a-zA-Z0-9_]+]] = tail([[node1]], 1)
+          |CHECK: connect [[result:[a-zA-Z0-9_]+]].bits, [[node2]]
+          |# Check for back-pressure (ready signal is driven in the opposite direction of bits + valid)
+          |CHECK: connect enq.ready, [[result]].ready
+          |CHECK: connect deq, [[result]]
+          |""".stripMargin)
   }
 
   "Decoupled.map" should "apply a function to a wrapped Bundle" in {