From 4bf9ae5a7e287c78d5be4dc7d5def93bdba8406b Mon Sep 17 00:00:00 2001 From: Avimitin Date: Mon, 29 Jul 2024 20:57:39 +0800 Subject: [PATCH] [ci] add VCS CI --- .github/workflows/vcs.yml | 105 +++++++++++++++++++++++++++++++++++ script/ci/src/Main.scala | 112 +++++++++++++++++++++----------------- tests/default.nix | 21 ++++++- 3 files changed, 186 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/vcs.yml diff --git a/.github/workflows/vcs.yml b/.github/workflows/vcs.yml new file mode 100644 index 0000000000..050c63e445 --- /dev/null +++ b/.github/workflows/vcs.yml @@ -0,0 +1,105 @@ +name: VCS +on: + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review + - labeled +env: + USER: runner + +# Cancel the current workflow when new commit pushed +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + gen-test-plan: + if: '! github.event.pull_request.draft' + name: "Generate test plan" + runs-on: [self-hosted, linux, nixos] + outputs: + testplan: ${{ steps.get-all-configs.outputs.out }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - id: "get-all-configs" + run: echo "out=$(nix run .#ci-helper generateTestPlan)" > $GITHUB_OUTPUT + + build-emulators: + name: "Build VCS Emulators" + needs: [gen-test-plan] + runs-on: [self-hosted, linux, nixos, BIGRAM] + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.gen-test-plan.outputs.testplan) }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: "Build verilator emulator" + run: | + nix build '.#t1.${{ matrix.config }}.ip.vcs-emu-trace' --impure --no-link --cores 64 + - name: "Build all testcases" + run: | + # Build testcases with vlen 1024 and vlen 4096 + nix build ".#t1.${{ matrix.config }}.cases.all" --max-jobs auto --no-link --cores 64 + + gen-matrix: + name: "Prepare for running testcases" + needs: [build-emulators] + runs-on: [self-hosted, linux, nixos, BIGRAM] + env: + RUNNERS: 70 + outputs: + ci-tests: ${{ steps.gen-matrix.outputs.matrix }} + steps: + # actions/checkout will use the "event" commit to checkout repository, + # which will lead to an unexpected issue that the "event" commit doesn't belongs to the repository, + # and causing the derivation build output cannot be cache correctly. + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - id: gen-matrix + name: "Generate test matrix" + run: | + echo -n matrix= >> "$GITHUB_OUTPUT" + nix run ".#ci-helper" -- generateCiMatrix --runnersAmount "$RUNNERS" >> "$GITHUB_OUTPUT" + + run-testcases: + name: "Run VCS" + needs: [gen-matrix] + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.gen-matrix.outputs.ci-tests) }} + runs-on: [self-hosted, linux, nixos] + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: "Run testcases" + run: | + nix run ".#ci-helper" -- runTests --jobs "${{ matrix.jobs }}" --testType "vcs" + + report: + name: "Report VCS CI result" + # Don't run report when: + # - user cancel ( we don't need report at this case ) + # - PR from outside repository ( we don't have permission to push commit into fork repository ) + if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }} + needs: [run-testcases] + runs-on: [self-hosted, linux, nixos] + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.head_ref }} + - name: "Print step summary" + run: | + nix run ".#ci-helper" -- postCI --failed-tests-file-path ./failed-tests.md --emuType vcs + cat ./failed-tests.md >> $GITHUB_STEP_SUMMARY diff --git a/script/ci/src/Main.scala b/script/ci/src/Main.scala index 7569955495..23a98e2007 100644 --- a/script/ci/src/Main.scala +++ b/script/ci/src/Main.scala @@ -144,7 +144,7 @@ object Main: @main def runTests( jobs: String, - dontBail: Flag = Flag(false) + testType: String = "verilator" ): Unit = if jobs == "" then Logger.info("No test found, exiting") @@ -157,32 +157,31 @@ object Main: val Array(config, caseName) = testName.split(",") println("\n") Logger.info( - s"${BOLD}[${index + 1}/${allJobs.length}]${RESET} Running test case $caseName with config $config" + s"${BOLD}[${index + 1}/${allJobs.length}]${RESET} Running VCS for test case $caseName with config $config" ) + val testAttr = testType.toLowerCase() match + case "verilator" => + s".#t1.$config.cases.$caseName.emu-result.with-offline" + case "vcs" => s".#t1.$config.cases.$caseName.emu-result.with-vcs" + case _ => Logger.fatal(s"Invalid test type ${testType}") val testResultPath = try os.Path( nixResolvePath( - s".#t1.$config.cases.$caseName.emu-result.with-offline" + testAttr, + if testType == "vcs" then Seq("--impure") else Seq() ) ) catch case _ => - Logger.error(s"Emulation for config $config, case $caseName fail") + Logger.error(s"VCS emulation for config $config, case $caseName fail") println("-" * 50) - println( - os.proc( - "nix", - "log", - s".#t1.$config.cases.$caseName.emu-result" - ).call() - .out - ) + println(os.proc("nix", "log", testAttr).call().out) println("-" * 50) Logger.fatal("Got error from emulation, exiting CI") - Logger.info("Checking RTL event with offline difftest") + Logger.info("Checking RTL event from VCS") val testSuccess = os.read(testResultPath / "offline-check-status").trim() == "0" if !testSuccess then @@ -193,10 +192,6 @@ object Main: val failedTests = findFailedTests() if failedTests.isEmpty then Logger.info(s"All tests passed") - else if dontBail.value then - Logger.error( - s"${BOLD}${failedTests.length} tests failed${RESET}" - ) else Logger.fatal( s"${BOLD}${failedTests.length} tests failed${RESET}" @@ -249,13 +244,17 @@ object Main: @arg( name = "cycle-update-file-path", doc = "specify the cycle update markdown file output path" - ) cycleUpdateFilePath: String + ) cycleUpdateFilePath: Option[String], + emuType: String = "verilator" ) = val failedTestsFile = os.Path(failedTestsFilePath, os.pwd) os.write.over(failedTestsFile, "## Failed Tests\n") - val cycleUpdateRecordFile = os.Path(cycleUpdateFilePath, os.pwd) - os.write.over(cycleUpdateRecordFile, "## Cycle Update\n") + if cycleUpdateFilePath.nonEmpty then + os.write.over( + os.Path(cycleUpdateFilePath.get, os.pwd), + "## Cycle Update\n" + ) os.walk(os.pwd / ".github" / "cases") .filter(_.last == "default.json") @@ -264,6 +263,11 @@ object Main: var cycleRecord = ujson.read(os.read(file)) Logger.info("Fetching CI results") + val resultAttr = emuType.toLowerCase() match + case "verilator" => + s".#t1.$config.cases._allEmuResult" + case "vcs" => s".#t1.$config.cases._allVCSEmuResult" + case _ => Logger.fatal(s"Invalid test type ${emuType}") val emuResultPath = os.Path(nixResolvePath(s".#t1.$config.cases._allEmuResult")) @@ -278,30 +282,34 @@ object Main: os.write.append(failedTestsFile, s"```text\n${journal}\n```\n") }) - Logger.info("Collecting cycle update info") - val perfCycleRegex = raw"total_cycles:\s(\d+)".r - val allCycleUpdates = os - .walk(emuResultPath) - .filter(path => path.last == "perf.txt") - .map(path => { - val cycle = os.read.lines(path).head match - case perfCycleRegex(cycle) => cycle.toInt - case _ => - throw new Exception("perf.txt file is not format as expected") - val caseName = path.segments.toSeq.reverse.drop(1).head - (caseName, cycle, cycleRecord.obj(caseName).num.toInt) - }) - .filter((_, newCycle, oldCycle) => newCycle != oldCycle) - .map: - case (caseName, newCycle, oldCycle) => - cycleRecord(caseName) = newCycle - if oldCycle == -1 then s"* 🆕 ${caseName}($config): NaN -> ${newCycle}" - else if oldCycle > newCycle then - s"* 🚀 $caseName($config): $oldCycle -> $newCycle" - else s"* 🐢 $caseName($config): $oldCycle -> $newCycle" - - os.write.append(cycleUpdateRecordFile, allCycleUpdates.mkString("\n")) - os.write.append(cycleUpdateRecordFile, "\n") + if cycleUpdateFilePath.nonEmpty then + Logger.info("Collecting cycle update info") + val perfCycleRegex = raw"total_cycles:\s(\d+)".r + val allCycleUpdates = os + .walk(emuResultPath) + .filter(path => path.last == "perf.txt") + .map(path => { + val cycle = os.read.lines(path).head match + case perfCycleRegex(cycle) => cycle.toInt + case _ => + throw new Exception("perf.txt file is not format as expected") + val caseName = path.segments.toSeq.reverse.drop(1).head + (caseName, cycle, cycleRecord.obj(caseName).num.toInt) + }) + .filter((_, newCycle, oldCycle) => newCycle != oldCycle) + .map: + case (caseName, newCycle, oldCycle) => + cycleRecord(caseName) = newCycle + if oldCycle == -1 then + s"* 🆕 ${caseName}($config): NaN -> ${newCycle}" + else if oldCycle > newCycle then + s"* 🚀 $caseName($config): $oldCycle -> $newCycle" + else s"* 🐢 $caseName($config): $oldCycle -> $newCycle" + + os.write.append( + os.Path(cycleUpdateFilePath.get, os.pwd), + allCycleUpdates.mkString("\n") + "\n" + ) os.write.over(file, ujson.write(cycleRecord, indent = 2)) end postCI @@ -316,14 +324,16 @@ object Main: println(ujson.write(Map("config" -> testPlans))) end generateTestPlan - def nixResolvePath(attr: String): String = + def nixResolvePath(attr: String, extraArgs: Seq[String] = Seq()): String = os.proc( - "nix", - "build", - "--no-link", - "--no-warn-dirty", - "--print-out-paths", - attr + Seq( + "nix", + "build", + "--no-link", + "--no-warn-dirty", + "--print-out-paths", + attr + ) ++ extraArgs ).call() .out .trim() diff --git a/tests/default.nix b/tests/default.nix index 83e5873ce5..11c6b73652 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -87,6 +87,25 @@ let in runCommand "catch-${configName}-all-emu-result-for-ci" { } script; + _allVCSEmuResult = + let + testPlan = builtins.fromJSON (lib.readFile ../.github/cases/${configName}/default.json); + # flattern the attr set to a list of test case derivations + # AttrSet (AttrSet Derivation) -> List Derivation + allCases = lib.filter (val: lib.isDerivation val && lib.hasAttr val.pname testPlan) + (lib.concatLists (map lib.attrValues (lib.attrValues scopeStripped))); + script = '' + mkdir -p $out + '' + (lib.concatMapStringsSep "\n" + (caseDrv: '' + _caseOutDir=$out/${caseDrv.pname} + mkdir -p "$_caseOutDir" + cp ${caseDrv.emu-result.with-vcs}/offline-check-* "$_caseOutDir"/ + '') + allCases); + in + runCommand "catch-${configName}-all-vcs-emu-result-for-ci" { } script; + all = let allCases = lib.filter lib.isDerivation @@ -103,4 +122,4 @@ let in runCommand "build-all-testcases" { } script; in -lib.recurseIntoAttrs (scopeStripped // { inherit all _allEmuResult; }) +lib.recurseIntoAttrs (scopeStripped // { inherit all _allEmuResult _allVCSEmuResult; })