From 8fed81b36bb9b533ea15687fc721f63713a327f7 Mon Sep 17 00:00:00 2001 From: Jiuyang Liu Date: Mon, 29 Jul 2024 03:25:52 +0800 Subject: [PATCH] [nix] refactor nix for vcs+verilator emulator --- .github/workflows/pr.yml | 4 +- difftest/online_vcs/default.nix | 8 ++- difftest/{default.nix => verilator.nix} | 3 +- nix/overlay.nix | 4 ++ nix/pkgs/vcs-fhs-env.nix | 91 +++++++++++++++++++++++++ nix/t1/default.nix | 68 +++++++++++++----- nix/t1/elaborate.nix | 2 +- nix/t1/rtl.nix | 11 +-- nix/t1/vcs.nix | 67 ++++++++++++++++++ tests/default.nix | 6 +- tests/make-emu-result.nix | 10 +-- 11 files changed, 235 insertions(+), 39 deletions(-) rename difftest/{default.nix => verilator.nix} (95%) create mode 100644 nix/pkgs/vcs-fhs-env.nix create mode 100644 nix/t1/vcs.nix diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index eaa9321b4d..94a3f19cc2 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -42,7 +42,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - name: "Build verilator emulator" run: | - nix build '.#t1.${{ matrix.config }}.ip.difftest' -L --no-link --cores 64 + nix build '.#t1.${{ matrix.config }}.ip.verilator-emu' -L --no-link --cores 64 - name: "Build all testcases" run: | # Build testcases with vlen 1024 and vlen 4096 @@ -81,7 +81,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} - name: "Build verilator emulator with trace" - run: nix build '.#t1.${{ matrix.config }}.ip.difftest-trace' -L --no-link --cores 64 + run: nix build '.#t1.${{ matrix.config }}.ip.verilator-emu-trace' -L --no-link --cores 64 test-emit: if: '! github.event.pull_request.draft' diff --git a/difftest/online_vcs/default.nix b/difftest/online_vcs/default.nix index 1800e962c8..b1400fe1fb 100644 --- a/difftest/online_vcs/default.nix +++ b/difftest/online_vcs/default.nix @@ -2,7 +2,7 @@ , elaborateConfig , rustPlatform , libspike -, spike_interfaces +, libspike_interfaces , enable-trace ? false , vcStaticHome }: @@ -29,9 +29,13 @@ rustPlatform.buildRustPackage { env = { VCS_LIB_DIR = "${vcStaticHome}/vcs-mx/linux64/lib"; SPIKE_LIB_DIR = "${libspike}/lib"; - SPIKE_INTERFACES_LIB_DIR = "${spike_interfaces}/lib"; + SPIKE_INTERFACES_LIB_DIR = "${libspike_interfaces}/lib"; DESIGN_VLEN = elaborateConfig.parameter.vLen; DESIGN_DLEN = elaborateConfig.parameter.dLen; + SPIKE_ISA_STRING = + "rv32gc" + + (builtins.concatStringsSep "_" elaborateConfig.parameter.extensions) + + "_Zvl${toString elaborateConfig.parameter.vLen}b"; }; cargoLock = { diff --git a/difftest/default.nix b/difftest/verilator.nix similarity index 95% rename from difftest/default.nix rename to difftest/verilator.nix index 876db0163c..9fb575a354 100644 --- a/difftest/default.nix +++ b/difftest/verilator.nix @@ -18,7 +18,8 @@ let self = rustPlatform.buildRustPackage { - name = "difftest"; + name = "verilator-emu" + (lib.optionalString verilated.enable-trace "-trace"); + src = with lib.fileset; toSource { root = ./.; fileset = unions [ diff --git a/nix/overlay.nix b/nix/overlay.nix index b25f766b8e..9306cffd7f 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -24,6 +24,10 @@ rec { circt-full = final.callPackage ./pkgs/circt-full.nix { }; rvv-codegen = final.callPackage ./pkgs/rvv-codegen.nix { }; add-determinism = final.callPackage ./pkgs/add-determinism { }; # faster strip-undetereminism + # Using VCS need to set VC_STATIC_HOME and SNPSLMD_LICENSE_FILE to impure env, and add sandbox dir to VC_STATIC_HOME + vcStaticHome = builtins.getEnv "VC_STATIC_HOME"; + snpslmdLicenseFile = builtins.getEnv "SNPSLMD_LICENSE_FILE"; + vcs-fhs-env = final.callPackage ./pkgs/vcs-fhs-env.nix { }; mill = let jre = final.jdk21; in (prev.mill.override { inherit jre; }).overrideAttrs (_: { diff --git a/nix/pkgs/vcs-fhs-env.nix b/nix/pkgs/vcs-fhs-env.nix new file mode 100644 index 0000000000..5cf0be588c --- /dev/null +++ b/nix/pkgs/vcs-fhs-env.nix @@ -0,0 +1,91 @@ +{ buildFHSEnv +, vcStaticHome +, snpslmdLicenseFile +}: +buildFHSEnv { + name = "vcs-fhs-env"; + + profile = '' + [ ! -e "${vcStaticHome}" ] && echo "env VC_STATIC_HOME not set" && exit 1 + [ ! -d "${vcStaticHome}" ] && echo "VC_STATIC_HOME not accessible" && exit 1 + [ -z "${snpslmdLicenseFile}" ] && echo "env SNPS LICENSE not set" && exit 1 + export VC_STATIC_HOME=${vcStaticHome} + + export TCL_TZ=UTC + export VC_STATIC_HOME=$VC_STATIC_HOME + export VCS_HOME=$VC_STATIC_HOME/vcs-mx + export VCS_TARGET_ARCH=amd64 + export VCS_ARCH_OVERRIDE=linux + export VERDI_HOME=$VC_STATIC_HOME/verdi + export NOVAS_HOME=$VC_STATIC_HOME/verdi + export SPYGLASS_HOME=$VC_STATIC_HOME/SG_COMPAT/SPYGLASS_HOME + export SNPS_VERDI_CBUG_LCA=1 + export SNPSLMD_LICENSE_FILE=${snpslmdLicenseFile} + + export PATH=$VC_STATIC_HOME/bin:$PATH + export PATH=$VC_STATIC_HOME/verdi/bin:$PATH + export PATH=$VC_STATIC_HOME/vcs-mx/bin:$PATH + export PATH=$VC_STATIC_HOME/SG_COMPAT/SPYGLASS_HOME/bin:$PATH + + export LD_LIBRARY_PATH=/usr/lib64/ + export LD_LIBRARY_PATH=$VC_STATIC_HOME/verdi/share/PLI/lib/LINUX64:$LD_LIBRARY_PATH + export LD_LIBRARY_PATH=$VC_STATIC_HOME/verdi/share/NPI/lib/LINUX64:$LD_LIBRARY_PATH + + export _oldVcsEnvPath="$PATH" + preHook() { + PATH="$PATH:$_oldVcsEnvPath" + } + export -f preHook + ''; + targetPkgs = (ps: with ps; [ + libGL + util-linux + libxcrypt-legacy + coreutils-full + ncurses5 + gmp5 + bzip2 + glib + bc + time + elfutils + ncurses5 + e2fsprogs + cyrus_sasl + expat + sqlite + nssmdns + (libkrb5.overrideAttrs rec { + version = "1.18.2"; + src = fetchurl { + url = "https://kerberos.org/dist/krb5/${lib.versions.majorMinor version}/krb5-${version}.tar.gz"; + hash = "sha256-xuTJ7BqYFBw/XWbd8aE1VJBQyfq06aRiDumyIIWHOuA="; + }; + sourceRoot = "krb5-${version}/src"; + }) + (gnugrep.overrideAttrs rec { + version = "3.1"; + doCheck = false; + src = fetchurl { + url = "mirror://gnu/grep/grep-${version}.tar.xz"; + hash = "sha256-22JcerO7PudXs5JqXPqNnhw5ka0kcHqD3eil7yv3oH4="; + }; + }) + keyutils + graphite2 + libpulseaudio + libxml2 + gcc + gnumake + xorg.libX11 + xorg.libXft + xorg.libXScrnSaver + xorg.libXext + xorg.libxcb + xorg.libXau + xorg.libXrender + xorg.libXcomposite + xorg.libXi + zlib + ]); +} diff --git a/nix/t1/default.nix b/nix/t1/default.nix index 1e12d07d5a..873db7ac3c 100644 --- a/nix/t1/default.nix +++ b/nix/t1/default.nix @@ -51,7 +51,7 @@ lib.makeScope newScope elaborateConfigJson = configPath; elaborateConfig = builtins.fromJSON (lib.readFile configPath); - cases = innerSelf.callPackage ../../tests { difftest = ip.difftest; difftest-trace = ip.difftest-trace; }; + cases = innerSelf.callPackage ../../tests { verilator-emu = ip.verilator-emu; verilator-emu-trace = ip.verilator-emu-trace; }; # for the convenience to use x86 cases on non-x86 machines, avoiding the extra build time cases-x86 = @@ -62,27 +62,63 @@ lib.makeScope newScope ip = rec { recurseForDerivations = true; - elaborate = innerSelf.callPackage ./elaborate.nix { target = "ip"; /* use-binder = true; */ }; + # T1 RTL. + elaborate = innerSelf.callPackage ./elaborate.nix { target = "ip"; }; mlirbc = innerSelf.callPackage ./mlirbc.nix { inherit elaborate; }; - rtl = innerSelf.callPackage ./rtl.nix { inherit mlirbc; }; - + rtl = innerSelf.callPackage ./rtl.nix { + inherit mlirbc; + mfcArgs = lib.escapeShellArgs [ + "-O=release" + "--disable-all-randomization" + "--split-verilog" + "--preserve-values=all" + "--strip-debug-info" + "--strip-fir-debug-info" + "--verification-flavor=sva" + "--lowering-options=verifLabels,omitVersionComment,emittedLineLength=240,locationInfoStyle=none" + ]; + }; omreader = self.omreader-unwrapped.mkWrapper { inherit mlirbc; }; - om = innerSelf.callPackage ./om.nix { inherit mlirbc; }; - emu-elaborate = innerSelf.callPackage ./elaborate.nix { target = "ipemu"; /* use-binder = true; */ }; + emu-elaborate = innerSelf.callPackage ./elaborate.nix { target = "ipemu"; }; emu-mlirbc = innerSelf.callPackage ./mlirbc.nix { elaborate = emu-elaborate; }; - emu-omreader = self.omreader-unwrapped.mkWrapper { mlirbc = emu-mlirbc; }; - emu-rtl = innerSelf.callPackage ./rtl.nix { mlirbc = emu-mlirbc; }; - - verilated = innerSelf.callPackage ./verilated.nix { rtl = emu-rtl; }; - verilated-trace = innerSelf.callPackage ./verilated.nix { rtl = emu-rtl; enable-trace = true; }; - - emu = innerSelf.callPackage ./ipemu.nix { rtl = ip.emu-rtl; stdenv = moldStdenv; }; - emu-trace = innerSelf.callPackage ./ipemu.nix { rtl = emu-rtl; stdenv = moldStdenv; do-trace = true; }; - difftest = innerSelf.callPackage ../../difftest/default.nix { inherit verilated; }; - difftest-trace = innerSelf.callPackage ../../difftest/default.nix { verilated = verilated-trace; }; + # T1 Verilator Emulator + verilator-emu-omreader = self.omreader-unwrapped.mkWrapper { mlirbc = emu-mlirbc; }; + verilator-emu-rtl = innerSelf.callPackage ./rtl.nix { + mlirbc = emu-mlirbc; + mfcArgs = lib.escapeShellArgs [ + "-O=release" + "--split-verilog" + "--preserve-values=all" + "--verification-flavor=if-else-fatal" + "--lowering-options=verifLabels,omitVersionComment" + "--strip-debug-info" + ]; + }; + verilator-emu-rtl-verilated = innerSelf.callPackage ./verilated.nix { rtl = verilator-emu-rtl; stdenv = moldStdenv; }; + verilator-emu-rtl-verilated-trace = innerSelf.callPackage ./verilated.nix { rtl = verilator-emu-rtl; stdenv = moldStdenv; enable-trace = true; }; + verilator-emu = innerSelf.callPackage ../../difftest/verilator.nix { verilated = verilator-emu-rtl-verilated; }; + verilator-emu-trace = innerSelf.callPackage ../../difftest/verilator.nix { verilated = verilator-emu-rtl-verilated-trace; }; + + # T1 VCS Emulator + vcs-emu-omreader = self.omreader-unwrapped.mkWrapper { mlirbc = emu-mlirbc; }; + vcs-emu-rtl = innerSelf.callPackage ./rtl.nix { + mlirbc = emu-mlirbc; + mfcArgs = lib.escapeShellArgs [ + "-O=release" + "--split-verilog" + "--preserve-values=all" + "--verification-flavor=sva" + "--lowering-options=verifLabels,omitVersionComment" + "--strip-debug-info" + ]; + }; + vcs-dpi-lib = innerSelf.callPackage ../../difftest/online_vcs { }; + vcs-dpi-lib-trace = vcs-dpi-lib.override { enable-trace = true; }; + vcs-emu-compiled = innerSelf.callPackage ./vcs.nix { inherit vcs-dpi-lib; rtl = vcs-emu-rtl; }; + vcs-emu-compiled-trace = innerSelf.callPackage ./vcs.nix { vcs-dpi-lib = vcs-dpi-lib-trace; rtl = vcs-emu-rtl; }; }; subsystem = rec { diff --git a/nix/t1/elaborate.nix b/nix/t1/elaborate.nix index 691a701cd3..a531ac1dff 100644 --- a/nix/t1/elaborate.nix +++ b/nix/t1/elaborate.nix @@ -12,7 +12,7 @@ }: assert lib.assertMsg - (lib.elem target [ "ip" "ipemu" "subsystem" "subsystememu" ]) + (lib.elem target [ "ip" "ipemu" ]) "Unknown elaborate target ${target}"; let diff --git a/nix/t1/rtl.nix b/nix/t1/rtl.nix index 9f3beed965..5b9d43369d 100644 --- a/nix/t1/rtl.nix +++ b/nix/t1/rtl.nix @@ -1,19 +1,12 @@ { stdenvNoCC , lib - +, mfcArgs , circt , mlirbc }: let - mfcArgs = lib.escapeShellArgs [ - "-O=debug" - "--split-verilog" - "--preserve-values=named" - "--lowering-options=verifLabels,omitVersionComment" - "--strip-debug-info" - ]; - fixupFilelist = lib.elem mlirbc.elaborateTarget [ "ipemu" "subsystememu" ]; + fixupFilelist = lib.elem mlirbc.elaborateTarget [ "ipemu" ]; in stdenvNoCC.mkDerivation { name = "t1-${mlirbc.elaborateConfig}-${mlirbc.elaborateTarget}-rtl"; diff --git a/nix/t1/vcs.nix b/nix/t1/vcs.nix new file mode 100644 index 0000000000..5d1668ec38 --- /dev/null +++ b/nix/t1/vcs.nix @@ -0,0 +1,67 @@ +{ lib +, bash +, stdenv +, configName +, rtl +, vcs-dpi-lib +, vcs-fhs-env +}: + +stdenv.mkDerivation { + name = "${configName}-vcs"; + + # require license + __noChroot = true; + dontPatchELF = true; + + src = rtl; + + buildPhase = '' + runHook preBuild + + echo "[nix] running VCS" + fhsBash=${vcs-fhs-env}/bin/vcs-fhs-env + VERDI_HOME=$("$fhsBash" -c "printenv VERDI_HOME") + "$fhsBash" vcs \ + -sverilog \ + -full64 \ + -timescale=1ns/1ps \ + -P $VERDI_HOME/share/PLI/VCS/LINUX64/novas.tab $VERDI_HOME/share/PLI/VCS/LINUX64/pli.a \ + ${lib.optionalString vcs-dpi-lib.enable-trace '' + -debug_access+pp+dmptf+thread \ + -kdb=common_elab,hgldd_all''} \ + -file filelist.f \ + ${vcs-dpi-lib}/lib/libdpi.a \ + -o t1-vcs-simulator + + runHook postBuild + ''; + + passthru = { + inherit (vcs-dpi-lib) enable-trace; + }; + + shellHook = '' + echo "[nix] entering fhs env" + ${vcs-fhs-env}/bin/vcs-fhs-env + ''; + + installPhase = '' + runHook preInstall + + mkdir -p $out/bin $out/lib + cp t1-vcs-simulator $out/lib + cp -r t1-vcs-simulator.daidir $out/lib + + # We need to carefully handle string escape here, so don't use makeWrapper + tee $out/bin/t1-vcs-simulator < $out/offline-check-journal