Skip to content

Commit

Permalink
[nix] refactor nix for vcs+verilator emulator
Browse files Browse the repository at this point in the history
  • Loading branch information
sequencer committed Jul 29, 2024
1 parent b5da9fe commit 8fed81b
Show file tree
Hide file tree
Showing 11 changed files with 235 additions and 39 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:
ref: ${{ github.event.pull_request.head.sha }}
- name: "Build verilator emulator"
run: |
nix build '.#t1.${{ matrix.config }}.ip.difftest' -L --no-link --cores 64
nix build '.#t1.${{ matrix.config }}.ip.verilator-emu' -L --no-link --cores 64
- name: "Build all testcases"
run: |
# Build testcases with vlen 1024 and vlen 4096
Expand Down Expand Up @@ -81,7 +81,7 @@ jobs:
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: "Build verilator emulator with trace"
run: nix build '.#t1.${{ matrix.config }}.ip.difftest-trace' -L --no-link --cores 64
run: nix build '.#t1.${{ matrix.config }}.ip.verilator-emu-trace' -L --no-link --cores 64

test-emit:
if: '! github.event.pull_request.draft'
Expand Down
8 changes: 6 additions & 2 deletions difftest/online_vcs/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
, elaborateConfig
, rustPlatform
, libspike
, spike_interfaces
, libspike_interfaces
, enable-trace ? false
, vcStaticHome
}:
Expand All @@ -29,9 +29,13 @@ rustPlatform.buildRustPackage {
env = {
VCS_LIB_DIR = "${vcStaticHome}/vcs-mx/linux64/lib";
SPIKE_LIB_DIR = "${libspike}/lib";
SPIKE_INTERFACES_LIB_DIR = "${spike_interfaces}/lib";
SPIKE_INTERFACES_LIB_DIR = "${libspike_interfaces}/lib";
DESIGN_VLEN = elaborateConfig.parameter.vLen;
DESIGN_DLEN = elaborateConfig.parameter.dLen;
SPIKE_ISA_STRING =
"rv32gc" +
(builtins.concatStringsSep "_" elaborateConfig.parameter.extensions)
+ "_Zvl${toString elaborateConfig.parameter.vLen}b";
};

cargoLock = {
Expand Down
3 changes: 2 additions & 1 deletion difftest/default.nix → difftest/verilator.nix
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@

let
self = rustPlatform.buildRustPackage {
name = "difftest";
name = "verilator-emu" + (lib.optionalString verilated.enable-trace "-trace");

src = with lib.fileset; toSource {
root = ./.;
fileset = unions [
Expand Down
4 changes: 4 additions & 0 deletions nix/overlay.nix
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ rec {
circt-full = final.callPackage ./pkgs/circt-full.nix { };
rvv-codegen = final.callPackage ./pkgs/rvv-codegen.nix { };
add-determinism = final.callPackage ./pkgs/add-determinism { }; # faster strip-undetereminism
# Using VCS need to set VC_STATIC_HOME and SNPSLMD_LICENSE_FILE to impure env, and add sandbox dir to VC_STATIC_HOME
vcStaticHome = builtins.getEnv "VC_STATIC_HOME";
snpslmdLicenseFile = builtins.getEnv "SNPSLMD_LICENSE_FILE";
vcs-fhs-env = final.callPackage ./pkgs/vcs-fhs-env.nix { };

mill = let jre = final.jdk21; in
(prev.mill.override { inherit jre; }).overrideAttrs (_: {
Expand Down
91 changes: 91 additions & 0 deletions nix/pkgs/vcs-fhs-env.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
{ buildFHSEnv
, vcStaticHome
, snpslmdLicenseFile
}:
buildFHSEnv {
name = "vcs-fhs-env";

profile = ''
[ ! -e "${vcStaticHome}" ] && echo "env VC_STATIC_HOME not set" && exit 1
[ ! -d "${vcStaticHome}" ] && echo "VC_STATIC_HOME not accessible" && exit 1
[ -z "${snpslmdLicenseFile}" ] && echo "env SNPS LICENSE not set" && exit 1
export VC_STATIC_HOME=${vcStaticHome}
export TCL_TZ=UTC
export VC_STATIC_HOME=$VC_STATIC_HOME
export VCS_HOME=$VC_STATIC_HOME/vcs-mx
export VCS_TARGET_ARCH=amd64
export VCS_ARCH_OVERRIDE=linux
export VERDI_HOME=$VC_STATIC_HOME/verdi
export NOVAS_HOME=$VC_STATIC_HOME/verdi
export SPYGLASS_HOME=$VC_STATIC_HOME/SG_COMPAT/SPYGLASS_HOME
export SNPS_VERDI_CBUG_LCA=1
export SNPSLMD_LICENSE_FILE=${snpslmdLicenseFile}
export PATH=$VC_STATIC_HOME/bin:$PATH
export PATH=$VC_STATIC_HOME/verdi/bin:$PATH
export PATH=$VC_STATIC_HOME/vcs-mx/bin:$PATH
export PATH=$VC_STATIC_HOME/SG_COMPAT/SPYGLASS_HOME/bin:$PATH
export LD_LIBRARY_PATH=/usr/lib64/
export LD_LIBRARY_PATH=$VC_STATIC_HOME/verdi/share/PLI/lib/LINUX64:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=$VC_STATIC_HOME/verdi/share/NPI/lib/LINUX64:$LD_LIBRARY_PATH
export _oldVcsEnvPath="$PATH"
preHook() {
PATH="$PATH:$_oldVcsEnvPath"
}
export -f preHook
'';
targetPkgs = (ps: with ps; [
libGL
util-linux
libxcrypt-legacy
coreutils-full
ncurses5
gmp5
bzip2
glib
bc
time
elfutils
ncurses5
e2fsprogs
cyrus_sasl
expat
sqlite
nssmdns
(libkrb5.overrideAttrs rec {
version = "1.18.2";
src = fetchurl {
url = "https://kerberos.org/dist/krb5/${lib.versions.majorMinor version}/krb5-${version}.tar.gz";
hash = "sha256-xuTJ7BqYFBw/XWbd8aE1VJBQyfq06aRiDumyIIWHOuA=";
};
sourceRoot = "krb5-${version}/src";
})
(gnugrep.overrideAttrs rec {
version = "3.1";
doCheck = false;
src = fetchurl {
url = "mirror://gnu/grep/grep-${version}.tar.xz";
hash = "sha256-22JcerO7PudXs5JqXPqNnhw5ka0kcHqD3eil7yv3oH4=";
};
})
keyutils
graphite2
libpulseaudio
libxml2
gcc
gnumake
xorg.libX11
xorg.libXft
xorg.libXScrnSaver
xorg.libXext
xorg.libxcb
xorg.libXau
xorg.libXrender
xorg.libXcomposite
xorg.libXi
zlib
]);
}
68 changes: 52 additions & 16 deletions nix/t1/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ lib.makeScope newScope
elaborateConfigJson = configPath;
elaborateConfig = builtins.fromJSON (lib.readFile configPath);

cases = innerSelf.callPackage ../../tests { difftest = ip.difftest; difftest-trace = ip.difftest-trace; };
cases = innerSelf.callPackage ../../tests { verilator-emu = ip.verilator-emu; verilator-emu-trace = ip.verilator-emu-trace; };

# for the convenience to use x86 cases on non-x86 machines, avoiding the extra build time
cases-x86 =
Expand All @@ -62,27 +62,63 @@ lib.makeScope newScope
ip = rec {
recurseForDerivations = true;

elaborate = innerSelf.callPackage ./elaborate.nix { target = "ip"; /* use-binder = true; */ };
# T1 RTL.
elaborate = innerSelf.callPackage ./elaborate.nix { target = "ip"; };
mlirbc = innerSelf.callPackage ./mlirbc.nix { inherit elaborate; };
rtl = innerSelf.callPackage ./rtl.nix { inherit mlirbc; };

rtl = innerSelf.callPackage ./rtl.nix {
inherit mlirbc;
mfcArgs = lib.escapeShellArgs [
"-O=release"
"--disable-all-randomization"
"--split-verilog"
"--preserve-values=all"
"--strip-debug-info"
"--strip-fir-debug-info"
"--verification-flavor=sva"
"--lowering-options=verifLabels,omitVersionComment,emittedLineLength=240,locationInfoStyle=none"
];
};
omreader = self.omreader-unwrapped.mkWrapper { inherit mlirbc; };

om = innerSelf.callPackage ./om.nix { inherit mlirbc; };

emu-elaborate = innerSelf.callPackage ./elaborate.nix { target = "ipemu"; /* use-binder = true; */ };
emu-elaborate = innerSelf.callPackage ./elaborate.nix { target = "ipemu"; };
emu-mlirbc = innerSelf.callPackage ./mlirbc.nix { elaborate = emu-elaborate; };
emu-omreader = self.omreader-unwrapped.mkWrapper { mlirbc = emu-mlirbc; };
emu-rtl = innerSelf.callPackage ./rtl.nix { mlirbc = emu-mlirbc; };

verilated = innerSelf.callPackage ./verilated.nix { rtl = emu-rtl; };
verilated-trace = innerSelf.callPackage ./verilated.nix { rtl = emu-rtl; enable-trace = true; };

emu = innerSelf.callPackage ./ipemu.nix { rtl = ip.emu-rtl; stdenv = moldStdenv; };
emu-trace = innerSelf.callPackage ./ipemu.nix { rtl = emu-rtl; stdenv = moldStdenv; do-trace = true; };

difftest = innerSelf.callPackage ../../difftest/default.nix { inherit verilated; };
difftest-trace = innerSelf.callPackage ../../difftest/default.nix { verilated = verilated-trace; };
# T1 Verilator Emulator
verilator-emu-omreader = self.omreader-unwrapped.mkWrapper { mlirbc = emu-mlirbc; };
verilator-emu-rtl = innerSelf.callPackage ./rtl.nix {
mlirbc = emu-mlirbc;
mfcArgs = lib.escapeShellArgs [
"-O=release"
"--split-verilog"
"--preserve-values=all"
"--verification-flavor=if-else-fatal"
"--lowering-options=verifLabels,omitVersionComment"
"--strip-debug-info"
];
};
verilator-emu-rtl-verilated = innerSelf.callPackage ./verilated.nix { rtl = verilator-emu-rtl; stdenv = moldStdenv; };
verilator-emu-rtl-verilated-trace = innerSelf.callPackage ./verilated.nix { rtl = verilator-emu-rtl; stdenv = moldStdenv; enable-trace = true; };
verilator-emu = innerSelf.callPackage ../../difftest/verilator.nix { verilated = verilator-emu-rtl-verilated; };
verilator-emu-trace = innerSelf.callPackage ../../difftest/verilator.nix { verilated = verilator-emu-rtl-verilated-trace; };

# T1 VCS Emulator
vcs-emu-omreader = self.omreader-unwrapped.mkWrapper { mlirbc = emu-mlirbc; };
vcs-emu-rtl = innerSelf.callPackage ./rtl.nix {
mlirbc = emu-mlirbc;
mfcArgs = lib.escapeShellArgs [
"-O=release"
"--split-verilog"
"--preserve-values=all"
"--verification-flavor=sva"
"--lowering-options=verifLabels,omitVersionComment"
"--strip-debug-info"
];
};
vcs-dpi-lib = innerSelf.callPackage ../../difftest/online_vcs { };
vcs-dpi-lib-trace = vcs-dpi-lib.override { enable-trace = true; };
vcs-emu-compiled = innerSelf.callPackage ./vcs.nix { inherit vcs-dpi-lib; rtl = vcs-emu-rtl; };
vcs-emu-compiled-trace = innerSelf.callPackage ./vcs.nix { vcs-dpi-lib = vcs-dpi-lib-trace; rtl = vcs-emu-rtl; };
};

subsystem = rec {
Expand Down
2 changes: 1 addition & 1 deletion nix/t1/elaborate.nix
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
}:

assert lib.assertMsg
(lib.elem target [ "ip" "ipemu" "subsystem" "subsystememu" ])
(lib.elem target [ "ip" "ipemu" ])
"Unknown elaborate target ${target}";

let
Expand Down
11 changes: 2 additions & 9 deletions nix/t1/rtl.nix
Original file line number Diff line number Diff line change
@@ -1,19 +1,12 @@
{ stdenvNoCC
, lib

, mfcArgs
, circt
, mlirbc
}:

let
mfcArgs = lib.escapeShellArgs [
"-O=debug"
"--split-verilog"
"--preserve-values=named"
"--lowering-options=verifLabels,omitVersionComment"
"--strip-debug-info"
];
fixupFilelist = lib.elem mlirbc.elaborateTarget [ "ipemu" "subsystememu" ];
fixupFilelist = lib.elem mlirbc.elaborateTarget [ "ipemu" ];
in
stdenvNoCC.mkDerivation {
name = "t1-${mlirbc.elaborateConfig}-${mlirbc.elaborateTarget}-rtl";
Expand Down
67 changes: 67 additions & 0 deletions nix/t1/vcs.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
{ lib
, bash
, stdenv
, configName
, rtl
, vcs-dpi-lib
, vcs-fhs-env
}:

stdenv.mkDerivation {
name = "${configName}-vcs";

# require license
__noChroot = true;
dontPatchELF = true;

src = rtl;

buildPhase = ''
runHook preBuild
echo "[nix] running VCS"
fhsBash=${vcs-fhs-env}/bin/vcs-fhs-env
VERDI_HOME=$("$fhsBash" -c "printenv VERDI_HOME")
"$fhsBash" vcs \
-sverilog \
-full64 \
-timescale=1ns/1ps \
-P $VERDI_HOME/share/PLI/VCS/LINUX64/novas.tab $VERDI_HOME/share/PLI/VCS/LINUX64/pli.a \
${lib.optionalString vcs-dpi-lib.enable-trace ''
-debug_access+pp+dmptf+thread \
-kdb=common_elab,hgldd_all''} \
-file filelist.f \
${vcs-dpi-lib}/lib/libdpi.a \
-o t1-vcs-simulator
runHook postBuild
'';

passthru = {
inherit (vcs-dpi-lib) enable-trace;
};

shellHook = ''
echo "[nix] entering fhs env"
${vcs-fhs-env}/bin/vcs-fhs-env
'';

installPhase = ''
runHook preInstall
mkdir -p $out/bin $out/lib
cp t1-vcs-simulator $out/lib
cp -r t1-vcs-simulator.daidir $out/lib
# We need to carefully handle string escape here, so don't use makeWrapper
tee $out/bin/t1-vcs-simulator <<EOF
#!${bash}/bin/bash
export LD_LIBRARY_PATH="$out/lib/t1-vcs-simulator.daidir:\$LD_LIBRARY_PATH"
_argv="\$@"
${vcs-fhs-env}/bin/vcs-fhs-env -c "$out/lib/t1-vcs-simulator \$_argv"
EOF
chmod +x $out/bin/t1-vcs-simulator
runHook postInstall
'';
}
6 changes: 3 additions & 3 deletions tests/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
, newScope
, rv32-stdenv
, runCommand
, difftest
, difftest-trace
, verilator-emu
, verilator-emu-trace
}:

let
Expand All @@ -20,7 +20,7 @@ let
scope = lib.recurseIntoAttrs (lib.makeScope newScope (casesSelf: {
recurseForDerivations = true;

inherit difftest difftest-trace;
inherit verilator-emu verilator-emu-trace;

makeEmuResult = casesSelf.callPackage ./make-emu-result.nix { };

Expand Down
10 changes: 5 additions & 5 deletions tests/make-emu-result.nix
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
, stdenvNoCC
, jq
, zstd
, difftest
, difftest-trace
, verilator-emu
, verilator-emu-trace
, elaborateConfigJson
}:

Expand All @@ -19,7 +19,7 @@ let

dontUnpack = true;

difftestDriver = "${difftest}/bin/online_drive";
difftestDriver = "${verilator-emu}/bin/online_drive";
difftestArgs = [
"--elf-file"
"${testCase}/bin/${testCase.pname}.elf"
Expand Down Expand Up @@ -76,7 +76,7 @@ let
'';

passthru.with-trace = self.overrideAttrs (old: {
difftestDriver = "${difftest-trace}/bin/online_drive";
difftestDriver = "${verilator-emu-trace}/bin/online_drive";
difftestArgs = old.difftestArgs ++ [ "--wave-path" "${placeholder "out"}/wave.fst" ];
postCheck = ''
if [ ! -r "$out/wave.fst" ]; then
Expand All @@ -89,7 +89,7 @@ let
passthru.with-offline = self.overrideAttrs (old: {
preInstall = ''
set +e
"${difftest}/bin/offline" \
"${verilator-emu}/bin/offline" \
--elf-file ${testCase}/bin/${testCase.pname}.elf \
--log-file $out/rtl-event.jsonl \
--log-level ERROR &> $out/offline-check-journal
Expand Down

0 comments on commit 8fed81b

Please sign in to comment.