fpga: Added flow and CI for implementation on VCU128 boards, added ov…

…erride files for Vivado compilation only

Bender.yml

@@ -58,9 +58,22 @@ sources:
     files:
       - scripts/spyglass/src/carfield_wrap.sv
 
-  - target: synthesis
+  - target: all(synthesis, not(fpga))
     files:
       - target/synth/carfield_synth_wrap.sv
+
+  - target: all(xilinx, fpga)
+    include_dirs:
+      - target/xilinx/src/overrides/include
+    files:
+      - target/xilinx/src/carfield_top_xilinx.sv
+      - target/xilinx/src/dram_wrapper.sv
+      # Override certain files due to vivado related errors
+      - target/xilinx/src/overrides/cv32e40p_fpu_wrap.sv
+      - target/xilinx/src/overrides/fpnew_wrapper.sv
+      - target/xilinx/src/overrides/tc_clk_xilinx.sv
+      - target/xilinx/src/overrides/tc_sram_xilinx.sv
+      - target/xilinx/src/overrides/riscv_ex_stage.sv
 
 vendor_package:
   - name: reggen

carfield.mk

@@ -72,7 +72,7 @@ endif
 ######################
 
 CAR_NONFREE_REMOTE ?= [email protected]:carfield/carfield-nonfree.git
-CAR_NONFREE_COMMIT ?= 5b6b7ed7c08ec079a6725f6b2c3ddf69fea3ba16
+CAR_NONFREE_COMMIT ?= dafcfee5ad118c16a9795c25ce6a6f68e8bd6177
 
 ## Clone the non-free verification IP for the Carfield TB
 car-nonfree-init:

hw/carfield.sv

@@ -21,6 +21,7 @@ module carfield
   import spatz_cluster_pkg::*;
 #(
   parameter cheshire_cfg_t Cfg = carfield_pkg::CarfieldCfgDefault,
+  parameter islands_cfg_t IslandsCfg = carfield_pkg::IslandsCfgDefault,
   parameter int unsigned HypNumPhys  = 2,
   parameter int unsigned HypNumChips = 2,
   parameter type reg_req_t           = logic,
@@ -1207,6 +1208,7 @@ assign safed_intrs = {
 };
 // verilog_lint: waive-stop line-length
 
+if (IslandsCfg.EnSafetyIsland) begin : gen_safety_island
 safety_island_synth_wrapper #(
   .SafetyIslandCfg          ( SafetyIslandCfg            ),
   .AxiAddrWidth             ( Cfg.AddrWidth              ),
@@ -1293,6 +1295,10 @@ safety_island_synth_wrapper #(
   .async_axi_out_r_wptr_i  ( axi_mst_ext_r_wptr  [SafetyIslandMstIdx] ),
   .async_axi_out_r_rptr_o  ( axi_mst_ext_r_rptr  [SafetyIslandMstIdx] )
 );
+end
+else begin : gen_no_safety_island
+  assign jtag_safety_island_tdo_o = jtag_safety_island_tdi_i;
+end
 
 // PULP integer cluster
 
@@ -1301,6 +1307,7 @@ assign car_regs_hw2reg.pulp_cluster_eoc.de  = 1'b1;
 assign car_regs_hw2reg.pulp_cluster_busy.de = 1'b1;
 assign car_regs_hw2reg.pulp_cluster_eoc.d = pulpcl_eoc;
 
+if (IslandsCfg.EnPulpCluster) begin : gen_pulp_cluster
 pulp_cluster #(
   .NB_CORES                       ( IntClusterNumCores        ),
   .NB_HWPE_PORTS                  ( IntClusterNumHwpePorts    ),
@@ -1400,6 +1407,7 @@ pulp_cluster #(
   .async_data_master_b_wptr_i  ( axi_mst_intcluster_b_wptr  ),
   .async_data_master_b_rptr_o  ( axi_mst_intcluster_b_rptr  )
 );
+end
 
 // Floating Point Spatz Cluster
 
@@ -1411,6 +1419,7 @@ logic [spatz_cluster_pkg::NumCores-1:0] spatzcl_mbox_intr;
 logic [spatz_cluster_pkg::NumCores-1:0] spatzcl_timer_intr = { chs_mti[FPClusterIntrHart1Idx], chs_mti[FPClusterIntrHart0Idx] };
 // verilog_lint: waive-stop line-length
 
+if (IslandsCfg.EnSpatzCluster) begin : gen_spatz_cluster
 spatz_cluster_wrapper #(
     .AxiAddrWidth             ( Cfg.AddrWidth     ),
     .AxiDataWidth             ( Cfg.AxiDataWidth  ),
@@ -1500,10 +1509,12 @@ spatz_cluster_wrapper #(
    .async_axi_out_r_rptr_o  ( axi_mst_ext_r_rptr  [FPClusterMstIdx] ),
    .cluster_probe_o (        )
   );
+end
 
 // Security Island
 logic secd_mbox_intr;
 
+if (IslandsCfg.EnOpenTitan) begin : gen_secure_subsystem
 secure_subsystem_synth_wrap #(
   .HartIdOffs            ( OpnTitHartIdOffs           ),
   .AxiAddrWidth          ( Cfg.AddrWidth              ),
@@ -1579,6 +1590,10 @@ secure_subsystem_synth_wrap #(
   .spi_host_SD_i    ( spih_ot_sd_i     ),
   .spi_host_SD_en_o ( spih_ot_sd_en_o  )
 );
+end
+else begin : gen_no_secure_subsystem
+  assign jtag_ot_tdo_o = jtag_ot_tdi_i;
+end
 
 // Security Island Mailbox
 // Host Clock Domain
@@ -1753,6 +1768,7 @@ axi_lite_mailbox_unit #(
 carfield_axi_slv_req_t axi_ethernet_req;
 carfield_axi_slv_rsp_t axi_ethernet_rsp;
 
+if (IslandsCfg.EnEthernet) begin : gen_ethernet
 axi_cdc_dst #(
   .LogDepth   ( LogDepth                   ),
   .aw_chan_t  ( carfield_axi_slv_aw_chan_t ),
@@ -1800,6 +1816,7 @@ axi_err_slv #(
   .slv_req_i  ( axi_ethernet_req       ),
   .slv_resp_o ( axi_ethernet_rsp       )
 );
+end
 
 // APB peripherals
 // Periph Clock Domain
@@ -2177,6 +2194,7 @@ assign reg_bus_hyper.ready = reg_hyper_rsp.ready;
 // CAN bus
 logic [63:0] can_timestamp;
 assign can_timestamp = '1;
+if (IslandsCfg.EnCan) begin : gen_can
 can_top_apb #(
   .rx_buffer_size   ( 32                    ),
   .txt_buffer_count ( 2                     ),
@@ -2201,5 +2219,6 @@ can_top_apb #(
   .s_apb_pwdata     ( apb_mst_req[CanIdx].pwdata  ),
   .s_apb_pwrite     ( apb_mst_req[CanIdx].pwrite  )
 );
+end
 
 endmodule

hw/carfield_pkg.sv

@@ -321,6 +321,27 @@ localparam cheshire_cfg_t CarfieldCfgDefault = '{
   default: '0
 };
 
+// Control which island to add (for FPGA)
+typedef struct packed {
+  bit     EnPulpCluster;
+  bit     EnSafetyIsland;
+  bit     EnSpatzCluster;
+  bit     EnOpenTitan;
+  bit     EnCan;
+  bit     EnEthernet;
+} islands_cfg_t;
+
+// Enable all islands by default
+localparam islands_cfg_t IslandsCfgDefault = '{
+  EnPulpCluster   : 1,
+  EnSafetyIsland  : 1,
+  EnSpatzCluster  : 1,
+  EnOpenTitan     : 1,
+  EnCan           : 1,
+  EnEthernet      : 1,
+  default         : '1
+};
+
 /*****************/
 /* L2 Parameters */
 /*****************/

target/xilinx/Makefile

@@ -0,0 +1,127 @@
+# Copyright 2022 ETH Zurich and University of Bologna.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Nicole Narr <[email protected]>
+# Christopher Reinwardt <[email protected]>
+# Cyril Koenig <[email protected]>
+
+BENDER       ?= bender
+
+PROJECT      ?= carfield
+
+# Board in     {genesys2, zcu102, vcu128}
+BOARD        ?= vcu128
+ip-dir       := xilinx
+
+# Derive from board
+bender-targets := -t fpga -t $(BOARD) -t cv64a6_imafdcsclic_sv39 -t cva6 -t mchan -t spatz -t cv32e40p_use_ff_regfile
+bender-defines := -D FEATURE_ICACHE_STAT -D PRIVATE_ICACHE -D HIERARCHY_ICACHE_32BIT -D CLUSTER_ALIAS -D PULP_FPGA_EMUL
+
+VIVADO       ?= vitis-2020.2 vivado
+
+# Select board specific variables
+ifeq ($(BOARD),vcu128)
+	XILINX_PART  ?= xcvu37p-fsvh2892-2L-e
+	XILINX_BOARD ?= xilinx.com:vcu128:part0:1.0
+	ips-names    := xlnx_clk_wiz xlnx_vio
+endif
+ifeq ($(BOARD),genesys2)
+	XILINX_PART  ?= xc7k325tffg900-2
+	XILINX_BOARD ?= digilentinc.com:genesys2:part0:1.1
+	ips-names    := xlnx_mig_7_ddr3 xlnx_clk_wiz xlnx_vio
+	FPGA_PATH    ?= xilinx_tcf/Digilent/200300A8C60DB
+endif
+ifeq ($(BOARD),zcu102)
+	XILINX_PART    ?= xczu9eg-ffvb1156-2-e
+	XILINX_BOARD   ?= xilinx.com:zcu102:part0:3.4
+	ips-names      := xlnx_mig_ddr4 xlnx_clk_wiz xlnx_vio
+endif
+
+# Derive from ips
+ips := $(addsuffix .xci ,$(basename $(ips-names)))
+bender-targets := $(bender-targets) $(addprefix -t ,$(basename $(ips)))
+
+# Select islands
+ifeq ($(GEN_PULP_CLUSTER), 1)
+bender-defines += -D GEN_PULP_CLUSTER=1
+endif
+ifeq ($(GEN_SAFETY_ISLAND), 1)
+bender-defines += -D GEN_SAFETY_ISLAND=1
+endif
+ifeq ($(GEN_SPATZ_CLUSTER), 1)
+bender-defines += -D GEN_SPATZ_CLUSTER=1
+endif
+ifeq ($(GEN_OPEN_TITAN), 1)
+bender-defines += -D GEN_OPEN_TITAN=1
+endif
+
+out := out
+bit := $(out)/$(PROJECT)_top_xilinx.bit
+mcs := $(out)/$(PROJECT)_top_xilinx.mcs
+
+VIVADOENV ?=  PROJECT=$(PROJECT)            \
+              BOARD=$(BOARD)                \
+              XILINX_PART=$(XILINX_PART)    \
+              XILINX_BOARD=$(XILINX_BOARD)  \
+              PORT=$(XILINX_PORT)           \
+              HOST=$(XILINX_HOST)           \
+              FPGA_PATH=$(FPGA_PATH)        \
+              BIT=$(BIT)
+
+# select IIS-internal tool commands if we run on IIS machines
+ifneq (,$(wildcard /etc/iis.version))
+	VIVADO ?= vitis-2020.2 vivado
+else
+	VIVADO ?= vivado
+endif
+
+MODE        ?= gui
+VIVADOFLAGS ?= -nojournal -mode $(MODE) -source scripts/prologue.tcl
+
+all: $(bit)
+
+# Generate mcs from bitstream
+$(mcs): $(bit)
+	$(VIVADOENV) $(VIVADO) $(VIVADOFLAGS) -source scripts/write_cfgmem.tcl -tclargs $@ $^
+
+$(bit): $(ips) scripts/add_sources.tcl
+	@mkdir -p $(out)
+	$(VIVADOENV) $(VIVADO) $(VIVADOFLAGS) -source scripts/run.tcl
+	cp $(PROJECT).runs/impl_1/$(PROJECT)* ./$(out)
+
+%.xci:
+	@echo "Generating IP $(basename $@)"
+	cd $(ip-dir)/$(basename $@) && $(MAKE) clean && $(VIVADOENV) VIVADO="$(VIVADO)" $(MAKE)
+	cp $(ip-dir)/$(basename $@)/$(basename $@).srcs/sources_1/ip/$(basename $@)/$@ $@
+
+gui:
+	@echo "Starting $(vivado) GUI"
+	@$(VIVADOENV) $(VIVADO) -nojournal -mode gui $(PROJECT).xpr &
+
+program: #$(bit)
+	@echo "Programming board $(BOARD) ($(XILINX_PART))"
+	$(VIVADOENV) $(VIVADO) $(VIVADOFLAGS) -source scripts/program.tcl
+
+clean:
+	rm -rf *.log *.jou *.str *.mif *.xci *.xpr .Xil/ $(out) $(PROJECT).cache $(PROJECT).hw $(PROJECT).ioplanning $(PROJECT).ip_user_files $(PROJECT).runs $(PROJECT).sim
+
+# Clean only top and copy back the IPs output here
+rebuild_top:
+	${MAKE} clean
+	rm -f scripts/add_sources.tcl
+	find xilinx -wholename "**/*.srcs/**/*.xci" | xargs -n 1 -I {} cp {} .
+	${MAKE} $(bit)
+
+# Bender
+scripts/add_sources.tcl: ../../Bender.yml
+	$(BENDER) script vivado $(bender-targets) $(bender-defines) > $@
+	cp $@ $@.bak
+	./scripts/overrides.sh $@
+	echo "" >> $@
+	echo "#Put the overrides folder at the head of the include list" >> $@
+	echo "set_property include_dirs [ \\" >> $@
+	echo "    concat \"\$$ROOT/target/xilinx/src/overrides/include \" [get_property include_dirs [current_fileset]] \\" >> $@
+	echo "] [current_fileset]" >> $@
+
+.PHONY: clean sim

target/xilinx/README.md

@@ -0,0 +1,13 @@
+# VCU128 emulation
+
+```bash
+# Build the bitstream:
+make
+# Re-build the bitstream without
+# re-building the IPs:
+make rebuild-top
+# Simulate with the IPs
+# Note you need to generate the
+# Vivado IP models before
+make sim
+```