fpga: Add VCU128 flow and other IPs

Bender.yml

@@ -53,4 +53,6 @@ sources:
   - target: all(fpga, xilinx)
     files:
       - target/xilinx/src/fan_ctrl.sv
+      - target/xilinx/src/dram_wrapper_xilinx.sv
+      - target/xilinx/src/phy_definitions.svh
       - target/xilinx/src/cheshire_top_xilinx.sv

README.md

@@ -19,6 +19,8 @@ source start.cheshire_soc.tcl
 run -all
 ```
 
+If you have access to our internal servers, you can run `make nonfree-init` to fetch additional resources we cannot make publically accessible. Note that these are *not required* to use anything provided in this repository.
+
 ## License
 
 Unless specified otherwise in the respective file headers, all code checked into this repository is made available under a permissive license. All hardware sources and tool scripts are licensed under the Solderpad Hardware License 0.51 (see `LICENSE`) with the exception of generated register file code (e.g. `hw/regs/*.sv`), which is generated by a fork of lowRISC's [`regtool`](https://github.com/lowRISC/opentitan/blob/master/util/regtool.py) and licensed under Apache 2.0. All software sources are licensed under Apache 2.0.

cheshire.mk

@@ -11,6 +11,9 @@ BENDER ?= bender
 VLOG_ARGS ?= -suppress 2583 -suppress 13314
 VSIM      ?= vsim
 
+# Define board for FPGA flow and/or device tree selection
+BOARD         ?= genesys2
+
 # Define used paths (prefixed to avoid name conflicts)
 CHS_ROOT      ?= $(shell $(BENDER) path cheshire)
 CHS_REG_DIR   := $(shell $(BENDER) path register_interface)
@@ -53,7 +56,7 @@ chs-clean-deps:
 ######################
 
 CHS_NONFREE_REMOTE ?= [email protected]:pulp-restricted/cheshire-nonfree.git
-CHS_NONFREE_COMMIT ?= dafd3c1
+CHS_NONFREE_COMMIT ?= 890a09d20bf200c4fbcc3d2b708a16ba89678306
 
 chs-nonfree-init:
 	git clone $(CHS_NONFREE_REMOTE) $(CHS_ROOT)/nonfree
@@ -155,25 +158,26 @@ CHS_SIM_ALL += $(CHS_ROOT)/target/sim/models/24FC1025.v
 CHS_SIM_ALL += $(CHS_ROOT)/target/sim/vsim/compile.cheshire_soc.tcl
 
 #############
-# FPGA Flow #
+# Emulation #
 #############
 
-$(CHS_ROOT)/target/xilinx/scripts/add_sources.tcl: Bender.yml
-	$(BENDER) script vivado -t fpga -t cv64a6_imafdcsclic_sv39 -t cva6 > $@
-
-CHS_XILINX_ALL += $(CHS_ROOT)/target/xilinx/scripts/add_sources.tcl
+include $(CHS_ROOT)/target/xilinx/xilinx.mk
+include $(CHS_XIL_DIR)/sim/sim.mk
+CHS_XILINX_ALL += $(CHS_XIL_DIR)/scripts/add_sources.tcl
+CHS_LINUX_IMG  += $(CHS_SW_DIR)/boot/linux-${BOARD}.gpt.bin
 
 #################################
 # Phonies (KEEP AT END OF FILE) #
 #################################
 
 .PHONY: chs-all chs-nonfree-init chs-clean-deps chs-sw-all chs-hw-all chs-bootrom-all chs-sim-all chs-xilinx-all
 
-CHS_ALL += $(CHS_SW_ALL) $(CHS_HW_ALL) $(CHS_SIM_ALL) $(CHS_XILINX_ALL)
+CHS_ALL += $(CHS_SW_ALL) $(CHS_HW_ALL) $(CHS_SIM_ALL)
 
 chs-all:         $(CHS_ALL)
 chs-sw-all:      $(CHS_SW_ALL)
 chs-hw-all:      $(CHS_HW_ALL)
 chs-bootrom-all: $(CHS_BOOTROM_ALL)
 chs-sim-all:     $(CHS_SIM_ALL)
 chs-xilinx-all:  $(CHS_XILINX_ALL)
+chs-linux-img:   $(CHS_LINUX_IMG)

docs/tg/xilinx.md

@@ -7,38 +7,109 @@ This page describes how to map Cheshire on Xilinx FPGAs to *execute baremetal pr
 We currently provide working setups for:
 
 - Digilent Genesys 2 with Vivado `>= 2020.2`
+- Xilinx VCU128 with Vivado `>= 2020.2`
 
 We are working on support for more boards in the future.
 
-## Implementation
+## Building the bistream
 
-Since the implementation steps and available features vary between boards, we provide instructions and document available features for each.
+Do to the structure of the Makefile flow. All the following commands are to be executed at the root of the Cheshire repository. If you want to see the targets that you will be using, you can find them in `sw/sw.mk` and `target/xilinx/xilinx.mk`.
 
-### Digilent Genesys 2
+First, make sure that you have generated all the RTL:
+
+```bash
+make chs-hw-all
+```
 
 Generate the bitstream `target/xilinx/out/cheshire_top_xilinx.bit` by running:
 
+```bash
+make chs-xil-all [VIVADO=version] [BOARD={genesys2,vcu128}] [MODE={batch,gui}] [INT-JTAG={0,1}]
 ```
-make -C target/xilinx
+
+See the argument list below:
+
+| Argument | Relevance | Description                                                                                                                           |
+|----------|-----------|---------------------------------------------------------------------------------------------------------------------------------------|
+| VIVADO   | all       | Vivado command to use **(default "vitis-2020.2 vivado")**                                                                             |
+| BOARD    | all       | `genesys-2` **(default)** <br>`vcu128`                                                                                                |
+| INT-JTAG | vcu128    | `0` Connect the RV debug module to an external JTAG chain<br>`1` Connect the RV debug module to the internal JTAG chain **(default)** |
+| MODE     | all       | `batch` Compile in Vivado shell<br>`gui` Compile in Vivado gui                                                                        |
+
+The build time takes a couple of hours.
+
+The above target used Bender and the file `Bender.yml` to generate the filelist required for Vivado. You can find it in 
+
+## Board specificities
+
+### Digilent Genesys 2
+> ##### Bootmode and switches
+> 
+> Before flashing the bitstream to your device, take note of the position of onboard switches, which control important functionality:
+> 
+> 
+>   | Switch | Function                                        |
+>   | ------ | ------------------------------------------------|
+>   | 1 .. 0 | Boot mode; see [Boot ROM](../um/sw.md#boot-rom) |
+>   | 5 .. 2 | Fan level; *do not* keep at 0                   |
+>   | 7      | Test mode; *leave at zero*                      |
+> 
+> The reset, JTAG TAP, UART, I2C, and VGA are all connected to their onboard logic or ports. The UART has *no flow control*. The microSD slot is connected to chip select 0 of the SPI host peripheral. Serial link and GPIOs are currently not available.
+> 
+### Xilinx VCU128
+> #### Bootmode and VIOs
+> 
+> As there are no switches on this board, the CVA6 bootmode (see [Boot ROM](../um/sw.md#boot-rom)) is selected by Xilinx VIOs that can be set in the Vivado GUI (see [Using Vivado GUI](#bringup_vivado_gui)).
+> 
+> #### External JTAG chain
+> 
+> The VCU128 development board only provides one JTAG chain, used by Vivado to program the bitstream, and interact with certain IPs (ILAs, VIOs, ...). The RV64 host also requires a JTAG chain to connect GDB to the debug-module in the bitstream. It is possible to use the same JTAG chain for both by using `INT-JTAG=1`. In this case no external cable is required but it will not be possible to use GDB (to debug the program running on the host) and communicate with the bitstream (to debug signals using ILAs) at the same time. By using `INT-JTAG=0` it is possible to add an external JTAG chain for the RV64 host through GPIOs. Since the VCU128 does not have GPIOs we use we use a Digilent JTAG-HS2 cable connected to the Xilinx XM105 FMC debug card. See the connections in `vcu128.xdc`.
+
+## Bare-metal bringup 
+
+### Programming the FPGA
+
+#### Using Vivado GUI <a name="bringup_vivado_gui"></a>
+
+If you have closed Vivado, or compiled in batch mode, you can open the Vivado GUI with:
+
+```bash
+make chs-xil-gui
 ```
 
-Before flashing the bitstream to your device, take note of the position of onboard switches, which control important functionality:
+You can now open the Hardware Manager and program the FPGA. Once done, Vivado will give you access the to Virtual Inputs Outputs (VIOs). You can now assert the following signals (on Cheshire top level).
 
+  | VIO               | Function                                                        |
+  | ----------------- | ----------------------------------------------------------------|
+  | vio_reset         | Positive edge-sensitive reset for the whole system              |
+  | vio_boot_mode     | Override the boot-mode switches described above                 |
+  | vio_boot_mode_sel | Select between 0: using boot mode switches 1: use boot mode VIO |
 
-  | Switch | Function                                        |
-  | ------ | ------------------------------------------------|
-  | 1 .. 0 | Boot mode; see [Boot ROM](../um/sw.md#boot-rom) |
-  | 5 .. 2 | Fan level; *do not* keep at 0                   |
-  | 7      | Test mode; *leave at zero*                      |
+#### Using command line <a name="bringup_vivado_cli"></a>
 
-The reset, JTAG TAP, UART, I2C, and VGA are all connected to their onboard logic or ports. The UART has *no flow control*. The microSD slot is connected to chip select 0 of the SPI host peripheral. Serial link and GPIOs are currently not available.
+A script `program.tcl` is available to flash the bitstream without opening Vivado GUI. You will need to give the following variable to access your board (see `target/xilinx/xilinx.mk`).
 
-## Debugging with OpenOCD
+- `XILINX_PORT`: Vivado opened port (**default 3121**)
+- `FPGA_PATH`: Vivado path to your FPGA (**default xilinx_tcf/Xilinx/[serial_id]**)
+- `XILINX_HOST`: Path to your Vivado server (**default localhost**)
 
-To establish a debug bridge over JTAG, ensure the target is in a debuggable state (for example by resetting into the idle boot mode 0) and launch OpenOCD with:
+Change the values to the appropriate ones (can be found in the Vivado GUI) and programm the board:
 
+```bash
+make chs-xil-program MODE=batch BOARD=vcu128
 ```
-openocd -f $(bender path ariane)/corev_apu/fpga/ariane.cfg
+
+### Loading binary and debugging with OpenOCD
+
+To establish a debug bridge over JTAG, ensure the target is in a debuggable state (for example by setting the boot mode to 0 before resetting) then launch OpenOCD with:
+
+```bash
+# VCU128 : Internal JTAG
+openocd -f util/openocd_vcu128.cfg
+# Genesys2 : Internal JTAG
+oprnocd -f util/openocd_genesys2.cfg
+# All boards : External JTAG (Digilent HS2)
+openocd -f util/openocd_hs2.cfg
 ```
 
 In another shell, launch a RISC-V GDB session attaching to OpenOCD:
@@ -78,7 +149,7 @@ continue
 
 You should see `Hello World!` output printed on the UART.
 
-### Boot from SD Card
+### Load from SD Card (Genesys2) <a name="bringup_flash_sd"></a>
 
 First, build an up-to-date a disk image for your desired binary. For `helloworld`:
 
@@ -95,39 +166,101 @@ sudo sgdisk -e /dev/<sdcard>
 
 The second command only ensures correctness of the partition layout; it moves the secondary GPT header at the end of the minimally sized image to the end of your actual SD card.
 
-Insert your SD card and reset into boot mode 1. You should see a `Hello World!` UART output.
+Insert your SD card and reset into __boot mode 1__. You should see a `Hello World!` UART output.
 
 ## Booting Linux
 
 To boot Linux, we must load the *OpenSBI* firmware, which takes over M mode and launches the U-boot bootloader. U-boot then loads Linux. For more details, see [Boot Flow](../um/sw.md#boot-flow).
 
-Clone the `cheshire` branch of CVA6 SDK and build the firmware (OpenSBI + U-boot) and Linux images (*this will take about 30 minutes*):
+Clone the `cheshire` branch of CVA6 SDK into `sw/deps/cva6-sdk` and build the firmware (OpenSBI + U-boot) and Linux images (*this will take about 30 minutes*):
 
-```
+```bash
 git submodule update --init --recursive sw/deps/cva6-sdk
 make -C sw/deps/cva6-sdk images
 ```
 
-In principle, we can boot Linux through JTAG by loading all images into memory, launching OpenSBI, and instructing U-boot to load the kernel directly from memory. Here, we focus on autonomous boot from SD card.
+In principle, we can boot Linux through JTAG by loading all images into memory, launching OpenSBI, and instructing U-boot to load the kernel directly from memory. Here, we focus on autonomous boot from SD card or SPI flash.
 
 In this case, OpenSBI is loaded by a regular baremetal program called the [Zero-Stage Loader](../um/sw.md#zero-stage-loader) (ZSL). The [boot ROM](../um/sw.md#boot-rom) loads the ZSL from SD card, which then loads the device tree and firmware from other SD card partitions into memory and launches OpenSBI.
 
 To create a full Linux disk image from the ZSL, device tree, firmware, and Linux, run:
 
+```bash
+# Note that the device tree's flavor depends on the board (see sw/boot/*.dts)
+make chs-linux-img BOARD=[genesys2, vcu128]
 ```
-make sw/boot/linux.gpt.bin
+
+### Digilent Genesys 2
+> 
+> Flash this image to an SD card as for the hello world (see [Load from SD Card](#bringup_flash_sd)), then insert the SD card and reset into boot mode 1. You should first see the ZSL print on the UART:
+> 
+> ```
+>  /\___/\       Boot mode:       1
+> ( o   o )      Real-time clock: ... Hz
+> (  =^=  )      System clock:    ... Hz
+> (        )     Read global ptr: 0x...
+> (    P    )    Read pointer:    0x...
+> (  U # L   )   Read argument:   0x...
+> (    P      )
+> (           ))))))))))
+> ```
+> You should then boot through OpenSBI, U-Boot, and Linux until you are dropped into a shell.
+> 
+### Xilinx VCU128
+> 
+> This board does not offer a SD card reader. We need to load the image in the integrated flash:
+> 
+> ```
+> make chs-xil-flash MODE=batch BOARD=vcu128
+> ```
+> 
+> Use the parameters defined in [Using command line](#bringup_vivado_cli) (defaults are in `target/xilinx/xilinx.mk`) to select your board:
+>
+> This script will erase your bitstream, once the flash has been written (c.a. 10min) you will need to re-program the bitstream on the board.
+
+## Add your own board
+
+If you wish to add a flow for a new FPGA board, please do the following steps:  
+_Please consider opening a pull request containing the necessary changes to integrate your new board (:_
+
+### Makefile
+
+Add your board on top of `target/xilinx/xilinx.mk`, in particular `XILINX_PART` and `XILINX_BOARD` are identifying the FPGA chip and board (can be found in VIvado GUI). The parameters identifying your personal device `XILINX_PORT`, `FPGA_PATH`, `XILINX_HOST` can be left empty for now.  
+You then need to define `ip-names` with the Xilinx IPs that you will be using: DDR3/4 depending on your board, Clock Wizard, VIOs. See next sections for more explanations.
+
+### Vivado IPs
+
+#### Re-arametrize existing IPs
+
+Cheshire's emulation requires a few Vivado IPs to work properly. They are defined and pre-compiled in `target/xilinx/xilinx/*`.  
+If you add a new board, you will need to reconfigure your IPs for this board. For instance, to use the _Vivado MIG DDR4 controller_, modify `target/xilinx/xilinx/xlnx_mig_ddr4/run.tcl`. There, add the relvant `$::env(BOARD)` entry with your configuration.  
+To know which configuration to use your board, you can open a blank project in Vivado GUI, create a blank block design, and instanciate the MIG DDR4 IP there. The Vivado TCL console should write the default parameters for your FPGA. You can later re-configure the IP in the block design and Vivado will print to the tcl console the modified parameters. Then yuo can copy these tcl lines to the `run.tcl` file. Make sure that you added your ip to `target/xilinx/xilinx.mk` under "ip-names".
+
+#### Add a new IP
+
+If your board require a new IP that has not been integrated already do the following :
+
+- Add a new folder `target/xilinx/xilinx/[your_ip]` taking the example of the `xlnx_mig_ddr4`.
+- Modify `target/xilinx/xilinx/[your_ip]/tcl/run.tcl` and `target/xilinx/xilinx/[your_ip]/Makefile` accordingly.
+- Add your IP to `target/xilinx/xilinx.mk` under "ip-names".
+
+#### Instantiate your IP
+
+Connect it's top module in the top-level: `target/xilinx/src/cheshire_top_xilinx.sv` (you can already ). If your IP is a DDR controller, please add it to `target/xilinx/src/dram_wrapper_xilinx.sv`. Note that this file contains a pipeline to resize AXI transactions from Cheshire to your controller.
+
+Add the relevant macro parameters to `target/xilinx/src/phy_definitions.sv` in order to disable your IP for non-relevant boards.
+
+#### Debug
+
+It is possible to use ILA (Integrated Logic Analyzers) in order to debug some signals on the running FPGA. Add the following before declaring your signals:
+
+```verilog
+  // Indicate that you need to debug a signal
+  (* dont_touch = "yes" *) (* mark_debug = "true" *) logic signal_d0;
+  // You can also use the following macro from phy_definitions.svh
+  `ila(ila_signal_d0, signal_d0)
 ```
 
-Flash this image to an SD card as you did in the previous section, then insert the SD card and reset into boot mode 1. You should first see the ZSL print on the UART:
+Then, re-build your bitstream.
 
-```
- /\___/\       Boot mode:       1
-( o   o )      Real-time clock: ... Hz
-(  =^=  )      System clock:    ... Hz
-(        )     Read global ptr: 0x...
-(    P    )    Read pointer:    0x...
-(  U # L   )   Read argument:   0x...
-(    P      )
-(           ))))))))))
-```
-You should then boot through OpenSBI, U-Boot, and Linux until you are dropped into a shell.
+It is also possible to simulate your IPs with `target/xilinx/sim` (undocumented yet).

sw/boot/cheshire.dts → sw/boot/cheshire.dtsi

@@ -7,11 +7,13 @@
 // Axel Vanoni <[email protected]>
 
 /dts-v1/;
+
 / {
   #address-cells = <2>;
   #size-cells = <2>;
   compatible = "eth,cheshire-dev";
   model = "eth,cheshire";
+
   chosen {
     stdout-path = "/soc/serial@3002000:115200";
   };
@@ -74,22 +76,16 @@
       interrupts = <2 3 4 5 6 7 8 9 10 11 12 13 14 15 16>;
       reg = <0x0 0x3003000 0x0 0x1000>;
     };
-    spi@3004000 {
+    spi: spi@3004000 {
       compatible = "opentitan,spi-host", "lowrisc,spi";
       interrupt-parent = <&PLIC0>;
       interrupts = <17 18>;
       reg = <0x0 0x3004000 0x0 0x1000>;
+      num-cs = <2>;
       clock-frequency = <50000000>;
       max-frequency = <25000000>;
       #address-cells = <1>;
       #size-cells = <0>;
-      mmc@0 {
-        compatible = "mmc-spi-slot";
-        reg = <0>;
-        spi-max-frequency = <25000000>;
-        voltage-ranges = <3300 3300>;
-        disable-wp;
-      };
     };
     vga@3007000 {
       compatible = "eth,axi-vga";

sw/boot/cheshire_genesys2.dts

@@ -0,0 +1,18 @@
+// Copyright 2022 ETH Zurich and University of Bologna.
+// Solderpad Hardware License, Version 0.51, see LICENSE for details.
+// SPDX-License-Identifier: SHL-0.51
+//
+// Cyril Koenig <[email protected]>
+
+/include/ "cheshire.dtsi"
+
+&spi {
+  boot-with = <0>;
+  mmc@0 {
+    compatible = "mmc-spi-slot";
+    reg = <0>; // CS
+    spi-max-frequency = <25000000>;
+    voltage-ranges = <3300 3300>;
+    disable-wp;
+  };
+};