diff --git a/Bender.yml b/Bender.yml index c21ec54dc8..1f978d1b97 100644 --- a/Bender.yml +++ b/Bender.yml @@ -48,9 +48,10 @@ sources: - core/include/cv32a6_imac_sv0_config_pkg.sv - core/include/riscv_pkg.sv - core/include/ariane_pkg.sv - - core/mmu_sv32/cva6_tlb_sv32.sv - - core/mmu_sv32/cva6_mmu_sv32.sv + - core/mmu_unify/cva6_tlb.sv + - core/mmu_unify/cva6_mmu.sv - core/mmu_sv32/cva6_ptw_sv32.sv + - core/mmu_unify/cva6_ptw.sv - core/cva6_accel_first_pass_decoder_stub.sv - target: cv32a6_imac_sv32 @@ -58,9 +59,10 @@ sources: - core/include/cv32a6_imac_sv32_config_pkg.sv - core/include/riscv_pkg.sv - core/include/ariane_pkg.sv - - core/mmu_sv32/cva6_tlb_sv32.sv - - core/mmu_sv32/cva6_mmu_sv32.sv + - core/mmu_unify/cva6_tlb.sv + - core/mmu_unify/cva6_mmu.sv - core/mmu_sv32/cva6_ptw_sv32.sv + - core/mmu_unify/cva6_ptw.sv - core/cva6_accel_first_pass_decoder_stub.sv - target: cv32a6_imafc_sv32 @@ -68,9 +70,10 @@ sources: - core/include/cv32a6_imafc_sv32_config_pkg.sv - core/include/riscv_pkg.sv - core/include/ariane_pkg.sv - - core/mmu_sv32/cva6_tlb_sv32.sv - - core/mmu_sv32/cva6_mmu_sv32.sv + - core/mmu_unify/cva6_tlb.sv + - core/mmu_unify/cva6_mmu.sv - core/mmu_sv32/cva6_ptw_sv32.sv + - core/mmu_unify/cva6_ptw.sv - core/cva6_accel_first_pass_decoder_stub.sv # included via target core/include/${TARGET_CFG}_config_pkg.sv diff --git a/ariane.core b/ariane.core index 501f296c14..577605276a 100644 --- a/ariane.core +++ b/ariane.core @@ -35,18 +35,21 @@ filesets: - src/miss_handler.sv - src/mmu_sv39/mmu.sv - src/mmu_sv32/cva6_mmu_sv32.sv + - src/mmu_unify/cva6_mmu.sv - src/mult.sv - src/nbdcache.sv - src/pcgen_stage.sv - src/perf_counters.sv - src/mmu_sv39/ptw.sv - src/mmu_sv32/cva6_ptw_sv32.sv + - src/mmu_unify/cva6_ptw.sv - src/regfile_ff.sv - src/scoreboard.sv - src/store_buffer.sv - src/store_unit.sv - src/mmu_sv39/tlb.sv - src/mmu_sv32/cva6_tlb_sv32.sv + - src/mmu_unify/cva6_tlb.sv file_type : systemVerilogSource depend : - pulp-platform.org::axi_mem_if diff --git a/core/Flist.cva6 b/core/Flist.cva6 index 46a541603e..aa8f20b281 100644 --- a/core/Flist.cva6 +++ b/core/Flist.cva6 @@ -187,4 +187,10 @@ ${CVA6_REPO_DIR}/core/mmu_sv32/cva6_ptw_sv32.sv ${CVA6_REPO_DIR}/core/mmu_sv32/cva6_tlb_sv32.sv ${CVA6_REPO_DIR}/core/mmu_sv32/cva6_shared_tlb_sv32.sv +// MMU Unify +${CVA6_REPO_DIR}/core/mmu_unify/cva6_mmu.sv +${CVA6_REPO_DIR}/core/mmu_unify/cva6_tlb.sv +${CVA6_REPO_DIR}/core/mmu_unify/cva6_shared_tlb.sv +${CVA6_REPO_DIR}/core/mmu_unify/cva6_ptw.sv + // end of manifest diff --git a/core/include/cv64a6_imafdc_sv39_config_pkg.sv b/core/include/cv64a6_imafdc_sv39_config_pkg.sv index 0865c0431d..7644769996 100644 --- a/core/include/cv64a6_imafdc_sv39_config_pkg.sv +++ b/core/include/cv64a6_imafdc_sv39_config_pkg.sv @@ -55,8 +55,8 @@ package cva6_config_pkg; localparam CVA6ConfigNrStorePipeRegs = 0; localparam CVA6ConfigNrLoadBufEntries = 2; - localparam CVA6ConfigInstrTlbEntries = 16; - localparam CVA6ConfigDataTlbEntries = 16; + localparam CVA6ConfigInstrTlbEntries = 2; + localparam CVA6ConfigDataTlbEntries = 2; localparam CVA6ConfigRASDepth = 2; localparam CVA6ConfigBTBEntries = 32; diff --git a/core/load_store_unit.sv b/core/load_store_unit.sv index 94e1c4f119..c735752643 100644 --- a/core/load_store_unit.sv +++ b/core/load_store_unit.sv @@ -140,41 +140,20 @@ module load_store_unit // ------------------- // MMU e.g.: TLBs/PTW // ------------------- - if (MMU_PRESENT && (riscv::XLEN == 64)) begin : gen_mmu_sv39 - mmu #( - .CVA6Cfg (CVA6Cfg), - .INSTR_TLB_ENTRIES(ariane_pkg::INSTR_TLB_ENTRIES), - .DATA_TLB_ENTRIES (ariane_pkg::DATA_TLB_ENTRIES), - .ASID_WIDTH (ASID_WIDTH) - ) i_cva6_mmu ( - // misaligned bypass - .misaligned_ex_i(misaligned_exception), - .lsu_is_store_i (st_translation_req), - .lsu_req_i (translation_req), - .lsu_vaddr_i (mmu_vaddr), - .lsu_valid_o (translation_valid), - .lsu_paddr_o (mmu_paddr), - .lsu_exception_o(mmu_exception), - .lsu_dtlb_hit_o (dtlb_hit), // send in the same cycle as the request - .lsu_dtlb_ppn_o (dtlb_ppn), // send in the same cycle as the request - // connecting PTW to D$ IF - .req_port_i (dcache_req_ports_i[0]), - .req_port_o (dcache_req_ports_o[0]), - // icache address translation requests - .icache_areq_i (icache_areq_i), - .asid_to_be_flushed_i, - .vaddr_to_be_flushed_i, - .icache_areq_o (icache_areq_o), - .pmpcfg_i, - .pmpaddr_i, - .* - ); - end else if (MMU_PRESENT && (riscv::XLEN == 32)) begin : gen_mmu_sv32 - cva6_mmu_sv32 #( + if (MMU_PRESENT) begin : gen_mmu + + localparam ASID_LEN = (riscv::XLEN == 64) ? 16 : 9; + localparam VPN_LEN = (riscv::XLEN == 64) ? 27 : 20; + localparam PT_LEVELS = (riscv::XLEN == 64) ? 3 : 2; + + cva6_mmu #( .CVA6Cfg (CVA6Cfg), .INSTR_TLB_ENTRIES(ariane_pkg::INSTR_TLB_ENTRIES), .DATA_TLB_ENTRIES (ariane_pkg::DATA_TLB_ENTRIES), - .ASID_WIDTH (ASID_WIDTH) + .ASID_WIDTH (ASID_WIDTH), + .ASID_LEN (ASID_LEN), + .VPN_LEN (VPN_LEN), + .PT_LEVELS (PT_LEVELS) ) i_cva6_mmu ( // misaligned bypass .misaligned_ex_i(misaligned_exception), diff --git a/core/mmu_unify/README.md b/core/mmu_unify/README.md new file mode 100644 index 0000000000..43a6523f6d --- /dev/null +++ b/core/mmu_unify/README.md @@ -0,0 +1 @@ +Unification of MMUs: sv32, sv39 and sv39x4 \ No newline at end of file diff --git a/core/mmu_unify/cva6_mmu.sv b/core/mmu_unify/cva6_mmu.sv new file mode 100644 index 0000000000..851bf01292 --- /dev/null +++ b/core/mmu_unify/cva6_mmu.sv @@ -0,0 +1,643 @@ +// Copyright (c) 2021 Thales. +// Copyright and related rights are licensed under the Solderpad Hardware +// License, Version 0.51 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law +// or agreed to in writing, software, hardware and materials distributed under +// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Author: Angela Gonzalez, PlanV Technology +// Date: 07/12/2023 +// Description: Memory Management Unit for CVA6, contains TLB and +// address translation unit. SV32 and SV39 as defined in RISC-V +// privilege specification 1.11-WIP. +// This module is an merge of the MMU Sv39 developed +// by Florian Zaruba and the MMU Sv32 developed by Sebastien Jacq. + + +module cva6_mmu + import ariane_pkg::*; +#( + parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, + parameter int unsigned INSTR_TLB_ENTRIES = 4, + parameter int unsigned DATA_TLB_ENTRIES = 4, + parameter int unsigned ASID_WIDTH = 1, + parameter int unsigned ASID_LEN = 1, + parameter int unsigned VPN_LEN = 1, + parameter int unsigned PT_LEVELS = 1 +) ( + input logic clk_i, + input logic rst_ni, + input logic flush_i, + input logic enable_translation_i, + input logic en_ld_st_translation_i, // enable virtual memory translation for load/stores + // IF interface + input icache_arsp_t icache_areq_i, + output icache_areq_t icache_areq_o, + // LSU interface + // this is a more minimalistic interface because the actual addressing logic is handled + // in the LSU as we distinguish load and stores, what we do here is simple address translation + input exception_t misaligned_ex_i, + input logic lsu_req_i, // request address translation + input logic [riscv::VLEN-1:0] lsu_vaddr_i, // virtual address in + input logic lsu_is_store_i, // the translation is requested by a store + // if we need to walk the page table we can't grant in the same cycle + // Cycle 0 + output logic lsu_dtlb_hit_o, // sent in the same cycle as the request if translation hits in the DTLB + output logic [riscv::PPNW-1:0] lsu_dtlb_ppn_o, // ppn (send same cycle as hit) + // Cycle 1 + output logic lsu_valid_o, // translation is valid + output logic [riscv::PLEN-1:0] lsu_paddr_o, // translated address + output exception_t lsu_exception_o, // address translation threw an exception + // General control signals + input riscv::priv_lvl_t priv_lvl_i, + input riscv::priv_lvl_t ld_st_priv_lvl_i, + input logic sum_i, + input logic mxr_i, + // input logic flag_mprv_i, + input logic [riscv::PPNW-1:0] satp_ppn_i, + input logic [ASID_WIDTH-1:0] asid_i, + input logic [ASID_WIDTH-1:0] asid_to_be_flushed_i, + input logic [riscv::VLEN-1:0] vaddr_to_be_flushed_i, + input logic flush_tlb_i, + // Performance counters + output logic itlb_miss_o, + output logic dtlb_miss_o, + // PTW memory interface + input dcache_req_o_t req_port_i, + output dcache_req_i_t req_port_o, + // PMP + input riscv::pmpcfg_t [15:0] pmpcfg_i, + input logic [15:0][riscv::PLEN-3:0] pmpaddr_i +); + + // memory management, pte for cva6 + localparam type pte_cva6_t = struct packed { + logic [riscv::PPNW-1:0] ppn; // PPN length for + logic [1:0] rsw; + logic d; + logic a; + logic g; + logic u; + logic x; + logic w; + logic r; + logic v; + }; + + localparam type tlb_update_cva6_t = struct packed { + logic valid; // valid flag + logic [PT_LEVELS-2:0] is_page; // + logic [VPN_LEN-1:0] vpn; // + logic [ASID_LEN-1:0] asid; // + pte_cva6_t content; + }; + + logic iaccess_err; // insufficient privilege to access this instruction page + logic daccess_err; // insufficient privilege to access this data page + logic ptw_active; // PTW is currently walking a page table + logic walking_instr; // PTW is walking because of an ITLB miss + logic ptw_error; // PTW threw an exception + logic ptw_access_exception; // PTW threw an access exception (PMPs) + logic [riscv::PLEN-1:0] ptw_bad_paddr; // PTW PMP exception bad physical addr + + logic [riscv::VLEN-1:0] update_vaddr; + // tlb_update_t update_ptw_itlb, update_ptw_dtlb; + tlb_update_cva6_t update_itlb, update_dtlb, update_shared_tlb; + + logic itlb_lu_access; + pte_cva6_t itlb_content; + logic [ PT_LEVELS-2:0] itlb_is_page; + logic itlb_lu_hit; + + logic dtlb_lu_access; + pte_cva6_t dtlb_content; + logic [ PT_LEVELS-2:0] dtlb_is_page; + logic dtlb_lu_hit; + + logic shared_tlb_access; + logic [riscv::VLEN-1:0] shared_tlb_vaddr; + logic shared_tlb_hit; + + logic itlb_req; + + // Assignments + assign itlb_lu_access = icache_areq_i.fetch_req; + assign dtlb_lu_access = lsu_req_i; + + + cva6_tlb #( + .CVA6Cfg (CVA6Cfg), + .TLB_ENTRIES (INSTR_TLB_ENTRIES), + .ASID_WIDTH (ASID_WIDTH), + .ASID_LEN (ASID_LEN), + .VPN_LEN (VPN_LEN), + .PT_LEVELS (PT_LEVELS), + .pte_cva6_t (pte_cva6_t), + .tlb_update_cva6_t(tlb_update_cva6_t) + ) i_itlb ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .flush_i(flush_tlb_i), + + .update_i(update_itlb), + + .lu_access_i (itlb_lu_access), + .lu_asid_i (asid_i), + .asid_to_be_flushed_i (asid_to_be_flushed_i), + .vaddr_to_be_flushed_i(vaddr_to_be_flushed_i), + .lu_vaddr_i (icache_areq_i.fetch_vaddr), + .lu_content_o (itlb_content), + + .lu_is_page_o(itlb_is_page), + .lu_hit_o(itlb_lu_hit) + ); + + cva6_tlb #( + .CVA6Cfg (CVA6Cfg), + .TLB_ENTRIES (DATA_TLB_ENTRIES), + .ASID_WIDTH (ASID_WIDTH), + .ASID_LEN (ASID_LEN), + .VPN_LEN (VPN_LEN), + .PT_LEVELS (PT_LEVELS), + .pte_cva6_t (pte_cva6_t), + .tlb_update_cva6_t(tlb_update_cva6_t) + ) i_dtlb ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .flush_i(flush_tlb_i), + + .update_i(update_dtlb), + + .lu_access_i (dtlb_lu_access), + .lu_asid_i (asid_i), + .asid_to_be_flushed_i (asid_to_be_flushed_i), + .vaddr_to_be_flushed_i(vaddr_to_be_flushed_i), + .lu_vaddr_i (lsu_vaddr_i), + .lu_content_o (dtlb_content), + + .lu_is_page_o(dtlb_is_page), + .lu_hit_o(dtlb_lu_hit) + ); + + cva6_shared_tlb #( + .CVA6Cfg (CVA6Cfg), + .SHARED_TLB_DEPTH (64), + .SHARED_TLB_WAYS (2), + .ASID_WIDTH (ASID_WIDTH), + .ASID_LEN (ASID_LEN), + .VPN_LEN (VPN_LEN), + .PT_LEVELS (PT_LEVELS), + .pte_cva6_t (pte_cva6_t), + .tlb_update_cva6_t(tlb_update_cva6_t) + ) i_shared_tlb ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .flush_i(flush_tlb_i), + + .enable_translation_i (enable_translation_i), + .en_ld_st_translation_i(en_ld_st_translation_i), + + .asid_i (asid_i), + // from TLBs + // did we miss? + .itlb_access_i(itlb_lu_access), + .itlb_hit_i (itlb_lu_hit), + .itlb_vaddr_i (icache_areq_i.fetch_vaddr), + + .dtlb_access_i(dtlb_lu_access), + .dtlb_hit_i (dtlb_lu_hit), + .dtlb_vaddr_i (lsu_vaddr_i), + + // to TLBs, update logic + .itlb_update_o(update_itlb), + .dtlb_update_o(update_dtlb), + + // Performance counters + .itlb_miss_o(itlb_miss_o), + .dtlb_miss_o(dtlb_miss_o), + + .shared_tlb_access_o(shared_tlb_access), + .shared_tlb_hit_o (shared_tlb_hit), + .shared_tlb_vaddr_o (shared_tlb_vaddr), + + .itlb_req_o (itlb_req), + // to update shared tlb + .shared_tlb_update_i(update_shared_tlb) + ); + + cva6_ptw #( + .CVA6Cfg (CVA6Cfg), + .ASID_WIDTH(ASID_WIDTH), + .VPN_LEN(VPN_LEN), + .PT_LEVELS(PT_LEVELS), + .pte_cva6_t(pte_cva6_t), + .tlb_update_cva6_t(tlb_update_cva6_t) + ) i_ptw ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .flush_i(flush_i), + + .ptw_active_o (ptw_active), + .walking_instr_o (walking_instr), + .ptw_error_o (ptw_error), + .ptw_access_exception_o(ptw_access_exception), + + .lsu_is_store_i(lsu_is_store_i), + // PTW memory interface + .req_port_i (req_port_i), + .req_port_o (req_port_o), + + // to Shared TLB, update logic + .shared_tlb_update_o(update_shared_tlb), + + .update_vaddr_o(update_vaddr), + + .asid_i(asid_i), + + // from shared TLB + // did we miss? + .shared_tlb_access_i(shared_tlb_access), + .shared_tlb_hit_i (shared_tlb_hit), + .shared_tlb_vaddr_i (shared_tlb_vaddr), + + .itlb_req_i(itlb_req), + + // from CSR file + .satp_ppn_i(satp_ppn_i), // ppn from satp + .mxr_i (mxr_i), + + // Performance counters + .shared_tlb_miss_o(), //open for now + + // PMP + .pmpcfg_i (pmpcfg_i), + .pmpaddr_i (pmpaddr_i), + .bad_paddr_o(ptw_bad_paddr) + + ); + + + + // ila_1 i_ila_1 ( + // .clk(clk_i), // input wire clk + // .probe0({req_port_o.address_tag, req_port_o.address_index}), + // .probe1(req_port_o.data_req), // input wire [63:0] probe1 + // .probe2(req_port_i.data_gnt), // input wire [0:0] probe2 + // .probe3(req_port_i.data_rdata), // input wire [0:0] probe3 + // .probe4(req_port_i.data_rvalid), // input wire [0:0] probe4 + // .probe5(ptw_error), // input wire [1:0] probe5 + // .probe6(update_vaddr), // input wire [0:0] probe6 + // .probe7(update_ptw_itlb.valid), // input wire [0:0] probe7 + // .probe8(update_ptw_dtlb.valid), // input wire [0:0] probe8 + // .probe9(dtlb_lu_access), // input wire [0:0] probe9 + // .probe10(lsu_vaddr_i), // input wire [0:0] probe10 + // .probe11(dtlb_lu_hit), // input wire [0:0] probe11 + // .probe12(itlb_lu_access), // input wire [0:0] probe12 + // .probe13(icache_areq_i.fetch_vaddr), // input wire [0:0] probe13 + // .probe14(itlb_lu_hit) // input wire [0:0] probe13 + // ); + + //----------------------- + // Instruction Interface + //----------------------- + logic match_any_execute_region; + logic pmp_instr_allow; + localparam PPNWMin = (riscv::PPNW - 1 > 29) ? 29 : riscv::PPNW - 1; + + assign icache_areq_o.fetch_paddr[11:0] = icache_areq_i.fetch_vaddr[11:0]; + assign icache_areq_o.fetch_paddr[riscv::PLEN-1:PPNWMin+1] = // + (enable_translation_i) ? // + itlb_content.ppn[riscv::PPNW-1:(riscv::PPNW-(riscv::PLEN-PPNWMin-1))] : // + riscv::PLEN'(icache_areq_i.fetch_vaddr[((riscv::PLEN > riscv::VLEN) ? riscv::VLEN : riscv::PLEN )-1:PPNWMin+1]); + + genvar a; + generate + + for (a = 0; a < PT_LEVELS - 1; a++) begin + assign icache_areq_o.fetch_paddr [PPNWMin-((VPN_LEN/PT_LEVELS)*(a)):PPNWMin-((VPN_LEN/PT_LEVELS)*(a+1))+1] = // + (enable_translation_i && (|itlb_is_page[a:0] == 0)) ? // + itlb_content.ppn [(riscv::PPNW - (riscv::PLEN - PPNWMin-1)-((VPN_LEN/PT_LEVELS)*(a))-1):(riscv::PPNW - (riscv::PLEN - PPNWMin-1)-((VPN_LEN/PT_LEVELS)*(a+1)))] : // + icache_areq_i.fetch_vaddr[PPNWMin-((VPN_LEN/PT_LEVELS)*(a)):PPNWMin-((VPN_LEN/PT_LEVELS)*(a+1))+1]; + end + + endgenerate + + // The instruction interface is a simple request response interface + always_comb begin : instr_interface + // MMU disabled: just pass through + icache_areq_o.fetch_valid = icache_areq_i.fetch_req; + // two potential exception sources: + // 1. HPTW threw an exception -> signal with a page fault exception + // 2. We got an access error because of insufficient permissions -> throw an access exception + icache_areq_o.fetch_exception = '0; + // Check whether we are allowed to access this memory region from a fetch perspective + iaccess_err = icache_areq_i.fetch_req && (((priv_lvl_i == riscv::PRIV_LVL_U) && ~itlb_content.u) + || ((priv_lvl_i == riscv::PRIV_LVL_S) && itlb_content.u)); + + // MMU enabled: address from TLB, request delayed until hit. Error when TLB + // hit and no access right or TLB hit and translated address not valid (e.g. + // AXI decode error), or when PTW performs walk due to ITLB miss and raises + // an error. + if (enable_translation_i) begin + // we work with SV39 or SV32, so if VM is enabled, check that all bits [riscv::VLEN-1:riscv::SV-1] are equal + if (icache_areq_i.fetch_req && !((&icache_areq_i.fetch_vaddr[riscv::VLEN-1:riscv::SV-1]) == 1'b1 || (|icache_areq_i.fetch_vaddr[riscv::VLEN-1:riscv::SV-1]) == 1'b0)) begin + icache_areq_o.fetch_exception = { + riscv::INSTR_ACCESS_FAULT, + {{riscv::XLEN - riscv::VLEN{1'b0}}, icache_areq_i.fetch_vaddr}, + 1'b1 + }; + end + + icache_areq_o.fetch_valid = 1'b0; + + // --------- + // ITLB Hit + // -------- + // if we hit the ITLB output the request signal immediately + if (itlb_lu_hit) begin + icache_areq_o.fetch_valid = icache_areq_i.fetch_req; + // we got an access error + if (iaccess_err) begin + // throw a page fault + icache_areq_o.fetch_exception = { + riscv::INSTR_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{1'b0}}, icache_areq_i.fetch_vaddr}, + 1'b1 + }; + end else if (!pmp_instr_allow) begin + icache_areq_o.fetch_exception = { + riscv::INSTR_ACCESS_FAULT, riscv::XLEN'(icache_areq_i.fetch_vaddr), 1'b1 + }; + end + end else + // --------- + // ITLB Miss + // --------- + // watch out for exceptions happening during walking the page table + if (ptw_active && walking_instr) begin + icache_areq_o.fetch_valid = ptw_error | ptw_access_exception; + if (ptw_error) + icache_areq_o.fetch_exception = { + riscv::INSTR_PAGE_FAULT, {{riscv::XLEN - riscv::VLEN{1'b0}}, update_vaddr}, 1'b1 + }; + else + icache_areq_o.fetch_exception = { + riscv::INSTR_ACCESS_FAULT, + ptw_bad_paddr[riscv::PLEN-1:(riscv::PLEN>riscv::VLEN)?(riscv::PLEN-riscv::VLEN) : 0], + 1'b1 + }; + end + end + // if it didn't match any execute region throw an `Instruction Access Fault` + // or: if we are not translating, check PMPs immediately on the paddr + if (!match_any_execute_region || (!enable_translation_i && !pmp_instr_allow)) begin + icache_areq_o.fetch_exception = { + riscv::INSTR_ACCESS_FAULT, + riscv::VLEN'(icache_areq_o.fetch_paddr[riscv::PLEN-1:(riscv::PLEN > riscv::VLEN) ? (riscv::PLEN - riscv::VLEN) : 0]), + 1'b1 + }; + end + end + + // check for execute flag on memory + assign match_any_execute_region = config_pkg::is_inside_execute_regions( + CVA6Cfg, {{64 - riscv::PLEN{1'b0}}, icache_areq_o.fetch_paddr} + ); + + // Instruction fetch + pmp #( + .CVA6Cfg (CVA6Cfg), + .PLEN (riscv::PLEN), + .PMP_LEN (riscv::PLEN - 2), + .NR_ENTRIES(CVA6Cfg.NrPMPEntries) + ) i_pmp_if ( + .addr_i (icache_areq_o.fetch_paddr), + .priv_lvl_i, + // we will always execute on the instruction fetch port + .access_type_i(riscv::ACCESS_EXEC), + // Configuration + .conf_addr_i (pmpaddr_i), + .conf_i (pmpcfg_i), + .allow_o (pmp_instr_allow) + ); + + //----------------------- + // Data Interface + //----------------------- + logic [riscv::VLEN-1:0] lsu_vaddr_n, lsu_vaddr_q; + pte_cva6_t dtlb_pte_n, dtlb_pte_q; + exception_t misaligned_ex_n, misaligned_ex_q; + logic lsu_req_n, lsu_req_q; + logic lsu_is_store_n, lsu_is_store_q; + logic dtlb_hit_n, dtlb_hit_q; + logic [PT_LEVELS-2:0] dtlb_is_page_n, dtlb_is_page_q; + + // check if we need to do translation or if we are always ready (e.g.: we are not translating anything) + assign lsu_dtlb_hit_o = (en_ld_st_translation_i) ? dtlb_lu_hit : 1'b1; + + // Wires to PMP checks + riscv::pmp_access_t pmp_access_type; + logic pmp_data_allow; + + assign lsu_paddr_o[11:0] = lsu_vaddr_q[11:0]; + assign lsu_paddr_o [riscv::PLEN-1:PPNWMin+1] = + (en_ld_st_translation_i && !misaligned_ex_q.valid) ? // + dtlb_pte_q.ppn[riscv::PPNW-1:(riscv::PPNW - (riscv::PLEN - PPNWMin-1))] : // + (riscv::PLEN-PPNWMin)'(lsu_vaddr_q[((riscv::PLEN > riscv::VLEN) ? riscv::VLEN : riscv::PLEN )-1:PPNWMin+1]); + + assign lsu_dtlb_ppn_o [11:0] = + (en_ld_st_translation_i && !misaligned_ex_q.valid) ? // + dtlb_content.ppn[11:0] : // + lsu_vaddr_n[23:12]; + + genvar i; + generate + for (i = 0; i < PT_LEVELS - 1; i++) begin + assign lsu_paddr_o [PPNWMin-((VPN_LEN/PT_LEVELS)*(i)):PPNWMin-((VPN_LEN/PT_LEVELS)*(i+1))+1] = // + (en_ld_st_translation_i && !misaligned_ex_q.valid && (|dtlb_is_page_q[i:0] == 0)) ? // + dtlb_pte_q.ppn [(riscv::PPNW - (riscv::PLEN - PPNWMin-1)-((VPN_LEN/PT_LEVELS)*(i))-1):(riscv::PPNW - (riscv::PLEN - PPNWMin-1)-((VPN_LEN/PT_LEVELS)*(i+1)))] : // + lsu_vaddr_q[PPNWMin-((VPN_LEN/PT_LEVELS)*(i)):PPNWMin-((VPN_LEN/PT_LEVELS)*(i+1))+1]; + + assign lsu_dtlb_ppn_o[PPNWMin-((VPN_LEN/PT_LEVELS)*(i)):PPNWMin-((VPN_LEN/PT_LEVELS)*(i+1))+1] = // + (en_ld_st_translation_i && !misaligned_ex_q.valid && (|dtlb_is_page_q[i:0] == 0)) ? // + dtlb_content.ppn[PPNWMin-((VPN_LEN/PT_LEVELS)*(i)):PPNWMin-((VPN_LEN/PT_LEVELS)*(i+1))+1] : // + (en_ld_st_translation_i && !misaligned_ex_q.valid && (|dtlb_is_page_q[i:0]!=0)? + lsu_vaddr_n[PPNWMin-((VPN_LEN/PT_LEVELS)*(i)):PPNWMin-((VPN_LEN/PT_LEVELS)*(i+1))+1]:// + (VPN_LEN/PT_LEVELS)'(lsu_vaddr_n[((riscv::PLEN > riscv::VLEN) ? riscv::VLEN -1 : (24 + (VPN_LEN/PT_LEVELS)*(PT_LEVELS-i-1) ) -1): (riscv::PLEN > riscv::VLEN) ? 24 :24 + (VPN_LEN/PT_LEVELS)*(PT_LEVELS-i-2)])); + end + if (riscv::IS_XLEN64) begin + assign lsu_dtlb_ppn_o[riscv::PPNW-1:PPNWMin+1] = (en_ld_st_translation_i && !misaligned_ex_q.valid) ? + dtlb_content.ppn[riscv::PPNW-1:PPNWMin+1] : + lsu_vaddr_n[riscv::PLEN-1:PPNWMin+1] ; + end + endgenerate + // The data interface is simpler and only consists of a request/response interface + always_comb begin : data_interface + // save request and DTLB response + lsu_vaddr_n = lsu_vaddr_i; + lsu_req_n = lsu_req_i; + misaligned_ex_n = misaligned_ex_i; + dtlb_pte_n = dtlb_content; + dtlb_hit_n = dtlb_lu_hit; + lsu_is_store_n = lsu_is_store_i; + dtlb_is_page_n = dtlb_is_page; + lsu_valid_o = lsu_req_q; + lsu_exception_o = misaligned_ex_q; + pmp_access_type = lsu_is_store_q ? riscv::ACCESS_WRITE : riscv::ACCESS_READ; + + // mute misaligned exceptions if there is no request otherwise they will throw accidental exceptions + misaligned_ex_n.valid = misaligned_ex_i.valid & lsu_req_i; + + // Check if the User flag is set, then we may only access it in supervisor mode + // if SUM is enabled + daccess_err = (ld_st_priv_lvl_i == riscv::PRIV_LVL_S && !sum_i && dtlb_pte_q.u) || // SUM is not set and we are trying to access a user page in supervisor mode + (ld_st_priv_lvl_i == riscv::PRIV_LVL_U && !dtlb_pte_q.u); // this is not a user page but we are in user mode and trying to access it + // translation is enabled and no misaligned exception occurred + if (en_ld_st_translation_i && !misaligned_ex_q.valid) begin + lsu_valid_o = 1'b0; + + // --------- + // DTLB Hit + // -------- + if (dtlb_hit_q && lsu_req_q) begin + lsu_valid_o = 1'b1; + // exception priority: + // PAGE_FAULTS have higher priority than ACCESS_FAULTS + // virtual memory based exceptions are PAGE_FAULTS + // physical memory based exceptions are ACCESS_FAULTS (PMA/PMP) + + // this is a store + if (lsu_is_store_q) begin + // check if the page is write-able and we are not violating privileges + // also check if the dirty flag is set + if (!dtlb_pte_q.w || daccess_err || !dtlb_pte_q.d) begin + lsu_exception_o = { + riscv::STORE_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, lsu_vaddr_q}, + 1'b1 + }; + // Check if any PMPs are violated + end else if (!pmp_data_allow) begin + lsu_exception_o = { + riscv::ST_ACCESS_FAULT, + riscv::XLEN'(lsu_paddr_o[riscv::PLEN-1:(riscv::PLEN > riscv::VLEN) ? (riscv::PLEN - riscv::VLEN) : 0]), + 1'b1 + }; + end + + // this is a load + end else begin + // check for sufficient access privileges - throw a page fault if necessary + if (daccess_err) begin + lsu_exception_o = { + riscv::LOAD_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, lsu_vaddr_q}, + 1'b1 + }; + // Check if any PMPs are violated + end else if (!pmp_data_allow) begin + lsu_exception_o = { + riscv::LD_ACCESS_FAULT, + lsu_paddr_o[riscv::PLEN-1:(riscv::PLEN>riscv::VLEN)?(riscv::PLEN-riscv::VLEN) : 0], + 1'b1 + }; + end + end + end else + + // --------- + // DTLB Miss + // --------- + // watch out for exceptions + if (ptw_active && !walking_instr) begin + // page table walker threw an exception + if (ptw_error) begin + // an error makes the translation valid + lsu_valid_o = 1'b1; + // the page table walker can only throw page faults + if (lsu_is_store_q) begin + lsu_exception_o = { + riscv::STORE_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, update_vaddr}, + 1'b1 + }; + end else begin + lsu_exception_o = { + riscv::LOAD_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, update_vaddr}, + 1'b1 + }; + end + end + + if (ptw_access_exception) begin + // an error makes the translation valid + lsu_valid_o = 1'b1; + // Any fault of the page table walk should be based of the original access type + lsu_exception_o = { + riscv::LD_ACCESS_FAULT, + ptw_bad_paddr[riscv::PLEN-1:(riscv::PLEN>riscv::VLEN)?(riscv::PLEN-riscv::VLEN) : 0], + 1'b1 + }; + end + end + end // If translation is not enabled, check the paddr immediately against PMPs + else if (lsu_req_q && !misaligned_ex_q.valid && !pmp_data_allow) begin + if (lsu_is_store_q) begin + lsu_exception_o = { + riscv::ST_ACCESS_FAULT, + lsu_paddr_o[riscv::PLEN-1:(riscv::PLEN>riscv::VLEN)?(riscv::PLEN-riscv::VLEN) : 0], + 1'b1 + }; + end else begin + lsu_exception_o = { + riscv::LD_ACCESS_FAULT, + lsu_paddr_o[riscv::PLEN-1:(riscv::PLEN>riscv::VLEN)?(riscv::PLEN-riscv::VLEN) : 0], + 1'b1 + }; + end + end + end + + // Load/store PMP check + pmp #( + .CVA6Cfg (CVA6Cfg), + .PLEN (riscv::PLEN), + .PMP_LEN (riscv::PLEN - 2), + .NR_ENTRIES(CVA6Cfg.NrPMPEntries) + ) i_pmp_data ( + .addr_i (lsu_paddr_o), + .priv_lvl_i (ld_st_priv_lvl_i), + .access_type_i(pmp_access_type), + // Configuration + .conf_addr_i (pmpaddr_i), + .conf_i (pmpcfg_i), + .allow_o (pmp_data_allow) + ); + + // ---------- + // Registers + // ---------- + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + lsu_vaddr_q <= '0; + lsu_req_q <= '0; + misaligned_ex_q <= '0; + dtlb_pte_q <= '0; + dtlb_hit_q <= '0; + lsu_is_store_q <= '0; + dtlb_is_page_q <= '0; + end else begin + lsu_vaddr_q <= lsu_vaddr_n; + lsu_req_q <= lsu_req_n; + misaligned_ex_q <= misaligned_ex_n; + dtlb_pte_q <= dtlb_pte_n; + dtlb_hit_q <= dtlb_hit_n; + lsu_is_store_q <= lsu_is_store_n; + dtlb_is_page_q <= dtlb_is_page_n; + end + end +endmodule \ No newline at end of file diff --git a/core/mmu_unify/cva6_ptw.sv b/core/mmu_unify/cva6_ptw.sv new file mode 100644 index 0000000000..1b9d2c7d46 --- /dev/null +++ b/core/mmu_unify/cva6_ptw.sv @@ -0,0 +1,418 @@ +// Copyright (c) 2021 Thales. +// Copyright and related rights are licensed under the Solderpad Hardware +// License, Version 0.51 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law +// or agreed to in writing, software, hardware and materials distributed under +// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Author: Angela Gonzalez PlanV Technology +// Date: 27/11/2023 +// +// +// Description: Hardware-PTW (Page-Table-Walker) for unified MMU. +// This module is a merge of the Sv32 PTW developed by Sebastien +// Jacq (Thales Research & Technology) and the Sv39 PTW developed +// by Florian Zaruba and David Schaffenrath. +// +// =========================================================================== // +// Revisions : +// Date Version Author Description +// 2023-11-27 0.1 A.Gonzalez PTW UNIFIED for CVA6 +// =========================================================================== // + +/* verilator lint_off WIDTH */ + +module cva6_ptw + import ariane_pkg::*; +#( + parameter type pte_cva6_t = logic, + parameter type tlb_update_cva6_t = logic, + parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, + parameter int ASID_WIDTH = 1, + parameter int unsigned VPN_LEN = 1, + parameter int unsigned PT_LEVELS = 1 +) ( + input logic clk_i, // Clock + input logic rst_ni, // Asynchronous reset active low + input logic flush_i, // flush everything, we need to do this because + // actually everything we do is speculative at this stage + // e.g.: there could be a CSR instruction that changes everything + output logic ptw_active_o, + output logic walking_instr_o, // set when walking for TLB + output logic ptw_error_o, // set when an error occurred + output logic ptw_access_exception_o, // set when an PMP access exception occured + + input logic lsu_is_store_i, // this translation was triggered by a store + // PTW memory interface + input dcache_req_o_t req_port_i, + output dcache_req_i_t req_port_o, + + // to Shared TLB, update logic + output tlb_update_cva6_t shared_tlb_update_o, + + output logic [riscv::VLEN-1:0] update_vaddr_o, + + input logic [ASID_WIDTH-1:0] asid_i, + + // from shared TLB + input logic shared_tlb_access_i, + input logic shared_tlb_hit_i, + input logic [riscv::VLEN-1:0] shared_tlb_vaddr_i, + + input logic itlb_req_i, + + // from CSR file + input logic [riscv::PPNW-1:0] satp_ppn_i, // ppn from satp + input logic mxr_i, + + // Performance counters + output logic shared_tlb_miss_o, + + // PMP + input riscv::pmpcfg_t [15:0] pmpcfg_i, + input logic [15:0][riscv::PLEN-3:0] pmpaddr_i, + output logic [riscv::PLEN-1:0] bad_paddr_o + +); + + // input registers + logic data_rvalid_q; + riscv::xlen_t data_rdata_q; + + pte_cva6_t pte; + assign pte = pte_cva6_t'(data_rdata_q[riscv::PPNW+9:0]); + + + enum logic [2:0] { + IDLE, + WAIT_GRANT, + PTE_LOOKUP, + WAIT_RVALID, + PROPAGATE_ERROR, + PROPAGATE_ACCESS_ERROR, + LATENCY + } + state_q, state_d; + + // page tables levels + logic [PT_LEVELS-1:0] misaligned_page; + logic [PT_LEVELS-2:0] ptw_lvl_n, ptw_lvl_q; + + // is this an instruction page table walk? + logic is_instr_ptw_q, is_instr_ptw_n; + logic global_mapping_q, global_mapping_n; + // latched tag signal + logic tag_valid_n, tag_valid_q; + // register the ASID + logic [ASID_WIDTH-1:0] tlb_update_asid_q, tlb_update_asid_n; + // register the VPN we need to walk + logic [riscv::VLEN-1:0] vaddr_q, vaddr_n; + logic [PT_LEVELS-2:0][(VPN_LEN/PT_LEVELS)-1:0] vaddr_lvl; + // 4 byte aligned physical pointer + logic [riscv::PLEN-1:0] ptw_pptr_q, ptw_pptr_n; + + // Assignments + assign update_vaddr_o = vaddr_q; + + assign ptw_active_o = (state_q != IDLE); + //assign walking_instr_o = is_instr_ptw_q; + assign walking_instr_o = is_instr_ptw_q; + // directly output the correct physical address + assign req_port_o.address_index = ptw_pptr_q[DCACHE_INDEX_WIDTH-1:0]; + assign req_port_o.address_tag = ptw_pptr_q[DCACHE_INDEX_WIDTH+DCACHE_TAG_WIDTH-1:DCACHE_INDEX_WIDTH]; + // we are never going to kill this request + assign req_port_o.kill_req = '0; + // we are never going to write with the HPTW + assign req_port_o.data_wdata = '0; + // we only issue one single request at a time + assign req_port_o.data_id = '0; + + // ----------- + // Shared TLB Update + // ----------- + assign shared_tlb_update_o.vpn = vaddr_q[riscv::SV-1:12]; + + + genvar x; + generate + for (x = 0; x < PT_LEVELS - 1; x++) begin + + // update the correct page table level + assign shared_tlb_update_o.is_page[x] = (ptw_lvl_q == (x)); + + // check if the ppn is correctly aligned: + // 6. If i > 0 and pa.ppn[i − 1 : 0] != 0, this is a misaligned superpage; stop and raise a page-fault + // exception. + assign misaligned_page[x] = (ptw_lvl_q == (x)) && (pte.ppn[(VPN_LEN/PT_LEVELS)*(PT_LEVELS-1-x)-1:0] != '0); + + //record the vaddr corresponding to each level + assign vaddr_lvl[x] = vaddr_q[12+((VPN_LEN/PT_LEVELS)*(PT_LEVELS-x-1))-1:12+((VPN_LEN/PT_LEVELS)*(PT_LEVELS-x-2))]; + end + endgenerate + + + + // output the correct ASID + assign shared_tlb_update_o.asid = tlb_update_asid_q; + // set the global mapping bit + assign shared_tlb_update_o.content = pte | (global_mapping_q << 5); + + + assign req_port_o.tag_valid = tag_valid_q; + + logic allow_access; + + assign bad_paddr_o = ptw_access_exception_o ? ptw_pptr_q : 'b0; + + pmp #( + .CVA6Cfg (CVA6Cfg), + .PLEN (riscv::PLEN), + .PMP_LEN (riscv::PLEN - 2), + .NR_ENTRIES(CVA6Cfg.NrPMPEntries) + ) i_pmp_ptw ( + .addr_i (ptw_pptr_q), + // PTW access are always checked as if in S-Mode... + .priv_lvl_i (riscv::PRIV_LVL_S), + // ...and they are always loads + .access_type_i(riscv::ACCESS_READ), + // Configuration + .conf_addr_i (pmpaddr_i), + .conf_i (pmpcfg_i), + .allow_o (allow_access) + ); + + + assign req_port_o.data_be = riscv::XLEN ==32? + be_gen_32(req_port_o.address_index[1:0], req_port_o.data_size): + be_gen(req_port_o.address_index[2:0], req_port_o.data_size); + + //------------------- + // Page table walker + //------------------- + // A virtual address va is translated into a physical address pa as follows: + // 1. Let a be sptbr.ppn × PAGESIZE, and let i = LEVELS-1. (For Sv39, + // PAGESIZE=2^12 and LEVELS=3.) + // 2. Let pte be the value of the PTE at address a+va.vpn[i]×PTESIZE. (For + // Sv32, PTESIZE=4.) + // 3. If pte.v = 0, or if pte.r = 0 and pte.w = 1, stop and raise an access + // exception. + // 4. Otherwise, the PTE is valid. If pte.r = 1 or pte.x = 1, go to step 5. + // Otherwise, this PTE is a pointer to the next level of the page table. + // Let i=i-1. If i < 0, stop and raise an access exception. Otherwise, let + // a = pte.ppn × PAGESIZE and go to step 2. + // 5. A leaf PTE has been found. Determine if the requested memory access + // is allowed by the pte.r, pte.w, and pte.x bits. If not, stop and + // raise an access exception. Otherwise, the translation is successful. + // Set pte.a to 1, and, if the memory access is a store, set pte.d to 1. + // The translated physical address is given as follows: + // - pa.pgoff = va.pgoff. + // - If i > 0, then this is a superpage translation and + // pa.ppn[i-1:0] = va.vpn[i-1:0]. + // - pa.ppn[LEVELS-1:i] = pte.ppn[LEVELS-1:i]. + always_comb begin : ptw + // default assignments + // PTW memory interface + tag_valid_n = 1'b0; + req_port_o.data_req = 1'b0; + req_port_o.data_size = 2'(PT_LEVELS); + req_port_o.data_we = 1'b0; + ptw_error_o = 1'b0; + ptw_access_exception_o = 1'b0; + shared_tlb_update_o.valid = 1'b0; + is_instr_ptw_n = is_instr_ptw_q; + ptw_lvl_n = ptw_lvl_q; + ptw_pptr_n = ptw_pptr_q; + state_d = state_q; + global_mapping_n = global_mapping_q; + // input registers + tlb_update_asid_n = tlb_update_asid_q; + vaddr_n = vaddr_q; + + shared_tlb_miss_o = 1'b0; + + case (state_q) + + IDLE: begin + // by default we start with the top-most page table + ptw_lvl_n = 0; + global_mapping_n = 1'b0; + is_instr_ptw_n = 1'b0; + // if we got a Shared TLB miss + if (shared_tlb_access_i & ~shared_tlb_hit_i) begin + ptw_pptr_n = { + satp_ppn_i, + shared_tlb_vaddr_i[riscv::SV-1:riscv::SV-(VPN_LEN/PT_LEVELS)], + (PT_LEVELS)'(0) + }; // SATP.PPN * PAGESIZE + VPN*PTESIZE = SATP.PPN * 2^(12) + VPN*4 + is_instr_ptw_n = itlb_req_i; + tlb_update_asid_n = asid_i; + vaddr_n = shared_tlb_vaddr_i; + state_d = WAIT_GRANT; + shared_tlb_miss_o = 1'b1; + end + end + WAIT_GRANT: begin + // send a request out + req_port_o.data_req = 1'b1; + // wait for the WAIT_GRANT + if (req_port_i.data_gnt) begin + // send the tag valid signal one cycle later + tag_valid_n = 1'b1; + state_d = PTE_LOOKUP; + end + end + + PTE_LOOKUP: begin + // we wait for the valid signal + if (data_rvalid_q) begin + + // check if the global mapping bit is set + if (pte.g) global_mapping_n = 1'b1; + + // ------------- + // Invalid PTE + // ------------- + // If pte.v = 0, or if pte.r = 0 and pte.w = 1, stop and raise a page-fault exception. + if (!pte.v || (!pte.r && pte.w)) state_d = PROPAGATE_ERROR; + // ----------- + // Valid PTE + // ----------- + else begin + //state_d = IDLE; + state_d = LATENCY; + // it is a valid PTE + // if pte.r = 1 or pte.x = 1 it is a valid PTE + if (pte.r || pte.x) begin + // Valid translation found (either 4M or 4K entry) + if (is_instr_ptw_q) begin + // ------------ + // Update ITLB + // ------------ + // If page is not executable, we can directly raise an error. This + // doesn't put a useless entry into the TLB. The same idea applies + // to the access flag since we let the access flag be managed by SW. + if (!pte.x || !pte.a) state_d = PROPAGATE_ERROR; + else shared_tlb_update_o.valid = 1'b1; + + end else begin + // ------------ + // Update DTLB + // ------------ + // Check if the access flag has been set, otherwise throw a page-fault + // and let the software handle those bits. + // If page is not readable (there are no write-only pages) + // we can directly raise an error. This doesn't put a useless + // entry into the TLB. + if (pte.a && (pte.r || (pte.x && mxr_i))) begin + shared_tlb_update_o.valid = 1'b1; + end else begin + state_d = PROPAGATE_ERROR; + end + // Request is a store: perform some additional checks + // If the request was a store and the page is not write-able, raise an error + // the same applies if the dirty flag is not set + if (lsu_is_store_i && (!pte.w || !pte.d)) begin + shared_tlb_update_o.valid = 1'b0; + state_d = PROPAGATE_ERROR; + end + end + //if there is a misaligned page, propagate error + if (|misaligned_page) begin + state_d = PROPAGATE_ERROR; + shared_tlb_update_o.valid = 1'b0; + end + // this is a pointer to the next TLB level + end else begin + // pointer to next level of page table + if (ptw_lvl_q == PT_LEVELS - 1) begin + // Should already be the last level page table => Error + ptw_lvl_n = PT_LEVELS - 1; + state_d = PROPAGATE_ERROR; + end else begin + // we are in the second level now + ptw_lvl_n = ptw_lvl_q + 1; + ptw_pptr_n = {pte.ppn, vaddr_lvl[ptw_lvl_q], (PT_LEVELS)'(0)}; + state_d = WAIT_GRANT; + end + end + end + + // Check if this access was actually allowed from a PMP perspective + if (!allow_access) begin + shared_tlb_update_o.valid = 1'b0; + // we have to return the failed address in bad_addr + ptw_pptr_n = ptw_pptr_q; + state_d = PROPAGATE_ACCESS_ERROR; + end + end + // we've got a data WAIT_GRANT so tell the cache that the tag is valid + end + // Propagate error to MMU/LSU + PROPAGATE_ERROR: begin + state_d = LATENCY; + ptw_error_o = 1'b1; + end + PROPAGATE_ACCESS_ERROR: begin + state_d = LATENCY; + ptw_access_exception_o = 1'b1; + end + // wait for the rvalid before going back to IDLE + WAIT_RVALID: begin + if (data_rvalid_q) state_d = IDLE; + end + LATENCY: begin + state_d = IDLE; + end + default: begin + state_d = IDLE; + end + endcase + + // ------- + // Flush + // ------- + // should we have flushed before we got an rvalid, wait for it until going back to IDLE + if (flush_i) begin + // on a flush check whether we are + // 1. in the PTE Lookup check whether we still need to wait for an rvalid + // 2. waiting for a grant, if so: wait for it + // if not, go back to idle + if (((state_q inside {PTE_LOOKUP, WAIT_RVALID}) && !data_rvalid_q) || + ((state_q == WAIT_GRANT) && req_port_i.data_gnt)) + state_d = WAIT_RVALID; + else state_d = LATENCY; + end + end + + // sequential process + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + state_q <= IDLE; + is_instr_ptw_q <= 1'b0; + ptw_lvl_q <= 0; + tag_valid_q <= 1'b0; + tlb_update_asid_q <= '0; + vaddr_q <= '0; + ptw_pptr_q <= '0; + global_mapping_q <= 1'b0; + data_rdata_q <= '0; + data_rvalid_q <= 1'b0; + end else begin + state_q <= state_d; + ptw_pptr_q <= ptw_pptr_n; + is_instr_ptw_q <= is_instr_ptw_n; + ptw_lvl_q <= ptw_lvl_n; + tag_valid_q <= tag_valid_n; + tlb_update_asid_q <= tlb_update_asid_n; + vaddr_q <= vaddr_n; + global_mapping_q <= global_mapping_n; + data_rdata_q <= req_port_i.data_rdata; + data_rvalid_q <= req_port_i.data_rvalid; + end + end + +endmodule +/* verilator lint_on WIDTH */ diff --git a/core/mmu_unify/cva6_shared_tlb.sv b/core/mmu_unify/cva6_shared_tlb.sv new file mode 100644 index 0000000000..f58958ef48 --- /dev/null +++ b/core/mmu_unify/cva6_shared_tlb.sv @@ -0,0 +1,397 @@ +// Copyright (c) 2023 Thales. +// Copyright and related rights are licensed under the Solderpad Hardware +// License, Version 0.51 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law +// or agreed to in writing, software, hardware and materials distributed under +// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Author: Angela Gonzalez PlanV Technology +// Date: 24/11/2023 +// +// Description: N-way associative shared TLB, it allows to reduce the number +// of ITLB and DTLB entries. +// + +/* verilator lint_off WIDTH */ + +module cva6_shared_tlb + import ariane_pkg::*; +#( + parameter type pte_cva6_t = logic, + parameter type tlb_update_cva6_t = logic, + parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, + parameter int SHARED_TLB_DEPTH = 64, + parameter int SHARED_TLB_WAYS = 2, + parameter int ASID_WIDTH = 1, + parameter int unsigned ASID_LEN = 1, + parameter int unsigned VPN_LEN = 1, + parameter int unsigned PT_LEVELS = 1 +) ( + input logic clk_i, // Clock + input logic rst_ni, // Asynchronous reset active low + input logic flush_i, + + input logic enable_translation_i, // CSRs indicate to enable ??? + input logic en_ld_st_translation_i, // enable virtual memory translation for load/stores + + input logic [ASID_WIDTH-1:0] asid_i, + + // from TLBs + // did we miss? + input logic itlb_access_i, + input logic itlb_hit_i, + input logic [riscv::VLEN-1:0] itlb_vaddr_i, + + input logic dtlb_access_i, + input logic dtlb_hit_i, + input logic [riscv::VLEN-1:0] dtlb_vaddr_i, + + // to TLBs, update logic + output tlb_update_cva6_t itlb_update_o, + output tlb_update_cva6_t dtlb_update_o, + + // Performance counters + output logic itlb_miss_o, + output logic dtlb_miss_o, + + output logic shared_tlb_access_o, + output logic shared_tlb_hit_o, + output logic [riscv::VLEN-1:0] shared_tlb_vaddr_o, + + output logic itlb_req_o, + + // Update shared TLB in case of miss + input tlb_update_cva6_t shared_tlb_update_i + +); + + function logic [SHARED_TLB_WAYS-1:0] shared_tlb_way_bin2oh(input logic [$clog2(SHARED_TLB_WAYS +)-1:0] in); + logic [SHARED_TLB_WAYS-1:0] out; + out = '0; + out[in] = 1'b1; + return out; + endfunction + + typedef struct packed { + logic [ASID_LEN-1:0] asid; + logic [PT_LEVELS-1:0][(VPN_LEN/PT_LEVELS)-1:0] vpn; + logic [PT_LEVELS-2:0] is_page; + } shared_tag_t; + + shared_tag_t shared_tag_wr; + shared_tag_t [SHARED_TLB_WAYS-1:0] shared_tag_rd; + + logic [SHARED_TLB_DEPTH-1:0][SHARED_TLB_WAYS-1:0] shared_tag_valid_q, shared_tag_valid_d; + + logic [ SHARED_TLB_WAYS-1:0] shared_tag_valid; + + logic [ SHARED_TLB_WAYS-1:0] tag_wr_en; + logic [$clog2(SHARED_TLB_DEPTH)-1:0] tag_wr_addr; + logic [ $bits(shared_tag_t)-1:0] tag_wr_data; + + logic [ SHARED_TLB_WAYS-1:0] tag_rd_en; + logic [$clog2(SHARED_TLB_DEPTH)-1:0] tag_rd_addr; + logic [ $bits(shared_tag_t)-1:0] tag_rd_data [SHARED_TLB_WAYS-1:0]; + + logic [ SHARED_TLB_WAYS-1:0] tag_req; + logic [ SHARED_TLB_WAYS-1:0] tag_we; + logic [$clog2(SHARED_TLB_DEPTH)-1:0] tag_addr; + + logic [ SHARED_TLB_WAYS-1:0] pte_wr_en; + logic [$clog2(SHARED_TLB_DEPTH)-1:0] pte_wr_addr; + logic [ $bits(pte_cva6_t)-1:0] pte_wr_data; + + logic [ SHARED_TLB_WAYS-1:0] pte_rd_en; + logic [$clog2(SHARED_TLB_DEPTH)-1:0] pte_rd_addr; + logic [ $bits(pte_cva6_t)-1:0] pte_rd_data [SHARED_TLB_WAYS-1:0]; + + logic [ SHARED_TLB_WAYS-1:0] pte_req; + logic [ SHARED_TLB_WAYS-1:0] pte_we; + logic [$clog2(SHARED_TLB_DEPTH)-1:0] pte_addr; + + logic [PT_LEVELS-1:0][(VPN_LEN/PT_LEVELS)-1:0] vpn_d, vpn_q; + logic [SHARED_TLB_WAYS-1:0][PT_LEVELS-1:0] vpn_match; + logic [SHARED_TLB_WAYS-1:0][PT_LEVELS-1:0] page_match; + logic [SHARED_TLB_WAYS-1:0][PT_LEVELS-1:0] level_match; + + pte_cva6_t [SHARED_TLB_WAYS-1:0] pte; + + logic [riscv::VLEN-1-12:0] itlb_vpn_q; + logic [riscv::VLEN-1-12:0] dtlb_vpn_q; + + logic [ASID_WIDTH-1:0] tlb_update_asid_q, tlb_update_asid_d; + + logic shared_tlb_access_q, shared_tlb_access_d; + logic shared_tlb_hit_d; + logic [riscv::VLEN-1:0] shared_tlb_vaddr_q, shared_tlb_vaddr_d; + + logic itlb_req_d, itlb_req_q; + logic dtlb_req_d, dtlb_req_q; + + // replacement strategy + logic [SHARED_TLB_WAYS-1:0] way_valid; + logic update_lfsr; // shift the LFSR + logic [$clog2(SHARED_TLB_WAYS)-1:0] inv_way; // first non-valid encountered + logic [$clog2(SHARED_TLB_WAYS)-1:0] rnd_way; // random index for replacement + logic [$clog2(SHARED_TLB_WAYS)-1:0] repl_way; // way to replace + logic [SHARED_TLB_WAYS-1:0] repl_way_oh_d; // way to replace (onehot) + logic all_ways_valid; // we need to switch repl strategy since all are valid + + assign shared_tlb_access_o = shared_tlb_access_q; + assign shared_tlb_hit_o = shared_tlb_hit_d; + assign shared_tlb_vaddr_o = shared_tlb_vaddr_q; + + assign itlb_req_o = itlb_req_q; + + genvar i, x; + generate + for (i = 0; i < SHARED_TLB_WAYS; i++) begin + //identify page_match for all TLB Entries + + for (x = 0; x < PT_LEVELS; x++) begin + assign page_match[i][x] = x == 0 ? 1 : shared_tag_rd[i].is_page[PT_LEVELS-1-x]; + assign vpn_match[i][x] = vpn_q[x] == shared_tag_rd[i].vpn[x]; + assign level_match[i][x] = &vpn_match[i][PT_LEVELS-1:x] & page_match[i][x]; + end + end + endgenerate + + genvar w; + generate + for (w = 0; w < PT_LEVELS; w++) begin + assign vpn_d[w] = (enable_translation_i & itlb_access_i & ~itlb_hit_i & ~dtlb_access_i) ? // + itlb_vaddr_i[12+((VPN_LEN/PT_LEVELS)*(w+1))-1:12+((VPN_LEN/PT_LEVELS)*w)] : // + ((en_ld_st_translation_i & dtlb_access_i & ~dtlb_hit_i) ? // + dtlb_vaddr_i[12+((VPN_LEN/PT_LEVELS)*(w+1))-1:12+((VPN_LEN/PT_LEVELS)*w)] : vpn_q[w]); + end + endgenerate + + + /////////////////////////////////////////////////////// + // tag comparison, hit generation + /////////////////////////////////////////////////////// + always_comb begin : itlb_dtlb_miss + itlb_miss_o = 1'b0; + dtlb_miss_o = 1'b0; + + tag_rd_en = '0; + pte_rd_en = '0; + + itlb_req_d = 1'b0; + dtlb_req_d = 1'b0; + + tlb_update_asid_d = tlb_update_asid_q; + + shared_tlb_access_d = '0; + shared_tlb_vaddr_d = shared_tlb_vaddr_q; + + tag_rd_addr = '0; + pte_rd_addr = '0; + + // if we got an ITLB miss + if (enable_translation_i & itlb_access_i & ~itlb_hit_i & ~dtlb_access_i) begin + tag_rd_en = '1; + tag_rd_addr = itlb_vaddr_i[12+:$clog2(SHARED_TLB_DEPTH)]; + pte_rd_en = '1; + pte_rd_addr = itlb_vaddr_i[12+:$clog2(SHARED_TLB_DEPTH)]; + + itlb_miss_o = 1'b1; + itlb_req_d = 1'b1; + + tlb_update_asid_d = asid_i; + + shared_tlb_access_d = 1'b1; + shared_tlb_vaddr_d = itlb_vaddr_i; + + // we got an DTLB miss + end else if (en_ld_st_translation_i & dtlb_access_i & ~dtlb_hit_i) begin + tag_rd_en = '1; + tag_rd_addr = dtlb_vaddr_i[12+:$clog2(SHARED_TLB_DEPTH)]; + pte_rd_en = '1; + pte_rd_addr = dtlb_vaddr_i[12+:$clog2(SHARED_TLB_DEPTH)]; + + dtlb_miss_o = 1'b1; + dtlb_req_d = 1'b1; + + tlb_update_asid_d = asid_i; + + shared_tlb_access_d = 1'b1; + shared_tlb_vaddr_d = dtlb_vaddr_i; + end + end //itlb_dtlb_miss + + always_comb begin : tag_comparison + shared_tlb_hit_d = 1'b0; + dtlb_update_o = '0; + itlb_update_o = '0; + //number of ways + for (int unsigned i = 0; i < SHARED_TLB_WAYS; i++) begin + if (shared_tag_valid[i] && ((tlb_update_asid_q == shared_tag_rd[i].asid) || pte[i].g)) begin + if (|level_match[i]) begin + shared_tlb_hit_d = 1'b1; + if (itlb_req_q) begin + itlb_update_o.valid = 1'b1; + itlb_update_o.vpn = itlb_vpn_q; + itlb_update_o.is_page = shared_tag_rd[i].is_page; + itlb_update_o.asid = tlb_update_asid_q; + itlb_update_o.content = pte[i]; + end else if (dtlb_req_q) begin + dtlb_update_o.valid = 1'b1; + dtlb_update_o.vpn = dtlb_vpn_q; + dtlb_update_o.is_page = shared_tag_rd[i].is_page; + dtlb_update_o.asid = tlb_update_asid_q; + dtlb_update_o.content = pte[i]; + end + end + end + end + end //tag_comparison + + // sequential process + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + itlb_vpn_q <= '0; + dtlb_vpn_q <= '0; + tlb_update_asid_q <= '0; + shared_tlb_access_q <= '0; + shared_tlb_vaddr_q <= '0; + shared_tag_valid_q <= '0; + vpn_q <= 0; + itlb_req_q <= '0; + dtlb_req_q <= '0; + shared_tag_valid <= '0; + end else begin + itlb_vpn_q <= itlb_vaddr_i[riscv::SV-1:12]; + dtlb_vpn_q <= dtlb_vaddr_i[riscv::SV-1:12]; + tlb_update_asid_q <= tlb_update_asid_d; + shared_tlb_access_q <= shared_tlb_access_d; + shared_tlb_vaddr_q <= shared_tlb_vaddr_d; + shared_tag_valid_q <= shared_tag_valid_d; + vpn_q <= vpn_d; + itlb_req_q <= itlb_req_d; + dtlb_req_q <= dtlb_req_d; + shared_tag_valid <= shared_tag_valid_q[tag_rd_addr]; + end + end + + // ------------------ + // Update and Flush + // ------------------ + always_comb begin : update_flush + shared_tag_valid_d = shared_tag_valid_q; + tag_wr_en = '0; + pte_wr_en = '0; + + if (flush_i) begin + shared_tag_valid_d = '0; + end else if (shared_tlb_update_i.valid) begin + for (int unsigned i = 0; i < SHARED_TLB_WAYS; i++) begin + if (repl_way_oh_d[i]) begin + shared_tag_valid_d[shared_tlb_update_i.vpn[$clog2(SHARED_TLB_DEPTH)-1:0]][i] = 1'b1; + tag_wr_en[i] = 1'b1; + pte_wr_en[i] = 1'b1; + end + end + end + end //update_flush + + assign shared_tag_wr.asid = shared_tlb_update_i.asid; + // assign shared_tag_wr.vpn[1] = shared_tlb_update_i.vpn[19:10]; + // assign shared_tag_wr.vpn[0] = shared_tlb_update_i.vpn[9:0]; + assign shared_tag_wr.is_page = shared_tlb_update_i.is_page; + + + genvar z; + generate + for (z = 0; z < PT_LEVELS; z++) begin + assign shared_tag_wr.vpn[z] = shared_tlb_update_i.vpn[((VPN_LEN/PT_LEVELS)*(z+1))-1:((VPN_LEN/PT_LEVELS)*z)]; + end + endgenerate + + + assign tag_wr_addr = shared_tlb_update_i.vpn[$clog2(SHARED_TLB_DEPTH)-1:0]; + assign tag_wr_data = shared_tag_wr; + + assign pte_wr_addr = shared_tlb_update_i.vpn[$clog2(SHARED_TLB_DEPTH)-1:0]; + assign pte_wr_data = shared_tlb_update_i.content; + + assign way_valid = shared_tag_valid_q[shared_tlb_update_i.vpn[$clog2(SHARED_TLB_DEPTH)-1:0]]; + assign repl_way = (all_ways_valid) ? rnd_way : inv_way; + assign update_lfsr = shared_tlb_update_i.valid & all_ways_valid; + assign repl_way_oh_d = (shared_tlb_update_i.valid) ? shared_tlb_way_bin2oh(repl_way) : '0; + + lzc #( + .WIDTH(SHARED_TLB_WAYS) + ) i_lzc ( + .in_i (~way_valid), + .cnt_o (inv_way), + .empty_o(all_ways_valid) + ); + + lfsr #( + .LfsrWidth(8), + .OutWidth ($clog2(SHARED_TLB_WAYS)) + ) i_lfsr ( + .clk_i (clk_i), + .rst_ni(rst_ni), + .en_i (update_lfsr), + .out_o (rnd_way) + ); + + /////////////////////////////////////////////////////// + // memory arrays and regs + /////////////////////////////////////////////////////// + + assign tag_req = tag_wr_en | tag_rd_en; + assign tag_we = tag_wr_en; + assign tag_addr = tag_wr_en ? tag_wr_addr : tag_rd_addr; + + assign pte_req = pte_wr_en | pte_rd_en; + assign pte_we = pte_wr_en; + assign pte_addr = pte_wr_en ? pte_wr_addr : pte_rd_addr; + + for (genvar i = 0; i < SHARED_TLB_WAYS; i++) begin : gen_sram + // Tag RAM + sram #( + .DATA_WIDTH($bits(shared_tag_t)), + .NUM_WORDS (SHARED_TLB_DEPTH) + ) tag_sram ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .req_i (tag_req[i]), + .we_i (tag_we[i]), + .addr_i (tag_addr), + .wuser_i('0), + .wdata_i(tag_wr_data), + .be_i ('1), + .ruser_o(), + .rdata_o(tag_rd_data[i]) + ); + + assign shared_tag_rd[i] = shared_tag_t'(tag_rd_data[i]); + + // PTE RAM + sram #( + .DATA_WIDTH($bits(pte_cva6_t)), + .NUM_WORDS (SHARED_TLB_DEPTH) + ) pte_sram ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .req_i (pte_req[i]), + .we_i (pte_we[i]), + .addr_i (pte_addr), + .wuser_i('0), + .wdata_i(pte_wr_data), + .be_i ('1), + .ruser_o(), + .rdata_o(pte_rd_data[i]) + ); + assign pte[i] = pte_cva6_t'(pte_rd_data[i]); + end +endmodule + +/* verilator lint_on WIDTH */ diff --git a/core/mmu_unify/cva6_tlb.sv b/core/mmu_unify/cva6_tlb.sv new file mode 100644 index 0000000000..61bbc4754c --- /dev/null +++ b/core/mmu_unify/cva6_tlb.sv @@ -0,0 +1,306 @@ +// Copyright (c) 2021 Thales. +// Copyright and related rights are licensed under the Solderpad Hardware +// License, Version 0.51 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law +// or agreed to in writing, software, hardware and materials distributed under +// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Author: Angela Gonzalez PlanV Technology +// Date: 20/11/2023 +// +// Description: Translation Lookaside Buffer, parameterizable to Sv32 or Sv39 , +// fully set-associative +// This module is an merge of the Sv32 TLB developed by Sebastien +// Jacq (Thales Research & Technology) and the Sv39 TLB developed +// by Florian Zaruba and David Schaffenrath. +// +// =========================================================================== // +// Revisions : +// Date Version Author Description +// 2023-11-20 0.1 A.Gonzalez Generic TLB for CVA6 +// =========================================================================== // + +module cva6_tlb + import ariane_pkg::*; +#( + parameter type pte_cva6_t = logic, + parameter type tlb_update_cva6_t = logic, + parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, + parameter int unsigned TLB_ENTRIES = 4, + parameter int unsigned ASID_WIDTH = 1, + parameter int unsigned ASID_LEN = 1, + parameter int unsigned VPN_LEN = 1, + parameter int unsigned PT_LEVELS = 1 +) ( + input logic clk_i, // Clock + input logic rst_ni, // Asynchronous reset active low + input logic flush_i, // Flush signal + // Update TLB + input tlb_update_cva6_t update_i, + // Lookup signals + input logic lu_access_i, + input logic [ASID_WIDTH-1:0] lu_asid_i, + input logic [riscv::VLEN-1:0] lu_vaddr_i, + output pte_cva6_t lu_content_o, + input logic [ASID_WIDTH-1:0] asid_to_be_flushed_i, + input logic [riscv::VLEN-1:0] vaddr_to_be_flushed_i, + output logic [PT_LEVELS-2:0] lu_is_page_o, + output logic lu_hit_o +); + + // Sv32 defines two levels of page tables, Sv39 defines 3 + struct packed { + logic [ASID_LEN-1:0] asid; + logic [PT_LEVELS-1:0][(VPN_LEN/PT_LEVELS)-1:0] vpn; + logic [PT_LEVELS-2:0] is_page; + logic valid; + } [TLB_ENTRIES-1:0] + tags_q, tags_n; + + pte_cva6_t [TLB_ENTRIES-1:0] content_q, content_n; + logic [TLB_ENTRIES-1:0][PT_LEVELS-1:0] vpn_match; + logic [TLB_ENTRIES-1:0][PT_LEVELS-1:0] page_match; + logic [TLB_ENTRIES-1:0][PT_LEVELS-1:0] level_match; + logic [TLB_ENTRIES-1:0][PT_LEVELS-1:0] vaddr_vpn_match; + logic [TLB_ENTRIES-1:0][PT_LEVELS-1:0] vaddr_level_match; + logic [TLB_ENTRIES-1:0] lu_hit; // to replacement logic + logic [TLB_ENTRIES-1:0] replace_en; // replace the following entry, set by replacement strategy + //------------- + // Translation + //------------- + + //at level 0 make page match always 1 + //build level match vector according to vpn_match and page_match + //a level has a match if all vpn of higher levels and current have a match, + //AND the page_match is also set + //At level 0 the page match is always set, so this level will have a match + //if all vpn levels match + genvar i, x; + generate + for (i = 0; i < TLB_ENTRIES; i++) begin + //identify page_match for all TLB Entries + // assign page_match[i] = (tags_q[i].is_page[PT_LEVELS-2:0])*2 +1; + + for (x = 0; x < PT_LEVELS; x++) begin + //identify page_match for all TLB Entries + assign page_match[i][x] = x == 0 ? 1 : tags_q[i].is_page[PT_LEVELS-1-x]; + //identify if vpn matches at all PT levels for all TLB entries + assign vpn_match[i][x] = lu_vaddr_i[12+((VPN_LEN/PT_LEVELS)*(x+1))-1:12+((VPN_LEN/PT_LEVELS)*x)] == tags_q[i].vpn[x]; + //identify if there is a hit at each PT level for all TLB entries + assign level_match[i][x] = &vpn_match[i][PT_LEVELS-1:x] & page_match[i][x]; + //identify if virtual address vpn matches at all PT levels for all TLB entries + assign vaddr_vpn_match[i][x] = vaddr_to_be_flushed_i[12+((VPN_LEN/PT_LEVELS)*(x+1))-1:12+((VPN_LEN/PT_LEVELS)*x)] == tags_q[i].vpn[x]; + //identify if there is a hit at each PT level for all TLB entries + assign vaddr_level_match[i][x] = &vaddr_vpn_match[i][PT_LEVELS-1:x] & page_match[i][x]; + //update vpn field in tags_n for each TLB when the update is valid and the tag needs to be replaced + assign tags_n[i].vpn[x] = (!flush_i && update_i.valid && replace_en[i]) ? update_i.vpn[(1+x)*(VPN_LEN/PT_LEVELS)-1:x*(VPN_LEN/PT_LEVELS)] : tags_q[i].vpn[x]; + end + end + endgenerate + + always_comb begin : translation + + // default assignment + lu_hit = '{default: 0}; + lu_hit_o = 1'b0; + lu_content_o = '{default: 0}; + lu_is_page_o = 0; + + for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin + // first level match, this may be a page, check the ASID flags as well + // if the entry is associated to a global address, don't match the ASID (ASID is don't care) + if (tags_q[i].valid && ((lu_asid_i == tags_q[i].asid[ASID_WIDTH-1:0]) || content_q[i].g)) begin + // find if there is a match at any level + if (|level_match[i]) begin + lu_is_page_o = tags_q[i].is_page; //the page size is indicated here + lu_content_o = content_q[i]; + lu_hit_o = 1'b1; + lu_hit[i] = 1'b1; + end + end + end + end + + logic asid_to_be_flushed_is0; // indicates that the ASID provided by SFENCE.VMA (rs2)is 0, active high + logic vaddr_to_be_flushed_is0; // indicates that the VADDR provided by SFENCE.VMA (rs1)is 0, active high + + assign asid_to_be_flushed_is0 = ~(|asid_to_be_flushed_i); + assign vaddr_to_be_flushed_is0 = ~(|vaddr_to_be_flushed_i); + + // ------------------ + // Update and Flush + // ------------------ + always_comb begin : update_flush + content_n = content_q; + + for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin + tags_n[i].asid = tags_q[i].asid; + tags_n[i].is_page = tags_q[i].is_page; + tags_n[i].valid = tags_q[i].valid; + if (flush_i) begin + // invalidate logic + // flush everything if ASID is 0 and vaddr is 0 ("SFENCE.VMA x0 x0" case) + if (asid_to_be_flushed_is0 && vaddr_to_be_flushed_is0) tags_n[i].valid = 1'b0; + // flush vaddr in all addressing space ("SFENCE.VMA vaddr x0" case), it should happen only for leaf pages + else if (asid_to_be_flushed_is0 && (|vaddr_level_match[i]) && (~vaddr_to_be_flushed_is0)) + tags_n[i].valid = 1'b0; + // the entry is flushed if it's not global and asid and vaddr both matches with the entry to be flushed ("SFENCE.VMA vaddr asid" case) + else if ((!content_q[i].g) && (|vaddr_level_match[i]) && (asid_to_be_flushed_i == tags_q[i].asid[ASID_WIDTH-1:0]) && (!vaddr_to_be_flushed_is0) && (!asid_to_be_flushed_is0)) + tags_n[i].valid = 1'b0; + // the entry is flushed if it's not global, and the asid matches and vaddr is 0. ("SFENCE.VMA 0 asid" case) + else if ((!content_q[i].g) && (vaddr_to_be_flushed_is0) && (asid_to_be_flushed_i == tags_q[i].asid[ASID_WIDTH-1:0]) && (!asid_to_be_flushed_is0)) + tags_n[i].valid = 1'b0; + // normal replacement + end else if (update_i.valid & replace_en[i]) begin + // update tag array + tags_n[i].asid = update_i.asid; + tags_n[i].is_page = update_i.is_page; + tags_n[i].valid = 1'b1; + + // and content as well + content_n[i] = update_i.content; + end + end + end + + // ----------------------------------------------- + // PLRU - Pseudo Least Recently Used Replacement + // ----------------------------------------------- + logic [2*(TLB_ENTRIES-1)-1:0] plru_tree_q, plru_tree_n; + logic en; + int unsigned idx_base, shift, new_index; + always_comb begin : plru_replacement + plru_tree_n = plru_tree_q; + en = '0; + idx_base = '0; + shift = '0; + new_index = '0; + // The PLRU-tree indexing: + // lvl0 0 + // / \ + // / \ + // lvl1 1 2 + // / \ / \ + // lvl2 3 4 5 6 + // / \ /\/\ /\ + // ... ... ... ... + // Just predefine which nodes will be set/cleared + // E.g. for a TLB with 8 entries, the for-loop is semantically + // equivalent to the following pseudo-code: + // unique case (1'b1) + // lu_hit[7]: plru_tree_n[0, 2, 6] = {1, 1, 1}; + // lu_hit[6]: plru_tree_n[0, 2, 6] = {1, 1, 0}; + // lu_hit[5]: plru_tree_n[0, 2, 5] = {1, 0, 1}; + // lu_hit[4]: plru_tree_n[0, 2, 5] = {1, 0, 0}; + // lu_hit[3]: plru_tree_n[0, 1, 4] = {0, 1, 1}; + // lu_hit[2]: plru_tree_n[0, 1, 4] = {0, 1, 0}; + // lu_hit[1]: plru_tree_n[0, 1, 3] = {0, 0, 1}; + // lu_hit[0]: plru_tree_n[0, 1, 3] = {0, 0, 0}; + // default: begin /* No hit */ end + // endcase + for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin + // we got a hit so update the pointer as it was least recently used + if (lu_hit[i] & lu_access_i) begin + // Set the nodes to the values we would expect + for (int unsigned lvl = 0; lvl < $clog2(TLB_ENTRIES); lvl++) begin + idx_base = $unsigned((2 ** lvl) - 1); + // lvl0 <=> MSB, lvl1 <=> MSB-1, ... + shift = $clog2(TLB_ENTRIES) - lvl; + // to circumvent the 32 bit integer arithmetic assignment + new_index = ~((i >> (shift - 1)) & 32'b1); + plru_tree_n[idx_base+(i>>shift)] = new_index[0]; + end + end + end + // Decode tree to write enable signals + // Next for-loop basically creates the following logic for e.g. an 8 entry + // TLB (note: pseudo-code obviously): + // replace_en[7] = &plru_tree_q[ 6, 2, 0]; //plru_tree_q[0,2,6]=={1,1,1} + // replace_en[6] = &plru_tree_q[~6, 2, 0]; //plru_tree_q[0,2,6]=={1,1,0} + // replace_en[5] = &plru_tree_q[ 5,~2, 0]; //plru_tree_q[0,2,5]=={1,0,1} + // replace_en[4] = &plru_tree_q[~5,~2, 0]; //plru_tree_q[0,2,5]=={1,0,0} + // replace_en[3] = &plru_tree_q[ 4, 1,~0]; //plru_tree_q[0,1,4]=={0,1,1} + // replace_en[2] = &plru_tree_q[~4, 1,~0]; //plru_tree_q[0,1,4]=={0,1,0} + // replace_en[1] = &plru_tree_q[ 3,~1,~0]; //plru_tree_q[0,1,3]=={0,0,1} + // replace_en[0] = &plru_tree_q[~3,~1,~0]; //plru_tree_q[0,1,3]=={0,0,0} + // For each entry traverse the tree. If every tree-node matches, + // the corresponding bit of the entry's index, this is + // the next entry to replace. + for (int unsigned i = 0; i < TLB_ENTRIES; i += 1) begin + en = 1'b1; + for (int unsigned lvl = 0; lvl < $clog2(TLB_ENTRIES); lvl++) begin + idx_base = $unsigned((2 ** lvl) - 1); + // lvl0 <=> MSB, lvl1 <=> MSB-1, ... + shift = $clog2(TLB_ENTRIES) - lvl; + + // en &= plru_tree_q[idx_base + (i>>shift)] == ((i >> (shift-1)) & 1'b1); + new_index = (i >> (shift - 1)) & 32'b1; + if (new_index[0]) begin + en &= plru_tree_q[idx_base+(i>>shift)]; + end else begin + en &= ~plru_tree_q[idx_base+(i>>shift)]; + end + end + replace_en[i] = en; + end + end + + // sequential process + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + tags_q <= '{default: 0}; + content_q <= '{default: 0}; + plru_tree_q <= '{default: 0}; + end else begin + tags_q <= tags_n; + content_q <= content_n; + plru_tree_q <= plru_tree_n; + end + end + //-------------- + // Sanity checks + //-------------- + + //pragma translate_off +`ifndef VERILATOR + + initial begin : p_assertions + assert ((TLB_ENTRIES % 2 == 0) && (TLB_ENTRIES > 1)) + else begin + $error("TLB size must be a multiple of 2 and greater than 1"); + $stop(); + end + assert (ASID_WIDTH >= 1) + else begin + $error("ASID width must be at least 1"); + $stop(); + end + end + + // Just for checking + function int countSetBits(logic [TLB_ENTRIES-1:0] vector); + automatic int count = 0; + foreach (vector[idx]) begin + count += vector[idx]; + end + return count; + endfunction + + assert property (@(posedge clk_i) (countSetBits(lu_hit) <= 1)) + else begin + $error("More then one hit in TLB!"); + $stop(); + end + assert property (@(posedge clk_i) (countSetBits(replace_en) <= 1)) + else begin + $error("More then one TLB entry selected for next replace!"); + $stop(); + end + +`endif + //pragma translate_on + +endmodule diff --git a/src_files.yml b/src_files.yml index 84173c67ca..b391245227 100644 --- a/src_files.yml +++ b/src_files.yml @@ -36,6 +36,7 @@ ariane: src/miss_handler.sv, src/mmu_sv39/mmu.sv, src/mmu_sv32/cva6_mmu_sv32.sv, + src/mmu_unify/cva6_mmu.sv, src/mult.sv, src/nbdcache.sv, src/vdregs.sv, @@ -45,12 +46,14 @@ ariane: src/perf_counters.sv, src/mmu_sv39/ptw.sv, src/mmu_sv32/cva6_ptw_sv32.sv, + src/mmu_unify/cva6_ptw.sv, src/re_name.sv, src/scoreboard.sv, src/store_buffer.sv, src/store_unit.sv, src/mmu_sv39/tlb.sv, src/mmu_sv32/cva6_tlb_sv32.sv, + src/mmu_unify/cva6_tlb.sv, src/acc_dispatcher.sv, src/debug/dm_csrs.sv, src/debug/dm_mem.sv,