[PATCH 00/17] Xvisor RISC-V nested virtualization support
129 views
Skip to first unread message
Anup Patel
Mar 17, 2022, 11:23:33 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
This series adds nested virtualization support to Xvisor RISC-V. Using
this series, we are able to boot Linux Guest inside KVM RISC-V running
as Xvisor Guest/VM.
The only constraint is that we require AIA CSRs in host hardware for
nested virtualization to work.
These patches can also be found ing riscv_nested_v1 branch at:
https://github.com/avpatel/xvisor-next.git
Anup Patel (17):
TESTS: riscv: Enable HUGETLBFS for Linux guest
TESTS: riscv: Enable SBI based earlycon for guest linux
TESTS: riscv: Add letter h to guest ISA string
TESTS: riscv: Improve SBI support in basic firmware
RISC-V: Add Xvisor specific SBI extension
TESTS: riscv: Generate VCPU isa string using Xvisor SBI extension
RISC-V: Fix typo in __sbi_rfence_v02() call to host
RISC-V: Treat Guest SBI HFENCE calls as NOPs
RISC-V: Add nested virtualization state in VCPU private context
RISC-V: Add more indentation in VCPU register dump prints
RISC-V: Improve H-extension CSR defines for emulation
RISC-V: Add helper functions for nested virtualization
RISC-V: Initial support for nested virtualization
ARCH: generic_mmu: Add mechanism to get guest page table mapping
RISC-V: Emulate HLV and HSV instructions for guest hypervisor
ARCH: generic_mmu: Add attributes and hardware tag for each page table
RISC-V: Emulate guest G-stage page table and guest HFENCE instructions
arch/arm/cpu/arm32ve/cpu_vcpu_helper.c | 10 +-
arch/arm/cpu/arm64/cpu_vcpu_helper.c | 10 +-
arch/arm/cpu/common/include/mmu_lpae.h | 7 +-
arch/arm/cpu/common/mmu_lpae.c | 7 +-
arch/common/generic_mmu.c | 96 +-
arch/common/include/generic_mmu.h | 49 +-
arch/riscv/cpu/generic/cpu_exception.c | 56 +-
arch/riscv/cpu/generic/cpu_init.c | 9 +-
arch/riscv/cpu/generic/cpu_mmu.c | 41 +-
arch/riscv/cpu/generic/cpu_sbi.c | 2 +-
arch/riscv/cpu/generic/cpu_vcpu_csr.c | 54 -
arch/riscv/cpu/generic/cpu_vcpu_fp.c | 60 +-
arch/riscv/cpu/generic/cpu_vcpu_helper.c | 219 ++-
arch/riscv/cpu/generic/cpu_vcpu_nested.c | 1529 +++++++++++++++++
arch/riscv/cpu/generic/cpu_vcpu_sbi.c | 13 +
arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c | 58 +-
arch/riscv/cpu/generic/cpu_vcpu_sbi_xvisor.c | 64 +
arch/riscv/cpu/generic/cpu_vcpu_trap.c | 998 ++++++++++-
arch/riscv/cpu/generic/include/arch_mmu.h | 7 +-
arch/riscv/cpu/generic/include/arch_regs.h | 34 +
arch/riscv/cpu/generic/include/cpu_hwcap.h | 9 +-
arch/riscv/cpu/generic/include/cpu_tlb.h | 4 +-
arch/riscv/cpu/generic/include/cpu_vcpu_csr.h | 37 -
arch/riscv/cpu/generic/include/cpu_vcpu_fp.h | 4 +-
.../cpu/generic/include/cpu_vcpu_helper.h | 9 +
.../cpu/generic/include/cpu_vcpu_nested.h | 110 ++
.../riscv/cpu/generic/include/cpu_vcpu_trap.h | 26 +-
.../cpu/generic/include/riscv_encoding.h | 239 ++-
arch/riscv/cpu/generic/objects.mk | 3 +-
libs/wboxtest/nested_mmu/nested_mmu_test.h | 3 +-
tests/riscv/common/basic/Makefile.inc | 1 +
tests/riscv/common/basic/arch_sbi.c | 265 +++
tests/riscv/common/basic/arch_sbi.h | 359 +++-
tests/riscv/virt32/basic/arch_board.c | 10 +-
tests/riscv/virt32/linux/linux_extra.config | 3 +
tests/riscv/virt32/virt32-guest.dts | 2 +-
tests/riscv/virt64/basic/arch_board.c | 10 +-
tests/riscv/virt64/linux/linux_extra.config | 3 +
tests/riscv/virt64/virt64-guest.dts | 2 +-
39 files changed, 4004 insertions(+), 418 deletions(-)
delete mode 100644 arch/riscv/cpu/generic/cpu_vcpu_csr.c
create mode 100644 arch/riscv/cpu/generic/cpu_vcpu_nested.c
create mode 100644 arch/riscv/cpu/generic/cpu_vcpu_sbi_xvisor.c
delete mode 100644 arch/riscv/cpu/generic/include/cpu_vcpu_csr.h
create mode 100644 arch/riscv/cpu/generic/include/cpu_vcpu_nested.h
create mode 100644 tests/riscv/common/basic/arch_sbi.c
--
2.25.1
this series, we are able to boot Linux Guest inside KVM RISC-V running
as Xvisor Guest/VM.
The only constraint is that we require AIA CSRs in host hardware for
nested virtualization to work.
These patches can also be found ing riscv_nested_v1 branch at:
https://github.com/avpatel/xvisor-next.git
Anup Patel (17):
TESTS: riscv: Enable HUGETLBFS for Linux guest
TESTS: riscv: Enable SBI based earlycon for guest linux
TESTS: riscv: Add letter h to guest ISA string
TESTS: riscv: Improve SBI support in basic firmware
RISC-V: Add Xvisor specific SBI extension
TESTS: riscv: Generate VCPU isa string using Xvisor SBI extension
RISC-V: Fix typo in __sbi_rfence_v02() call to host
RISC-V: Treat Guest SBI HFENCE calls as NOPs
RISC-V: Add nested virtualization state in VCPU private context
RISC-V: Add more indentation in VCPU register dump prints
RISC-V: Improve H-extension CSR defines for emulation
RISC-V: Add helper functions for nested virtualization
RISC-V: Initial support for nested virtualization
ARCH: generic_mmu: Add mechanism to get guest page table mapping
RISC-V: Emulate HLV and HSV instructions for guest hypervisor
ARCH: generic_mmu: Add attributes and hardware tag for each page table
RISC-V: Emulate guest G-stage page table and guest HFENCE instructions
arch/arm/cpu/arm32ve/cpu_vcpu_helper.c | 10 +-
arch/arm/cpu/arm64/cpu_vcpu_helper.c | 10 +-
arch/arm/cpu/common/include/mmu_lpae.h | 7 +-
arch/arm/cpu/common/mmu_lpae.c | 7 +-
arch/common/generic_mmu.c | 96 +-
arch/common/include/generic_mmu.h | 49 +-
arch/riscv/cpu/generic/cpu_exception.c | 56 +-
arch/riscv/cpu/generic/cpu_init.c | 9 +-
arch/riscv/cpu/generic/cpu_mmu.c | 41 +-
arch/riscv/cpu/generic/cpu_sbi.c | 2 +-
arch/riscv/cpu/generic/cpu_vcpu_csr.c | 54 -
arch/riscv/cpu/generic/cpu_vcpu_fp.c | 60 +-
arch/riscv/cpu/generic/cpu_vcpu_helper.c | 219 ++-
arch/riscv/cpu/generic/cpu_vcpu_nested.c | 1529 +++++++++++++++++
arch/riscv/cpu/generic/cpu_vcpu_sbi.c | 13 +
arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c | 58 +-
arch/riscv/cpu/generic/cpu_vcpu_sbi_xvisor.c | 64 +
arch/riscv/cpu/generic/cpu_vcpu_trap.c | 998 ++++++++++-
arch/riscv/cpu/generic/include/arch_mmu.h | 7 +-
arch/riscv/cpu/generic/include/arch_regs.h | 34 +
arch/riscv/cpu/generic/include/cpu_hwcap.h | 9 +-
arch/riscv/cpu/generic/include/cpu_tlb.h | 4 +-
arch/riscv/cpu/generic/include/cpu_vcpu_csr.h | 37 -
arch/riscv/cpu/generic/include/cpu_vcpu_fp.h | 4 +-
.../cpu/generic/include/cpu_vcpu_helper.h | 9 +
.../cpu/generic/include/cpu_vcpu_nested.h | 110 ++
.../riscv/cpu/generic/include/cpu_vcpu_trap.h | 26 +-
.../cpu/generic/include/riscv_encoding.h | 239 ++-
arch/riscv/cpu/generic/objects.mk | 3 +-
libs/wboxtest/nested_mmu/nested_mmu_test.h | 3 +-
tests/riscv/common/basic/Makefile.inc | 1 +
tests/riscv/common/basic/arch_sbi.c | 265 +++
tests/riscv/common/basic/arch_sbi.h | 359 +++-
tests/riscv/virt32/basic/arch_board.c | 10 +-
tests/riscv/virt32/linux/linux_extra.config | 3 +
tests/riscv/virt32/virt32-guest.dts | 2 +-
tests/riscv/virt64/basic/arch_board.c | 10 +-
tests/riscv/virt64/linux/linux_extra.config | 3 +
tests/riscv/virt64/virt64-guest.dts | 2 +-
39 files changed, 4004 insertions(+), 418 deletions(-)
delete mode 100644 arch/riscv/cpu/generic/cpu_vcpu_csr.c
create mode 100644 arch/riscv/cpu/generic/cpu_vcpu_nested.c
create mode 100644 arch/riscv/cpu/generic/cpu_vcpu_sbi_xvisor.c
delete mode 100644 arch/riscv/cpu/generic/include/cpu_vcpu_csr.h
create mode 100644 arch/riscv/cpu/generic/include/cpu_vcpu_nested.h
create mode 100644 tests/riscv/common/basic/arch_sbi.c
--
2.25.1
Anup Patel
Mar 17, 2022, 11:23:35 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
We should have HUGETLBFS enabled by default for Linux guest so
that we can tune nested virtualization performance.
Signed-off-by: Anup Patel <apa...@ventanamicro.com>
---
tests/riscv/virt32/linux/linux_extra.config | 1 +
tests/riscv/virt64/linux/linux_extra.config | 1 +
2 files changed, 2 insertions(+)
diff --git a/tests/riscv/virt32/linux/linux_extra.config b/tests/riscv/virt32/linux/linux_extra.config
index 34be5895..3c1859d6 100644
--- a/tests/riscv/virt32/linux/linux_extra.config
+++ b/tests/riscv/virt32/linux/linux_extra.config
@@ -1,3 +1,4 @@
CONFIG_HIGH_RES_TIMERS=y
CONFIG_NO_HZ_IDLE=y
+CONFIG_HUGETLBFS=y
CONFIG_DRM=n
diff --git a/tests/riscv/virt64/linux/linux_extra.config b/tests/riscv/virt64/linux/linux_extra.config
index 34be5895..3c1859d6 100644
--- a/tests/riscv/virt64/linux/linux_extra.config
+++ b/tests/riscv/virt64/linux/linux_extra.config
@@ -1,3 +1,4 @@
CONFIG_HIGH_RES_TIMERS=y
CONFIG_NO_HZ_IDLE=y
+CONFIG_HUGETLBFS=y
CONFIG_DRM=n
--
2.25.1
that we can tune nested virtualization performance.
Signed-off-by: Anup Patel <apa...@ventanamicro.com>
---
tests/riscv/virt32/linux/linux_extra.config | 1 +
tests/riscv/virt64/linux/linux_extra.config | 1 +
2 files changed, 2 insertions(+)
diff --git a/tests/riscv/virt32/linux/linux_extra.config b/tests/riscv/virt32/linux/linux_extra.config
index 34be5895..3c1859d6 100644
--- a/tests/riscv/virt32/linux/linux_extra.config
+++ b/tests/riscv/virt32/linux/linux_extra.config
@@ -1,3 +1,4 @@
CONFIG_HIGH_RES_TIMERS=y
CONFIG_NO_HZ_IDLE=y
+CONFIG_HUGETLBFS=y
CONFIG_DRM=n
diff --git a/tests/riscv/virt64/linux/linux_extra.config b/tests/riscv/virt64/linux/linux_extra.config
index 34be5895..3c1859d6 100644
--- a/tests/riscv/virt64/linux/linux_extra.config
+++ b/tests/riscv/virt64/linux/linux_extra.config
@@ -1,3 +1,4 @@
CONFIG_HIGH_RES_TIMERS=y
CONFIG_NO_HZ_IDLE=y
+CONFIG_HUGETLBFS=y
CONFIG_DRM=n
--
2.25.1
Anup Patel
Mar 17, 2022, 11:23:37 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
The SBI based earlycon (i.e. early prints) is much faster compared
to MMIO based earlycon (i.e. uart8250 early prints) for Linux running
under a Guest hypervisor (i.e. nested virtualization). This patch
enables SBI based earlycon for RISC-V guest linux.
tests/riscv/virt32/linux/linux_extra.config | 2 ++
tests/riscv/virt64/linux/linux_extra.config | 2 ++
2 files changed, 4 insertions(+)
diff --git a/tests/riscv/virt32/linux/linux_extra.config b/tests/riscv/virt32/linux/linux_extra.config
index 3c1859d6..ccf7985f 100644
--- a/tests/riscv/virt32/linux/linux_extra.config
+++ b/tests/riscv/virt32/linux/linux_extra.config
@@ -1,4 +1,6 @@
CONFIG_HIGH_RES_TIMERS=y
CONFIG_NO_HZ_IDLE=y
CONFIG_HUGETLBFS=y
+CONFIG_RISCV_SBI_V01=y
+CONFIG_HVC_RISCV_SBI=n
--- a/tests/riscv/virt64/linux/linux_extra.config
+++ b/tests/riscv/virt64/linux/linux_extra.config
@@ -1,4 +1,6 @@
CONFIG_HIGH_RES_TIMERS=y
CONFIG_NO_HZ_IDLE=y
CONFIG_HUGETLBFS=y
+CONFIG_RISCV_SBI_V01=y
+CONFIG_HVC_RISCV_SBI=n
CONFIG_DRM=n
--
2.25.1
to MMIO based earlycon (i.e. uart8250 early prints) for Linux running
under a Guest hypervisor (i.e. nested virtualization). This patch
enables SBI based earlycon for RISC-V guest linux.
tests/riscv/virt32/linux/linux_extra.config | 2 ++
tests/riscv/virt64/linux/linux_extra.config | 2 ++
2 files changed, 4 insertions(+)
diff --git a/tests/riscv/virt32/linux/linux_extra.config b/tests/riscv/virt32/linux/linux_extra.config
index 3c1859d6..ccf7985f 100644
--- a/tests/riscv/virt32/linux/linux_extra.config
+++ b/tests/riscv/virt32/linux/linux_extra.config
@@ -1,4 +1,6 @@
CONFIG_HIGH_RES_TIMERS=y
CONFIG_NO_HZ_IDLE=y
CONFIG_HUGETLBFS=y
+CONFIG_RISCV_SBI_V01=y
+CONFIG_HVC_RISCV_SBI=n
CONFIG_DRM=n
diff --git a/tests/riscv/virt64/linux/linux_extra.config b/tests/riscv/virt64/linux/linux_extra.config
index 3c1859d6..ccf7985f 100644
diff --git a/tests/riscv/virt64/linux/linux_extra.config b/tests/riscv/virt64/linux/linux_extra.config
--- a/tests/riscv/virt64/linux/linux_extra.config
+++ b/tests/riscv/virt64/linux/linux_extra.config
@@ -1,4 +1,6 @@
CONFIG_HIGH_RES_TIMERS=y
CONFIG_NO_HZ_IDLE=y
CONFIG_HUGETLBFS=y
+CONFIG_RISCV_SBI_V01=y
+CONFIG_HVC_RISCV_SBI=n
CONFIG_DRM=n
--
2.25.1
Anup Patel
Mar 17, 2022, 11:23:39 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
Nested virtualization is naturally supported by RISC-V H-extension
so Xvisor nested virtualization is assumed to be always enabled.
tests/riscv/virt32/basic/arch_board.c | 2 +-
tests/riscv/virt32/virt32-guest.dts | 2 +-
tests/riscv/virt64/basic/arch_board.c | 2 +-
tests/riscv/virt64/virt64-guest.dts | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/tests/riscv/virt32/basic/arch_board.c b/tests/riscv/virt32/basic/arch_board.c
index 2495d3a9..74937356 100644
--- a/tests/riscv/virt32/basic/arch_board.c
+++ b/tests/riscv/virt32/basic/arch_board.c
@@ -179,7 +179,7 @@ void arch_board_fdt_fixup(void *fdt_addr)
}
ret = fdt_setprop_string(fdt_addr, cpu_offset,
- "riscv,isa", "rv32imacfd");
+ "riscv,isa", "rv32imacfdh");
if (ret < 0) {
basic_printf("Failed to set %s property of /cpus/%s "
"DT node\n", "riscv,isa", name);
diff --git a/tests/riscv/virt32/virt32-guest.dts b/tests/riscv/virt32/virt32-guest.dts
index 6049525d..5f8a6377 100644
--- a/tests/riscv/virt32/virt32-guest.dts
+++ b/tests/riscv/virt32/virt32-guest.dts
@@ -16,7 +16,7 @@
vcpu_template {
device_type = "vcpu";
compatible = "riscv,generic";
- riscv,isa = "rv32imafdcsu";
+ riscv,isa = "rv32imafdch";
start_pc = <0x00000000>;
poweroff;
};
diff --git a/tests/riscv/virt64/basic/arch_board.c b/tests/riscv/virt64/basic/arch_board.c
index 8bd18c43..b20623a5 100644
--- a/tests/riscv/virt64/basic/arch_board.c
+++ b/tests/riscv/virt64/basic/arch_board.c
@@ -179,7 +179,7 @@ void arch_board_fdt_fixup(void *fdt_addr)
}
ret = fdt_setprop_string(fdt_addr, cpu_offset,
- "riscv,isa", "rv64imacfd");
+ "riscv,isa", "rv64imacfdh");
if (ret < 0) {
basic_printf("Failed to set %s property of /cpus/%s "
"DT node\n", "riscv,isa", name);
diff --git a/tests/riscv/virt64/virt64-guest.dts b/tests/riscv/virt64/virt64-guest.dts
index 8d769eac..fa87d5ef 100644
--- a/tests/riscv/virt64/virt64-guest.dts
+++ b/tests/riscv/virt64/virt64-guest.dts
@@ -16,7 +16,7 @@
vcpu_template {
device_type = "vcpu";
compatible = "riscv,generic";
- riscv,isa = "rv64imafdcsu";
+ riscv,isa = "rv64imafdch";
start_pc = <0x00000000>;
poweroff;
};
--
2.25.1
so Xvisor nested virtualization is assumed to be always enabled.
tests/riscv/virt32/basic/arch_board.c | 2 +-
tests/riscv/virt32/virt32-guest.dts | 2 +-
tests/riscv/virt64/basic/arch_board.c | 2 +-
tests/riscv/virt64/virt64-guest.dts | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/tests/riscv/virt32/basic/arch_board.c b/tests/riscv/virt32/basic/arch_board.c
index 2495d3a9..74937356 100644
--- a/tests/riscv/virt32/basic/arch_board.c
+++ b/tests/riscv/virt32/basic/arch_board.c
@@ -179,7 +179,7 @@ void arch_board_fdt_fixup(void *fdt_addr)
}
ret = fdt_setprop_string(fdt_addr, cpu_offset,
- "riscv,isa", "rv32imacfd");
+ "riscv,isa", "rv32imacfdh");
if (ret < 0) {
basic_printf("Failed to set %s property of /cpus/%s "
"DT node\n", "riscv,isa", name);
diff --git a/tests/riscv/virt32/virt32-guest.dts b/tests/riscv/virt32/virt32-guest.dts
index 6049525d..5f8a6377 100644
--- a/tests/riscv/virt32/virt32-guest.dts
+++ b/tests/riscv/virt32/virt32-guest.dts
@@ -16,7 +16,7 @@
vcpu_template {
device_type = "vcpu";
compatible = "riscv,generic";
- riscv,isa = "rv32imafdcsu";
+ riscv,isa = "rv32imafdch";
start_pc = <0x00000000>;
poweroff;
};
diff --git a/tests/riscv/virt64/basic/arch_board.c b/tests/riscv/virt64/basic/arch_board.c
index 8bd18c43..b20623a5 100644
--- a/tests/riscv/virt64/basic/arch_board.c
+++ b/tests/riscv/virt64/basic/arch_board.c
@@ -179,7 +179,7 @@ void arch_board_fdt_fixup(void *fdt_addr)
}
ret = fdt_setprop_string(fdt_addr, cpu_offset,
- "riscv,isa", "rv64imacfd");
+ "riscv,isa", "rv64imacfdh");
if (ret < 0) {
basic_printf("Failed to set %s property of /cpus/%s "
"DT node\n", "riscv,isa", name);
diff --git a/tests/riscv/virt64/virt64-guest.dts b/tests/riscv/virt64/virt64-guest.dts
index 8d769eac..fa87d5ef 100644
--- a/tests/riscv/virt64/virt64-guest.dts
+++ b/tests/riscv/virt64/virt64-guest.dts
@@ -16,7 +16,7 @@
vcpu_template {
device_type = "vcpu";
compatible = "riscv,generic";
- riscv,isa = "rv64imafdcsu";
+ riscv,isa = "rv64imafdch";
start_pc = <0x00000000>;
poweroff;
};
--
2.25.1
Anup Patel
Mar 17, 2022, 11:23:42 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
Currently, the SBI support in basic firmware only uses SBI v0.1
specification so this patch improves it to use SBI v0.2 (or higher).
tests/riscv/common/basic/Makefile.inc | 1 +
tests/riscv/common/basic/arch_sbi.c | 219 ++++++++++++++++
tests/riscv/common/basic/arch_sbi.h | 357 +++++++++++++++++++-------
tests/riscv/virt32/basic/arch_board.c | 5 +-
tests/riscv/virt64/basic/arch_board.c | 5 +-
5 files changed, 494 insertions(+), 93 deletions(-)
create mode 100644 tests/riscv/common/basic/arch_sbi.c
diff --git a/tests/riscv/common/basic/Makefile.inc b/tests/riscv/common/basic/Makefile.inc
index 9fa5abed..0bf7701e 100644
--- a/tests/riscv/common/basic/Makefile.inc
+++ b/tests/riscv/common/basic/Makefile.inc
@@ -43,6 +43,7 @@ ARCH_DIR=$(arch_dir)
ARCH_OBJS= $(ARCH_OBJ_DIR)/arch_entry.o \
$(ARCH_OBJ_DIR)/arch_cache.o \
$(ARCH_OBJ_DIR)/arch_irq.o \
+ $(ARCH_OBJ_DIR)/arch_sbi.o \
$(ARCH_OBJ_DIR)/arch_linux.o \
$(ARCH_OBJ_DIR)/arch_mmu.o \
$(board_objs)
diff --git a/tests/riscv/common/basic/arch_sbi.c b/tests/riscv/common/basic/arch_sbi.c
new file mode 100644
index 00000000..3a3e14b9
--- /dev/null
+++ b/tests/riscv/common/basic/arch_sbi.c
@@ -0,0 +1,219 @@
+/**
+ * Copyright (c) 2022 Ventana Micro Systems Inc.
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * @file arch_sbi.c
+ * @author Anup Patel (apa...@ventanamicro.com)
+ * @brief Supervisor binary interface (SBI) source
+ */
+
+#include <basic_stdio.h>
+#include <arch_sbi.h>
+
+struct sbiret {
+ long error;
+ long value;
+};
+
+static struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0,
+ unsigned long arg1, unsigned long arg2,
+ unsigned long arg3, unsigned long arg4,
+ unsigned long arg5)
+{
+ struct sbiret ret;
+
+ register unsigned long a0 asm ("a0") = (arg0);
+ register unsigned long a1 asm ("a1") = (arg1);
+ register unsigned long a2 asm ("a2") = (arg2);
+ register unsigned long a3 asm ("a3") = (arg3);
+ register unsigned long a4 asm ("a4") = (arg4);
+ register unsigned long a5 asm ("a5") = (arg5);
+ register unsigned long a6 asm ("a6") = (fid);
+ register unsigned long a7 asm ("a7") = (ext);
+ asm volatile ("ecall"
+ : "+r" (a0), "+r" (a1)
+ : "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6), "r" (a7)
+ : "memory");
+ ret.error = a0;
+ ret.value = a1;
+
+ return ret;
+}
+
+static int sbi_probe_extension(long extid)
+{
+ struct sbiret ret;
+
+ ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_PROBE_EXT, extid,
+ 0, 0, 0, 0, 0);
+ if (!ret.error && ret.value)
+ return ret.value;
+
+ return -1;
+}
+
+static unsigned long sbi_spec_version = SBI_SPEC_VERSION_DEFAULT;
+
+int sbi_spec_is_0_1(void)
+{
+ return (sbi_spec_version == SBI_SPEC_VERSION_DEFAULT) ? 1 : 0;
+}
+
+unsigned long sbi_major_version(void)
+{
+ return (sbi_spec_version >> SBI_SPEC_VERSION_MAJOR_SHIFT) &
+ SBI_SPEC_VERSION_MAJOR_MASK;
+}
+
+unsigned long sbi_minor_version(void)
+{
+ return sbi_spec_version & SBI_SPEC_VERSION_MINOR_MASK;
+}
+
+void sbi_console_putchar(int ch)
+{
+ sbi_ecall(SBI_EXT_0_1_CONSOLE_PUTCHAR, 0, ch, 0, 0, 0, 0, 0);
+}
+
+int sbi_console_getchar(void)
+{
+ struct sbiret ret;
+
+ ret = sbi_ecall(SBI_EXT_0_1_CONSOLE_GETCHAR, 0, 0, 0, 0, 0, 0, 0);
+
+ return ret.error;
+}
+
+void sbi_set_timer(u64 stime_value)
+{
+#if __riscv_xlen == 32
+ sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value,
+ stime_value >> 32, 0, 0, 0, 0);
+#else
+ sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, 0, 0, 0, 0, 0);
+#endif
+}
+
+void sbi_clear_timer(void)
+{
+#if __riscv_xlen == 32
+ sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, -1UL, -1UL, 0, 0, 0, 0);
+#else
+ sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, -1UL, 0, 0, 0, 0, 0);
+#endif
+}
+
+void sbi_clear_ipi(void)
+{
+ sbi_ecall(SBI_EXT_0_1_CLEAR_IPI, 0, 0, 0, 0, 0, 0, 0);
+}
+
+void sbi_send_ipi(const unsigned long *hart_mask)
+{
+ sbi_ecall(SBI_EXT_0_1_SEND_IPI, 0, (unsigned long)hart_mask,
+ 0, 0, 0, 0, 0);
+}
+
+void sbi_remote_fence_i(const unsigned long *hart_mask)
+{
+ sbi_ecall(SBI_EXT_0_1_REMOTE_FENCE_I, 0,
+ (unsigned long)hart_mask, 0, 0, 0, 0, 0);
+}
+
+void sbi_remote_sfence_vma(const unsigned long *hart_mask,
+ unsigned long start, unsigned long size)
+{
+ sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA, 0,
+ (unsigned long)hart_mask, start, size, 0, 0, 0);
+}
+
+void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask,
+ unsigned long start, unsigned long size,
+ unsigned long asid)
+{
+ sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID, 0,
+ (unsigned long)hart_mask, start, size, asid, 0, 0);
+}
+
+static void __sbi_srst_reset(unsigned long type, unsigned long reason)
+{
+ sbi_ecall(SBI_EXT_SRST, SBI_EXT_SRST_RESET, type, reason,
+ 0, 0, 0, 0);
+}
+
+void sbi_shutdown(void)
+{
+ /* Must have SBI SRST extension */
+ if (!sbi_spec_is_0_1() && (sbi_probe_extension(SBI_EXT_SRST) > 0)) {
+ __sbi_srst_reset(SBI_SRST_RESET_TYPE_SHUTDOWN,
+ SBI_SRST_RESET_REASON_NONE);
+ } else {
+ sbi_ecall(SBI_EXT_0_1_SHUTDOWN, 0, 0, 0, 0, 0, 0, 0);
+ }
+}
+
+void sbi_reset(void)
+{
+ /* Must have SBI SRST extension */
+ if (!sbi_spec_is_0_1() && (sbi_probe_extension(SBI_EXT_SRST) > 0)) {
+ __sbi_srst_reset(SBI_SRST_RESET_TYPE_COLD_REBOOT,
+ SBI_SRST_RESET_REASON_NONE);
+ } else {
+ sbi_ecall(SBI_EXT_0_1_SHUTDOWN, 0, 0, 0, 0, 0, 0, 0);
+ }
+}
+
+static long sbi_ext_base_func(long fid)
+{
+ struct sbiret ret;
+
+ ret = sbi_ecall(SBI_EXT_BASE, fid, 0, 0, 0, 0, 0, 0);
+ if (!ret.error)
+ return ret.value;
+ else
+ return ret.error;
+}
+
+#define sbi_get_spec_version() \
+ sbi_ext_base_func(SBI_EXT_BASE_GET_SPEC_VERSION)
+
+#define sbi_get_firmware_id() \
+ sbi_ext_base_func(SBI_EXT_BASE_GET_IMP_ID)
+
+#define sbi_get_firmware_version() \
+ sbi_ext_base_func(SBI_EXT_BASE_GET_IMP_VERSION)
+
+void sbi_init(void)
+{
+ int ret;
+
+ ret = sbi_get_spec_version();
+ if (ret > 0)
+ sbi_spec_version = ret;
+
+ basic_printf("RISC-V SBI specification v%d.%d detected\n",
+ (unsigned int)sbi_major_version(),
+ (unsigned int)sbi_minor_version());
+
+ if (!sbi_spec_is_0_1()) {
+ basic_printf("RISC-V SBI implementation ID=0x%lx Version=0x%lx\n",
+ sbi_get_firmware_id(),
+ sbi_get_firmware_version());
+ }
+
+ basic_printf("\n");
+}
diff --git a/tests/riscv/common/basic/arch_sbi.h b/tests/riscv/common/basic/arch_sbi.h
index 369638de..87db4a29 100644
--- a/tests/riscv/common/basic/arch_sbi.h
+++ b/tests/riscv/common/basic/arch_sbi.h
@@ -33,94 +33,273 @@
#include <arch_types.h>
-#define SBI_SET_TIMER 0
-#define SBI_CONSOLE_PUTCHAR 1
-#define SBI_CONSOLE_GETCHAR 2
-#define SBI_CLEAR_IPI 3
-#define SBI_SEND_IPI 4
-#define SBI_REMOTE_FENCE_I 5
-#define SBI_REMOTE_SFENCE_VMA 6
-#define SBI_REMOTE_SFENCE_VMA_ASID 7
-#define SBI_SHUTDOWN 8
-
-#define SBI_CALL(which, arg0, arg1, arg2) ({ \
- register unsigned long a0 asm ("a0") = (unsigned long)(arg0); \
- register unsigned long a1 asm ("a1") = (unsigned long)(arg1); \
- register unsigned long a2 asm ("a2") = (unsigned long)(arg2); \
- register unsigned long a7 asm ("a7") = (unsigned long)(which); \
- asm volatile ("ecall" \
- : "+r" (a0) \
- : "r" (a1), "r" (a2), "r" (a7) \
- : "memory"); \
- a0; \
-})
-
-/* Lazy implementations until SBI is finalized */
-#define SBI_CALL_0(which) SBI_CALL(which, 0, 0, 0)
-#define SBI_CALL_1(which, arg0) SBI_CALL(which, arg0, 0, 0)
-#define SBI_CALL_2(which, arg0, arg1) SBI_CALL(which, arg0, arg1, 0)
-
-static inline void sbi_console_putchar(int ch)
-{
- SBI_CALL_1(SBI_CONSOLE_PUTCHAR, ch);
-}
-
-static inline int sbi_console_getchar(void)
-{
- return SBI_CALL_0(SBI_CONSOLE_GETCHAR);
-}
-
-static inline void sbi_set_timer(u64 stime_value)
-{
-#if __riscv_xlen == 32
- SBI_CALL_2(SBI_SET_TIMER, stime_value, stime_value >> 32);
-#else
- SBI_CALL_1(SBI_SET_TIMER, stime_value);
-#endif
-}
-
-static inline void sbi_clear_timer(void)
-{
-#if __riscv_xlen == 32
- SBI_CALL_2(SBI_SET_TIMER, -1UL, -1UL);
-#else
- SBI_CALL_1(SBI_SET_TIMER, -1UL);
-#endif
-}
-
-static inline void sbi_shutdown(void)
-{
- SBI_CALL_0(SBI_SHUTDOWN);
-}
-
-static inline void sbi_clear_ipi(void)
-{
- SBI_CALL_0(SBI_CLEAR_IPI);
-}
-
-static inline void sbi_send_ipi(const unsigned long *hart_mask)
-{
- SBI_CALL_1(SBI_SEND_IPI, hart_mask);
-}
-
-static inline void sbi_remote_fence_i(const unsigned long *hart_mask)
-{
- SBI_CALL_1(SBI_REMOTE_FENCE_I, hart_mask);
-}
-
-static inline void sbi_remote_sfence_vma(const unsigned long *hart_mask,
- unsigned long start,
- unsigned long size)
-{
- SBI_CALL_1(SBI_REMOTE_SFENCE_VMA, hart_mask);
-}
-
-static inline void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask,
- unsigned long start,
- unsigned long size,
- unsigned long asid)
-{
- SBI_CALL_1(SBI_REMOTE_SFENCE_VMA_ASID, hart_mask);
-}
+/* SBI Extension IDs */
+#define SBI_EXT_0_1_SET_TIMER 0x0
+#define SBI_EXT_0_1_CONSOLE_PUTCHAR 0x1
+#define SBI_EXT_0_1_CONSOLE_GETCHAR 0x2
+#define SBI_EXT_0_1_CLEAR_IPI 0x3
+#define SBI_EXT_0_1_SEND_IPI 0x4
+#define SBI_EXT_0_1_REMOTE_FENCE_I 0x5
+#define SBI_EXT_0_1_REMOTE_SFENCE_VMA 0x6
+#define SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID 0x7
+#define SBI_EXT_0_1_SHUTDOWN 0x8
+#define SBI_EXT_BASE 0x10
+#define SBI_EXT_TIME 0x54494D45
+#define SBI_EXT_IPI 0x735049
+#define SBI_EXT_RFENCE 0x52464E43
+#define SBI_EXT_HSM 0x48534D
+#define SBI_EXT_SRST 0x53525354
+#define SBI_EXT_PMU 0x504D55
+
+/* SBI function IDs for BASE extension */
+#define SBI_EXT_BASE_GET_SPEC_VERSION 0x0
+#define SBI_EXT_BASE_GET_IMP_ID 0x1
+#define SBI_EXT_BASE_GET_IMP_VERSION 0x2
+#define SBI_EXT_BASE_PROBE_EXT 0x3
+#define SBI_EXT_BASE_GET_MVENDORID 0x4
+#define SBI_EXT_BASE_GET_MARCHID 0x5
+#define SBI_EXT_BASE_GET_MIMPID 0x6
+
+/* SBI function IDs for TIME extension */
+#define SBI_EXT_TIME_SET_TIMER 0x0
+
+/* SBI function IDs for IPI extension */
+#define SBI_EXT_IPI_SEND_IPI 0x0
+
+/* SBI function IDs for RFENCE extension */
+#define SBI_EXT_RFENCE_REMOTE_FENCE_I 0x0
+#define SBI_EXT_RFENCE_REMOTE_SFENCE_VMA 0x1
+#define SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID 0x2
+#define SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID 0x3
+#define SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA 0x4
+#define SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID 0x5
+#define SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA 0x6
+
+/* SBI function IDs for HSM extension */
+#define SBI_EXT_HSM_HART_START 0x0
+#define SBI_EXT_HSM_HART_STOP 0x1
+#define SBI_EXT_HSM_HART_GET_STATUS 0x2
+#define SBI_EXT_HSM_HART_SUSPEND 0x3
+
+#define SBI_HSM_STATE_STARTED 0x0
+#define SBI_HSM_STATE_STOPPED 0x1
+#define SBI_HSM_STATE_START_PENDING 0x2
+#define SBI_HSM_STATE_STOP_PENDING 0x3
+#define SBI_HSM_STATE_SUSPENDED 0x4
+#define SBI_HSM_STATE_SUSPEND_PENDING 0x5
+#define SBI_HSM_STATE_RESUME_PENDING 0x6
+
+#define SBI_HSM_SUSP_BASE_MASK 0x7fffffff
+#define SBI_HSM_SUSP_NON_RET_BIT 0x80000000
+#define SBI_HSM_SUSP_PLAT_BASE 0x10000000
+
+#define SBI_HSM_SUSPEND_RET_DEFAULT 0x00000000
+#define SBI_HSM_SUSPEND_RET_PLATFORM SBI_HSM_SUSP_PLAT_BASE
+#define SBI_HSM_SUSPEND_RET_LAST SBI_HSM_SUSP_BASE_MASK
+#define SBI_HSM_SUSPEND_NON_RET_DEFAULT SBI_HSM_SUSP_NON_RET_BIT
+#define SBI_HSM_SUSPEND_NON_RET_PLATFORM (SBI_HSM_SUSP_NON_RET_BIT | \
+ SBI_HSM_SUSP_PLAT_BASE)
+#define SBI_HSM_SUSPEND_NON_RET_LAST (SBI_HSM_SUSP_NON_RET_BIT | \
+ SBI_HSM_SUSP_BASE_MASK)
+
+/* SBI function IDs for SRST extension */
+#define SBI_EXT_SRST_RESET 0x0
+
+#define SBI_SRST_RESET_TYPE_SHUTDOWN 0x0
+#define SBI_SRST_RESET_TYPE_COLD_REBOOT 0x1
+#define SBI_SRST_RESET_TYPE_WARM_REBOOT 0x2
+#define SBI_SRST_RESET_TYPE_LAST SBI_SRST_RESET_TYPE_WARM_REBOOT
+
+#define SBI_SRST_RESET_REASON_NONE 0x0
+#define SBI_SRST_RESET_REASON_SYSFAIL 0x1
+
+/* SBI function IDs for PMU extension */
+#define SBI_EXT_PMU_NUM_COUNTERS 0x0
+#define SBI_EXT_PMU_COUNTER_GET_INFO 0x1
+#define SBI_EXT_PMU_COUNTER_CFG_MATCH 0x2
+#define SBI_EXT_PMU_COUNTER_START 0x3
+#define SBI_EXT_PMU_COUNTER_STOP 0x4
+#define SBI_EXT_PMU_COUNTER_FW_READ 0x5
+
+/** General pmu event codes specified in SBI PMU extension */
+enum sbi_pmu_hw_generic_events_t {
+ SBI_PMU_HW_NO_EVENT = 0,
+ SBI_PMU_HW_CPU_CYCLES = 1,
+ SBI_PMU_HW_INSTRUCTIONS = 2,
+ SBI_PMU_HW_CACHE_REFERENCES = 3,
+ SBI_PMU_HW_CACHE_MISSES = 4,
+ SBI_PMU_HW_BRANCH_INSTRUCTIONS = 5,
+ SBI_PMU_HW_BRANCH_MISSES = 6,
+ SBI_PMU_HW_BUS_CYCLES = 7,
+ SBI_PMU_HW_STALLED_CYCLES_FRONTEND = 8,
+ SBI_PMU_HW_STALLED_CYCLES_BACKEND = 9,
+ SBI_PMU_HW_REF_CPU_CYCLES = 10,
+
+ SBI_PMU_HW_GENERAL_MAX,
+};
+
+/**
+ * Generalized hardware cache events:
+ *
+ * { L1-D, L1-I, LLC, ITLB, DTLB, BPU, NODE } x
+ * { read, write, prefetch } x
+ * { accesses, misses }
+ */
+enum sbi_pmu_hw_cache_id {
+ SBI_PMU_HW_CACHE_L1D = 0,
+ SBI_PMU_HW_CACHE_L1I = 1,
+ SBI_PMU_HW_CACHE_LL = 2,
+ SBI_PMU_HW_CACHE_DTLB = 3,
+ SBI_PMU_HW_CACHE_ITLB = 4,
+ SBI_PMU_HW_CACHE_BPU = 5,
+ SBI_PMU_HW_CACHE_NODE = 6,
+
+ SBI_PMU_HW_CACHE_MAX,
+};
+
+enum sbi_pmu_hw_cache_op_id {
+ SBI_PMU_HW_CACHE_OP_READ = 0,
+ SBI_PMU_HW_CACHE_OP_WRITE = 1,
+ SBI_PMU_HW_CACHE_OP_PREFETCH = 2,
+
+ SBI_PMU_HW_CACHE_OP_MAX,
+};
+
+enum sbi_pmu_hw_cache_op_result_id {
+ SBI_PMU_HW_CACHE_RESULT_ACCESS = 0,
+ SBI_PMU_HW_CACHE_RESULT_MISS = 1,
+
+ SBI_PMU_HW_CACHE_RESULT_MAX,
+};
+
+/**
+ * Special "firmware" events provided by the OpenSBI, even if the hardware
+ * does not support performance events. These events are encoded as a raw
+ * event type in Linux kernel perf framework.
+ */
+enum sbi_pmu_fw_event_code_id {
+ SBI_PMU_FW_MISALIGNED_LOAD = 0,
+ SBI_PMU_FW_MISALIGNED_STORE = 1,
+ SBI_PMU_FW_ACCESS_LOAD = 2,
+ SBI_PMU_FW_ACCESS_STORE = 3,
+ SBI_PMU_FW_ILLEGAL_INSN = 4,
+ SBI_PMU_FW_SET_TIMER = 5,
+ SBI_PMU_FW_IPI_SENT = 6,
+ SBI_PMU_FW_IPI_RECVD = 7,
+ SBI_PMU_FW_FENCE_I_SENT = 8,
+ SBI_PMU_FW_FENCE_I_RECVD = 9,
+ SBI_PMU_FW_SFENCE_VMA_SENT = 10,
+ SBI_PMU_FW_SFENCE_VMA_RCVD = 11,
+ SBI_PMU_FW_SFENCE_VMA_ASID_SENT = 12,
+ SBI_PMU_FW_SFENCE_VMA_ASID_RCVD = 13,
+
+ SBI_PMU_FW_HFENCE_GVMA_SENT = 14,
+ SBI_PMU_FW_HFENCE_GVMA_RCVD = 15,
+ SBI_PMU_FW_HFENCE_GVMA_VMID_SENT = 16,
+ SBI_PMU_FW_HFENCE_GVMA_VMID_RCVD = 17,
+
+ SBI_PMU_FW_HFENCE_VVMA_SENT = 18,
+ SBI_PMU_FW_HFENCE_VVMA_RCVD = 19,
+ SBI_PMU_FW_HFENCE_VVMA_ASID_SENT = 20,
+ SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD = 21,
+ SBI_PMU_FW_MAX,
+};
+
+/** SBI PMU event idx type */
+enum sbi_pmu_event_type_id {
+ SBI_PMU_EVENT_TYPE_HW = 0x0,
+ SBI_PMU_EVENT_TYPE_HW_CACHE = 0x1,
+ SBI_PMU_EVENT_TYPE_HW_RAW = 0x2,
+ SBI_PMU_EVENT_TYPE_FW = 0xf,
+ SBI_PMU_EVENT_TYPE_MAX,
+};
+
+/** SBI PMU counter type */
+enum sbi_pmu_ctr_type {
+ SBI_PMU_CTR_TYPE_HW = 0,
+ SBI_PMU_CTR_TYPE_FW,
+};
+
+/* Helper macros to decode event idx */
+#define SBI_PMU_EVENT_IDX_OFFSET 20
+#define SBI_PMU_EVENT_IDX_MASK 0xFFFFF
+#define SBI_PMU_EVENT_IDX_CODE_MASK 0xFFFF
+#define SBI_PMU_EVENT_IDX_TYPE_MASK 0xF0000
+#define SBI_PMU_EVENT_RAW_IDX 0x20000
+
+#define SBI_PMU_EVENT_IDX_INVALID 0xFFFFFFFF
+
+/* Flags defined for config matching function */
+#define SBI_PMU_CFG_FLAG_SKIP_MATCH (1 << 0)
+#define SBI_PMU_CFG_FLAG_CLEAR_VALUE (1 << 1)
+#define SBI_PMU_CFG_FLAG_AUTO_START (1 << 2)
+#define SBI_PMU_CFG_FLAG_SET_VUINH (1 << 3)
+#define SBI_PMU_CFG_FLAG_SET_VSINH (1 << 4)
+#define SBI_PMU_CFG_FLAG_SET_UINH (1 << 5)
+#define SBI_PMU_CFG_FLAG_SET_SINH (1 << 6)
+#define SBI_PMU_CFG_FLAG_SET_MINH (1 << 7)
+
+/* Flags defined for counter start function */
+#define SBI_PMU_START_FLAG_SET_INIT_VALUE (1 << 0)
+
+/* Flags defined for counter stop function */
+#define SBI_PMU_STOP_FLAG_RESET (1 << 0)
+
+/* SBI base specification related macros */
+#define SBI_SPEC_VERSION_MAJOR_SHIFT 24
+#define SBI_SPEC_VERSION_MAJOR_MASK 0x7f
+#define SBI_SPEC_VERSION_MINOR_MASK 0xffffff
+#define SBI_EXT_VENDOR_START 0x09000000
+#define SBI_EXT_VENDOR_END 0x09FFFFFF
+#define SBI_EXT_FIRMWARE_START 0x0A000000
+#define SBI_EXT_FIRMWARE_END 0x0AFFFFFF
+
+/* SBI return error codes */
+#define SBI_SUCCESS 0
+#define SBI_ERR_FAILED -1
+#define SBI_ERR_NOT_SUPPORTED -2
+#define SBI_ERR_INVALID_PARAM -3
+#define SBI_ERR_DENIED -4
+#define SBI_ERR_INVALID_ADDRESS -5
+#define SBI_ERR_ALREADY_AVAILABLE -6
+#define SBI_ERR_ALREADY_STARTED -7
+#define SBI_ERR_ALREADY_STOPPED -8
+
+#define SBI_LAST_ERR SBI_ERR_ALREADY_STOPPED
+
+#define SBI_SPEC_VERSION_DEFAULT 0x1
+
+int sbi_spec_is_0_1(void);
+
+unsigned long sbi_major_version(void);
+
+unsigned long sbi_minor_version(void);
+
+void sbi_console_putchar(int ch);
+
+int sbi_console_getchar(void);
+
+void sbi_set_timer(u64 stime_value);
+
+void sbi_clear_timer(void);
+
+void sbi_shutdown(void);
+
+void sbi_clear_ipi(void);
+
+void sbi_send_ipi(const unsigned long *hart_mask);
+
+void sbi_remote_fence_i(const unsigned long *hart_mask);
+
+void sbi_remote_sfence_vma(const unsigned long *hart_mask,
+ unsigned long start, unsigned long size);
+
+void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask,
+ unsigned long start, unsigned long size,
+ unsigned long asid);
+
+void sbi_reset(void);
+
+void sbi_init(void);
#endif
diff --git a/tests/riscv/virt32/basic/arch_board.c b/tests/riscv/virt32/basic/arch_board.c
index 74937356..1546cbb4 100644
--- a/tests/riscv/virt32/basic/arch_board.c
+++ b/tests/riscv/virt32/basic/arch_board.c
@@ -23,6 +23,7 @@
#include <arch_types.h>
#include <arch_board.h>
+#include <arch_sbi.h>
#include <basic_stdio.h>
#include <basic_heap.h>
#include <basic_string.h>
@@ -72,12 +73,12 @@
void arch_board_reset(void)
{
- /* Nothing to do */
+ sbi_reset();
}
void arch_board_init(void)
{
- /* Nothing to do */
+ sbi_init();
}
char *arch_board_name(void)
diff --git a/tests/riscv/virt64/basic/arch_board.c b/tests/riscv/virt64/basic/arch_board.c
index b20623a5..565fddbe 100644
--- a/tests/riscv/virt64/basic/arch_board.c
+++ b/tests/riscv/virt64/basic/arch_board.c
@@ -23,6 +23,7 @@
#include <arch_types.h>
#include <arch_board.h>
+#include <arch_sbi.h>
#include <basic_stdio.h>
#include <basic_heap.h>
#include <basic_string.h>
@@ -72,12 +73,12 @@
void arch_board_reset(void)
{
- /* Nothing to do */
+ sbi_reset();
}
void arch_board_init(void)
{
- /* Nothing to do */
+ sbi_init();
}
char *arch_board_name(void)
--
2.25.1
specification so this patch improves it to use SBI v0.2 (or higher).
tests/riscv/common/basic/Makefile.inc | 1 +
tests/riscv/common/basic/arch_sbi.c | 219 ++++++++++++++++
tests/riscv/common/basic/arch_sbi.h | 357 +++++++++++++++++++-------
tests/riscv/virt32/basic/arch_board.c | 5 +-
tests/riscv/virt64/basic/arch_board.c | 5 +-
5 files changed, 494 insertions(+), 93 deletions(-)
create mode 100644 tests/riscv/common/basic/arch_sbi.c
diff --git a/tests/riscv/common/basic/Makefile.inc b/tests/riscv/common/basic/Makefile.inc
index 9fa5abed..0bf7701e 100644
--- a/tests/riscv/common/basic/Makefile.inc
+++ b/tests/riscv/common/basic/Makefile.inc
@@ -43,6 +43,7 @@ ARCH_DIR=$(arch_dir)
ARCH_OBJS= $(ARCH_OBJ_DIR)/arch_entry.o \
$(ARCH_OBJ_DIR)/arch_cache.o \
$(ARCH_OBJ_DIR)/arch_irq.o \
+ $(ARCH_OBJ_DIR)/arch_sbi.o \
$(ARCH_OBJ_DIR)/arch_linux.o \
$(ARCH_OBJ_DIR)/arch_mmu.o \
$(board_objs)
diff --git a/tests/riscv/common/basic/arch_sbi.c b/tests/riscv/common/basic/arch_sbi.c
new file mode 100644
index 00000000..3a3e14b9
--- /dev/null
+++ b/tests/riscv/common/basic/arch_sbi.c
@@ -0,0 +1,219 @@
+/**
+ * Copyright (c) 2022 Ventana Micro Systems Inc.
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * @file arch_sbi.c
+ * @author Anup Patel (apa...@ventanamicro.com)
+ * @brief Supervisor binary interface (SBI) source
+ */
+
+#include <basic_stdio.h>
+#include <arch_sbi.h>
+
+struct sbiret {
+ long error;
+ long value;
+};
+
+static struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0,
+ unsigned long arg1, unsigned long arg2,
+ unsigned long arg3, unsigned long arg4,
+ unsigned long arg5)
+{
+ struct sbiret ret;
+
+ register unsigned long a0 asm ("a0") = (arg0);
+ register unsigned long a1 asm ("a1") = (arg1);
+ register unsigned long a2 asm ("a2") = (arg2);
+ register unsigned long a3 asm ("a3") = (arg3);
+ register unsigned long a4 asm ("a4") = (arg4);
+ register unsigned long a5 asm ("a5") = (arg5);
+ register unsigned long a6 asm ("a6") = (fid);
+ register unsigned long a7 asm ("a7") = (ext);
+ asm volatile ("ecall"
+ : "+r" (a0), "+r" (a1)
+ : "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6), "r" (a7)
+ : "memory");
+ ret.error = a0;
+ ret.value = a1;
+
+ return ret;
+}
+
+static int sbi_probe_extension(long extid)
+{
+ struct sbiret ret;
+
+ ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_PROBE_EXT, extid,
+ 0, 0, 0, 0, 0);
+ if (!ret.error && ret.value)
+ return ret.value;
+
+ return -1;
+}
+
+static unsigned long sbi_spec_version = SBI_SPEC_VERSION_DEFAULT;
+
+int sbi_spec_is_0_1(void)
+{
+ return (sbi_spec_version == SBI_SPEC_VERSION_DEFAULT) ? 1 : 0;
+}
+
+unsigned long sbi_major_version(void)
+{
+ return (sbi_spec_version >> SBI_SPEC_VERSION_MAJOR_SHIFT) &
+ SBI_SPEC_VERSION_MAJOR_MASK;
+}
+
+unsigned long sbi_minor_version(void)
+{
+ return sbi_spec_version & SBI_SPEC_VERSION_MINOR_MASK;
+}
+
+void sbi_console_putchar(int ch)
+{
+ sbi_ecall(SBI_EXT_0_1_CONSOLE_PUTCHAR, 0, ch, 0, 0, 0, 0, 0);
+}
+
+int sbi_console_getchar(void)
+{
+ struct sbiret ret;
+
+ ret = sbi_ecall(SBI_EXT_0_1_CONSOLE_GETCHAR, 0, 0, 0, 0, 0, 0, 0);
+
+ return ret.error;
+}
+
+void sbi_set_timer(u64 stime_value)
+{
+#if __riscv_xlen == 32
+ sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value,
+ stime_value >> 32, 0, 0, 0, 0);
+#else
+ sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, 0, 0, 0, 0, 0);
+#endif
+}
+
+void sbi_clear_timer(void)
+{
+#if __riscv_xlen == 32
+ sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, -1UL, -1UL, 0, 0, 0, 0);
+#else
+ sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, -1UL, 0, 0, 0, 0, 0);
+#endif
+}
+
+void sbi_clear_ipi(void)
+{
+ sbi_ecall(SBI_EXT_0_1_CLEAR_IPI, 0, 0, 0, 0, 0, 0, 0);
+}
+
+void sbi_send_ipi(const unsigned long *hart_mask)
+{
+ sbi_ecall(SBI_EXT_0_1_SEND_IPI, 0, (unsigned long)hart_mask,
+ 0, 0, 0, 0, 0);
+}
+
+void sbi_remote_fence_i(const unsigned long *hart_mask)
+{
+ sbi_ecall(SBI_EXT_0_1_REMOTE_FENCE_I, 0,
+ (unsigned long)hart_mask, 0, 0, 0, 0, 0);
+}
+
+void sbi_remote_sfence_vma(const unsigned long *hart_mask,
+ unsigned long start, unsigned long size)
+{
+ sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA, 0,
+ (unsigned long)hart_mask, start, size, 0, 0, 0);
+}
+
+void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask,
+ unsigned long start, unsigned long size,
+ unsigned long asid)
+{
+ sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID, 0,
+ (unsigned long)hart_mask, start, size, asid, 0, 0);
+}
+
+static void __sbi_srst_reset(unsigned long type, unsigned long reason)
+{
+ sbi_ecall(SBI_EXT_SRST, SBI_EXT_SRST_RESET, type, reason,
+ 0, 0, 0, 0);
+}
+
+void sbi_shutdown(void)
+{
+ /* Must have SBI SRST extension */
+ if (!sbi_spec_is_0_1() && (sbi_probe_extension(SBI_EXT_SRST) > 0)) {
+ __sbi_srst_reset(SBI_SRST_RESET_TYPE_SHUTDOWN,
+ SBI_SRST_RESET_REASON_NONE);
+ } else {
+ sbi_ecall(SBI_EXT_0_1_SHUTDOWN, 0, 0, 0, 0, 0, 0, 0);
+ }
+}
+
+void sbi_reset(void)
+{
+ /* Must have SBI SRST extension */
+ if (!sbi_spec_is_0_1() && (sbi_probe_extension(SBI_EXT_SRST) > 0)) {
+ __sbi_srst_reset(SBI_SRST_RESET_TYPE_COLD_REBOOT,
+ SBI_SRST_RESET_REASON_NONE);
+ } else {
+ sbi_ecall(SBI_EXT_0_1_SHUTDOWN, 0, 0, 0, 0, 0, 0, 0);
+ }
+}
+
+static long sbi_ext_base_func(long fid)
+{
+ struct sbiret ret;
+
+ ret = sbi_ecall(SBI_EXT_BASE, fid, 0, 0, 0, 0, 0, 0);
+ if (!ret.error)
+ return ret.value;
+ else
+ return ret.error;
+}
+
+#define sbi_get_spec_version() \
+ sbi_ext_base_func(SBI_EXT_BASE_GET_SPEC_VERSION)
+
+#define sbi_get_firmware_id() \
+ sbi_ext_base_func(SBI_EXT_BASE_GET_IMP_ID)
+
+#define sbi_get_firmware_version() \
+ sbi_ext_base_func(SBI_EXT_BASE_GET_IMP_VERSION)
+
+void sbi_init(void)
+{
+ int ret;
+
+ ret = sbi_get_spec_version();
+ if (ret > 0)
+ sbi_spec_version = ret;
+
+ basic_printf("RISC-V SBI specification v%d.%d detected\n",
+ (unsigned int)sbi_major_version(),
+ (unsigned int)sbi_minor_version());
+
+ if (!sbi_spec_is_0_1()) {
+ basic_printf("RISC-V SBI implementation ID=0x%lx Version=0x%lx\n",
+ sbi_get_firmware_id(),
+ sbi_get_firmware_version());
+ }
+
+ basic_printf("\n");
+}
diff --git a/tests/riscv/common/basic/arch_sbi.h b/tests/riscv/common/basic/arch_sbi.h
index 369638de..87db4a29 100644
--- a/tests/riscv/common/basic/arch_sbi.h
+++ b/tests/riscv/common/basic/arch_sbi.h
@@ -33,94 +33,273 @@
#include <arch_types.h>
-#define SBI_SET_TIMER 0
-#define SBI_CONSOLE_PUTCHAR 1
-#define SBI_CONSOLE_GETCHAR 2
-#define SBI_CLEAR_IPI 3
-#define SBI_SEND_IPI 4
-#define SBI_REMOTE_FENCE_I 5
-#define SBI_REMOTE_SFENCE_VMA 6
-#define SBI_REMOTE_SFENCE_VMA_ASID 7
-#define SBI_SHUTDOWN 8
-
-#define SBI_CALL(which, arg0, arg1, arg2) ({ \
- register unsigned long a0 asm ("a0") = (unsigned long)(arg0); \
- register unsigned long a1 asm ("a1") = (unsigned long)(arg1); \
- register unsigned long a2 asm ("a2") = (unsigned long)(arg2); \
- register unsigned long a7 asm ("a7") = (unsigned long)(which); \
- asm volatile ("ecall" \
- : "+r" (a0) \
- : "r" (a1), "r" (a2), "r" (a7) \
- : "memory"); \
- a0; \
-})
-
-/* Lazy implementations until SBI is finalized */
-#define SBI_CALL_0(which) SBI_CALL(which, 0, 0, 0)
-#define SBI_CALL_1(which, arg0) SBI_CALL(which, arg0, 0, 0)
-#define SBI_CALL_2(which, arg0, arg1) SBI_CALL(which, arg0, arg1, 0)
-
-static inline void sbi_console_putchar(int ch)
-{
- SBI_CALL_1(SBI_CONSOLE_PUTCHAR, ch);
-}
-
-static inline int sbi_console_getchar(void)
-{
- return SBI_CALL_0(SBI_CONSOLE_GETCHAR);
-}
-
-static inline void sbi_set_timer(u64 stime_value)
-{
-#if __riscv_xlen == 32
- SBI_CALL_2(SBI_SET_TIMER, stime_value, stime_value >> 32);
-#else
- SBI_CALL_1(SBI_SET_TIMER, stime_value);
-#endif
-}
-
-static inline void sbi_clear_timer(void)
-{
-#if __riscv_xlen == 32
- SBI_CALL_2(SBI_SET_TIMER, -1UL, -1UL);
-#else
- SBI_CALL_1(SBI_SET_TIMER, -1UL);
-#endif
-}
-
-static inline void sbi_shutdown(void)
-{
- SBI_CALL_0(SBI_SHUTDOWN);
-}
-
-static inline void sbi_clear_ipi(void)
-{
- SBI_CALL_0(SBI_CLEAR_IPI);
-}
-
-static inline void sbi_send_ipi(const unsigned long *hart_mask)
-{
- SBI_CALL_1(SBI_SEND_IPI, hart_mask);
-}
-
-static inline void sbi_remote_fence_i(const unsigned long *hart_mask)
-{
- SBI_CALL_1(SBI_REMOTE_FENCE_I, hart_mask);
-}
-
-static inline void sbi_remote_sfence_vma(const unsigned long *hart_mask,
- unsigned long start,
- unsigned long size)
-{
- SBI_CALL_1(SBI_REMOTE_SFENCE_VMA, hart_mask);
-}
-
-static inline void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask,
- unsigned long start,
- unsigned long size,
- unsigned long asid)
-{
- SBI_CALL_1(SBI_REMOTE_SFENCE_VMA_ASID, hart_mask);
-}
+/* SBI Extension IDs */
+#define SBI_EXT_0_1_SET_TIMER 0x0
+#define SBI_EXT_0_1_CONSOLE_PUTCHAR 0x1
+#define SBI_EXT_0_1_CONSOLE_GETCHAR 0x2
+#define SBI_EXT_0_1_CLEAR_IPI 0x3
+#define SBI_EXT_0_1_SEND_IPI 0x4
+#define SBI_EXT_0_1_REMOTE_FENCE_I 0x5
+#define SBI_EXT_0_1_REMOTE_SFENCE_VMA 0x6
+#define SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID 0x7
+#define SBI_EXT_0_1_SHUTDOWN 0x8
+#define SBI_EXT_BASE 0x10
+#define SBI_EXT_TIME 0x54494D45
+#define SBI_EXT_IPI 0x735049
+#define SBI_EXT_RFENCE 0x52464E43
+#define SBI_EXT_HSM 0x48534D
+#define SBI_EXT_SRST 0x53525354
+#define SBI_EXT_PMU 0x504D55
+
+/* SBI function IDs for BASE extension */
+#define SBI_EXT_BASE_GET_SPEC_VERSION 0x0
+#define SBI_EXT_BASE_GET_IMP_ID 0x1
+#define SBI_EXT_BASE_GET_IMP_VERSION 0x2
+#define SBI_EXT_BASE_PROBE_EXT 0x3
+#define SBI_EXT_BASE_GET_MVENDORID 0x4
+#define SBI_EXT_BASE_GET_MARCHID 0x5
+#define SBI_EXT_BASE_GET_MIMPID 0x6
+
+/* SBI function IDs for TIME extension */
+#define SBI_EXT_TIME_SET_TIMER 0x0
+
+/* SBI function IDs for IPI extension */
+#define SBI_EXT_IPI_SEND_IPI 0x0
+
+/* SBI function IDs for RFENCE extension */
+#define SBI_EXT_RFENCE_REMOTE_FENCE_I 0x0
+#define SBI_EXT_RFENCE_REMOTE_SFENCE_VMA 0x1
+#define SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID 0x2
+#define SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID 0x3
+#define SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA 0x4
+#define SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID 0x5
+#define SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA 0x6
+
+/* SBI function IDs for HSM extension */
+#define SBI_EXT_HSM_HART_START 0x0
+#define SBI_EXT_HSM_HART_STOP 0x1
+#define SBI_EXT_HSM_HART_GET_STATUS 0x2
+#define SBI_EXT_HSM_HART_SUSPEND 0x3
+
+#define SBI_HSM_STATE_STARTED 0x0
+#define SBI_HSM_STATE_STOPPED 0x1
+#define SBI_HSM_STATE_START_PENDING 0x2
+#define SBI_HSM_STATE_STOP_PENDING 0x3
+#define SBI_HSM_STATE_SUSPENDED 0x4
+#define SBI_HSM_STATE_SUSPEND_PENDING 0x5
+#define SBI_HSM_STATE_RESUME_PENDING 0x6
+
+#define SBI_HSM_SUSP_BASE_MASK 0x7fffffff
+#define SBI_HSM_SUSP_NON_RET_BIT 0x80000000
+#define SBI_HSM_SUSP_PLAT_BASE 0x10000000
+
+#define SBI_HSM_SUSPEND_RET_DEFAULT 0x00000000
+#define SBI_HSM_SUSPEND_RET_PLATFORM SBI_HSM_SUSP_PLAT_BASE
+#define SBI_HSM_SUSPEND_RET_LAST SBI_HSM_SUSP_BASE_MASK
+#define SBI_HSM_SUSPEND_NON_RET_DEFAULT SBI_HSM_SUSP_NON_RET_BIT
+#define SBI_HSM_SUSPEND_NON_RET_PLATFORM (SBI_HSM_SUSP_NON_RET_BIT | \
+ SBI_HSM_SUSP_PLAT_BASE)
+#define SBI_HSM_SUSPEND_NON_RET_LAST (SBI_HSM_SUSP_NON_RET_BIT | \
+ SBI_HSM_SUSP_BASE_MASK)
+
+/* SBI function IDs for SRST extension */
+#define SBI_EXT_SRST_RESET 0x0
+
+#define SBI_SRST_RESET_TYPE_SHUTDOWN 0x0
+#define SBI_SRST_RESET_TYPE_COLD_REBOOT 0x1
+#define SBI_SRST_RESET_TYPE_WARM_REBOOT 0x2
+#define SBI_SRST_RESET_TYPE_LAST SBI_SRST_RESET_TYPE_WARM_REBOOT
+
+#define SBI_SRST_RESET_REASON_NONE 0x0
+#define SBI_SRST_RESET_REASON_SYSFAIL 0x1
+
+/* SBI function IDs for PMU extension */
+#define SBI_EXT_PMU_NUM_COUNTERS 0x0
+#define SBI_EXT_PMU_COUNTER_GET_INFO 0x1
+#define SBI_EXT_PMU_COUNTER_CFG_MATCH 0x2
+#define SBI_EXT_PMU_COUNTER_START 0x3
+#define SBI_EXT_PMU_COUNTER_STOP 0x4
+#define SBI_EXT_PMU_COUNTER_FW_READ 0x5
+
+/** General pmu event codes specified in SBI PMU extension */
+enum sbi_pmu_hw_generic_events_t {
+ SBI_PMU_HW_NO_EVENT = 0,
+ SBI_PMU_HW_CPU_CYCLES = 1,
+ SBI_PMU_HW_INSTRUCTIONS = 2,
+ SBI_PMU_HW_CACHE_REFERENCES = 3,
+ SBI_PMU_HW_CACHE_MISSES = 4,
+ SBI_PMU_HW_BRANCH_INSTRUCTIONS = 5,
+ SBI_PMU_HW_BRANCH_MISSES = 6,
+ SBI_PMU_HW_BUS_CYCLES = 7,
+ SBI_PMU_HW_STALLED_CYCLES_FRONTEND = 8,
+ SBI_PMU_HW_STALLED_CYCLES_BACKEND = 9,
+ SBI_PMU_HW_REF_CPU_CYCLES = 10,
+
+ SBI_PMU_HW_GENERAL_MAX,
+};
+
+/**
+ * Generalized hardware cache events:
+ *
+ * { L1-D, L1-I, LLC, ITLB, DTLB, BPU, NODE } x
+ * { read, write, prefetch } x
+ * { accesses, misses }
+ */
+enum sbi_pmu_hw_cache_id {
+ SBI_PMU_HW_CACHE_L1D = 0,
+ SBI_PMU_HW_CACHE_L1I = 1,
+ SBI_PMU_HW_CACHE_LL = 2,
+ SBI_PMU_HW_CACHE_DTLB = 3,
+ SBI_PMU_HW_CACHE_ITLB = 4,
+ SBI_PMU_HW_CACHE_BPU = 5,
+ SBI_PMU_HW_CACHE_NODE = 6,
+
+ SBI_PMU_HW_CACHE_MAX,
+};
+
+enum sbi_pmu_hw_cache_op_id {
+ SBI_PMU_HW_CACHE_OP_READ = 0,
+ SBI_PMU_HW_CACHE_OP_WRITE = 1,
+ SBI_PMU_HW_CACHE_OP_PREFETCH = 2,
+
+ SBI_PMU_HW_CACHE_OP_MAX,
+};
+
+enum sbi_pmu_hw_cache_op_result_id {
+ SBI_PMU_HW_CACHE_RESULT_ACCESS = 0,
+ SBI_PMU_HW_CACHE_RESULT_MISS = 1,
+
+ SBI_PMU_HW_CACHE_RESULT_MAX,
+};
+
+/**
+ * Special "firmware" events provided by the OpenSBI, even if the hardware
+ * does not support performance events. These events are encoded as a raw
+ * event type in Linux kernel perf framework.
+ */
+enum sbi_pmu_fw_event_code_id {
+ SBI_PMU_FW_MISALIGNED_LOAD = 0,
+ SBI_PMU_FW_MISALIGNED_STORE = 1,
+ SBI_PMU_FW_ACCESS_LOAD = 2,
+ SBI_PMU_FW_ACCESS_STORE = 3,
+ SBI_PMU_FW_ILLEGAL_INSN = 4,
+ SBI_PMU_FW_SET_TIMER = 5,
+ SBI_PMU_FW_IPI_SENT = 6,
+ SBI_PMU_FW_IPI_RECVD = 7,
+ SBI_PMU_FW_FENCE_I_SENT = 8,
+ SBI_PMU_FW_FENCE_I_RECVD = 9,
+ SBI_PMU_FW_SFENCE_VMA_SENT = 10,
+ SBI_PMU_FW_SFENCE_VMA_RCVD = 11,
+ SBI_PMU_FW_SFENCE_VMA_ASID_SENT = 12,
+ SBI_PMU_FW_SFENCE_VMA_ASID_RCVD = 13,
+
+ SBI_PMU_FW_HFENCE_GVMA_SENT = 14,
+ SBI_PMU_FW_HFENCE_GVMA_RCVD = 15,
+ SBI_PMU_FW_HFENCE_GVMA_VMID_SENT = 16,
+ SBI_PMU_FW_HFENCE_GVMA_VMID_RCVD = 17,
+
+ SBI_PMU_FW_HFENCE_VVMA_SENT = 18,
+ SBI_PMU_FW_HFENCE_VVMA_RCVD = 19,
+ SBI_PMU_FW_HFENCE_VVMA_ASID_SENT = 20,
+ SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD = 21,
+ SBI_PMU_FW_MAX,
+};
+
+/** SBI PMU event idx type */
+enum sbi_pmu_event_type_id {
+ SBI_PMU_EVENT_TYPE_HW = 0x0,
+ SBI_PMU_EVENT_TYPE_HW_CACHE = 0x1,
+ SBI_PMU_EVENT_TYPE_HW_RAW = 0x2,
+ SBI_PMU_EVENT_TYPE_FW = 0xf,
+ SBI_PMU_EVENT_TYPE_MAX,
+};
+
+/** SBI PMU counter type */
+enum sbi_pmu_ctr_type {
+ SBI_PMU_CTR_TYPE_HW = 0,
+ SBI_PMU_CTR_TYPE_FW,
+};
+
+/* Helper macros to decode event idx */
+#define SBI_PMU_EVENT_IDX_OFFSET 20
+#define SBI_PMU_EVENT_IDX_MASK 0xFFFFF
+#define SBI_PMU_EVENT_IDX_CODE_MASK 0xFFFF
+#define SBI_PMU_EVENT_IDX_TYPE_MASK 0xF0000
+#define SBI_PMU_EVENT_RAW_IDX 0x20000
+
+#define SBI_PMU_EVENT_IDX_INVALID 0xFFFFFFFF
+
+/* Flags defined for config matching function */
+#define SBI_PMU_CFG_FLAG_SKIP_MATCH (1 << 0)
+#define SBI_PMU_CFG_FLAG_CLEAR_VALUE (1 << 1)
+#define SBI_PMU_CFG_FLAG_AUTO_START (1 << 2)
+#define SBI_PMU_CFG_FLAG_SET_VUINH (1 << 3)
+#define SBI_PMU_CFG_FLAG_SET_VSINH (1 << 4)
+#define SBI_PMU_CFG_FLAG_SET_UINH (1 << 5)
+#define SBI_PMU_CFG_FLAG_SET_SINH (1 << 6)
+#define SBI_PMU_CFG_FLAG_SET_MINH (1 << 7)
+
+/* Flags defined for counter start function */
+#define SBI_PMU_START_FLAG_SET_INIT_VALUE (1 << 0)
+
+/* Flags defined for counter stop function */
+#define SBI_PMU_STOP_FLAG_RESET (1 << 0)
+
+/* SBI base specification related macros */
+#define SBI_SPEC_VERSION_MAJOR_SHIFT 24
+#define SBI_SPEC_VERSION_MAJOR_MASK 0x7f
+#define SBI_SPEC_VERSION_MINOR_MASK 0xffffff
+#define SBI_EXT_VENDOR_START 0x09000000
+#define SBI_EXT_VENDOR_END 0x09FFFFFF
+#define SBI_EXT_FIRMWARE_START 0x0A000000
+#define SBI_EXT_FIRMWARE_END 0x0AFFFFFF
+
+/* SBI return error codes */
+#define SBI_SUCCESS 0
+#define SBI_ERR_FAILED -1
+#define SBI_ERR_NOT_SUPPORTED -2
+#define SBI_ERR_INVALID_PARAM -3
+#define SBI_ERR_DENIED -4
+#define SBI_ERR_INVALID_ADDRESS -5
+#define SBI_ERR_ALREADY_AVAILABLE -6
+#define SBI_ERR_ALREADY_STARTED -7
+#define SBI_ERR_ALREADY_STOPPED -8
+
+#define SBI_LAST_ERR SBI_ERR_ALREADY_STOPPED
+
+#define SBI_SPEC_VERSION_DEFAULT 0x1
+
+int sbi_spec_is_0_1(void);
+
+unsigned long sbi_major_version(void);
+
+unsigned long sbi_minor_version(void);
+
+void sbi_console_putchar(int ch);
+
+int sbi_console_getchar(void);
+
+void sbi_set_timer(u64 stime_value);
+
+void sbi_clear_timer(void);
+
+void sbi_shutdown(void);
+
+void sbi_clear_ipi(void);
+
+void sbi_send_ipi(const unsigned long *hart_mask);
+
+void sbi_remote_fence_i(const unsigned long *hart_mask);
+
+void sbi_remote_sfence_vma(const unsigned long *hart_mask,
+ unsigned long start, unsigned long size);
+
+void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask,
+ unsigned long start, unsigned long size,
+ unsigned long asid);
+
+void sbi_reset(void);
+
+void sbi_init(void);
#endif
diff --git a/tests/riscv/virt32/basic/arch_board.c b/tests/riscv/virt32/basic/arch_board.c
index 74937356..1546cbb4 100644
--- a/tests/riscv/virt32/basic/arch_board.c
+++ b/tests/riscv/virt32/basic/arch_board.c
@@ -23,6 +23,7 @@
#include <arch_types.h>
#include <arch_board.h>
+#include <arch_sbi.h>
#include <basic_stdio.h>
#include <basic_heap.h>
#include <basic_string.h>
@@ -72,12 +73,12 @@
void arch_board_reset(void)
{
- /* Nothing to do */
+ sbi_reset();
}
void arch_board_init(void)
{
- /* Nothing to do */
+ sbi_init();
}
char *arch_board_name(void)
diff --git a/tests/riscv/virt64/basic/arch_board.c b/tests/riscv/virt64/basic/arch_board.c
index b20623a5..565fddbe 100644
--- a/tests/riscv/virt64/basic/arch_board.c
+++ b/tests/riscv/virt64/basic/arch_board.c
@@ -23,6 +23,7 @@
#include <arch_types.h>
#include <arch_board.h>
+#include <arch_sbi.h>
#include <basic_stdio.h>
#include <basic_heap.h>
#include <basic_string.h>
@@ -72,12 +73,12 @@
void arch_board_reset(void)
{
- /* Nothing to do */
+ sbi_reset();
}
void arch_board_init(void)
{
- /* Nothing to do */
+ sbi_init();
}
char *arch_board_name(void)
--
2.25.1
Anup Patel
Mar 17, 2022, 11:23:44 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
We add Xvisor specific SBI extension to advertise VCPU capabilities
to Guest/VM. This will help basic firmware to contruct VCPU ISA
string based on VCPU capabilities virtualized by Xvisor.
Signed-off-by: Anup Patel Anup Patel <apa...@ventanamicro.com>
---
arch/riscv/cpu/generic/cpu_vcpu_sbi.c | 2 +
arch/riscv/cpu/generic/cpu_vcpu_sbi_xvisor.c | 64 ++++++++++++++++++++
arch/riscv/cpu/generic/objects.mk | 1 +
3 files changed, 67 insertions(+)
create mode 100644 arch/riscv/cpu/generic/cpu_vcpu_sbi_xvisor.c
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_sbi.c b/arch/riscv/cpu/generic/cpu_vcpu_sbi.c
index 60b01d0b..fbd6a8f6 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_sbi.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_sbi.c
@@ -35,6 +35,7 @@ extern const struct cpu_vcpu_sbi_extension vcpu_sbi_base;
extern const struct cpu_vcpu_sbi_extension vcpu_sbi_hsm;
extern const struct cpu_vcpu_sbi_extension vcpu_sbi_srst;
extern const struct cpu_vcpu_sbi_extension vcpu_sbi_legacy;
+extern const struct cpu_vcpu_sbi_extension vcpu_sbi_xvisor;
static const struct cpu_vcpu_sbi_extension *vcpu_sbi[] = {
&vcpu_sbi_time,
@@ -44,6 +45,7 @@ static const struct cpu_vcpu_sbi_extension *vcpu_sbi[] = {
&vcpu_sbi_hsm,
&vcpu_sbi_srst,
&vcpu_sbi_legacy,
+ &vcpu_sbi_xvisor,
};
const struct cpu_vcpu_sbi_extension *cpu_vcpu_sbi_find_extension(
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_sbi_xvisor.c b/arch/riscv/cpu/generic/cpu_vcpu_sbi_xvisor.c
new file mode 100644
index 00000000..ef3b2664
--- /dev/null
+++ b/arch/riscv/cpu/generic/cpu_vcpu_sbi_xvisor.c
@@ -0,0 +1,64 @@
+ */
+
+#include <vmm_error.h>
+#include <vmm_macros.h>
+#include <vmm_manager.h>
+#include <vmm_guest_aspace.h>
+
+#include <cpu_hwcap.h>
+#include <cpu_vcpu_sbi.h>
+#include <riscv_sbi.h>
+
+#define SBI_EXT_XVISOR (SBI_EXT_FIRMWARE_START + \
+ CPU_VCPU_SBI_IMPID)
+
+#define SBI_EXT_XVISOR_ISA_EXT 0x0
+
+static int vcpu_sbi_xvisor_ecall(struct vmm_vcpu *vcpu,
+ unsigned long ext_id, unsigned long func_id,
+ unsigned long *args, unsigned long *out_val,
+ struct cpu_vcpu_trap *out_trap)
+{
+ switch (func_id) {
+ case SBI_EXT_XVISOR_ISA_EXT:
+ if (args[0] < RISCV_ISA_EXT_MAX) {
+ *out_val = __riscv_isa_extension_available(
+ riscv_priv(vcpu)->isa,
+ args[0]);
+ } else {
+ return SBI_ERR_INVALID_PARAM;
+ }
+ break;
+ default:
+ return SBI_ERR_NOT_SUPPORTED;
+ }
+
+ return 0;
+}
+
+const struct cpu_vcpu_sbi_extension vcpu_sbi_xvisor = {
+ .extid_start = SBI_EXT_XVISOR,
+ .extid_end = SBI_EXT_XVISOR,
+ .handle = vcpu_sbi_xvisor_ecall,
+};
diff --git a/arch/riscv/cpu/generic/objects.mk b/arch/riscv/cpu/generic/objects.mk
index 9a3acbb2..bbb9a6b5 100644
--- a/arch/riscv/cpu/generic/objects.mk
+++ b/arch/riscv/cpu/generic/objects.mk
@@ -87,6 +87,7 @@ cpu-objs-y+= cpu_vcpu_sbi_base.o
cpu-objs-y+= cpu_vcpu_sbi_legacy.o
cpu-objs-y+= cpu_vcpu_sbi_replace.o
cpu-objs-y+= cpu_vcpu_sbi_hsm.o
+cpu-objs-y+= cpu_vcpu_sbi_xvisor.o
cpu-objs-y+= cpu_vcpu_switch.o
cpu-objs-y+= cpu_vcpu_timer.o
cpu-objs-y+= cpu_vcpu_trap.o
--
2.25.1
to Guest/VM. This will help basic firmware to contruct VCPU ISA
string based on VCPU capabilities virtualized by Xvisor.
Signed-off-by: Anup Patel Anup Patel <apa...@ventanamicro.com>
---
arch/riscv/cpu/generic/cpu_vcpu_sbi.c | 2 +
arch/riscv/cpu/generic/cpu_vcpu_sbi_xvisor.c | 64 ++++++++++++++++++++
arch/riscv/cpu/generic/objects.mk | 1 +
3 files changed, 67 insertions(+)
create mode 100644 arch/riscv/cpu/generic/cpu_vcpu_sbi_xvisor.c
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_sbi.c b/arch/riscv/cpu/generic/cpu_vcpu_sbi.c
index 60b01d0b..fbd6a8f6 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_sbi.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_sbi.c
@@ -35,6 +35,7 @@ extern const struct cpu_vcpu_sbi_extension vcpu_sbi_base;
extern const struct cpu_vcpu_sbi_extension vcpu_sbi_hsm;
extern const struct cpu_vcpu_sbi_extension vcpu_sbi_srst;
extern const struct cpu_vcpu_sbi_extension vcpu_sbi_legacy;
+extern const struct cpu_vcpu_sbi_extension vcpu_sbi_xvisor;
static const struct cpu_vcpu_sbi_extension *vcpu_sbi[] = {
&vcpu_sbi_time,
@@ -44,6 +45,7 @@ static const struct cpu_vcpu_sbi_extension *vcpu_sbi[] = {
&vcpu_sbi_hsm,
&vcpu_sbi_srst,
&vcpu_sbi_legacy,
+ &vcpu_sbi_xvisor,
};
const struct cpu_vcpu_sbi_extension *cpu_vcpu_sbi_find_extension(
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_sbi_xvisor.c b/arch/riscv/cpu/generic/cpu_vcpu_sbi_xvisor.c
new file mode 100644
index 00000000..ef3b2664
--- /dev/null
+++ b/arch/riscv/cpu/generic/cpu_vcpu_sbi_xvisor.c
@@ -0,0 +1,64 @@
+/**
+ * Copyright (c) 2022 Ventana Micro Systems Inc.
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * @file cpu_vcpu_sbi_xvisor.c
+ * Copyright (c) 2022 Ventana Micro Systems Inc.
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * @author Anup Patel (apa...@ventanamicro.com)
+ * @brief source of SBI Xvisor extension
+ */
+
+#include <vmm_error.h>
+#include <vmm_macros.h>
+#include <vmm_manager.h>
+#include <vmm_guest_aspace.h>
+
+#include <cpu_hwcap.h>
+#include <cpu_vcpu_sbi.h>
+#include <riscv_sbi.h>
+
+#define SBI_EXT_XVISOR (SBI_EXT_FIRMWARE_START + \
+ CPU_VCPU_SBI_IMPID)
+
+#define SBI_EXT_XVISOR_ISA_EXT 0x0
+
+static int vcpu_sbi_xvisor_ecall(struct vmm_vcpu *vcpu,
+ unsigned long ext_id, unsigned long func_id,
+ unsigned long *args, unsigned long *out_val,
+ struct cpu_vcpu_trap *out_trap)
+{
+ switch (func_id) {
+ case SBI_EXT_XVISOR_ISA_EXT:
+ if (args[0] < RISCV_ISA_EXT_MAX) {
+ *out_val = __riscv_isa_extension_available(
+ riscv_priv(vcpu)->isa,
+ args[0]);
+ } else {
+ return SBI_ERR_INVALID_PARAM;
+ }
+ break;
+ default:
+ return SBI_ERR_NOT_SUPPORTED;
+ }
+
+ return 0;
+}
+
+const struct cpu_vcpu_sbi_extension vcpu_sbi_xvisor = {
+ .extid_start = SBI_EXT_XVISOR,
+ .extid_end = SBI_EXT_XVISOR,
+ .handle = vcpu_sbi_xvisor_ecall,
+};
diff --git a/arch/riscv/cpu/generic/objects.mk b/arch/riscv/cpu/generic/objects.mk
index 9a3acbb2..bbb9a6b5 100644
--- a/arch/riscv/cpu/generic/objects.mk
+++ b/arch/riscv/cpu/generic/objects.mk
@@ -87,6 +87,7 @@ cpu-objs-y+= cpu_vcpu_sbi_base.o
cpu-objs-y+= cpu_vcpu_sbi_legacy.o
cpu-objs-y+= cpu_vcpu_sbi_replace.o
cpu-objs-y+= cpu_vcpu_sbi_hsm.o
+cpu-objs-y+= cpu_vcpu_sbi_xvisor.o
cpu-objs-y+= cpu_vcpu_switch.o
cpu-objs-y+= cpu_vcpu_timer.o
cpu-objs-y+= cpu_vcpu_trap.o
--
2.25.1
Anup Patel
Mar 17, 2022, 11:23:47 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
We should generate VCPU isa string in basic firmware using Xvisor
specific SBI extension so that Guest/VM is in sync with Xvisor
VCPU capabilities.
tests/riscv/common/basic/arch_sbi.c | 46 +++++++++++++++++++++++++++
tests/riscv/common/basic/arch_sbi.h | 6 ++--
tests/riscv/virt32/basic/arch_board.c | 5 +--
tests/riscv/virt64/basic/arch_board.c | 5 +--
4 files changed, 56 insertions(+), 6 deletions(-)
diff --git a/tests/riscv/common/basic/arch_sbi.c b/tests/riscv/common/basic/arch_sbi.c
index 3a3e14b9..eebcdaef 100644
--- a/tests/riscv/common/basic/arch_sbi.c
+++ b/tests/riscv/common/basic/arch_sbi.c
@@ -22,6 +22,7 @@
*/
#include <basic_stdio.h>
+#include <basic_string.h>
#include <arch_sbi.h>
struct sbiret {
@@ -177,6 +178,51 @@ void sbi_reset(void)
}
}
+#define SBI_EXT_XVISOR (SBI_EXT_FIRMWARE_START + 0x2)
+#define SBI_EXT_XVISOR_ISA_EXT 0x0
+
+unsigned long sbi_xvisor_isa_string(char *out_isa, unsigned long max_len)
+ size_t i, valid_isa_len;
+ const char *valid_isa_order = "iemafdqclbjtpvnhkorwyg";
+
+ if (!out_isa || (max_len - pos) < 5)
+ return pos;
+
+#if __riscv_xlen == 64
+ basic_strcpy(&out_isa[pos], "rv64");
+#elif __riscv_xlen == 32
+ basic_strcpy(&out_isa[pos], "rv32");
+#else
+#error "Unexpected __riscv_xlen"
+#endif
+ pos += 4;
+
+ if (sbi_spec_is_0_1() || (sbi_probe_extension(SBI_EXT_XVISOR) <= 0)) {
+ if (max_len > 3) {
+ out_isa[pos++] = 'g';
+ out_isa[pos++] = 'c';
+ pos += 2;
+ return pos;
+ }
+ }
+
+ valid_isa_len = basic_strlen(valid_isa_order);
+ for (i = 0; i < valid_isa_len && pos < max_len; i++) {
+ ret = sbi_ecall(SBI_EXT_XVISOR, SBI_EXT_XVISOR_ISA_EXT,
+ valid_isa_order[i] - 'a', 0, 0, 0, 0, 0);
+ if (ret.error || !ret.value)
+ continue;
+
+ out_isa[pos++] = valid_isa_order[i];
+ }
+ out_isa[pos++] = '\0';
+
+ return pos;
+}
+
static long sbi_ext_base_func(long fid)
{
struct sbiret ret;
diff --git a/tests/riscv/common/basic/arch_sbi.h b/tests/riscv/common/basic/arch_sbi.h
index 87db4a29..fe3df663 100644
--- a/tests/riscv/common/basic/arch_sbi.h
+++ b/tests/riscv/common/basic/arch_sbi.h
@@ -283,8 +283,6 @@ void sbi_set_timer(u64 stime_value);
void sbi_clear_timer(void);
-void sbi_shutdown(void);
-
void sbi_clear_ipi(void);
void sbi_send_ipi(const unsigned long *hart_mask);
@@ -298,8 +296,12 @@ void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask,
+void sbi_shutdown(void);
+
void sbi_reset(void);
+unsigned long sbi_xvisor_isa_string(char *out_isa, unsigned long max_len);
+
--- a/tests/riscv/virt32/basic/arch_board.c
+++ b/tests/riscv/virt32/basic/arch_board.c
@@ -104,8 +104,8 @@ void arch_board_linux_default_cmdline(char *cmdline, u32 cmdline_sz)
void arch_board_fdt_fixup(void *fdt_addr)
{
- char name[64];
u32 i, *vals;
+ char name[64], isa[256];
int ret, cpus_offset, cpu_offset, intc_offset, plic_offset;
u32 timebase_freq = (u32)vminfo_clocksource_freq(VIRT_VMINFO);
u32 vcpu_count = vminfo_vcpu_count(VIRT_VMINFO);
@@ -179,8 +179,9 @@ void arch_board_fdt_fixup(void *fdt_addr)
return;
}
+ sbi_xvisor_isa_string(isa, sizeof(isa));
ret = fdt_setprop_string(fdt_addr, cpu_offset,
- "riscv,isa", "rv32imacfdh");
+ "riscv,isa", isa);
index 565fddbe..0ce85c45 100644
--- a/tests/riscv/virt64/basic/arch_board.c
+++ b/tests/riscv/virt64/basic/arch_board.c
@@ -104,8 +104,8 @@ void arch_board_linux_default_cmdline(char *cmdline, u32 cmdline_sz)
void arch_board_fdt_fixup(void *fdt_addr)
{
- char name[64];
u32 i, *vals;
+ char name[64], isa[256];
int ret, cpus_offset, cpu_offset, intc_offset, plic_offset;
u32 timebase_freq = (u32)vminfo_clocksource_freq(VIRT_VMINFO);
u32 vcpu_count = vminfo_vcpu_count(VIRT_VMINFO);
@@ -179,8 +179,9 @@ void arch_board_fdt_fixup(void *fdt_addr)
return;
}
+ sbi_xvisor_isa_string(isa, sizeof(isa));
ret = fdt_setprop_string(fdt_addr, cpu_offset,
- "riscv,isa", "rv64imacfdh");
+ "riscv,isa", isa);
2.25.1
specific SBI extension so that Guest/VM is in sync with Xvisor
VCPU capabilities.
tests/riscv/common/basic/arch_sbi.c | 46 +++++++++++++++++++++++++++
tests/riscv/common/basic/arch_sbi.h | 6 ++--
tests/riscv/virt32/basic/arch_board.c | 5 +--
tests/riscv/virt64/basic/arch_board.c | 5 +--
4 files changed, 56 insertions(+), 6 deletions(-)
diff --git a/tests/riscv/common/basic/arch_sbi.c b/tests/riscv/common/basic/arch_sbi.c
index 3a3e14b9..eebcdaef 100644
--- a/tests/riscv/common/basic/arch_sbi.c
+++ b/tests/riscv/common/basic/arch_sbi.c
@@ -22,6 +22,7 @@
*/
#include <basic_stdio.h>
+#include <basic_string.h>
#include <arch_sbi.h>
struct sbiret {
@@ -177,6 +178,51 @@ void sbi_reset(void)
}
}
+#define SBI_EXT_XVISOR (SBI_EXT_FIRMWARE_START + 0x2)
+#define SBI_EXT_XVISOR_ISA_EXT 0x0
+
+unsigned long sbi_xvisor_isa_string(char *out_isa, unsigned long max_len)
+{
+ struct sbiret ret;
+ unsigned long pos = 0;
+ struct sbiret ret;
+ size_t i, valid_isa_len;
+ const char *valid_isa_order = "iemafdqclbjtpvnhkorwyg";
+
+ if (!out_isa || (max_len - pos) < 5)
+ return pos;
+
+#if __riscv_xlen == 64
+ basic_strcpy(&out_isa[pos], "rv64");
+#elif __riscv_xlen == 32
+ basic_strcpy(&out_isa[pos], "rv32");
+#else
+#error "Unexpected __riscv_xlen"
+#endif
+ pos += 4;
+
+ if (sbi_spec_is_0_1() || (sbi_probe_extension(SBI_EXT_XVISOR) <= 0)) {
+ if (max_len > 3) {
+ out_isa[pos++] = 'g';
+ out_isa[pos++] = 'c';
+ pos += 2;
+ return pos;
+ }
+ }
+
+ valid_isa_len = basic_strlen(valid_isa_order);
+ for (i = 0; i < valid_isa_len && pos < max_len; i++) {
+ ret = sbi_ecall(SBI_EXT_XVISOR, SBI_EXT_XVISOR_ISA_EXT,
+ valid_isa_order[i] - 'a', 0, 0, 0, 0, 0);
+ if (ret.error || !ret.value)
+ continue;
+
+ out_isa[pos++] = valid_isa_order[i];
+ }
+ out_isa[pos++] = '\0';
+
+ return pos;
+}
+
static long sbi_ext_base_func(long fid)
{
struct sbiret ret;
diff --git a/tests/riscv/common/basic/arch_sbi.h b/tests/riscv/common/basic/arch_sbi.h
index 87db4a29..fe3df663 100644
--- a/tests/riscv/common/basic/arch_sbi.h
+++ b/tests/riscv/common/basic/arch_sbi.h
@@ -283,8 +283,6 @@ void sbi_set_timer(u64 stime_value);
void sbi_clear_timer(void);
-void sbi_shutdown(void);
-
void sbi_clear_ipi(void);
void sbi_send_ipi(const unsigned long *hart_mask);
@@ -298,8 +296,12 @@ void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask,
unsigned long start, unsigned long size,
unsigned long asid);
+void sbi_shutdown(void);
+
void sbi_reset(void);
+unsigned long sbi_xvisor_isa_string(char *out_isa, unsigned long max_len);
+
void sbi_init(void);
#endif
diff --git a/tests/riscv/virt32/basic/arch_board.c b/tests/riscv/virt32/basic/arch_board.c
index 1546cbb4..9dd11f3d 100644
#endif
diff --git a/tests/riscv/virt32/basic/arch_board.c b/tests/riscv/virt32/basic/arch_board.c
--- a/tests/riscv/virt32/basic/arch_board.c
+++ b/tests/riscv/virt32/basic/arch_board.c
@@ -104,8 +104,8 @@ void arch_board_linux_default_cmdline(char *cmdline, u32 cmdline_sz)
void arch_board_fdt_fixup(void *fdt_addr)
{
- char name[64];
u32 i, *vals;
+ char name[64], isa[256];
int ret, cpus_offset, cpu_offset, intc_offset, plic_offset;
u32 timebase_freq = (u32)vminfo_clocksource_freq(VIRT_VMINFO);
u32 vcpu_count = vminfo_vcpu_count(VIRT_VMINFO);
@@ -179,8 +179,9 @@ void arch_board_fdt_fixup(void *fdt_addr)
return;
}
+ sbi_xvisor_isa_string(isa, sizeof(isa));
ret = fdt_setprop_string(fdt_addr, cpu_offset,
- "riscv,isa", "rv32imacfdh");
+ "riscv,isa", isa);
if (ret < 0) {
basic_printf("Failed to set %s property of /cpus/%s "
"DT node\n", "riscv,isa", name);
diff --git a/tests/riscv/virt64/basic/arch_board.c b/tests/riscv/virt64/basic/arch_board.c
basic_printf("Failed to set %s property of /cpus/%s "
"DT node\n", "riscv,isa", name);
index 565fddbe..0ce85c45 100644
--- a/tests/riscv/virt64/basic/arch_board.c
+++ b/tests/riscv/virt64/basic/arch_board.c
@@ -104,8 +104,8 @@ void arch_board_linux_default_cmdline(char *cmdline, u32 cmdline_sz)
void arch_board_fdt_fixup(void *fdt_addr)
{
- char name[64];
u32 i, *vals;
+ char name[64], isa[256];
int ret, cpus_offset, cpu_offset, intc_offset, plic_offset;
u32 timebase_freq = (u32)vminfo_clocksource_freq(VIRT_VMINFO);
u32 vcpu_count = vminfo_vcpu_count(VIRT_VMINFO);
@@ -179,8 +179,9 @@ void arch_board_fdt_fixup(void *fdt_addr)
return;
}
+ sbi_xvisor_isa_string(isa, sizeof(isa));
ret = fdt_setprop_string(fdt_addr, cpu_offset,
- "riscv,isa", "rv64imacfdh");
+ "riscv,isa", isa);
if (ret < 0) {
basic_printf("Failed to set %s property of /cpus/%s "
"DT node\n", "riscv,isa", name);
--
basic_printf("Failed to set %s property of /cpus/%s "
"DT node\n", "riscv,isa", name);
2.25.1
Anup Patel
Mar 17, 2022, 11:23:50 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
The hmask and hbase parameters are swapped in __sbi_rfence_v02() calls
__sbi_rfence_v02_real() for hart_mask == NULL.
arch/riscv/cpu/generic/cpu_sbi.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/riscv/cpu/generic/cpu_sbi.c b/arch/riscv/cpu/generic/cpu_sbi.c
index 12ff28bd..36c1cc9b 100644
--- a/arch/riscv/cpu/generic/cpu_sbi.c
+++ b/arch/riscv/cpu/generic/cpu_sbi.c
@@ -294,7 +294,7 @@ static int __sbi_rfence_v02(unsigned long fid,
int result;
if (!hart_mask) {
- return __sbi_rfence_v02_real(fid, 0UL, -1UL,
+ return __sbi_rfence_v02_real(fid, -1UL, 0UL,
start, size, arg4);
}
--
2.25.1
__sbi_rfence_v02_real() for hart_mask == NULL.
arch/riscv/cpu/generic/cpu_sbi.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/riscv/cpu/generic/cpu_sbi.c b/arch/riscv/cpu/generic/cpu_sbi.c
index 12ff28bd..36c1cc9b 100644
--- a/arch/riscv/cpu/generic/cpu_sbi.c
+++ b/arch/riscv/cpu/generic/cpu_sbi.c
@@ -294,7 +294,7 @@ static int __sbi_rfence_v02(unsigned long fid,
int result;
if (!hart_mask) {
- return __sbi_rfence_v02_real(fid, 0UL, -1UL,
+ return __sbi_rfence_v02_real(fid, -1UL, 0UL,
start, size, arg4);
}
--
2.25.1
Anup Patel
Mar 17, 2022, 11:23:52 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
Instead of returning SBI_ERR_NOT_SUPPORTED for Guest SBI HFENCE calls,
we should treat these SBI calls as NOPs so that subsequent or future
patches can provide complete implementation.
arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c b/arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c
index 3cd226f9..eb7c3acb 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c
@@ -100,8 +100,10 @@ static int vcpu_sbi_rfence_ecall(struct vmm_vcpu *vcpu,
case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID:
/*
* TODO: these SBI calls will be implemented as part
- * of nested virtualization support.
+ * of nested virtualization support so treat these
+ * calls as NOPs.
*/
+ break;
default:
return SBI_ERR_NOT_SUPPORTED;
};
--
2.25.1
we should treat these SBI calls as NOPs so that subsequent or future
patches can provide complete implementation.
arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c b/arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c
index 3cd226f9..eb7c3acb 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c
@@ -100,8 +100,10 @@ static int vcpu_sbi_rfence_ecall(struct vmm_vcpu *vcpu,
case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID:
/*
* TODO: these SBI calls will be implemented as part
- * of nested virtualization support.
+ * of nested virtualization support so treat these
+ * calls as NOPs.
*/
+ break;
default:
return SBI_ERR_NOT_SUPPORTED;
};
--
2.25.1
Anup Patel
Mar 17, 2022, 11:23:54 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
This patch extends nested virtualization state in RISC-V VCPU
private context.
arch/riscv/cpu/generic/cpu_vcpu_helper.c | 41 +++++++++++++++-------
arch/riscv/cpu/generic/include/arch_regs.h | 34 ++++++++++++++++++
2 files changed, 63 insertions(+), 12 deletions(-)
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_helper.c b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
index 899b266b..adb0c72a 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_helper.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
@@ -123,9 +123,14 @@ int arch_guest_init(struct vmm_guest *guest)
guest->arch_priv = NULL;
return VMM_ENOMEM;
}
+
priv->guest_serial = vmm_malloc(sizeof(struct riscv_guest_serial));
- if (!priv->guest_serial)
+ if (!priv->guest_serial) {
+ mmu_pgtbl_free(riscv_guest_priv(guest)->pgtbl);
+ vmm_free(guest->arch_priv);
+ guest->arch_priv = NULL;
return VMM_ENOMEM;
+ }
gserial = riscv_guest_serial(guest);
sname = guest_fdt_find_serial_node(guest->name);
@@ -138,8 +143,13 @@ int arch_guest_init(struct vmm_guest *guest)
gserial->vser_client.notifier_call = &guest_vserial_notification;
gserial->vser_client.priority = 0;
rc = vmm_vserial_register_client(&gserial->vser_client);
- if (rc)
+ if (rc) {
+ vmm_free(gserial);
+ mmu_pgtbl_free(riscv_guest_priv(guest)->pgtbl);
+ vmm_free(guest->arch_priv);
+ guest->arch_priv = NULL;
return rc;
+ }
}
return VMM_OK;
@@ -224,18 +234,18 @@ int arch_vcpu_init(struct vmm_vcpu *vcpu)
rc = vmm_devtree_read_string(vcpu->node,
VMM_DEVTREE_COMPATIBLE_ATTR_NAME, &attr);
if (rc) {
- goto done;
+ goto fail;
}
if (strcmp(attr, "riscv,generic") != 0) {
rc = VMM_EINVALID;
- goto done;
+ goto fail;
}
/* Alloc private context */
vcpu->arch_priv = vmm_zalloc(sizeof(struct riscv_priv));
if (!vcpu->arch_priv) {
rc = VMM_ENOMEM;
- goto done;
+ goto fail;
}
/* Set register width */
@@ -245,10 +255,8 @@ int arch_vcpu_init(struct vmm_vcpu *vcpu)
riscv_priv(vcpu)->isa =
vmm_zalloc(bitmap_estimate_size(RISCV_ISA_EXT_MAX));
if (!riscv_priv(vcpu)->isa) {
- vmm_free(vcpu->arch_priv);
- vcpu->arch_priv = NULL;
rc = VMM_ENOMEM;
- goto done;
+ goto fail_free_priv;
}
/* Parse VCPU ISA string */
@@ -256,17 +264,17 @@ int arch_vcpu_init(struct vmm_vcpu *vcpu)
rc = vmm_devtree_read_string(vcpu->node, "riscv,isa", &attr);
if (rc || !attr) {
rc = VMM_EINVALID;
- goto done;
+ goto fail_free_isa;
}
rc = riscv_isa_parse_string(attr, &riscv_priv(vcpu)->xlen,
riscv_priv(vcpu)->isa,
RISCV_ISA_EXT_MAX);
if (rc) {
- goto done;
+ goto fail_free_isa;
}
if (riscv_priv(vcpu)->xlen > riscv_xlen) {
rc = VMM_EINVALID;
- goto done;
+ goto fail_free_isa;
}
riscv_priv(vcpu)->isa[0] &= RISCV_ISA_ALLOWED;
}
@@ -302,7 +310,16 @@ int arch_vcpu_init(struct vmm_vcpu *vcpu)
cpu_vcpu_fp_init(vcpu);
riscv_timer_event_init(vcpu, &riscv_timer_priv(vcpu));
-done:
+
+ return VMM_OK;
+
+fail_free_isa:
+ vmm_free(riscv_priv(vcpu)->isa);
+ riscv_priv(vcpu)->isa = NULL;
+fail_free_priv:
+ vmm_free(vcpu->arch_priv);
+ vcpu->arch_priv = NULL;
+fail:
return rc;
}
diff --git a/arch/riscv/cpu/generic/include/arch_regs.h b/arch/riscv/cpu/generic/include/arch_regs.h
index 81fb53fb..c9f51ae6 100644
--- a/arch/riscv/cpu/generic/include/arch_regs.h
+++ b/arch/riscv/cpu/generic/include/arch_regs.h
@@ -154,6 +154,36 @@ union riscv_priv_fp {
struct riscv_priv_fp_d d;
};
+struct riscv_priv_nested {
+ /* Nested virt state */
+ bool virt;
+ /* Nested software TLB */
+ void *swtlb;
+ /* Nested shadow page table */
+ struct mmu_pgtbl *pgtbl;
+ /* Nested CSR state */
+ unsigned long hstatus;
+ unsigned long hedeleg;
+ unsigned long hideleg;
+ unsigned long hvip;
+ unsigned long hcounteren;
+ unsigned long htimedelta;
+ unsigned long htimedeltah;
+ unsigned long htval;
+ unsigned long htinst;
+ unsigned long hgatp;
+ unsigned long vsstatus;
+ unsigned long vsie;
+ unsigned long vstvec;
+ unsigned long vsscratch;
+ unsigned long vsepc;
+ unsigned long vscause;
+ unsigned long vstval;
+ unsigned long vsatp;
+ /* Nested AIA CSR state */
+ unsigned long hvictl;
+};
+
struct riscv_priv {
/* Register width */
unsigned long xlen;
@@ -171,6 +201,8 @@ struct riscv_priv {
unsigned long vstval;
unsigned long vsatp;
unsigned long scounteren;
+ /* Nested state */
+ struct riscv_priv_nested nested;
/* FP state */
union riscv_priv_fp fp;
/* Opaque pointer to timer data */
@@ -188,6 +220,8 @@ struct riscv_guest_priv {
#define riscv_regs(vcpu) (&((vcpu)->regs))
#define riscv_priv(vcpu) ((struct riscv_priv *)((vcpu)->arch_priv))
+#define riscv_nested_priv(vcpu) (&riscv_priv(vcpu)->nested)
+#define riscv_nested_virt(vcpu) (riscv_nested_priv(vcpu)->virt)
#define riscv_fp_priv(vcpu) (&riscv_priv(vcpu)->fp)
#define riscv_timer_priv(vcpu) (riscv_priv(vcpu)->timer_priv)
#define riscv_guest_priv(guest) ((struct riscv_guest_priv *)((guest)->arch_priv))
--
2.25.1
private context.
arch/riscv/cpu/generic/cpu_vcpu_helper.c | 41 +++++++++++++++-------
arch/riscv/cpu/generic/include/arch_regs.h | 34 ++++++++++++++++++
2 files changed, 63 insertions(+), 12 deletions(-)
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_helper.c b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
index 899b266b..adb0c72a 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_helper.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
@@ -123,9 +123,14 @@ int arch_guest_init(struct vmm_guest *guest)
guest->arch_priv = NULL;
return VMM_ENOMEM;
}
+
priv->guest_serial = vmm_malloc(sizeof(struct riscv_guest_serial));
- if (!priv->guest_serial)
+ if (!priv->guest_serial) {
+ mmu_pgtbl_free(riscv_guest_priv(guest)->pgtbl);
+ vmm_free(guest->arch_priv);
+ guest->arch_priv = NULL;
return VMM_ENOMEM;
+ }
gserial = riscv_guest_serial(guest);
sname = guest_fdt_find_serial_node(guest->name);
@@ -138,8 +143,13 @@ int arch_guest_init(struct vmm_guest *guest)
gserial->vser_client.notifier_call = &guest_vserial_notification;
gserial->vser_client.priority = 0;
rc = vmm_vserial_register_client(&gserial->vser_client);
- if (rc)
+ if (rc) {
+ vmm_free(gserial);
+ mmu_pgtbl_free(riscv_guest_priv(guest)->pgtbl);
+ vmm_free(guest->arch_priv);
+ guest->arch_priv = NULL;
return rc;
+ }
}
return VMM_OK;
@@ -224,18 +234,18 @@ int arch_vcpu_init(struct vmm_vcpu *vcpu)
rc = vmm_devtree_read_string(vcpu->node,
VMM_DEVTREE_COMPATIBLE_ATTR_NAME, &attr);
if (rc) {
- goto done;
+ goto fail;
}
if (strcmp(attr, "riscv,generic") != 0) {
rc = VMM_EINVALID;
- goto done;
+ goto fail;
}
/* Alloc private context */
vcpu->arch_priv = vmm_zalloc(sizeof(struct riscv_priv));
if (!vcpu->arch_priv) {
rc = VMM_ENOMEM;
- goto done;
+ goto fail;
}
/* Set register width */
@@ -245,10 +255,8 @@ int arch_vcpu_init(struct vmm_vcpu *vcpu)
riscv_priv(vcpu)->isa =
vmm_zalloc(bitmap_estimate_size(RISCV_ISA_EXT_MAX));
if (!riscv_priv(vcpu)->isa) {
- vmm_free(vcpu->arch_priv);
- vcpu->arch_priv = NULL;
rc = VMM_ENOMEM;
- goto done;
+ goto fail_free_priv;
}
/* Parse VCPU ISA string */
@@ -256,17 +264,17 @@ int arch_vcpu_init(struct vmm_vcpu *vcpu)
rc = vmm_devtree_read_string(vcpu->node, "riscv,isa", &attr);
if (rc || !attr) {
rc = VMM_EINVALID;
- goto done;
+ goto fail_free_isa;
}
rc = riscv_isa_parse_string(attr, &riscv_priv(vcpu)->xlen,
riscv_priv(vcpu)->isa,
RISCV_ISA_EXT_MAX);
if (rc) {
- goto done;
+ goto fail_free_isa;
}
if (riscv_priv(vcpu)->xlen > riscv_xlen) {
rc = VMM_EINVALID;
- goto done;
+ goto fail_free_isa;
}
riscv_priv(vcpu)->isa[0] &= RISCV_ISA_ALLOWED;
}
@@ -302,7 +310,16 @@ int arch_vcpu_init(struct vmm_vcpu *vcpu)
cpu_vcpu_fp_init(vcpu);
riscv_timer_event_init(vcpu, &riscv_timer_priv(vcpu));
-done:
+
+ return VMM_OK;
+
+fail_free_isa:
+ vmm_free(riscv_priv(vcpu)->isa);
+ riscv_priv(vcpu)->isa = NULL;
+fail_free_priv:
+ vmm_free(vcpu->arch_priv);
+ vcpu->arch_priv = NULL;
+fail:
return rc;
}
diff --git a/arch/riscv/cpu/generic/include/arch_regs.h b/arch/riscv/cpu/generic/include/arch_regs.h
index 81fb53fb..c9f51ae6 100644
--- a/arch/riscv/cpu/generic/include/arch_regs.h
+++ b/arch/riscv/cpu/generic/include/arch_regs.h
@@ -154,6 +154,36 @@ union riscv_priv_fp {
struct riscv_priv_fp_d d;
};
+struct riscv_priv_nested {
+ /* Nested virt state */
+ bool virt;
+ /* Nested software TLB */
+ void *swtlb;
+ /* Nested shadow page table */
+ struct mmu_pgtbl *pgtbl;
+ /* Nested CSR state */
+ unsigned long hstatus;
+ unsigned long hedeleg;
+ unsigned long hideleg;
+ unsigned long hvip;
+ unsigned long hcounteren;
+ unsigned long htimedelta;
+ unsigned long htimedeltah;
+ unsigned long htval;
+ unsigned long htinst;
+ unsigned long hgatp;
+ unsigned long vsstatus;
+ unsigned long vsie;
+ unsigned long vstvec;
+ unsigned long vsscratch;
+ unsigned long vsepc;
+ unsigned long vscause;
+ unsigned long vstval;
+ unsigned long vsatp;
+ /* Nested AIA CSR state */
+ unsigned long hvictl;
+};
+
struct riscv_priv {
/* Register width */
unsigned long xlen;
@@ -171,6 +201,8 @@ struct riscv_priv {
unsigned long vstval;
unsigned long vsatp;
unsigned long scounteren;
+ /* Nested state */
+ struct riscv_priv_nested nested;
/* FP state */
union riscv_priv_fp fp;
/* Opaque pointer to timer data */
@@ -188,6 +220,8 @@ struct riscv_guest_priv {
#define riscv_regs(vcpu) (&((vcpu)->regs))
#define riscv_priv(vcpu) ((struct riscv_priv *)((vcpu)->arch_priv))
+#define riscv_nested_priv(vcpu) (&riscv_priv(vcpu)->nested)
+#define riscv_nested_virt(vcpu) (riscv_nested_priv(vcpu)->virt)
#define riscv_fp_priv(vcpu) (&riscv_priv(vcpu)->fp)
#define riscv_timer_priv(vcpu) (riscv_priv(vcpu)->timer_priv)
#define riscv_guest_priv(guest) ((struct riscv_guest_priv *)((guest)->arch_priv))
--
2.25.1
Anup Patel
Mar 17, 2022, 11:23:57 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
We add more indentation in VCPU register dump prints so that we can
differentiate VCPU nested virtualization registers using some prefix.
arch/riscv/cpu/generic/cpu_vcpu_fp.c | 4 +-
arch/riscv/cpu/generic/cpu_vcpu_helper.c | 58 ++++++++++++------------
2 files changed, 31 insertions(+), 31 deletions(-)
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_fp.c b/arch/riscv/cpu/generic/cpu_vcpu_fp.c
index 53ba2afa..e4ccebaa 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_fp.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_fp.c
@@ -83,9 +83,9 @@ void cpu_vcpu_fp_dump_regs(struct vmm_chardev *cdev, struct vmm_vcpu *vcpu)
return;
vmm_cprintf(cdev, "\n");
- vmm_cprintf(cdev, " fcsr=0x%08x\n", priv->fp.d.fcsr);
+ vmm_cprintf(cdev, " fcsr=0x%08x\n", priv->fp.d.fcsr);
for (i = 0; i < array_size(priv->fp.d.f) / 2; i++) {
- vmm_cprintf(cdev, " f%02d=0x%016"PRIx64
+ vmm_cprintf(cdev, " f%02d=0x%016"PRIx64
" f%02d=0x%016"PRIx64"\n",
(2*i), priv->fp.d.f[2*i],
(2*i + 1), priv->fp.d.f[2*i + 1]);
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_helper.c b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
index adb0c72a..46833382 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_helper.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
@@ -429,41 +429,41 @@ void arch_vcpu_post_switch(struct vmm_vcpu *vcpu,
void cpu_vcpu_dump_general_regs(struct vmm_chardev *cdev,
arch_regs_t *regs)
{
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" zero", regs->zero, " ra", regs->ra);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" sp", regs->sp, " gp", regs->gp);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" tp", regs->tp, " s0", regs->s0);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" s1", regs->s1, " a0", regs->a0);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" a1", regs->a1, " a2", regs->a2);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" a3", regs->a3, " a4", regs->a4);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" a5", regs->a5, " a6", regs->a6);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" a7", regs->a7, " s2", regs->s2);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" s3", regs->s3, " s4", regs->s4);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" s5", regs->s5, " s6", regs->s6);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" s7", regs->s7, " s8", regs->s8);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" s9", regs->s9, " s10", regs->s10);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" s11", regs->s11, " t0", regs->t0);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" t1", regs->t1, " t2", regs->t2);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" t3", regs->t3, " t4", regs->t4);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" t5", regs->t5, " t6", regs->t6);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" sepc", regs->sepc, " sstatus", regs->sstatus);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" hstatus", regs->hstatus, " sp_exec", regs->sp_exec);
}
@@ -483,28 +483,28 @@ void cpu_vcpu_dump_private_regs(struct vmm_chardev *cdev,
}
vmm_cprintf(cdev, "\n");
- vmm_cprintf(cdev, " %s=%s\n",
+ vmm_cprintf(cdev, " %s=%s\n",
" isa", isa);
vmm_cprintf(cdev, "\n");
#ifdef CONFIG_64BIT
- vmm_cprintf(cdev, "%s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR"\n",
" htimedelta", (ulong)gpriv->time_delta);
#else
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" htimedelta", (ulong)(gpriv->time_delta),
"htimedeltah", (ulong)(gpriv->time_delta >> 32));
#endif
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" hie", priv->hie, " hip", priv->hip);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" hvip", priv->hvip, " vsstatus", priv->vsstatus);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" vsatp", priv->vsatp, " vstvec", priv->vstvec);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" vsscratch", priv->vsscratch, " vsepc", priv->vsepc);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" vscause", priv->vscause, " vstval", priv->vstval);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR"\n",
" scounteren", priv->scounteren);
cpu_vcpu_fp_dump_regs(cdev, vcpu);
@@ -514,9 +514,9 @@ void cpu_vcpu_dump_exception_regs(struct vmm_chardev *cdev,
unsigned long scause, unsigned long stval,
unsigned long htval, unsigned long htinst)
{
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" scause", scause, " stval", stval);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" htval", htval, " htinst", htinst);
}
--
2.25.1
differentiate VCPU nested virtualization registers using some prefix.
arch/riscv/cpu/generic/cpu_vcpu_fp.c | 4 +-
arch/riscv/cpu/generic/cpu_vcpu_helper.c | 58 ++++++++++++------------
2 files changed, 31 insertions(+), 31 deletions(-)
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_fp.c b/arch/riscv/cpu/generic/cpu_vcpu_fp.c
index 53ba2afa..e4ccebaa 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_fp.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_fp.c
@@ -83,9 +83,9 @@ void cpu_vcpu_fp_dump_regs(struct vmm_chardev *cdev, struct vmm_vcpu *vcpu)
return;
vmm_cprintf(cdev, "\n");
- vmm_cprintf(cdev, " fcsr=0x%08x\n", priv->fp.d.fcsr);
+ vmm_cprintf(cdev, " fcsr=0x%08x\n", priv->fp.d.fcsr);
for (i = 0; i < array_size(priv->fp.d.f) / 2; i++) {
- vmm_cprintf(cdev, " f%02d=0x%016"PRIx64
+ vmm_cprintf(cdev, " f%02d=0x%016"PRIx64
" f%02d=0x%016"PRIx64"\n",
(2*i), priv->fp.d.f[2*i],
(2*i + 1), priv->fp.d.f[2*i + 1]);
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_helper.c b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
index adb0c72a..46833382 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_helper.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
@@ -429,41 +429,41 @@ void arch_vcpu_post_switch(struct vmm_vcpu *vcpu,
void cpu_vcpu_dump_general_regs(struct vmm_chardev *cdev,
arch_regs_t *regs)
{
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" zero", regs->zero, " ra", regs->ra);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" sp", regs->sp, " gp", regs->gp);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" tp", regs->tp, " s0", regs->s0);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" s1", regs->s1, " a0", regs->a0);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" a1", regs->a1, " a2", regs->a2);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" a3", regs->a3, " a4", regs->a4);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" a5", regs->a5, " a6", regs->a6);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" a7", regs->a7, " s2", regs->s2);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" s3", regs->s3, " s4", regs->s4);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" s5", regs->s5, " s6", regs->s6);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" s7", regs->s7, " s8", regs->s8);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" s9", regs->s9, " s10", regs->s10);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" s11", regs->s11, " t0", regs->t0);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" t1", regs->t1, " t2", regs->t2);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" t3", regs->t3, " t4", regs->t4);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" t5", regs->t5, " t6", regs->t6);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" sepc", regs->sepc, " sstatus", regs->sstatus);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" hstatus", regs->hstatus, " sp_exec", regs->sp_exec);
}
@@ -483,28 +483,28 @@ void cpu_vcpu_dump_private_regs(struct vmm_chardev *cdev,
}
vmm_cprintf(cdev, "\n");
- vmm_cprintf(cdev, " %s=%s\n",
+ vmm_cprintf(cdev, " %s=%s\n",
" isa", isa);
vmm_cprintf(cdev, "\n");
#ifdef CONFIG_64BIT
- vmm_cprintf(cdev, "%s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR"\n",
" htimedelta", (ulong)gpriv->time_delta);
#else
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" htimedelta", (ulong)(gpriv->time_delta),
"htimedeltah", (ulong)(gpriv->time_delta >> 32));
#endif
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" hie", priv->hie, " hip", priv->hip);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" hvip", priv->hvip, " vsstatus", priv->vsstatus);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" vsatp", priv->vsatp, " vstvec", priv->vstvec);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" vsscratch", priv->vsscratch, " vsepc", priv->vsepc);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" vscause", priv->vscause, " vstval", priv->vstval);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR"\n",
" scounteren", priv->scounteren);
cpu_vcpu_fp_dump_regs(cdev, vcpu);
@@ -514,9 +514,9 @@ void cpu_vcpu_dump_exception_regs(struct vmm_chardev *cdev,
unsigned long scause, unsigned long stval,
unsigned long htval, unsigned long htinst)
{
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" scause", scause, " stval", stval);
- vmm_cprintf(cdev, "%s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ vmm_cprintf(cdev, " %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
" htval", htval, " htinst", htinst);
}
--
2.25.1
Anup Patel
Mar 17, 2022, 11:24:00 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
We would be emulating H-extension CSRs and instructions for nested
virtualization so this patch improves some of the CSR defines to make
them nested virtualization friendly.
arch/riscv/cpu/generic/cpu_exception.c | 18 +-
arch/riscv/cpu/generic/cpu_init.c | 6 +-
arch/riscv/cpu/generic/cpu_mmu.c | 4 +-
.../cpu/generic/include/riscv_encoding.h | 239 +++++++++++++++++-
4 files changed, 239 insertions(+), 28 deletions(-)
diff --git a/arch/riscv/cpu/generic/cpu_exception.c b/arch/riscv/cpu/generic/cpu_exception.c
index dde0a703..76fded9d 100644
--- a/arch/riscv/cpu/generic/cpu_exception.c
+++ b/arch/riscv/cpu/generic/cpu_exception.c
@@ -165,30 +165,18 @@ void do_handle_exception(arch_regs_t *regs)
int __cpuinit arch_cpu_irq_setup(void)
{
- unsigned long hideleg, hedeleg;
extern unsigned long _handle_exception[];
extern unsigned long _handle_hyp_exception[];
if (riscv_isa_extension_available(NULL, h)) {
/* Update HIDELEG */
- hideleg = 0;
- hideleg |= (1UL << IRQ_VS_SOFT);
- hideleg |= (1UL << IRQ_VS_TIMER);
- hideleg |= (1UL << IRQ_VS_EXT);
- csr_write(CSR_HIDELEG, hideleg);
+ csr_write(CSR_HIDELEG, HIDELEG_DEFAULT);
/* Update HEDELEG */
- hedeleg = 0;
- hedeleg |= (1UL << CAUSE_MISALIGNED_FETCH);
- hedeleg |= (1UL << CAUSE_BREAKPOINT);
- hedeleg |= (1UL << CAUSE_USER_ECALL);
- hedeleg |= (1UL << CAUSE_FETCH_PAGE_FAULT);
- hedeleg |= (1UL << CAUSE_LOAD_PAGE_FAULT);
- hedeleg |= (1UL << CAUSE_STORE_PAGE_FAULT);
- csr_write(CSR_HEDELEG, hedeleg);
+ csr_write(CSR_HEDELEG, HEDELEG_DEFAULT);
/* Update HCOUNTEREN */
- csr_write(CSR_HCOUNTEREN, -1UL);
+ csr_write(CSR_HCOUNTEREN, HCOUNTEREN_DEFAULT);
/* Setup final exception handler with hypervisor enabled */
csr_write(CSR_STVEC, (virtual_addr_t)&_handle_hyp_exception);
diff --git a/arch/riscv/cpu/generic/cpu_init.c b/arch/riscv/cpu/generic/cpu_init.c
index ba7d1342..9e9a7ce1 100644
--- a/arch/riscv/cpu/generic/cpu_init.c
+++ b/arch/riscv/cpu/generic/cpu_init.c
@@ -277,13 +277,13 @@ int __init arch_cpu_nascent_init(void)
/* Setup Stage2 mode and Stage2 VMID bits */
if (riscv_isa_extension_available(NULL, h)) {
- csr_write(CSR_HGATP, HGATP_VMID_MASK);
- val = csr_read(CSR_HGATP) & HGATP_VMID_MASK;
+ csr_write(CSR_HGATP, HGATP_VMID);
+ val = csr_read(CSR_HGATP) & HGATP_VMID;
riscv_stage2_vmid_bits = fls_long(val >> HGATP_VMID_SHIFT);
#ifdef CONFIG_64BIT
/* Try Sv48 MMU mode */
- csr_write(CSR_HGATP, HGATP_VMID_MASK |
+ csr_write(CSR_HGATP, HGATP_VMID |
(HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT));
val = csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT;
if (val == HGATP_MODE_SV48X4) {
diff --git a/arch/riscv/cpu/generic/cpu_mmu.c b/arch/riscv/cpu/generic/cpu_mmu.c
index 632af339..853cc9ad 100644
--- a/arch/riscv/cpu/generic/cpu_mmu.c
+++ b/arch/riscv/cpu/generic/cpu_mmu.c
@@ -680,7 +680,7 @@ physical_addr_t arch_mmu_stage2_current_pgtbl_addr(void)
u32 arch_mmu_stage2_current_vmid(void)
{
- return (csr_read(CSR_HGATP) & HGATP_VMID_MASK) >> HGATP_VMID_SHIFT;
+ return (csr_read(CSR_HGATP) & HGATP_VMID) >> HGATP_VMID_SHIFT;
}
int arch_mmu_stage2_change_pgtbl(u32 vmid, physical_addr_t tbl_phys)
@@ -688,7 +688,7 @@ int arch_mmu_stage2_change_pgtbl(u32 vmid, physical_addr_t tbl_phys)
unsigned long hgatp;
hgatp = riscv_stage2_mode << HGATP_MODE_SHIFT;
- hgatp |= ((unsigned long)vmid << HGATP_VMID_SHIFT) & HGATP_VMID_MASK;
+ hgatp |= ((unsigned long)vmid << HGATP_VMID_SHIFT) & HGATP_VMID;
hgatp |= (tbl_phys >> PGTBL_PAGE_SIZE_SHIFT) & HGATP_PPN;
csr_write(CSR_HGATP, hgatp);
diff --git a/arch/riscv/cpu/generic/include/riscv_encoding.h b/arch/riscv/cpu/generic/include/riscv_encoding.h
index c8f5d7ce..992cdd43 100644
--- a/arch/riscv/cpu/generic/include/riscv_encoding.h
+++ b/arch/riscv/cpu/generic/include/riscv_encoding.h
@@ -91,6 +91,7 @@
#define SSTATUS_SIE MSTATUS_SIE
#define SSTATUS_SPIE_SHIFT MSTATUS_SPIE_SHIFT
#define SSTATUS_SPIE MSTATUS_SPIE
+#define SSTATUS_UBE MSTATUS_UBE
#define SSTATUS_SPP_SHIFT MSTATUS_SPP_SHIFT
#define SSTATUS_SPP MSTATUS_SPP
#define SSTATUS_SUM MSTATUS_SUM
@@ -101,8 +102,10 @@
#ifdef CONFIG_64BIT
#define SSTATUS_SD SSTATUS64_SD
+#define SSTATUS_UXL SSTATUS64_UXL
#else
#define SSTATUS_SD SSTATUS32_SD
+#define SSTATUS_UXL _UL(0x0)
#endif
#define SSTATUS_FS MSTATUS_FS
@@ -120,7 +123,13 @@
#ifdef CONFIG_64BIT
#define HSTATUS_VSXL _UL(0x300000000)
#define HSTATUS_VSXL_SHIFT 32
+#else
+#define HSTATUS_VSXL _UL(0x0)
+#define HSTATUS_VSXL_SHIFT 0
#endif
+#define HSTATUS_VSXL_RV32 _UL(0x1)
+#define HSTATUS_VSXL_RV64 _UL(0x2)
+#define HSTATUS_VSXL_RV128 _UL(0x3)
#define HSTATUS_VTSR _UL(0x00400000)
#define HSTATUS_VTW _UL(0x00200000)
#define HSTATUS_VTVM _UL(0x00100000)
@@ -169,6 +178,7 @@
#define HVIP_VSSIP MIP_VSSIP
#define HVIP_VSTIP MIP_VSTIP
#define HVIP_VSEIP MIP_VSEIP
+#define HVIP_WRITEABLE (HVIP_VSSIP | HVIP_VSTIP | HVIP_VSEIP)
#define HIP_VSSIP MIP_VSSIP
#define HIP_VSTIP MIP_VSTIP
@@ -180,6 +190,34 @@
#define HIE_VSEIE MIP_VSEIP
#define HIE_SGEIE MIP_SGEIP
+#define HIDELEG_WRITEABLE HVIP_WRITEABLE
+#define HIDELEG_DEFAULT HIDELEG_WRITEABLE
+
+#define HEDELEG_WRITEABLE \
+ ((_UL(1) << CAUSE_MISALIGNED_FETCH) | \
+ (_UL(1) << CAUSE_FETCH_ACCESS) | \
+ (_UL(1) << CAUSE_ILLEGAL_INSTRUCTION) | \
+ (_UL(1) << CAUSE_BREAKPOINT) | \
+ (_UL(1) << CAUSE_MISALIGNED_LOAD) | \
+ (_UL(1) << CAUSE_LOAD_ACCESS) | \
+ (_UL(1) << CAUSE_MISALIGNED_STORE) | \
+ (_UL(1) << CAUSE_STORE_ACCESS) | \
+ (_UL(1) << CAUSE_USER_ECALL) | \
+ (_UL(1) << CAUSE_FETCH_PAGE_FAULT) | \
+ (_UL(1) << CAUSE_LOAD_PAGE_FAULT) | \
+ (_UL(1) << CAUSE_STORE_PAGE_FAULT))
+#define HEDELEG_DEFAULT \
+ ((_UL(1) << CAUSE_MISALIGNED_FETCH) | \
+ (_UL(1) << CAUSE_BREAKPOINT) | \
+ (_UL(1) << CAUSE_USER_ECALL) | \
+ (_UL(1) << CAUSE_FETCH_PAGE_FAULT) | \
+ (_UL(1) << CAUSE_LOAD_PAGE_FAULT) | \
+ (_UL(1) << CAUSE_STORE_PAGE_FAULT))
+
+#define HCOUNTEREN_WRITEABLE _UL(0xffffffff)
+#define HCOUNTEREN_DEFAULT HCOUNTEREN_WRITEABLE
+
+#define VSIE_WRITEABLE (SIE_SSIE | SIE_STIE | SIE_SEIE)
/* Privilege Modes */
#define PRV_U _UL(0)
@@ -213,35 +251,41 @@
#define HGATP_MODE_SV48X4 _UL(9)
#define HGATP32_MODE_SHIFT 31
+#define HGATP32_MODE _UL(0x80000000)
#define HGATP32_VMID_SHIFT 22
-#define HGATP32_VMID_MASK _UL(0x1FC00000)
+#define HGATP32_VMID _UL(0x1FC00000)
#define HGATP32_PPN _UL(0x003FFFFF)
#define HGATP64_MODE_SHIFT 60
+#define HGATP64_MODE _ULL(0xF000000000000000)
#define HGATP64_VMID_SHIFT 44
-#define HGATP64_VMID_MASK _ULL(0x03FFF00000000000)
+#define HGATP64_VMID _ULL(0x03FFF00000000000)
#define HGATP64_PPN _ULL(0x00000FFFFFFFFFFF)
#ifdef CONFIG_64BIT
#define SATP_PPN SATP64_PPN
#define SATP_ASID_SHIFT SATP64_ASID_SHIFT
-#define SATP_ASID_MASK SATP64_ASID_MASK
+#define SATP_ASID SATP64_ASID
#define SATP_MODE_SHIFT SATP64_MODE_SHIFT
+#define SATP_MODE SATP64_MODE
#define HGATP_PPN HGATP64_PPN
#define HGATP_VMID_SHIFT HGATP64_VMID_SHIFT
-#define HGATP_VMID_MASK HGATP64_VMID_MASK
+#define HGATP_VMID HGATP64_VMID
#define HGATP_MODE_SHIFT HGATP64_MODE_SHIFT
+#define HGATP_MODE HGATP64_MODE
#else
#define SATP_PPN SATP32_PPN
#define SATP_ASID_SHIFT SATP32_ASID_SHIFT
-#define SATP_ASID_MASK SATP32_ASID_MASK
+#define SATP_ASID SATP32_ASID
#define SATP_MODE_SHIFT SATP32_MODE_SHIFT
+#define SATP_MODE SATP32_MODE
#define HGATP_PPN HGATP32_PPN
#define HGATP_VMID_SHIFT HGATP32_VMID_SHIFT
-#define HGATP_VMID_MASK HGATP32_VMID_MASK
+#define HGATP_VMID HGATP32_VMID
#define HGATP_MODE_SHIFT HGATP32_MODE_SHIFT
+#define HGATP_MODE HGATP32_MODE
#endif
@@ -875,8 +919,69 @@
#define INSN_MATCH_C_FSWSP 0xe002
#define INSN_MASK_C_FSWSP 0xe003
-#define INSN_MASK_WFI 0xffffff00
-#define INSN_MATCH_WFI 0x10500000
+#define INSN_MATCH_SRET 0x10200073
+#define INSN_MASK_SRET 0xffffffff
+
+#define INSN_MATCH_WFI 0x10500073
+#define INSN_MASK_WFI 0xffffffff
+
+#define INSN_MATCH_HFENCE_VVMA 0x22000073
+#define INSN_MASK_HFENCE_VVMA 0xfe007fff
+
+#define INSN_MATCH_HFENCE_GVMA 0x62000073
+#define INSN_MASK_HFENCE_GVMA 0xfe007fff
+
+#define INSN_MATCH_HLV_B 0x60004073
+#define INSN_MASK_HLV_B 0xfff0707f
+
+#define INSN_MATCH_HLV_BU 0x60104073
+#define INSN_MASK_HLV_BU 0xfff0707f
+
+#define INSN_MATCH_HLV_H 0x64004073
+#define INSN_MASK_HLV_H 0xfff0707f
+
+#define INSN_MATCH_HLV_HU 0x64104073
+#define INSN_MASK_HLV_HU 0xfff0707f
+
+#define INSN_MATCH_HLVX_HU 0x64304073
+#define INSN_MASK_HLVX_HU 0xfff0707f
+
+#define INSN_MATCH_HLV_W 0x68004073
+#define INSN_MASK_HLV_W 0xfff0707f
+
+#define INSN_MATCH_HLV_WU 0x68104073
+#define INSN_MASK_HLV_WU 0xfff0707f
+
+#define INSN_MATCH_HLVX_WU 0x68304073
+#define INSN_MASK_HLVX_WU 0xfff0707f
+
+#define INSN_MATCH_HLV_D 0x6c004073
+#define INSN_MASK_HLV_D 0xfff0707f
+
+#define INSN_MATCH_HSV_B 0x62004073
+#define INSN_MASK_HSV_B 0xfe007fff
+
+#define INSN_MATCH_HSV_H 0x66004073
+#define INSN_MASK_HSV_H 0xfe007fff
+
+#define INSN_MATCH_HSV_W 0x6a004073
+#define INSN_MASK_HSV_W 0xfe007fff
+
+#define INSN_MATCH_HSV_D 0x6e004073
+#define INSN_MASK_HSV_D 0xfe007fff
+
+#define INSN_MATCH_CSRRW 0x1073
+#define INSN_MASK_CSRRW 0x707f
+#define INSN_MATCH_CSRRS 0x2073
+#define INSN_MASK_CSRRS 0x707f
+#define INSN_MATCH_CSRRC 0x3073
+#define INSN_MASK_CSRRC 0x707f
+#define INSN_MATCH_CSRRWI 0x5073
+#define INSN_MASK_CSRRWI 0x707f
+#define INSN_MATCH_CSRRSI 0x6073
+#define INSN_MASK_CSRRSI 0x707f
+#define INSN_MATCH_CSRRCI 0x7073
+#define INSN_MASK_CSRRCI 0x707f
#define INSN_16BIT_MASK 0x3
#define INSN_32BIT_MASK 0x1c
@@ -900,6 +1005,7 @@
#define SH_RS1 15
#define SH_RS2 20
#define SH_RS2C 2
+#define MASK_RX 0x1f
#define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1))
#define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \
@@ -947,4 +1053,121 @@
(s32)(((insn) >> 7) & 0x1f))
#define MASK_FUNCT3 0x7000
+#define INSN_OPC_LOAD 0x03
+#define INSN_OPC_STORE 0x23
+
+#define INSN_OPC_FP_LOAD 0x7
+#define INSN_OPC_FP_STORE 0x27
+
+#define INSN_OPC_LB (INSN_OPC_LOAD | (0x0 << 12))
+#define INSN_OPC_LH (INSN_OPC_LOAD | (0x1 << 12))
+#define INSN_OPC_LW (INSN_OPC_LOAD | (0x2 << 12))
+#define INSN_OPC_LD (INSN_OPC_LOAD | (0x3 << 12))
+#define INSN_OPC_LBU (INSN_OPC_LOAD | (0x4 << 12))
+#define INSN_OPC_LHU (INSN_OPC_LOAD | (0x5 << 12))
+#define INSN_OPC_LWU (INSN_OPC_LOAD | (0x6 << 12))
+
+#define INSN_OPC_SB (INSN_OPC_STORE | (0x0 << 12))
+#define INSN_OPC_SH (INSN_OPC_STORE | (0x1 << 12))
+#define INSN_OPC_SW (INSN_OPC_STORE | (0x2 << 12))
+#define INSN_OPC_SD (INSN_OPC_STORE | (0x3 << 12))
+
+#define INSN_OPC_FLW (INSN_OPC_FP_LOAD | (0x2 << 12))
+#define INSN_OPC_FLD (INSN_OPC_FP_LOAD | (0x3 << 12))
+
+#define INSN_OPC_FSW (INSN_OPC_FP_STORE | (0x2 << 12))
+#define INSN_OPC_FSD (INSN_OPC_FP_STORE | (0x3 << 12))
+
+#define INSN_GET_RM(insn) (((insn) >> 12) & 0x7)
+#define INSN_GET_RS3(insn) (((insn) >> 27) & 0x1f)
+#define INSN_GET_RS1(insn) (((insn) >> 15) & 0x1f)
+#define INSN_GET_RS2(insn) (((insn) >> 20) & 0x1f)
+#define INSN_GET_RD(insn) (((insn) >> 7) & 0x1f)
+#define INSN_GET_IMM(insn) ((s64)(((insn) << 32) >> 52))
+
+#define INSN_SET_RS1(insn, val) \
+do { \
+ (insn) &= ~(0x1fUL << 15); \
+ (insn) |= (((val) & 0x1fUL) << 15); \
+} while (0)
+#define INSN_SET_RS2(insn, val) \
+do { \
+ (insn) &= ~(0x1fUL << 20); \
+ (insn) |= (((val) & 0x1fUL) << 20); \
+} while (0)
+#define INSN_SET_RD(insn, val) \
+do { \
+ (insn) &= ~(0x1fUL << 7); \
+ (insn) |= (((val) & 0x1fUL) << 7); \
+} while (0)
+#define INSN_SET_I_IMM(insn, val) \
+do { \
+ (insn) &= ~(0xfffUL << 20); \
+ (insn) |= (((val) & 0xfffUL) << 20); \
+} while (0)
+#define INSN_SET_S_IMM(insn, val) \
+do { \
+ (insn) &= ~(0x1fUL << 7); \
+ (insn) |= (((val) & 0x1fUL) << 7); \
+ (insn) &= ~(0x7fUL << 25); \
+ (insn) |= ((((val) >> 5) & 0x7fUL) << 25); \
+} while (0)
+
+#define INSN_GET_C_RD(insn) INSN_GET_RD(insn)
+#define INSN_GET_C_RS1(insn) INSN_GET_RD(insn)
+#define INSN_GET_C_RS2(insn) (((insn) >> 2) & 0x1f)
+#define INSN_GET_C_RS1S(insn) (8 + (((insn) >> 7) & 0x7))
+#define INSN_GET_C_RS2S(insn) (8 + (((insn) >> 2) & 0x7))
+#define INSN_GET_C_FUNC(insn) (((insn) >> 13) & 0x7)
+#define INSN_GET_C_OP(insn) ((insn) & 0x3)
+
+#define INSN_GET_C_LWSP_IMM(insn) \
+(((((insn) >> 4) & 0x7) << 2) | \
+ ((((insn) >> 12) & 0x1) << 5) | \
+ ((((insn) >> 2) & 0x3) << 6))
+#define INSN_GET_C_LDSP_IMM(insn) \
+(((((insn) >> 5) & 0x3) << 3) | \
+ ((((insn) >> 12) & 0x1) << 5) | \
+ ((((insn) >> 2) & 0x7) << 6))
+#define INSN_GET_C_SWSP_IMM(insn) \
+(((((insn) >> 9) & 0xf) << 2) | \
+ ((((insn) >> 7) & 0x3) << 6))
+#define INSN_GET_C_SDSP_IMM(insn) \
+(((((insn) >> 10) & 0x7) << 3) | \
+ ((((insn) >> 7) & 0x7) << 6))
+
+#define INSN_GET_C_LW_IMM(inss) \
+(((((insn) >> 6) & 0x1) << 2) | \
+ ((((insn) >> 10) & 0x7) << 3) | \
+ ((((insn) >> 5) & 0x1) << 6))
+#define INSN_GET_C_LD_IMM(insn) \
+(((((insn) >> 10) & 0x7) << 3) | \
+ ((((insn) >> 5) & 0x3) << 6))
+#define INSN_GET_C_SW_IMM(insn) INSN_GET_C_LW_IMM(insn)
+#define INSN_GET_C_SD_IMM(insn) INSN_GET_C_LD_IMM(insn)
+
+/* RVC Quadrants */
+#define INSN_C_OP_QUAD0 0x0
+#define INSN_C_OP_QUAD1 0x1
+#define INSN_C_OP_QUAD2 0x2
+
+/* RVC Quadrant 0 */
+#define INSN_C_FUNC_ADDI4SPN 0x0
+#define INSN_C_FUNC_FLD_LQ 0x1
+#define INSN_C_FUNC_LW 0x2
+#define INSN_C_FUNC_FLW_LD 0x3
+#define INSN_C_FUNC_FSD_SQ 0x5
+#define INSN_C_FUNC_SW 0x6
+#define INSN_C_FUNC_FSW_SD 0x7
+
+/* RVC Quadrant 2 */
+#define INSN_C_FUNC_SLLI_SLLI64 0x0
+#define INSN_C_FUNC_FLDSP_LQSP 0x1
+#define INSN_C_FUNC_LWSP 0x2
+#define INSN_C_FUNC_FLWSP_LDSP 0x3
+#define INSN_C_FUNC_JR_MV_EBREAK_JALR_ADD 0x4
+#define INSN_C_FUNC_FSDSP_SQSP 0x5
+#define INSN_C_FUNC_SWSP 0x6
+#define INSN_C_FUNC_FSWSP_SDSP 0x7
+
#endif
--
2.25.1
virtualization so this patch improves some of the CSR defines to make
them nested virtualization friendly.
arch/riscv/cpu/generic/cpu_exception.c | 18 +-
arch/riscv/cpu/generic/cpu_init.c | 6 +-
arch/riscv/cpu/generic/cpu_mmu.c | 4 +-
.../cpu/generic/include/riscv_encoding.h | 239 +++++++++++++++++-
4 files changed, 239 insertions(+), 28 deletions(-)
diff --git a/arch/riscv/cpu/generic/cpu_exception.c b/arch/riscv/cpu/generic/cpu_exception.c
index dde0a703..76fded9d 100644
--- a/arch/riscv/cpu/generic/cpu_exception.c
+++ b/arch/riscv/cpu/generic/cpu_exception.c
@@ -165,30 +165,18 @@ void do_handle_exception(arch_regs_t *regs)
int __cpuinit arch_cpu_irq_setup(void)
{
- unsigned long hideleg, hedeleg;
extern unsigned long _handle_exception[];
extern unsigned long _handle_hyp_exception[];
if (riscv_isa_extension_available(NULL, h)) {
/* Update HIDELEG */
- hideleg = 0;
- hideleg |= (1UL << IRQ_VS_SOFT);
- hideleg |= (1UL << IRQ_VS_TIMER);
- hideleg |= (1UL << IRQ_VS_EXT);
- csr_write(CSR_HIDELEG, hideleg);
+ csr_write(CSR_HIDELEG, HIDELEG_DEFAULT);
/* Update HEDELEG */
- hedeleg = 0;
- hedeleg |= (1UL << CAUSE_MISALIGNED_FETCH);
- hedeleg |= (1UL << CAUSE_BREAKPOINT);
- hedeleg |= (1UL << CAUSE_USER_ECALL);
- hedeleg |= (1UL << CAUSE_FETCH_PAGE_FAULT);
- hedeleg |= (1UL << CAUSE_LOAD_PAGE_FAULT);
- hedeleg |= (1UL << CAUSE_STORE_PAGE_FAULT);
- csr_write(CSR_HEDELEG, hedeleg);
+ csr_write(CSR_HEDELEG, HEDELEG_DEFAULT);
/* Update HCOUNTEREN */
- csr_write(CSR_HCOUNTEREN, -1UL);
+ csr_write(CSR_HCOUNTEREN, HCOUNTEREN_DEFAULT);
/* Setup final exception handler with hypervisor enabled */
csr_write(CSR_STVEC, (virtual_addr_t)&_handle_hyp_exception);
diff --git a/arch/riscv/cpu/generic/cpu_init.c b/arch/riscv/cpu/generic/cpu_init.c
index ba7d1342..9e9a7ce1 100644
--- a/arch/riscv/cpu/generic/cpu_init.c
+++ b/arch/riscv/cpu/generic/cpu_init.c
@@ -277,13 +277,13 @@ int __init arch_cpu_nascent_init(void)
/* Setup Stage2 mode and Stage2 VMID bits */
if (riscv_isa_extension_available(NULL, h)) {
- csr_write(CSR_HGATP, HGATP_VMID_MASK);
- val = csr_read(CSR_HGATP) & HGATP_VMID_MASK;
+ csr_write(CSR_HGATP, HGATP_VMID);
+ val = csr_read(CSR_HGATP) & HGATP_VMID;
riscv_stage2_vmid_bits = fls_long(val >> HGATP_VMID_SHIFT);
#ifdef CONFIG_64BIT
/* Try Sv48 MMU mode */
- csr_write(CSR_HGATP, HGATP_VMID_MASK |
+ csr_write(CSR_HGATP, HGATP_VMID |
(HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT));
val = csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT;
if (val == HGATP_MODE_SV48X4) {
diff --git a/arch/riscv/cpu/generic/cpu_mmu.c b/arch/riscv/cpu/generic/cpu_mmu.c
index 632af339..853cc9ad 100644
--- a/arch/riscv/cpu/generic/cpu_mmu.c
+++ b/arch/riscv/cpu/generic/cpu_mmu.c
@@ -680,7 +680,7 @@ physical_addr_t arch_mmu_stage2_current_pgtbl_addr(void)
u32 arch_mmu_stage2_current_vmid(void)
{
- return (csr_read(CSR_HGATP) & HGATP_VMID_MASK) >> HGATP_VMID_SHIFT;
+ return (csr_read(CSR_HGATP) & HGATP_VMID) >> HGATP_VMID_SHIFT;
}
int arch_mmu_stage2_change_pgtbl(u32 vmid, physical_addr_t tbl_phys)
@@ -688,7 +688,7 @@ int arch_mmu_stage2_change_pgtbl(u32 vmid, physical_addr_t tbl_phys)
unsigned long hgatp;
hgatp = riscv_stage2_mode << HGATP_MODE_SHIFT;
- hgatp |= ((unsigned long)vmid << HGATP_VMID_SHIFT) & HGATP_VMID_MASK;
+ hgatp |= ((unsigned long)vmid << HGATP_VMID_SHIFT) & HGATP_VMID;
hgatp |= (tbl_phys >> PGTBL_PAGE_SIZE_SHIFT) & HGATP_PPN;
csr_write(CSR_HGATP, hgatp);
diff --git a/arch/riscv/cpu/generic/include/riscv_encoding.h b/arch/riscv/cpu/generic/include/riscv_encoding.h
index c8f5d7ce..992cdd43 100644
--- a/arch/riscv/cpu/generic/include/riscv_encoding.h
+++ b/arch/riscv/cpu/generic/include/riscv_encoding.h
@@ -91,6 +91,7 @@
#define SSTATUS_SIE MSTATUS_SIE
#define SSTATUS_SPIE_SHIFT MSTATUS_SPIE_SHIFT
#define SSTATUS_SPIE MSTATUS_SPIE
+#define SSTATUS_UBE MSTATUS_UBE
#define SSTATUS_SPP_SHIFT MSTATUS_SPP_SHIFT
#define SSTATUS_SPP MSTATUS_SPP
#define SSTATUS_SUM MSTATUS_SUM
@@ -101,8 +102,10 @@
#ifdef CONFIG_64BIT
#define SSTATUS_SD SSTATUS64_SD
+#define SSTATUS_UXL SSTATUS64_UXL
#else
#define SSTATUS_SD SSTATUS32_SD
+#define SSTATUS_UXL _UL(0x0)
#endif
#define SSTATUS_FS MSTATUS_FS
@@ -120,7 +123,13 @@
#ifdef CONFIG_64BIT
#define HSTATUS_VSXL _UL(0x300000000)
#define HSTATUS_VSXL_SHIFT 32
+#else
+#define HSTATUS_VSXL _UL(0x0)
+#define HSTATUS_VSXL_SHIFT 0
#endif
+#define HSTATUS_VSXL_RV32 _UL(0x1)
+#define HSTATUS_VSXL_RV64 _UL(0x2)
+#define HSTATUS_VSXL_RV128 _UL(0x3)
#define HSTATUS_VTSR _UL(0x00400000)
#define HSTATUS_VTW _UL(0x00200000)
#define HSTATUS_VTVM _UL(0x00100000)
@@ -169,6 +178,7 @@
#define HVIP_VSSIP MIP_VSSIP
#define HVIP_VSTIP MIP_VSTIP
#define HVIP_VSEIP MIP_VSEIP
+#define HVIP_WRITEABLE (HVIP_VSSIP | HVIP_VSTIP | HVIP_VSEIP)
#define HIP_VSSIP MIP_VSSIP
#define HIP_VSTIP MIP_VSTIP
@@ -180,6 +190,34 @@
#define HIE_VSEIE MIP_VSEIP
#define HIE_SGEIE MIP_SGEIP
+#define HIDELEG_WRITEABLE HVIP_WRITEABLE
+#define HIDELEG_DEFAULT HIDELEG_WRITEABLE
+
+#define HEDELEG_WRITEABLE \
+ ((_UL(1) << CAUSE_MISALIGNED_FETCH) | \
+ (_UL(1) << CAUSE_FETCH_ACCESS) | \
+ (_UL(1) << CAUSE_ILLEGAL_INSTRUCTION) | \
+ (_UL(1) << CAUSE_BREAKPOINT) | \
+ (_UL(1) << CAUSE_MISALIGNED_LOAD) | \
+ (_UL(1) << CAUSE_LOAD_ACCESS) | \
+ (_UL(1) << CAUSE_MISALIGNED_STORE) | \
+ (_UL(1) << CAUSE_STORE_ACCESS) | \
+ (_UL(1) << CAUSE_USER_ECALL) | \
+ (_UL(1) << CAUSE_FETCH_PAGE_FAULT) | \
+ (_UL(1) << CAUSE_LOAD_PAGE_FAULT) | \
+ (_UL(1) << CAUSE_STORE_PAGE_FAULT))
+#define HEDELEG_DEFAULT \
+ ((_UL(1) << CAUSE_MISALIGNED_FETCH) | \
+ (_UL(1) << CAUSE_BREAKPOINT) | \
+ (_UL(1) << CAUSE_USER_ECALL) | \
+ (_UL(1) << CAUSE_FETCH_PAGE_FAULT) | \
+ (_UL(1) << CAUSE_LOAD_PAGE_FAULT) | \
+ (_UL(1) << CAUSE_STORE_PAGE_FAULT))
+
+#define HCOUNTEREN_WRITEABLE _UL(0xffffffff)
+#define HCOUNTEREN_DEFAULT HCOUNTEREN_WRITEABLE
+
+#define VSIE_WRITEABLE (SIE_SSIE | SIE_STIE | SIE_SEIE)
/* Privilege Modes */
#define PRV_U _UL(0)
@@ -213,35 +251,41 @@
#define HGATP_MODE_SV48X4 _UL(9)
#define HGATP32_MODE_SHIFT 31
+#define HGATP32_MODE _UL(0x80000000)
#define HGATP32_VMID_SHIFT 22
-#define HGATP32_VMID_MASK _UL(0x1FC00000)
+#define HGATP32_VMID _UL(0x1FC00000)
#define HGATP32_PPN _UL(0x003FFFFF)
#define HGATP64_MODE_SHIFT 60
+#define HGATP64_MODE _ULL(0xF000000000000000)
#define HGATP64_VMID_SHIFT 44
-#define HGATP64_VMID_MASK _ULL(0x03FFF00000000000)
+#define HGATP64_VMID _ULL(0x03FFF00000000000)
#define HGATP64_PPN _ULL(0x00000FFFFFFFFFFF)
#ifdef CONFIG_64BIT
#define SATP_PPN SATP64_PPN
#define SATP_ASID_SHIFT SATP64_ASID_SHIFT
-#define SATP_ASID_MASK SATP64_ASID_MASK
+#define SATP_ASID SATP64_ASID
#define SATP_MODE_SHIFT SATP64_MODE_SHIFT
+#define SATP_MODE SATP64_MODE
#define HGATP_PPN HGATP64_PPN
#define HGATP_VMID_SHIFT HGATP64_VMID_SHIFT
-#define HGATP_VMID_MASK HGATP64_VMID_MASK
+#define HGATP_VMID HGATP64_VMID
#define HGATP_MODE_SHIFT HGATP64_MODE_SHIFT
+#define HGATP_MODE HGATP64_MODE
#else
#define SATP_PPN SATP32_PPN
#define SATP_ASID_SHIFT SATP32_ASID_SHIFT
-#define SATP_ASID_MASK SATP32_ASID_MASK
+#define SATP_ASID SATP32_ASID
#define SATP_MODE_SHIFT SATP32_MODE_SHIFT
+#define SATP_MODE SATP32_MODE
#define HGATP_PPN HGATP32_PPN
#define HGATP_VMID_SHIFT HGATP32_VMID_SHIFT
-#define HGATP_VMID_MASK HGATP32_VMID_MASK
+#define HGATP_VMID HGATP32_VMID
#define HGATP_MODE_SHIFT HGATP32_MODE_SHIFT
+#define HGATP_MODE HGATP32_MODE
#endif
@@ -875,8 +919,69 @@
#define INSN_MATCH_C_FSWSP 0xe002
#define INSN_MASK_C_FSWSP 0xe003
-#define INSN_MASK_WFI 0xffffff00
-#define INSN_MATCH_WFI 0x10500000
+#define INSN_MATCH_SRET 0x10200073
+#define INSN_MASK_SRET 0xffffffff
+
+#define INSN_MATCH_WFI 0x10500073
+#define INSN_MASK_WFI 0xffffffff
+
+#define INSN_MATCH_HFENCE_VVMA 0x22000073
+#define INSN_MASK_HFENCE_VVMA 0xfe007fff
+
+#define INSN_MATCH_HFENCE_GVMA 0x62000073
+#define INSN_MASK_HFENCE_GVMA 0xfe007fff
+
+#define INSN_MATCH_HLV_B 0x60004073
+#define INSN_MASK_HLV_B 0xfff0707f
+
+#define INSN_MATCH_HLV_BU 0x60104073
+#define INSN_MASK_HLV_BU 0xfff0707f
+
+#define INSN_MATCH_HLV_H 0x64004073
+#define INSN_MASK_HLV_H 0xfff0707f
+
+#define INSN_MATCH_HLV_HU 0x64104073
+#define INSN_MASK_HLV_HU 0xfff0707f
+
+#define INSN_MATCH_HLVX_HU 0x64304073
+#define INSN_MASK_HLVX_HU 0xfff0707f
+
+#define INSN_MATCH_HLV_W 0x68004073
+#define INSN_MASK_HLV_W 0xfff0707f
+
+#define INSN_MATCH_HLV_WU 0x68104073
+#define INSN_MASK_HLV_WU 0xfff0707f
+
+#define INSN_MATCH_HLVX_WU 0x68304073
+#define INSN_MASK_HLVX_WU 0xfff0707f
+
+#define INSN_MATCH_HLV_D 0x6c004073
+#define INSN_MASK_HLV_D 0xfff0707f
+
+#define INSN_MATCH_HSV_B 0x62004073
+#define INSN_MASK_HSV_B 0xfe007fff
+
+#define INSN_MATCH_HSV_H 0x66004073
+#define INSN_MASK_HSV_H 0xfe007fff
+
+#define INSN_MATCH_HSV_W 0x6a004073
+#define INSN_MASK_HSV_W 0xfe007fff
+
+#define INSN_MATCH_HSV_D 0x6e004073
+#define INSN_MASK_HSV_D 0xfe007fff
+
+#define INSN_MATCH_CSRRW 0x1073
+#define INSN_MASK_CSRRW 0x707f
+#define INSN_MATCH_CSRRS 0x2073
+#define INSN_MASK_CSRRS 0x707f
+#define INSN_MATCH_CSRRC 0x3073
+#define INSN_MASK_CSRRC 0x707f
+#define INSN_MATCH_CSRRWI 0x5073
+#define INSN_MASK_CSRRWI 0x707f
+#define INSN_MATCH_CSRRSI 0x6073
+#define INSN_MASK_CSRRSI 0x707f
+#define INSN_MATCH_CSRRCI 0x7073
+#define INSN_MASK_CSRRCI 0x707f
#define INSN_16BIT_MASK 0x3
#define INSN_32BIT_MASK 0x1c
@@ -900,6 +1005,7 @@
#define SH_RS1 15
#define SH_RS2 20
#define SH_RS2C 2
+#define MASK_RX 0x1f
#define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1))
#define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \
@@ -947,4 +1053,121 @@
(s32)(((insn) >> 7) & 0x1f))
#define MASK_FUNCT3 0x7000
+#define INSN_OPC_LOAD 0x03
+#define INSN_OPC_STORE 0x23
+
+#define INSN_OPC_FP_LOAD 0x7
+#define INSN_OPC_FP_STORE 0x27
+
+#define INSN_OPC_LB (INSN_OPC_LOAD | (0x0 << 12))
+#define INSN_OPC_LH (INSN_OPC_LOAD | (0x1 << 12))
+#define INSN_OPC_LW (INSN_OPC_LOAD | (0x2 << 12))
+#define INSN_OPC_LD (INSN_OPC_LOAD | (0x3 << 12))
+#define INSN_OPC_LBU (INSN_OPC_LOAD | (0x4 << 12))
+#define INSN_OPC_LHU (INSN_OPC_LOAD | (0x5 << 12))
+#define INSN_OPC_LWU (INSN_OPC_LOAD | (0x6 << 12))
+
+#define INSN_OPC_SB (INSN_OPC_STORE | (0x0 << 12))
+#define INSN_OPC_SH (INSN_OPC_STORE | (0x1 << 12))
+#define INSN_OPC_SW (INSN_OPC_STORE | (0x2 << 12))
+#define INSN_OPC_SD (INSN_OPC_STORE | (0x3 << 12))
+
+#define INSN_OPC_FLW (INSN_OPC_FP_LOAD | (0x2 << 12))
+#define INSN_OPC_FLD (INSN_OPC_FP_LOAD | (0x3 << 12))
+
+#define INSN_OPC_FSW (INSN_OPC_FP_STORE | (0x2 << 12))
+#define INSN_OPC_FSD (INSN_OPC_FP_STORE | (0x3 << 12))
+
+#define INSN_GET_RM(insn) (((insn) >> 12) & 0x7)
+#define INSN_GET_RS3(insn) (((insn) >> 27) & 0x1f)
+#define INSN_GET_RS1(insn) (((insn) >> 15) & 0x1f)
+#define INSN_GET_RS2(insn) (((insn) >> 20) & 0x1f)
+#define INSN_GET_RD(insn) (((insn) >> 7) & 0x1f)
+#define INSN_GET_IMM(insn) ((s64)(((insn) << 32) >> 52))
+
+#define INSN_SET_RS1(insn, val) \
+do { \
+ (insn) &= ~(0x1fUL << 15); \
+ (insn) |= (((val) & 0x1fUL) << 15); \
+} while (0)
+#define INSN_SET_RS2(insn, val) \
+do { \
+ (insn) &= ~(0x1fUL << 20); \
+ (insn) |= (((val) & 0x1fUL) << 20); \
+} while (0)
+#define INSN_SET_RD(insn, val) \
+do { \
+ (insn) &= ~(0x1fUL << 7); \
+ (insn) |= (((val) & 0x1fUL) << 7); \
+} while (0)
+#define INSN_SET_I_IMM(insn, val) \
+do { \
+ (insn) &= ~(0xfffUL << 20); \
+ (insn) |= (((val) & 0xfffUL) << 20); \
+} while (0)
+#define INSN_SET_S_IMM(insn, val) \
+do { \
+ (insn) &= ~(0x1fUL << 7); \
+ (insn) |= (((val) & 0x1fUL) << 7); \
+ (insn) &= ~(0x7fUL << 25); \
+ (insn) |= ((((val) >> 5) & 0x7fUL) << 25); \
+} while (0)
+
+#define INSN_GET_C_RD(insn) INSN_GET_RD(insn)
+#define INSN_GET_C_RS1(insn) INSN_GET_RD(insn)
+#define INSN_GET_C_RS2(insn) (((insn) >> 2) & 0x1f)
+#define INSN_GET_C_RS1S(insn) (8 + (((insn) >> 7) & 0x7))
+#define INSN_GET_C_RS2S(insn) (8 + (((insn) >> 2) & 0x7))
+#define INSN_GET_C_FUNC(insn) (((insn) >> 13) & 0x7)
+#define INSN_GET_C_OP(insn) ((insn) & 0x3)
+
+#define INSN_GET_C_LWSP_IMM(insn) \
+(((((insn) >> 4) & 0x7) << 2) | \
+ ((((insn) >> 12) & 0x1) << 5) | \
+ ((((insn) >> 2) & 0x3) << 6))
+#define INSN_GET_C_LDSP_IMM(insn) \
+(((((insn) >> 5) & 0x3) << 3) | \
+ ((((insn) >> 12) & 0x1) << 5) | \
+ ((((insn) >> 2) & 0x7) << 6))
+#define INSN_GET_C_SWSP_IMM(insn) \
+(((((insn) >> 9) & 0xf) << 2) | \
+ ((((insn) >> 7) & 0x3) << 6))
+#define INSN_GET_C_SDSP_IMM(insn) \
+(((((insn) >> 10) & 0x7) << 3) | \
+ ((((insn) >> 7) & 0x7) << 6))
+
+#define INSN_GET_C_LW_IMM(inss) \
+(((((insn) >> 6) & 0x1) << 2) | \
+ ((((insn) >> 10) & 0x7) << 3) | \
+ ((((insn) >> 5) & 0x1) << 6))
+#define INSN_GET_C_LD_IMM(insn) \
+(((((insn) >> 10) & 0x7) << 3) | \
+ ((((insn) >> 5) & 0x3) << 6))
+#define INSN_GET_C_SW_IMM(insn) INSN_GET_C_LW_IMM(insn)
+#define INSN_GET_C_SD_IMM(insn) INSN_GET_C_LD_IMM(insn)
+
+/* RVC Quadrants */
+#define INSN_C_OP_QUAD0 0x0
+#define INSN_C_OP_QUAD1 0x1
+#define INSN_C_OP_QUAD2 0x2
+
+/* RVC Quadrant 0 */
+#define INSN_C_FUNC_ADDI4SPN 0x0
+#define INSN_C_FUNC_FLD_LQ 0x1
+#define INSN_C_FUNC_LW 0x2
+#define INSN_C_FUNC_FLW_LD 0x3
+#define INSN_C_FUNC_FSD_SQ 0x5
+#define INSN_C_FUNC_SW 0x6
+#define INSN_C_FUNC_FSW_SD 0x7
+
+/* RVC Quadrant 2 */
+#define INSN_C_FUNC_SLLI_SLLI64 0x0
+#define INSN_C_FUNC_FLDSP_LQSP 0x1
+#define INSN_C_FUNC_LWSP 0x2
+#define INSN_C_FUNC_FLWSP_LDSP 0x3
+#define INSN_C_FUNC_JR_MV_EBREAK_JALR_ADD 0x4
+#define INSN_C_FUNC_FSDSP_SQSP 0x5
+#define INSN_C_FUNC_SWSP 0x6
+#define INSN_C_FUNC_FSWSP_SDSP 0x7
+
#endif
--
2.25.1
Anup Patel
Mar 17, 2022, 11:24:03 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
This patch adds nested virtualization friendly helper functions to
update interrupt delegation, G-stage page table and time delta.
arch/riscv/cpu/generic/cpu_init.c | 3 +
arch/riscv/cpu/generic/cpu_vcpu_helper.c | 95 +++++++++++++++----
arch/riscv/cpu/generic/include/cpu_hwcap.h | 9 +-
arch/riscv/cpu/generic/include/cpu_vcpu_fp.h | 4 +-
.../cpu/generic/include/cpu_vcpu_helper.h | 9 ++
5 files changed, 100 insertions(+), 20 deletions(-)
diff --git a/arch/riscv/cpu/generic/cpu_init.c b/arch/riscv/cpu/generic/cpu_init.c
index 9e9a7ce1..64799fd8 100644
--- a/arch/riscv/cpu/generic/cpu_init.c
+++ b/arch/riscv/cpu/generic/cpu_init.c
@@ -178,6 +178,8 @@ unsigned long riscv_stage2_mode = HGATP_MODE_SV39X4;
unsigned long riscv_stage2_mode = HGATP_MODE_SV32X4;
#endif
unsigned long riscv_stage2_vmid_bits = 0;
+unsigned long riscv_stage2_vmid_nested = 0;
+bool riscv_stage2_use_vmid = false;
unsigned long riscv_timer_hz = 0;
bool riscv_aia_available = true;
@@ -280,6 +282,7 @@ int __init arch_cpu_nascent_init(void)
csr_write(CSR_HGATP, HGATP_VMID);
#ifdef CONFIG_64BIT
/* Try Sv48 MMU mode */
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_helper.c b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
index 46833382..4ff619a7 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_helper.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
@@ -389,15 +389,6 @@ void arch_vcpu_switch(struct vmm_vcpu *tvcpu,
memcpy(regs, riscv_regs(vcpu), sizeof(*regs));
if (vcpu->is_normal) {
priv = riscv_priv(vcpu);
-#ifdef CONFIG_64BIT
- csr_write(CSR_HTIMEDELTA,
- riscv_guest_priv(vcpu->guest)->time_delta);
-#else
- csr_write(CSR_HTIMEDELTA,
- (u32)(riscv_guest_priv(vcpu->guest)->time_delta));
- csr_write(CSR_HTIMEDELTAH,
- (u32)(riscv_guest_priv(vcpu->guest)->time_delta >> 32));
-#endif
csr_write(CSR_HIE, priv->hie);
csr_write(CSR_HVIP, priv->hvip);
csr_write(CSR_VSSTATUS, priv->vsstatus);
@@ -409,14 +400,9 @@ void arch_vcpu_switch(struct vmm_vcpu *tvcpu,
csr_write(CSR_VSATP, priv->vsatp);
csr_write(CSR_SCOUNTEREN, priv->scounteren);
cpu_vcpu_fp_restore(vcpu, regs);
- if (CONFIG_MAX_GUEST_COUNT <= (1UL << riscv_stage2_vmid_bits)) {
- mmu_stage2_change_pgtbl(vcpu->guest->id,
- riscv_guest_priv(vcpu->guest)->pgtbl);
- } else {
- mmu_stage2_change_pgtbl(0,
- riscv_guest_priv(vcpu->guest)->pgtbl);
- __hfence_gvma_all();
- }
+ cpu_vcpu_time_delta_update(vcpu, riscv_nested_virt(vcpu));
+ cpu_vcpu_gstage_update(vcpu, riscv_nested_virt(vcpu));
+ cpu_vcpu_irq_deleg_update(vcpu, riscv_nested_virt(vcpu));
}
}
@@ -426,6 +412,81 @@ void arch_vcpu_post_switch(struct vmm_vcpu *vcpu,
/* Nothing to do here */
}
+void cpu_vcpu_irq_deleg_update(struct vmm_vcpu *vcpu, bool nested_virt)
+{
+ if (nested_virt) {
+ /* Disable interrupt delegation */
+ csr_write(CSR_HIDELEG, 0);
+
+ /* Enable sip/siph and sie/sieh trapping */
+ if (riscv_aia_available) {
+ csr_set(CSR_HVICTL, HVICTL_VTI);
+ }
+ } else {
+ /* Enable interrupt delegation */
+ csr_write(CSR_HIDELEG, HIDELEG_DEFAULT);
+
+ /* Disable sip/siph and sie/sieh trapping */
+ if (riscv_aia_available) {
+ csr_clear(CSR_HVICTL, HVICTL_VTI);
+ }
+ }
+}
+
+void cpu_vcpu_time_delta_update(struct vmm_vcpu *vcpu, bool nested_virt)
+{
+ u64 vtdelta, tdelta = riscv_guest_priv(vcpu->guest)->time_delta;
+
+ if (nested_virt) {
+ vtdelta = riscv_nested_priv(vcpu)->htimedelta;
+#ifndef CONFIG_64BIT
+ vtdelta |= ((u64)riscv_nested_priv(vcpu)->htimedeltah) << 32;
+#endif
+ tdelta += vtdelta;
+ }
+
+#ifdef CONFIG_64BIT
+ csr_write(CSR_HTIMEDELTA, tdelta);
+#else
+ csr_write(CSR_HTIMEDELTA, (u32)tdelta);
+ csr_write(CSR_HTIMEDELTAH, (u32)(tdelta >> 32));
+#endif
+}
+
+void cpu_vcpu_gstage_update(struct vmm_vcpu *vcpu, bool nested_virt)
+{
+ if (riscv_stage2_vmid_available()) {
+ if (nested_virt) {
+ mmu_stage2_change_pgtbl(
+ riscv_stage2_vmid_nested + vcpu->guest->id,
+ riscv_nested_priv(vcpu)->pgtbl);
+ } else {
+ mmu_stage2_change_pgtbl(vcpu->guest->id,
+ riscv_guest_priv(vcpu->guest)->pgtbl);
+ }
+ } else {
+ if (nested_virt) {
+ mmu_stage2_change_pgtbl(0,
+ riscv_nested_priv(vcpu)->pgtbl);
+ } else {
+ mmu_stage2_change_pgtbl(0,
+ riscv_guest_priv(vcpu->guest)->pgtbl);
+ }
+
+ /*
+ * Invalidate entries related to all guests from both
+ * G-stage TLB and VS-stage TLB.
+ *
+ * NOTE: Due to absence of VMID, there is not VMID tagging
+ * in VS-stage TLB as well so to avoid one Guest seeing
+ * VS-stage mappings of other Guest we have to invalidate
+ * VS-stage TLB enteries as well.
+ */
+ __hfence_gvma_all();
+ __hfence_vvma_all();
+ }
+}
+
index 2ed9bfe9..c7c87829 100644
--- a/arch/riscv/cpu/generic/include/cpu_hwcap.h
+++ b/arch/riscv/cpu/generic/include/cpu_hwcap.h
@@ -115,9 +115,16 @@ extern unsigned long riscv_xlen;
/** RISC-V Stage2 MMU mode */
extern unsigned long riscv_stage2_mode;
-/** Available RISC-V Stage2 VMID bits */
+/** RISC-V Stage2 VMID bits */
extern unsigned long riscv_stage2_vmid_bits;
+/** RISC-V Stage2 VMID nested */
+extern unsigned long riscv_stage2_vmid_nested;
+
+/** RISC-V Stage2 VMID available for Guest */
+#define riscv_stage2_vmid_available() \
+ (CONFIG_MAX_GUEST_COUNT <= riscv_stage2_vmid_nested)
+
/** RISC-V Time Base Frequency */
extern unsigned long riscv_timer_hz;
diff --git a/arch/riscv/cpu/generic/include/cpu_vcpu_fp.h b/arch/riscv/cpu/generic/include/cpu_vcpu_fp.h
index 25b74b45..38e1ac06 100644
--- a/arch/riscv/cpu/generic/include/cpu_vcpu_fp.h
+++ b/arch/riscv/cpu/generic/include/cpu_vcpu_fp.h
@@ -21,8 +21,8 @@
* @brief header of VCPU FP functions
*/
-#ifndef _CPU_VCPU_HELPER_H__
-#define _CPU_VCPU_HELPER_H__
+#ifndef _CPU_VCPU_FP_H__
+#define _CPU_VCPU_FP_H__
#include <vmm_types.h>
#include <vmm_manager.h>
diff --git a/arch/riscv/cpu/generic/include/cpu_vcpu_helper.h b/arch/riscv/cpu/generic/include/cpu_vcpu_helper.h
index 694af455..fc587323 100644
--- a/arch/riscv/cpu/generic/include/cpu_vcpu_helper.h
+++ b/arch/riscv/cpu/generic/include/cpu_vcpu_helper.h
@@ -26,6 +26,15 @@
#include <vmm_types.h>
#include <vmm_manager.h>
+/** Function to update interrupt delegation */
+void cpu_vcpu_irq_deleg_update(struct vmm_vcpu *vcpu, bool nested_virt);
+
+/** Function to update time delta */
+void cpu_vcpu_time_delta_update(struct vmm_vcpu *vcpu, bool nested_virt);
+
+/** Function to update G-stage page table */
+void cpu_vcpu_gstage_update(struct vmm_vcpu *vcpu, bool nested_virt);
+
/** Function to dump general registers */
void cpu_vcpu_dump_general_regs(struct vmm_chardev *cdev,
arch_regs_t *regs);
--
2.25.1
update interrupt delegation, G-stage page table and time delta.
arch/riscv/cpu/generic/cpu_init.c | 3 +
arch/riscv/cpu/generic/cpu_vcpu_helper.c | 95 +++++++++++++++----
arch/riscv/cpu/generic/include/cpu_hwcap.h | 9 +-
arch/riscv/cpu/generic/include/cpu_vcpu_fp.h | 4 +-
.../cpu/generic/include/cpu_vcpu_helper.h | 9 ++
5 files changed, 100 insertions(+), 20 deletions(-)
diff --git a/arch/riscv/cpu/generic/cpu_init.c b/arch/riscv/cpu/generic/cpu_init.c
index 9e9a7ce1..64799fd8 100644
--- a/arch/riscv/cpu/generic/cpu_init.c
+++ b/arch/riscv/cpu/generic/cpu_init.c
@@ -178,6 +178,8 @@ unsigned long riscv_stage2_mode = HGATP_MODE_SV39X4;
unsigned long riscv_stage2_mode = HGATP_MODE_SV32X4;
#endif
unsigned long riscv_stage2_vmid_bits = 0;
+unsigned long riscv_stage2_vmid_nested = 0;
+bool riscv_stage2_use_vmid = false;
unsigned long riscv_timer_hz = 0;
bool riscv_aia_available = true;
@@ -280,6 +282,7 @@ int __init arch_cpu_nascent_init(void)
csr_write(CSR_HGATP, HGATP_VMID);
val = csr_read(CSR_HGATP) & HGATP_VMID;
riscv_stage2_vmid_bits = fls_long(val >> HGATP_VMID_SHIFT);
+ riscv_stage2_vmid_nested = (1UL << riscv_stage2_vmid_bits) / 2;
riscv_stage2_vmid_bits = fls_long(val >> HGATP_VMID_SHIFT);
#ifdef CONFIG_64BIT
/* Try Sv48 MMU mode */
index 46833382..4ff619a7 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_helper.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
@@ -389,15 +389,6 @@ void arch_vcpu_switch(struct vmm_vcpu *tvcpu,
memcpy(regs, riscv_regs(vcpu), sizeof(*regs));
if (vcpu->is_normal) {
priv = riscv_priv(vcpu);
-#ifdef CONFIG_64BIT
- csr_write(CSR_HTIMEDELTA,
- riscv_guest_priv(vcpu->guest)->time_delta);
-#else
- csr_write(CSR_HTIMEDELTA,
- (u32)(riscv_guest_priv(vcpu->guest)->time_delta));
- csr_write(CSR_HTIMEDELTAH,
- (u32)(riscv_guest_priv(vcpu->guest)->time_delta >> 32));
-#endif
csr_write(CSR_HIE, priv->hie);
csr_write(CSR_HVIP, priv->hvip);
csr_write(CSR_VSSTATUS, priv->vsstatus);
@@ -409,14 +400,9 @@ void arch_vcpu_switch(struct vmm_vcpu *tvcpu,
csr_write(CSR_VSATP, priv->vsatp);
csr_write(CSR_SCOUNTEREN, priv->scounteren);
cpu_vcpu_fp_restore(vcpu, regs);
- if (CONFIG_MAX_GUEST_COUNT <= (1UL << riscv_stage2_vmid_bits)) {
- mmu_stage2_change_pgtbl(vcpu->guest->id,
- riscv_guest_priv(vcpu->guest)->pgtbl);
- } else {
- mmu_stage2_change_pgtbl(0,
- riscv_guest_priv(vcpu->guest)->pgtbl);
- __hfence_gvma_all();
- }
+ cpu_vcpu_time_delta_update(vcpu, riscv_nested_virt(vcpu));
+ cpu_vcpu_gstage_update(vcpu, riscv_nested_virt(vcpu));
+ cpu_vcpu_irq_deleg_update(vcpu, riscv_nested_virt(vcpu));
}
}
@@ -426,6 +412,81 @@ void arch_vcpu_post_switch(struct vmm_vcpu *vcpu,
/* Nothing to do here */
}
+void cpu_vcpu_irq_deleg_update(struct vmm_vcpu *vcpu, bool nested_virt)
+{
+ if (nested_virt) {
+ /* Disable interrupt delegation */
+ csr_write(CSR_HIDELEG, 0);
+
+ /* Enable sip/siph and sie/sieh trapping */
+ if (riscv_aia_available) {
+ csr_set(CSR_HVICTL, HVICTL_VTI);
+ }
+ } else {
+ /* Enable interrupt delegation */
+ csr_write(CSR_HIDELEG, HIDELEG_DEFAULT);
+
+ /* Disable sip/siph and sie/sieh trapping */
+ if (riscv_aia_available) {
+ csr_clear(CSR_HVICTL, HVICTL_VTI);
+ }
+ }
+}
+
+void cpu_vcpu_time_delta_update(struct vmm_vcpu *vcpu, bool nested_virt)
+{
+ u64 vtdelta, tdelta = riscv_guest_priv(vcpu->guest)->time_delta;
+
+ if (nested_virt) {
+ vtdelta = riscv_nested_priv(vcpu)->htimedelta;
+#ifndef CONFIG_64BIT
+ vtdelta |= ((u64)riscv_nested_priv(vcpu)->htimedeltah) << 32;
+#endif
+ tdelta += vtdelta;
+ }
+
+#ifdef CONFIG_64BIT
+ csr_write(CSR_HTIMEDELTA, tdelta);
+#else
+ csr_write(CSR_HTIMEDELTA, (u32)tdelta);
+ csr_write(CSR_HTIMEDELTAH, (u32)(tdelta >> 32));
+#endif
+}
+
+void cpu_vcpu_gstage_update(struct vmm_vcpu *vcpu, bool nested_virt)
+{
+ if (riscv_stage2_vmid_available()) {
+ if (nested_virt) {
+ mmu_stage2_change_pgtbl(
+ riscv_stage2_vmid_nested + vcpu->guest->id,
+ riscv_nested_priv(vcpu)->pgtbl);
+ } else {
+ mmu_stage2_change_pgtbl(vcpu->guest->id,
+ riscv_guest_priv(vcpu->guest)->pgtbl);
+ }
+ } else {
+ if (nested_virt) {
+ mmu_stage2_change_pgtbl(0,
+ riscv_nested_priv(vcpu)->pgtbl);
+ } else {
+ mmu_stage2_change_pgtbl(0,
+ riscv_guest_priv(vcpu->guest)->pgtbl);
+ }
+
+ /*
+ * Invalidate entries related to all guests from both
+ * G-stage TLB and VS-stage TLB.
+ *
+ * NOTE: Due to absence of VMID, there is not VMID tagging
+ * in VS-stage TLB as well so to avoid one Guest seeing
+ * VS-stage mappings of other Guest we have to invalidate
+ * VS-stage TLB enteries as well.
+ */
+ __hfence_gvma_all();
+ __hfence_vvma_all();
+ }
+}
+
void cpu_vcpu_dump_general_regs(struct vmm_chardev *cdev,
arch_regs_t *regs)
{
diff --git a/arch/riscv/cpu/generic/include/cpu_hwcap.h b/arch/riscv/cpu/generic/include/cpu_hwcap.h
arch_regs_t *regs)
{
index 2ed9bfe9..c7c87829 100644
--- a/arch/riscv/cpu/generic/include/cpu_hwcap.h
+++ b/arch/riscv/cpu/generic/include/cpu_hwcap.h
@@ -115,9 +115,16 @@ extern unsigned long riscv_xlen;
/** RISC-V Stage2 MMU mode */
extern unsigned long riscv_stage2_mode;
-/** Available RISC-V Stage2 VMID bits */
+/** RISC-V Stage2 VMID bits */
extern unsigned long riscv_stage2_vmid_bits;
+/** RISC-V Stage2 VMID nested */
+extern unsigned long riscv_stage2_vmid_nested;
+
+/** RISC-V Stage2 VMID available for Guest */
+#define riscv_stage2_vmid_available() \
+ (CONFIG_MAX_GUEST_COUNT <= riscv_stage2_vmid_nested)
+
/** RISC-V Time Base Frequency */
extern unsigned long riscv_timer_hz;
diff --git a/arch/riscv/cpu/generic/include/cpu_vcpu_fp.h b/arch/riscv/cpu/generic/include/cpu_vcpu_fp.h
index 25b74b45..38e1ac06 100644
--- a/arch/riscv/cpu/generic/include/cpu_vcpu_fp.h
+++ b/arch/riscv/cpu/generic/include/cpu_vcpu_fp.h
@@ -21,8 +21,8 @@
* @brief header of VCPU FP functions
*/
-#ifndef _CPU_VCPU_HELPER_H__
-#define _CPU_VCPU_HELPER_H__
+#ifndef _CPU_VCPU_FP_H__
+#define _CPU_VCPU_FP_H__
#include <vmm_types.h>
#include <vmm_manager.h>
diff --git a/arch/riscv/cpu/generic/include/cpu_vcpu_helper.h b/arch/riscv/cpu/generic/include/cpu_vcpu_helper.h
index 694af455..fc587323 100644
--- a/arch/riscv/cpu/generic/include/cpu_vcpu_helper.h
+++ b/arch/riscv/cpu/generic/include/cpu_vcpu_helper.h
@@ -26,6 +26,15 @@
#include <vmm_types.h>
#include <vmm_manager.h>
+/** Function to update interrupt delegation */
+void cpu_vcpu_irq_deleg_update(struct vmm_vcpu *vcpu, bool nested_virt);
+
+/** Function to update time delta */
+void cpu_vcpu_time_delta_update(struct vmm_vcpu *vcpu, bool nested_virt);
+
+/** Function to update G-stage page table */
+void cpu_vcpu_gstage_update(struct vmm_vcpu *vcpu, bool nested_virt);
+
/** Function to dump general registers */
void cpu_vcpu_dump_general_regs(struct vmm_chardev *cdev,
arch_regs_t *regs);
--
2.25.1
Anup Patel
Mar 17, 2022, 11:24:06 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
This patch adds initial support for nested virtualizaiton with only
placeholder (or stub) functions for emulating hfences, hlv/hsv, and
shadow g-stage page table. These stub function will be filled-up by
subsequent patches.
arch/riscv/cpu/generic/cpu_exception.c | 38 +
arch/riscv/cpu/generic/cpu_vcpu_csr.c | 54 -
arch/riscv/cpu/generic/cpu_vcpu_fp.c | 56 +-
arch/riscv/cpu/generic/cpu_vcpu_helper.c | 27 +-
arch/riscv/cpu/generic/cpu_vcpu_nested.c | 587 ++++++++++
arch/riscv/cpu/generic/cpu_vcpu_sbi.c | 11 +
arch/riscv/cpu/generic/cpu_vcpu_trap.c | 998 ++++++++++++++++--
arch/riscv/cpu/generic/include/cpu_vcpu_csr.h | 37 -
.../cpu/generic/include/cpu_vcpu_nested.h | 105 ++
.../riscv/cpu/generic/include/cpu_vcpu_trap.h | 26 +-
arch/riscv/cpu/generic/objects.mk | 2 +-
11 files changed, 1758 insertions(+), 183 deletions(-)
index 76fded9d..b08ee243 100644
--- a/arch/riscv/cpu/generic/cpu_exception.c
+++ b/arch/riscv/cpu/generic/cpu_exception.c
@@ -58,6 +58,14 @@ void do_handle_irq(arch_regs_t *regs, unsigned long cause)
vmm_scheduler_irq_enter(regs, FALSE);
+ if (cause == IRQ_VS_SOFT ||
+ cause == IRQ_VS_TIMER ||
+ cause == IRQ_VS_EXT) {
+ rc = cpu_vcpu_redirect_vsirq(vmm_scheduler_current_vcpu(),
+ regs, cause);
+ goto done;
+ }
+
/* NOTE: Only exec <= 0xFFFFFFFFUL will be handled */
if (cause <= 0xFFFFFFFFUL) {
rc = vmm_host_active_irq_exec(cause);
@@ -65,10 +73,13 @@ void do_handle_irq(arch_regs_t *regs, unsigned long cause)
rc = VMM_EINVALID;
}
+done:
if (rc) {
do_error(vmm_scheduler_current_vcpu(), regs,
cause | SCAUSE_INTERRUPT_MASK,
"interrupt handling failed", rc, TRUE);
+ } else {
+ cpu_vcpu_take_vsirq(vmm_scheduler_current_vcpu(), regs);
}
vmm_scheduler_irq_exit(regs);
@@ -100,6 +111,31 @@ void do_handle_trap(arch_regs_t *regs, unsigned long cause)
}
switch (cause) {
+ case CAUSE_MISALIGNED_FETCH:
+ case CAUSE_FETCH_ACCESS:
+ case CAUSE_ILLEGAL_INSTRUCTION:
+ case CAUSE_BREAKPOINT:
+ case CAUSE_MISALIGNED_LOAD:
+ case CAUSE_LOAD_ACCESS:
+ case CAUSE_MISALIGNED_STORE:
+ case CAUSE_STORE_ACCESS:
+ case CAUSE_USER_ECALL:
+ case CAUSE_FETCH_PAGE_FAULT:
+ case CAUSE_LOAD_PAGE_FAULT:
+ case CAUSE_STORE_PAGE_FAULT:
+ msg = "general fault failed";
+ if (regs->hstatus & HSTATUS_SPV) {
+ trap.sepc = regs->sepc;
+ trap.scause = cause;
+ trap.stval = csr_read(CSR_STVAL);
+ trap.htval = csr_read(CSR_HTVAL);
+ trap.htinst = csr_read(CSR_HTINST);
+ rc = cpu_vcpu_general_fault(vcpu, regs, &trap);
+ panic = FALSE;
+ } else {
+ rc = VMM_EINVALID;
+ }
+ break;
case CAUSE_FETCH_GUEST_PAGE_FAULT:
case CAUSE_LOAD_GUEST_PAGE_FAULT:
case CAUSE_STORE_GUEST_PAGE_FAULT:
@@ -147,6 +183,8 @@ void do_handle_trap(arch_regs_t *regs, unsigned long cause)
done:
if (rc) {
do_error(vcpu, regs, cause, msg, rc, panic);
+ } else {
+ cpu_vcpu_take_vsirq(vcpu, regs);
}
vmm_scheduler_irq_exit(regs);
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_csr.c b/arch/riscv/cpu/generic/cpu_vcpu_csr.c
deleted file mode 100644
index 76856ca7..00000000
--- a/arch/riscv/cpu/generic/cpu_vcpu_csr.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Copyright (c) 2019 Anup Patel.
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- *
- * @file cpu_vcpu_csr.c
- * @author Anup Patel (an...@brainfault.org)
- * @brief source for VCPU CSR read/write handling
- */
-
-#include <vmm_error.h>
-#include <vmm_stdio.h>
-#include <cpu_vcpu_csr.h>
-
-int cpu_vcpu_csr_read(struct vmm_vcpu *vcpu,
- unsigned long csr_num,
- unsigned long *csr_val)
-{
- /*
- * We don't have any CSRs to emulate because runtime
- * M-mode firmware (i.e. OpenSBI) takes care of it
- */
- vmm_printf("%s: vcpu=%s invalid csr_num=0x%lx\n",
- __func__, (vcpu) ? vcpu->name : "(null)", csr_num);
-
- return VMM_ENOTSUPP;
-}
-
-int cpu_vcpu_csr_write(struct vmm_vcpu *vcpu,
- unsigned long csr_num,
- unsigned long csr_val)
-{
- /*
- * We don't have any CSRs to emulate because runtime
- * M-mode firmware (i.e. OpenSBI) takes care of it
- */
- vmm_printf("%s: vcpu=%s invalid csr_num=0x%lx\n",
- __func__, (vcpu) ? vcpu->name : "(null)", csr_num);
-
- return VMM_ENOTSUPP;
-}
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_fp.c b/arch/riscv/cpu/generic/cpu_vcpu_fp.c
index e4ccebaa..7a1f907a 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_fp.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_fp.c
@@ -45,31 +45,51 @@ static inline void cpu_vcpu_fp_clean(arch_regs_t *regs)
regs->sstatus |= SSTATUS_FS_CLEAN;
}
-void cpu_vcpu_fp_save(struct vmm_vcpu *vcpu, arch_regs_t *regs)
+static inline void cpu_vcpu_fp_force_save(struct vmm_vcpu *vcpu)
+{
+ unsigned long *isa = riscv_priv(vcpu)->isa;
+
+ if (riscv_isa_extension_available(isa, d))
+ __cpu_vcpu_fp_d_save(&riscv_priv(vcpu)->fp.d);
+ else if (riscv_isa_extension_available(isa, f))
+ __cpu_vcpu_fp_f_save(&riscv_priv(vcpu)->fp.f);
+}
+
+static inline void cpu_vcpu_fp_force_restore(struct vmm_vcpu *vcpu)
{
- unsigned long *isa;
+ unsigned long *isa = riscv_priv(vcpu)->isa;
- if ((regs->sstatus & SSTATUS_FS) == SSTATUS_FS_DIRTY) {
- isa = riscv_priv(vcpu)->isa;
- if (riscv_isa_extension_available(isa, d))
- __cpu_vcpu_fp_d_save(&riscv_priv(vcpu)->fp.d);
- else if (riscv_isa_extension_available(isa, f))
- __cpu_vcpu_fp_f_save(&riscv_priv(vcpu)->fp.f);
- cpu_vcpu_fp_clean(regs);
+ if (riscv_isa_extension_available(isa, d))
+ __cpu_vcpu_fp_d_restore(&riscv_priv(vcpu)->fp.d);
+ else if (riscv_isa_extension_available(isa, f))
+ __cpu_vcpu_fp_f_restore(&riscv_priv(vcpu)->fp.f);
+}
+
+void cpu_vcpu_fp_save(struct vmm_vcpu *vcpu, arch_regs_t *regs)
+{
+ if (riscv_nested_virt(vcpu)) {
+ /* Always save FP state when nested virtualization is ON */
+ cpu_vcpu_fp_force_save(vcpu);
+ } else {
+ /* Lazy save FP state when nested virtualization is OFF */
+ if ((regs->sstatus & SSTATUS_FS) == SSTATUS_FS_DIRTY) {
+ cpu_vcpu_fp_force_save(vcpu);
+ cpu_vcpu_fp_clean(regs);
+ }
}
}
void cpu_vcpu_fp_restore(struct vmm_vcpu *vcpu, arch_regs_t *regs)
{
- unsigned long *isa;
-
- if ((regs->sstatus & SSTATUS_FS) != SSTATUS_FS_OFF) {
- isa = riscv_priv(vcpu)->isa;
- if (riscv_isa_extension_available(isa, d))
- __cpu_vcpu_fp_d_restore(&riscv_priv(vcpu)->fp.d);
- else if (riscv_isa_extension_available(isa, f))
- __cpu_vcpu_fp_f_restore(&riscv_priv(vcpu)->fp.f);
- cpu_vcpu_fp_clean(regs);
+ if (riscv_nested_virt(vcpu)) {
+ /* Always restore FP state when nested virtualization is ON */
+ cpu_vcpu_fp_force_restore(vcpu);
+ } else {
+ /* Lazy restore FP state when nested virtualization is OFF */
+ if ((regs->sstatus & SSTATUS_FS) != SSTATUS_FS_OFF) {
+ cpu_vcpu_fp_force_restore(vcpu);
+ cpu_vcpu_fp_clean(regs);
+ }
}
}
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_helper.c b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
index 4ff619a7..e33bce24 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_helper.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
@@ -39,6 +39,8 @@
#include <cpu_tlb.h>
#include <cpu_sbi.h>
#include <cpu_vcpu_fp.h>
+#include <cpu_vcpu_trap.h>
+#include <cpu_vcpu_nested.h>
#include <cpu_vcpu_helper.h>
#include <cpu_vcpu_timer.h>
#include <cpu_guest_serial.h>
@@ -52,8 +54,7 @@
riscv_isa_extension_mask(f) | \
riscv_isa_extension_mask(i) | \
riscv_isa_extension_mask(m) | \
- riscv_isa_extension_mask(s) | \
- riscv_isa_extension_mask(u))
+ riscv_isa_extension_mask(h))
static char *guest_fdt_find_serial_node(char *guest_name)
{
@@ -277,6 +278,18 @@ int arch_vcpu_init(struct vmm_vcpu *vcpu)
+ /* H-extension only available when AIA CSRs are available */
+ if (!riscv_aia_available) {
+ riscv_priv(vcpu)->isa[0] &=
+ ~riscv_isa_extension_mask(h);
+ }
+
+ /* Initialize nested state */
+ rc = cpu_vcpu_nested_init(vcpu);
+ if (rc) {
+ goto fail_free_isa;
+ }
}
/* Set a0 to VCPU sub-id (i.e. virtual HARTID) */
@@ -306,6 +319,9 @@ int arch_vcpu_init(struct vmm_vcpu *vcpu)
/* By default, make CY, TM, and IR counters accessible in VU mode */
riscv_priv(vcpu)->scounteren = 7;
+ /* Reset nested state */
+ cpu_vcpu_nested_reset(vcpu);
+
/* Initialize FP state */
cpu_vcpu_fp_init(vcpu);
@@ -345,10 +361,14 @@ int arch_vcpu_deinit(struct vmm_vcpu *vcpu)
return VMM_OK;
}
+ /* Cleanup timer */
rc = riscv_timer_event_deinit(vcpu, &riscv_timer_priv(vcpu));
if (rc)
return rc;
+ /* Cleanup nested state */
+ cpu_vcpu_nested_deinit(vcpu);
+
/* Free ISA bitmap */
vmm_free(riscv_priv(vcpu)->isa);
riscv_priv(vcpu)->isa = NULL;
@@ -403,6 +423,7 @@ void arch_vcpu_switch(struct vmm_vcpu *tvcpu,
cpu_vcpu_time_delta_update(vcpu, riscv_nested_virt(vcpu));
cpu_vcpu_gstage_update(vcpu, riscv_nested_virt(vcpu));
cpu_vcpu_irq_deleg_update(vcpu, riscv_nested_virt(vcpu));
+ cpu_vcpu_take_vsirq(vcpu, regs);
}
}
@@ -568,6 +589,8 @@ void cpu_vcpu_dump_private_regs(struct vmm_chardev *cdev,
+
cpu_vcpu_fp_dump_regs(cdev, vcpu);
}
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_nested.c b/arch/riscv/cpu/generic/cpu_vcpu_nested.c
new file mode 100644
index 00000000..5f8774de
--- /dev/null
+++ b/arch/riscv/cpu/generic/cpu_vcpu_nested.c
@@ -0,0 +1,587 @@
+#include <generic_mmu.h>
+
+#include <cpu_hwcap.h>
+#include <cpu_vcpu_helper.h>
+#include <cpu_vcpu_nested.h>
+#include <cpu_vcpu_trap.h>
+#include <riscv_csr.h>
+
+int cpu_vcpu_nested_init(struct vmm_vcpu *vcpu)
+{
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ npriv->pgtbl = mmu_pgtbl_alloc(MMU_STAGE2, -1);
+ if (!npriv->pgtbl) {
+ return VMM_ENOMEM;
+ }
+{
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ npriv->virt = FALSE;
+#ifdef CONFIG_64BIT
+ npriv->hstatus = HSTATUS_VSXL_RV64 << HSTATUS_VSXL_SHIFT;
+#else
+ npriv->hstatus = 0;
+#endif
+ npriv->hedeleg = 0;
+ npriv->hideleg = 0;
+ npriv->hvip = 0;
+ npriv->hcounteren = 0;
+ npriv->htimedelta = 0;
+ npriv->htimedeltah = 0;
+ npriv->htval = 0;
+ npriv->htinst = 0;
+ npriv->hgatp = 0;
+ npriv->vsstatus = 0;
+ npriv->vsie = 0;
+ npriv->vstvec = 0;
+ npriv->vsscratch = 0;
+ npriv->vsepc = 0;
+ npriv->vscause = 0;
+ npriv->vstval = 0;
+ npriv->vsatp = 0;
+
+ npriv->hvictl = 0;
+}
+
+void cpu_vcpu_nested_deinit(struct vmm_vcpu *vcpu)
+{
+ mmu_pgtbl_free(riscv_nested_priv(vcpu)->pgtbl);
+}
+
+void cpu_vcpu_nested_dump_regs(struct vmm_chardev *cdev,
+ struct vmm_vcpu *vcpu)
+{
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ if (!riscv_isa_extension_available(riscv_priv(vcpu)->isa, h))
+ return;
+
+ vmm_cprintf(cdev, "\n");
+ vmm_cprintf(cdev, "\n");
+#ifdef CONFIG_64BIT
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR"\n",
+ " htimedelta", npriv->htimedelta);
+#else
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " htimedelta", npriv->htimedelta,
+ "htimedeltah", npriv->htimedeltah);
+#endif
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " hstatus", npriv->hstatus, " hgatp", npriv->hgatp);
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " hedeleg", npriv->hedeleg, " hideleg", npriv->hideleg);
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " hvip", npriv->hvip, " hcounteren", npriv->hcounteren);
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " htval", npriv->htval, " htinst", npriv->htinst);
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " vsstatus", npriv->vsstatus, " vsie", npriv->vsie);
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " vsatp", npriv->vsatp, " vstvec", npriv->vstvec);
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " vsscratch", npriv->vsscratch, " vsepc", npriv->vsepc);
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " vscause", npriv->vscause, " vstval", npriv->vstval);
+
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR"\n",
+ " hvictl", npriv->hvictl);
+}
+
+int cpu_vcpu_nested_smode_csr_rmw(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ unsigned int csr_num, unsigned long *val,
+ unsigned long new_val, unsigned long wr_mask)
+{
+ int csr_shift = 0;
+ bool read_only = FALSE;
+ unsigned long *csr, zero = 0, writeable_mask = 0;
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ /*
+ * These CSRs should never trap for virtual-HS/U modes because
+ * we only emulate these CSRs for virtual-VS/VU modes.
+ */
+ if (!riscv_nested_virt(vcpu)) {
+ return VMM_EINVALID;
+ }
+
+ /*
+ * Access of these CSRs from virtual-VU mode should be forwarded
+ * as illegal instruction trap to virtual-HS mode.
+ */
+ if (!(regs->hstatus & HSTATUS_SPVP)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ switch (csr_num) {
+ case CSR_SIE:
+ if (npriv->hvictl & HVICTL_VTI) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+ csr = &npriv->vsie;
+ writeable_mask = VSIE_WRITEABLE & (npriv->hideleg >> 1);
+ break;
+ case CSR_SIEH:
+ if (npriv->hvictl & HVICTL_VTI) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+ csr = &zero;
+ break;
+ case CSR_SIP:
+ if (npriv->hvictl & HVICTL_VTI) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+ csr = &npriv->hvip;
+ csr_shift = 1;
+ writeable_mask = HVIP_VSSIP & npriv->hideleg;
+ break;
+ case CSR_SIPH:
+ if (npriv->hvictl & HVICTL_VTI) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+ csr = &zero;
+ break;
+ default:
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ if (val) {
+ *val = (csr_shift < 0) ?
+ (*csr) << -csr_shift : (*csr) >> csr_shift;
+ }
+
+ if (read_only) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ } else {
+ writeable_mask = (csr_shift < 0) ?
+ writeable_mask >> -csr_shift :
+ writeable_mask << csr_shift;
+ wr_mask = (csr_shift < 0) ?
+ wr_mask >> -csr_shift : wr_mask << csr_shift;
+ new_val = (csr_shift < 0) ?
+ new_val >> -csr_shift : new_val << csr_shift;
+ wr_mask &= writeable_mask;
+ *csr = (*csr & ~wr_mask) | (new_val & wr_mask);
+ }
+ unsigned int csr_num, unsigned long *val,
+ unsigned long new_val, unsigned long wr_mask)
+{
+ int csr_shift = 0;
+ bool read_only = FALSE;
+ unsigned int csr_priv = (csr_num >> 8) & 0x3;
+ unsigned long *csr, mode, zero = 0, writeable_mask = 0;
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded
+ * to virtual-HS mode as a virtual instruction trap.
+ */
+ if (riscv_nested_virt(vcpu)) {
+ return (csr_priv == (PRV_S + 1)) ?
+ TRAP_RETURN_VIRTUAL_INSN : TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ /*
+ * If H-extension is not available for VCPU then forward trap
+ * as illegal instruction trap to virtual-HS mode.
+ */
+ if (!riscv_isa_extension_available(riscv_priv(vcpu)->isa, h)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ /*
+ * H-extension CSRs not allowed in virtual-U mode so forward trap
+ * as illegal instruction trap to virtual-HS mode.
+ */
+ if (!(regs->hstatus & HSTATUS_SPVP)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ switch (csr_num) {
+ case CSR_HSTATUS:
+ csr = &npriv->hstatus;
+ writeable_mask = HSTATUS_VTSR | HSTATUS_VTW | HSTATUS_VTVM |
+ HSTATUS_HU | HSTATUS_SPVP | HSTATUS_SPV |
+ HSTATUS_GVA;
+ if (wr_mask & HSTATUS_SPV) {
+ /*
+ * Enable (or Disable) host SRET trapping for
+ * virtual-HS mode. This will be auto-disabled
+ * by cpu_vcpu_nested_set_virt() upon SRET trap
+ * from virtual-HS mode.
+ */
+ regs->hstatus &= ~HSTATUS_VTSR;
+ regs->hstatus |= (new_val & HSTATUS_SPV) ?
+ HSTATUS_VTSR : 0;
+ }
+ break;
+ case CSR_HEDELEG:
+ csr = &npriv->hedeleg;
+ writeable_mask = HEDELEG_WRITEABLE;
+ break;
+ case CSR_HIDELEG:
+ csr = &npriv->hideleg;
+ writeable_mask = HIDELEG_WRITEABLE;
+ break;
+ case CSR_HVIP:
+ csr = &npriv->hvip;
+ writeable_mask = HVIP_WRITEABLE;
+ break;
+ case CSR_HIE:
+ csr = &npriv->vsie;
+ csr_shift = -1;
+ writeable_mask = HVIP_WRITEABLE;
+ break;
+ case CSR_HIP:
+ csr = &npriv->hvip;
+ writeable_mask = HVIP_VSSIP;
+ break;
+ case CSR_HGEIP:
+ csr = &zero;
+ read_only = TRUE;
+ break;
+ case CSR_HGEIE:
+ csr = &zero;
+ break;
+ case CSR_HCOUNTEREN:
+ csr = &npriv->hcounteren;
+ writeable_mask = HCOUNTEREN_WRITEABLE;
+ break;
+ case CSR_HTIMEDELTA:
+ csr = &npriv->htimedelta;
+ writeable_mask = -1UL;
+ break;
+#ifndef CONFIG_64BIT
+ case CSR_HTIMEDELTAH:
+ csr = &npriv->htimedeltah;
+ writeable_mask = -1UL;
+ break;
+#endif
+ case CSR_HTVAL:
+ csr = &npriv->htval;
+ writeable_mask = -1UL;
+ break;
+ case CSR_HTINST:
+ csr = &npriv->htinst;
+ writeable_mask = -1UL;
+ break;
+ case CSR_HGATP:
+ csr = &npriv->hgatp;
+ writeable_mask = HGATP_MODE | HGATP_VMID | HGATP_PPN;
+ if (wr_mask & HGATP_MODE) {
+ mode = (new_val & HGATP_MODE) >> HGATP_MODE_SHIFT;
+ switch (mode) {
+ /*
+ * We (intentionally) support only Sv39x4 on RV64
+ * and Sv32x4 on RV32 for guest G-stage so that
+ * software page table walks on guest G-stage is
+ * faster.
+ */
+#ifdef CONFIG_64BIT
+ case HGATP_MODE_SV39X4:
+ if (riscv_stage2_mode != HGATP_MODE_SV48X4 &&
+ riscv_stage2_mode != HGATP_MODE_SV39X4) {
+ mode = HGATP_MODE_OFF;
+ }
+ break;
+#else
+ case HGATP_MODE_SV32X4:
+ if (riscv_stage2_mode != HGATP_MODE_SV32X4) {
+ mode = HGATP_MODE_OFF;
+ }
+ break;
+#endif
+ default:
+ mode = HGATP_MODE_OFF;
+ break;
+ }
+ new_val &= ~HGATP_MODE;
+ new_val |= (mode << HGATP_MODE_SHIFT) & HGATP_MODE;
+ }
+ break;
+ case CSR_VSSTATUS:
+ csr = &npriv->vsstatus;
+ writeable_mask = SSTATUS_SIE | SSTATUS_SPIE | SSTATUS_UBE |
+ SSTATUS_SPP | SSTATUS_SUM | SSTATUS_MXR |
+ SSTATUS_FS | SSTATUS_UXL;
+ break;
+ case CSR_VSIP:
+ csr = &npriv->hvip;
+ csr_shift = 1;
+ writeable_mask = HVIP_VSSIP & npriv->hideleg;
+ break;
+ case CSR_VSIE:
+ csr = &npriv->vsie;
+ writeable_mask = VSIE_WRITEABLE & (npriv->hideleg >> 1);
+ break;
+ case CSR_VSTVEC:
+ csr = &npriv->vstvec;
+ writeable_mask = -1UL;
+ break;
+ case CSR_VSSCRATCH:
+ csr = &npriv->vsscratch;
+ writeable_mask = -1UL;
+ break;
+ case CSR_VSEPC:
+ csr = &npriv->vsepc;
+ writeable_mask = -1UL;
+ break;
+ case CSR_VSCAUSE:
+ csr = &npriv->vscause;
+ writeable_mask = 0x1fUL;
+ break;
+ case CSR_VSTVAL:
+ csr = &npriv->vstval;
+ writeable_mask = -1UL;
+ break;
+ case CSR_VSATP:
+ csr = &npriv->vsatp;
+ writeable_mask = SATP_MODE | SATP_ASID | SATP_PPN;
+ if (wr_mask & SATP_MODE) {
+ mode = (new_val & SATP_MODE) >> SATP_MODE_SHIFT;
+ switch (mode) {
+#ifdef CONFIG_64BIT
+ case SATP_MODE_SV48:
+ if (riscv_stage1_mode != SATP_MODE_SV48) {
+ mode = SATP_MODE_OFF;
+ }
+ break;
+ case SATP_MODE_SV39:
+ if (riscv_stage1_mode != SATP_MODE_SV48 &&
+ riscv_stage1_mode != SATP_MODE_SV39) {
+ mode = SATP_MODE_OFF;
+ }
+ break;
+#else
+ case SATP_MODE_SV32:
+ if (riscv_stage1_mode != SATP_MODE_SV32) {
+ mode = SATP_MODE_OFF;
+ }
+ break;
+#endif
+ default:
+ mode = SATP_MODE_OFF;
+ break;
+ }
+ new_val &= ~SATP_MODE;
+ new_val |= (mode << SATP_MODE_SHIFT) & SATP_MODE;
+ }
+ break;
+ case CSR_HVICTL:
+ csr = &npriv->hvictl;
+ writeable_mask = HVICTL_VTI | HVICTL_IID |
+ HVICTL_IPRIOM | HVICTL_IPRIO;
+ break;
+ default:
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ if (val) {
+ *val = (csr_shift < 0) ?
+ (*csr) << -csr_shift : (*csr) >> csr_shift;
+ }
+
+ if (read_only) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ } else {
+ writeable_mask = (csr_shift < 0) ?
+ writeable_mask >> -csr_shift :
+ writeable_mask << csr_shift;
+ wr_mask = (csr_shift < 0) ?
+ wr_mask >> -csr_shift : wr_mask << csr_shift;
+ new_val = (csr_shift < 0) ?
+ new_val >> -csr_shift : new_val << csr_shift;
+ wr_mask &= writeable_mask;
+ *csr = (*csr & ~wr_mask) | (new_val & wr_mask);
+ }
+ bool trap_from_smode,
+ const struct cpu_vcpu_trap *trap,
+ return VMM_OK;
+}
+
+void cpu_vcpu_nested_hfence_vvma(struct vmm_vcpu *vcpu,
+ unsigned long *vaddr, unsigned int *asid)
+{
+ /* TODO: */
+}
+
+void cpu_vcpu_nested_hfence_gvma(struct vmm_vcpu *vcpu,
+ physical_addr_t *gaddr, unsigned int *vmid)
+{
+ /* TODO: */
+}
+
+int cpu_vcpu_nested_hlv(struct vmm_vcpu *vcpu, unsigned long vaddr,
+ bool hlvx, void *data, unsigned long len,
+ unsigned long *out_scause,
+ unsigned long *out_stval,
+ unsigned long *out_htval)
+{
+ /* TODO: */
+ return VMM_OK;
+}
+
+int cpu_vcpu_nested_hsv(struct vmm_vcpu *vcpu, unsigned long vaddr,
+ const void *data, unsigned long len,
+ unsigned long *out_scause,
+ unsigned long *out_stval,
+ unsigned long *out_htval)
+{
+ /* TODO: */
+ return VMM_OK;
+}
+
+void cpu_vcpu_nested_set_virt(struct vmm_vcpu *vcpu, struct arch_regs *regs,
+ enum nested_set_virt_event event, bool virt,
+ bool spvp, bool gva)
+{
+ unsigned long tmp;
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ /* If H-extension is not available for VCPU then do nothing */
+ if (!riscv_isa_extension_available(riscv_priv(vcpu)->isa, h)) {
+ return;
+ }
+
+ /* Skip hardware CSR update if no change in virt state */
+ if (virt == npriv->virt)
+ goto skip_csr_update;
+
+ /* Swap hcounteren and hedeleg CSRs */
+ npriv->hcounteren = csr_swap(CSR_HCOUNTEREN, npriv->hcounteren);
+ npriv->hedeleg = csr_swap(CSR_HEDELEG, npriv->hedeleg);
+
+ /* Update interrupt delegation */
+ cpu_vcpu_irq_deleg_update(vcpu, virt);
+
+ /* Update time delta */
+ cpu_vcpu_time_delta_update(vcpu, virt);
+
+ /* Update G-stage page table */
+ cpu_vcpu_gstage_update(vcpu, virt);
+
+ /* Swap hardware vs<xyz> CSRs except vsie and vsstatus */
+ npriv->vstvec = csr_swap(CSR_VSTVEC, npriv->vstvec);
+ npriv->vsscratch = csr_swap(CSR_VSSCRATCH, npriv->vsscratch);
+ npriv->vsepc = csr_swap(CSR_VSEPC, npriv->vsepc);
+ npriv->vscause = csr_swap(CSR_VSCAUSE, npriv->vscause);
+ npriv->vstval = csr_swap(CSR_VSTVAL, npriv->vstval);
+ npriv->vsatp = csr_swap(CSR_VSATP, npriv->vsatp);
+
+ /* Update vsstatus CSR */
+ if (virt) {
+ /* Nested virtualization state changing from OFF to ON */
+
+ /*
+ * Update vsstatus in following manner:
+ * 1) Swap hardware vsstatus (i.e. virtual-HS mode sstatus)
+ * with vsstatus in nested virtualization context (i.e.
+ * virtual-VS mode sstatus)
+ * 2) Swap host sstatus.FS (i.e. HS mode sstatus.FS) with
+ * the vsstatus.FS saved in nested virtualization context
+ * (i.e. virtual-HS mode sstatus.FS)
+ */
+ npriv->vsstatus = csr_swap(CSR_VSSTATUS, npriv->vsstatus);
+ tmp = regs->sstatus & SSTATUS_FS;
+ regs->sstatus &= ~SSTATUS_FS;
+ regs->sstatus |= (npriv->vsstatus & SSTATUS_FS);
+ npriv->vsstatus &= ~SSTATUS_FS;
+ npriv->vsstatus |= tmp;
+ } else {
+ /* Nested virtualization state changing from ON to OFF */
+
+ /*
+ * Update vsstatus in following manner:
+ * 1) Swap host sstatus.FS (i.e. virtual-HS mode sstatus.FS)
+ * with vsstatus.FS saved in the nested virtualization
+ * context (i.e. HS mode sstatus.FS)
+ * 2) Swap hardware vsstatus (i.e. virtual-VS mode sstatus)
+ * with vsstatus in nested virtualization context (i.e.
+ * virtual-HS mode sstatus)
+ */
+ tmp = regs->sstatus & SSTATUS_FS;
+ regs->sstatus &= ~SSTATUS_FS;
+ regs->sstatus |= (npriv->vsstatus & SSTATUS_FS);
+ npriv->vsstatus &= ~SSTATUS_FS;
+ npriv->vsstatus |= tmp;
+ npriv->vsstatus = csr_swap(CSR_VSSTATUS, npriv->vsstatus);
+ }
+
+skip_csr_update:
+ if (event != NESTED_SET_VIRT_EVENT_SRET) {
+ /* Update Guest hstatus.SPV bit */
+ npriv->hstatus &= ~HSTATUS_SPV;
+ npriv->hstatus |= (npriv->virt) ? HSTATUS_SPV : 0;
+
+ /* Update Guest hstatus.SPVP bit */
+ if (npriv->virt) {
+ npriv->hstatus &= ~HSTATUS_SPVP;
+ if (spvp)
+ npriv->hstatus |= HSTATUS_SPVP;
+ }
+
+ /* Update Guest hstatus.GVA bit */
+ if (event == NESTED_SET_VIRT_EVENT_TRAP) {
+ npriv->hstatus &= ~HSTATUS_GVA;
+ npriv->hstatus |= (gva) ? HSTATUS_GVA : 0;
+ }
+ }
+
+ /* Update host SRET and VM trapping */
+ regs->hstatus &= ~HSTATUS_VTSR;
+ if (virt && (npriv->hstatus & HSTATUS_VTSR)) {
+ regs->hstatus |= HSTATUS_VTSR;
+ }
+ regs->hstatus &= ~HSTATUS_VTVM;
+ if (virt && (npriv->hstatus & HSTATUS_VTVM)) {
+ regs->hstatus |= HSTATUS_VTVM;
+ }
+
+ /* Update virt flag */
+ npriv->virt = virt;
+}
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_sbi.c b/arch/riscv/cpu/generic/cpu_vcpu_sbi.c
index fbd6a8f6..9788150e 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_sbi.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_sbi.c
@@ -74,6 +74,17 @@ int cpu_vcpu_sbi_ecall(struct vmm_vcpu *vcpu, ulong cause,
bool is_0_1_spec = FALSE;
unsigned long args[6];
+ /* Forward SBI calls from virtual-VS mode to virtual-HS mode */
+ if (riscv_nested_virt(vcpu)) {
+ trap.sepc = regs->sepc;
+ trap.scause = CAUSE_VIRTUAL_SUPERVISOR_ECALL;
+ trap.stval = 0;
+ trap.htval = 0;
+ trap.htinst = 0;
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ return VMM_OK;
+ }
+
args[0] = regs->a0;
args[1] = regs->a1;
args[2] = regs->a2;
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_trap.c b/arch/riscv/cpu/generic/cpu_vcpu_trap.c
index 953e6e4b..e990ed4a 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_trap.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_trap.c
@@ -30,46 +30,86 @@
#include <libs/stringlib.h>
#include <generic_mmu.h>
-#include <cpu_vcpu_csr.h>
+#include <cpu_hwcap.h>
+#include <cpu_vcpu_nested.h>
#include <cpu_vcpu_trap.h>
#include <cpu_vcpu_unpriv.h>
-int cpu_vcpu_redirect_trap(struct vmm_vcpu *vcpu,
- arch_regs_t *regs,
- struct cpu_vcpu_trap *trap)
+void cpu_vcpu_redirect_smode_trap(arch_regs_t *regs,
+ struct cpu_vcpu_trap *trap, bool prev_spp)
{
+ /* Read Guest sstatus */
unsigned long vsstatus = csr_read(CSR_VSSTATUS);
- /* Change Guest SSTATUS.SPP bit */
+ /* Change Guest sstatus.SPP bit */
vsstatus &= ~SSTATUS_SPP;
- if (regs->sstatus & SSTATUS_SPP)
+ if (prev_spp)
vsstatus |= SSTATUS_SPP;
- /* Change Guest SSTATUS.SPIE bit */
+ /* Change Guest sstatus.SPIE bit */
vsstatus &= ~SSTATUS_SPIE;
if (vsstatus & SSTATUS_SIE)
vsstatus |= SSTATUS_SPIE;
- /* Clear Guest SSTATUS.SIE bit */
+ /* Clear Guest sstatus.SIE bit */
vsstatus &= ~SSTATUS_SIE;
- /* Update Guest SSTATUS */
+ /* Update Guest sstatus */
csr_write(CSR_VSSTATUS, vsstatus);
- /* Update Guest SCAUSE, STVAL, and SEPC */
+ /* Update Guest scause, stval, and sepc */
csr_write(CSR_VSCAUSE, trap->scause);
csr_write(CSR_VSTVAL, trap->stval);
csr_write(CSR_VSEPC, trap->sepc);
- /* Set Guest PC to Guest exception vector */
+ /* Set next PC to exception vector */
regs->sepc = csr_read(CSR_VSTVEC);
- return 0;
+ /* Set next privilege mode to supervisor */
+ regs->sstatus |= SSTATUS_SPP;
+}
+
+void cpu_vcpu_redirect_trap(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ struct cpu_vcpu_trap *trap)
+{
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+ bool prev_spp = (regs->sstatus & SSTATUS_SPP) ? TRUE : FALSE;
+ bool gva = FALSE;
+
+ /* Determine GVA bit state */
+ switch (trap->scause) {
+ case CAUSE_MISALIGNED_FETCH:
+ case CAUSE_FETCH_ACCESS:
+ case CAUSE_MISALIGNED_LOAD:
+ case CAUSE_LOAD_ACCESS:
+ case CAUSE_MISALIGNED_STORE:
+ case CAUSE_STORE_ACCESS:
+ case CAUSE_FETCH_PAGE_FAULT:
+ case CAUSE_LOAD_PAGE_FAULT:
+ case CAUSE_STORE_PAGE_FAULT:
+ case CAUSE_FETCH_GUEST_PAGE_FAULT:
+ case CAUSE_LOAD_GUEST_PAGE_FAULT:
+ case CAUSE_STORE_GUEST_PAGE_FAULT:
+ gva = TRUE;
+ break;
+ default:
+ break;
+ }
+
+ /* Turn-off nested virtualization for virtual-HS mode */
+ cpu_vcpu_nested_set_virt(vcpu, regs, NESTED_SET_VIRT_EVENT_TRAP,
+ FALSE, prev_spp, gva);
+
+ /* Update Guest HTVAL and HTINST */
+ npriv->htval = trap->htval;
+ npriv->htinst = trap->htinst;
+
+ /* Update Guest supervisor state */
+ cpu_vcpu_redirect_smode_trap(regs, trap, prev_spp);
}
static int cpu_vcpu_stage2_map(struct vmm_vcpu *vcpu,
- arch_regs_t *regs,
- physical_addr_t fault_addr)
+ physical_addr_t fault_addr)
{
int rc, rc1;
u32 reg_flags = 0x0, pg_reg_flags = 0x0;
@@ -160,9 +200,9 @@ static int cpu_vcpu_emulate_load(struct vmm_vcpu *vcpu,
u16 data16;
u32 data32;
u64 data64;
+ struct cpu_vcpu_trap trap;
unsigned long insn, insn_len;
int rc = VMM_OK, shift = 0, len = 0;
- struct cpu_vcpu_trap trap = { 0 };
if (htinst & 0x1) {
/*
@@ -176,12 +216,18 @@ static int cpu_vcpu_emulate_load(struct vmm_vcpu *vcpu,
* Bit[0] == 0 implies trapped instruction value is
* zero or special value.
*/
+ trap.sepc = 0;
+ trap.scause = 0;
+ trap.stval = 0;
+ trap.htval = 0;
+ trap.htinst = 0;
insn = __cpu_vcpu_unpriv_read_insn(regs->sepc, &trap);
if (trap.scause) {
if (trap.scause == CAUSE_LOAD_PAGE_FAULT)
trap.scause = CAUSE_FETCH_PAGE_FAULT;
trap.sepc = trap.stval = regs->sepc;
- return cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ return VMM_OK;
}
insn_len = INSN_LEN(insn);
}
@@ -287,8 +333,8 @@ static int cpu_vcpu_emulate_store(struct vmm_vcpu *vcpu,
u32 data32;
u64 data64;
int rc = VMM_OK, len = 0;
+ struct cpu_vcpu_trap trap;
unsigned long data, insn, insn_len;
- struct cpu_vcpu_trap trap = { 0 };
if (htinst & 0x1) {
/*
@@ -298,16 +344,22 @@ static int cpu_vcpu_emulate_store(struct vmm_vcpu *vcpu,
insn = htinst | INSN_16BIT_MASK;
insn_len = (htinst & 0x2) ? INSN_LEN(insn) : 2;
} else {
- /*
+ /*
* Bit[0] == 0 implies trapped instruction value is
* zero or special value.
*/
+ trap.sepc = 0;
+ trap.scause = 0;
+ trap.stval = 0;
+ trap.htval = 0;
+ trap.htinst = 0;
insn = __cpu_vcpu_unpriv_read_insn(regs->sepc, &trap);
if (trap.scause) {
if (trap.scause == CAUSE_LOAD_PAGE_FAULT)
trap.scause = CAUSE_FETCH_PAGE_FAULT;
trap.sepc = trap.stval = regs->sepc;
- return cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ return VMM_OK;
}
insn_len = INSN_LEN(insn);
}
@@ -384,9 +436,31 @@ int cpu_vcpu_page_fault(struct vmm_vcpu *vcpu,
arch_regs_t *regs,
struct cpu_vcpu_trap *trap)
{
+ int rc;
struct vmm_region *reg;
+ struct cpu_vcpu_trap otrap;
physical_addr_t fault_addr;
+ if (riscv_nested_virt(vcpu)) {
+ otrap.sepc = 0;
+ otrap.scause = 0;
+ otrap.stval = 0;
+ otrap.htval = 0;
+ otrap.htinst = 0;
+ rc = cpu_vcpu_nested_page_fault(vcpu,
+ (regs->hstatus & HSTATUS_SPVP) ? TRUE : FALSE,
+ trap, &otrap);
+ if (rc) {
+ return rc;
+ }
+
+ if (otrap.scause) {
+ cpu_vcpu_redirect_trap(vcpu, regs, &otrap);
+ }
fault_addr |= ((physical_addr_t)trap->stval & 0x3);
@@ -397,18 +471,18 @@ int cpu_vcpu_page_fault(struct vmm_vcpu *vcpu,
/* Emulate load/store instructions for virtual device */
switch (trap->scause) {
case CAUSE_LOAD_GUEST_PAGE_FAULT:
- return cpu_vcpu_emulate_load(vcpu, regs,
- fault_addr, trap->htinst);
+ return cpu_vcpu_emulate_load(vcpu, regs, fault_addr,
+ trap->htinst);
case CAUSE_STORE_GUEST_PAGE_FAULT:
- return cpu_vcpu_emulate_store(vcpu, regs,
- fault_addr, trap->htinst);
+ return cpu_vcpu_emulate_store(vcpu, regs, fault_addr,
+ trap->htinst);
default:
return VMM_ENOTSUPP;
};
}
/* Mapping does not exist hence create one */
- return cpu_vcpu_stage2_map(vcpu, regs, fault_addr);
+ return cpu_vcpu_stage2_map(vcpu, fault_addr);
}
static int truly_illegal_insn(struct vmm_vcpu *vcpu,
@@ -421,75 +495,788 @@ static int truly_illegal_insn(struct vmm_vcpu *vcpu,
trap.sepc = regs->sepc;
trap.scause = CAUSE_ILLEGAL_INSTRUCTION;
trap.stval = insn;
- return cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ trap.htval = 0;
+ trap.htinst = 0;
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+
+ return VMM_OK;
}
-static int system_opcode_insn(struct vmm_vcpu *vcpu,
+static int truly_virtual_insn(struct vmm_vcpu *vcpu,
arch_regs_t *regs,
ulong insn)
{
- int rc = VMM_OK, do_write, rs1_num;
- ulong rs1_val, csr_num, csr_val, new_csr_val;
+ struct cpu_vcpu_trap trap;
- if ((insn & INSN_MASK_WFI) == INSN_MATCH_WFI) {
- /* Wait for irq with default timeout */
- vmm_vcpu_irq_wait_timeout(vcpu, 0);
- goto done;
- }
+ /* Redirect trap to Guest VCPU */
+ trap.sepc = regs->sepc;
+ trap.scause = CAUSE_VIRTUAL_INST_FAULT;
+ trap.stval = insn;
+ trap.htval = 0;
+ trap.htinst = 0;
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
- rs1_num = (insn >> 15) & 0x1f;
- rs1_val = GET_RS1(insn, regs);
- csr_num = insn >> 20;
+ return VMM_OK;
+}
- rc = cpu_vcpu_csr_read(vcpu, csr_num, &csr_val);
- if (rc == VMM_EINVALID) {
- return truly_illegal_insn(vcpu, regs, insn);
- }
- if (rc) {
- return rc;
- }
+struct system_opcode_func {
+ ulong mask;
+ ulong match;
+ /*
+ * Possible return values are as follows:
+ * 1) Returns < 0 for error case
+ * 2) Return == 0 to increment PC and continue
+ * 3) Return == 1 to inject illegal instruction trap and continue
+ * 4) Return == 2 to inject virtual instruction trap and continue
+ * 5) Return == 3 to do nothing and continue
+ */
+ int (*func)(struct vmm_vcpu *vcpu, arch_regs_t *regs, ulong insn);
+};
- do_write = rs1_num;
+struct csr_func {
+ unsigned int csr_num;
+ /*
+ * Possible return values are as follows:
+ * 1) Returns < 0 for error case
+ * 2) Return == 0 to increment PC and continue
+ * 3) Return == 1 to inject illegal instruction trap and continue
+ * 4) Return == 2 to inject virtual instruction trap and continue
+ * 5) Return == 3 to do nothing and continue
+ */
+ int (*rmw_func)(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ unsigned int csr_num, unsigned long *val,
+ unsigned long new_val, unsigned long wr_mask);
+};
+
+static const struct csr_func csr_funcs[] = {
+ {
+ .csr_num = CSR_SIE,
+ .rmw_func = cpu_vcpu_nested_smode_csr_rmw,
+ },
+ {
+ .csr_num = CSR_SIEH,
+ .rmw_func = cpu_vcpu_nested_smode_csr_rmw,
+ },
+ {
+ .csr_num = CSR_SIP,
+ .rmw_func = cpu_vcpu_nested_smode_csr_rmw,
+ },
+ {
+ .csr_num = CSR_SIPH,
+ .rmw_func = cpu_vcpu_nested_smode_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HSTATUS,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HEDELEG,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HIDELEG,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HVIP,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HIE,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HIP,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HGEIP,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HGEIE,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HCOUNTEREN,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HTIMEDELTA,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HTIMEDELTAH,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HTVAL,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HTINST,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HGATP,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSSTATUS,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSIP,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSIE,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSTVEC,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSSCRATCH,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSEPC,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSCAUSE,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSTVAL,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSATP,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+};
+
+static int csr_insn(struct vmm_vcpu *vcpu, arch_regs_t *regs, ulong insn)
+{
+ int i, rc = TRAP_RETURN_ILLEGAL_INSN;
+ unsigned int csr_num = insn >> SH_RS2;
+ unsigned int rs1_num = (insn >> SH_RS1) & MASK_RX;
+ ulong rs1_val = GET_RS1(insn, regs);
+ const struct csr_func *tcfn, *cfn = NULL;
+ ulong val = 0, wr_mask = 0, new_val = 0;
+
+ /* Decode the CSR instruction */
switch (GET_RM(insn)) {
case 1:
- new_csr_val = rs1_val;
- do_write = 1;
+ wr_mask = -1UL;
+ new_val = rs1_val;
break;
case 2:
- new_csr_val = csr_val | rs1_val;
+ wr_mask = rs1_val;
+ new_val = -1UL;
break;
- case 3: new_csr_val = csr_val & ~rs1_val;
+ case 3:
+ wr_mask = rs1_val;
+ new_val = 0;
break;
case 5:
- new_csr_val = rs1_num;
- do_write = 1;
+ wr_mask = -1UL;
+ new_val = rs1_num;
break;
case 6:
- new_csr_val = csr_val | rs1_num;
+ wr_mask = rs1_num;
+ new_val = -1UL;
break;
case 7:
- new_csr_val = csr_val & ~rs1_num;
+ wr_mask = rs1_num;
+ new_val = 0;
break;
default:
- return truly_illegal_insn(vcpu, regs, insn);
- };
+ return rc;
+ }
- if (do_write) {
- rc = cpu_vcpu_csr_write(vcpu, csr_num, new_csr_val);
- if (rc == VMM_EINVALID) {
- return truly_illegal_insn(vcpu, regs, insn);
+ /* Find CSR function */
+ for (i = 0; i < array_size(csr_funcs); i++) {
+ tcfn = &csr_funcs[i];
+ if (tcfn->csr_num == csr_num) {
+ cfn = tcfn;
+ break;
}
- if (rc) {
- return rc;
+ }
+ if (!cfn || !cfn->rmw_func) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ /* Do CSR emulation */
+ rc = cfn->rmw_func(vcpu, regs, csr_num, &val, new_val, wr_mask);
+ if (rc) {
+ return rc;
+ }
+
+ /* Update destination register for CSR reads */
+ if ((insn >> SH_RD) & MASK_RX) {
+ SET_RD(insn, regs, val);
+ }
+{
+ bool next_virt;
+ unsigned long vsstatus, next_sepc, next_spp;
+
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded to
+ * virtual-HS mode as a virtual instruction trap.
+ */
+ if (riscv_nested_virt(vcpu)) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-U mode should be forwarded to virtual-HS mode
+ * as illegal instruction trap.
+ */
+ if (!(regs->hstatus & HSTATUS_SPVP)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+ vsstatus = csr_read(CSR_VSSTATUS);
+
+ /*
+ * Find next nested virtualization mode, next privilege mode,
+ * and next sepc
+ */
+ next_virt = (riscv_nested_priv(vcpu)->hstatus & HSTATUS_SPV) ?
+ TRUE : FALSE;
+ next_sepc = csr_read(CSR_VSEPC);
+ next_spp = vsstatus & SSTATUS_SPP;
+
+ /* Update Guest sstatus.sie */
+ vsstatus &= ~SSTATUS_SIE;
+ vsstatus |= (vsstatus & SSTATUS_SPIE) ? SSTATUS_SIE : 0;
+ csr_write(CSR_VSSTATUS, vsstatus);
+
+ /* Update return address and return privilege mode*/
+ regs->sepc = next_sepc;
+ regs->sstatus &= ~SSTATUS_SPP;
+ regs->sstatus |= next_spp;
+
+ /* Set nested virtualization state based on guest hstatus.SPV */
+ cpu_vcpu_nested_set_virt(vcpu, regs, NESTED_SET_VIRT_EVENT_SRET,
+ next_virt, FALSE, FALSE);
+
+ return TRAP_RETURN_CONTINUE;
+}
+
+static int wfi_insn(struct vmm_vcpu *vcpu, arch_regs_t *regs, ulong insn)
+{
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded to
+ * virtual-HS mode as a virtual instruction trap.
+ */
+ if (riscv_nested_virt(vcpu)) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-U mode should be forwarded to virtual-HS mode
+ * as illegal instruction trap.
+ */
+ if (!(regs->hstatus & HSTATUS_SPVP)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ /* Wait for irq with default timeout */
+ vmm_vcpu_irq_wait_timeout(vcpu, 0);
+ return VMM_OK;
+}
+
+static int hfence_vvma_insn(struct vmm_vcpu *vcpu,
+ arch_regs_t *regs, ulong insn)
+{
+ unsigned int rs1_num = (insn >> SH_RS1) & MASK_RX;
+ unsigned int rs2_num = (insn >> SH_RS2) & MASK_RX;
+ unsigned long vaddr = GET_RS1(insn, regs);
+ unsigned int asid = GET_RS2(insn, regs);
+
+ /*
+ * If H-extension is not available for VCPU then forward trap
+ * as illegal instruction trap to virtual-HS mode.
+ */
+ if (!riscv_isa_extension_available(riscv_priv(vcpu)->isa, h)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded to
+ * virtual-HS mode as a virtual instruction trap.
+ */
+ if (riscv_nested_virt(vcpu)) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-U mode should be forwarded to virtual-HS mode
+ * as illegal instruction trap.
+ */
+ if (!(regs->hstatus & HSTATUS_SPVP)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ if (!rs1_num && !rs2_num) {
+ cpu_vcpu_nested_hfence_vvma(vcpu, NULL, NULL);
+ } else if (!rs1_num && rs2_num) {
+ cpu_vcpu_nested_hfence_vvma(vcpu, NULL, &asid);
+ } else if (rs1_num && !rs2_num) {
+ cpu_vcpu_nested_hfence_vvma(vcpu, &vaddr, NULL);
+ } else {
+ cpu_vcpu_nested_hfence_vvma(vcpu, &vaddr, &asid);
+ }
+ arch_regs_t *regs, ulong insn)
+{
+ unsigned int rs1_num = (insn >> SH_RS1) & MASK_RX;
+ unsigned int rs2_num = (insn >> SH_RS2) & MASK_RX;
+ physical_addr_t gaddr = GET_RS1(insn, regs) << 2;
+ unsigned int vmid = GET_RS2(insn, regs);
+
+ /*
+ * If H-extension is not available for VCPU then forward trap
+ * as illegal instruction trap to virtual-HS mode.
+ */
+ if (!riscv_isa_extension_available(riscv_priv(vcpu)->isa, h)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded to
+ * virtual-HS mode as a virtual instruction trap.
+ */
+ if (riscv_nested_virt(vcpu)) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-U mode should be forwarded to virtual-HS mode
+ * as illegal instruction trap.
+ */
+ if (!(regs->hstatus & HSTATUS_SPVP)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ if (!rs1_num && !rs2_num) {
+ cpu_vcpu_nested_hfence_gvma(vcpu, NULL, NULL);
+ } else if (!rs1_num && rs2_num) {
+ cpu_vcpu_nested_hfence_gvma(vcpu, NULL, &vmid);
+ } else if (rs1_num && !rs2_num) {
+ cpu_vcpu_nested_hfence_gvma(vcpu, &gaddr, NULL);
+ } else {
+ cpu_vcpu_nested_hfence_gvma(vcpu, &gaddr, &vmid);
+ }
+ ulong data_ulong;
+ u64 data_u64;
+ u32 data_u32;
+ u16 data_u16;
+ u8 data_u8;
+};
+
+static int hlv_insn(struct vmm_vcpu *vcpu, arch_regs_t *regs, ulong insn)
+{
+ int rc;
+ void *data;
+ bool hlvx = FALSE;
+ union hxv_reg_data v;
+ struct cpu_vcpu_trap trap;
+ unsigned long shift = 0, len, vaddr;
+
+ /*
+ * If H-extension is not available for VCPU then forward trap
+ * as illegal instruction trap to virtual-HS mode.
+ */
+ if (!riscv_isa_extension_available(riscv_priv(vcpu)->isa, h)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded to
+ * virtual-HS mode as a virtual instruction trap.
+ */
+ if (riscv_nested_virt(vcpu)) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-U mode should be forwarded to virtual-HS mode
+ * as illegal instruction trap when guest hstatus.HU == 0.
+ */
+ if (!(regs->hstatus & HSTATUS_SPVP) &&
+ !(riscv_nested_priv(vcpu)->hstatus & HSTATUS_HU)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ vaddr = GET_RS1(insn, regs);
+ v.data_u64 = 0;
+
+ if ((insn & INSN_MASK_HLV_B) == INSN_MATCH_HLV_B) {
+ data = &v.data_u8;
+ len = 1;
+ shift = (sizeof(long) - 1) * 8;
+ } else if ((insn & INSN_MASK_HLV_BU) == INSN_MATCH_HLV_BU) {
+ data = &v.data_u8;
+ len = 1;
+ } else if ((insn & INSN_MASK_HLV_H) == INSN_MATCH_HLV_H) {
+ data = &v.data_u16;
+ len = 2;
+ shift = (sizeof(long) - 2) * 8;
+ } else if ((insn & INSN_MASK_HLV_HU) == INSN_MATCH_HLV_HU) {
+ data = &v.data_u16;
+ len = 2;
+ } else if ((insn & INSN_MASK_HLVX_HU) == INSN_MATCH_HLVX_HU) {
+ data = &v.data_u16;
+ len = 2;
+ hlvx = TRUE;
+ } else if ((insn & INSN_MASK_HLV_W) == INSN_MATCH_HLV_W) {
+ data = &v.data_u32;
+ len = 4;
+ shift = (sizeof(long) - 4) * 8;
+ } else if ((insn & INSN_MASK_HLV_WU) == INSN_MATCH_HLV_WU) {
+ data = &v.data_u32;
+ len = 4;
+ } else if ((insn & INSN_MASK_HLVX_WU) == INSN_MATCH_HLVX_WU) {
+ data = &v.data_u32;
+ len = 4;
+ hlvx = TRUE;
+ } else if ((insn & INSN_MASK_HLV_D) == INSN_MATCH_HLV_D) {
+ data = &v.data_u64;
+ len = 8;
+ } else {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ trap.sepc = regs->sepc;
+ trap.scause = 0;
+ trap.stval = 0;
+ trap.htval = 0;
+ trap.htinst = insn;
+
+ rc = cpu_vcpu_nested_hlv(vcpu, vaddr, hlvx, data, len,
+ &trap.scause, &trap.stval, &trap.htval);
+ if (rc) {
+ return rc;
+ }
+
+ if (trap.scause) {
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ return TRAP_RETURN_CONTINUE;
+ } else {
+ SET_RD(insn, regs, ((long)(v.data_ulong << shift)) >> shift);
+ }
+{
+ int rc;
+ void *data;
+ union hxv_reg_data v;
+ unsigned long len, vaddr;
+ struct cpu_vcpu_trap trap;
+
+ /*
+ * If H-extension is not available for VCPU then forward trap
+ * as illegal instruction trap to virtual-HS mode.
+ */
+ if (!riscv_isa_extension_available(riscv_priv(vcpu)->isa, h)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded to
+ * virtual-HS mode as a virtual instruction trap.
+ */
+ if (riscv_nested_virt(vcpu)) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-U mode should be forwarded to virtual-HS mode
+ * as illegal instruction trap when guest hstatus.HU == 0.
+ */
+ if (!(regs->hstatus & HSTATUS_SPVP) &&
+ !(riscv_nested_priv(vcpu)->hstatus & HSTATUS_HU)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ vaddr = GET_RS1(insn, regs);
+ v.data_ulong = GET_RS2(insn, regs);
+
+ if ((insn & INSN_MASK_HSV_B) == INSN_MATCH_HSV_B) {
+ data = &v.data_u8;
+ len = 1;
+ } else if ((insn & INSN_MASK_HSV_H) == INSN_MATCH_HSV_H) {
+ data = &v.data_u16;
+ len = 2;
+ } else if ((insn & INSN_MASK_HSV_W) == INSN_MATCH_HSV_W) {
+ data = &v.data_u32;
+ len = 4;
+ } else if ((insn & INSN_MASK_HSV_D) == INSN_MATCH_HSV_D) {
+ data = &v.data_u64;
+ len = 8;
+ } else {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ trap.sepc = regs->sepc;
+ trap.scause = 0;
+ trap.stval = 0;
+ trap.htval = 0;
+ trap.htinst = insn;
+
+ rc = cpu_vcpu_nested_hsv(vcpu, vaddr, data, len,
+ &trap.scause, &trap.stval, &trap.htval);
+ if (rc) {
+ return rc;
+ }
+
+ if (trap.scause) {
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ return TRAP_RETURN_CONTINUE;
+ }
+ {
+ .mask = INSN_MASK_CSRRW,
+ .match = INSN_MATCH_CSRRW,
+ .func = csr_insn,
+ },
+ {
+ .mask = INSN_MASK_CSRRS,
+ .match = INSN_MATCH_CSRRS,
+ .func = csr_insn,
+ },
+ {
+ .mask = INSN_MASK_CSRRC,
+ .match = INSN_MATCH_CSRRC,
+ .func = csr_insn,
+ },
+ {
+ .mask = INSN_MASK_CSRRWI,
+ .match = INSN_MATCH_CSRRWI,
+ .func = csr_insn,
+ },
+ {
+ .mask = INSN_MASK_CSRRSI,
+ .match = INSN_MATCH_CSRRSI,
+ .func = csr_insn,
+ },
+ {
+ .mask = INSN_MASK_CSRRCI,
+ .match = INSN_MATCH_CSRRCI,
+ .func = csr_insn,
+ },
+ {
+ .mask = INSN_MASK_SRET,
+ .match = INSN_MATCH_SRET,
+ .func = sret_insn,
+ },
+ {
+ .mask = INSN_MASK_WFI,
+ .match = INSN_MATCH_WFI,
+ .func = wfi_insn,
+ },
+ {
+ .mask = INSN_MASK_HFENCE_VVMA,
+ .match = INSN_MATCH_HFENCE_VVMA,
+ .func = hfence_vvma_insn,
+ },
+ {
+ .mask = INSN_MASK_HFENCE_GVMA,
+ .match = INSN_MATCH_HFENCE_GVMA,
+ .func = hfence_gvma_insn,
+ },
+ {
+ .mask = INSN_MASK_HLV_B,
+ .match = INSN_MATCH_HLV_B,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HLV_BU,
+ .match = INSN_MATCH_HLV_BU,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HLV_H,
+ .match = INSN_MATCH_HLV_H,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HLV_HU,
+ .match = INSN_MATCH_HLV_HU,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HLVX_HU,
+ .match = INSN_MATCH_HLVX_HU,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HLV_W,
+ .match = INSN_MATCH_HLV_W,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HLV_WU,
+ .match = INSN_MATCH_HLV_WU,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HLVX_WU,
+ .match = INSN_MATCH_HLVX_WU,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HLV_D,
+ .match = INSN_MATCH_HLV_D,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HSV_B,
+ .match = INSN_MATCH_HSV_B,
+ .func = hsv_insn,
+ },
+ {
+ .mask = INSN_MASK_HSV_H,
+ .match = INSN_MATCH_HSV_H,
+ .func = hsv_insn,
+ },
+ {
+ .mask = INSN_MASK_HSV_W,
+ .match = INSN_MATCH_HSV_W,
+ .func = hsv_insn,
+ },
+ {
+ .mask = INSN_MASK_HSV_D,
+ .match = INSN_MATCH_HSV_D,
+ .func = hsv_insn,
+ },
+};
+
+static int system_opcode_insn(struct vmm_vcpu *vcpu,
+ arch_regs_t *regs,
+ ulong insn)
+{
+ int i, rc = TRAP_RETURN_ILLEGAL_INSN;
+ const struct system_opcode_func *ifn;
+
+ for (i = 0; i < array_size(system_opcode_funcs); i++) {
+ ifn = &system_opcode_funcs[i];
+ if ((insn & ifn->mask) == ifn->match) {
+ rc = ifn->func(vcpu, regs, insn);
+ break;
}
}
- SET_RD(insn, regs, csr_val);
+ if (rc == TRAP_RETURN_ILLEGAL_INSN)
+ return truly_illegal_insn(vcpu, regs, insn);
+ else if (rc == TRAP_RETURN_VIRTUAL_INSN)
+ return truly_virtual_insn(vcpu, regs, insn);
-done:
- regs->sepc += 4;
+ if (!rc) {
+ regs->sepc += INSN_LEN(insn);
+ }
- return rc;
+ return (rc < 0) ? rc : VMM_OK;
+}
+
+int cpu_vcpu_general_fault(struct vmm_vcpu *vcpu,
+ arch_regs_t *regs, struct cpu_vcpu_trap *trap)
+{
+ struct cpu_vcpu_trap itrap = { 0 };
+
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded
+ * to virtual-HS mode.
+ */
+
+ if (!riscv_nested_virt(vcpu)) {
+ return VMM_EINVALID;
+ }
+
+ /*
+ * Blindly forward all general faults to virtual-HS mode
+ * except illegal instruction fault
+ */
+ if (trap->scause != CAUSE_ILLEGAL_INSTRUCTION) {
+ cpu_vcpu_redirect_trap(vcpu, regs, trap);
+ return VMM_OK;
+ }
+
+ /* Update trap->stval for illegal instruction fault */
+ if (unlikely((trap->stval & INSN_16BIT_MASK) != INSN_16BIT_MASK)) {
+ if (trap->stval == 0) {
+ trap->stval = __cpu_vcpu_unpriv_read_insn(regs->sepc,
+ &itrap);
+ if (itrap.scause) {
+ if (itrap.scause == CAUSE_LOAD_PAGE_FAULT)
+ itrap.scause = CAUSE_FETCH_PAGE_FAULT;
+ itrap.stval = itrap.sepc = regs->sepc;
+ cpu_vcpu_redirect_trap(vcpu, regs, &itrap);
+ return VMM_OK;
+ }
+ }
+ }
+
+ /* Forward illegal instruction fault */
+ return truly_illegal_insn(vcpu, regs, trap->stval);
+}
+
+int cpu_vcpu_illegal_insn_fault(struct vmm_vcpu *vcpu,
+ arch_regs_t *regs,
+ unsigned long stval)
+{
+ unsigned long insn = stval;
+ struct cpu_vcpu_trap trap = { 0 };
+
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded to
+ * virtual-HS mode as a illegal instruction trap.
+ */
+
+ if (!riscv_nested_virt(vcpu)) {
+ return VMM_EINVALID;
+ }
+
+ if (unlikely((insn & INSN_16BIT_MASK) != INSN_16BIT_MASK)) {
+ if (insn == 0) {
+ insn = __cpu_vcpu_unpriv_read_insn(regs->sepc, &trap);
+ if (trap.scause) {
+ if (trap.scause == CAUSE_LOAD_PAGE_FAULT)
+ trap.scause = CAUSE_FETCH_PAGE_FAULT;
+ trap.stval = trap.sepc = regs->sepc;
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ return VMM_OK;
+ }
+ }
+ }
+
+ return truly_illegal_insn(vcpu, regs, insn);
}
int cpu_vcpu_virtual_insn_fault(struct vmm_vcpu *vcpu,
@@ -499,18 +1286,18 @@ int cpu_vcpu_virtual_insn_fault(struct vmm_vcpu *vcpu,
unsigned long insn = stval;
struct cpu_vcpu_trap trap = { 0 };
- if (unlikely((insn & 3) != 3)) {
+ if (unlikely((insn & INSN_16BIT_MASK) != INSN_16BIT_MASK)) {
if (insn == 0) {
insn = __cpu_vcpu_unpriv_read_insn(regs->sepc, &trap);
if (trap.scause) {
if (trap.scause == CAUSE_LOAD_PAGE_FAULT)
trap.scause = CAUSE_FETCH_PAGE_FAULT;
trap.stval = trap.sepc = regs->sepc;
- return cpu_vcpu_redirect_trap(vcpu, regs,
- &trap);
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ return VMM_OK;
}
}
- if ((insn & 3) != 3)
+ if ((insn & INSN_16BIT_MASK) != INSN_16BIT_MASK)
return truly_illegal_insn(vcpu, regs, insn);
}
@@ -521,3 +1308,78 @@ int cpu_vcpu_virtual_insn_fault(struct vmm_vcpu *vcpu,
return truly_illegal_insn(vcpu, regs, insn);
};
}
+
+void cpu_vcpu_take_vsirq(struct vmm_vcpu *vcpu, struct arch_regs *regs)
+{
+ int vsirq;
+ bool next_spp;
+ unsigned long irqs;
+ struct cpu_vcpu_trap trap;
+ struct riscv_priv_nested *npriv;
+
+ /* Do nothing for Orphan VCPUs */
+ if (!vcpu->is_normal) {
+ return;
+ }
+
+ /* Do nothing if virt state is OFF */
+ npriv = riscv_nested_priv(vcpu);
+ if (!npriv->virt) {
+ return;
+ }
+
+ /*
+ * Determine whether we are resuming in virtual-VS mode
+ * or virtual-VU mode
+ */
+ next_spp = (regs->sstatus & SSTATUS_SPP) ? TRUE : FALSE;
+
+ /*
+ * Do nothing if we going to virtual-VS mode and
+ * interrupts are disabled
+ */
+ if (next_spp && !(csr_read(CSR_VSSTATUS) & SSTATUS_SIE)) {
+ return;
+ }
+
+ /* Determine virtual-VS mode interrupt number */
+ vsirq = 0;
+ irqs = npriv->hvip;
+ irqs &= npriv->vsie << 1;
+ irqs &= npriv->hideleg;
+ if (irqs & MIP_VSEIP) {
+ vsirq = IRQ_S_EXT;
+ } else if (irqs & MIP_VSTIP) {
+ vsirq = IRQ_S_TIMER;
+ } else if (irqs & MIP_VSSIP) {
+ vsirq = IRQ_S_SOFT;
+ }
+
+ /* Take virtual-VS mode interrupt */
+ if (vsirq > 0) {
+ trap.scause = SCAUSE_INTERRUPT_MASK | vsirq;
+ trap.sepc = regs->sepc;
+ trap.stval = 0;
+ trap.htval = 0;
+ trap.htinst = 0;
+ cpu_vcpu_redirect_smode_trap(regs, &trap, next_spp);
+ }
+
+ return;
+}
+
+int cpu_vcpu_redirect_vsirq(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ unsigned long irq)
+{
+ struct cpu_vcpu_trap trap = { 0 };
+
+ if (!vcpu || !vcpu->is_normal || !riscv_nested_virt(vcpu)) {
+ return VMM_EFAIL;
+ }
+
+ trap.sepc = regs->sepc;
+ trap.scause = SCAUSE_INTERRUPT_MASK | (irq - 1);
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
deleted file mode 100644
index cc1bb580..00000000
--- a/arch/riscv/cpu/generic/include/cpu_vcpu_csr.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Copyright (c) 2019 Anup Patel.
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- *
- * @file cpu_vcpu_csr.h
- * @author Anup Patel (an...@brainfault.org)
- * @brief header for VCPU CSR read/write handling
- */
-#ifndef _CPU_VCPU_CSR_H__
-#define _CPU_VCPU_CSR_H__
-
-#include <vmm_types.h>
-#include <vmm_manager.h>
-
-int cpu_vcpu_csr_read(struct vmm_vcpu *vcpu,
- unsigned long csr_num,
- unsigned long *csr_val);
-
-int cpu_vcpu_csr_write(struct vmm_vcpu *vcpu,
- unsigned long csr_num,
- unsigned long csr_val);
-
-#endif
diff --git a/arch/riscv/cpu/generic/include/cpu_vcpu_nested.h b/arch/riscv/cpu/generic/include/cpu_vcpu_nested.h
new file mode 100644
index 00000000..669690d3
--- /dev/null
+++ b/arch/riscv/cpu/generic/include/cpu_vcpu_nested.h
@@ -0,0 +1,105 @@
+ */
+
+#ifndef _CPU_VCPU_NESTED_H__
+#define _CPU_VCPU_NESTED_H__
+
+#include <vmm_types.h>
+
+struct vmm_vcpu;
+struct cpu_vcpu_trap;
+struct arch_regs;
+
+/** Function to init nested state */
+int cpu_vcpu_nested_init(struct vmm_vcpu *vcpu);
+
+/** Function to reset nested state */
+void cpu_vcpu_nested_reset(struct vmm_vcpu *vcpu);
+
+/** Function to initialize nested state */
+void cpu_vcpu_nested_deinit(struct vmm_vcpu *vcpu);
+
+/** Function to dump nested registers */
+void cpu_vcpu_nested_dump_regs(struct vmm_chardev *cdev,
+ struct vmm_vcpu *vcpu);
+
+/** Function to access nested non-virt CSRs */
+int cpu_vcpu_nested_smode_csr_rmw(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ unsigned int csr_num, unsigned long *val,
+ unsigned long new_val, unsigned long wr_mask);
+
+/** Function to access nested virt CSRs */
+int cpu_vcpu_nested_hext_csr_rmw(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ unsigned int csr_num, unsigned long *val,
+ unsigned long new_val, unsigned long wr_mask);
+
+/** Function to handle nested page fault */
+int cpu_vcpu_nested_page_fault(struct vmm_vcpu *vcpu,
+ bool trap_from_smode,
+ const struct cpu_vcpu_trap *trap,
+void cpu_vcpu_nested_hfence_vvma(struct vmm_vcpu *vcpu,
+ unsigned long *vaddr, unsigned int *asid);
+
+/** Function to handle nested hfence.gvma instruction */
+void cpu_vcpu_nested_hfence_gvma(struct vmm_vcpu *vcpu,
+ physical_addr_t *gaddr, unsigned int *vmid);
+
+/**
+ * Function to handle nested hlv instruction
+ * @returns (< 0) error code upon failure and (>= 0) trap return value
+ * upon success
+ */
+int cpu_vcpu_nested_hlv(struct vmm_vcpu *vcpu, unsigned long vaddr,
+ bool hlvx, void *data, unsigned long len,
+ unsigned long *out_scause,
+ unsigned long *out_stval,
+ unsigned long *out_htval);
+
+/**
+ * Function to handle nested hsv instruction
+ * @returns (< 0) error code upon failure and (>= 0) trap return value
+ * upon success
+ */
+int cpu_vcpu_nested_hsv(struct vmm_vcpu *vcpu, unsigned long vaddr,
+ const void *data, unsigned long len,
+ unsigned long *out_scause,
+ unsigned long *out_stval,
+ unsigned long *out_htval);
+
+enum nested_set_virt_event {
+ NESTED_SET_VIRT_EVENT_TRAP = 0,
+ NESTED_SET_VIRT_EVENT_SRET,
+};
+
+/**
+ * Function to change nested virtualization state
+ * NOTE: This can also update Guest hstatus.SPV and hstatus.SPVP bits
+ */
+void cpu_vcpu_nested_set_virt(struct vmm_vcpu *vcpu, struct arch_regs *regs,
+ enum nested_set_virt_event event, bool virt,
+ bool spvp, bool gva);
+
+#endif
diff --git a/arch/riscv/cpu/generic/include/cpu_vcpu_trap.h b/arch/riscv/cpu/generic/include/cpu_vcpu_trap.h
index cb0ffbb6..d77bba87 100644
--- a/arch/riscv/cpu/generic/include/cpu_vcpu_trap.h
+++ b/arch/riscv/cpu/generic/include/cpu_vcpu_trap.h
@@ -44,18 +44,38 @@ struct cpu_vcpu_trap {
unsigned long htinst;
};
-int cpu_vcpu_redirect_trap(struct vmm_vcpu *vcpu,
- arch_regs_t *regs,
- struct cpu_vcpu_trap *trap);
+enum trap_return {
+ TRAP_RETURN_OK=0,
+ TRAP_RETURN_ILLEGAL_INSN,
+ TRAP_RETURN_VIRTUAL_INSN,
+ TRAP_RETURN_CONTINUE
+};
+
+void cpu_vcpu_update_trap(struct vmm_vcpu *vcpu, arch_regs_t *regs);
+
+void cpu_vcpu_redirect_smode_trap(arch_regs_t *regs,
+ struct cpu_vcpu_trap *trap, bool prev_spp);
+
+void cpu_vcpu_redirect_trap(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ struct cpu_vcpu_trap *trap);
int cpu_vcpu_page_fault(struct vmm_vcpu *vcpu,
arch_regs_t *regs,
struct cpu_vcpu_trap *trap);
+int cpu_vcpu_general_fault(struct vmm_vcpu *vcpu,
+ arch_regs_t *regs,
+ struct cpu_vcpu_trap *trap);
+
int cpu_vcpu_virtual_insn_fault(struct vmm_vcpu *vcpu,
arch_regs_t *regs,
unsigned long stval);
+void cpu_vcpu_take_vsirq(struct vmm_vcpu *vcpu, arch_regs_t *regs);
+
+int cpu_vcpu_redirect_vsirq(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ unsigned long irq);
+
#endif
#endif
diff --git a/arch/riscv/cpu/generic/objects.mk b/arch/riscv/cpu/generic/objects.mk
index bbb9a6b5..69daf98a 100644
--- a/arch/riscv/cpu/generic/objects.mk
+++ b/arch/riscv/cpu/generic/objects.mk
@@ -79,7 +79,7 @@ cpu-objs-$(CONFIG_SMP)+=cpu_smp_ops_default.o
cpu-objs-$(CONFIG_SMP)+=cpu_smp_ops_sbi.o
cpu-objs-$(CONFIG_SMP)+=cpu_sbi_ipi.o
cpu-objs-y+= cpu_vcpu_helper.o
-cpu-objs-y+= cpu_vcpu_csr.o
+cpu-objs-y+= cpu_vcpu_nested.o
cpu-objs-y+= cpu_vcpu_fp.o
cpu-objs-y+= cpu_vcpu_irq.o
cpu-objs-y+= cpu_vcpu_sbi.o
--
2.25.1
placeholder (or stub) functions for emulating hfences, hlv/hsv, and
shadow g-stage page table. These stub function will be filled-up by
subsequent patches.
arch/riscv/cpu/generic/cpu_exception.c | 38 +
arch/riscv/cpu/generic/cpu_vcpu_csr.c | 54 -
arch/riscv/cpu/generic/cpu_vcpu_fp.c | 56 +-
arch/riscv/cpu/generic/cpu_vcpu_helper.c | 27 +-
arch/riscv/cpu/generic/cpu_vcpu_nested.c | 587 ++++++++++
arch/riscv/cpu/generic/cpu_vcpu_sbi.c | 11 +
arch/riscv/cpu/generic/cpu_vcpu_trap.c | 998 ++++++++++++++++--
arch/riscv/cpu/generic/include/cpu_vcpu_csr.h | 37 -
.../cpu/generic/include/cpu_vcpu_nested.h | 105 ++
.../riscv/cpu/generic/include/cpu_vcpu_trap.h | 26 +-
arch/riscv/cpu/generic/objects.mk | 2 +-
11 files changed, 1758 insertions(+), 183 deletions(-)
delete mode 100644 arch/riscv/cpu/generic/cpu_vcpu_csr.c
create mode 100644 arch/riscv/cpu/generic/cpu_vcpu_nested.c
create mode 100644 arch/riscv/cpu/generic/cpu_vcpu_nested.c
delete mode 100644 arch/riscv/cpu/generic/include/cpu_vcpu_csr.h
create mode 100644 arch/riscv/cpu/generic/include/cpu_vcpu_nested.h
diff --git a/arch/riscv/cpu/generic/cpu_exception.c b/arch/riscv/cpu/generic/cpu_exception.c
create mode 100644 arch/riscv/cpu/generic/include/cpu_vcpu_nested.h
index 76fded9d..b08ee243 100644
--- a/arch/riscv/cpu/generic/cpu_exception.c
+++ b/arch/riscv/cpu/generic/cpu_exception.c
@@ -58,6 +58,14 @@ void do_handle_irq(arch_regs_t *regs, unsigned long cause)
vmm_scheduler_irq_enter(regs, FALSE);
+ if (cause == IRQ_VS_SOFT ||
+ cause == IRQ_VS_TIMER ||
+ cause == IRQ_VS_EXT) {
+ rc = cpu_vcpu_redirect_vsirq(vmm_scheduler_current_vcpu(),
+ regs, cause);
+ goto done;
+ }
+
/* NOTE: Only exec <= 0xFFFFFFFFUL will be handled */
if (cause <= 0xFFFFFFFFUL) {
rc = vmm_host_active_irq_exec(cause);
@@ -65,10 +73,13 @@ void do_handle_irq(arch_regs_t *regs, unsigned long cause)
rc = VMM_EINVALID;
}
+done:
if (rc) {
do_error(vmm_scheduler_current_vcpu(), regs,
cause | SCAUSE_INTERRUPT_MASK,
"interrupt handling failed", rc, TRUE);
+ } else {
+ cpu_vcpu_take_vsirq(vmm_scheduler_current_vcpu(), regs);
}
vmm_scheduler_irq_exit(regs);
@@ -100,6 +111,31 @@ void do_handle_trap(arch_regs_t *regs, unsigned long cause)
}
switch (cause) {
+ case CAUSE_MISALIGNED_FETCH:
+ case CAUSE_FETCH_ACCESS:
+ case CAUSE_ILLEGAL_INSTRUCTION:
+ case CAUSE_BREAKPOINT:
+ case CAUSE_MISALIGNED_LOAD:
+ case CAUSE_LOAD_ACCESS:
+ case CAUSE_MISALIGNED_STORE:
+ case CAUSE_STORE_ACCESS:
+ case CAUSE_USER_ECALL:
+ case CAUSE_FETCH_PAGE_FAULT:
+ case CAUSE_LOAD_PAGE_FAULT:
+ case CAUSE_STORE_PAGE_FAULT:
+ msg = "general fault failed";
+ if (regs->hstatus & HSTATUS_SPV) {
+ trap.sepc = regs->sepc;
+ trap.scause = cause;
+ trap.stval = csr_read(CSR_STVAL);
+ trap.htval = csr_read(CSR_HTVAL);
+ trap.htinst = csr_read(CSR_HTINST);
+ rc = cpu_vcpu_general_fault(vcpu, regs, &trap);
+ panic = FALSE;
+ } else {
+ rc = VMM_EINVALID;
+ }
+ break;
case CAUSE_FETCH_GUEST_PAGE_FAULT:
case CAUSE_LOAD_GUEST_PAGE_FAULT:
case CAUSE_STORE_GUEST_PAGE_FAULT:
@@ -147,6 +183,8 @@ void do_handle_trap(arch_regs_t *regs, unsigned long cause)
done:
if (rc) {
do_error(vcpu, regs, cause, msg, rc, panic);
+ } else {
+ cpu_vcpu_take_vsirq(vcpu, regs);
}
vmm_scheduler_irq_exit(regs);
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_csr.c b/arch/riscv/cpu/generic/cpu_vcpu_csr.c
deleted file mode 100644
index 76856ca7..00000000
--- a/arch/riscv/cpu/generic/cpu_vcpu_csr.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Copyright (c) 2019 Anup Patel.
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- *
- * @file cpu_vcpu_csr.c
- * @author Anup Patel (an...@brainfault.org)
- * @brief source for VCPU CSR read/write handling
- */
-
-#include <vmm_error.h>
-#include <vmm_stdio.h>
-#include <cpu_vcpu_csr.h>
-
-int cpu_vcpu_csr_read(struct vmm_vcpu *vcpu,
- unsigned long csr_num,
- unsigned long *csr_val)
-{
- /*
- * We don't have any CSRs to emulate because runtime
- * M-mode firmware (i.e. OpenSBI) takes care of it
- */
- vmm_printf("%s: vcpu=%s invalid csr_num=0x%lx\n",
- __func__, (vcpu) ? vcpu->name : "(null)", csr_num);
-
- return VMM_ENOTSUPP;
-}
-
-int cpu_vcpu_csr_write(struct vmm_vcpu *vcpu,
- unsigned long csr_num,
- unsigned long csr_val)
-{
- /*
- * We don't have any CSRs to emulate because runtime
- * M-mode firmware (i.e. OpenSBI) takes care of it
- */
- vmm_printf("%s: vcpu=%s invalid csr_num=0x%lx\n",
- __func__, (vcpu) ? vcpu->name : "(null)", csr_num);
-
- return VMM_ENOTSUPP;
-}
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_fp.c b/arch/riscv/cpu/generic/cpu_vcpu_fp.c
index e4ccebaa..7a1f907a 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_fp.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_fp.c
@@ -45,31 +45,51 @@ static inline void cpu_vcpu_fp_clean(arch_regs_t *regs)
regs->sstatus |= SSTATUS_FS_CLEAN;
}
-void cpu_vcpu_fp_save(struct vmm_vcpu *vcpu, arch_regs_t *regs)
+static inline void cpu_vcpu_fp_force_save(struct vmm_vcpu *vcpu)
+{
+ unsigned long *isa = riscv_priv(vcpu)->isa;
+
+ if (riscv_isa_extension_available(isa, d))
+ __cpu_vcpu_fp_d_save(&riscv_priv(vcpu)->fp.d);
+ else if (riscv_isa_extension_available(isa, f))
+ __cpu_vcpu_fp_f_save(&riscv_priv(vcpu)->fp.f);
+}
+
+static inline void cpu_vcpu_fp_force_restore(struct vmm_vcpu *vcpu)
{
- unsigned long *isa;
+ unsigned long *isa = riscv_priv(vcpu)->isa;
- if ((regs->sstatus & SSTATUS_FS) == SSTATUS_FS_DIRTY) {
- isa = riscv_priv(vcpu)->isa;
- if (riscv_isa_extension_available(isa, d))
- __cpu_vcpu_fp_d_save(&riscv_priv(vcpu)->fp.d);
- else if (riscv_isa_extension_available(isa, f))
- __cpu_vcpu_fp_f_save(&riscv_priv(vcpu)->fp.f);
- cpu_vcpu_fp_clean(regs);
+ if (riscv_isa_extension_available(isa, d))
+ __cpu_vcpu_fp_d_restore(&riscv_priv(vcpu)->fp.d);
+ else if (riscv_isa_extension_available(isa, f))
+ __cpu_vcpu_fp_f_restore(&riscv_priv(vcpu)->fp.f);
+}
+
+void cpu_vcpu_fp_save(struct vmm_vcpu *vcpu, arch_regs_t *regs)
+{
+ if (riscv_nested_virt(vcpu)) {
+ /* Always save FP state when nested virtualization is ON */
+ cpu_vcpu_fp_force_save(vcpu);
+ } else {
+ /* Lazy save FP state when nested virtualization is OFF */
+ if ((regs->sstatus & SSTATUS_FS) == SSTATUS_FS_DIRTY) {
+ cpu_vcpu_fp_force_save(vcpu);
+ cpu_vcpu_fp_clean(regs);
+ }
}
}
void cpu_vcpu_fp_restore(struct vmm_vcpu *vcpu, arch_regs_t *regs)
{
- unsigned long *isa;
-
- if ((regs->sstatus & SSTATUS_FS) != SSTATUS_FS_OFF) {
- isa = riscv_priv(vcpu)->isa;
- if (riscv_isa_extension_available(isa, d))
- __cpu_vcpu_fp_d_restore(&riscv_priv(vcpu)->fp.d);
- else if (riscv_isa_extension_available(isa, f))
- __cpu_vcpu_fp_f_restore(&riscv_priv(vcpu)->fp.f);
- cpu_vcpu_fp_clean(regs);
+ if (riscv_nested_virt(vcpu)) {
+ /* Always restore FP state when nested virtualization is ON */
+ cpu_vcpu_fp_force_restore(vcpu);
+ } else {
+ /* Lazy restore FP state when nested virtualization is OFF */
+ if ((regs->sstatus & SSTATUS_FS) != SSTATUS_FS_OFF) {
+ cpu_vcpu_fp_force_restore(vcpu);
+ cpu_vcpu_fp_clean(regs);
+ }
}
}
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_helper.c b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
index 4ff619a7..e33bce24 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_helper.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
@@ -39,6 +39,8 @@
#include <cpu_tlb.h>
#include <cpu_sbi.h>
#include <cpu_vcpu_fp.h>
+#include <cpu_vcpu_trap.h>
+#include <cpu_vcpu_nested.h>
#include <cpu_vcpu_helper.h>
#include <cpu_vcpu_timer.h>
#include <cpu_guest_serial.h>
@@ -52,8 +54,7 @@
riscv_isa_extension_mask(f) | \
riscv_isa_extension_mask(i) | \
riscv_isa_extension_mask(m) | \
- riscv_isa_extension_mask(s) | \
- riscv_isa_extension_mask(u))
+ riscv_isa_extension_mask(h))
static char *guest_fdt_find_serial_node(char *guest_name)
{
@@ -277,6 +278,18 @@ int arch_vcpu_init(struct vmm_vcpu *vcpu)
goto fail_free_isa;
}
riscv_priv(vcpu)->isa[0] &= RISCV_ISA_ALLOWED;
+
}
riscv_priv(vcpu)->isa[0] &= RISCV_ISA_ALLOWED;
+ /* H-extension only available when AIA CSRs are available */
+ if (!riscv_aia_available) {
+ riscv_priv(vcpu)->isa[0] &=
+ ~riscv_isa_extension_mask(h);
+ }
+
+ /* Initialize nested state */
+ rc = cpu_vcpu_nested_init(vcpu);
+ if (rc) {
+ goto fail_free_isa;
+ }
}
/* Set a0 to VCPU sub-id (i.e. virtual HARTID) */
@@ -306,6 +319,9 @@ int arch_vcpu_init(struct vmm_vcpu *vcpu)
/* By default, make CY, TM, and IR counters accessible in VU mode */
riscv_priv(vcpu)->scounteren = 7;
+ /* Reset nested state */
+ cpu_vcpu_nested_reset(vcpu);
+
/* Initialize FP state */
cpu_vcpu_fp_init(vcpu);
@@ -345,10 +361,14 @@ int arch_vcpu_deinit(struct vmm_vcpu *vcpu)
return VMM_OK;
}
+ /* Cleanup timer */
rc = riscv_timer_event_deinit(vcpu, &riscv_timer_priv(vcpu));
if (rc)
return rc;
+ /* Cleanup nested state */
+ cpu_vcpu_nested_deinit(vcpu);
+
/* Free ISA bitmap */
vmm_free(riscv_priv(vcpu)->isa);
riscv_priv(vcpu)->isa = NULL;
@@ -403,6 +423,7 @@ void arch_vcpu_switch(struct vmm_vcpu *tvcpu,
cpu_vcpu_time_delta_update(vcpu, riscv_nested_virt(vcpu));
cpu_vcpu_gstage_update(vcpu, riscv_nested_virt(vcpu));
cpu_vcpu_irq_deleg_update(vcpu, riscv_nested_virt(vcpu));
+ cpu_vcpu_take_vsirq(vcpu, regs);
}
}
@@ -568,6 +589,8 @@ void cpu_vcpu_dump_private_regs(struct vmm_chardev *cdev,
vmm_cprintf(cdev, " %s=0x%"PRIADDR"\n",
" scounteren", priv->scounteren);
+ cpu_vcpu_nested_dump_regs(cdev, vcpu);
" scounteren", priv->scounteren);
+
cpu_vcpu_fp_dump_regs(cdev, vcpu);
}
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_nested.c b/arch/riscv/cpu/generic/cpu_vcpu_nested.c
new file mode 100644
index 00000000..5f8774de
--- /dev/null
+++ b/arch/riscv/cpu/generic/cpu_vcpu_nested.c
@@ -0,0 +1,587 @@
+/**
+ * Copyright (c) 2022 Ventana Micro Systems Inc.
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * @file cpu_vcpu_nested.c
+ * Copyright (c) 2022 Ventana Micro Systems Inc.
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * @author Anup Patel (apa...@ventanamicro.com)
+ * @brief source of VCPU nested functions
+ */
+
+#include <vmm_error.h>
+#include <vmm_stdio.h>
+
+#include <vmm_error.h>
+#include <generic_mmu.h>
+
+#include <cpu_hwcap.h>
+#include <cpu_vcpu_helper.h>
+#include <cpu_vcpu_nested.h>
+#include <cpu_vcpu_trap.h>
+#include <riscv_csr.h>
+
+int cpu_vcpu_nested_init(struct vmm_vcpu *vcpu)
+{
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ npriv->pgtbl = mmu_pgtbl_alloc(MMU_STAGE2, -1);
+ if (!npriv->pgtbl) {
+ return VMM_ENOMEM;
+ }
+
+ return VMM_OK;
+}
+
+void cpu_vcpu_nested_reset(struct vmm_vcpu *vcpu)
+ return VMM_OK;
+}
+
+{
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ npriv->virt = FALSE;
+#ifdef CONFIG_64BIT
+ npriv->hstatus = HSTATUS_VSXL_RV64 << HSTATUS_VSXL_SHIFT;
+#else
+ npriv->hstatus = 0;
+#endif
+ npriv->hedeleg = 0;
+ npriv->hideleg = 0;
+ npriv->hvip = 0;
+ npriv->hcounteren = 0;
+ npriv->htimedelta = 0;
+ npriv->htimedeltah = 0;
+ npriv->htval = 0;
+ npriv->htinst = 0;
+ npriv->hgatp = 0;
+ npriv->vsstatus = 0;
+ npriv->vsie = 0;
+ npriv->vstvec = 0;
+ npriv->vsscratch = 0;
+ npriv->vsepc = 0;
+ npriv->vscause = 0;
+ npriv->vstval = 0;
+ npriv->vsatp = 0;
+
+ npriv->hvictl = 0;
+}
+
+void cpu_vcpu_nested_deinit(struct vmm_vcpu *vcpu)
+{
+ mmu_pgtbl_free(riscv_nested_priv(vcpu)->pgtbl);
+}
+
+void cpu_vcpu_nested_dump_regs(struct vmm_chardev *cdev,
+ struct vmm_vcpu *vcpu)
+{
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ if (!riscv_isa_extension_available(riscv_priv(vcpu)->isa, h))
+ return;
+
+ vmm_cprintf(cdev, "\n");
+ vmm_cprintf(cdev, " %s=%s\n",
+ " virt", (npriv->virt) ? "on" : "off");
+ vmm_cprintf(cdev, "\n");
+#ifdef CONFIG_64BIT
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR"\n",
+ " htimedelta", npriv->htimedelta);
+#else
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " htimedelta", npriv->htimedelta,
+ "htimedeltah", npriv->htimedeltah);
+#endif
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " hstatus", npriv->hstatus, " hgatp", npriv->hgatp);
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " hedeleg", npriv->hedeleg, " hideleg", npriv->hideleg);
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " hvip", npriv->hvip, " hcounteren", npriv->hcounteren);
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " htval", npriv->htval, " htinst", npriv->htinst);
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " vsstatus", npriv->vsstatus, " vsie", npriv->vsie);
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " vsatp", npriv->vsatp, " vstvec", npriv->vstvec);
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " vsscratch", npriv->vsscratch, " vsepc", npriv->vsepc);
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR" %s=0x%"PRIADDR"\n",
+ " vscause", npriv->vscause, " vstval", npriv->vstval);
+
+ vmm_cprintf(cdev, "(V) %s=0x%"PRIADDR"\n",
+ " hvictl", npriv->hvictl);
+}
+
+int cpu_vcpu_nested_smode_csr_rmw(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ unsigned int csr_num, unsigned long *val,
+ unsigned long new_val, unsigned long wr_mask)
+{
+ int csr_shift = 0;
+ bool read_only = FALSE;
+ unsigned long *csr, zero = 0, writeable_mask = 0;
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ /*
+ * These CSRs should never trap for virtual-HS/U modes because
+ * we only emulate these CSRs for virtual-VS/VU modes.
+ */
+ if (!riscv_nested_virt(vcpu)) {
+ return VMM_EINVALID;
+ }
+
+ /*
+ * Access of these CSRs from virtual-VU mode should be forwarded
+ * as illegal instruction trap to virtual-HS mode.
+ */
+ if (!(regs->hstatus & HSTATUS_SPVP)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ switch (csr_num) {
+ case CSR_SIE:
+ if (npriv->hvictl & HVICTL_VTI) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+ csr = &npriv->vsie;
+ writeable_mask = VSIE_WRITEABLE & (npriv->hideleg >> 1);
+ break;
+ case CSR_SIEH:
+ if (npriv->hvictl & HVICTL_VTI) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+ csr = &zero;
+ break;
+ case CSR_SIP:
+ if (npriv->hvictl & HVICTL_VTI) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+ csr = &npriv->hvip;
+ csr_shift = 1;
+ writeable_mask = HVIP_VSSIP & npriv->hideleg;
+ break;
+ case CSR_SIPH:
+ if (npriv->hvictl & HVICTL_VTI) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+ csr = &zero;
+ break;
+ default:
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ if (val) {
+ *val = (csr_shift < 0) ?
+ (*csr) << -csr_shift : (*csr) >> csr_shift;
+ }
+
+ if (read_only) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ } else {
+ writeable_mask = (csr_shift < 0) ?
+ writeable_mask >> -csr_shift :
+ writeable_mask << csr_shift;
+ wr_mask = (csr_shift < 0) ?
+ wr_mask >> -csr_shift : wr_mask << csr_shift;
+ new_val = (csr_shift < 0) ?
+ new_val >> -csr_shift : new_val << csr_shift;
+ wr_mask &= writeable_mask;
+ *csr = (*csr & ~wr_mask) | (new_val & wr_mask);
+ }
+
+ return VMM_OK;
+}
+
+int cpu_vcpu_nested_hext_csr_rmw(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ return VMM_OK;
+}
+
+ unsigned int csr_num, unsigned long *val,
+ unsigned long new_val, unsigned long wr_mask)
+{
+ int csr_shift = 0;
+ bool read_only = FALSE;
+ unsigned int csr_priv = (csr_num >> 8) & 0x3;
+ unsigned long *csr, mode, zero = 0, writeable_mask = 0;
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded
+ * to virtual-HS mode as a virtual instruction trap.
+ */
+ if (riscv_nested_virt(vcpu)) {
+ return (csr_priv == (PRV_S + 1)) ?
+ TRAP_RETURN_VIRTUAL_INSN : TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ /*
+ * If H-extension is not available for VCPU then forward trap
+ * as illegal instruction trap to virtual-HS mode.
+ */
+ if (!riscv_isa_extension_available(riscv_priv(vcpu)->isa, h)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ /*
+ * H-extension CSRs not allowed in virtual-U mode so forward trap
+ * as illegal instruction trap to virtual-HS mode.
+ */
+ if (!(regs->hstatus & HSTATUS_SPVP)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ switch (csr_num) {
+ case CSR_HSTATUS:
+ csr = &npriv->hstatus;
+ writeable_mask = HSTATUS_VTSR | HSTATUS_VTW | HSTATUS_VTVM |
+ HSTATUS_HU | HSTATUS_SPVP | HSTATUS_SPV |
+ HSTATUS_GVA;
+ if (wr_mask & HSTATUS_SPV) {
+ /*
+ * Enable (or Disable) host SRET trapping for
+ * virtual-HS mode. This will be auto-disabled
+ * by cpu_vcpu_nested_set_virt() upon SRET trap
+ * from virtual-HS mode.
+ */
+ regs->hstatus &= ~HSTATUS_VTSR;
+ regs->hstatus |= (new_val & HSTATUS_SPV) ?
+ HSTATUS_VTSR : 0;
+ }
+ break;
+ case CSR_HEDELEG:
+ csr = &npriv->hedeleg;
+ writeable_mask = HEDELEG_WRITEABLE;
+ break;
+ case CSR_HIDELEG:
+ csr = &npriv->hideleg;
+ writeable_mask = HIDELEG_WRITEABLE;
+ break;
+ case CSR_HVIP:
+ csr = &npriv->hvip;
+ writeable_mask = HVIP_WRITEABLE;
+ break;
+ case CSR_HIE:
+ csr = &npriv->vsie;
+ csr_shift = -1;
+ writeable_mask = HVIP_WRITEABLE;
+ break;
+ case CSR_HIP:
+ csr = &npriv->hvip;
+ writeable_mask = HVIP_VSSIP;
+ break;
+ case CSR_HGEIP:
+ csr = &zero;
+ read_only = TRUE;
+ break;
+ case CSR_HGEIE:
+ csr = &zero;
+ break;
+ case CSR_HCOUNTEREN:
+ csr = &npriv->hcounteren;
+ writeable_mask = HCOUNTEREN_WRITEABLE;
+ break;
+ case CSR_HTIMEDELTA:
+ csr = &npriv->htimedelta;
+ writeable_mask = -1UL;
+ break;
+#ifndef CONFIG_64BIT
+ case CSR_HTIMEDELTAH:
+ csr = &npriv->htimedeltah;
+ writeable_mask = -1UL;
+ break;
+#endif
+ case CSR_HTVAL:
+ csr = &npriv->htval;
+ writeable_mask = -1UL;
+ break;
+ case CSR_HTINST:
+ csr = &npriv->htinst;
+ writeable_mask = -1UL;
+ break;
+ case CSR_HGATP:
+ csr = &npriv->hgatp;
+ writeable_mask = HGATP_MODE | HGATP_VMID | HGATP_PPN;
+ if (wr_mask & HGATP_MODE) {
+ mode = (new_val & HGATP_MODE) >> HGATP_MODE_SHIFT;
+ switch (mode) {
+ /*
+ * We (intentionally) support only Sv39x4 on RV64
+ * and Sv32x4 on RV32 for guest G-stage so that
+ * software page table walks on guest G-stage is
+ * faster.
+ */
+#ifdef CONFIG_64BIT
+ case HGATP_MODE_SV39X4:
+ if (riscv_stage2_mode != HGATP_MODE_SV48X4 &&
+ riscv_stage2_mode != HGATP_MODE_SV39X4) {
+ mode = HGATP_MODE_OFF;
+ }
+ break;
+#else
+ case HGATP_MODE_SV32X4:
+ if (riscv_stage2_mode != HGATP_MODE_SV32X4) {
+ mode = HGATP_MODE_OFF;
+ }
+ break;
+#endif
+ default:
+ mode = HGATP_MODE_OFF;
+ break;
+ }
+ new_val &= ~HGATP_MODE;
+ new_val |= (mode << HGATP_MODE_SHIFT) & HGATP_MODE;
+ }
+ break;
+ case CSR_VSSTATUS:
+ csr = &npriv->vsstatus;
+ writeable_mask = SSTATUS_SIE | SSTATUS_SPIE | SSTATUS_UBE |
+ SSTATUS_SPP | SSTATUS_SUM | SSTATUS_MXR |
+ SSTATUS_FS | SSTATUS_UXL;
+ break;
+ case CSR_VSIP:
+ csr = &npriv->hvip;
+ csr_shift = 1;
+ writeable_mask = HVIP_VSSIP & npriv->hideleg;
+ break;
+ case CSR_VSIE:
+ csr = &npriv->vsie;
+ writeable_mask = VSIE_WRITEABLE & (npriv->hideleg >> 1);
+ break;
+ case CSR_VSTVEC:
+ csr = &npriv->vstvec;
+ writeable_mask = -1UL;
+ break;
+ case CSR_VSSCRATCH:
+ csr = &npriv->vsscratch;
+ writeable_mask = -1UL;
+ break;
+ case CSR_VSEPC:
+ csr = &npriv->vsepc;
+ writeable_mask = -1UL;
+ break;
+ case CSR_VSCAUSE:
+ csr = &npriv->vscause;
+ writeable_mask = 0x1fUL;
+ break;
+ case CSR_VSTVAL:
+ csr = &npriv->vstval;
+ writeable_mask = -1UL;
+ break;
+ case CSR_VSATP:
+ csr = &npriv->vsatp;
+ writeable_mask = SATP_MODE | SATP_ASID | SATP_PPN;
+ if (wr_mask & SATP_MODE) {
+ mode = (new_val & SATP_MODE) >> SATP_MODE_SHIFT;
+ switch (mode) {
+#ifdef CONFIG_64BIT
+ case SATP_MODE_SV48:
+ if (riscv_stage1_mode != SATP_MODE_SV48) {
+ mode = SATP_MODE_OFF;
+ }
+ break;
+ case SATP_MODE_SV39:
+ if (riscv_stage1_mode != SATP_MODE_SV48 &&
+ riscv_stage1_mode != SATP_MODE_SV39) {
+ mode = SATP_MODE_OFF;
+ }
+ break;
+#else
+ case SATP_MODE_SV32:
+ if (riscv_stage1_mode != SATP_MODE_SV32) {
+ mode = SATP_MODE_OFF;
+ }
+ break;
+#endif
+ default:
+ mode = SATP_MODE_OFF;
+ break;
+ }
+ new_val &= ~SATP_MODE;
+ new_val |= (mode << SATP_MODE_SHIFT) & SATP_MODE;
+ }
+ break;
+ case CSR_HVICTL:
+ csr = &npriv->hvictl;
+ writeable_mask = HVICTL_VTI | HVICTL_IID |
+ HVICTL_IPRIOM | HVICTL_IPRIO;
+ break;
+ default:
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ if (val) {
+ *val = (csr_shift < 0) ?
+ (*csr) << -csr_shift : (*csr) >> csr_shift;
+ }
+
+ if (read_only) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ } else {
+ writeable_mask = (csr_shift < 0) ?
+ writeable_mask >> -csr_shift :
+ writeable_mask << csr_shift;
+ wr_mask = (csr_shift < 0) ?
+ wr_mask >> -csr_shift : wr_mask << csr_shift;
+ new_val = (csr_shift < 0) ?
+ new_val >> -csr_shift : new_val << csr_shift;
+ wr_mask &= writeable_mask;
+ *csr = (*csr & ~wr_mask) | (new_val & wr_mask);
+ }
+
+ return VMM_OK;
+}
+
+int cpu_vcpu_nested_page_fault(struct vmm_vcpu *vcpu,
+ return VMM_OK;
+}
+
+ bool trap_from_smode,
+ const struct cpu_vcpu_trap *trap,
+ struct cpu_vcpu_trap *out_trap)
+{
+ /* TODO: */
+{
+ return VMM_OK;
+}
+
+void cpu_vcpu_nested_hfence_vvma(struct vmm_vcpu *vcpu,
+ unsigned long *vaddr, unsigned int *asid)
+{
+ /* TODO: */
+}
+
+void cpu_vcpu_nested_hfence_gvma(struct vmm_vcpu *vcpu,
+ physical_addr_t *gaddr, unsigned int *vmid)
+{
+ /* TODO: */
+}
+
+int cpu_vcpu_nested_hlv(struct vmm_vcpu *vcpu, unsigned long vaddr,
+ bool hlvx, void *data, unsigned long len,
+ unsigned long *out_scause,
+ unsigned long *out_stval,
+ unsigned long *out_htval)
+{
+ /* TODO: */
+ return VMM_OK;
+}
+
+int cpu_vcpu_nested_hsv(struct vmm_vcpu *vcpu, unsigned long vaddr,
+ const void *data, unsigned long len,
+ unsigned long *out_scause,
+ unsigned long *out_stval,
+ unsigned long *out_htval)
+{
+ /* TODO: */
+ return VMM_OK;
+}
+
+void cpu_vcpu_nested_set_virt(struct vmm_vcpu *vcpu, struct arch_regs *regs,
+ enum nested_set_virt_event event, bool virt,
+ bool spvp, bool gva)
+{
+ unsigned long tmp;
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ /* If H-extension is not available for VCPU then do nothing */
+ if (!riscv_isa_extension_available(riscv_priv(vcpu)->isa, h)) {
+ return;
+ }
+
+ /* Skip hardware CSR update if no change in virt state */
+ if (virt == npriv->virt)
+ goto skip_csr_update;
+
+ /* Swap hcounteren and hedeleg CSRs */
+ npriv->hcounteren = csr_swap(CSR_HCOUNTEREN, npriv->hcounteren);
+ npriv->hedeleg = csr_swap(CSR_HEDELEG, npriv->hedeleg);
+
+ /* Update interrupt delegation */
+ cpu_vcpu_irq_deleg_update(vcpu, virt);
+
+ /* Update time delta */
+ cpu_vcpu_time_delta_update(vcpu, virt);
+
+ /* Update G-stage page table */
+ cpu_vcpu_gstage_update(vcpu, virt);
+
+ /* Swap hardware vs<xyz> CSRs except vsie and vsstatus */
+ npriv->vstvec = csr_swap(CSR_VSTVEC, npriv->vstvec);
+ npriv->vsscratch = csr_swap(CSR_VSSCRATCH, npriv->vsscratch);
+ npriv->vsepc = csr_swap(CSR_VSEPC, npriv->vsepc);
+ npriv->vscause = csr_swap(CSR_VSCAUSE, npriv->vscause);
+ npriv->vstval = csr_swap(CSR_VSTVAL, npriv->vstval);
+ npriv->vsatp = csr_swap(CSR_VSATP, npriv->vsatp);
+
+ /* Update vsstatus CSR */
+ if (virt) {
+ /* Nested virtualization state changing from OFF to ON */
+
+ /*
+ * Update vsstatus in following manner:
+ * 1) Swap hardware vsstatus (i.e. virtual-HS mode sstatus)
+ * with vsstatus in nested virtualization context (i.e.
+ * virtual-VS mode sstatus)
+ * 2) Swap host sstatus.FS (i.e. HS mode sstatus.FS) with
+ * the vsstatus.FS saved in nested virtualization context
+ * (i.e. virtual-HS mode sstatus.FS)
+ */
+ npriv->vsstatus = csr_swap(CSR_VSSTATUS, npriv->vsstatus);
+ tmp = regs->sstatus & SSTATUS_FS;
+ regs->sstatus &= ~SSTATUS_FS;
+ regs->sstatus |= (npriv->vsstatus & SSTATUS_FS);
+ npriv->vsstatus &= ~SSTATUS_FS;
+ npriv->vsstatus |= tmp;
+ } else {
+ /* Nested virtualization state changing from ON to OFF */
+
+ /*
+ * Update vsstatus in following manner:
+ * 1) Swap host sstatus.FS (i.e. virtual-HS mode sstatus.FS)
+ * with vsstatus.FS saved in the nested virtualization
+ * context (i.e. HS mode sstatus.FS)
+ * 2) Swap hardware vsstatus (i.e. virtual-VS mode sstatus)
+ * with vsstatus in nested virtualization context (i.e.
+ * virtual-HS mode sstatus)
+ */
+ tmp = regs->sstatus & SSTATUS_FS;
+ regs->sstatus &= ~SSTATUS_FS;
+ regs->sstatus |= (npriv->vsstatus & SSTATUS_FS);
+ npriv->vsstatus &= ~SSTATUS_FS;
+ npriv->vsstatus |= tmp;
+ npriv->vsstatus = csr_swap(CSR_VSSTATUS, npriv->vsstatus);
+ }
+
+skip_csr_update:
+ if (event != NESTED_SET_VIRT_EVENT_SRET) {
+ /* Update Guest hstatus.SPV bit */
+ npriv->hstatus &= ~HSTATUS_SPV;
+ npriv->hstatus |= (npriv->virt) ? HSTATUS_SPV : 0;
+
+ /* Update Guest hstatus.SPVP bit */
+ if (npriv->virt) {
+ npriv->hstatus &= ~HSTATUS_SPVP;
+ if (spvp)
+ npriv->hstatus |= HSTATUS_SPVP;
+ }
+
+ /* Update Guest hstatus.GVA bit */
+ if (event == NESTED_SET_VIRT_EVENT_TRAP) {
+ npriv->hstatus &= ~HSTATUS_GVA;
+ npriv->hstatus |= (gva) ? HSTATUS_GVA : 0;
+ }
+ }
+
+ /* Update host SRET and VM trapping */
+ regs->hstatus &= ~HSTATUS_VTSR;
+ if (virt && (npriv->hstatus & HSTATUS_VTSR)) {
+ regs->hstatus |= HSTATUS_VTSR;
+ }
+ regs->hstatus &= ~HSTATUS_VTVM;
+ if (virt && (npriv->hstatus & HSTATUS_VTVM)) {
+ regs->hstatus |= HSTATUS_VTVM;
+ }
+
+ /* Update virt flag */
+ npriv->virt = virt;
+}
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_sbi.c b/arch/riscv/cpu/generic/cpu_vcpu_sbi.c
index fbd6a8f6..9788150e 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_sbi.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_sbi.c
@@ -74,6 +74,17 @@ int cpu_vcpu_sbi_ecall(struct vmm_vcpu *vcpu, ulong cause,
bool is_0_1_spec = FALSE;
unsigned long args[6];
+ /* Forward SBI calls from virtual-VS mode to virtual-HS mode */
+ if (riscv_nested_virt(vcpu)) {
+ trap.sepc = regs->sepc;
+ trap.scause = CAUSE_VIRTUAL_SUPERVISOR_ECALL;
+ trap.stval = 0;
+ trap.htval = 0;
+ trap.htinst = 0;
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ return VMM_OK;
+ }
+
args[0] = regs->a0;
args[1] = regs->a1;
args[2] = regs->a2;
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_trap.c b/arch/riscv/cpu/generic/cpu_vcpu_trap.c
index 953e6e4b..e990ed4a 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_trap.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_trap.c
@@ -30,46 +30,86 @@
#include <libs/stringlib.h>
#include <generic_mmu.h>
-#include <cpu_vcpu_csr.h>
+#include <cpu_hwcap.h>
+#include <cpu_vcpu_nested.h>
#include <cpu_vcpu_trap.h>
#include <cpu_vcpu_unpriv.h>
-int cpu_vcpu_redirect_trap(struct vmm_vcpu *vcpu,
- arch_regs_t *regs,
- struct cpu_vcpu_trap *trap)
+void cpu_vcpu_redirect_smode_trap(arch_regs_t *regs,
+ struct cpu_vcpu_trap *trap, bool prev_spp)
{
+ /* Read Guest sstatus */
unsigned long vsstatus = csr_read(CSR_VSSTATUS);
- /* Change Guest SSTATUS.SPP bit */
+ /* Change Guest sstatus.SPP bit */
vsstatus &= ~SSTATUS_SPP;
- if (regs->sstatus & SSTATUS_SPP)
+ if (prev_spp)
vsstatus |= SSTATUS_SPP;
- /* Change Guest SSTATUS.SPIE bit */
+ /* Change Guest sstatus.SPIE bit */
vsstatus &= ~SSTATUS_SPIE;
if (vsstatus & SSTATUS_SIE)
vsstatus |= SSTATUS_SPIE;
- /* Clear Guest SSTATUS.SIE bit */
+ /* Clear Guest sstatus.SIE bit */
vsstatus &= ~SSTATUS_SIE;
- /* Update Guest SSTATUS */
+ /* Update Guest sstatus */
csr_write(CSR_VSSTATUS, vsstatus);
- /* Update Guest SCAUSE, STVAL, and SEPC */
+ /* Update Guest scause, stval, and sepc */
csr_write(CSR_VSCAUSE, trap->scause);
csr_write(CSR_VSTVAL, trap->stval);
csr_write(CSR_VSEPC, trap->sepc);
- /* Set Guest PC to Guest exception vector */
+ /* Set next PC to exception vector */
regs->sepc = csr_read(CSR_VSTVEC);
- return 0;
+ /* Set next privilege mode to supervisor */
+ regs->sstatus |= SSTATUS_SPP;
+}
+
+void cpu_vcpu_redirect_trap(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ struct cpu_vcpu_trap *trap)
+{
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+ bool prev_spp = (regs->sstatus & SSTATUS_SPP) ? TRUE : FALSE;
+ bool gva = FALSE;
+
+ /* Determine GVA bit state */
+ switch (trap->scause) {
+ case CAUSE_MISALIGNED_FETCH:
+ case CAUSE_FETCH_ACCESS:
+ case CAUSE_MISALIGNED_LOAD:
+ case CAUSE_LOAD_ACCESS:
+ case CAUSE_MISALIGNED_STORE:
+ case CAUSE_STORE_ACCESS:
+ case CAUSE_FETCH_PAGE_FAULT:
+ case CAUSE_LOAD_PAGE_FAULT:
+ case CAUSE_STORE_PAGE_FAULT:
+ case CAUSE_FETCH_GUEST_PAGE_FAULT:
+ case CAUSE_LOAD_GUEST_PAGE_FAULT:
+ case CAUSE_STORE_GUEST_PAGE_FAULT:
+ gva = TRUE;
+ break;
+ default:
+ break;
+ }
+
+ /* Turn-off nested virtualization for virtual-HS mode */
+ cpu_vcpu_nested_set_virt(vcpu, regs, NESTED_SET_VIRT_EVENT_TRAP,
+ FALSE, prev_spp, gva);
+
+ /* Update Guest HTVAL and HTINST */
+ npriv->htval = trap->htval;
+ npriv->htinst = trap->htinst;
+
+ /* Update Guest supervisor state */
+ cpu_vcpu_redirect_smode_trap(regs, trap, prev_spp);
}
static int cpu_vcpu_stage2_map(struct vmm_vcpu *vcpu,
- arch_regs_t *regs,
- physical_addr_t fault_addr)
+ physical_addr_t fault_addr)
{
int rc, rc1;
u32 reg_flags = 0x0, pg_reg_flags = 0x0;
@@ -160,9 +200,9 @@ static int cpu_vcpu_emulate_load(struct vmm_vcpu *vcpu,
u16 data16;
u32 data32;
u64 data64;
+ struct cpu_vcpu_trap trap;
unsigned long insn, insn_len;
int rc = VMM_OK, shift = 0, len = 0;
- struct cpu_vcpu_trap trap = { 0 };
if (htinst & 0x1) {
/*
@@ -176,12 +216,18 @@ static int cpu_vcpu_emulate_load(struct vmm_vcpu *vcpu,
* Bit[0] == 0 implies trapped instruction value is
* zero or special value.
*/
+ trap.sepc = 0;
+ trap.scause = 0;
+ trap.stval = 0;
+ trap.htval = 0;
+ trap.htinst = 0;
insn = __cpu_vcpu_unpriv_read_insn(regs->sepc, &trap);
if (trap.scause) {
if (trap.scause == CAUSE_LOAD_PAGE_FAULT)
trap.scause = CAUSE_FETCH_PAGE_FAULT;
trap.sepc = trap.stval = regs->sepc;
- return cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ return VMM_OK;
}
insn_len = INSN_LEN(insn);
}
@@ -287,8 +333,8 @@ static int cpu_vcpu_emulate_store(struct vmm_vcpu *vcpu,
u32 data32;
u64 data64;
int rc = VMM_OK, len = 0;
+ struct cpu_vcpu_trap trap;
unsigned long data, insn, insn_len;
- struct cpu_vcpu_trap trap = { 0 };
if (htinst & 0x1) {
/*
@@ -298,16 +344,22 @@ static int cpu_vcpu_emulate_store(struct vmm_vcpu *vcpu,
insn = htinst | INSN_16BIT_MASK;
insn_len = (htinst & 0x2) ? INSN_LEN(insn) : 2;
} else {
- /*
+ /*
* Bit[0] == 0 implies trapped instruction value is
* zero or special value.
*/
+ trap.sepc = 0;
+ trap.scause = 0;
+ trap.stval = 0;
+ trap.htval = 0;
+ trap.htinst = 0;
insn = __cpu_vcpu_unpriv_read_insn(regs->sepc, &trap);
if (trap.scause) {
if (trap.scause == CAUSE_LOAD_PAGE_FAULT)
trap.scause = CAUSE_FETCH_PAGE_FAULT;
trap.sepc = trap.stval = regs->sepc;
- return cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ return VMM_OK;
}
insn_len = INSN_LEN(insn);
}
@@ -384,9 +436,31 @@ int cpu_vcpu_page_fault(struct vmm_vcpu *vcpu,
arch_regs_t *regs,
struct cpu_vcpu_trap *trap)
{
+ int rc;
struct vmm_region *reg;
+ struct cpu_vcpu_trap otrap;
physical_addr_t fault_addr;
+ if (riscv_nested_virt(vcpu)) {
+ otrap.sepc = 0;
+ otrap.scause = 0;
+ otrap.stval = 0;
+ otrap.htval = 0;
+ otrap.htinst = 0;
+ rc = cpu_vcpu_nested_page_fault(vcpu,
+ (regs->hstatus & HSTATUS_SPVP) ? TRUE : FALSE,
+ trap, &otrap);
+ if (rc) {
+ return rc;
+ }
+
+ if (otrap.scause) {
+ cpu_vcpu_redirect_trap(vcpu, regs, &otrap);
+ }
+
+ return VMM_OK;
+ }
+
fault_addr = ((physical_addr_t)trap->htval << 2);
+ return VMM_OK;
+ }
+
fault_addr |= ((physical_addr_t)trap->stval & 0x3);
@@ -397,18 +471,18 @@ int cpu_vcpu_page_fault(struct vmm_vcpu *vcpu,
/* Emulate load/store instructions for virtual device */
switch (trap->scause) {
case CAUSE_LOAD_GUEST_PAGE_FAULT:
- return cpu_vcpu_emulate_load(vcpu, regs,
- fault_addr, trap->htinst);
+ return cpu_vcpu_emulate_load(vcpu, regs, fault_addr,
+ trap->htinst);
case CAUSE_STORE_GUEST_PAGE_FAULT:
- return cpu_vcpu_emulate_store(vcpu, regs,
- fault_addr, trap->htinst);
+ return cpu_vcpu_emulate_store(vcpu, regs, fault_addr,
+ trap->htinst);
default:
return VMM_ENOTSUPP;
};
}
/* Mapping does not exist hence create one */
- return cpu_vcpu_stage2_map(vcpu, regs, fault_addr);
+ return cpu_vcpu_stage2_map(vcpu, fault_addr);
}
static int truly_illegal_insn(struct vmm_vcpu *vcpu,
@@ -421,75 +495,788 @@ static int truly_illegal_insn(struct vmm_vcpu *vcpu,
trap.sepc = regs->sepc;
trap.scause = CAUSE_ILLEGAL_INSTRUCTION;
trap.stval = insn;
- return cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ trap.htval = 0;
+ trap.htinst = 0;
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+
+ return VMM_OK;
}
-static int system_opcode_insn(struct vmm_vcpu *vcpu,
+static int truly_virtual_insn(struct vmm_vcpu *vcpu,
arch_regs_t *regs,
ulong insn)
{
- int rc = VMM_OK, do_write, rs1_num;
- ulong rs1_val, csr_num, csr_val, new_csr_val;
+ struct cpu_vcpu_trap trap;
- if ((insn & INSN_MASK_WFI) == INSN_MATCH_WFI) {
- /* Wait for irq with default timeout */
- vmm_vcpu_irq_wait_timeout(vcpu, 0);
- goto done;
- }
+ /* Redirect trap to Guest VCPU */
+ trap.sepc = regs->sepc;
+ trap.scause = CAUSE_VIRTUAL_INST_FAULT;
+ trap.stval = insn;
+ trap.htval = 0;
+ trap.htinst = 0;
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
- rs1_num = (insn >> 15) & 0x1f;
- rs1_val = GET_RS1(insn, regs);
- csr_num = insn >> 20;
+ return VMM_OK;
+}
- rc = cpu_vcpu_csr_read(vcpu, csr_num, &csr_val);
- if (rc == VMM_EINVALID) {
- return truly_illegal_insn(vcpu, regs, insn);
- }
- if (rc) {
- return rc;
- }
+struct system_opcode_func {
+ ulong mask;
+ ulong match;
+ /*
+ * Possible return values are as follows:
+ * 1) Returns < 0 for error case
+ * 2) Return == 0 to increment PC and continue
+ * 3) Return == 1 to inject illegal instruction trap and continue
+ * 4) Return == 2 to inject virtual instruction trap and continue
+ * 5) Return == 3 to do nothing and continue
+ */
+ int (*func)(struct vmm_vcpu *vcpu, arch_regs_t *regs, ulong insn);
+};
- do_write = rs1_num;
+struct csr_func {
+ unsigned int csr_num;
+ /*
+ * Possible return values are as follows:
+ * 1) Returns < 0 for error case
+ * 2) Return == 0 to increment PC and continue
+ * 3) Return == 1 to inject illegal instruction trap and continue
+ * 4) Return == 2 to inject virtual instruction trap and continue
+ * 5) Return == 3 to do nothing and continue
+ */
+ int (*rmw_func)(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ unsigned int csr_num, unsigned long *val,
+ unsigned long new_val, unsigned long wr_mask);
+};
+
+static const struct csr_func csr_funcs[] = {
+ {
+ .csr_num = CSR_SIE,
+ .rmw_func = cpu_vcpu_nested_smode_csr_rmw,
+ },
+ {
+ .csr_num = CSR_SIEH,
+ .rmw_func = cpu_vcpu_nested_smode_csr_rmw,
+ },
+ {
+ .csr_num = CSR_SIP,
+ .rmw_func = cpu_vcpu_nested_smode_csr_rmw,
+ },
+ {
+ .csr_num = CSR_SIPH,
+ .rmw_func = cpu_vcpu_nested_smode_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HSTATUS,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HEDELEG,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HIDELEG,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HVIP,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HIE,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HIP,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HGEIP,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HGEIE,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HCOUNTEREN,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HTIMEDELTA,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HTIMEDELTAH,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HTVAL,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HTINST,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HGATP,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSSTATUS,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSIP,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSIE,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSTVEC,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSSCRATCH,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSEPC,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSCAUSE,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSTVAL,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_VSATP,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+};
+
+static int csr_insn(struct vmm_vcpu *vcpu, arch_regs_t *regs, ulong insn)
+{
+ int i, rc = TRAP_RETURN_ILLEGAL_INSN;
+ unsigned int csr_num = insn >> SH_RS2;
+ unsigned int rs1_num = (insn >> SH_RS1) & MASK_RX;
+ ulong rs1_val = GET_RS1(insn, regs);
+ const struct csr_func *tcfn, *cfn = NULL;
+ ulong val = 0, wr_mask = 0, new_val = 0;
+
+ /* Decode the CSR instruction */
switch (GET_RM(insn)) {
case 1:
- new_csr_val = rs1_val;
- do_write = 1;
+ wr_mask = -1UL;
+ new_val = rs1_val;
break;
case 2:
- new_csr_val = csr_val | rs1_val;
+ wr_mask = rs1_val;
+ new_val = -1UL;
break;
- case 3: new_csr_val = csr_val & ~rs1_val;
+ case 3:
+ wr_mask = rs1_val;
+ new_val = 0;
break;
case 5:
- new_csr_val = rs1_num;
- do_write = 1;
+ wr_mask = -1UL;
+ new_val = rs1_num;
break;
case 6:
- new_csr_val = csr_val | rs1_num;
+ wr_mask = rs1_num;
+ new_val = -1UL;
break;
case 7:
- new_csr_val = csr_val & ~rs1_num;
+ wr_mask = rs1_num;
+ new_val = 0;
break;
default:
- return truly_illegal_insn(vcpu, regs, insn);
- };
+ return rc;
+ }
- if (do_write) {
- rc = cpu_vcpu_csr_write(vcpu, csr_num, new_csr_val);
- if (rc == VMM_EINVALID) {
- return truly_illegal_insn(vcpu, regs, insn);
+ /* Find CSR function */
+ for (i = 0; i < array_size(csr_funcs); i++) {
+ tcfn = &csr_funcs[i];
+ if (tcfn->csr_num == csr_num) {
+ cfn = tcfn;
+ break;
}
- if (rc) {
- return rc;
+ }
+ if (!cfn || !cfn->rmw_func) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ /* Do CSR emulation */
+ rc = cfn->rmw_func(vcpu, regs, csr_num, &val, new_val, wr_mask);
+ if (rc) {
+ return rc;
+ }
+
+ /* Update destination register for CSR reads */
+ if ((insn >> SH_RD) & MASK_RX) {
+ SET_RD(insn, regs, val);
+ }
+
+ return VMM_OK;
+}
+
+static int sret_insn(struct vmm_vcpu *vcpu, arch_regs_t *regs, ulong insn)
+ return VMM_OK;
+}
+
+{
+ bool next_virt;
+ unsigned long vsstatus, next_sepc, next_spp;
+
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded to
+ * virtual-HS mode as a virtual instruction trap.
+ */
+ if (riscv_nested_virt(vcpu)) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-U mode should be forwarded to virtual-HS mode
+ * as illegal instruction trap.
+ */
+ if (!(regs->hstatus & HSTATUS_SPVP)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+ vsstatus = csr_read(CSR_VSSTATUS);
+
+ /*
+ * Find next nested virtualization mode, next privilege mode,
+ * and next sepc
+ */
+ next_virt = (riscv_nested_priv(vcpu)->hstatus & HSTATUS_SPV) ?
+ TRUE : FALSE;
+ next_sepc = csr_read(CSR_VSEPC);
+ next_spp = vsstatus & SSTATUS_SPP;
+
+ /* Update Guest sstatus.sie */
+ vsstatus &= ~SSTATUS_SIE;
+ vsstatus |= (vsstatus & SSTATUS_SPIE) ? SSTATUS_SIE : 0;
+ csr_write(CSR_VSSTATUS, vsstatus);
+
+ /* Update return address and return privilege mode*/
+ regs->sepc = next_sepc;
+ regs->sstatus &= ~SSTATUS_SPP;
+ regs->sstatus |= next_spp;
+
+ /* Set nested virtualization state based on guest hstatus.SPV */
+ cpu_vcpu_nested_set_virt(vcpu, regs, NESTED_SET_VIRT_EVENT_SRET,
+ next_virt, FALSE, FALSE);
+
+ return TRAP_RETURN_CONTINUE;
+}
+
+static int wfi_insn(struct vmm_vcpu *vcpu, arch_regs_t *regs, ulong insn)
+{
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded to
+ * virtual-HS mode as a virtual instruction trap.
+ */
+ if (riscv_nested_virt(vcpu)) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-U mode should be forwarded to virtual-HS mode
+ * as illegal instruction trap.
+ */
+ if (!(regs->hstatus & HSTATUS_SPVP)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ /* Wait for irq with default timeout */
+ vmm_vcpu_irq_wait_timeout(vcpu, 0);
+ return VMM_OK;
+}
+
+static int hfence_vvma_insn(struct vmm_vcpu *vcpu,
+ arch_regs_t *regs, ulong insn)
+{
+ unsigned int rs1_num = (insn >> SH_RS1) & MASK_RX;
+ unsigned int rs2_num = (insn >> SH_RS2) & MASK_RX;
+ unsigned long vaddr = GET_RS1(insn, regs);
+ unsigned int asid = GET_RS2(insn, regs);
+
+ /*
+ * If H-extension is not available for VCPU then forward trap
+ * as illegal instruction trap to virtual-HS mode.
+ */
+ if (!riscv_isa_extension_available(riscv_priv(vcpu)->isa, h)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded to
+ * virtual-HS mode as a virtual instruction trap.
+ */
+ if (riscv_nested_virt(vcpu)) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-U mode should be forwarded to virtual-HS mode
+ * as illegal instruction trap.
+ */
+ if (!(regs->hstatus & HSTATUS_SPVP)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ if (!rs1_num && !rs2_num) {
+ cpu_vcpu_nested_hfence_vvma(vcpu, NULL, NULL);
+ } else if (!rs1_num && rs2_num) {
+ cpu_vcpu_nested_hfence_vvma(vcpu, NULL, &asid);
+ } else if (rs1_num && !rs2_num) {
+ cpu_vcpu_nested_hfence_vvma(vcpu, &vaddr, NULL);
+ } else {
+ cpu_vcpu_nested_hfence_vvma(vcpu, &vaddr, &asid);
+ }
+
+ return VMM_OK;
+}
+
+static int hfence_gvma_insn(struct vmm_vcpu *vcpu,
+ return VMM_OK;
+}
+
+ arch_regs_t *regs, ulong insn)
+{
+ unsigned int rs1_num = (insn >> SH_RS1) & MASK_RX;
+ unsigned int rs2_num = (insn >> SH_RS2) & MASK_RX;
+ physical_addr_t gaddr = GET_RS1(insn, regs) << 2;
+ unsigned int vmid = GET_RS2(insn, regs);
+
+ /*
+ * If H-extension is not available for VCPU then forward trap
+ * as illegal instruction trap to virtual-HS mode.
+ */
+ if (!riscv_isa_extension_available(riscv_priv(vcpu)->isa, h)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded to
+ * virtual-HS mode as a virtual instruction trap.
+ */
+ if (riscv_nested_virt(vcpu)) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-U mode should be forwarded to virtual-HS mode
+ * as illegal instruction trap.
+ */
+ if (!(regs->hstatus & HSTATUS_SPVP)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ if (!rs1_num && !rs2_num) {
+ cpu_vcpu_nested_hfence_gvma(vcpu, NULL, NULL);
+ } else if (!rs1_num && rs2_num) {
+ cpu_vcpu_nested_hfence_gvma(vcpu, NULL, &vmid);
+ } else if (rs1_num && !rs2_num) {
+ cpu_vcpu_nested_hfence_gvma(vcpu, &gaddr, NULL);
+ } else {
+ cpu_vcpu_nested_hfence_gvma(vcpu, &gaddr, &vmid);
+ }
+
+ return VMM_OK;
+}
+
+union hxv_reg_data {
+ return VMM_OK;
+}
+
+ ulong data_ulong;
+ u64 data_u64;
+ u32 data_u32;
+ u16 data_u16;
+ u8 data_u8;
+};
+
+static int hlv_insn(struct vmm_vcpu *vcpu, arch_regs_t *regs, ulong insn)
+{
+ int rc;
+ void *data;
+ bool hlvx = FALSE;
+ union hxv_reg_data v;
+ struct cpu_vcpu_trap trap;
+ unsigned long shift = 0, len, vaddr;
+
+ /*
+ * If H-extension is not available for VCPU then forward trap
+ * as illegal instruction trap to virtual-HS mode.
+ */
+ if (!riscv_isa_extension_available(riscv_priv(vcpu)->isa, h)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded to
+ * virtual-HS mode as a virtual instruction trap.
+ */
+ if (riscv_nested_virt(vcpu)) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-U mode should be forwarded to virtual-HS mode
+ * as illegal instruction trap when guest hstatus.HU == 0.
+ */
+ if (!(regs->hstatus & HSTATUS_SPVP) &&
+ !(riscv_nested_priv(vcpu)->hstatus & HSTATUS_HU)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ vaddr = GET_RS1(insn, regs);
+ v.data_u64 = 0;
+
+ if ((insn & INSN_MASK_HLV_B) == INSN_MATCH_HLV_B) {
+ data = &v.data_u8;
+ len = 1;
+ shift = (sizeof(long) - 1) * 8;
+ } else if ((insn & INSN_MASK_HLV_BU) == INSN_MATCH_HLV_BU) {
+ data = &v.data_u8;
+ len = 1;
+ } else if ((insn & INSN_MASK_HLV_H) == INSN_MATCH_HLV_H) {
+ data = &v.data_u16;
+ len = 2;
+ shift = (sizeof(long) - 2) * 8;
+ } else if ((insn & INSN_MASK_HLV_HU) == INSN_MATCH_HLV_HU) {
+ data = &v.data_u16;
+ len = 2;
+ } else if ((insn & INSN_MASK_HLVX_HU) == INSN_MATCH_HLVX_HU) {
+ data = &v.data_u16;
+ len = 2;
+ hlvx = TRUE;
+ } else if ((insn & INSN_MASK_HLV_W) == INSN_MATCH_HLV_W) {
+ data = &v.data_u32;
+ len = 4;
+ shift = (sizeof(long) - 4) * 8;
+ } else if ((insn & INSN_MASK_HLV_WU) == INSN_MATCH_HLV_WU) {
+ data = &v.data_u32;
+ len = 4;
+ } else if ((insn & INSN_MASK_HLVX_WU) == INSN_MATCH_HLVX_WU) {
+ data = &v.data_u32;
+ len = 4;
+ hlvx = TRUE;
+ } else if ((insn & INSN_MASK_HLV_D) == INSN_MATCH_HLV_D) {
+ data = &v.data_u64;
+ len = 8;
+ } else {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ trap.sepc = regs->sepc;
+ trap.scause = 0;
+ trap.stval = 0;
+ trap.htval = 0;
+ trap.htinst = insn;
+
+ rc = cpu_vcpu_nested_hlv(vcpu, vaddr, hlvx, data, len,
+ &trap.scause, &trap.stval, &trap.htval);
+ if (rc) {
+ return rc;
+ }
+
+ if (trap.scause) {
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ return TRAP_RETURN_CONTINUE;
+ } else {
+ SET_RD(insn, regs, ((long)(v.data_ulong << shift)) >> shift);
+ }
+
+ return VMM_OK;
+}
+
+static int hsv_insn(struct vmm_vcpu *vcpu, arch_regs_t *regs, ulong insn)
+ return VMM_OK;
+}
+
+{
+ int rc;
+ void *data;
+ union hxv_reg_data v;
+ unsigned long len, vaddr;
+ struct cpu_vcpu_trap trap;
+
+ /*
+ * If H-extension is not available for VCPU then forward trap
+ * as illegal instruction trap to virtual-HS mode.
+ */
+ if (!riscv_isa_extension_available(riscv_priv(vcpu)->isa, h)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded to
+ * virtual-HS mode as a virtual instruction trap.
+ */
+ if (riscv_nested_virt(vcpu)) {
+ return TRAP_RETURN_VIRTUAL_INSN;
+ }
+
+ /*
+ * Trap from virtual-U mode should be forwarded to virtual-HS mode
+ * as illegal instruction trap when guest hstatus.HU == 0.
+ */
+ if (!(regs->hstatus & HSTATUS_SPVP) &&
+ !(riscv_nested_priv(vcpu)->hstatus & HSTATUS_HU)) {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ vaddr = GET_RS1(insn, regs);
+ v.data_ulong = GET_RS2(insn, regs);
+
+ if ((insn & INSN_MASK_HSV_B) == INSN_MATCH_HSV_B) {
+ data = &v.data_u8;
+ len = 1;
+ } else if ((insn & INSN_MASK_HSV_H) == INSN_MATCH_HSV_H) {
+ data = &v.data_u16;
+ len = 2;
+ } else if ((insn & INSN_MASK_HSV_W) == INSN_MATCH_HSV_W) {
+ data = &v.data_u32;
+ len = 4;
+ } else if ((insn & INSN_MASK_HSV_D) == INSN_MATCH_HSV_D) {
+ data = &v.data_u64;
+ len = 8;
+ } else {
+ return TRAP_RETURN_ILLEGAL_INSN;
+ }
+
+ trap.sepc = regs->sepc;
+ trap.scause = 0;
+ trap.stval = 0;
+ trap.htval = 0;
+ trap.htinst = insn;
+
+ rc = cpu_vcpu_nested_hsv(vcpu, vaddr, data, len,
+ &trap.scause, &trap.stval, &trap.htval);
+ if (rc) {
+ return rc;
+ }
+
+ if (trap.scause) {
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ return TRAP_RETURN_CONTINUE;
+ }
+
+ return VMM_OK;
+}
+
+static const struct system_opcode_func system_opcode_funcs[] = {
+ return VMM_OK;
+}
+
+ {
+ .mask = INSN_MASK_CSRRW,
+ .match = INSN_MATCH_CSRRW,
+ .func = csr_insn,
+ },
+ {
+ .mask = INSN_MASK_CSRRS,
+ .match = INSN_MATCH_CSRRS,
+ .func = csr_insn,
+ },
+ {
+ .mask = INSN_MASK_CSRRC,
+ .match = INSN_MATCH_CSRRC,
+ .func = csr_insn,
+ },
+ {
+ .mask = INSN_MASK_CSRRWI,
+ .match = INSN_MATCH_CSRRWI,
+ .func = csr_insn,
+ },
+ {
+ .mask = INSN_MASK_CSRRSI,
+ .match = INSN_MATCH_CSRRSI,
+ .func = csr_insn,
+ },
+ {
+ .mask = INSN_MASK_CSRRCI,
+ .match = INSN_MATCH_CSRRCI,
+ .func = csr_insn,
+ },
+ {
+ .mask = INSN_MASK_SRET,
+ .match = INSN_MATCH_SRET,
+ .func = sret_insn,
+ },
+ {
+ .mask = INSN_MASK_WFI,
+ .match = INSN_MATCH_WFI,
+ .func = wfi_insn,
+ },
+ {
+ .mask = INSN_MASK_HFENCE_VVMA,
+ .match = INSN_MATCH_HFENCE_VVMA,
+ .func = hfence_vvma_insn,
+ },
+ {
+ .mask = INSN_MASK_HFENCE_GVMA,
+ .match = INSN_MATCH_HFENCE_GVMA,
+ .func = hfence_gvma_insn,
+ },
+ {
+ .mask = INSN_MASK_HLV_B,
+ .match = INSN_MATCH_HLV_B,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HLV_BU,
+ .match = INSN_MATCH_HLV_BU,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HLV_H,
+ .match = INSN_MATCH_HLV_H,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HLV_HU,
+ .match = INSN_MATCH_HLV_HU,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HLVX_HU,
+ .match = INSN_MATCH_HLVX_HU,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HLV_W,
+ .match = INSN_MATCH_HLV_W,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HLV_WU,
+ .match = INSN_MATCH_HLV_WU,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HLVX_WU,
+ .match = INSN_MATCH_HLVX_WU,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HLV_D,
+ .match = INSN_MATCH_HLV_D,
+ .func = hlv_insn,
+ },
+ {
+ .mask = INSN_MASK_HSV_B,
+ .match = INSN_MATCH_HSV_B,
+ .func = hsv_insn,
+ },
+ {
+ .mask = INSN_MASK_HSV_H,
+ .match = INSN_MATCH_HSV_H,
+ .func = hsv_insn,
+ },
+ {
+ .mask = INSN_MASK_HSV_W,
+ .match = INSN_MATCH_HSV_W,
+ .func = hsv_insn,
+ },
+ {
+ .mask = INSN_MASK_HSV_D,
+ .match = INSN_MATCH_HSV_D,
+ .func = hsv_insn,
+ },
+};
+
+static int system_opcode_insn(struct vmm_vcpu *vcpu,
+ arch_regs_t *regs,
+ ulong insn)
+{
+ int i, rc = TRAP_RETURN_ILLEGAL_INSN;
+ const struct system_opcode_func *ifn;
+
+ for (i = 0; i < array_size(system_opcode_funcs); i++) {
+ ifn = &system_opcode_funcs[i];
+ if ((insn & ifn->mask) == ifn->match) {
+ rc = ifn->func(vcpu, regs, insn);
+ break;
}
}
- SET_RD(insn, regs, csr_val);
+ if (rc == TRAP_RETURN_ILLEGAL_INSN)
+ return truly_illegal_insn(vcpu, regs, insn);
+ else if (rc == TRAP_RETURN_VIRTUAL_INSN)
+ return truly_virtual_insn(vcpu, regs, insn);
-done:
- regs->sepc += 4;
+ if (!rc) {
+ regs->sepc += INSN_LEN(insn);
+ }
- return rc;
+ return (rc < 0) ? rc : VMM_OK;
+}
+
+int cpu_vcpu_general_fault(struct vmm_vcpu *vcpu,
+ arch_regs_t *regs, struct cpu_vcpu_trap *trap)
+{
+ struct cpu_vcpu_trap itrap = { 0 };
+
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded
+ * to virtual-HS mode.
+ */
+
+ if (!riscv_nested_virt(vcpu)) {
+ return VMM_EINVALID;
+ }
+
+ /*
+ * Blindly forward all general faults to virtual-HS mode
+ * except illegal instruction fault
+ */
+ if (trap->scause != CAUSE_ILLEGAL_INSTRUCTION) {
+ cpu_vcpu_redirect_trap(vcpu, regs, trap);
+ return VMM_OK;
+ }
+
+ /* Update trap->stval for illegal instruction fault */
+ if (unlikely((trap->stval & INSN_16BIT_MASK) != INSN_16BIT_MASK)) {
+ if (trap->stval == 0) {
+ trap->stval = __cpu_vcpu_unpriv_read_insn(regs->sepc,
+ &itrap);
+ if (itrap.scause) {
+ if (itrap.scause == CAUSE_LOAD_PAGE_FAULT)
+ itrap.scause = CAUSE_FETCH_PAGE_FAULT;
+ itrap.stval = itrap.sepc = regs->sepc;
+ cpu_vcpu_redirect_trap(vcpu, regs, &itrap);
+ return VMM_OK;
+ }
+ }
+ }
+
+ /* Forward illegal instruction fault */
+ return truly_illegal_insn(vcpu, regs, trap->stval);
+}
+
+int cpu_vcpu_illegal_insn_fault(struct vmm_vcpu *vcpu,
+ arch_regs_t *regs,
+ unsigned long stval)
+{
+ unsigned long insn = stval;
+ struct cpu_vcpu_trap trap = { 0 };
+
+ /*
+ * Trap from virtual-VS and virtual-VU modes should be forwarded to
+ * virtual-HS mode as a illegal instruction trap.
+ */
+
+ if (!riscv_nested_virt(vcpu)) {
+ return VMM_EINVALID;
+ }
+
+ if (unlikely((insn & INSN_16BIT_MASK) != INSN_16BIT_MASK)) {
+ if (insn == 0) {
+ insn = __cpu_vcpu_unpriv_read_insn(regs->sepc, &trap);
+ if (trap.scause) {
+ if (trap.scause == CAUSE_LOAD_PAGE_FAULT)
+ trap.scause = CAUSE_FETCH_PAGE_FAULT;
+ trap.stval = trap.sepc = regs->sepc;
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ return VMM_OK;
+ }
+ }
+ }
+
+ return truly_illegal_insn(vcpu, regs, insn);
}
int cpu_vcpu_virtual_insn_fault(struct vmm_vcpu *vcpu,
@@ -499,18 +1286,18 @@ int cpu_vcpu_virtual_insn_fault(struct vmm_vcpu *vcpu,
unsigned long insn = stval;
struct cpu_vcpu_trap trap = { 0 };
- if (unlikely((insn & 3) != 3)) {
+ if (unlikely((insn & INSN_16BIT_MASK) != INSN_16BIT_MASK)) {
if (insn == 0) {
insn = __cpu_vcpu_unpriv_read_insn(regs->sepc, &trap);
if (trap.scause) {
if (trap.scause == CAUSE_LOAD_PAGE_FAULT)
trap.scause = CAUSE_FETCH_PAGE_FAULT;
trap.stval = trap.sepc = regs->sepc;
- return cpu_vcpu_redirect_trap(vcpu, regs,
- &trap);
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+ return VMM_OK;
}
}
- if ((insn & 3) != 3)
+ if ((insn & INSN_16BIT_MASK) != INSN_16BIT_MASK)
return truly_illegal_insn(vcpu, regs, insn);
}
@@ -521,3 +1308,78 @@ int cpu_vcpu_virtual_insn_fault(struct vmm_vcpu *vcpu,
return truly_illegal_insn(vcpu, regs, insn);
};
}
+
+void cpu_vcpu_take_vsirq(struct vmm_vcpu *vcpu, struct arch_regs *regs)
+{
+ int vsirq;
+ bool next_spp;
+ unsigned long irqs;
+ struct cpu_vcpu_trap trap;
+ struct riscv_priv_nested *npriv;
+
+ /* Do nothing for Orphan VCPUs */
+ if (!vcpu->is_normal) {
+ return;
+ }
+
+ /* Do nothing if virt state is OFF */
+ npriv = riscv_nested_priv(vcpu);
+ if (!npriv->virt) {
+ return;
+ }
+
+ /*
+ * Determine whether we are resuming in virtual-VS mode
+ * or virtual-VU mode
+ */
+ next_spp = (regs->sstatus & SSTATUS_SPP) ? TRUE : FALSE;
+
+ /*
+ * Do nothing if we going to virtual-VS mode and
+ * interrupts are disabled
+ */
+ if (next_spp && !(csr_read(CSR_VSSTATUS) & SSTATUS_SIE)) {
+ return;
+ }
+
+ /* Determine virtual-VS mode interrupt number */
+ vsirq = 0;
+ irqs = npriv->hvip;
+ irqs &= npriv->vsie << 1;
+ irqs &= npriv->hideleg;
+ if (irqs & MIP_VSEIP) {
+ vsirq = IRQ_S_EXT;
+ } else if (irqs & MIP_VSTIP) {
+ vsirq = IRQ_S_TIMER;
+ } else if (irqs & MIP_VSSIP) {
+ vsirq = IRQ_S_SOFT;
+ }
+
+ /* Take virtual-VS mode interrupt */
+ if (vsirq > 0) {
+ trap.scause = SCAUSE_INTERRUPT_MASK | vsirq;
+ trap.sepc = regs->sepc;
+ trap.stval = 0;
+ trap.htval = 0;
+ trap.htinst = 0;
+ cpu_vcpu_redirect_smode_trap(regs, &trap, next_spp);
+ }
+
+ return;
+}
+
+int cpu_vcpu_redirect_vsirq(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ unsigned long irq)
+{
+ struct cpu_vcpu_trap trap = { 0 };
+
+ if (!vcpu || !vcpu->is_normal || !riscv_nested_virt(vcpu)) {
+ return VMM_EFAIL;
+ }
+
+ trap.sepc = regs->sepc;
+ trap.scause = SCAUSE_INTERRUPT_MASK | (irq - 1);
+ cpu_vcpu_redirect_trap(vcpu, regs, &trap);
+
+ return VMM_OK;
+}
diff --git a/arch/riscv/cpu/generic/include/cpu_vcpu_csr.h b/arch/riscv/cpu/generic/include/cpu_vcpu_csr.h
+ return VMM_OK;
+}
deleted file mode 100644
index cc1bb580..00000000
--- a/arch/riscv/cpu/generic/include/cpu_vcpu_csr.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Copyright (c) 2019 Anup Patel.
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- *
- * @file cpu_vcpu_csr.h
- * @author Anup Patel (an...@brainfault.org)
- * @brief header for VCPU CSR read/write handling
- */
-#ifndef _CPU_VCPU_CSR_H__
-#define _CPU_VCPU_CSR_H__
-
-#include <vmm_types.h>
-#include <vmm_manager.h>
-
-int cpu_vcpu_csr_read(struct vmm_vcpu *vcpu,
- unsigned long csr_num,
- unsigned long *csr_val);
-
-int cpu_vcpu_csr_write(struct vmm_vcpu *vcpu,
- unsigned long csr_num,
- unsigned long csr_val);
-
-#endif
diff --git a/arch/riscv/cpu/generic/include/cpu_vcpu_nested.h b/arch/riscv/cpu/generic/include/cpu_vcpu_nested.h
new file mode 100644
index 00000000..669690d3
--- /dev/null
+++ b/arch/riscv/cpu/generic/include/cpu_vcpu_nested.h
@@ -0,0 +1,105 @@
+/**
+ * Copyright (c) 2022 Ventana Micro Systems Inc.
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * @file cpu_vcpu_nested.h
+ * Copyright (c) 2022 Ventana Micro Systems Inc.
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * @author Anup Patel (apa...@ventanamicro.com)
+ * @brief header of VCPU nested functions
+ */
+
+#ifndef _CPU_VCPU_NESTED_H__
+#define _CPU_VCPU_NESTED_H__
+
+#include <vmm_types.h>
+
+struct vmm_vcpu;
+struct cpu_vcpu_trap;
+struct arch_regs;
+
+/** Function to init nested state */
+int cpu_vcpu_nested_init(struct vmm_vcpu *vcpu);
+
+/** Function to reset nested state */
+void cpu_vcpu_nested_reset(struct vmm_vcpu *vcpu);
+
+/** Function to initialize nested state */
+void cpu_vcpu_nested_deinit(struct vmm_vcpu *vcpu);
+
+/** Function to dump nested registers */
+void cpu_vcpu_nested_dump_regs(struct vmm_chardev *cdev,
+ struct vmm_vcpu *vcpu);
+
+/** Function to access nested non-virt CSRs */
+int cpu_vcpu_nested_smode_csr_rmw(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ unsigned int csr_num, unsigned long *val,
+ unsigned long new_val, unsigned long wr_mask);
+
+/** Function to access nested virt CSRs */
+int cpu_vcpu_nested_hext_csr_rmw(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ unsigned int csr_num, unsigned long *val,
+ unsigned long new_val, unsigned long wr_mask);
+
+/** Function to handle nested page fault */
+int cpu_vcpu_nested_page_fault(struct vmm_vcpu *vcpu,
+ bool trap_from_smode,
+ const struct cpu_vcpu_trap *trap,
+ struct cpu_vcpu_trap *out_trap);
+
+/** Function to handle nested hfence.vvma instruction */
+
+void cpu_vcpu_nested_hfence_vvma(struct vmm_vcpu *vcpu,
+ unsigned long *vaddr, unsigned int *asid);
+
+/** Function to handle nested hfence.gvma instruction */
+void cpu_vcpu_nested_hfence_gvma(struct vmm_vcpu *vcpu,
+ physical_addr_t *gaddr, unsigned int *vmid);
+
+/**
+ * Function to handle nested hlv instruction
+ * @returns (< 0) error code upon failure and (>= 0) trap return value
+ * upon success
+ */
+int cpu_vcpu_nested_hlv(struct vmm_vcpu *vcpu, unsigned long vaddr,
+ bool hlvx, void *data, unsigned long len,
+ unsigned long *out_scause,
+ unsigned long *out_stval,
+ unsigned long *out_htval);
+
+/**
+ * Function to handle nested hsv instruction
+ * @returns (< 0) error code upon failure and (>= 0) trap return value
+ * upon success
+ */
+int cpu_vcpu_nested_hsv(struct vmm_vcpu *vcpu, unsigned long vaddr,
+ const void *data, unsigned long len,
+ unsigned long *out_scause,
+ unsigned long *out_stval,
+ unsigned long *out_htval);
+
+enum nested_set_virt_event {
+ NESTED_SET_VIRT_EVENT_TRAP = 0,
+ NESTED_SET_VIRT_EVENT_SRET,
+};
+
+/**
+ * Function to change nested virtualization state
+ * NOTE: This can also update Guest hstatus.SPV and hstatus.SPVP bits
+ */
+void cpu_vcpu_nested_set_virt(struct vmm_vcpu *vcpu, struct arch_regs *regs,
+ enum nested_set_virt_event event, bool virt,
+ bool spvp, bool gva);
+
+#endif
diff --git a/arch/riscv/cpu/generic/include/cpu_vcpu_trap.h b/arch/riscv/cpu/generic/include/cpu_vcpu_trap.h
index cb0ffbb6..d77bba87 100644
--- a/arch/riscv/cpu/generic/include/cpu_vcpu_trap.h
+++ b/arch/riscv/cpu/generic/include/cpu_vcpu_trap.h
@@ -44,18 +44,38 @@ struct cpu_vcpu_trap {
unsigned long htinst;
};
-int cpu_vcpu_redirect_trap(struct vmm_vcpu *vcpu,
- arch_regs_t *regs,
- struct cpu_vcpu_trap *trap);
+enum trap_return {
+ TRAP_RETURN_OK=0,
+ TRAP_RETURN_ILLEGAL_INSN,
+ TRAP_RETURN_VIRTUAL_INSN,
+ TRAP_RETURN_CONTINUE
+};
+
+void cpu_vcpu_update_trap(struct vmm_vcpu *vcpu, arch_regs_t *regs);
+
+void cpu_vcpu_redirect_smode_trap(arch_regs_t *regs,
+ struct cpu_vcpu_trap *trap, bool prev_spp);
+
+void cpu_vcpu_redirect_trap(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ struct cpu_vcpu_trap *trap);
int cpu_vcpu_page_fault(struct vmm_vcpu *vcpu,
arch_regs_t *regs,
struct cpu_vcpu_trap *trap);
+int cpu_vcpu_general_fault(struct vmm_vcpu *vcpu,
+ arch_regs_t *regs,
+ struct cpu_vcpu_trap *trap);
+
int cpu_vcpu_virtual_insn_fault(struct vmm_vcpu *vcpu,
arch_regs_t *regs,
unsigned long stval);
+void cpu_vcpu_take_vsirq(struct vmm_vcpu *vcpu, arch_regs_t *regs);
+
+int cpu_vcpu_redirect_vsirq(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ unsigned long irq);
+
#endif
#endif
diff --git a/arch/riscv/cpu/generic/objects.mk b/arch/riscv/cpu/generic/objects.mk
index bbb9a6b5..69daf98a 100644
--- a/arch/riscv/cpu/generic/objects.mk
+++ b/arch/riscv/cpu/generic/objects.mk
@@ -79,7 +79,7 @@ cpu-objs-$(CONFIG_SMP)+=cpu_smp_ops_default.o
cpu-objs-$(CONFIG_SMP)+=cpu_smp_ops_sbi.o
cpu-objs-$(CONFIG_SMP)+=cpu_sbi_ipi.o
cpu-objs-y+= cpu_vcpu_helper.o
-cpu-objs-y+= cpu_vcpu_csr.o
+cpu-objs-y+= cpu_vcpu_nested.o
cpu-objs-y+= cpu_vcpu_fp.o
cpu-objs-y+= cpu_vcpu_irq.o
cpu-objs-y+= cpu_vcpu_sbi.o
--
2.25.1
Anup Patel
Mar 17, 2022, 11:24:07 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
To implemented nested virtualization on any architecture, we require
a function to get guest page table mapping. We add mmu_get_guest_page()
in generic MMU for this purpose.
arch/common/generic_mmu.c | 59 +++++++++++++++++++++++++++++++
arch/common/include/generic_mmu.h | 18 ++++++++++
2 files changed, 77 insertions(+)
diff --git a/arch/common/generic_mmu.c b/arch/common/generic_mmu.c
index d6b8b03e..2c468f04 100644
--- a/arch/common/generic_mmu.c
+++ b/arch/common/generic_mmu.c
@@ -750,6 +750,65 @@ int mmu_find_pte(struct mmu_pgtbl *pgtbl, physical_addr_t ia,
return VMM_OK;
}
+int mmu_get_guest_page(physical_addr_t pgtbl_guest_pa, int stage, int level,
+ const struct mmu_get_guest_page_ops *ops,
+ void *opaque, physical_addr_t guest_ia,
+ struct mmu_page *pg)
+{
+ int idx;
+ arch_pte_t pte;
+ physical_addr_t pte_pa;
+
+ if (stage <= MMU_STAGE_UNKNOWN ||
+ MMU_STAGE_MAX <= stage ||
+ arch_mmu_start_level(stage) < level ||
+ !ops || !pg)
+ return VMM_EINVALID;
+
+ if (level < 0)
+ level = arch_mmu_start_level(stage);
+
+ idx = ops->gpa2hpa(opaque, stage, level, pgtbl_guest_pa, &pte_pa);
+ if (idx) {
+ if (idx == VMM_EFAULT) {
+ ops->setfault(opaque, stage, level, guest_ia);
+ }
+ return idx;
+ }
+
+ idx = arch_mmu_level_index(guest_ia, stage, level);
+ if (vmm_host_memory_read(pte_pa + idx * sizeof(pte),
+ &pte, sizeof(pte), TRUE) != sizeof(pte)) {
+ ops->setfault(opaque, stage, level, guest_ia);
+ return VMM_EFAULT;
+ }
+
+ if (!arch_mmu_pte_is_valid(&pte, stage, level)) {
+ ops->setfault(opaque, stage, level, guest_ia);
+ return VMM_EFAULT;
+ }
+ if ((level == 0) &&
+ arch_mmu_pte_is_table(&pte, stage, level)) {
+ ops->setfault(opaque, stage, level, guest_ia);
+ return VMM_EFAULT;
+ }
+
+ if ((level > 0) && arch_mmu_pte_is_table(&pte, stage, level)) {
+ pte_pa = arch_mmu_pte_table_addr(&pte, stage, level);
+ return mmu_get_guest_page(pte_pa, stage, level - 1,
+ ops, opaque, guest_ia, pg);
+ }
+
+ memset(pg, 0, sizeof(struct mmu_page));
+
+ pg->ia = guest_ia & arch_mmu_level_map_mask(stage, level);
+ pg->oa = arch_mmu_pte_addr(&pte, stage, level);
+ pg->sz = arch_mmu_level_block_size(stage, level);
+ arch_mmu_pte_flags(&pte, stage, level, &pg->flags);
void (*fn)(struct mmu_pgtbl *, arch_pte_t *, void *),
void *opaque)
diff --git a/arch/common/include/generic_mmu.h b/arch/common/include/generic_mmu.h
index 22d49e14..9e53c9c4 100644
--- a/arch/common/include/generic_mmu.h
+++ b/arch/common/include/generic_mmu.h
@@ -128,6 +128,24 @@ int mmu_map_page(struct mmu_pgtbl *pgtbl, struct mmu_page *pg);
int mmu_find_pte(struct mmu_pgtbl *pgtbl, physical_addr_t ia,
arch_pte_t **ptep, struct mmu_pgtbl **pgtblp);
+struct mmu_get_guest_page_ops {
+ void (*setfault)(void *opaque, int stage, int level,
+ physical_addr_t guest_ia);
+ int (*gpa2hpa)(void *opaque, int stage, int level,
+ physical_addr_t guest_pa,
+ physical_addr_t *out_host_pa);
+};
+
+/**
+ * Get guest page table entry
+ *
+ * Returns VMM_OK on success, VMM_EFAULT on trap and VMM_Exxx on failure.
+ */
+int mmu_get_guest_page(physical_addr_t pgtbl_guest_ia, int stage, int level,
+ const struct mmu_get_guest_page_ops *ops,
+ void *opaque, physical_addr_t guest_ia,
+ struct mmu_page *pg);
+
void mmu_walk_address(struct mmu_pgtbl *pgtbl, physical_addr_t ia,
void (*fn)(struct mmu_pgtbl *, arch_pte_t *, void *),
void *opaque);
--
2.25.1
a function to get guest page table mapping. We add mmu_get_guest_page()
in generic MMU for this purpose.
arch/common/generic_mmu.c | 59 +++++++++++++++++++++++++++++++
arch/common/include/generic_mmu.h | 18 ++++++++++
2 files changed, 77 insertions(+)
diff --git a/arch/common/generic_mmu.c b/arch/common/generic_mmu.c
index d6b8b03e..2c468f04 100644
--- a/arch/common/generic_mmu.c
+++ b/arch/common/generic_mmu.c
@@ -750,6 +750,65 @@ int mmu_find_pte(struct mmu_pgtbl *pgtbl, physical_addr_t ia,
return VMM_OK;
}
+int mmu_get_guest_page(physical_addr_t pgtbl_guest_pa, int stage, int level,
+ const struct mmu_get_guest_page_ops *ops,
+ void *opaque, physical_addr_t guest_ia,
+ struct mmu_page *pg)
+{
+ int idx;
+ arch_pte_t pte;
+ physical_addr_t pte_pa;
+
+ if (stage <= MMU_STAGE_UNKNOWN ||
+ MMU_STAGE_MAX <= stage ||
+ arch_mmu_start_level(stage) < level ||
+ !ops || !pg)
+ return VMM_EINVALID;
+
+ if (level < 0)
+ level = arch_mmu_start_level(stage);
+
+ idx = ops->gpa2hpa(opaque, stage, level, pgtbl_guest_pa, &pte_pa);
+ if (idx) {
+ if (idx == VMM_EFAULT) {
+ ops->setfault(opaque, stage, level, guest_ia);
+ }
+ return idx;
+ }
+
+ idx = arch_mmu_level_index(guest_ia, stage, level);
+ if (vmm_host_memory_read(pte_pa + idx * sizeof(pte),
+ &pte, sizeof(pte), TRUE) != sizeof(pte)) {
+ ops->setfault(opaque, stage, level, guest_ia);
+ return VMM_EFAULT;
+ }
+
+ if (!arch_mmu_pte_is_valid(&pte, stage, level)) {
+ ops->setfault(opaque, stage, level, guest_ia);
+ return VMM_EFAULT;
+ }
+ if ((level == 0) &&
+ arch_mmu_pte_is_table(&pte, stage, level)) {
+ ops->setfault(opaque, stage, level, guest_ia);
+ return VMM_EFAULT;
+ }
+
+ if ((level > 0) && arch_mmu_pte_is_table(&pte, stage, level)) {
+ pte_pa = arch_mmu_pte_table_addr(&pte, stage, level);
+ return mmu_get_guest_page(pte_pa, stage, level - 1,
+ ops, opaque, guest_ia, pg);
+ }
+
+ memset(pg, 0, sizeof(struct mmu_page));
+
+ pg->ia = guest_ia & arch_mmu_level_map_mask(stage, level);
+ pg->oa = arch_mmu_pte_addr(&pte, stage, level);
+ pg->sz = arch_mmu_level_block_size(stage, level);
+ arch_mmu_pte_flags(&pte, stage, level, &pg->flags);
+
+ return VMM_OK;
+}
+
void mmu_walk_address(struct mmu_pgtbl *pgtbl, physical_addr_t ia,
+ return VMM_OK;
+}
+
void (*fn)(struct mmu_pgtbl *, arch_pte_t *, void *),
void *opaque)
diff --git a/arch/common/include/generic_mmu.h b/arch/common/include/generic_mmu.h
index 22d49e14..9e53c9c4 100644
--- a/arch/common/include/generic_mmu.h
+++ b/arch/common/include/generic_mmu.h
@@ -128,6 +128,24 @@ int mmu_map_page(struct mmu_pgtbl *pgtbl, struct mmu_page *pg);
int mmu_find_pte(struct mmu_pgtbl *pgtbl, physical_addr_t ia,
arch_pte_t **ptep, struct mmu_pgtbl **pgtblp);
+struct mmu_get_guest_page_ops {
+ void (*setfault)(void *opaque, int stage, int level,
+ physical_addr_t guest_ia);
+ int (*gpa2hpa)(void *opaque, int stage, int level,
+ physical_addr_t guest_pa,
+ physical_addr_t *out_host_pa);
+};
+
+/**
+ * Get guest page table entry
+ *
+ * Returns VMM_OK on success, VMM_EFAULT on trap and VMM_Exxx on failure.
+ */
+int mmu_get_guest_page(physical_addr_t pgtbl_guest_ia, int stage, int level,
+ const struct mmu_get_guest_page_ops *ops,
+ void *opaque, physical_addr_t guest_ia,
+ struct mmu_page *pg);
+
void mmu_walk_address(struct mmu_pgtbl *pgtbl, physical_addr_t ia,
void (*fn)(struct mmu_pgtbl *, arch_pte_t *, void *),
void *opaque);
--
2.25.1
Anup Patel
Mar 17, 2022, 11:24:09 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
The guest hypervisor running in virtual-HS mode can use HLV and
HSV instructions for unprivileged access so let's emulate these
instructions in host hypervisor.
arch/riscv/cpu/generic/cpu_vcpu_nested.c | 550 ++++++++++++++++++++++-
1 file changed, 548 insertions(+), 2 deletions(-)
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_nested.c b/arch/riscv/cpu/generic/cpu_vcpu_nested.c
index 5f8774de..e5d95a97 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_nested.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_nested.c
@@ -23,6 +23,8 @@
#include <vmm_error.h>
#include <vmm_stdio.h>
+#include <vmm_guest_aspace.h>
+#include <vmm_host_aspace.h>
#include <generic_mmu.h>
#include <cpu_hwcap.h>
@@ -31,6 +33,488 @@
#include <cpu_vcpu_trap.h>
#include <riscv_csr.h>
+#ifdef CONFIG_64BIT
+#define NESTED_MMU_OFF_BLOCK_SIZE PGTBL_L2_BLOCK_SIZE
+#else
+#define NESTED_MMU_OFF_BLOCK_SIZE PGTBL_L1_BLOCK_SIZE
+#endif
+
+enum nested_xlate_access {
+ NESTED_XLATE_LOAD = 0,
+ NESTED_XLATE_STORE,
+ NESTED_XLATE_FETCH,
+};
+
+struct nested_xlate_context {
+ /* VCPU for which translation is being done */
+ struct vmm_vcpu *vcpu;
+
+ /* Original access type for fault generation */
+ enum nested_xlate_access original_access;
+
+ /* Details from CSR or instruction */
+ bool smode;
+ unsigned long sstatus;
+ bool hlvx;
+
+ /* Final host region details */
+ physical_addr_t host_pa;
+ physical_size_t host_sz;
+ u32 host_reg_flags;
+
+ /* Fault details */
+ physical_size_t nostage_page_sz;
+ physical_size_t gstage_page_sz;
+ physical_size_t vsstage_page_sz;
+ unsigned long scause;
+ unsigned long stval;
+ unsigned long htval;
+};
+
+#define nested_xlate_context_init(__x, __v, __a, __smode, __sstatus, __hlvx)\
+do { \
+ (__x)->vcpu = (__v); \
+ (__x)->original_access = (__a); \
+ (__x)->smode = (__smode); \
+ (__x)->sstatus = (__sstatus); \
+ (__x)->hlvx = (__hlvx); \
+ (__x)->host_pa = 0; \
+ (__x)->host_sz = 0; \
+ (__x)->host_reg_flags = 0; \
+ (__x)->nostage_page_sz = 0; \
+ (__x)->gstage_page_sz = 0; \
+ (__x)->vsstage_page_sz = 0; \
+ (__x)->scause = 0; \
+ (__x)->stval = 0; \
+ (__x)->htval = 0; \
+} while (0)
+
+static int nested_nostage_perm_check(enum nested_xlate_access guest_access,
+ u32 reg_flags)
+{
+ if ((guest_access == NESTED_XLATE_LOAD) ||
+ (guest_access == NESTED_XLATE_FETCH)) {
+ if (!(reg_flags & (VMM_REGION_ISRAM | VMM_REGION_ISROM))) {
+ return VMM_EFAULT;
+ }
+ } else if (guest_access == NESTED_XLATE_STORE) {
+ if (!(reg_flags & VMM_REGION_ISRAM)) {
+ physical_addr_t guest_hpa,
+ physical_size_t block_size,
+ enum nested_xlate_access guest_access,
+ physical_addr_t *out_host_pa,
+ physical_size_t *out_host_sz,
+ u32 *out_host_reg_flags)
+{
+ int rc;
+ u32 reg_flags = 0x0;
+ physical_size_t availsz = 0;
+ physical_addr_t inaddr, outaddr = 0;
+
+ inaddr = guest_hpa & ~(block_size - 1);
+
+ /* Map host physical address */
+ rc = vmm_guest_physical_map(guest, inaddr, block_size,
+ &outaddr, &availsz, ®_flags);
+ if (rc || (availsz < block_size)) {
+ rc = nested_nostage_perm_check(guest_access, reg_flags);
+ if (out_host_pa) {
+ *out_host_pa = outaddr;
+ }
+ if (out_host_sz) {
+ *out_host_sz = block_size;
+ }
+ if (out_host_reg_flags) {
+ *out_host_reg_flags = reg_flags;
+ }
+ physical_addr_t guest_gpa)
+{
+ /* We should never have non-zero scause here. */
+ BUG_ON(xc->scause);
+
+ switch (xc->original_access) {
+ case NESTED_XLATE_LOAD:
+ xc->scause = CAUSE_LOAD_GUEST_PAGE_FAULT;
+ xc->htval = guest_gpa >> 2;
+ break;
+ case NESTED_XLATE_STORE:
+ xc->scause = CAUSE_STORE_GUEST_PAGE_FAULT;
+ xc->htval = guest_gpa >> 2;
+ break;
+ case NESTED_XLATE_FETCH:
+ xc->scause = CAUSE_FETCH_GUEST_PAGE_FAULT;
+ xc->htval = guest_gpa >> 2;
+static int nested_xlate_nostage(struct nested_xlate_context *xc,
+ physical_addr_t guest_hpa,
+ enum nested_xlate_access guest_access)
+{
+ int rc;
+ u32 outflags = 0;
+ physical_size_t outsz = 0;
+ physical_addr_t outaddr = 0;
+
+ /* Translate host physical address with L0 block size */
+ rc = nested_xlate_nostage_single(xc->vcpu->guest, guest_hpa,
+ PGTBL_L0_BLOCK_SIZE, guest_access,
+ &outaddr, &outsz, &outflags);
+ rc = nested_xlate_nostage_single(xc->vcpu->guest, guest_hpa,
+ PGTBL_L1_BLOCK_SIZE, guest_access,
+ &outaddr, &outsz, &outflags);
+ if (rc) {
+ goto done;
+ }
+
+#ifdef CONFIG_64BIT
+ /* Try to translate host physical address with L2 block size */
+ rc = nested_xlate_nostage_single(xc->vcpu->guest, guest_hpa,
+ PGTBL_L2_BLOCK_SIZE, guest_access,
+ &outaddr, &outsz, &outflags);
+ if (rc) {
+ goto done;
+ }
+#endif
+
+done:
+ /* Update return values */
+ xc->host_pa = outaddr;
+ xc->host_sz = outsz;
+ xc->host_reg_flags = outflags;
+ physical_addr_t guest_gpa)
+{
+ struct nested_xlate_context *xc = opaque;
+
+ xc->gstage_page_sz = arch_mmu_level_block_size(stage, level);
+ nested_gstage_write_fault(xc, guest_gpa);
+}
+
+static int nested_gstage_gpa2hpa(void *opaque, int stage, int level,
+ physical_addr_t guest_hpa,
+ physical_addr_t *out_host_pa)
+{
+ int rc;
+ physical_size_t outsz = 0;
+ physical_addr_t outaddr = 0;
+ struct nested_xlate_context *xc = opaque;
+
+ rc = nested_xlate_nostage_single(xc->vcpu->guest, guest_hpa,
+ PGTBL_L0_BLOCK_SIZE,
+ NESTED_XLATE_LOAD,
+ &outaddr, &outsz, NULL);
+ .gpa2hpa = nested_gstage_gpa2hpa,
+ .setfault = nested_gstage_setfault,
+};
+
+/* Translate guest gpa to host pa */
+static int nested_xlate_gstage(struct nested_xlate_context *xc,
+ physical_addr_t guest_gpa,
+ enum nested_xlate_access guest_access)
+{
+ int rc;
+ unsigned long mode;
+ struct mmu_page page;
+ bool perm_fault = FALSE;
+ physical_addr_t pgtlb, guest_hpa;
+ struct riscv_priv_nested *npriv = riscv_nested_priv(xc->vcpu);
+
+ /* Find guest G-stage page */
+ mode = (npriv->hgatp & HGATP_MODE) >> HGATP_MODE_SHIFT;
+ if (mode == HGATP_MODE_OFF) {
+ memset(&page, 0, sizeof(page));
+ page.sz = NESTED_MMU_OFF_BLOCK_SIZE;
+ page.ia = guest_gpa & ~(page.sz - 1);
+ page.oa = guest_gpa & ~(page.sz - 1);
+ page.flags.dirty = 1;
+ page.flags.accessed = 1;
+ page.flags.global = 1;
+ page.flags.user = 1;
+ page.flags.read = 1;
+ page.flags.write = 1;
+ page.flags.execute = 1;
+ page.flags.valid = 1;
+ } else {
+ rc = 3;
+ break;
+ case HGATP_MODE_SV39X4:
+ rc = 2;
+ rc = mmu_get_guest_page(pgtlb, MMU_STAGE2, rc,
+ &nested_xlate_gstage_ops, xc,
+ guest_gpa, &page);
+ if (!page.flags.user) {
+ perm_fault = TRUE;
+ } else if (guest_access == NESTED_XLATE_FETCH || xc->hlvx) {
+ perm_fault = !page.flags.execute;
+ } else if (guest_access == NESTED_XLATE_LOAD) {
+ perm_fault = !(page.flags.read ||
+ ((xc->sstatus & SSTATUS_MXR) && page.flags.execute)) ||
+ !page.flags.accessed;
+ } else if (guest_access == NESTED_XLATE_STORE) {
+ perm_fault = !page.flags.read ||
+ !page.flags.write ||
+ !page.flags.accessed ||
+ !page.flags.dirty;
+ }
+ if (perm_fault) {
+ xc->gstage_page_sz = page.sz;
+ nested_gstage_write_fault(xc, guest_gpa);
+ guest_hpa = page.oa | (guest_gpa & (page.sz - 1));
+
+ /* Translate guest hpa to host pa */
+ rc = nested_xlate_nostage(xc, guest_hpa, guest_access);
+ if (rc) {
+ if (rc == VMM_EFAULT) {
+ xc->nostage_page_sz = page.sz;
+ nested_gstage_write_fault(xc, guest_gpa);
+ }
+ if (page.sz <= xc->host_sz) {
+ xc->host_pa |= guest_hpa & (xc->host_sz - 1);
+ xc->host_sz = page.sz;
+ xc->host_pa &= ~(xc->host_sz - 1);
+ } else {
+ page.ia = guest_gpa & ~(xc->host_sz - 1);
+ page.oa = guest_hpa & ~(xc->host_sz - 1);
+ page.sz = xc->host_sz;
+ }
+ physical_addr_t guest_gva)
+{
+ switch (xc->original_access) {
+ case NESTED_XLATE_LOAD:
+ if (!xc->scause) {
+ xc->scause = CAUSE_LOAD_PAGE_FAULT;
+ }
+ xc->stval = guest_gva;
+ break;
+ case NESTED_XLATE_STORE:
+ if (!xc->scause) {
+ xc->scause = CAUSE_STORE_PAGE_FAULT;
+ }
+ xc->stval = guest_gva;
+ break;
+ case NESTED_XLATE_FETCH:
+ if (!xc->scause) {
+ xc->scause = CAUSE_FETCH_PAGE_FAULT;
+ }
+ xc->stval = guest_gva;
+ physical_addr_t guest_gva)
+{
+ struct nested_xlate_context *xc = opaque;
+
+ xc->vsstage_page_sz = arch_mmu_level_block_size(stage, level);
+ nested_vsstage_write_fault(xc, guest_gva);
+}
+
+static int nested_vsstage_gpa2hpa(void *opaque, int stage, int level,
+ physical_addr_t guest_gpa,
+ physical_addr_t *out_host_pa)
+{
+ int rc;
+ struct nested_xlate_context *xc = opaque;
+
+ rc = nested_xlate_gstage(xc, guest_gpa, NESTED_XLATE_LOAD);
+ .setfault = nested_vsstage_setfault,
+ .gpa2hpa = nested_vsstage_gpa2hpa,
+};
+
+/* Translate guest gva to host pa */
+static int nested_xlate_vsstage(struct nested_xlate_context *xc,
+ physical_addr_t guest_gva,
+ enum nested_xlate_access guest_access)
+{
+ int rc;
+ unsigned long mode;
+ struct mmu_page page;
+ bool perm_fault = FALSE;
+ physical_addr_t pgtlb, guest_gpa;
+ struct riscv_priv_nested *npriv = riscv_nested_priv(xc->vcpu);
+
+ /* Get guest VS-stage page */
+ mode = (npriv->vsatp & SATP_MODE) >> SATP_MODE_SHIFT;
+ if (mode == SATP_MODE_OFF) {
+ memset(&page, 0, sizeof(page));
+ page.sz = NESTED_MMU_OFF_BLOCK_SIZE;
+ page.ia = guest_gva & ~(page.sz - 1);
+ page.oa = guest_gva & ~(page.sz - 1);
+ page.flags.dirty = 1;
+ page.flags.accessed = 1;
+ page.flags.global = 1;
+ page.flags.user = 1;
+ page.flags.read = 1;
+ page.flags.write = 1;
+ page.flags.execute = 1;
+ page.flags.valid = 1;
+ } else {
+ rc = mmu_get_guest_page(pgtlb, MMU_STAGE1, rc,
+ &nested_xlate_vsstage_ops, xc,
+ guest_gva, &page);
+ if (page.flags.user ?
+ xc->smode && (guest_access == NESTED_XLATE_FETCH ||
+ (xc->sstatus & SSTATUS_SUM)) :
+ !xc->smode) {
+ perm_fault = TRUE;
+ } else if (guest_access == NESTED_XLATE_FETCH || xc->hlvx) {
+ perm_fault = !page.flags.execute;
+ } else if (guest_access == NESTED_XLATE_LOAD) {
+ perm_fault = !(page.flags.read ||
+ ((xc->sstatus & SSTATUS_MXR) && page.flags.execute)) ||
+ !page.flags.accessed;
+ } else if (guest_access == NESTED_XLATE_STORE) {
+ perm_fault = !page.flags.read ||
+ !page.flags.write ||
+ !page.flags.accessed ||
+ !page.flags.dirty;
+ }
+ if (perm_fault) {
+ xc->vsstage_page_sz = page.sz;
+ nested_vsstage_write_fault(xc, guest_gva);
+ guest_gpa = page.oa | (guest_gva & (page.sz - 1));
+
+ /* Translate guest gpa to host pa */
+ rc = nested_xlate_gstage(xc, guest_gpa, guest_access);
+ if (rc) {
+ if (rc == VMM_EFAULT) {
+ nested_vsstage_write_fault(xc, guest_gva);
+ }
+ if (page.sz <= xc->host_sz) {
+ xc->host_pa |= guest_gpa & (xc->host_sz - 1);
+ xc->host_sz = page.sz;
+ xc->host_pa &= ~(xc->host_sz - 1);
+ } else {
+ page.ia = guest_gva & ~(xc->host_sz - 1);
+ page.oa = guest_gpa & ~(xc->host_sz - 1);
+ page.sz = xc->host_sz;
+ }
{
unsigned long *out_stval,
unsigned long *out_htval)
{
- /* TODO: */
+ int rc;
+ physical_addr_t hpa;
+ struct nested_xlate_context xc;
+ if (vaddr & (len - 1)) {
+ *out_scause = CAUSE_MISALIGNED_LOAD;
+ *out_stval = vaddr;
+ *out_htval = 0;
+ (npriv->hstatus & HSTATUS_SPVP) ? TRUE : FALSE,
+ csr_read(CSR_VSSTATUS), hlvx);
+ rc = nested_xlate_vsstage(&xc, vaddr, NESTED_XLATE_LOAD);
+ if (rc) {
+ if (rc == VMM_EFAULT) {
+ *out_scause = xc.scause;
+ *out_stval = xc.stval;
+ *out_htval = xc.htval;
+ return 0;
+ }
+ if (vmm_host_memory_read(hpa, data, len, TRUE) != len) {
+ return VMM_EIO;
+ }
+
return VMM_OK;
}
@@ -472,7 +987,38 @@ int cpu_vcpu_nested_hsv(struct vmm_vcpu *vcpu, unsigned long vaddr,
unsigned long *out_stval,
unsigned long *out_htval)
{
- /* TODO: */
+ int rc;
+ physical_addr_t hpa;
+ struct nested_xlate_context xc;
+ if (vaddr & (len - 1)) {
+ *out_scause = CAUSE_MISALIGNED_STORE;
+ *out_stval = vaddr;
+ *out_htval = 0;
+ (npriv->hstatus & HSTATUS_SPVP) ? TRUE : FALSE,
+ csr_read(CSR_VSSTATUS), FALSE);
+ rc = nested_xlate_vsstage(&xc, vaddr, NESTED_XLATE_STORE);
+ if (rc) {
+ if (rc == VMM_EFAULT) {
+ *out_scause = xc.scause;
+ *out_stval = xc.stval;
+ *out_htval = xc.htval;
+ return 0;
+ }
+ if (vmm_host_memory_write(hpa, (void *)data, len, TRUE) != len) {
+ return VMM_EIO;
+ }
+
return VMM_OK;
}
--
2.25.1
HSV instructions for unprivileged access so let's emulate these
instructions in host hypervisor.
arch/riscv/cpu/generic/cpu_vcpu_nested.c | 550 ++++++++++++++++++++++-
1 file changed, 548 insertions(+), 2 deletions(-)
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_nested.c b/arch/riscv/cpu/generic/cpu_vcpu_nested.c
index 5f8774de..e5d95a97 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_nested.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_nested.c
@@ -23,6 +23,8 @@
#include <vmm_error.h>
#include <vmm_stdio.h>
+#include <vmm_guest_aspace.h>
+#include <vmm_host_aspace.h>
#include <generic_mmu.h>
#include <cpu_hwcap.h>
@@ -31,6 +33,488 @@
#include <cpu_vcpu_trap.h>
#include <riscv_csr.h>
+#ifdef CONFIG_64BIT
+#define NESTED_MMU_OFF_BLOCK_SIZE PGTBL_L2_BLOCK_SIZE
+#else
+#define NESTED_MMU_OFF_BLOCK_SIZE PGTBL_L1_BLOCK_SIZE
+#endif
+
+enum nested_xlate_access {
+ NESTED_XLATE_LOAD = 0,
+ NESTED_XLATE_STORE,
+ NESTED_XLATE_FETCH,
+};
+
+struct nested_xlate_context {
+ /* VCPU for which translation is being done */
+ struct vmm_vcpu *vcpu;
+
+ /* Original access type for fault generation */
+ enum nested_xlate_access original_access;
+
+ /* Details from CSR or instruction */
+ bool smode;
+ unsigned long sstatus;
+ bool hlvx;
+
+ /* Final host region details */
+ physical_addr_t host_pa;
+ physical_size_t host_sz;
+ u32 host_reg_flags;
+
+ /* Fault details */
+ physical_size_t nostage_page_sz;
+ physical_size_t gstage_page_sz;
+ physical_size_t vsstage_page_sz;
+ unsigned long scause;
+ unsigned long stval;
+ unsigned long htval;
+};
+
+#define nested_xlate_context_init(__x, __v, __a, __smode, __sstatus, __hlvx)\
+do { \
+ (__x)->vcpu = (__v); \
+ (__x)->original_access = (__a); \
+ (__x)->smode = (__smode); \
+ (__x)->sstatus = (__sstatus); \
+ (__x)->hlvx = (__hlvx); \
+ (__x)->host_pa = 0; \
+ (__x)->host_sz = 0; \
+ (__x)->host_reg_flags = 0; \
+ (__x)->nostage_page_sz = 0; \
+ (__x)->gstage_page_sz = 0; \
+ (__x)->vsstage_page_sz = 0; \
+ (__x)->scause = 0; \
+ (__x)->stval = 0; \
+ (__x)->htval = 0; \
+} while (0)
+
+static int nested_nostage_perm_check(enum nested_xlate_access guest_access,
+ u32 reg_flags)
+{
+ if ((guest_access == NESTED_XLATE_LOAD) ||
+ (guest_access == NESTED_XLATE_FETCH)) {
+ if (!(reg_flags & (VMM_REGION_ISRAM | VMM_REGION_ISROM))) {
+ return VMM_EFAULT;
+ }
+ } else if (guest_access == NESTED_XLATE_STORE) {
+ if (!(reg_flags & VMM_REGION_ISRAM)) {
+ return VMM_EFAULT;
+ }
+ }
+
+ }
+ }
+
+ return VMM_OK;
+}
+
+static int nested_xlate_nostage_single(struct vmm_guest *guest,
+}
+
+ physical_addr_t guest_hpa,
+ physical_size_t block_size,
+ enum nested_xlate_access guest_access,
+ physical_addr_t *out_host_pa,
+ physical_size_t *out_host_sz,
+ u32 *out_host_reg_flags)
+{
+ int rc;
+ u32 reg_flags = 0x0;
+ physical_size_t availsz = 0;
+ physical_addr_t inaddr, outaddr = 0;
+
+ inaddr = guest_hpa & ~(block_size - 1);
+
+ /* Map host physical address */
+ rc = vmm_guest_physical_map(guest, inaddr, block_size,
+ &outaddr, &availsz, ®_flags);
+ if (rc || (availsz < block_size)) {
+ return VMM_EFAULT;
+ }
+
+ /* Check region permissions */
+ }
+
+ rc = nested_nostage_perm_check(guest_access, reg_flags);
+ if (rc) {
+ return rc;
+ }
+
+ /* Update return values */
+ return rc;
+ }
+
+ if (out_host_pa) {
+ *out_host_pa = outaddr;
+ }
+ if (out_host_sz) {
+ *out_host_sz = block_size;
+ }
+ if (out_host_reg_flags) {
+ *out_host_reg_flags = reg_flags;
+ }
+
+ return VMM_OK;
+}
+
+static void nested_gstage_write_fault(struct nested_xlate_context *xc,
+ return VMM_OK;
+}
+
+ physical_addr_t guest_gpa)
+{
+ /* We should never have non-zero scause here. */
+ BUG_ON(xc->scause);
+
+ switch (xc->original_access) {
+ case NESTED_XLATE_LOAD:
+ xc->scause = CAUSE_LOAD_GUEST_PAGE_FAULT;
+ xc->htval = guest_gpa >> 2;
+ break;
+ case NESTED_XLATE_STORE:
+ xc->scause = CAUSE_STORE_GUEST_PAGE_FAULT;
+ xc->htval = guest_gpa >> 2;
+ break;
+ case NESTED_XLATE_FETCH:
+ xc->scause = CAUSE_FETCH_GUEST_PAGE_FAULT;
+ xc->htval = guest_gpa >> 2;
+ break;
+ default:
+ break;
+ };
+}
+
+/* Translate guest hpa to host pa */
+ default:
+ break;
+ };
+}
+
+static int nested_xlate_nostage(struct nested_xlate_context *xc,
+ physical_addr_t guest_hpa,
+ enum nested_xlate_access guest_access)
+{
+ int rc;
+ u32 outflags = 0;
+ physical_size_t outsz = 0;
+ physical_addr_t outaddr = 0;
+
+ /* Translate host physical address with L0 block size */
+ rc = nested_xlate_nostage_single(xc->vcpu->guest, guest_hpa,
+ PGTBL_L0_BLOCK_SIZE, guest_access,
+ &outaddr, &outsz, &outflags);
+ if (rc) {
+ return rc;
+ }
+
+ /* Try to translate host physical address with L1 block size */
+ return rc;
+ }
+
+ rc = nested_xlate_nostage_single(xc->vcpu->guest, guest_hpa,
+ PGTBL_L1_BLOCK_SIZE, guest_access,
+ &outaddr, &outsz, &outflags);
+ if (rc) {
+ goto done;
+ }
+
+#ifdef CONFIG_64BIT
+ /* Try to translate host physical address with L2 block size */
+ rc = nested_xlate_nostage_single(xc->vcpu->guest, guest_hpa,
+ PGTBL_L2_BLOCK_SIZE, guest_access,
+ &outaddr, &outsz, &outflags);
+ if (rc) {
+ goto done;
+ }
+#endif
+
+done:
+ /* Update return values */
+ xc->host_pa = outaddr;
+ xc->host_sz = outsz;
+ xc->host_reg_flags = outflags;
+
+ return VMM_OK;
+}
+
+static void nested_gstage_setfault(void *opaque, int stage, int level,
+ return VMM_OK;
+}
+
+ physical_addr_t guest_gpa)
+{
+ struct nested_xlate_context *xc = opaque;
+
+ xc->gstage_page_sz = arch_mmu_level_block_size(stage, level);
+ nested_gstage_write_fault(xc, guest_gpa);
+}
+
+static int nested_gstage_gpa2hpa(void *opaque, int stage, int level,
+ physical_addr_t guest_hpa,
+ physical_addr_t *out_host_pa)
+{
+ int rc;
+ physical_size_t outsz = 0;
+ physical_addr_t outaddr = 0;
+ struct nested_xlate_context *xc = opaque;
+
+ rc = nested_xlate_nostage_single(xc->vcpu->guest, guest_hpa,
+ PGTBL_L0_BLOCK_SIZE,
+ NESTED_XLATE_LOAD,
+ &outaddr, &outsz, NULL);
+ if (rc) {
+ return rc;
+ }
+
+ *out_host_pa = outaddr | (guest_hpa & (outsz - 1));
+ return rc;
+ }
+
+ return VMM_OK;
+}
+
+static struct mmu_get_guest_page_ops nested_xlate_gstage_ops = {
+}
+
+ .gpa2hpa = nested_gstage_gpa2hpa,
+ .setfault = nested_gstage_setfault,
+};
+
+/* Translate guest gpa to host pa */
+static int nested_xlate_gstage(struct nested_xlate_context *xc,
+ physical_addr_t guest_gpa,
+ enum nested_xlate_access guest_access)
+{
+ int rc;
+ unsigned long mode;
+ struct mmu_page page;
+ bool perm_fault = FALSE;
+ physical_addr_t pgtlb, guest_hpa;
+ struct riscv_priv_nested *npriv = riscv_nested_priv(xc->vcpu);
+
+ /* Find guest G-stage page */
+ mode = (npriv->hgatp & HGATP_MODE) >> HGATP_MODE_SHIFT;
+ if (mode == HGATP_MODE_OFF) {
+ memset(&page, 0, sizeof(page));
+ page.sz = NESTED_MMU_OFF_BLOCK_SIZE;
+ page.ia = guest_gpa & ~(page.sz - 1);
+ page.oa = guest_gpa & ~(page.sz - 1);
+ page.flags.dirty = 1;
+ page.flags.accessed = 1;
+ page.flags.global = 1;
+ page.flags.user = 1;
+ page.flags.read = 1;
+ page.flags.write = 1;
+ page.flags.execute = 1;
+ page.flags.valid = 1;
+ } else {
+ switch (mode) {
+#ifdef CONFIG_64BIT
+ case HGATP_MODE_SV48X4:
+#ifdef CONFIG_64BIT
+ rc = 3;
+ break;
+ case HGATP_MODE_SV39X4:
+ rc = 2;
+ break;
+#else
+ case HGATP_MODE_SV32X4:
+ rc = 1;
+#else
+ case HGATP_MODE_SV32X4:
+ break;
+#endif
+ default:
+#endif
+ default:
+ return VMM_EFAIL;
+ }
+
+ pgtlb = (npriv->hgatp & HGATP_PPN) << PGTBL_PAGE_SIZE_SHIFT;
+ }
+
+ rc = mmu_get_guest_page(pgtlb, MMU_STAGE2, rc,
+ &nested_xlate_gstage_ops, xc,
+ guest_gpa, &page);
+ if (rc) {
+ return rc;
+ }
+ }
+
+ /* Check guest G-stage page permissions */
+ return rc;
+ }
+ }
+
+ if (!page.flags.user) {
+ perm_fault = TRUE;
+ } else if (guest_access == NESTED_XLATE_FETCH || xc->hlvx) {
+ perm_fault = !page.flags.execute;
+ } else if (guest_access == NESTED_XLATE_LOAD) {
+ perm_fault = !(page.flags.read ||
+ ((xc->sstatus & SSTATUS_MXR) && page.flags.execute)) ||
+ !page.flags.accessed;
+ } else if (guest_access == NESTED_XLATE_STORE) {
+ perm_fault = !page.flags.read ||
+ !page.flags.write ||
+ !page.flags.accessed ||
+ !page.flags.dirty;
+ }
+ if (perm_fault) {
+ xc->gstage_page_sz = page.sz;
+ nested_gstage_write_fault(xc, guest_gpa);
+ return VMM_EFAULT;
+ }
+
+ /* Calculate guest hpa */
+ }
+
+ guest_hpa = page.oa | (guest_gpa & (page.sz - 1));
+
+ /* Translate guest hpa to host pa */
+ rc = nested_xlate_nostage(xc, guest_hpa, guest_access);
+ if (rc) {
+ if (rc == VMM_EFAULT) {
+ xc->nostage_page_sz = page.sz;
+ nested_gstage_write_fault(xc, guest_gpa);
+ }
+ return rc;
+ }
+
+ /* Update output address and size */
+ }
+
+ if (page.sz <= xc->host_sz) {
+ xc->host_pa |= guest_hpa & (xc->host_sz - 1);
+ xc->host_sz = page.sz;
+ xc->host_pa &= ~(xc->host_sz - 1);
+ } else {
+ page.ia = guest_gpa & ~(xc->host_sz - 1);
+ page.oa = guest_hpa & ~(xc->host_sz - 1);
+ page.sz = xc->host_sz;
+ }
+
+ return VMM_OK;
+}
+
+static void nested_vsstage_write_fault(struct nested_xlate_context *xc,
+ return VMM_OK;
+}
+
+ physical_addr_t guest_gva)
+{
+ switch (xc->original_access) {
+ case NESTED_XLATE_LOAD:
+ if (!xc->scause) {
+ xc->scause = CAUSE_LOAD_PAGE_FAULT;
+ }
+ xc->stval = guest_gva;
+ break;
+ case NESTED_XLATE_STORE:
+ if (!xc->scause) {
+ xc->scause = CAUSE_STORE_PAGE_FAULT;
+ }
+ xc->stval = guest_gva;
+ break;
+ case NESTED_XLATE_FETCH:
+ if (!xc->scause) {
+ xc->scause = CAUSE_FETCH_PAGE_FAULT;
+ }
+ xc->stval = guest_gva;
+ break;
+ default:
+ break;
+ };
+}
+
+static void nested_vsstage_setfault(void *opaque, int stage, int level,
+ default:
+ break;
+ };
+}
+
+ physical_addr_t guest_gva)
+{
+ struct nested_xlate_context *xc = opaque;
+
+ xc->vsstage_page_sz = arch_mmu_level_block_size(stage, level);
+ nested_vsstage_write_fault(xc, guest_gva);
+}
+
+static int nested_vsstage_gpa2hpa(void *opaque, int stage, int level,
+ physical_addr_t guest_gpa,
+ physical_addr_t *out_host_pa)
+{
+ int rc;
+ struct nested_xlate_context *xc = opaque;
+
+ rc = nested_xlate_gstage(xc, guest_gpa, NESTED_XLATE_LOAD);
+ if (rc) {
+ return rc;
+ }
+
+ *out_host_pa = xc->host_pa | (guest_gpa & (xc->host_sz - 1));
+ return rc;
+ }
+
+ return VMM_OK;
+}
+
+static struct mmu_get_guest_page_ops nested_xlate_vsstage_ops = {
+}
+
+ .setfault = nested_vsstage_setfault,
+ .gpa2hpa = nested_vsstage_gpa2hpa,
+};
+
+/* Translate guest gva to host pa */
+static int nested_xlate_vsstage(struct nested_xlate_context *xc,
+ physical_addr_t guest_gva,
+ enum nested_xlate_access guest_access)
+{
+ int rc;
+ unsigned long mode;
+ struct mmu_page page;
+ bool perm_fault = FALSE;
+ physical_addr_t pgtlb, guest_gpa;
+ struct riscv_priv_nested *npriv = riscv_nested_priv(xc->vcpu);
+
+ /* Get guest VS-stage page */
+ mode = (npriv->vsatp & SATP_MODE) >> SATP_MODE_SHIFT;
+ if (mode == SATP_MODE_OFF) {
+ memset(&page, 0, sizeof(page));
+ page.sz = NESTED_MMU_OFF_BLOCK_SIZE;
+ page.ia = guest_gva & ~(page.sz - 1);
+ page.oa = guest_gva & ~(page.sz - 1);
+ page.flags.dirty = 1;
+ page.flags.accessed = 1;
+ page.flags.global = 1;
+ page.flags.user = 1;
+ page.flags.read = 1;
+ page.flags.write = 1;
+ page.flags.execute = 1;
+ page.flags.valid = 1;
+ } else {
+ switch (mode) {
+#ifdef CONFIG_64BIT
+ case SATP_MODE_SV48:
+ rc = 3;
+#ifdef CONFIG_64BIT
+ case SATP_MODE_SV48:
+ break;
+ case SATP_MODE_SV39:
+ rc = 2;
+ case SATP_MODE_SV39:
+ break;
+#else
+ case SATP_MODE_SV32:
+ rc = 1;
+#else
+ case SATP_MODE_SV32:
+ break;
+#endif
+ default:
+#endif
+ default:
+ return VMM_EFAIL;
+ }
+
+ pgtlb = (npriv->vsatp & SATP_PPN) << PGTBL_PAGE_SIZE_SHIFT;
+ }
+
+ rc = mmu_get_guest_page(pgtlb, MMU_STAGE1, rc,
+ &nested_xlate_vsstage_ops, xc,
+ guest_gva, &page);
+ if (rc) {
+ return rc;
+ }
+ }
+
+ /* Check guest VS-stage page permissions */
+ return rc;
+ }
+ }
+
+ if (page.flags.user ?
+ xc->smode && (guest_access == NESTED_XLATE_FETCH ||
+ (xc->sstatus & SSTATUS_SUM)) :
+ !xc->smode) {
+ perm_fault = TRUE;
+ } else if (guest_access == NESTED_XLATE_FETCH || xc->hlvx) {
+ perm_fault = !page.flags.execute;
+ } else if (guest_access == NESTED_XLATE_LOAD) {
+ perm_fault = !(page.flags.read ||
+ ((xc->sstatus & SSTATUS_MXR) && page.flags.execute)) ||
+ !page.flags.accessed;
+ } else if (guest_access == NESTED_XLATE_STORE) {
+ perm_fault = !page.flags.read ||
+ !page.flags.write ||
+ !page.flags.accessed ||
+ !page.flags.dirty;
+ }
+ if (perm_fault) {
+ xc->vsstage_page_sz = page.sz;
+ nested_vsstage_write_fault(xc, guest_gva);
+ return VMM_EFAULT;
+ }
+
+ /* Calculate guest gpa */
+ }
+
+ guest_gpa = page.oa | (guest_gva & (page.sz - 1));
+
+ /* Translate guest gpa to host pa */
+ rc = nested_xlate_gstage(xc, guest_gpa, guest_access);
+ if (rc) {
+ if (rc == VMM_EFAULT) {
+ nested_vsstage_write_fault(xc, guest_gva);
+ }
+ return rc;
+ }
+
+ /* Update output address and size */
+ }
+
+ if (page.sz <= xc->host_sz) {
+ xc->host_pa |= guest_gpa & (xc->host_sz - 1);
+ xc->host_sz = page.sz;
+ xc->host_pa &= ~(xc->host_sz - 1);
+ } else {
+ page.ia = guest_gva & ~(xc->host_sz - 1);
+ page.oa = guest_gpa & ~(xc->host_sz - 1);
+ page.sz = xc->host_sz;
+ }
+
+ return VMM_OK;
+}
+
int cpu_vcpu_nested_init(struct vmm_vcpu *vcpu)
+ return VMM_OK;
+}
+
{
struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
@@ -462,7 +946,38 @@ int cpu_vcpu_nested_hlv(struct vmm_vcpu *vcpu, unsigned long vaddr,
unsigned long *out_stval,
unsigned long *out_htval)
{
- /* TODO: */
+ int rc;
+ physical_addr_t hpa;
+ struct nested_xlate_context xc;
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ /* Don't handle misaligned HLV */
+
+ if (vaddr & (len - 1)) {
+ *out_scause = CAUSE_MISALIGNED_LOAD;
+ *out_stval = vaddr;
+ *out_htval = 0;
+ return VMM_OK;
+ }
+
+ nested_xlate_context_init(&xc, vcpu, NESTED_XLATE_LOAD,
+ }
+
+ (npriv->hstatus & HSTATUS_SPVP) ? TRUE : FALSE,
+ csr_read(CSR_VSSTATUS), hlvx);
+ rc = nested_xlate_vsstage(&xc, vaddr, NESTED_XLATE_LOAD);
+ if (rc) {
+ if (rc == VMM_EFAULT) {
+ *out_scause = xc.scause;
+ *out_stval = xc.stval;
+ *out_htval = xc.htval;
+ return 0;
+ }
+ return rc;
+ }
+
+ hpa = xc.host_pa | (vaddr & (xc.host_sz - 1));
+ }
+
+ if (vmm_host_memory_read(hpa, data, len, TRUE) != len) {
+ return VMM_EIO;
+ }
+
return VMM_OK;
}
@@ -472,7 +987,38 @@ int cpu_vcpu_nested_hsv(struct vmm_vcpu *vcpu, unsigned long vaddr,
unsigned long *out_stval,
unsigned long *out_htval)
{
- /* TODO: */
+ int rc;
+ physical_addr_t hpa;
+ struct nested_xlate_context xc;
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ /* Don't handle misaligned HSV */
+
+ if (vaddr & (len - 1)) {
+ *out_scause = CAUSE_MISALIGNED_STORE;
+ *out_stval = vaddr;
+ *out_htval = 0;
+ return VMM_OK;
+ }
+
+ nested_xlate_context_init(&xc, vcpu, NESTED_XLATE_STORE,
+ }
+
+ (npriv->hstatus & HSTATUS_SPVP) ? TRUE : FALSE,
+ csr_read(CSR_VSSTATUS), FALSE);
+ rc = nested_xlate_vsstage(&xc, vaddr, NESTED_XLATE_STORE);
+ if (rc) {
+ if (rc == VMM_EFAULT) {
+ *out_scause = xc.scause;
+ *out_stval = xc.stval;
+ *out_htval = xc.htval;
+ return 0;
+ }
+ return rc;
+ }
+
+ hpa = xc.host_pa | (vaddr & (xc.host_sz - 1));
+ }
+
+ if (vmm_host_memory_write(hpa, (void *)data, len, TRUE) != len) {
+ return VMM_EIO;
+ }
+
return VMM_OK;
}
--
2.25.1
Anup Patel
Mar 17, 2022, 11:24:13 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
We make generic mmu_pgtbl more extensible by adding attributes and
hardware tag (i.e. ASID or VMID) for each page table. The page table
attributes include flags such as: type of TLB flush (i.e. remote or
local), whether hardware tag is valid, etc.
arch/arm/cpu/arm32ve/cpu_vcpu_helper.c | 10 ++++--
arch/arm/cpu/arm64/cpu_vcpu_helper.c | 10 ++++--
arch/arm/cpu/common/include/mmu_lpae.h | 7 ++--
arch/arm/cpu/common/mmu_lpae.c | 7 ++--
arch/common/generic_mmu.c | 37 +++++++++++++++++-----
arch/common/include/generic_mmu.h | 31 ++++++++++++++++--
arch/riscv/cpu/generic/cpu_mmu.c | 37 ++++++++++++++++------
arch/riscv/cpu/generic/cpu_vcpu_helper.c | 32 ++++++++-----------
arch/riscv/cpu/generic/cpu_vcpu_nested.c | 8 ++++-
arch/riscv/cpu/generic/include/arch_mmu.h | 7 ++--
arch/riscv/cpu/generic/include/cpu_tlb.h | 4 +--
libs/wboxtest/nested_mmu/nested_mmu_test.h | 3 +-
12 files changed, 136 insertions(+), 57 deletions(-)
diff --git a/arch/arm/cpu/arm32ve/cpu_vcpu_helper.c b/arch/arm/cpu/arm32ve/cpu_vcpu_helper.c
index 85b9de4f..b0c10fe4 100644
--- a/arch/arm/cpu/arm32ve/cpu_vcpu_helper.c
+++ b/arch/arm/cpu/arm32ve/cpu_vcpu_helper.c
@@ -441,13 +441,18 @@ int cpu_vcpu_spsr_update(struct vmm_vcpu *vcpu,
int arch_guest_init(struct vmm_guest *guest)
{
+ u32 pgtbl_attr;
+
if (!guest->reset_count) {
guest->arch_priv = vmm_zalloc(sizeof(struct arm_guest_priv));
if (!guest->arch_priv) {
return VMM_ENOMEM;
}
- arm_guest_priv(guest)->ttbl = mmu_pgtbl_alloc(MMU_STAGE2, -1);
+ pgtbl_attr = MMU_ATTR_REMOTE_TLB_FLUSH;
+ pgtbl_attr |= MMU_ATTR_HW_TAG_VALID;
+ arm_guest_priv(guest)->ttbl = mmu_pgtbl_alloc(MMU_STAGE2, -1,
+ pgtbl_attr, guest->id);
if (!arm_guest_priv(guest)->ttbl) {
vmm_free(guest->arch_priv);
guest->arch_priv = NULL;
@@ -936,8 +941,7 @@ void arch_vcpu_switch(struct vmm_vcpu *tvcpu,
write_hcptr(arm_priv(vcpu)->hcptr);
write_hstr(arm_priv(vcpu)->hstr);
/* Update hypervisor Stage2 MMU context */
- mmu_stage2_change_pgtbl(vcpu->guest->id,
- arm_guest_priv(vcpu->guest)->ttbl);
+ mmu_stage2_change_pgtbl(arm_guest_priv(vcpu->guest)->ttbl);
/* Flush TLB if moved to new host CPU */
if (arm_priv(vcpu)->last_hcpu != vmm_smp_processor_id()) {
/* Invalidate all guest TLB enteries because
diff --git a/arch/arm/cpu/arm64/cpu_vcpu_helper.c b/arch/arm/cpu/arm64/cpu_vcpu_helper.c
index badb9739..8fae3bcb 100644
--- a/arch/arm/cpu/arm64/cpu_vcpu_helper.c
+++ b/arch/arm/cpu/arm64/cpu_vcpu_helper.c
@@ -303,13 +303,18 @@ void cpu_vcpu_reg_write(struct vmm_vcpu *vcpu,
int arch_guest_init(struct vmm_guest *guest)
{
+ u32 pgtbl_attr;
+
if (!guest->reset_count) {
guest->arch_priv = vmm_malloc(sizeof(struct arm_guest_priv));
if (!guest->arch_priv) {
return VMM_ENOMEM;
}
- arm_guest_priv(guest)->ttbl = mmu_pgtbl_alloc(MMU_STAGE2, -1);
+ pgtbl_attr = MMU_ATTR_REMOTE_TLB_FLUSH;
+ pgtbl_attr |= MMU_ATTR_HW_TAG_VALID;
+ arm_guest_priv(guest)->ttbl = mmu_pgtbl_alloc(MMU_STAGE2, -1,
+ pgtbl_attr, guest->id);
if (!arm_guest_priv(guest)->ttbl) {
vmm_free(guest->arch_priv);
guest->arch_priv = NULL;
@@ -786,8 +791,7 @@ void arch_vcpu_switch(struct vmm_vcpu *tvcpu,
msr(cptr_el2, arm_priv(vcpu)->cptr);
msr(hstr_el2, arm_priv(vcpu)->hstr);
/* Update hypervisor Stage2 MMU context */
- mmu_stage2_change_pgtbl(vcpu->guest->id,
- arm_guest_priv(vcpu->guest)->ttbl);
+ mmu_stage2_change_pgtbl(arm_guest_priv(vcpu->guest)->ttbl);
/* Flush TLB if moved to new host CPU */
if (arm_priv(vcpu)->last_hcpu != vmm_smp_processor_id()) {
/* Invalidate all guest TLB enteries because
diff --git a/arch/arm/cpu/common/include/mmu_lpae.h b/arch/arm/cpu/common/include/mmu_lpae.h
index 6d86d350..a90662db 100644
--- a/arch/arm/cpu/common/include/mmu_lpae.h
+++ b/arch/arm/cpu/common/include/mmu_lpae.h
@@ -172,10 +172,10 @@ int arch_mmu_pgtbl_align_order(int stage, int level);
int arch_mmu_pgtbl_size_order(int stage, int level);
-void arch_mmu_stage2_tlbflush(bool remote,
+void arch_mmu_stage2_tlbflush(bool remote, bool use_vmid, u32 vmid,
physical_addr_t gpa, physical_size_t gsz);
-void arch_mmu_stage1_tlbflush(bool remote,
+void arch_mmu_stage1_tlbflush(bool remote, bool use_asid, u32 asid,
virtual_addr_t va, virtual_size_t sz);
bool arch_mmu_valid_block_size(physical_size_t sz);
@@ -225,7 +225,8 @@ physical_addr_t arch_mmu_stage2_current_pgtbl_addr(void);
u32 arch_mmu_stage2_current_vmid(void);
-int arch_mmu_stage2_change_pgtbl(u32 vmid, physical_addr_t tbl_phys);
+int arch_mmu_stage2_change_pgtbl(bool have_vmid, u32 vmid,
+ physical_addr_t tbl_phys);
#endif /* !__ASSEMBLY__ */
diff --git a/arch/arm/cpu/common/mmu_lpae.c b/arch/arm/cpu/common/mmu_lpae.c
index 1dd67b6d..db8646d3 100644
--- a/arch/arm/cpu/common/mmu_lpae.c
+++ b/arch/arm/cpu/common/mmu_lpae.c
@@ -44,13 +44,13 @@ int arch_mmu_pgtbl_size_order(int stage, int level)
return TTBL_L3_BLOCK_SHIFT;
}
-void arch_mmu_stage2_tlbflush(bool remote,
+void arch_mmu_stage2_tlbflush(bool remote, bool use_vmid, u32 vmid,
physical_addr_t gpa, physical_size_t gsz)
{
cpu_invalid_ipa_guest_tlb(gpa);
}
-void arch_mmu_stage1_tlbflush(bool remote,
+void arch_mmu_stage1_tlbflush(bool remote, bool use_asid, u32 asid,
virtual_addr_t va, virtual_size_t sz)
{
cpu_invalid_va_hypervisor_tlb(va);
@@ -440,7 +440,8 @@ u32 arch_mmu_stage2_current_vmid(void)
return cpu_stage2_vmid();
}
-int arch_mmu_stage2_change_pgtbl(u32 vmid, physical_addr_t tbl_phys)
+int arch_mmu_stage2_change_pgtbl(bool have_vmid, u32 vmid,
+ physical_addr_t tbl_phys)
{
cpu_stage2_update(tbl_phys, vmid);
diff --git a/arch/common/generic_mmu.c b/arch/common/generic_mmu.c
index 2c468f04..d617a5cb 100644
--- a/arch/common/generic_mmu.c
+++ b/arch/common/generic_mmu.c
@@ -389,7 +389,7 @@ static int mmu_pgtbl_deattach(struct mmu_pgtbl *child)
return VMM_OK;
}
-struct mmu_pgtbl *mmu_pgtbl_alloc(int stage, int level)
+struct mmu_pgtbl *mmu_pgtbl_alloc(int stage, int level, u32 attr, u32 hw_tag)
{
struct mmu_pgtbl *pgtbl;
@@ -409,6 +409,8 @@ struct mmu_pgtbl *mmu_pgtbl_alloc(int stage, int level)
pgtbl->parent = NULL;
pgtbl->stage = stage;
pgtbl->level = level;
+ pgtbl->attr = attr;
+ pgtbl->hw_tag = hw_tag;
pgtbl->map_ia = 0;
INIT_SPIN_LOCK(&pgtbl->tbl_lock);
pgtbl->pte_cnt = 0;
@@ -454,6 +456,8 @@ int mmu_pgtbl_free(struct mmu_pgtbl *pgtbl)
stage = pgtbl->stage;
pgtbl->stage = 0;
pgtbl->level = 0;
+ pgtbl->attr = 0;
+ pgtbl->hw_tag = 0;
pgtbl->map_ia = 0;
if (stage == MMU_STAGE1)
@@ -503,7 +507,8 @@ struct mmu_pgtbl *mmu_pgtbl_get_child(struct mmu_pgtbl *parent,
return NULL;
}
- child = mmu_pgtbl_alloc(parent->stage, parent->level - 1);
+ child = mmu_pgtbl_alloc(parent->stage, parent->level - 1,
+ parent->attr, parent->hw_tag);
if (!child) {
return NULL;
}
@@ -620,9 +625,17 @@ int mmu_unmap_page(struct mmu_pgtbl *pgtbl, struct mmu_page *pg)
arch_mmu_pte_sync(&pte[index], pgtbl->stage, pgtbl->level);
if (pgtbl->stage == MMU_STAGE1) {
- arch_mmu_stage1_tlbflush(TRUE, pg->ia, blksz);
+ arch_mmu_stage1_tlbflush(
+ mmu_pgtbl_need_remote_tlbflush(pgtbl),
+ mmu_pgtbl_has_hw_tag(pgtbl),
+ mmu_pgtbl_hw_tag(pgtbl),
+ pg->ia, blksz);
} else {
- arch_mmu_stage2_tlbflush(TRUE, pg->ia, blksz);
+ arch_mmu_stage2_tlbflush(
+ mmu_pgtbl_need_remote_tlbflush(pgtbl),
+ mmu_pgtbl_has_hw_tag(pgtbl),
+ mmu_pgtbl_hw_tag(pgtbl),
+ pg->ia, blksz);
}
pgtbl->pte_cnt--;
@@ -683,9 +696,17 @@ int mmu_map_page(struct mmu_pgtbl *pgtbl, struct mmu_page *pg)
arch_mmu_pte_sync(&pte[index], pgtbl->stage, pgtbl->level);
if (pgtbl->stage == MMU_STAGE1) {
- arch_mmu_stage1_tlbflush(TRUE, pg->ia, blksz);
+ arch_mmu_stage1_tlbflush(
+ mmu_pgtbl_need_remote_tlbflush(pgtbl),
+ mmu_pgtbl_has_hw_tag(pgtbl),
+ mmu_pgtbl_hw_tag(pgtbl),
+ pg->ia, blksz);
} else {
- arch_mmu_stage2_tlbflush(TRUE, pg->ia, blksz);
+ arch_mmu_stage2_tlbflush(
+ mmu_pgtbl_need_remote_tlbflush(pgtbl),
+ mmu_pgtbl_has_hw_tag(pgtbl),
+ mmu_pgtbl_hw_tag(pgtbl),
+ pg->ia, blksz);
}
pgtbl->pte_cnt++;
@@ -1145,7 +1166,7 @@ int arch_cpu_aspace_memory_read(virtual_addr_t tmp_va,
arch_mmu_pte_set(pte , MMU_STAGE1, pgtbl_level, src, flags);
arch_mmu_pte_sync(pte, MMU_STAGE1, pgtbl_level);
- arch_mmu_stage1_tlbflush(FALSE, tmp_va, VMM_PAGE_SIZE);
+ arch_mmu_stage1_tlbflush(FALSE, FALSE, 0, tmp_va, VMM_PAGE_SIZE);
switch (len) {
case 1:
@@ -1187,7 +1208,7 @@ int arch_cpu_aspace_memory_write(virtual_addr_t tmp_va,
arch_mmu_pte_set(pte , MMU_STAGE1, pgtbl_level, dst, flags);
arch_mmu_pte_sync(pte, MMU_STAGE1, pgtbl_level);
- arch_mmu_stage1_tlbflush(FALSE, tmp_va, VMM_PAGE_SIZE);
+ arch_mmu_stage1_tlbflush(FALSE, FALSE, 0, tmp_va, VMM_PAGE_SIZE);
switch (len) {
case 1:
diff --git a/arch/common/include/generic_mmu.h b/arch/common/include/generic_mmu.h
index 9e53c9c4..e1866c5c 100644
--- a/arch/common/include/generic_mmu.h
+++ b/arch/common/include/generic_mmu.h
@@ -44,12 +44,18 @@ enum mmu_stage {
MMU_STAGE_MAX
};
+/** MMU page table attributes */
+#define MMU_ATTR_REMOTE_TLB_FLUSH (1 << 0)
+#define MMU_ATTR_HW_TAG_VALID (1 << 1)
+
/** MMU page table */
struct mmu_pgtbl {
struct dlist head;
struct mmu_pgtbl *parent;
enum mmu_stage stage;
int level;
+ u32 attr;
+ u32 hw_tag;
physical_addr_t map_ia;
physical_addr_t tbl_pa;
vmm_spinlock_t tbl_lock; /*< Lock to protect table contents,
@@ -74,7 +80,7 @@ u64 mmu_pgtbl_count(int stage, int level);
struct mmu_pgtbl *mmu_pgtbl_find(int stage, physical_addr_t tbl_pa);
-struct mmu_pgtbl *mmu_pgtbl_alloc(int stage, int level);
+struct mmu_pgtbl *mmu_pgtbl_alloc(int stage, int level, u32 attr, u32 hw_tag);
int mmu_pgtbl_free(struct mmu_pgtbl *pgtbl);
@@ -88,6 +94,23 @@ static inline int mmu_pgtbl_level(struct mmu_pgtbl *pgtbl)
return (pgtbl) ? pgtbl->level : -1;
}
+static inline bool mmu_pgtbl_need_remote_tlbflush(struct mmu_pgtbl *pgtbl)
+{
+ return (pgtbl && (pgtbl->attr & MMU_ATTR_REMOTE_TLB_FLUSH)) ?
+ TRUE : FALSE;
+}
+
+static inline bool mmu_pgtbl_has_hw_tag(struct mmu_pgtbl *pgtbl)
+{
+ return (pgtbl && (pgtbl->attr & MMU_ATTR_HW_TAG_VALID)) ?
+ TRUE : FALSE;
+}
+
+static inline u32 mmu_pgtbl_hw_tag(struct mmu_pgtbl *pgtbl)
+{
+ return (pgtbl) ? pgtbl->hw_tag : 0;
+}
+
static inline physical_addr_t mmu_pgtbl_map_addr(struct mmu_pgtbl *pgtbl)
{
return (pgtbl) ? pgtbl->map_ia : 0;
@@ -198,9 +221,11 @@ static inline u32 mmu_stage2_current_vmid(void)
return arch_mmu_stage2_current_vmid();
}
-static inline int mmu_stage2_change_pgtbl(u32 vmid, struct mmu_pgtbl *pgtbl)
+static inline int mmu_stage2_change_pgtbl(struct mmu_pgtbl *pgtbl)
{
- return arch_mmu_stage2_change_pgtbl(vmid, pgtbl->tbl_pa);
+ return arch_mmu_stage2_change_pgtbl(mmu_pgtbl_has_hw_tag(pgtbl),
+ mmu_pgtbl_hw_tag(pgtbl),
+ pgtbl->tbl_pa);
}
#endif
diff --git a/arch/riscv/cpu/generic/cpu_mmu.c b/arch/riscv/cpu/generic/cpu_mmu.c
index 853cc9ad..d8a1d1c8 100644
--- a/arch/riscv/cpu/generic/cpu_mmu.c
+++ b/arch/riscv/cpu/generic/cpu_mmu.c
@@ -63,29 +63,47 @@ int arch_mmu_pgtbl_size_order(int stage, int level)
return PGTBL_PAGE_SIZE_SHIFT;
}
-void arch_mmu_stage2_tlbflush(bool remote,
+void arch_mmu_stage2_tlbflush(bool remote, bool use_vmid, u32 vmid,
physical_addr_t gpa, physical_size_t gsz)
{
physical_addr_t off;
if (remote) {
- sbi_remote_hfence_gvma(NULL, gpa, gsz);
+ if (use_vmid) {
+ sbi_remote_hfence_gvma_vmid(NULL, gpa, gsz, vmid);
+ } else {
+ sbi_remote_hfence_gvma(NULL, gpa, gsz);
+ }
} else {
- for (off = 0; off < gsz; off += VMM_PAGE_SIZE)
- __hfence_gvma_gpa((gpa + off) >> 2);
+ if (use_vmid) {
+ for (off = 0; off < gsz; off += VMM_PAGE_SIZE)
+ __hfence_gvma_vmid_gpa((gpa + off) >> 2, vmid);
+ } else {
+ for (off = 0; off < gsz; off += VMM_PAGE_SIZE)
+ __hfence_gvma_gpa((gpa + off) >> 2);
+ }
}
}
-void arch_mmu_stage1_tlbflush(bool remote,
+void arch_mmu_stage1_tlbflush(bool remote, bool use_asid, u32 asid,
virtual_addr_t va, virtual_size_t sz)
{
virtual_addr_t off;
if (remote) {
- sbi_remote_sfence_vma(NULL, va, sz);
+ if (use_asid) {
+ sbi_remote_sfence_vma_asid(NULL, va, sz, asid);
+ } else {
+ sbi_remote_sfence_vma(NULL, va, sz);
+ }
} else {
- for (off = 0; off < sz; off += VMM_PAGE_SIZE)
- __sfence_vma_va(va + off);
+ if (use_asid) {
+ for (off = 0; off < sz; off += VMM_PAGE_SIZE)
+ __sfence_vma_asid_va(asid, va + off);
+ } else {
+ for (off = 0; off < sz; off += VMM_PAGE_SIZE)
+ __sfence_vma_va(va + off);
+ }
}
}
@@ -683,7 +701,8 @@ u32 arch_mmu_stage2_current_vmid(void)
+int arch_mmu_stage2_change_pgtbl(bool have_vmid, u32 vmid,
+ physical_addr_t tbl_phys)
{
unsigned long hgatp;
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_helper.c b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
index e33bce24..75862d93 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_helper.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
@@ -102,6 +102,7 @@ int arch_guest_init(struct vmm_guest *guest)
{
struct riscv_guest_priv *priv;
struct riscv_guest_serial *gserial;
+ u32 pgtbl_attr, pgtbl_hw_tag;
char *sname;
int rc;
@@ -118,7 +119,14 @@ int arch_guest_init(struct vmm_guest *guest)
priv->time_delta = -get_cycles64();
- priv->pgtbl = mmu_pgtbl_alloc(MMU_STAGE2, -1);
+ pgtbl_hw_tag = 0;
+ pgtbl_attr = MMU_ATTR_REMOTE_TLB_FLUSH;
+ if (riscv_stage2_vmid_available()) {
+ pgtbl_hw_tag = guest->id;
+ pgtbl_attr |= MMU_ATTR_HW_TAG_VALID;
+ }
+ priv->pgtbl = mmu_pgtbl_alloc(MMU_STAGE2, -1,
+ pgtbl_attr, pgtbl_hw_tag);
if (!priv->pgtbl) {
vmm_free(guest->arch_priv);
guest->arch_priv = NULL;
@@ -476,24 +484,12 @@ void cpu_vcpu_time_delta_update(struct vmm_vcpu *vcpu, bool nested_virt)
void cpu_vcpu_gstage_update(struct vmm_vcpu *vcpu, bool nested_virt)
{
- if (riscv_stage2_vmid_available()) {
- if (nested_virt) {
- mmu_stage2_change_pgtbl(
- riscv_stage2_vmid_nested + vcpu->guest->id,
- riscv_nested_priv(vcpu)->pgtbl);
- } else {
- if (nested_virt) {
- mmu_stage2_change_pgtbl(0,
- riscv_nested_priv(vcpu)->pgtbl);
+ riscv_nested_priv(vcpu)->pgtbl :
+ riscv_guest_priv(vcpu->guest)->pgtbl;
+ mmu_stage2_change_pgtbl(pgtbl);
+ if (!mmu_pgtbl_has_hw_tag(pgtbl)) {
/*
index e5d95a97..ad074747 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_nested.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_nested.c
@@ -517,9 +517,15 @@ static int nested_xlate_vsstage(struct nested_xlate_context *xc,
int cpu_vcpu_nested_init(struct vmm_vcpu *vcpu)
{
+ u32 pgtbl_attr = 0, pgtbl_hw_tag = 0;
+ if (riscv_stage2_vmid_available()) {
+ pgtbl_hw_tag = riscv_stage2_vmid_nested + vcpu->guest->id;
+ pgtbl_attr |= MMU_ATTR_HW_TAG_VALID;
+ }
+ npriv->pgtbl = mmu_pgtbl_alloc(MMU_STAGE2, -1, pgtbl_attr,
+ pgtbl_hw_tag);
if (!npriv->pgtbl) {
return VMM_ENOMEM;
}
diff --git a/arch/riscv/cpu/generic/include/arch_mmu.h b/arch/riscv/cpu/generic/include/arch_mmu.h
index e67b1176..b78cbc76 100644
--- a/arch/riscv/cpu/generic/include/arch_mmu.h
+++ b/arch/riscv/cpu/generic/include/arch_mmu.h
@@ -133,10 +133,10 @@ int arch_mmu_pgtbl_align_order(int stage, int level);
int arch_mmu_pgtbl_size_order(int stage, int level);
-void arch_mmu_stage2_tlbflush(bool remote,
+void arch_mmu_stage2_tlbflush(bool remote, bool use_vmid, u32 vmid,
physical_addr_t gpa, physical_size_t gsz);
-void arch_mmu_stage1_tlbflush(bool remote,
+void arch_mmu_stage1_tlbflush(bool remote, bool use_asid, u32 asid,
virtual_addr_t va, virtual_size_t sz);
bool arch_mmu_valid_block_size(physical_size_t sz);
@@ -186,7 +186,8 @@ physical_addr_t arch_mmu_stage2_current_pgtbl_addr(void);
u32 arch_mmu_stage2_current_vmid(void);
-int arch_mmu_stage2_change_pgtbl(u32 vmid, physical_addr_t tbl_phys);
+int arch_mmu_stage2_change_pgtbl(bool have_vmid, u32 vmid,
+ physical_addr_t tbl_phys);
#endif
diff --git a/arch/riscv/cpu/generic/include/cpu_tlb.h b/arch/riscv/cpu/generic/include/cpu_tlb.h
index 3ed19ade..853fb067 100644
--- a/arch/riscv/cpu/generic/include/cpu_tlb.h
+++ b/arch/riscv/cpu/generic/include/cpu_tlb.h
@@ -52,7 +52,7 @@ void __hfence_vvma_all(void);
inline void __sfence_vma_asid_va(unsigned long asid, unsigned long va)
{
- __asm__ __volatile__("sfence.vma %0 %1"
+ __asm__ __volatile__("sfence.vma %0, %1"
:
: "r"(va),"r"(asid)
: "memory");
@@ -60,7 +60,7 @@ inline void __sfence_vma_asid_va(unsigned long asid, unsigned long va)
inline void __sfence_vma_asid(unsigned long asid)
{
- __asm__ __volatile__("sfence.vma x0 %1"
+ __asm__ __volatile__("sfence.vma x0, %1"
:
: "r"(asid)
: "memory");
diff --git a/libs/wboxtest/nested_mmu/nested_mmu_test.h b/libs/wboxtest/nested_mmu/nested_mmu_test.h
index 07b73822..33b1f425 100755
--- a/libs/wboxtest/nested_mmu/nested_mmu_test.h
+++ b/libs/wboxtest/nested_mmu/nested_mmu_test.h
@@ -122,7 +122,8 @@ do { \
DPRINTF((__cdev), "%s: Allocating Stage%s page table ", \
(__test)->name, \
((__stage) == MMU_STAGE2) ? "2" : "1"); \
- *(__output_pgtbl_double_ptr) = mmu_pgtbl_alloc((__stage), -1); \
+ *(__output_pgtbl_double_ptr) = mmu_pgtbl_alloc((__stage), -1, \
+ MMU_ATTR_REMOTE_TLB_FLUSH, 0); \
DPRINTF((__cdev), "%s", \
(!*(__output_pgtbl_double_ptr)) ? "(failed)\n" : ""); \
if (!*(__output_pgtbl_double_ptr)) { \
--
2.25.1
hardware tag (i.e. ASID or VMID) for each page table. The page table
attributes include flags such as: type of TLB flush (i.e. remote or
local), whether hardware tag is valid, etc.
arch/arm/cpu/arm32ve/cpu_vcpu_helper.c | 10 ++++--
arch/arm/cpu/arm64/cpu_vcpu_helper.c | 10 ++++--
arch/arm/cpu/common/include/mmu_lpae.h | 7 ++--
arch/arm/cpu/common/mmu_lpae.c | 7 ++--
arch/common/generic_mmu.c | 37 +++++++++++++++++-----
arch/common/include/generic_mmu.h | 31 ++++++++++++++++--
arch/riscv/cpu/generic/cpu_mmu.c | 37 ++++++++++++++++------
arch/riscv/cpu/generic/cpu_vcpu_helper.c | 32 ++++++++-----------
arch/riscv/cpu/generic/cpu_vcpu_nested.c | 8 ++++-
arch/riscv/cpu/generic/include/arch_mmu.h | 7 ++--
arch/riscv/cpu/generic/include/cpu_tlb.h | 4 +--
libs/wboxtest/nested_mmu/nested_mmu_test.h | 3 +-
12 files changed, 136 insertions(+), 57 deletions(-)
diff --git a/arch/arm/cpu/arm32ve/cpu_vcpu_helper.c b/arch/arm/cpu/arm32ve/cpu_vcpu_helper.c
index 85b9de4f..b0c10fe4 100644
--- a/arch/arm/cpu/arm32ve/cpu_vcpu_helper.c
+++ b/arch/arm/cpu/arm32ve/cpu_vcpu_helper.c
@@ -441,13 +441,18 @@ int cpu_vcpu_spsr_update(struct vmm_vcpu *vcpu,
int arch_guest_init(struct vmm_guest *guest)
{
+ u32 pgtbl_attr;
+
if (!guest->reset_count) {
guest->arch_priv = vmm_zalloc(sizeof(struct arm_guest_priv));
if (!guest->arch_priv) {
return VMM_ENOMEM;
}
- arm_guest_priv(guest)->ttbl = mmu_pgtbl_alloc(MMU_STAGE2, -1);
+ pgtbl_attr = MMU_ATTR_REMOTE_TLB_FLUSH;
+ pgtbl_attr |= MMU_ATTR_HW_TAG_VALID;
+ arm_guest_priv(guest)->ttbl = mmu_pgtbl_alloc(MMU_STAGE2, -1,
+ pgtbl_attr, guest->id);
if (!arm_guest_priv(guest)->ttbl) {
vmm_free(guest->arch_priv);
guest->arch_priv = NULL;
@@ -936,8 +941,7 @@ void arch_vcpu_switch(struct vmm_vcpu *tvcpu,
write_hcptr(arm_priv(vcpu)->hcptr);
write_hstr(arm_priv(vcpu)->hstr);
/* Update hypervisor Stage2 MMU context */
- mmu_stage2_change_pgtbl(vcpu->guest->id,
- arm_guest_priv(vcpu->guest)->ttbl);
+ mmu_stage2_change_pgtbl(arm_guest_priv(vcpu->guest)->ttbl);
/* Flush TLB if moved to new host CPU */
if (arm_priv(vcpu)->last_hcpu != vmm_smp_processor_id()) {
/* Invalidate all guest TLB enteries because
diff --git a/arch/arm/cpu/arm64/cpu_vcpu_helper.c b/arch/arm/cpu/arm64/cpu_vcpu_helper.c
index badb9739..8fae3bcb 100644
--- a/arch/arm/cpu/arm64/cpu_vcpu_helper.c
+++ b/arch/arm/cpu/arm64/cpu_vcpu_helper.c
@@ -303,13 +303,18 @@ void cpu_vcpu_reg_write(struct vmm_vcpu *vcpu,
int arch_guest_init(struct vmm_guest *guest)
{
+ u32 pgtbl_attr;
+
if (!guest->reset_count) {
guest->arch_priv = vmm_malloc(sizeof(struct arm_guest_priv));
if (!guest->arch_priv) {
return VMM_ENOMEM;
}
- arm_guest_priv(guest)->ttbl = mmu_pgtbl_alloc(MMU_STAGE2, -1);
+ pgtbl_attr = MMU_ATTR_REMOTE_TLB_FLUSH;
+ pgtbl_attr |= MMU_ATTR_HW_TAG_VALID;
+ arm_guest_priv(guest)->ttbl = mmu_pgtbl_alloc(MMU_STAGE2, -1,
+ pgtbl_attr, guest->id);
if (!arm_guest_priv(guest)->ttbl) {
vmm_free(guest->arch_priv);
guest->arch_priv = NULL;
@@ -786,8 +791,7 @@ void arch_vcpu_switch(struct vmm_vcpu *tvcpu,
msr(cptr_el2, arm_priv(vcpu)->cptr);
msr(hstr_el2, arm_priv(vcpu)->hstr);
/* Update hypervisor Stage2 MMU context */
- mmu_stage2_change_pgtbl(vcpu->guest->id,
- arm_guest_priv(vcpu->guest)->ttbl);
+ mmu_stage2_change_pgtbl(arm_guest_priv(vcpu->guest)->ttbl);
/* Flush TLB if moved to new host CPU */
if (arm_priv(vcpu)->last_hcpu != vmm_smp_processor_id()) {
/* Invalidate all guest TLB enteries because
diff --git a/arch/arm/cpu/common/include/mmu_lpae.h b/arch/arm/cpu/common/include/mmu_lpae.h
index 6d86d350..a90662db 100644
--- a/arch/arm/cpu/common/include/mmu_lpae.h
+++ b/arch/arm/cpu/common/include/mmu_lpae.h
@@ -172,10 +172,10 @@ int arch_mmu_pgtbl_align_order(int stage, int level);
int arch_mmu_pgtbl_size_order(int stage, int level);
-void arch_mmu_stage2_tlbflush(bool remote,
+void arch_mmu_stage2_tlbflush(bool remote, bool use_vmid, u32 vmid,
physical_addr_t gpa, physical_size_t gsz);
-void arch_mmu_stage1_tlbflush(bool remote,
+void arch_mmu_stage1_tlbflush(bool remote, bool use_asid, u32 asid,
virtual_addr_t va, virtual_size_t sz);
bool arch_mmu_valid_block_size(physical_size_t sz);
@@ -225,7 +225,8 @@ physical_addr_t arch_mmu_stage2_current_pgtbl_addr(void);
u32 arch_mmu_stage2_current_vmid(void);
-int arch_mmu_stage2_change_pgtbl(u32 vmid, physical_addr_t tbl_phys);
+int arch_mmu_stage2_change_pgtbl(bool have_vmid, u32 vmid,
+ physical_addr_t tbl_phys);
#endif /* !__ASSEMBLY__ */
diff --git a/arch/arm/cpu/common/mmu_lpae.c b/arch/arm/cpu/common/mmu_lpae.c
index 1dd67b6d..db8646d3 100644
--- a/arch/arm/cpu/common/mmu_lpae.c
+++ b/arch/arm/cpu/common/mmu_lpae.c
@@ -44,13 +44,13 @@ int arch_mmu_pgtbl_size_order(int stage, int level)
return TTBL_L3_BLOCK_SHIFT;
}
-void arch_mmu_stage2_tlbflush(bool remote,
+void arch_mmu_stage2_tlbflush(bool remote, bool use_vmid, u32 vmid,
physical_addr_t gpa, physical_size_t gsz)
{
cpu_invalid_ipa_guest_tlb(gpa);
}
-void arch_mmu_stage1_tlbflush(bool remote,
+void arch_mmu_stage1_tlbflush(bool remote, bool use_asid, u32 asid,
virtual_addr_t va, virtual_size_t sz)
{
cpu_invalid_va_hypervisor_tlb(va);
@@ -440,7 +440,8 @@ u32 arch_mmu_stage2_current_vmid(void)
return cpu_stage2_vmid();
}
-int arch_mmu_stage2_change_pgtbl(u32 vmid, physical_addr_t tbl_phys)
+int arch_mmu_stage2_change_pgtbl(bool have_vmid, u32 vmid,
+ physical_addr_t tbl_phys)
{
cpu_stage2_update(tbl_phys, vmid);
diff --git a/arch/common/generic_mmu.c b/arch/common/generic_mmu.c
index 2c468f04..d617a5cb 100644
--- a/arch/common/generic_mmu.c
+++ b/arch/common/generic_mmu.c
@@ -389,7 +389,7 @@ static int mmu_pgtbl_deattach(struct mmu_pgtbl *child)
return VMM_OK;
}
-struct mmu_pgtbl *mmu_pgtbl_alloc(int stage, int level)
+struct mmu_pgtbl *mmu_pgtbl_alloc(int stage, int level, u32 attr, u32 hw_tag)
{
struct mmu_pgtbl *pgtbl;
@@ -409,6 +409,8 @@ struct mmu_pgtbl *mmu_pgtbl_alloc(int stage, int level)
pgtbl->parent = NULL;
pgtbl->stage = stage;
pgtbl->level = level;
+ pgtbl->attr = attr;
+ pgtbl->hw_tag = hw_tag;
pgtbl->map_ia = 0;
INIT_SPIN_LOCK(&pgtbl->tbl_lock);
pgtbl->pte_cnt = 0;
@@ -454,6 +456,8 @@ int mmu_pgtbl_free(struct mmu_pgtbl *pgtbl)
stage = pgtbl->stage;
pgtbl->stage = 0;
pgtbl->level = 0;
+ pgtbl->attr = 0;
+ pgtbl->hw_tag = 0;
pgtbl->map_ia = 0;
if (stage == MMU_STAGE1)
@@ -503,7 +507,8 @@ struct mmu_pgtbl *mmu_pgtbl_get_child(struct mmu_pgtbl *parent,
return NULL;
}
- child = mmu_pgtbl_alloc(parent->stage, parent->level - 1);
+ child = mmu_pgtbl_alloc(parent->stage, parent->level - 1,
+ parent->attr, parent->hw_tag);
if (!child) {
return NULL;
}
@@ -620,9 +625,17 @@ int mmu_unmap_page(struct mmu_pgtbl *pgtbl, struct mmu_page *pg)
arch_mmu_pte_sync(&pte[index], pgtbl->stage, pgtbl->level);
if (pgtbl->stage == MMU_STAGE1) {
- arch_mmu_stage1_tlbflush(TRUE, pg->ia, blksz);
+ arch_mmu_stage1_tlbflush(
+ mmu_pgtbl_need_remote_tlbflush(pgtbl),
+ mmu_pgtbl_has_hw_tag(pgtbl),
+ mmu_pgtbl_hw_tag(pgtbl),
+ pg->ia, blksz);
} else {
- arch_mmu_stage2_tlbflush(TRUE, pg->ia, blksz);
+ arch_mmu_stage2_tlbflush(
+ mmu_pgtbl_need_remote_tlbflush(pgtbl),
+ mmu_pgtbl_has_hw_tag(pgtbl),
+ mmu_pgtbl_hw_tag(pgtbl),
+ pg->ia, blksz);
}
pgtbl->pte_cnt--;
@@ -683,9 +696,17 @@ int mmu_map_page(struct mmu_pgtbl *pgtbl, struct mmu_page *pg)
arch_mmu_pte_sync(&pte[index], pgtbl->stage, pgtbl->level);
if (pgtbl->stage == MMU_STAGE1) {
- arch_mmu_stage1_tlbflush(TRUE, pg->ia, blksz);
+ arch_mmu_stage1_tlbflush(
+ mmu_pgtbl_need_remote_tlbflush(pgtbl),
+ mmu_pgtbl_has_hw_tag(pgtbl),
+ mmu_pgtbl_hw_tag(pgtbl),
+ pg->ia, blksz);
} else {
- arch_mmu_stage2_tlbflush(TRUE, pg->ia, blksz);
+ arch_mmu_stage2_tlbflush(
+ mmu_pgtbl_need_remote_tlbflush(pgtbl),
+ mmu_pgtbl_has_hw_tag(pgtbl),
+ mmu_pgtbl_hw_tag(pgtbl),
+ pg->ia, blksz);
}
pgtbl->pte_cnt++;
@@ -1145,7 +1166,7 @@ int arch_cpu_aspace_memory_read(virtual_addr_t tmp_va,
arch_mmu_pte_set(pte , MMU_STAGE1, pgtbl_level, src, flags);
arch_mmu_pte_sync(pte, MMU_STAGE1, pgtbl_level);
- arch_mmu_stage1_tlbflush(FALSE, tmp_va, VMM_PAGE_SIZE);
+ arch_mmu_stage1_tlbflush(FALSE, FALSE, 0, tmp_va, VMM_PAGE_SIZE);
switch (len) {
case 1:
@@ -1187,7 +1208,7 @@ int arch_cpu_aspace_memory_write(virtual_addr_t tmp_va,
arch_mmu_pte_set(pte , MMU_STAGE1, pgtbl_level, dst, flags);
arch_mmu_pte_sync(pte, MMU_STAGE1, pgtbl_level);
- arch_mmu_stage1_tlbflush(FALSE, tmp_va, VMM_PAGE_SIZE);
+ arch_mmu_stage1_tlbflush(FALSE, FALSE, 0, tmp_va, VMM_PAGE_SIZE);
switch (len) {
case 1:
diff --git a/arch/common/include/generic_mmu.h b/arch/common/include/generic_mmu.h
index 9e53c9c4..e1866c5c 100644
--- a/arch/common/include/generic_mmu.h
+++ b/arch/common/include/generic_mmu.h
@@ -44,12 +44,18 @@ enum mmu_stage {
MMU_STAGE_MAX
};
+/** MMU page table attributes */
+#define MMU_ATTR_REMOTE_TLB_FLUSH (1 << 0)
+#define MMU_ATTR_HW_TAG_VALID (1 << 1)
+
/** MMU page table */
struct mmu_pgtbl {
struct dlist head;
struct mmu_pgtbl *parent;
enum mmu_stage stage;
int level;
+ u32 attr;
+ u32 hw_tag;
physical_addr_t map_ia;
physical_addr_t tbl_pa;
vmm_spinlock_t tbl_lock; /*< Lock to protect table contents,
@@ -74,7 +80,7 @@ u64 mmu_pgtbl_count(int stage, int level);
struct mmu_pgtbl *mmu_pgtbl_find(int stage, physical_addr_t tbl_pa);
-struct mmu_pgtbl *mmu_pgtbl_alloc(int stage, int level);
+struct mmu_pgtbl *mmu_pgtbl_alloc(int stage, int level, u32 attr, u32 hw_tag);
int mmu_pgtbl_free(struct mmu_pgtbl *pgtbl);
@@ -88,6 +94,23 @@ static inline int mmu_pgtbl_level(struct mmu_pgtbl *pgtbl)
return (pgtbl) ? pgtbl->level : -1;
}
+static inline bool mmu_pgtbl_need_remote_tlbflush(struct mmu_pgtbl *pgtbl)
+{
+ return (pgtbl && (pgtbl->attr & MMU_ATTR_REMOTE_TLB_FLUSH)) ?
+ TRUE : FALSE;
+}
+
+static inline bool mmu_pgtbl_has_hw_tag(struct mmu_pgtbl *pgtbl)
+{
+ return (pgtbl && (pgtbl->attr & MMU_ATTR_HW_TAG_VALID)) ?
+ TRUE : FALSE;
+}
+
+static inline u32 mmu_pgtbl_hw_tag(struct mmu_pgtbl *pgtbl)
+{
+ return (pgtbl) ? pgtbl->hw_tag : 0;
+}
+
static inline physical_addr_t mmu_pgtbl_map_addr(struct mmu_pgtbl *pgtbl)
{
return (pgtbl) ? pgtbl->map_ia : 0;
@@ -198,9 +221,11 @@ static inline u32 mmu_stage2_current_vmid(void)
return arch_mmu_stage2_current_vmid();
}
-static inline int mmu_stage2_change_pgtbl(u32 vmid, struct mmu_pgtbl *pgtbl)
+static inline int mmu_stage2_change_pgtbl(struct mmu_pgtbl *pgtbl)
{
- return arch_mmu_stage2_change_pgtbl(vmid, pgtbl->tbl_pa);
+ return arch_mmu_stage2_change_pgtbl(mmu_pgtbl_has_hw_tag(pgtbl),
+ mmu_pgtbl_hw_tag(pgtbl),
+ pgtbl->tbl_pa);
}
#endif
diff --git a/arch/riscv/cpu/generic/cpu_mmu.c b/arch/riscv/cpu/generic/cpu_mmu.c
index 853cc9ad..d8a1d1c8 100644
--- a/arch/riscv/cpu/generic/cpu_mmu.c
+++ b/arch/riscv/cpu/generic/cpu_mmu.c
@@ -63,29 +63,47 @@ int arch_mmu_pgtbl_size_order(int stage, int level)
return PGTBL_PAGE_SIZE_SHIFT;
}
-void arch_mmu_stage2_tlbflush(bool remote,
+void arch_mmu_stage2_tlbflush(bool remote, bool use_vmid, u32 vmid,
physical_addr_t gpa, physical_size_t gsz)
{
physical_addr_t off;
if (remote) {
- sbi_remote_hfence_gvma(NULL, gpa, gsz);
+ if (use_vmid) {
+ sbi_remote_hfence_gvma_vmid(NULL, gpa, gsz, vmid);
+ } else {
+ sbi_remote_hfence_gvma(NULL, gpa, gsz);
+ }
} else {
- for (off = 0; off < gsz; off += VMM_PAGE_SIZE)
- __hfence_gvma_gpa((gpa + off) >> 2);
+ if (use_vmid) {
+ for (off = 0; off < gsz; off += VMM_PAGE_SIZE)
+ __hfence_gvma_vmid_gpa((gpa + off) >> 2, vmid);
+ } else {
+ for (off = 0; off < gsz; off += VMM_PAGE_SIZE)
+ __hfence_gvma_gpa((gpa + off) >> 2);
+ }
}
}
-void arch_mmu_stage1_tlbflush(bool remote,
+void arch_mmu_stage1_tlbflush(bool remote, bool use_asid, u32 asid,
virtual_addr_t va, virtual_size_t sz)
{
virtual_addr_t off;
if (remote) {
- sbi_remote_sfence_vma(NULL, va, sz);
+ if (use_asid) {
+ sbi_remote_sfence_vma_asid(NULL, va, sz, asid);
+ } else {
+ sbi_remote_sfence_vma(NULL, va, sz);
+ }
} else {
- for (off = 0; off < sz; off += VMM_PAGE_SIZE)
- __sfence_vma_va(va + off);
+ if (use_asid) {
+ for (off = 0; off < sz; off += VMM_PAGE_SIZE)
+ __sfence_vma_asid_va(asid, va + off);
+ } else {
+ for (off = 0; off < sz; off += VMM_PAGE_SIZE)
+ __sfence_vma_va(va + off);
+ }
}
}
@@ -683,7 +701,8 @@ u32 arch_mmu_stage2_current_vmid(void)
return (csr_read(CSR_HGATP) & HGATP_VMID) >> HGATP_VMID_SHIFT;
}
-int arch_mmu_stage2_change_pgtbl(u32 vmid, physical_addr_t tbl_phys)
}
+int arch_mmu_stage2_change_pgtbl(bool have_vmid, u32 vmid,
+ physical_addr_t tbl_phys)
{
unsigned long hgatp;
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_helper.c b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
index e33bce24..75862d93 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_helper.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_helper.c
@@ -102,6 +102,7 @@ int arch_guest_init(struct vmm_guest *guest)
{
struct riscv_guest_priv *priv;
struct riscv_guest_serial *gserial;
+ u32 pgtbl_attr, pgtbl_hw_tag;
char *sname;
int rc;
@@ -118,7 +119,14 @@ int arch_guest_init(struct vmm_guest *guest)
priv->time_delta = -get_cycles64();
- priv->pgtbl = mmu_pgtbl_alloc(MMU_STAGE2, -1);
+ pgtbl_hw_tag = 0;
+ pgtbl_attr = MMU_ATTR_REMOTE_TLB_FLUSH;
+ if (riscv_stage2_vmid_available()) {
+ pgtbl_hw_tag = guest->id;
+ pgtbl_attr |= MMU_ATTR_HW_TAG_VALID;
+ }
+ priv->pgtbl = mmu_pgtbl_alloc(MMU_STAGE2, -1,
+ pgtbl_attr, pgtbl_hw_tag);
if (!priv->pgtbl) {
vmm_free(guest->arch_priv);
guest->arch_priv = NULL;
@@ -476,24 +484,12 @@ void cpu_vcpu_time_delta_update(struct vmm_vcpu *vcpu, bool nested_virt)
void cpu_vcpu_gstage_update(struct vmm_vcpu *vcpu, bool nested_virt)
- if (riscv_stage2_vmid_available()) {
- if (nested_virt) {
- mmu_stage2_change_pgtbl(
- riscv_stage2_vmid_nested + vcpu->guest->id,
- riscv_nested_priv(vcpu)->pgtbl);
- } else {
- mmu_stage2_change_pgtbl(vcpu->guest->id,
- riscv_guest_priv(vcpu->guest)->pgtbl);
- }
- } else {
- riscv_guest_priv(vcpu->guest)->pgtbl);
- }
- if (nested_virt) {
- mmu_stage2_change_pgtbl(0,
- riscv_nested_priv(vcpu)->pgtbl);
- } else {
- mmu_stage2_change_pgtbl(0,
- riscv_guest_priv(vcpu->guest)->pgtbl);
- }
+ struct mmu_pgtbl *pgtbl = (nested_virt) ?
- mmu_stage2_change_pgtbl(0,
- riscv_guest_priv(vcpu->guest)->pgtbl);
- }
+ riscv_nested_priv(vcpu)->pgtbl :
+ riscv_guest_priv(vcpu->guest)->pgtbl;
+ mmu_stage2_change_pgtbl(pgtbl);
+ if (!mmu_pgtbl_has_hw_tag(pgtbl)) {
/*
* Invalidate entries related to all guests from both
* G-stage TLB and VS-stage TLB.
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_nested.c b/arch/riscv/cpu/generic/cpu_vcpu_nested.c
index e5d95a97..ad074747 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_nested.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_nested.c
@@ -517,9 +517,15 @@ static int nested_xlate_vsstage(struct nested_xlate_context *xc,
int cpu_vcpu_nested_init(struct vmm_vcpu *vcpu)
{
+ u32 pgtbl_attr = 0, pgtbl_hw_tag = 0;
struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
- npriv->pgtbl = mmu_pgtbl_alloc(MMU_STAGE2, -1);
+ if (riscv_stage2_vmid_available()) {
+ pgtbl_hw_tag = riscv_stage2_vmid_nested + vcpu->guest->id;
+ pgtbl_attr |= MMU_ATTR_HW_TAG_VALID;
+ }
+ npriv->pgtbl = mmu_pgtbl_alloc(MMU_STAGE2, -1, pgtbl_attr,
+ pgtbl_hw_tag);
if (!npriv->pgtbl) {
return VMM_ENOMEM;
}
diff --git a/arch/riscv/cpu/generic/include/arch_mmu.h b/arch/riscv/cpu/generic/include/arch_mmu.h
index e67b1176..b78cbc76 100644
--- a/arch/riscv/cpu/generic/include/arch_mmu.h
+++ b/arch/riscv/cpu/generic/include/arch_mmu.h
@@ -133,10 +133,10 @@ int arch_mmu_pgtbl_align_order(int stage, int level);
int arch_mmu_pgtbl_size_order(int stage, int level);
-void arch_mmu_stage2_tlbflush(bool remote,
+void arch_mmu_stage2_tlbflush(bool remote, bool use_vmid, u32 vmid,
physical_addr_t gpa, physical_size_t gsz);
-void arch_mmu_stage1_tlbflush(bool remote,
+void arch_mmu_stage1_tlbflush(bool remote, bool use_asid, u32 asid,
virtual_addr_t va, virtual_size_t sz);
bool arch_mmu_valid_block_size(physical_size_t sz);
@@ -186,7 +186,8 @@ physical_addr_t arch_mmu_stage2_current_pgtbl_addr(void);
u32 arch_mmu_stage2_current_vmid(void);
-int arch_mmu_stage2_change_pgtbl(u32 vmid, physical_addr_t tbl_phys);
+int arch_mmu_stage2_change_pgtbl(bool have_vmid, u32 vmid,
+ physical_addr_t tbl_phys);
#endif
diff --git a/arch/riscv/cpu/generic/include/cpu_tlb.h b/arch/riscv/cpu/generic/include/cpu_tlb.h
index 3ed19ade..853fb067 100644
--- a/arch/riscv/cpu/generic/include/cpu_tlb.h
+++ b/arch/riscv/cpu/generic/include/cpu_tlb.h
@@ -52,7 +52,7 @@ void __hfence_vvma_all(void);
inline void __sfence_vma_asid_va(unsigned long asid, unsigned long va)
{
- __asm__ __volatile__("sfence.vma %0 %1"
+ __asm__ __volatile__("sfence.vma %0, %1"
:
: "r"(va),"r"(asid)
: "memory");
@@ -60,7 +60,7 @@ inline void __sfence_vma_asid_va(unsigned long asid, unsigned long va)
inline void __sfence_vma_asid(unsigned long asid)
{
- __asm__ __volatile__("sfence.vma x0 %1"
+ __asm__ __volatile__("sfence.vma x0, %1"
:
: "r"(asid)
: "memory");
diff --git a/libs/wboxtest/nested_mmu/nested_mmu_test.h b/libs/wboxtest/nested_mmu/nested_mmu_test.h
index 07b73822..33b1f425 100755
--- a/libs/wboxtest/nested_mmu/nested_mmu_test.h
+++ b/libs/wboxtest/nested_mmu/nested_mmu_test.h
@@ -122,7 +122,8 @@ do { \
DPRINTF((__cdev), "%s: Allocating Stage%s page table ", \
(__test)->name, \
((__stage) == MMU_STAGE2) ? "2" : "1"); \
- *(__output_pgtbl_double_ptr) = mmu_pgtbl_alloc((__stage), -1); \
+ *(__output_pgtbl_double_ptr) = mmu_pgtbl_alloc((__stage), -1, \
+ MMU_ATTR_REMOTE_TLB_FLUSH, 0); \
DPRINTF((__cdev), "%s", \
(!*(__output_pgtbl_double_ptr)) ? "(failed)\n" : ""); \
if (!*(__output_pgtbl_double_ptr)) { \
--
2.25.1
Anup Patel
Mar 17, 2022, 11:24:15 AM3/17/22
to xvisor...@googlegroups.com, Anup Patel, Anup Patel
The guest G-stage will need to do two translations: first translate
guest gpa to host gpa and next translate host gpa to host pa. Also,
the guest HFENCE instruction will apply only to guest gpa.
To emulate guest G-stage and guest HFENCE instruction, we implement:
1) Shadow page table which will directly map guest gpa to host pa
by doing software walk on the guest G-stage page table pointed
by guest HGATP CSR
2) Software TLB which will help us optimize software walk on the
guest G-stage page table and also help us emulate guest HFENCE
instructions
arch/riscv/cpu/generic/cpu_vcpu_nested.c | 444 ++++++++++++++++--
arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c | 58 ++-
.../cpu/generic/include/cpu_vcpu_nested.h | 5 +
3 files changed, 474 insertions(+), 33 deletions(-)
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_nested.c b/arch/riscv/cpu/generic/cpu_vcpu_nested.c
index ad074747..9fcb25ff 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_nested.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_nested.c
@@ -23,16 +23,219 @@
#include <vmm_error.h>
#include <vmm_stdio.h>
+#include <vmm_heap.h>
#include <vmm_guest_aspace.h>
#include <vmm_host_aspace.h>
#include <generic_mmu.h>
+#include <libs/list.h>
#include <cpu_hwcap.h>
+#include <cpu_tlb.h>
#include <cpu_vcpu_helper.h>
#include <cpu_vcpu_nested.h>
#include <cpu_vcpu_trap.h>
#include <riscv_csr.h>
+/*
+ * We share the same host VMID between virtual-HS/U modes and
+ * virtual-VS/VU modes. To achieve this, we flush all guest TLB
+ * entries upon:
+ * 1) Change in nested virt state (ON => OFF or OFF => ON)
+ * 2) Change in guest hgatp.VMID
+ */
+
+#define NESTED_SWTLB_ITLB_MAX_ENTRY 128
+#define NESTED_SWTLB_DTLB_MAX_ENTRY 128
+#define NESTED_SWTLB_MAX_ENTRY (NESTED_SWTLB_ITLB_MAX_ENTRY + \
+ NESTED_SWTLB_DTLB_MAX_ENTRY)
+
+struct nested_swtlb_entry {
+ struct dlist head;
+ struct mmu_page page;
+ struct mmu_page shadow_page;
+ u32 shadow_reg_flags;
+};
+
+struct nested_swtlb_xtlb {
+ struct dlist active_list;
+ struct dlist free_list;
+};
+
+struct nested_swtlb {
+ struct nested_swtlb_xtlb itlb;
+ struct nested_swtlb_xtlb dtlb;
+ struct nested_swtlb_entry *entries;
+};
+
+static const struct nested_swtlb_entry *nested_swtlb_lookup(
+ struct vmm_vcpu *vcpu,
+ physical_addr_t guest_gpa)
+{
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+ struct nested_swtlb *swtlb = npriv->swtlb;
+ struct nested_swtlb_entry *swte;
+
+ list_for_each_entry(swte, &swtlb->itlb.active_list, head) {
+ if (swte->page.ia <= guest_gpa &&
+ guest_gpa < (swte->page.ia + swte->page.sz)) {
+ list_del(&swte->head);
+ list_add(&swte->head, &swtlb->itlb.active_list);
+ return swte;
+ }
+ }
+
+ list_for_each_entry(swte, &swtlb->dtlb.active_list, head) {
+ if (swte->page.ia <= guest_gpa &&
+ guest_gpa < (swte->page.ia + swte->page.sz)) {
+ list_del(&swte->head);
+ list_add(&swte->head, &swtlb->dtlb.active_list);
+ return swte;
+ }
+ }
+
+ return NULL;
+}
+
+static void nested_swtlb_update(struct vmm_vcpu *vcpu, bool itlb,
+ const struct mmu_page *page,
+ const struct mmu_page *shadow_page,
+ u32 shadow_reg_flags)
+{
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+ struct nested_swtlb *swtlb = npriv->swtlb;
+ struct nested_swtlb_entry *swte;
+ struct nested_swtlb_xtlb *xtlb;
+ int rc;
+
+ xtlb = (itlb) ? &swtlb->itlb : &swtlb->dtlb;
+
+ if (!list_empty(&xtlb->free_list)) {
+ swte = list_entry(list_pop(&xtlb->free_list),
+ struct nested_swtlb_entry, head);
+ } else if (!list_empty(&xtlb->active_list)) {
+ swte = list_entry(list_pop_tail(&xtlb->active_list),
+ struct nested_swtlb_entry, head);
+ rc = mmu_unmap_page(npriv->pgtbl, &swte->shadow_page);
+ if (rc) {
+ vmm_panic("%s: shadow page unmap failed (error %d)\n",
+ __func__, rc);
+ }
+ } else {
+ BUG_ON(1);
+ }
+
+ memcpy(&swte->page, page, sizeof(swte->page));
+ memcpy(&swte->shadow_page, shadow_page, sizeof(swte->shadow_page));
+ swte->shadow_reg_flags = shadow_reg_flags;
+
+ rc = mmu_map_page(npriv->pgtbl, &swte->shadow_page);
+ if (rc) {
+ vmm_panic("%s: shadow page map failed (error %d)\n",
+ __func__, rc);
+ }
+
+ list_add(&swte->head, &xtlb->active_list);
+}
+
+void cpu_vcpu_nested_swtlb_flush(struct vmm_vcpu *vcpu,
+ physical_addr_t guest_gpa,
+ physical_size_t guest_gpa_size)
+{
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+ struct nested_swtlb *swtlb = npriv->swtlb;
+ physical_addr_t start, end, pstart, pend;
+ struct nested_swtlb_entry *swte, *nswte;
+ int rc;
+
+ if (!guest_gpa && !guest_gpa_size) {
+ start = 0;
+ end = ~start;
+ } else {
+ start = guest_gpa;
+ end = guest_gpa + guest_gpa_size - 1;
+ }
+
+ list_for_each_entry_safe(swte, nswte, &swtlb->itlb.active_list, head) {
+ pstart = swte->page.ia;
+ pend = swte->page.ia + swte->page.sz - 1;
+ if (end < pstart || pend < start)
+ continue;
+
+ list_del(&swte->head);
+
+ rc = mmu_unmap_page(npriv->pgtbl, &swte->shadow_page);
+ if (rc) {
+ vmm_panic("%s: shadow page unmap failed (error %d)\n",
+ __func__, rc);
+ }
+
+ list_add_tail(&swte->head, &swtlb->itlb.free_list);
+ }
+
+ list_for_each_entry_safe(swte, nswte, &swtlb->dtlb.active_list, head) {
+ pstart = swte->page.ia;
+ pend = swte->page.ia + swte->page.sz - 1;
+ if (end < pstart || pend < start)
+ continue;
+
+ list_del(&swte->head);
+
+ rc = mmu_unmap_page(npriv->pgtbl, &swte->shadow_page);
+ if (rc) {
+ vmm_panic("%s: shadow page unmap failed (error %d)\n",
+ __func__, rc);
+ }
+
+ list_add_tail(&swte->head, &swtlb->dtlb.free_list);
+ }
+}
+
+static int nested_swtlb_init(struct vmm_vcpu *vcpu)
+{
+ int i;
+ struct nested_swtlb *swtlb;
+ struct nested_swtlb_entry *swte;
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ swtlb = vmm_zalloc(sizeof(*swtlb));
+ if (!swtlb) {
+ return VMM_ENOMEM;
+ }
+ INIT_LIST_HEAD(&swtlb->itlb.active_list);
+ INIT_LIST_HEAD(&swtlb->itlb.free_list);
+ INIT_LIST_HEAD(&swtlb->dtlb.active_list);
+ INIT_LIST_HEAD(&swtlb->dtlb.free_list);
+
+ swtlb->entries = vmm_zalloc(sizeof(*swtlb->entries) *
+ NESTED_SWTLB_MAX_ENTRY);
+ if (!swtlb->entries) {
+ vmm_free(swtlb);
+ swte = &swtlb->entries[i];
+ INIT_LIST_HEAD(&swte->head);
+ if (i < NESTED_SWTLB_ITLB_MAX_ENTRY) {
+ list_add_tail(&swte->head, &swtlb->itlb.free_list);
+ } else {
+ list_add_tail(&swte->head, &swtlb->dtlb.free_list);
+ }
+ }
+
+ npriv->swtlb = swtlb;
+{
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+ struct nested_swtlb *swtlb = npriv->swtlb;
+
+ vmm_free(swtlb->entries);
+ vmm_free(swtlb);
+}
+
#ifdef CONFIG_64BIT
#define NESTED_MMU_OFF_BLOCK_SIZE PGTBL_L2_BLOCK_SIZE
#else
@@ -259,14 +462,17 @@ static int nested_xlate_gstage(struct nested_xlate_context *xc,
{
int rc;
unsigned long mode;
- struct mmu_page page;
bool perm_fault = FALSE;
physical_addr_t pgtlb, guest_hpa;
+ struct mmu_page page, shadow_page;
+ const struct nested_swtlb_entry *swte = NULL;
+ if ((swte = nested_swtlb_lookup(xc->vcpu, guest_gpa))) {
+ memcpy(&page, &swte->page, sizeof(page));
+ } else if (mode == HGATP_MODE_OFF) {
memset(&page, 0, sizeof(page));
page.sz = NESTED_MMU_OFF_BLOCK_SIZE;
return VMM_EFAULT;
}
- /* Calculate guest hpa */
- guest_hpa = page.oa | (guest_gpa & (page.sz - 1));
+ /* Update host region details */
+ if (swte) {
+ /* Get host details from software TLB entry */
+ xc->host_pa = swte->shadow_page.oa;
+ xc->host_sz = swte->shadow_page.sz;
+ xc->host_reg_flags = swte->shadow_reg_flags;
+
+ /* Check shadow page permissions */
+ rc = VMM_EFAULT;
+ switch (guest_access) {
+ case NESTED_XLATE_LOAD:
+ if (swte->shadow_page.flags.read) {
+ rc = VMM_OK;
+ }
+ swte->shadow_page.flags.write) {
+ rc = VMM_OK;
+ }
+ rc = VMM_OK;
+ }
- rc = nested_xlate_nostage(xc, guest_hpa, guest_access);
- if (rc) {
- if (rc == VMM_EFAULT) {
- xc->nostage_page_sz = page.sz;
- nested_gstage_write_fault(xc, guest_gpa);
- }
- /* Update output address and size */
- if (page.sz <= xc->host_sz) {
- xc->host_pa |= guest_hpa & (xc->host_sz - 1);
- xc->host_sz = page.sz;
- xc->host_pa &= ~(xc->host_sz - 1);
- } else {
- page.ia = guest_gpa & ~(xc->host_sz - 1);
- page.oa = guest_hpa & ~(xc->host_sz - 1);
- page.sz = xc->host_sz;
+ /* Prepare shadow page */
+ memset(&shadow_page, 0, sizeof(shadow_page));
+ shadow_page.ia = page.ia;
+ shadow_page.oa = xc->host_pa;
+ shadow_page.sz = xc->host_sz;
+ shadow_page.flags.dirty = 1;
+ shadow_page.flags.accessed = 1;
+ shadow_page.flags.global = 0;
+ shadow_page.flags.user = 1;
+ shadow_page.flags.read = 0;
+ if ((xc->host_reg_flags &
+ (VMM_REGION_ISRAM | VMM_REGION_ISROM)) &&
+ page.flags.read) {
+ shadow_page.flags.read = 1;
+ }
+ shadow_page.flags.write = 0;
+ if ((xc->host_reg_flags & VMM_REGION_ISRAM) &&
+ page.flags.write) {
+ shadow_page.flags.write = 1;
+ }
+ shadow_page.flags.execute = page.flags.execute;
+ shadow_page.flags.valid = 1;
+
+ /* Update software TLB */
+ nested_swtlb_update(xc->vcpu,
+ (guest_access == NESTED_XLATE_FETCH) ? TRUE : FALSE,
+ &page, &shadow_page, xc->host_reg_flags);
}
return VMM_OK;
@@ -517,6 +789,7 @@ static int nested_xlate_vsstage(struct nested_xlate_context *xc,
int cpu_vcpu_nested_init(struct vmm_vcpu *vcpu)
{
+ int rc;
return VMM_ENOMEM;
}
+ rc = nested_swtlb_init(vcpu);
+ if (rc) {
+ mmu_pgtbl_free(npriv->pgtbl);
}
@@ -537,6 +816,7 @@ void cpu_vcpu_nested_reset(struct vmm_vcpu *vcpu)
{
npriv->virt = FALSE;
#ifdef CONFIG_64BIT
void cpu_vcpu_nested_deinit(struct vmm_vcpu *vcpu)
{
- mmu_pgtbl_free(riscv_nested_priv(vcpu)->pgtbl);
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ nested_swtlb_deinit(vcpu);
+ mmu_pgtbl_free(npriv->pgtbl);
}
void cpu_vcpu_nested_dump_regs(struct vmm_chardev *cdev,
@@ -694,7 +977,7 @@ int cpu_vcpu_nested_hext_csr_rmw(struct vmm_vcpu *vcpu, arch_regs_t *regs,
int csr_shift = 0;
- bool read_only = FALSE;
+ bool read_only = FALSE, nuke_swtlb = FALSE;
}
new_val &= ~HGATP_MODE;
+ nuke_swtlb = TRUE;
+ }
+ }
+ if (wr_mask & HGATP_VMID) {
+ if ((new_val ^ npriv->hgatp) & HGATP_VMID) {
+ nuke_swtlb = TRUE;
+ }
}
break;
case CSR_VSSTATUS:
@@ -922,6 +1213,10 @@ int cpu_vcpu_nested_hext_csr_rmw(struct vmm_vcpu *vcpu, arch_regs_t *regs,
+ cpu_vcpu_nested_swtlb_flush(vcpu, 0, 0);
+ }
+
return VMM_OK;
}
@@ -930,20 +1225,115 @@ int cpu_vcpu_nested_page_fault(struct vmm_vcpu *vcpu,
const struct cpu_vcpu_trap *trap,
struct cpu_vcpu_trap *out_trap)
{
- /* TODO: */
- return VMM_OK;
+ int rc = VMM_OK;
+ physical_addr_t guest_gpa;
+ struct nested_xlate_context xc;
+ enum nested_xlate_access guest_access;
+
+ /*
+ * This function is called to handle guest page faults
+ * from virtual-VS/VU modes.
+ *
+ * We perform a guest gpa to host pa translation for
+ * the faulting guest gpa so that nested software TLB
+ * and shadow page table is updated.
+ */
+
+ guest_gpa = ((physical_addr_t)trap->htval << 2);
+ guest_gpa |= ((physical_addr_t)trap->stval & 0x3);
+ switch (trap->scause) {
+ case CAUSE_LOAD_GUEST_PAGE_FAULT:
+ guest_access = NESTED_XLATE_LOAD;
+ break;
+ case CAUSE_STORE_GUEST_PAGE_FAULT:
+ guest_access = NESTED_XLATE_STORE;
+ break;
+ case CAUSE_FETCH_GUEST_PAGE_FAULT:
+ guest_access = NESTED_XLATE_FETCH;
+ break;
+ default:
+ memcpy(out_trap, trap, sizeof(*out_trap));
+ csr_read(CSR_VSSTATUS), FALSE);
+ rc = nested_xlate_gstage(&xc, guest_gpa, guest_access);
+ out_trap->scause = xc.scause;
+ out_trap->stval = trap->stval;
+ out_trap->htval = xc.htval;
+ out_trap->htinst = trap->htinst;
+ rc = VMM_OK;
+ }
+
+ return rc;
}
void cpu_vcpu_nested_hfence_vvma(struct vmm_vcpu *vcpu,
- /* TODO: */
+ unsigned long hgatp;
+ * VS-stage TLB entries for virtual-VS/VU modes.
+ *
+ * When host G-stage VMID is not available, we flush all guest
+ * TLB (both G-stage and VS-stage) entries upon nested virt
+ * state change so the HFENCE.VVMA becomes a NOP in this case.
+ */
+
+ if (!mmu_pgtbl_has_hw_tag(npriv->pgtbl)) {
+ return;
+ }
+
+ hgatp = mmu_pgtbl_hw_tag(npriv->pgtbl);
+ hgatp = csr_swap(CSR_HGATP, hgatp << HGATP_VMID_SHIFT);
+
+ if (!vaddr && !asid) {
+ __hfence_vvma_all();
+ } else if (!vaddr && asid) {
+ __hfence_vvma_asid(*asid);
+ } else if (vaddr && !asid) {
+ __hfence_vvma_va(*vaddr);
+ } else {
+ __hfence_vvma_asid_va(*vaddr, *asid);
+ }
+
+ csr_write(CSR_HGATP, hgatp);
}
void cpu_vcpu_nested_hfence_gvma(struct vmm_vcpu *vcpu,
- /* TODO: */
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+ unsigned long current_vmid =
+ (npriv->hgatp & HGATP_VMID) >> HGATP_VMID_SHIFT;
+
+ /*
+ * The HFENCE.GVMA instructions help virtual-HS mode flush
+ * G-stage TLB entries for virtual-VS/VU modes.
+ *
+ * Irrespective whether host G-stage VMID is available or not,
+ * we flush all software TLB (only G-stage) entries upon guest
+ * hgatp.VMID change so the HFENCE.GVMA instruction becomes a
+ * NOP for virtual-HS mode when the current guest hgatp.VMID
+ * is different from the VMID specified in the HFENCE.GVMA
+ * instruction.
+ */
+
+ if (vmid && current_vmid != *vmid) {
+ return;
+ }
+
+ if (gaddr) {
+ cpu_vcpu_nested_swtlb_flush(vcpu, *gaddr, 1);
+ } else {
+ cpu_vcpu_nested_swtlb_flush(vcpu, 0, 0);
+ }
index eb7c3acb..4c5cb44c 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c
@@ -29,6 +29,8 @@
#include <cpu_sbi.h>
#include <cpu_vcpu_sbi.h>
#include <cpu_vcpu_timer.h>
+#include <cpu_vcpu_nested.h>
+#include <generic_mmu.h>
#include <riscv_sbi.h>
static int vcpu_sbi_time_ecall(struct vmm_vcpu *vcpu,
@@ -63,7 +65,8 @@ static int vcpu_sbi_rfence_ecall(struct vmm_vcpu *vcpu,
struct vmm_vcpu *rvcpu;
struct vmm_cpumask cm, hm;
struct vmm_guest *guest = vcpu->guest;
- unsigned long hmask = args[0], hbase = args[1];
+ unsigned long hgatp, hmask = args[0], hbase = args[1];
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
vmm_cpumask_clear(&cm);
vmm_manager_for_each_guest_vcpu(rvcpu, guest) {
@@ -96,13 +99,56 @@ static int vcpu_sbi_rfence_ecall(struct vmm_vcpu *vcpu,
break;
case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA:
case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID:
+ /* Flush the nested software TLB of calling VCPU */
+ cpu_vcpu_nested_swtlb_flush(vcpu, args[2], args[3]);
+
+ if (mmu_pgtbl_has_hw_tag(npriv->pgtbl)) {
+ /*
+ * We use two VMIDs for nested virtualization:
+ * one for virtual-HS/U modes and another for
+ * virtual-VS/VU modes. This means we need to
+ * restrict guest remote HFENCE.GVMA to VMID
+ * used for virtual-VS/VU modes.
+ */
+ sbi_remote_hfence_gvma_vmid(vmm_cpumask_bits(&hm),
+ args[2], args[3],
+ mmu_pgtbl_has_hw_tag(npriv->pgtbl));
+ } else {
+ /*
+ * No VMID support so we do remote HFENCE.GVMA
+ * accross all VMIDs.
+ */
+ sbi_remote_hfence_gvma(vmm_cpumask_bits(&hm),
+ args[2], args[3]);
+ }
+ break;
case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA:
case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID:
- /*
- * TODO: these SBI calls will be implemented as part
- * of nested virtualization support so treat these
- * calls as NOPs.
- */
+ hgatp = 0;
+ if (mmu_pgtbl_has_hw_tag(npriv->pgtbl)) {
+ /*
+ * We use two VMIDs for nested virtualization:
+ * one for virtual-HS/U modes and another for
+ * virtual-VS/VU modes. This means we need to
+ * switch hgatp.VMID before doing forwarding
+ * SBI call to host firmware.
+ */
+ hgatp = mmu_pgtbl_hw_tag(npriv->pgtbl);
+ hgatp = csr_swap(CSR_HGATP, hgatp << HGATP_VMID_SHIFT);
+ }
+
+ if (func_id == SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA) {
+ sbi_remote_hfence_vvma(vmm_cpumask_bits(&hm),
+ args[2], args[3]);
+ } else {
+ sbi_remote_hfence_vvma_asid(
+ vmm_cpumask_bits(&hm),
+ args[2], args[3], args[4]);
+ }
+
+ if (mmu_pgtbl_has_hw_tag(npriv->pgtbl)) {
+ csr_write(CSR_HGATP, hgatp);
index 669690d3..274633e5 100644
--- a/arch/riscv/cpu/generic/include/cpu_vcpu_nested.h
+++ b/arch/riscv/cpu/generic/include/cpu_vcpu_nested.h
@@ -30,6 +30,11 @@ struct vmm_vcpu;
struct cpu_vcpu_trap;
struct arch_regs;
+/** Function to flush nested software TLB */
+void cpu_vcpu_nested_swtlb_flush(struct vmm_vcpu *vcpu,
+ physical_addr_t guest_gpa,
+ physical_size_t guest_gpa_size);
+
--
2.25.1
guest gpa to host gpa and next translate host gpa to host pa. Also,
the guest HFENCE instruction will apply only to guest gpa.
To emulate guest G-stage and guest HFENCE instruction, we implement:
1) Shadow page table which will directly map guest gpa to host pa
by doing software walk on the guest G-stage page table pointed
by guest HGATP CSR
2) Software TLB which will help us optimize software walk on the
guest G-stage page table and also help us emulate guest HFENCE
instructions
arch/riscv/cpu/generic/cpu_vcpu_nested.c | 444 ++++++++++++++++--
arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c | 58 ++-
.../cpu/generic/include/cpu_vcpu_nested.h | 5 +
3 files changed, 474 insertions(+), 33 deletions(-)
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_nested.c b/arch/riscv/cpu/generic/cpu_vcpu_nested.c
index ad074747..9fcb25ff 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_nested.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_nested.c
@@ -23,16 +23,219 @@
#include <vmm_error.h>
#include <vmm_stdio.h>
+#include <vmm_heap.h>
#include <vmm_guest_aspace.h>
#include <vmm_host_aspace.h>
#include <generic_mmu.h>
+#include <libs/list.h>
#include <cpu_hwcap.h>
+#include <cpu_tlb.h>
#include <cpu_vcpu_helper.h>
#include <cpu_vcpu_nested.h>
#include <cpu_vcpu_trap.h>
#include <riscv_csr.h>
+/*
+ * We share the same host VMID between virtual-HS/U modes and
+ * virtual-VS/VU modes. To achieve this, we flush all guest TLB
+ * entries upon:
+ * 1) Change in nested virt state (ON => OFF or OFF => ON)
+ * 2) Change in guest hgatp.VMID
+ */
+
+#define NESTED_SWTLB_ITLB_MAX_ENTRY 128
+#define NESTED_SWTLB_DTLB_MAX_ENTRY 128
+#define NESTED_SWTLB_MAX_ENTRY (NESTED_SWTLB_ITLB_MAX_ENTRY + \
+ NESTED_SWTLB_DTLB_MAX_ENTRY)
+
+struct nested_swtlb_entry {
+ struct dlist head;
+ struct mmu_page page;
+ struct mmu_page shadow_page;
+ u32 shadow_reg_flags;
+};
+
+struct nested_swtlb_xtlb {
+ struct dlist active_list;
+ struct dlist free_list;
+};
+
+struct nested_swtlb {
+ struct nested_swtlb_xtlb itlb;
+ struct nested_swtlb_xtlb dtlb;
+ struct nested_swtlb_entry *entries;
+};
+
+static const struct nested_swtlb_entry *nested_swtlb_lookup(
+ struct vmm_vcpu *vcpu,
+ physical_addr_t guest_gpa)
+{
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+ struct nested_swtlb *swtlb = npriv->swtlb;
+ struct nested_swtlb_entry *swte;
+
+ list_for_each_entry(swte, &swtlb->itlb.active_list, head) {
+ if (swte->page.ia <= guest_gpa &&
+ guest_gpa < (swte->page.ia + swte->page.sz)) {
+ list_del(&swte->head);
+ list_add(&swte->head, &swtlb->itlb.active_list);
+ return swte;
+ }
+ }
+
+ list_for_each_entry(swte, &swtlb->dtlb.active_list, head) {
+ if (swte->page.ia <= guest_gpa &&
+ guest_gpa < (swte->page.ia + swte->page.sz)) {
+ list_del(&swte->head);
+ list_add(&swte->head, &swtlb->dtlb.active_list);
+ return swte;
+ }
+ }
+
+ return NULL;
+}
+
+static void nested_swtlb_update(struct vmm_vcpu *vcpu, bool itlb,
+ const struct mmu_page *page,
+ const struct mmu_page *shadow_page,
+ u32 shadow_reg_flags)
+{
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+ struct nested_swtlb *swtlb = npriv->swtlb;
+ struct nested_swtlb_entry *swte;
+ struct nested_swtlb_xtlb *xtlb;
+ int rc;
+
+ xtlb = (itlb) ? &swtlb->itlb : &swtlb->dtlb;
+
+ if (!list_empty(&xtlb->free_list)) {
+ swte = list_entry(list_pop(&xtlb->free_list),
+ struct nested_swtlb_entry, head);
+ } else if (!list_empty(&xtlb->active_list)) {
+ swte = list_entry(list_pop_tail(&xtlb->active_list),
+ struct nested_swtlb_entry, head);
+ rc = mmu_unmap_page(npriv->pgtbl, &swte->shadow_page);
+ if (rc) {
+ vmm_panic("%s: shadow page unmap failed (error %d)\n",
+ __func__, rc);
+ }
+ } else {
+ BUG_ON(1);
+ }
+
+ memcpy(&swte->page, page, sizeof(swte->page));
+ memcpy(&swte->shadow_page, shadow_page, sizeof(swte->shadow_page));
+ swte->shadow_reg_flags = shadow_reg_flags;
+
+ rc = mmu_map_page(npriv->pgtbl, &swte->shadow_page);
+ if (rc) {
+ vmm_panic("%s: shadow page map failed (error %d)\n",
+ __func__, rc);
+ }
+
+ list_add(&swte->head, &xtlb->active_list);
+}
+
+void cpu_vcpu_nested_swtlb_flush(struct vmm_vcpu *vcpu,
+ physical_addr_t guest_gpa,
+ physical_size_t guest_gpa_size)
+{
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+ struct nested_swtlb *swtlb = npriv->swtlb;
+ physical_addr_t start, end, pstart, pend;
+ struct nested_swtlb_entry *swte, *nswte;
+ int rc;
+
+ if (!guest_gpa && !guest_gpa_size) {
+ start = 0;
+ end = ~start;
+ } else {
+ start = guest_gpa;
+ end = guest_gpa + guest_gpa_size - 1;
+ }
+
+ list_for_each_entry_safe(swte, nswte, &swtlb->itlb.active_list, head) {
+ pstart = swte->page.ia;
+ pend = swte->page.ia + swte->page.sz - 1;
+ if (end < pstart || pend < start)
+ continue;
+
+ list_del(&swte->head);
+
+ rc = mmu_unmap_page(npriv->pgtbl, &swte->shadow_page);
+ if (rc) {
+ vmm_panic("%s: shadow page unmap failed (error %d)\n",
+ __func__, rc);
+ }
+
+ list_add_tail(&swte->head, &swtlb->itlb.free_list);
+ }
+
+ list_for_each_entry_safe(swte, nswte, &swtlb->dtlb.active_list, head) {
+ pstart = swte->page.ia;
+ pend = swte->page.ia + swte->page.sz - 1;
+ if (end < pstart || pend < start)
+ continue;
+
+ list_del(&swte->head);
+
+ rc = mmu_unmap_page(npriv->pgtbl, &swte->shadow_page);
+ if (rc) {
+ vmm_panic("%s: shadow page unmap failed (error %d)\n",
+ __func__, rc);
+ }
+
+ list_add_tail(&swte->head, &swtlb->dtlb.free_list);
+ }
+}
+
+static int nested_swtlb_init(struct vmm_vcpu *vcpu)
+{
+ int i;
+ struct nested_swtlb *swtlb;
+ struct nested_swtlb_entry *swte;
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ swtlb = vmm_zalloc(sizeof(*swtlb));
+ if (!swtlb) {
+ return VMM_ENOMEM;
+ }
+ INIT_LIST_HEAD(&swtlb->itlb.active_list);
+ INIT_LIST_HEAD(&swtlb->itlb.free_list);
+ INIT_LIST_HEAD(&swtlb->dtlb.active_list);
+ INIT_LIST_HEAD(&swtlb->dtlb.free_list);
+
+ swtlb->entries = vmm_zalloc(sizeof(*swtlb->entries) *
+ NESTED_SWTLB_MAX_ENTRY);
+ if (!swtlb->entries) {
+ vmm_free(swtlb);
+ return VMM_ENOMEM;
+ }
+
+ for (i = 0; i < NESTED_SWTLB_MAX_ENTRY; i++) {
+ }
+
+ swte = &swtlb->entries[i];
+ INIT_LIST_HEAD(&swte->head);
+ if (i < NESTED_SWTLB_ITLB_MAX_ENTRY) {
+ list_add_tail(&swte->head, &swtlb->itlb.free_list);
+ } else {
+ list_add_tail(&swte->head, &swtlb->dtlb.free_list);
+ }
+ }
+
+ npriv->swtlb = swtlb;
+ return VMM_OK;
+}
+
+static void nested_swtlb_deinit(struct vmm_vcpu *vcpu)
+}
+
+{
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+ struct nested_swtlb *swtlb = npriv->swtlb;
+
+ vmm_free(swtlb->entries);
+ vmm_free(swtlb);
+}
+
#ifdef CONFIG_64BIT
#define NESTED_MMU_OFF_BLOCK_SIZE PGTBL_L2_BLOCK_SIZE
#else
@@ -259,14 +462,17 @@ static int nested_xlate_gstage(struct nested_xlate_context *xc,
{
int rc;
unsigned long mode;
- struct mmu_page page;
bool perm_fault = FALSE;
physical_addr_t pgtlb, guest_hpa;
+ struct mmu_page page, shadow_page;
+ const struct nested_swtlb_entry *swte = NULL;
struct riscv_priv_nested *npriv = riscv_nested_priv(xc->vcpu);
/* Find guest G-stage page */
mode = (npriv->hgatp & HGATP_MODE) >> HGATP_MODE_SHIFT;
- if (mode == HGATP_MODE_OFF) {
+ if ((swte = nested_swtlb_lookup(xc->vcpu, guest_gpa))) {
+ memcpy(&page, &swte->page, sizeof(page));
+ } else if (mode == HGATP_MODE_OFF) {
memset(&page, 0, sizeof(page));
page.sz = NESTED_MMU_OFF_BLOCK_SIZE;
page.ia = guest_gpa & ~(page.sz - 1);
@@ -327,28 +533,94 @@ static int nested_xlate_gstage(struct nested_xlate_context *xc,
return VMM_EFAULT;
}
- /* Calculate guest hpa */
- guest_hpa = page.oa | (guest_gpa & (page.sz - 1));
+ /* Update host region details */
+ if (swte) {
+ /* Get host details from software TLB entry */
+ xc->host_pa = swte->shadow_page.oa;
+ xc->host_sz = swte->shadow_page.sz;
+ xc->host_reg_flags = swte->shadow_reg_flags;
+
+ /* Check shadow page permissions */
+ rc = VMM_EFAULT;
+ switch (guest_access) {
+ case NESTED_XLATE_LOAD:
+ if (swte->shadow_page.flags.read) {
+ rc = VMM_OK;
+ }
+ break;
+ case NESTED_XLATE_STORE:
+ if (swte->shadow_page.flags.read &&
+ case NESTED_XLATE_STORE:
+ swte->shadow_page.flags.write) {
+ rc = VMM_OK;
+ }
+ break;
+ case NESTED_XLATE_FETCH:
+ if (swte->shadow_page.flags.execute) {
+ case NESTED_XLATE_FETCH:
+ rc = VMM_OK;
+ }
+ break;
+ default:
+ break;
+ }
+ default:
+ break;
+ }
+ if (rc) {
+ if (rc == VMM_EFAULT) {
+ xc->nostage_page_sz = page.sz;
+ nested_gstage_write_fault(xc, guest_gpa);
+ }
+ return rc;
+ }
+ } else {
+ if (rc == VMM_EFAULT) {
+ xc->nostage_page_sz = page.sz;
+ nested_gstage_write_fault(xc, guest_gpa);
+ }
+ return rc;
+ }
+ /* Calculate guest hpa */
+ guest_hpa = page.oa | (guest_gpa & (page.sz - 1));
+
+ /* Translate guest hpa to host pa */
+ rc = nested_xlate_nostage(xc, guest_hpa, guest_access);
+ if (rc) {
+ if (rc == VMM_EFAULT) {
+ xc->nostage_page_sz = page.sz;
+ nested_gstage_write_fault(xc, guest_gpa);
+ }
+ return rc;
+ }
- /* Translate guest hpa to host pa */
+ guest_hpa = page.oa | (guest_gpa & (page.sz - 1));
+
+ /* Translate guest hpa to host pa */
+ rc = nested_xlate_nostage(xc, guest_hpa, guest_access);
+ if (rc) {
+ if (rc == VMM_EFAULT) {
+ xc->nostage_page_sz = page.sz;
+ nested_gstage_write_fault(xc, guest_gpa);
+ }
+ return rc;
+ }
- rc = nested_xlate_nostage(xc, guest_hpa, guest_access);
- if (rc) {
- if (rc == VMM_EFAULT) {
- xc->nostage_page_sz = page.sz;
- nested_gstage_write_fault(xc, guest_gpa);
+ /* Update output address and size */
+ if (page.sz <= xc->host_sz) {
+ xc->host_pa |= guest_hpa & (xc->host_sz - 1);
+ xc->host_sz = page.sz;
+ xc->host_pa &= ~(xc->host_sz - 1);
+ } else {
+ page.ia = guest_gpa & ~(xc->host_sz - 1);
+ page.oa = guest_hpa & ~(xc->host_sz - 1);
+ page.sz = xc->host_sz;
}
- return rc;
+ if (page.sz <= xc->host_sz) {
+ xc->host_pa |= guest_hpa & (xc->host_sz - 1);
+ xc->host_sz = page.sz;
+ xc->host_pa &= ~(xc->host_sz - 1);
+ } else {
+ page.ia = guest_gpa & ~(xc->host_sz - 1);
+ page.oa = guest_hpa & ~(xc->host_sz - 1);
+ page.sz = xc->host_sz;
}
- }
- /* Update output address and size */
- if (page.sz <= xc->host_sz) {
- xc->host_pa |= guest_hpa & (xc->host_sz - 1);
- xc->host_sz = page.sz;
- xc->host_pa &= ~(xc->host_sz - 1);
- } else {
- page.ia = guest_gpa & ~(xc->host_sz - 1);
- page.oa = guest_hpa & ~(xc->host_sz - 1);
- page.sz = xc->host_sz;
+ /* Prepare shadow page */
+ memset(&shadow_page, 0, sizeof(shadow_page));
+ shadow_page.ia = page.ia;
+ shadow_page.oa = xc->host_pa;
+ shadow_page.sz = xc->host_sz;
+ shadow_page.flags.dirty = 1;
+ shadow_page.flags.accessed = 1;
+ shadow_page.flags.global = 0;
+ shadow_page.flags.user = 1;
+ shadow_page.flags.read = 0;
+ if ((xc->host_reg_flags &
+ (VMM_REGION_ISRAM | VMM_REGION_ISROM)) &&
+ page.flags.read) {
+ shadow_page.flags.read = 1;
+ }
+ shadow_page.flags.write = 0;
+ if ((xc->host_reg_flags & VMM_REGION_ISRAM) &&
+ page.flags.write) {
+ shadow_page.flags.write = 1;
+ }
+ shadow_page.flags.execute = page.flags.execute;
+ shadow_page.flags.valid = 1;
+
+ /* Update software TLB */
+ nested_swtlb_update(xc->vcpu,
+ (guest_access == NESTED_XLATE_FETCH) ? TRUE : FALSE,
+ &page, &shadow_page, xc->host_reg_flags);
}
return VMM_OK;
@@ -517,6 +789,7 @@ static int nested_xlate_vsstage(struct nested_xlate_context *xc,
int cpu_vcpu_nested_init(struct vmm_vcpu *vcpu)
{
+ int rc;
u32 pgtbl_attr = 0, pgtbl_hw_tag = 0;
struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
@@ -530,6 +803,12 @@ int cpu_vcpu_nested_init(struct vmm_vcpu *vcpu)
struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
return VMM_ENOMEM;
}
+ rc = nested_swtlb_init(vcpu);
+ if (rc) {
+ mmu_pgtbl_free(npriv->pgtbl);
+ return rc;
+ }
+
return VMM_OK;
+ }
+
}
@@ -537,6 +816,7 @@ void cpu_vcpu_nested_reset(struct vmm_vcpu *vcpu)
{
struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+ cpu_vcpu_nested_swtlb_flush(vcpu, 0, 0);
npriv->virt = FALSE;
#ifdef CONFIG_64BIT
npriv->hstatus = HSTATUS_VSXL_RV64 << HSTATUS_VSXL_SHIFT;
@@ -566,7 +846,10 @@ void cpu_vcpu_nested_reset(struct vmm_vcpu *vcpu)
void cpu_vcpu_nested_deinit(struct vmm_vcpu *vcpu)
{
- mmu_pgtbl_free(riscv_nested_priv(vcpu)->pgtbl);
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ nested_swtlb_deinit(vcpu);
+ mmu_pgtbl_free(npriv->pgtbl);
}
void cpu_vcpu_nested_dump_regs(struct vmm_chardev *cdev,
@@ -694,7 +977,7 @@ int cpu_vcpu_nested_hext_csr_rmw(struct vmm_vcpu *vcpu, arch_regs_t *regs,
unsigned long new_val, unsigned long wr_mask)
{
int csr_shift = 0;
- bool read_only = FALSE;
+ bool read_only = FALSE, nuke_swtlb = FALSE;
unsigned int csr_priv = (csr_num >> 8) & 0x3;
unsigned long *csr, mode, zero = 0, writeable_mask = 0;
struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
@@ -824,6 +1107,14 @@ int cpu_vcpu_nested_hext_csr_rmw(struct vmm_vcpu *vcpu, arch_regs_t *regs,
}
new_val &= ~HGATP_MODE;
new_val |= (mode << HGATP_MODE_SHIFT) & HGATP_MODE;
+ if ((new_val ^ npriv->hgatp) & HGATP_MODE) {
+ nuke_swtlb = TRUE;
+ }
+ }
+ if (wr_mask & HGATP_VMID) {
+ if ((new_val ^ npriv->hgatp) & HGATP_VMID) {
+ nuke_swtlb = TRUE;
+ }
}
break;
case CSR_VSSTATUS:
@@ -922,6 +1213,10 @@ int cpu_vcpu_nested_hext_csr_rmw(struct vmm_vcpu *vcpu, arch_regs_t *regs,
*csr = (*csr & ~wr_mask) | (new_val & wr_mask);
}
+ if (nuke_swtlb) {
}
+ cpu_vcpu_nested_swtlb_flush(vcpu, 0, 0);
+ }
+
return VMM_OK;
}
@@ -930,20 +1225,115 @@ int cpu_vcpu_nested_page_fault(struct vmm_vcpu *vcpu,
const struct cpu_vcpu_trap *trap,
struct cpu_vcpu_trap *out_trap)
{
- /* TODO: */
- return VMM_OK;
+ int rc = VMM_OK;
+ physical_addr_t guest_gpa;
+ struct nested_xlate_context xc;
+ enum nested_xlate_access guest_access;
+
+ /*
+ * This function is called to handle guest page faults
+ * from virtual-VS/VU modes.
+ *
+ * We perform a guest gpa to host pa translation for
+ * the faulting guest gpa so that nested software TLB
+ * and shadow page table is updated.
+ */
+
+ guest_gpa = ((physical_addr_t)trap->htval << 2);
+ guest_gpa |= ((physical_addr_t)trap->stval & 0x3);
+ switch (trap->scause) {
+ case CAUSE_LOAD_GUEST_PAGE_FAULT:
+ guest_access = NESTED_XLATE_LOAD;
+ break;
+ case CAUSE_STORE_GUEST_PAGE_FAULT:
+ guest_access = NESTED_XLATE_STORE;
+ break;
+ case CAUSE_FETCH_GUEST_PAGE_FAULT:
+ guest_access = NESTED_XLATE_FETCH;
+ break;
+ default:
+ memcpy(out_trap, trap, sizeof(*out_trap));
+ return VMM_OK;
+ }
+
+ nested_xlate_context_init(&xc, vcpu, guest_access, trap_from_smode,
+ }
+
+ csr_read(CSR_VSSTATUS), FALSE);
+ rc = nested_xlate_gstage(&xc, guest_gpa, guest_access);
+ if (rc == VMM_EFAULT) {
+ out_trap->sepc = trap->sepc;
+ out_trap->scause = xc.scause;
+ out_trap->stval = trap->stval;
+ out_trap->htval = xc.htval;
+ out_trap->htinst = trap->htinst;
+ rc = VMM_OK;
+ }
+
+ return rc;
}
void cpu_vcpu_nested_hfence_vvma(struct vmm_vcpu *vcpu,
unsigned long *vaddr, unsigned int *asid)
{
- /* TODO: */
+ unsigned long hgatp;
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+
+ /*
+ * The HFENCE.VVMA instructions help virtual-HS mode flush
+
+ /*
+ * VS-stage TLB entries for virtual-VS/VU modes.
+ *
+ * When host G-stage VMID is not available, we flush all guest
+ * TLB (both G-stage and VS-stage) entries upon nested virt
+ * state change so the HFENCE.VVMA becomes a NOP in this case.
+ */
+
+ if (!mmu_pgtbl_has_hw_tag(npriv->pgtbl)) {
+ return;
+ }
+
+ hgatp = mmu_pgtbl_hw_tag(npriv->pgtbl);
+ hgatp = csr_swap(CSR_HGATP, hgatp << HGATP_VMID_SHIFT);
+
+ if (!vaddr && !asid) {
+ __hfence_vvma_all();
+ } else if (!vaddr && asid) {
+ __hfence_vvma_asid(*asid);
+ } else if (vaddr && !asid) {
+ __hfence_vvma_va(*vaddr);
+ } else {
+ __hfence_vvma_asid_va(*vaddr, *asid);
+ }
+
+ csr_write(CSR_HGATP, hgatp);
}
void cpu_vcpu_nested_hfence_gvma(struct vmm_vcpu *vcpu,
physical_addr_t *gaddr, unsigned int *vmid)
{
- /* TODO: */
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
+ unsigned long current_vmid =
+ (npriv->hgatp & HGATP_VMID) >> HGATP_VMID_SHIFT;
+
+ /*
+ * The HFENCE.GVMA instructions help virtual-HS mode flush
+ * G-stage TLB entries for virtual-VS/VU modes.
+ *
+ * Irrespective whether host G-stage VMID is available or not,
+ * we flush all software TLB (only G-stage) entries upon guest
+ * hgatp.VMID change so the HFENCE.GVMA instruction becomes a
+ * NOP for virtual-HS mode when the current guest hgatp.VMID
+ * is different from the VMID specified in the HFENCE.GVMA
+ * instruction.
+ */
+
+ if (vmid && current_vmid != *vmid) {
+ return;
+ }
+
+ if (gaddr) {
+ cpu_vcpu_nested_swtlb_flush(vcpu, *gaddr, 1);
+ } else {
+ cpu_vcpu_nested_swtlb_flush(vcpu, 0, 0);
+ }
}
int cpu_vcpu_nested_hlv(struct vmm_vcpu *vcpu, unsigned long vaddr,
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c b/arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c
int cpu_vcpu_nested_hlv(struct vmm_vcpu *vcpu, unsigned long vaddr,
index eb7c3acb..4c5cb44c 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_sbi_replace.c
@@ -29,6 +29,8 @@
#include <cpu_sbi.h>
#include <cpu_vcpu_sbi.h>
#include <cpu_vcpu_timer.h>
+#include <cpu_vcpu_nested.h>
+#include <generic_mmu.h>
#include <riscv_sbi.h>
static int vcpu_sbi_time_ecall(struct vmm_vcpu *vcpu,
@@ -63,7 +65,8 @@ static int vcpu_sbi_rfence_ecall(struct vmm_vcpu *vcpu,
struct vmm_vcpu *rvcpu;
struct vmm_cpumask cm, hm;
struct vmm_guest *guest = vcpu->guest;
- unsigned long hmask = args[0], hbase = args[1];
+ unsigned long hgatp, hmask = args[0], hbase = args[1];
+ struct riscv_priv_nested *npriv = riscv_nested_priv(vcpu);
vmm_cpumask_clear(&cm);
vmm_manager_for_each_guest_vcpu(rvcpu, guest) {
@@ -96,13 +99,56 @@ static int vcpu_sbi_rfence_ecall(struct vmm_vcpu *vcpu,
break;
case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA:
case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID:
+ /* Flush the nested software TLB of calling VCPU */
+ cpu_vcpu_nested_swtlb_flush(vcpu, args[2], args[3]);
+
+ if (mmu_pgtbl_has_hw_tag(npriv->pgtbl)) {
+ /*
+ * We use two VMIDs for nested virtualization:
+ * one for virtual-HS/U modes and another for
+ * virtual-VS/VU modes. This means we need to
+ * restrict guest remote HFENCE.GVMA to VMID
+ * used for virtual-VS/VU modes.
+ */
+ sbi_remote_hfence_gvma_vmid(vmm_cpumask_bits(&hm),
+ args[2], args[3],
+ mmu_pgtbl_has_hw_tag(npriv->pgtbl));
+ } else {
+ /*
+ * No VMID support so we do remote HFENCE.GVMA
+ * accross all VMIDs.
+ */
+ sbi_remote_hfence_gvma(vmm_cpumask_bits(&hm),
+ args[2], args[3]);
+ }
+ break;
case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA:
case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID:
- /*
- * TODO: these SBI calls will be implemented as part
- * of nested virtualization support so treat these
- * calls as NOPs.
- */
+ hgatp = 0;
+ if (mmu_pgtbl_has_hw_tag(npriv->pgtbl)) {
+ /*
+ * We use two VMIDs for nested virtualization:
+ * one for virtual-HS/U modes and another for
+ * virtual-VS/VU modes. This means we need to
+ * switch hgatp.VMID before doing forwarding
+ * SBI call to host firmware.
+ */
+ hgatp = mmu_pgtbl_hw_tag(npriv->pgtbl);
+ hgatp = csr_swap(CSR_HGATP, hgatp << HGATP_VMID_SHIFT);
+ }
+
+ if (func_id == SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA) {
+ sbi_remote_hfence_vvma(vmm_cpumask_bits(&hm),
+ args[2], args[3]);
+ } else {
+ sbi_remote_hfence_vvma_asid(
+ vmm_cpumask_bits(&hm),
+ args[2], args[3], args[4]);
+ }
+
+ if (mmu_pgtbl_has_hw_tag(npriv->pgtbl)) {
+ csr_write(CSR_HGATP, hgatp);
+ }
break;
default:
return SBI_ERR_NOT_SUPPORTED;
diff --git a/arch/riscv/cpu/generic/include/cpu_vcpu_nested.h b/arch/riscv/cpu/generic/include/cpu_vcpu_nested.h
break;
default:
return SBI_ERR_NOT_SUPPORTED;
index 669690d3..274633e5 100644
--- a/arch/riscv/cpu/generic/include/cpu_vcpu_nested.h
+++ b/arch/riscv/cpu/generic/include/cpu_vcpu_nested.h
@@ -30,6 +30,11 @@ struct vmm_vcpu;
struct cpu_vcpu_trap;
struct arch_regs;
+/** Function to flush nested software TLB */
+void cpu_vcpu_nested_swtlb_flush(struct vmm_vcpu *vcpu,
+ physical_addr_t guest_gpa,
+ physical_size_t guest_gpa_size);
+
/** Function to init nested state */
int cpu_vcpu_nested_init(struct vmm_vcpu *vcpu);
--
2.25.1
Anup Patel
Mar 23, 2022, 12:46:37 AM3/23/22
to Anup Patel, Xvisor Devel
Regards,
Anup
Reply all
Reply to author
Forward
0 new messages