[PATCH 0/3] Fixes for nested virtualization in Xvisor RISC-V

[Anup Patel]

Running KVM RISC-V inside Xvisor RISC-V was broken for Linux-6.0 so
this series does the required fixes.

Anup Patel (3):
RISC-V: Emulate dummy henvcfg[h] CSR for the guest hypervisor
TESTS: Don't disable CONFIG_PROFILING in update-linux-defconfig.sh
EMULATORS: plic: Fix number of irq lines

arch/riscv/cpu/generic/cpu_vcpu_nested.c | 6 +++++
arch/riscv/cpu/generic/cpu_vcpu_trap.c | 8 ++++++
.../cpu/generic/include/riscv_encoding.h | 27 +++++++++++++++++--
emulators/pic/plic.c | 3 ++-
.../vexpress-a15/linux/linux_extra.config | 1 +
tests/arm32/virt-v7/linux/linux_extra.config | 1 +
tests/arm64/virt-v8/linux/linux_extra.config | 1 +
.../common/scripts/update-linux-defconfig.sh | 1 -
tests/riscv/virt32/linux/virt32.dts | 2 +-
tests/riscv/virt32/virt32-guest.dts | 2 +-
tests/riscv/virt64/linux/virt64.dts | 2 +-
tests/riscv/virt64/virt64-guest.dts | 2 +-
12 files changed, 48 insertions(+), 8 deletions(-)

--
2.34.1

[Anup Patel]

The latest KVM RISC-V programes henvcfg[h] CSR so let us emulate
a dummy (read zero and write ignore) henvcfg[h] CSR for the guest
hypervisor.

Signed-off-by: Anup Patel <apa...@ventanamicro.com>
---

arch/riscv/cpu/generic/cpu_vcpu_nested.c | 6 +++++
arch/riscv/cpu/generic/cpu_vcpu_trap.c | 8 ++++++
.../cpu/generic/include/riscv_encoding.h | 27 +++++++++++++++++--

3 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/arch/riscv/cpu/generic/cpu_vcpu_nested.c b/arch/riscv/cpu/generic/cpu_vcpu_nested.c
index 4d3d4f08..9a96d9cd 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_nested.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_nested.c
@@ -1143,6 +1143,12 @@ int cpu_vcpu_nested_hext_csr_rmw(struct vmm_vcpu *vcpu, arch_regs_t *regs,
}
}
break;
+ case CSR_HENVCFG:
+#ifndef CONFIG_64BIT
+ case CSR_HENVCFGH:
+#endif
+ csr = &zero;
+ break;
case CSR_VSSTATUS:
csr = &npriv->vsstatus;
writeable_mask = SSTATUS_SIE | SSTATUS_SPIE | SSTATUS_UBE |
diff --git a/arch/riscv/cpu/generic/cpu_vcpu_trap.c b/arch/riscv/cpu/generic/cpu_vcpu_trap.c
index 81140ac4..a2085a08 100644
--- a/arch/riscv/cpu/generic/cpu_vcpu_trap.c
+++ b/arch/riscv/cpu/generic/cpu_vcpu_trap.c
@@ -621,6 +621,14 @@ static const struct csr_func csr_funcs[] = {
.csr_num = CSR_HGATP,
.rmw_func = cpu_vcpu_nested_hext_csr_rmw,
},
+ {
+ .csr_num = CSR_HENVCFG,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
+ {
+ .csr_num = CSR_HENVCFGH,
+ .rmw_func = cpu_vcpu_nested_hext_csr_rmw,
+ },
{
.csr_num = CSR_VSSTATUS,
.rmw_func = cpu_vcpu_nested_hext_csr_rmw,
diff --git a/arch/riscv/cpu/generic/include/riscv_encoding.h b/arch/riscv/cpu/generic/include/riscv_encoding.h
index de79cf30..1124d4ba 100644
--- a/arch/riscv/cpu/generic/include/riscv_encoding.h
+++ b/arch/riscv/cpu/generic/include/riscv_encoding.h
@@ -359,6 +359,18 @@
#define HVICTL_IPRIOM 0x00000100
#define HVICTL_IPRIO 0x000000ff

+/* xENVCFG flags */
+#define ENVCFG_STCE (_AC(1, ULL) << 63)
+#define ENVCFG_PBMTE (_AC(1, ULL) << 62)
+#define ENVCFG_CBZE (_AC(1, UL) << 7)
+#define ENVCFG_CBCFE (_AC(1, UL) << 6)
+#define ENVCFG_CBIE_SHIFT 4
+#define ENVCFG_CBIE (_AC(0x3, UL) << ENVCFG_CBIE_SHIFT)
+#define ENVCFG_CBIE_ILL _AC(0x0, UL)
+#define ENVCFG_CBIE_FLUSH _AC(0x1, UL)
+#define ENVCFG_CBIE_INV _AC(0x3, UL)
+#define ENVCFG_FIOM _AC(0x1, UL)
+
/* ===== User-level CSRs ===== */

/* User Trap Setup (N-extension) */
@@ -482,6 +494,12 @@
#define CSR_SIEH 0x114
#define CSR_SIPH 0x154

+/* Supervisor Configuration */
+#define CSR_SENVCFG 0x10a
+
+/* Counter Overflow CSR */
+#define CSR_SCOUNTOVF 0xda0
+
/* ===== Hypervisor-level CSRs ===== */

/* Hypervisor Trap Setup (H-extension) */
@@ -506,6 +524,10 @@
#define CSR_HTIMEDELTA 0x605
#define CSR_HTIMEDELTAH 0x615

+/* Hypervisor Configuration (H-extension) */
+#define CSR_HENVCFG 0x60a
+#define CSR_HENVCFGH 0x61a
+
/* Virtual Supervisor Registers (H-extension) */
#define CSR_VSSTATUS 0x200
#define CSR_VSIE 0x204
@@ -782,8 +804,9 @@
#define CSR_MHPMEVENT30H 0x73e
#define CSR_MHPMEVENT31H 0x73f

-/* Counter Overflow CSR */
-#define CSR_SCOUNTOVF 0xda0
+/* Machine Configuration */
+#define CSR_MENVCFG 0x30a
+#define CSR_MENVCFGH 0x31a

/* Debug/Trace Registers */
#define CSR_TSELECT 0x7a0
--
2.34.1

[Anup Patel]

Disabling CONFIG_PROFILING in update-linux-defconfig.sh will disable
profiling for all guest types. Instead, we should explicitly disable
CONFIG_PROFILING from linux_extra.config of appropriate guest types.

Signed-off-by: Anup Patel <apa...@ventanamicro.com>
---

tests/arm32/vexpress-a15/linux/linux_extra.config | 1 +

tests/arm32/virt-v7/linux/linux_extra.config | 1 +
tests/arm64/virt-v8/linux/linux_extra.config | 1 +

tests/common/scripts/update-linux-defconfig.sh | 1 -
4 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/arm32/vexpress-a15/linux/linux_extra.config b/tests/arm32/vexpress-a15/linux/linux_extra.config
index c933e79c..56b150d2 100644
--- a/tests/arm32/vexpress-a15/linux/linux_extra.config
+++ b/tests/arm32/vexpress-a15/linux/linux_extra.config
@@ -1,3 +1,4 @@
CONFIG_NO_HZ_FULL=n
CONFIG_NO_HZ_IDLE=y
CONFIG_DRM=n
+CONFIG_PROFILING=n
diff --git a/tests/arm32/virt-v7/linux/linux_extra.config b/tests/arm32/virt-v7/linux/linux_extra.config
index e4bafbe2..6549843c 100644
--- a/tests/arm32/virt-v7/linux/linux_extra.config
+++ b/tests/arm32/virt-v7/linux/linux_extra.config
@@ -1,2 +1,3 @@
CONFIG_NO_HZ_FULL=n
CONFIG_NO_HZ_IDLE=y
+CONFIG_PROFILING=n
diff --git a/tests/arm64/virt-v8/linux/linux_extra.config b/tests/arm64/virt-v8/linux/linux_extra.config
index 993a43f9..f8684c39 100644
--- a/tests/arm64/virt-v8/linux/linux_extra.config
+++ b/tests/arm64/virt-v8/linux/linux_extra.config
@@ -29,3 +29,4 @@ CONFIG_ARCH_VEXPRESS=y
CONFIG_ARCH_XGENE=n
CONFIG_ARCH_ZX=n
CONFIG_ARCH_ZYNQMP=n
+CONFIG_PROFILING=n
diff --git a/tests/common/scripts/update-linux-defconfig.sh b/tests/common/scripts/update-linux-defconfig.sh
index 68d50949..70ae9e3f 100755
--- a/tests/common/scripts/update-linux-defconfig.sh
+++ b/tests/common/scripts/update-linux-defconfig.sh
@@ -133,7 +133,6 @@ LINUX_OPTIONS+=" CONFIG_RPMSG_VIRTIO=y"
LINUX_OPTIONS+=" CONFIG_DRM=y"
LINUX_OPTIONS+=" CONFIG_DRM_VIRTIO_GPU=y"

-LINUX_OPTIONS+=" CONFIG_PROFILING=n"
LINUX_OPTIONS+=" CONFIG_OPROFILE=n"
LINUX_OPTIONS+=" CONFIG_MTD=n"
LINUX_OPTIONS+=" CONFIG_SOUND=n"
--
2.34.1

[Anup Patel]

The IRQ#0 is a special/reserved interrupt line in PLIC so we
should emulate one extra line to account this.

Signed-off-by: Anup Patel <apa...@ventanamicro.com>
---

emulators/pic/plic.c | 3 ++-

tests/riscv/virt32/linux/virt32.dts | 2 +-
tests/riscv/virt32/virt32-guest.dts | 2 +-
tests/riscv/virt64/linux/virt64.dts | 2 +-
tests/riscv/virt64/virt64-guest.dts | 2 +-

5 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/emulators/pic/plic.c b/emulators/pic/plic.c
index 0f243929..812187b9 100644
--- a/emulators/pic/plic.c
+++ b/emulators/pic/plic.c
@@ -575,8 +575,9 @@ static int plic_emulator_probe(struct vmm_guest *guest,
}

if (vmm_devtree_read_u32(edev->node, "num_irq", &s->num_irq)) {
- s->num_irq = MAX_DEVICES;
+ s->num_irq = MAX_DEVICES - 1;
}
+ s->num_irq += 1; /* IRQ0 is dummy */
if (s->num_irq > MAX_DEVICES) {
rc = VMM_EINVALID;
goto plic_emulator_probe_freestate_fail;
diff --git a/tests/riscv/virt32/linux/virt32.dts b/tests/riscv/virt32/linux/virt32.dts
index 3332ea7b..f786919d 100644
--- a/tests/riscv/virt32/linux/virt32.dts
+++ b/tests/riscv/virt32/linux/virt32.dts
@@ -47,7 +47,7 @@
interrupt-controller;
#interrupt-cells = <1>;
riscv,max-priority = <7>;
- riscv,ndev = <32>;
+ riscv,ndev = <31>;
reg = <0x0 0x0c000000 0 0x04000000>;
};

diff --git a/tests/riscv/virt32/virt32-guest.dts b/tests/riscv/virt32/virt32-guest.dts
index 5f8a6377..7f418fe2 100644
--- a/tests/riscv/virt32/virt32-guest.dts
+++ b/tests/riscv/virt32/virt32-guest.dts
@@ -41,7 +41,7 @@
device_type = "pic";
compatible = "sifive,plic0";
parent_irq = <10>;
- num_irq = <32>;
+ num_irq = <31>;
max_priority = <7>;
};

diff --git a/tests/riscv/virt64/linux/virt64.dts b/tests/riscv/virt64/linux/virt64.dts
index d1687729..a43b8b1e 100644
--- a/tests/riscv/virt64/linux/virt64.dts
+++ b/tests/riscv/virt64/linux/virt64.dts
@@ -47,7 +47,7 @@
interrupt-controller;
#interrupt-cells = <1>;
riscv,max-priority = <7>;
- riscv,ndev = <32>;
+ riscv,ndev = <31>;
reg = <0x0 0x0c000000 0 0x04000000>;
};

diff --git a/tests/riscv/virt64/virt64-guest.dts b/tests/riscv/virt64/virt64-guest.dts
index fa87d5ef..66f4dd59 100644
--- a/tests/riscv/virt64/virt64-guest.dts
+++ b/tests/riscv/virt64/virt64-guest.dts
@@ -41,7 +41,7 @@
device_type = "pic";
compatible = "sifive,plic0";
parent_irq = <10>;
- num_irq = <32>;
+ num_irq = <31>;
max_priority = <7>;
};

--
2.34.1

[Anup Patel]

On Thu, Sep 1, 2022 at 9:03 PM Anup Patel <apa...@ventanamicro.com> wrote:
>
> Running KVM RISC-V inside Xvisor RISC-V was broken for Linux-6.0 so
> this series does the required fixes.
>
> Anup Patel (3):
> RISC-V: Emulate dummy henvcfg[h] CSR for the guest hypervisor
> TESTS: Don't disable CONFIG_PROFILING in update-linux-defconfig.sh
> EMULATORS: plic: Fix number of irq lines

Applied this series to the xvisor-next repo

Thanks,
Anup

>
> arch/riscv/cpu/generic/cpu_vcpu_nested.c | 6 +++++
> arch/riscv/cpu/generic/cpu_vcpu_trap.c | 8 ++++++
> .../cpu/generic/include/riscv_encoding.h | 27 +++++++++++++++++--
> emulators/pic/plic.c | 3 ++-
> .../vexpress-a15/linux/linux_extra.config | 1 +
> tests/arm32/virt-v7/linux/linux_extra.config | 1 +
> tests/arm64/virt-v8/linux/linux_extra.config | 1 +
> .../common/scripts/update-linux-defconfig.sh | 1 -
> tests/riscv/virt32/linux/virt32.dts | 2 +-
> tests/riscv/virt32/virt32-guest.dts | 2 +-
> tests/riscv/virt64/linux/virt64.dts | 2 +-
> tests/riscv/virt64/virt64-guest.dts | 2 +-
> 12 files changed, 48 insertions(+), 8 deletions(-)
>
> --
> 2.34.1
>

> --
> You received this message because you are subscribed to the Google Groups "Xvisor Development" group.
> To unsubscribe from this group and stop receiving emails from it, send an email to xvisor-devel...@googlegroups.com.
> To view this discussion on the web visit https://groups.google.com/d/msgid/xvisor-devel/20220901153326.108782-1-apatel%40ventanamicro.com.