virtio-ivshmem-*: only works on the 2nd try

[Philipp Rosenberger]

Hi,

I have managed to get virtio-ivshmem console and block running. But I
observed a strange behavior. I do the following:

1. Boot up the board.
2. Enable the rootcell.
3. echo "110a 4106 110a 4106 ffc002 ffffff" > \
/sys/bus/pci/drivers/uio_ivshmem/new_id
4. virtio-ivshmem-block /dev/uio0 /path/to/disk.image
5. boot linux-inmate
6. virtio-ivshmem 0000:00:0f.0: backend not ready
7. kernel panic

If I redo the sets 4 and 5 the inmates starts as expected and I can
access the disk.image via /dev/vda.

I found, the the virtio-ivshmem-block tool waits for an interrupt if
'state[peer_id] != VIRTIO_STATE_RESET'. But there is no interrupt.

> 129: 0 0 0 IR-PCI-MSI 245760-edge uio_ivshmem[0000:00:0f.0]
> 130: 0 0 0 IR-PCI-MSI 245761-edge uio_ivshmem[0000:00:0f.0]

After the first run the 'state[peer_id]' is already set to
VIRTIO_STATE_RESET. Now the linux inmate starts as expected.

> 129: 0 18 0 IR-PCI-MSI 245760-edge uio_ivshmem[0000:00:0f.0]
> 130: 0 0 136 IR-PCI-MSI 245761-edge uio_ivshmem[0000:00:0f.0]

Best regards,
Philipp

My root-cell:
> #define USE_VIRTIO_BLK
>
> #ifndef USE_IVSHMEM_NET
> # define NET_MEM 0
> # define NET_PCI 0
> #else
> # define NET_MEM 4
> # define NET_PCI 1
> #endif
>
> #ifndef USE_IVSHMEM_DEMO
> # define DEMO_MEM 0
> # define DEMO_PCI 0
> #else
> # define DEMO_MEM 5
> # define DEMO_PCI 1
> #endif
>
> #ifndef USE_VIRTIO_CON
> # define CON_MEM 0
> # define CON_PCI 0
> #else
> # define CON_MEM 4
> # define CON_PCI 1
> #endif
>
> #ifndef USE_VIRTIO_BLK
> # define BLK_MEM 0
> # define BLK_PCI 0
> #else
> # define BLK_MEM 4
> # define BLK_PCI 1
> #endif
>
> #define INTERCELL_MEM (NET_MEM + DEMO_MEM + CON_MEM + BLK_MEM)
> #define INTERCELL_PCI (NET_PCI + DEMO_PCI + CON_PCI + BLK_PCI)
>
> #include <jailhouse/types.h>
> #include <jailhouse/cell-config.h>
>
> struct {
> struct jailhouse_system header;
> __u64 cpus[1];
> struct jailhouse_memory mem_regions[43 + INTERCELL_MEM];
> struct jailhouse_irqchip irqchips[1];
> struct jailhouse_pio pio_regions[15];
> struct jailhouse_pci_device pci_devices[14 + INTERCELL_PCI];
> struct jailhouse_pci_capability pci_caps[35];
> } __attribute__((packed)) config = {
> .header = {
> .signature = JAILHOUSE_SYSTEM_SIGNATURE,
> .revision = JAILHOUSE_CONFIG_REVISION,
> .flags = JAILHOUSE_SYS_VIRTUAL_DEBUG_CONSOLE,
> .hypervisor_memory = {
> .phys_start = 0x100000000,
> .size = 0x600000,
> },
> .debug_console = {
> .address = 0x3f8,
> .type = JAILHOUSE_CON_TYPE_8250,
> .flags = JAILHOUSE_CON_ACCESS_PIO |
> JAILHOUSE_CON_REGDIST_1,
> },
> .platform_info = {
> .pci_mmconfig_base = 0xe0000000,
> .pci_mmconfig_end_bus = 0xff,
> .x86 = {
> .pm_timer_address = 0x1808,
> .vtd_interrupt_limit = 256,
> .iommu_units = {
> {
> .type = JAILHOUSE_IOMMU_INTEL,
> .base = 0xfed90000,
> .size = 0x1000,
> },
> {
> .type = JAILHOUSE_IOMMU_INTEL,
> .base = 0xfed91000,
> .size = 0x1000,
> },
> },
> },
> },
> .root_cell = {
> .name = "RootCell",
> .cpu_set_size = sizeof(config.cpus),
> .num_memory_regions = ARRAY_SIZE(config.mem_regions),
> .num_irqchips = ARRAY_SIZE(config.irqchips),
> .num_pio_regions = ARRAY_SIZE(config.pio_regions),
> .num_pci_devices = ARRAY_SIZE(config.pci_devices),
> .num_pci_caps = ARRAY_SIZE(config.pci_caps),
> },
> },
>
> .cpus = {
> 0x000000000000000f,
> },
>
> .mem_regions = {
> #if defined(USE_IVSHMEM_NET)
> /* IVSHMEM shared memory regions (networking) */
> JAILHOUSE_SHMEM_NET_REGIONS(0x16f000000, 0),
> #endif /* USE_IVSHMEM_NET */
>
> #if defined(USE_IVSHMEM_DEMO)
> /* IVSHMEM shared memory regions (demo) */
> {
> .phys_start = 0x16f100000,
> .virt_start = 0x16f100000,
> .size = 0x1000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_ROOTSHARED,
> },
> {
> .phys_start = 0x16f101000,
> .virt_start = 0x16f101000,
> .size = 0x9000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_ROOTSHARED,
> },
> {
> .phys_start = 0x16f10a000,
> .virt_start = 0x16f10a000,
> .size = 0x2000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_ROOTSHARED,
> },
> {
> .phys_start = 0x16f10c000,
> .virt_start = 0x16f10c000,
> .size = 0x2000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_ROOTSHARED,
> },
> {
> .phys_start = 0x16f10e000,
> .virt_start = 0x16f10e000,
> .size = 0x2000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_ROOTSHARED,
> },
> #endif /* USE_IVSHMEM_DEMO */
>
> #if defined(USE_VIRTIO_CON)
> /* IVSHMEM shared memory region (virtio-con back-end) */
> {
> .phys_start = 0x16f200000,
> .virt_start = 0x16f200000,
> .size = 0x1000,
> .flags = JAILHOUSE_MEM_READ,
> },
> {
> .phys_start = 0x16f201000,
> .virt_start = 0x16f201000,
> .size = 0xf000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> { 0 },
> { 0 },
> #endif /* USE_VIRTIO_CON */
>
> #if defined(USE_VIRTIO_BLK)
> /* IVSHMEM shared memory region (virtio-blk back-end) */
> {
> .phys_start = 0x16f300000,
> .virt_start = 0x16f300000,
> .size = 0x1000,
> .flags = JAILHOUSE_MEM_READ,
> },
> {
> .phys_start = 0x16f301000,
> .virt_start = 0x16f301000,
> .size = 0xdf000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> { 0 },
> { 0 },
> #endif /* USE_VIRTIO_BLK */
>
> /* MemRegion: 00000000-0009dbff : System RAM */
> {
> .phys_start = 0x0,
> .virt_start = 0x0,
> .size = 0x9e000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_EXECUTE | JAILHOUSE_MEM_DMA,
> },
> /* MemRegion: 00100000-4e1fffff : System RAM */
> {
> .phys_start = 0x100000,
> .virt_start = 0x100000,
> .size = 0x4e100000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_EXECUTE | JAILHOUSE_MEM_DMA,
> },
> /* MemRegion: 4e200000-4fffffff : Kernel */
> {
> .phys_start = 0x4e200000,
> .virt_start = 0x4e200000,
> .size = 0x1e00000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_EXECUTE | JAILHOUSE_MEM_DMA,
> },
> /* MemRegion: 50000000-87d9ffff : System RAM */
> {
> .phys_start = 0x50000000,
> .virt_start = 0x50000000,
> .size = 0x37da0000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_EXECUTE | JAILHOUSE_MEM_DMA,
> },
> /* MemRegion: 87da0000-87da0fff : ACPI Non-volatile Storage */
> {
> .phys_start = 0x87da0000,
> .virt_start = 0x87da0000,
> .size = 0x1000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: 87dcb000-8c4f8fff : System RAM */
> {
> .phys_start = 0x87dcb000,
> .virt_start = 0x87dcb000,
> .size = 0x472e000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_EXECUTE | JAILHOUSE_MEM_DMA,
> },
> /* MemRegion: 8c56f000-8c5a3fff : ACPI Tables */
> {
> .phys_start = 0x8c56f000,
> .virt_start = 0x8c56f000,
> .size = 0x35000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: 8c5a4000-8ce96fff : ACPI Non-volatile Storage */
> {
> .phys_start = 0x8c5a4000,
> .virt_start = 0x8c5a4000,
> .size = 0x8f3000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: 8d2ff000-8d2fffff : System RAM */
> {
> .phys_start = 0x8d2ff000,
> .virt_start = 0x8d2ff000,
> .size = 0x1000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_EXECUTE | JAILHOUSE_MEM_DMA,
> },
> /* MemRegion: 8d400000-8dffffff : RAM buffer */
> {
> .phys_start = 0x8d400000,
> .virt_start = 0x8d400000,
> .size = 0xc00000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_EXECUTE | JAILHOUSE_MEM_DMA,
> },
> /* MemRegion: c0000000-cfffffff : 0000:00:02.0 */
> {
> .phys_start = 0xc0000000,
> .virt_start = 0xc0000000,
> .size = 0x10000000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: de000000-deffffff : 0000:00:02.0 */
> {
> .phys_start = 0xde000000,
> .virt_start = 0xde000000,
> .size = 0x1000000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: df100000-df11ffff : e1000e */
> {
> .phys_start = 0xdf100000,
> .virt_start = 0xdf100000,
> .size = 0x20000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: df120000-df12ffff : ICH HD audio */
> {
> .phys_start = 0xdf120000,
> .virt_start = 0xdf120000,
> .size = 0x10000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: df130000-df13ffff : xhci-hcd */
> {
> .phys_start = 0xdf130000,
> .virt_start = 0xdf130000,
> .size = 0x10000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: df140000-df143fff : ICH HD audio */
> {
> .phys_start = 0xdf140000,
> .virt_start = 0xdf140000,
> .size = 0x4000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: df144000-df147fff : 0000:00:1f.2 */
> {
> .phys_start = 0xdf144000,
> .virt_start = 0xdf144000,
> .size = 0x4000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: df148000-df149fff : ahci */
> {
> .phys_start = 0xdf148000,
> .virt_start = 0xdf148000,
> .size = 0x2000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: df14a000-df14a0ff : 0000:00:1f.4 */
> {
> .phys_start = 0xdf14a000,
> .virt_start = 0xdf14a000,
> .size = 0x1000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: df14b000-df14b7ff : ahci */
> {
> .phys_start = 0xdf14b000,
> .virt_start = 0xdf14b000,
> .size = 0x1000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: df14c000-df14c0ff : ahci */
> {
> .phys_start = 0xdf14c000,
> .virt_start = 0xdf14c000,
> .size = 0x1000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: df14d000-df14dfff : 0000:00:14.2 */
> {
> .phys_start = 0xdf14d000,
> .virt_start = 0xdf14d000,
> .size = 0x1000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: df14e000-df14efff : 0000:00:08.0 */
> {
> .phys_start = 0xdf14e000,
> .virt_start = 0xdf14e000,
> .size = 0x1000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: dffe0000-dfffffff : pnp 00:08 */
> {
> .phys_start = 0xdffe0000,
> .virt_start = 0xdffe0000,
> .size = 0x20000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: fd000000-fdabffff : pnp 00:09 */
> {
> .phys_start = 0xfd000000,
> .virt_start = 0xfd000000,
> .size = 0xac0000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: fdac0000-fdacffff : pnp 00:0b */
> {
> .phys_start = 0xfdac0000,
> .virt_start = 0xfdac0000,
> .size = 0x10000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: fdad0000-fdadffff : pnp 00:09 */
> {
> .phys_start = 0xfdad0000,
> .virt_start = 0xfdad0000,
> .size = 0x10000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: fdae0000-fdaeffff : pnp 00:0b */
> {
> .phys_start = 0xfdae0000,
> .virt_start = 0xfdae0000,
> .size = 0x10000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: fdaf0000-fdafffff : pnp 00:0b */
> {
> .phys_start = 0xfdaf0000,
> .virt_start = 0xfdaf0000,
> .size = 0x10000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: fdc6000c-fdc6000f : iTCO_wdt */
> {
> .phys_start = 0xfdc6000c,
> .virt_start = 0xfdc6000c,
> .size = 0x1000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: fe036000-fe03bfff : pnp 00:09 */
> {
> .phys_start = 0xfe036000,
> .virt_start = 0xfe036000,
> .size = 0x6000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: fe03d000-fe3fffff : pnp 00:09 */
> {
> .phys_start = 0xfe03d000,
> .virt_start = 0xfe03d000,
> .size = 0x3c3000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: fe410000-fe7fffff : pnp 00:09 */
> {
> .phys_start = 0xfe410000,
> .virt_start = 0xfe410000,
> .size = 0x3f0000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: fed00000-fed003ff : PNP0103:00 */
> {
> .phys_start = 0xfed00000,
> .virt_start = 0xfed00000,
> .size = 0x1000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: fed10000-fed17fff : pnp 00:08 */
> {
> .phys_start = 0xfed10000,
> .virt_start = 0xfed10000,
> .size = 0x8000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: fed18000-fed18fff : pnp 00:08 */
> {
> .phys_start = 0xfed18000,
> .virt_start = 0xfed18000,
> .size = 0x1000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: fed19000-fed19fff : pnp 00:08 */
> {
> .phys_start = 0xfed19000,
> .virt_start = 0xfed19000,
> .size = 0x1000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: fed20000-fed3ffff : pnp 00:08 */
> {
> .phys_start = 0xfed20000,
> .virt_start = 0xfed20000,
> .size = 0x20000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: fed45000-fed8ffff : pnp 00:08 */
> {
> .phys_start = 0xfed45000,
> .virt_start = 0xfed45000,
> .size = 0x4b000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: 000c0000-000dffff : ROMs */
> {
> .phys_start = 0xc0000,
> .virt_start = 0xc0000,
> .size = 0x20000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: 8c51b000-8c53afff : ACPI DMAR RMRR */
> /* PCI device: 00:14.0 */
> {
> .phys_start = 0x8c51b000,
> .virt_start = 0x8c51b000,
> .size = 0x20000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_EXECUTE | JAILHOUSE_MEM_DMA,
> },
> /* MemRegion: 8d800000-8fffffff : ACPI DMAR RMRR */
> /* PCI device: 00:02.0 */
> {
> .phys_start = 0x8d800000,
> .virt_start = 0x8d800000,
> .size = 0x2800000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_EXECUTE | JAILHOUSE_MEM_DMA,
> },
> /* MemRegion: 100600000-1051fffff : JAILHOUSE Inmate Memory */
> {
> .phys_start = 0x100600000,
> .virt_start = 0x100600000,
> .size = 0x4c00000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> },
>
> .irqchips = {
> /* IOAPIC 2, GSI base 0 */
> {
> .address = 0xfec00000,
> .id = 0x1f0f8,
> .pin_bitmap = {
> 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
> },
> },
> },
>
> .pio_regions = {
> /* Port I/O: 0000-001f : dma1 */
> /* PIO_RANGE(0x0, 0x20), */
> /* Port I/O: 0020-0021 : pic1 */
> /* PIO_RANGE(0x20, 0x2), */
> /* Port I/O: 0040-0043 : timer0 */
> PIO_RANGE(0x40, 0x4),
> /* Port I/O: 0050-0053 : timer1 */
> /* PIO_RANGE(0x50, 0x4), */
> /* Port I/O: 0060-0060 : keyboard */
> PIO_RANGE(0x60, 0x1),
> /* Port I/O: 0064-0064 : keyboard */
> PIO_RANGE(0x64, 0x1),
> /* Port I/O: 0070-0077 : rtc0 */
> PIO_RANGE(0x70, 0x8),
> /* Port I/O: 0080-008f : dma page reg */
> /* PIO_RANGE(0x80, 0x10), */
> /* Port I/O: 00a0-00a1 : pic2 */
> /* PIO_RANGE(0xa0, 0x2), */
> /* Port I/O: 00c0-00df : dma2 */
> /* PIO_RANGE(0xc0, 0x20), */
> /* Port I/O: 00f0-00f0 : PNP0C04:00 */
> /* PIO_RANGE(0xf0, 0x1), */
> /* Port I/O: 02e8-02ef : CGT0502:00 */
> /* PIO_RANGE(0x2e8, 0x8), */
> /* Port I/O: 02f8-02ff : serial */
> PIO_RANGE(0x2f8, 0x8),
> /* Port I/O: 03c0-03df : vga+ */
> PIO_RANGE(0x3c0, 0x20),
> /* Port I/O: 03e8-03ef : CGT0501:00 */
> /* PIO_RANGE(0x3e8, 0x8), */
> /* Port I/O: 03f8-03ff : serial */
> PIO_RANGE(0x3f8, 0x8),
> /* Port I/O: 0400-041f : iTCO_wdt */
> /* PIO_RANGE(0x400, 0x20), */
> /* Port I/O: 0680-069f : pnp 00:03 */
> /* PIO_RANGE(0x680, 0x20), */
> /* Port I/O: 0800-087f : pnp 00:04 */
> /* PIO_RANGE(0x800, 0x80), */
> /* Port I/O: 0a00-0a1f : pnp 00:07 */
> /* PIO_RANGE(0xa00, 0x20), */
> /* Port I/O: 0d00-17ff : PCI Bus 0000:00 (partial/HACK) */
> PIO_RANGE(0x0d00, 0xb00),
> /* Port I/O: 1800-1803 : ACPI PM1a_EVT_BLK */
> PIO_RANGE(0x1800, 0x4),
> /* Port I/O: 1880-189f : ACPI GPE0_BLK */
> PIO_RANGE(0x1880, 0x20),
> /* Port I/O: f000-f03f : 0000:00:02.0 */
> PIO_RANGE(0xf000, 0x40),
> /* Port I/O: f040-f05f : 0000:00:1f.4 */
> PIO_RANGE(0xf040, 0x20),
> /* Port I/O: f060-f07f : 0000:00:17.0 */
> PIO_RANGE(0xf060, 0x20),
> /* Port I/O: f080-f083 : 0000:00:17.0 */
> PIO_RANGE(0xf080, 0x4),
> /* Port I/O: f090-f097 : 0000:00:17.0 */
> PIO_RANGE(0xf090, 0x8),
> },
>
> .pci_devices = {
> /* PCIDevice: 00:00.0 */
> {
> .type = JAILHOUSE_PCI_TYPE_DEVICE,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0x0,
> .bar_mask = {
> 0x00000000, 0x00000000, 0x00000000,
> 0x00000000, 0x00000000, 0x00000000,
> },
> .caps_start = 0,
> .num_caps = 1,
> .num_msi_vectors = 0,
> .msi_64bits = 0,
> .msi_maskable = 0,
> .num_msix_vectors = 0,
> .msix_region_size = 0x0,
> .msix_address = 0x0,
> },
> /* PCIDevice: 00:02.0 */
> {
> .type = JAILHOUSE_PCI_TYPE_DEVICE,
> .iommu = 0,
> .domain = 0x0,
> .bdf = 0x10,
> .bar_mask = {
> 0xff000000, 0xffffffff, 0xf0000000,
> 0xffffffff, 0xffffffc0, 0x00000000,
> },
> .caps_start = 1,
> .num_caps = 7,
> .num_msi_vectors = 1,
> .msi_64bits = 0,
> .msi_maskable = 0,
> .num_msix_vectors = 0,
> .msix_region_size = 0x0,
> .msix_address = 0x0,
> },
> /* PCIDevice: 00:08.0 */
> {
> .type = JAILHOUSE_PCI_TYPE_DEVICE,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0x40,
> .bar_mask = {
> 0xfffff000, 0xffffffff, 0x00000000,
> 0x00000000, 0x00000000, 0x00000000,
> },
> .caps_start = 8,
> .num_caps = 3,
> .num_msi_vectors = 1,
> .msi_64bits = 0,
> .msi_maskable = 0,
> .num_msix_vectors = 0,
> .msix_region_size = 0x0,
> .msix_address = 0x0,
> },
> /* PCIDevice: 00:14.0 */
> {
> .type = JAILHOUSE_PCI_TYPE_DEVICE,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0xa0,
> .bar_mask = {
> 0xffff0000, 0xffffffff, 0x00000000,
> 0x00000000, 0x00000000, 0x00000000,
> },
> .caps_start = 11,
> .num_caps = 2,
> .num_msi_vectors = 8,
> .msi_64bits = 1,
> .msi_maskable = 0,
> .num_msix_vectors = 0,
> .msix_region_size = 0x0,
> .msix_address = 0x0,
> },
> /* PCIDevice: 00:14.2 */
> {
> .type = JAILHOUSE_PCI_TYPE_DEVICE,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0xa2,
> .bar_mask = {
> 0xfffff000, 0xffffffff, 0x00000000,
> 0x00000000, 0x00000000, 0x00000000,
> },
> .caps_start = 13,
> .num_caps = 2,
> .num_msi_vectors = 1,
> .msi_64bits = 0,
> .msi_maskable = 0,
> .num_msix_vectors = 0,
> .msix_region_size = 0x0,
> .msix_address = 0x0,
> },
> /* PCIDevice: 00:17.0 */
> {
> .type = JAILHOUSE_PCI_TYPE_DEVICE,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0xb8,
> .bar_mask = {
> 0xffffe000, 0xffffff00, 0xfffffff8,
> 0xfffffffc, 0xffffffe0, 0xfffff800,
> },
> .caps_start = 15,
> .num_caps = 3,
> .num_msi_vectors = 1,
> .msi_64bits = 0,
> .msi_maskable = 0,
> .num_msix_vectors = 0,
> .msix_region_size = 0x0,
> .msix_address = 0x0,
> },
> /* PCIDevice: 00:1c.0 */
> {
> .type = JAILHOUSE_PCI_TYPE_BRIDGE,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0xe0,
> .bar_mask = {
> 0x00000000, 0x00000000, 0x00000000,
> 0x00000000, 0x00000000, 0x00000000,
> },
> .caps_start = 18,
> .num_caps = 5,
> .num_msi_vectors = 1,
> .msi_64bits = 0,
> .msi_maskable = 0,
> .num_msix_vectors = 0,
> .msix_region_size = 0x0,
> .msix_address = 0x0,
> },
> /* PCIDevice: 00:1c.6 */
> {
> .type = JAILHOUSE_PCI_TYPE_BRIDGE,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0xe6,
> .bar_mask = {
> 0x00000000, 0x00000000, 0x00000000,
> 0x00000000, 0x00000000, 0x00000000,
> },
> .caps_start = 23,
> .num_caps = 7,
> .num_msi_vectors = 1,
> .msi_64bits = 0,
> .msi_maskable = 0,
> .num_msix_vectors = 0,
> .msix_region_size = 0x0,
> .msix_address = 0x0,
> },
> /* PCIDevice: 00:1f.0 */
> {
> .type = JAILHOUSE_PCI_TYPE_DEVICE,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0xf8,
> .bar_mask = {
> 0x00000000, 0x00000000, 0x00000000,
> 0x00000000, 0x00000000, 0x00000000,
> },
> .caps_start = 0,
> .num_caps = 0,
> .num_msi_vectors = 0,
> .msi_64bits = 0,
> .msi_maskable = 0,
> .num_msix_vectors = 0,
> .msix_region_size = 0x0,
> .msix_address = 0x0,
> },
> /* PCIDevice: 00:1f.2 */
> {
> .type = JAILHOUSE_PCI_TYPE_DEVICE,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0xfa,
> .bar_mask = {
> 0xffffc000, 0x00000000, 0x00000000,
> 0x00000000, 0x00000000, 0x00000000,
> },
> .caps_start = 0,
> .num_caps = 0,
> .num_msi_vectors = 0,
> .msi_64bits = 0,
> .msi_maskable = 0,
> .num_msix_vectors = 0,
> .msix_region_size = 0x0,
> .msix_address = 0x0,
> },
> /* PCIDevice: 00:1f.3 */
> {
> .type = JAILHOUSE_PCI_TYPE_DEVICE,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0xfb,
> .bar_mask = {
> 0xffffc000, 0xffffffff, 0x00000000,
> 0x00000000, 0xffff0000, 0xffffffff,
> },
> .caps_start = 30,
> .num_caps = 2,
> .num_msi_vectors = 1,
> .msi_64bits = 1,
> .msi_maskable = 0,
> .num_msix_vectors = 0,
> .msix_region_size = 0x0,
> .msix_address = 0x0,
> },
> /* PCIDevice: 00:1f.4 */
> {
> .type = JAILHOUSE_PCI_TYPE_DEVICE,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0xfc,
> .bar_mask = {
> 0xffffff00, 0xffffffff, 0x00000000,
> 0x00000000, 0xffffffe0, 0x00000000,
> },
> .caps_start = 0,
> .num_caps = 0,
> .num_msi_vectors = 0,
> .msi_64bits = 0,
> .msi_maskable = 0,
> .num_msix_vectors = 0,
> .msix_region_size = 0x0,
> .msix_address = 0x0,
> },
> /* PCIDevice: 00:1f.6 */
> {
> .type = JAILHOUSE_PCI_TYPE_DEVICE,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0xfe,
> .bar_mask = {
> 0xfffe0000, 0x00000000, 0x00000000,
> 0x00000000, 0x00000000, 0x00000000,
> },
> .caps_start = 32,
> .num_caps = 3,
> .num_msi_vectors = 1,
> .msi_64bits = 1,
> .msi_maskable = 0,
> .num_msix_vectors = 0,
> .msix_region_size = 0x0,
> .msix_address = 0x0,
> },
> #if defined(USE_IVSHMEM_NET)
> { /* IVSHMEM (networking) */
> .type = JAILHOUSE_PCI_TYPE_IVSHMEM,
> .iommu = 1,
> .domain = 0x0000,
> .bdf = 0x0c << 3,
> .bar_mask = JAILHOUSE_IVSHMEM_BAR_MASK_MSIX,
> .num_msix_vectors = 2,
> .shmem_regions_start = 0,
> .shmem_dev_id = 0,
> .shmem_peers = 2,
> .shmem_protocol = JAILHOUSE_SHMEM_PROTO_VETH,
> },
> #endif /* USE_IVSHMEM_NET */
>
> #if defined(USE_IVSHMEM_DEMO)
> { /* IVSHMEM (demo) */
> .type = JAILHOUSE_PCI_TYPE_IVSHMEM,
> .iommu = 1,
> .domain = 0x0000,
> .bdf = 0x0d << 3,
> .bar_mask = JAILHOUSE_IVSHMEM_BAR_MASK_MSIX,
> .num_msix_vectors = 16,
> .shmem_regions_start = NET_MEM,
> .shmem_dev_id = 0,
> .shmem_peers = 3,
> .shmem_protocol = JAILHOUSE_SHMEM_PROTO_UNDEFINED,
> },
> #endif /* USE_IVSHMEM_DEMO */
>
> #if defined(USE_VIRTIO_CON)
> { /* IVSHMEM (virtio-con back-end) */
> .type = JAILHOUSE_PCI_TYPE_IVSHMEM,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0x0e << 3,
> .bar_mask = JAILHOUSE_IVSHMEM_BAR_MASK_MSIX,
> .num_msix_vectors = 3,
> .shmem_regions_start = NET_MEM + DEMO_MEM,
> .shmem_dev_id = 0,
> .shmem_peers = 2,
> .shmem_protocol = JAILHOUSE_SHMEM_PROTO_VIRTIO_BACK +
> VIRTIO_DEV_CONSOLE,
> },
> #endif /* USE_VIRTIO_CON */
>
> #if defined(USE_VIRTIO_BLK)
> { /* IVSHMEM (virtio-blk back-end) */
> .type = JAILHOUSE_PCI_TYPE_IVSHMEM,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0x0f << 3,
> .bar_mask = JAILHOUSE_IVSHMEM_BAR_MASK_MSIX,
> .num_msix_vectors = 2,
> .shmem_regions_start = NET_MEM + DEMO_MEM + CON_MEM,
> .shmem_dev_id = 0,
> .shmem_peers = 2,
> .shmem_protocol = JAILHOUSE_SHMEM_PROTO_VIRTIO_BACK +
> VIRTIO_DEV_BLOCK,
> },
> #endif /* USE_VIRTIO_BLK*/
> },
>
> .pci_caps = {
> /* PCIDevice: 00:00.0 */
> {
> .id = PCI_CAP_ID_VNDR,
> .start = 0xe0,
> .len = 0x2,
> .flags = 0,
> },
> /* PCIDevice: 00:02.0 */
> {
> .id = PCI_CAP_ID_VNDR,
> .start = 0x40,
> .len = 0x2,
> .flags = 0,
> },
> {
> .id = PCI_CAP_ID_EXP,
> .start = 0x70,
> .len = 0x3c,
> .flags = 0,
> },
> {
> .id = PCI_CAP_ID_MSI,
> .start = 0xac,
> .len = 0xa,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_CAP_ID_PM,
> .start = 0xd0,
> .len = 0x8,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_EXT_CAP_ID_PASID | JAILHOUSE_PCI_EXT_CAP,
> .start = 0x100,
> .len = 0x8,
> .flags = 0,
> },
> {
> .id = PCI_EXT_CAP_ID_ATS | JAILHOUSE_PCI_EXT_CAP,
> .start = 0x200,
> .len = 0x4,
> .flags = 0,
> },
> {
> .id = PCI_EXT_CAP_ID_PRI | JAILHOUSE_PCI_EXT_CAP,
> .start = 0x300,
> .len = 0x4,
> .flags = 0,
> },
> /* PCIDevice: 00:08.0 */
> {
> .id = PCI_CAP_ID_MSI,
> .start = 0x90,
> .len = 0xa,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_CAP_ID_PM,
> .start = 0xdc,
> .len = 0x8,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_CAP_ID_AF,
> .start = 0xf0,
> .len = 0x2,
> .flags = 0,
> },
> /* PCIDevice: 00:14.0 */
> {
> .id = PCI_CAP_ID_PM,
> .start = 0x70,
> .len = 0x8,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_CAP_ID_MSI,
> .start = 0x80,
> .len = 0xe,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> /* PCIDevice: 00:14.2 */
> {
> .id = PCI_CAP_ID_PM,
> .start = 0x50,
> .len = 0x8,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_CAP_ID_MSI,
> .start = 0x80,
> .len = 0xa,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> /* PCIDevice: 00:17.0 */
> {
> .id = PCI_CAP_ID_MSI,
> .start = 0x80,
> .len = 0xa,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_CAP_ID_PM,
> .start = 0x70,
> .len = 0x8,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_CAP_ID_SATA,
> .start = 0xa8,
> .len = 0x2,
> .flags = 0,
> },
> /* PCIDevice: 00:1c.0 */
> {
> .id = PCI_CAP_ID_EXP,
> .start = 0x40,
> .len = 0x3c,
> .flags = 0,
> },
> {
> .id = PCI_CAP_ID_MSI,
> .start = 0x80,
> .len = 0xa,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_CAP_ID_SSVID,
> .start = 0x90,
> .len = 0x2,
> .flags = 0,
> },
> {
> .id = PCI_CAP_ID_PM,
> .start = 0xa0,
> .len = 0x8,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = 0x0 | JAILHOUSE_PCI_EXT_CAP,
> .start = 0x100,
> .len = 0x4,
> .flags = 0,
> },
> /* PCIDevice: 00:1c.6 */
> {
> .id = PCI_CAP_ID_EXP,
> .start = 0x40,
> .len = 0x3c,
> .flags = 0,
> },
> {
> .id = PCI_CAP_ID_MSI,
> .start = 0x80,
> .len = 0xa,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_CAP_ID_SSVID,
> .start = 0x90,
> .len = 0x2,
> .flags = 0,
> },
> {
> .id = PCI_CAP_ID_PM,
> .start = 0xa0,
> .len = 0x8,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_EXT_CAP_ID_ERR | JAILHOUSE_PCI_EXT_CAP,
> .start = 0x100,
> .len = 0x40,
> .flags = 0,
> },
> {
> .id = PCI_EXT_CAP_ID_ACS | JAILHOUSE_PCI_EXT_CAP,
> .start = 0x140,
> .len = 0x8,
> .flags = 0,
> },
> {
> .id = PCI_EXT_CAP_ID_SECPCI | JAILHOUSE_PCI_EXT_CAP,
> .start = 0x220,
> .len = 0x10,
> .flags = 0,
> },
> /* PCIDevice: 00:1f.3 */
> {
> .id = PCI_CAP_ID_PM,
> .start = 0x50,
> .len = 0x8,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_CAP_ID_MSI,
> .start = 0x60,
> .len = 0xe,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> /* PCIDevice: 00:1f.6 */
> {
> .id = PCI_CAP_ID_PM,
> .start = 0xc8,
> .len = 0x8,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_CAP_ID_MSI,
> .start = 0xd0,
> .len = 0xe,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_CAP_ID_AF,
> .start = 0xe0,
> .len = 0x2,
> .flags = 0,
> },
> },
> };


My inmate:
> #define USE_VIRTIO_BLK
>
> #ifndef USE_IVSHMEM_NET
> # define NET_MEM 0
> # define NET_PCI 0
> #else
> # define NET_MEM 4
> # define NET_PCI 1
> #endif
>
> #ifndef USE_IVSHMEM_DEMO
> # define DEMO_MEM 0
> # define DEMO_PCI 0
> #else
> # define DEMO_MEM 5
> # define DEMO_PCI 1
> #endif
>
> #ifndef USE_VIRTIO_CON
> # define CON_MEM 0
> # define CON_PCI 0
> #else
> # define CON_MEM 4
> # define CON_PCI 1
> #endif
>
> #ifndef USE_VIRTIO_BLK
> # define BLK_MEM 0
> # define BLK_PCI 0
> #else
> # define BLK_MEM 4
> # define BLK_PCI 1
> #endif
>
> #define INTERCELL_MEM (NET_MEM + DEMO_MEM + CON_MEM + BLK_MEM)
> #define INTERCELL_PCI (NET_PCI + DEMO_PCI + CON_PCI + BLK_PCI)
>
> #include <jailhouse/types.h>
> #include <jailhouse/cell-config.h>
>
> struct {
> struct jailhouse_cell_desc cell;
> __u64 cpus[1];
> struct jailhouse_memory mem_regions[5 + INTERCELL_MEM];
> struct jailhouse_cache cache_regions[1];
> struct jailhouse_irqchip irqchips[1];
> struct jailhouse_pio pio_regions[4];
> struct jailhouse_pci_device pci_devices[1 + INTERCELL_PCI];
> struct jailhouse_pci_capability pci_caps[13];
>
> } __attribute__((packed)) config = {
> .cell = {
> .signature = JAILHOUSE_CELL_DESC_SIGNATURE,
> .revision = JAILHOUSE_CONFIG_REVISION,
> .name = "my-inmate",
> .flags = JAILHOUSE_CELL_PASSIVE_COMMREG |
> JAILHOUSE_CELL_VIRTUAL_CONSOLE_PERMITTED,
>
> .cpu_set_size = sizeof(config.cpus),
> .num_memory_regions = ARRAY_SIZE(config.mem_regions),
> .num_cache_regions = ARRAY_SIZE(config.cache_regions),
> .num_irqchips = ARRAY_SIZE(config.irqchips),
> .num_pio_regions = ARRAY_SIZE(config.pio_regions),
> .num_pci_devices = ARRAY_SIZE(config.pci_devices),
> .num_pci_caps = ARRAY_SIZE(config.pci_caps),
> },
>
> /* CPU 4 */
> .cpus = {
> 0x8,
> },
>
> .mem_regions = {
> #if defined(USE_IVSHMEM_NET)
> /* IVSHMEM shared memory regions (networking) */
> JAILHOUSE_SHMEM_NET_REGIONS(0x16f000000, 1),
> #endif /* USE_IVSHMEM_NET */
>
> #if defined(USE_IVSHMEM_DEMO)
> /* IVSHMEM shared memory regions (demo) */
> {
> .phys_start = 0x16f100000,
> .virt_start = 0x16f100000,
> .size = 0x1000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_ROOTSHARED,
> },
> {
> .phys_start = 0x16f101000,
> .virt_start = 0x16f101000,
> .size = 0x9000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_ROOTSHARED,
> },
> {
> .phys_start = 0x16f10a000,
> .virt_start = 0x16f10a000,
> .size = 0x2000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_ROOTSHARED,
> },
> {
> .phys_start = 0x16f10c000,
> .virt_start = 0x16f10c000,
> .size = 0x2000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_ROOTSHARED,
> },
> {
> .phys_start = 0x16f10e000,
> .virt_start = 0x16f10e000,
> .size = 0x2000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_ROOTSHARED,
> },
> #endif /* USE_IVSHMEM_DEMO */
>
> #if defined(USE_VIRTIO_CON)
> /* IVSHMEM shared memory region (virtio-con front) */
> {
> .phys_start = 0x16f200000,
> .virt_start = 0x16f200000,
> .size = 0x1000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_ROOTSHARED,
> },
> {
> .phys_start = 0x16f201000,
> .virt_start = 0x16f201000,
> .size = 0xf000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_ROOTSHARED,
> },
> { 0 },
> { 0 },
> #endif /* USE_VIRTIO_CON */
>
> #if defined(USE_VIRTIO_BLK)
> /* IVSHMEM shared memory region (virtio-blk front) */
> {
> .phys_start = 0x16f300000,
> .virt_start = 0x16f300000,
> .size = 0x1000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_ROOTSHARED,
> },
> {
> .phys_start = 0x16f301000,
> .virt_start = 0x16f301000,
> .size = 0xdf000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_ROOTSHARED,
> },
> { 0 },
> { 0 },
> #endif /* USE_VIRTIO_BLK) */
>
> /* low RAM: 1 MiB */ {
> .phys_start = 0x100700000,
> .virt_start = 0,
> .size = 0x00100000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_EXECUTE | JAILHOUSE_MEM_DMA |
> JAILHOUSE_MEM_LOADABLE,
> },
> /* communication region */
> {
> .virt_start = 0x00100000,
> .size = 0x00001000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_COMM_REGION,
> },
> /* high RAM: 249 MiB */ {
> .phys_start = 0x100800000,
> .virt_start = 0x00200000,
> .size = 0xf900000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
> JAILHOUSE_MEM_EXECUTE | JAILHOUSE_MEM_DMA |
> JAILHOUSE_MEM_LOADABLE,
> },
> /* MemRegion: df000000-df01ffff : 02:00.0/igb */
> {
> .phys_start = 0xdf000000,
> .virt_start = 0xdf000000,
> .size = 0x20000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> /* MemRegion: df021000-df023fff : 02:00.0/igb */
> {
> .phys_start = 0xdf021000,
> .virt_start = 0xdf021000,
> .size = 0x3000,
> .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
> },
> },
>
> .cache_regions = {
> {
> .start = 0,
> .size = 2,
> .type = JAILHOUSE_CACHE_L3,
> },
> },
>
> .irqchips = {
> /* IOAPIC 2, GSI base 0 */
> {
> .address = 0xfec00000,
> .id = 0x1f0f8,
> .pin_bitmap = {
> (1 << 4),
> },
> },
> },
>
> .pio_regions = {
> PIO_RANGE(0x4e, 4),
> PIO_RANGE(0x2e, 4),
> PIO_RANGE(0x110, 8),
> /* Serial device: ttyS0 with IRQ 4 (see above) */
> PIO_RANGE(0x3f8, 8),
> },
>
> .pci_devices = {
> #if defined(USE_IVSHMEM_NET)
> {
> .type = JAILHOUSE_PCI_TYPE_IVSHMEM,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0x0c << 3,
> .bar_mask = JAILHOUSE_IVSHMEM_BAR_MASK_MSIX,
> .num_msix_vectors = 2,
> .shmem_regions_start = 0,
> .shmem_dev_id = 1,
> .shmem_peers = 2,
> .shmem_protocol = JAILHOUSE_SHMEM_PROTO_VETH,
> },
> #endif /* USE_IVSHMEM_NET */
>
> #if defined(USE_IVSHMEM_DEMO)
> {
> .type = JAILHOUSE_PCI_TYPE_IVSHMEM,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0x0d << 3,
> .bar_mask = JAILHOUSE_IVSHMEM_BAR_MASK_MSIX,
> .num_msix_vectors = 16,
> .shmem_regions_start = NET_MEM,
> .shmem_dev_id = 2,
> .shmem_peers = 3,
> .shmem_protocol = JAILHOUSE_SHMEM_PROTO_UNDEFINED,
> },
> #endif /* USE_IVSHMEM_DEMO */
>
> #if defined(USE_VIRTIO_CON)
> {
> .type = JAILHOUSE_PCI_TYPE_IVSHMEM,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0x0e << 3,
> .bar_mask = JAILHOUSE_IVSHMEM_BAR_MASK_MSIX,
> .num_msix_vectors = 3,
> .shmem_regions_start = NET_MEM + DEMO_MEM,
> .shmem_dev_id = 1,
> .shmem_peers = 2,
> .shmem_protocol = JAILHOUSE_SHMEM_PROTO_VIRTIO_FRONT +
> VIRTIO_DEV_CONSOLE,
> },
> #endif /* USE_VIRTIO_CON */
>
> #if defined(USE_VIRTIO_BLK)
> {
> .type = JAILHOUSE_PCI_TYPE_IVSHMEM,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0x0f << 3,
> .bar_mask = JAILHOUSE_IVSHMEM_BAR_MASK_MSIX,
> .num_msix_vectors = 2,
> .shmem_regions_start = NET_MEM + DEMO_MEM + CON_MEM,
> .shmem_dev_id = 1,
> .shmem_peers = 2,
> .shmem_protocol = JAILHOUSE_SHMEM_PROTO_VIRTIO_FRONT +
> VIRTIO_DEV_BLOCK,
> },
> #endif /* USE_VIRTIO_BLK */
>
> /* PCIDevice: igb/02:00.0 */
> {
> .type = JAILHOUSE_PCI_TYPE_DEVICE,
> .iommu = 1,
> .domain = 0x0,
> .bdf = 0x200,
> .bar_mask = {
> 0xfffe0000, 0x00000000, 0x00000000,
> 0xffffc000, 0x00000000, 0x00000000,
> },
> .caps_start = 0,
> .num_caps = 6,
> .num_msi_vectors = 1,
> .msi_64bits = 1,
> .msi_maskable = 1,
> .num_msix_vectors = 5,
> .msix_region_size = 0x1000,
> .msix_address = 0xdf020000,
> },
> },
>
> .pci_caps = {
> /* PCIDevice: igb/02:00.0 */
> {
> .id = PCI_CAP_ID_PM,
> .start = 0x40,
> .len = 0x8,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_CAP_ID_MSI,
> .start = 0x50,
> .len = 0x18,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_CAP_ID_MSIX,
> .start = 0x70,
> .len = 0xc,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_CAP_ID_EXP,
> .start = 0xa0,
> .len = 0x3c,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_EXT_CAP_ID_ERR | JAILHOUSE_PCI_EXT_CAP,
> .start = 0x100,
> .len = 0x40,
> .flags = JAILHOUSE_PCICAPS_WRITE,
> },
> {
> .id = PCI_EXT_CAP_ID_TPH | JAILHOUSE_PCI_EXT_CAP,
> .start = 0x1a0,
> .len = 0x4,
> .flags = 0,
> },
> },
> };

[Jan Kiszka]

On 10.03.20 15:41, Philipp Rosenberger wrote:
> Hi,
>
> I have managed to get virtio-ivshmem console and block running. But I
> observed a strange behavior. I do the following:
>
> 1. Boot up the board.
> 2. Enable the rootcell.
> 3. echo "110a 4106 110a 4106 ffc002 ffffff" > \
> /sys/bus/pci/drivers/uio_ivshmem/new_id
> 4. virtio-ivshmem-block /dev/uio0 /path/to/disk.image
> 5. boot linux-inmate
> 6. virtio-ivshmem 0000:00:0f.0: backend not ready
> 7. kernel panic
>
> If I redo the sets 4 and 5 the inmates starts as expected and I can
> access the disk.image via /dev/vda.
>
> I found, the the virtio-ivshmem-block tool waits for an interrupt if
> 'state[peer_id] != VIRTIO_STATE_RESET'. But there is no interrupt.

The state memory should be zeroed, provided the peer is not running. You
will only get an interrupt during the peer setup when it switches it
state from (expected) RESET to READY. Maybe we miss some proper
initialization of the shared state memory in Jailhouse.

Can you confirm that the state memory is in a random state on first
startup? And that it changes as expected for the peer to READY once the
non-root Linux boots?

Jan

--
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux

[Jan Kiszka]

Does this help?

diff --git a/hypervisor/ivshmem.c b/hypervisor/ivshmem.c
index 22283a85..a5cdcf93 100644
--- a/hypervisor/ivshmem.c
+++ b/hypervisor/ivshmem.c
@@ -101,13 +101,8 @@ static void ivshmem_trigger_interrupt(struct ivshmem_endpoint *ive,
spin_unlock(&ive->irq_lock);
}

-static void ivshmem_write_state(struct ivshmem_endpoint *ive, u32 new_state)
+static u32 *ivshmem_map_state_table(struct ivshmem_endpoint *ive)
{
- const struct jailhouse_pci_device *dev_info = ive->device->info;
- u32 *state_table = (u32 *)TEMPORARY_MAPPING_BASE;
- struct ivshmem_endpoint *target_ive;
- unsigned int id;
-
/*
* Cannot fail: upper levels of page table were already created by
* paging_init, and we always map single pages, thus only update the
@@ -115,9 +110,20 @@ static void ivshmem_write_state(struct ivshmem_endpoint *ive, u32 new_state)
*/
paging_create(&this_cpu_data()->pg_structs,
ive->shmem[0].phys_start, PAGE_SIZE,
- (unsigned long)state_table, PAGE_DEFAULT_FLAGS,
+ TEMPORARY_MAPPING_BASE, PAGE_DEFAULT_FLAGS,
PAGING_NON_COHERENT | PAGING_NO_HUGE);

+ return (u32 *)TEMPORARY_MAPPING_BASE;
+}
+
+
+static void ivshmem_write_state(struct ivshmem_endpoint *ive, u32 new_state)
+{
+ const struct jailhouse_pci_device *dev_info = ive->device->info;
+ u32 *state_table = ivshmem_map_state_table(ive);
+ struct ivshmem_endpoint *target_ive;
+ unsigned int id;
+
state_table[dev_info->shmem_dev_id] = new_state;
memory_barrier();

@@ -435,12 +441,15 @@ int ivshmem_init(struct cell *cell, struct pci_device *device)
}

link->peers++;
- ive = &link->eps[dev_info->shmem_dev_id];
+ ive = &link->eps[id];

ive->device = device;
ive->link = link;
ive->shmem = jailhouse_cell_mem_regions(cell->config) +
dev_info->shmem_regions_start;
+ if (link->peers == 1)
+ memset(ivshmem_map_state_table(ive), 0,
+ dev_info->shmem_peers * sizeof(u32));
device->ivshmem_endpoint = ive;

device->cell = cell;

[Philipp Rosenberger]

Hi Jan,

On 10.03.20 16:08, Jan Kiszka wrote:

> On 10.03.20 15:41, Philipp Rosenberger wrote:
>> Hi,
>>
>> I have managed to get virtio-ivshmem console and block running. But I
>> observed a strange behavior. I do the following:
>>
>> 1. Boot up the board.

devmemdump 0x16f300000 0x1000 | hexdump -C | head
00000000 07 26 8b 17 4d 0f 2e 83 06 26 8a 17 4c 0f 2f a3
00000010 16 ae 9a 97 5c 8f 3f 03 36 97 bc 97 1c 8e 3f 82

>> 2. Enable the rootcell.

devmemdump 0x16f300000 0x1000 | hexdump -C | head -n 2
00000000 00 00 00 00 4d 0f 2e 83 06 26 8a 17 4c 0f 2f a3
00000010 16 ae 9a 97 5c 8f 3f 03 36 97 bc 97 1c 8e 3f 82

>> 3. echo "110a 4106 110a 4106 ffc002 ffffff" > \
>>    /sys/bus/pci/drivers/uio_ivshmem/new_id
>> 4. virtio-ivshmem-block /dev/uio0 /path/to/disk.image

devmemdump 0x16f300000 0x1000 | hexdump -C | head -n 2
00000000 00 00 00 00 4d 0f 2e 83 06 26 8a 17 4c 0f 2f a3
00000010 16 ae 9a 97 5c 8f 3f 03 36 97 bc 97 1c 8e 3f 82

>> 5. boot linux-inmate
>> 6. virtio-ivshmem 0000:00:0f.0: backend not ready
>> 7. kernel panic

devmemdump 0x16f300000 0x1000 | hexdump -C | head -n 2
00000000 00 00 00 00 00 00 00 00 06 26 8a 17 4c 0f 2f a3
00000010 16 ae 9a 97 5c 8f 3f 03 36 97 bc 97 1c 8e 3f 82

>> If I redo the sets 4 and 5 the inmates starts as expected and I can
>> access the disk.image via /dev/vda.

repeated step 4:
devmemdump 0x16f300000 0x1000 | hexdump -C | head -n 2
00000000 01 00 00 00 00 00 00 00 06 26 8a 17 4c 0f 2f a3
00000010 16 ae 9a 97 5c 8f 3f 03 36 97 bc 97 1c 8e 3f 82

repeated step 5:
devmemdump 0x16f300000 0x1000 | hexdump -C | head -n 2
00000000 01 00 00 00 01 00 00 00 06 26 8a 17 4c 0f 2f a3
00000010 16 ae 9a 97 5c 8f 3f 03 36 97 bc 97 1c 8e 3f 82

>> I found, the the virtio-ivshmem-block tool waits for an interrupt if
>> 'state[peer_id] != VIRTIO_STATE_RESET'. But there is no interrupt.
>
> The state memory should be zeroed, provided the peer is not running. You
> will only get an interrupt during the peer setup when it switches it
> state from (expected) RESET to READY. Maybe we miss some proper
> initialization of the shared state memory in Jailhouse.
>
> Can you confirm that the state memory is in a random state on first
> startup? And that it changes as expected for the peer to READY once the
> non-root Linux boots?

I hope I have done this right. The 'state' is stored in the first
region. In my case '.phys_start = 0x16f300000'.

You can see how the first two 32 bit words change. The devmemdump tool
just maps /dev/mem at the given address and writes it to stdout.

I've tested the following after step 1 and it also worked:
# devmemdump 0x16f300000 0x1000 | hexdump -C | head -n 2
00000000 07 26 8b 17 4d 0f 2e 83 06 26 8a 17 4c 0f 2f 83
00000010 16 a6 9a 97 5c 8f 3f 03 16 a7 9a 96 5c 8e 3f 02
# busybox devmem 0x16f300000 32 0
# busybox devmem 0x16f300004 32 0
# devmemdump 0x16f300000 0x1000 | hexdump -C | head -n 2
00000000 00 00 00 00 00 00 00 00 06 26 8a 17 4c 0f 2f 83
00000010 16 a6 9a 97 5c 8f 3f 03 16 a7 9a 96 5c 8e 3f 02

Best regards,
Philipp

> Jan
>

[Philipp Rosenberger]

Yeah, this works nicely.
Thank you!

Best regards,
Philipp

Linutronix GmbH | Bahnhofstrasse 3 | D-88690 Uhldingen-Mühlhofen
Phone: +49 7556 25 999 0; Fax.: +49 7556 25 999 99

Hinweise zum Datenschutz finden Sie hier (Informations on data privacy
can be found here):
https://linutronix.de/kontakt/Datenschutz.php

Linutronix GmbH
Firmensitz (Registered Office): Uhldingen-Mühlhofen
Registergericht (Registration Court): Amtsgericht Freiburg i.Br., HRB700 806
Geschäftsführer (Managing Directors): Heinz Egger, Thomas Gleixner

[Jan Kiszka]

Yes, your logs look consistent with my theory, and my patch should fix it.

I suppose I tested too often in friendly virtual environments...

Thanks for reporting!