[PATCH 0/4] AMD IVRS parser and related

Valentine Sinitsyn

unread,

Feb 25, 2015, 2:25:27 PM2/25/15

to jailho...@googlegroups.com, Valentine Sinitsyn

Hi all,

This commit introduces AMD IVRS parser and related configuration file changes.
All of these is required for AMD IOMMU support.

parse_ivrs() is somewhat minimal implementation, so everyone with AMD hardware
is encouraged to try it and report features that are missing.

Don't expect AMD IOMMU to work though, this is only a preparatory step.
You can watch (slow) AMD IOMMU support progress at [1].

Regards,
Valentine

1. https://github.com/vsinitsyn/jailhouse/tree/amd-vi

Valentine Sinitsyn (4):
tools: Prepare for AMD IOMMU support
core: Introduce AMD IOMMU config data structures
tools: Implement ACPI IVRS table parser
configs: Add IOMMU to F2A88XM-HD3 config

configs/f2a88xm-hd3.c | 35 +++--
hypervisor/include/jailhouse/cell-config.h | 6 +-
tools/jailhouse-config-create | 208 +++++++++++++++++++++++++----
tools/root-cell-config.c.tmpl | 9 ++
4 files changed, 223 insertions(+), 35 deletions(-)

--
2.3.0

Valentine Sinitsyn

unread,

Feb 25, 2015, 2:25:28 PM2/25/15

to jailho...@googlegroups.com, Valentine Sinitsyn

VT-d and AMD-Vi impose slightly different requirements on PCI devices
configuration (eg PCI root complex). Move sanity checks to parse_dmar()
to account for these discrepancies,

Signed-off-by: Valentine Sinitsyn <valentine...@gmail.com>
---
tools/jailhouse-config-create | 6 ++++++
1 file changed, 6 insertions(+)

diff --git a/tools/jailhouse-config-create b/tools/jailhouse-config-create
index c3eebbe..c6abe63 100755
--- a/tools/jailhouse-config-create
+++ b/tools/jailhouse-config-create
@@ -702,6 +702,12 @@ def parse_dmar(pcidevices, ioapics):
f.seek(struct_len - offset, os.SEEK_CUR)

f.close()
+
+ for d in pcidevices:
+ if d.iommu is None:
+ raise RuntimeError('PCI device %02x:%02x.%x outside the scope of an '
+ 'IOMMU' % (d.bus, d.dev, d.fn))
+
return units, regions

--
2.3.0

Valentine Sinitsyn

unread,

Feb 25, 2015, 2:25:30 PM2/25/15

to jailho...@googlegroups.com, Valentine Sinitsyn

Adapt platform_info to store either VT-d (dmar_units_base) or AMD IOMMU
(currently, iommu_base) data.

Signed-off-by: Valentine Sinitsyn <valentine...@gmail.com>
---

It is likely more AMD-specific flags will be needed for IOMMU support,
that's why we introduce a union and not simply rename dmar_unit_base.

hypervisor/include/jailhouse/cell-config.h | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/hypervisor/include/jailhouse/cell-config.h b/hypervisor/include/jailhouse/cell-config.h
index e533127..d9f0606 100644
--- a/hypervisor/include/jailhouse/cell-config.h
+++ b/hypervisor/include/jailhouse/cell-config.h
@@ -90,6 +90,7 @@ struct jailhouse_pci_capability {
} __attribute__((packed));

#define JAILHOUSE_MAX_DMAR_UNITS 8
+#define JAILHOUSE_MAX_IOMMU_UNITS 1

struct jailhouse_system {
struct jailhouse_memory hypervisor_memory;
@@ -100,7 +101,10 @@ struct jailhouse_system {
__u8 mmconfig_end_bus;
__u8 padding[5];
__u16 pm_timer_address;
- __u64 dmar_unit_base[JAILHOUSE_MAX_DMAR_UNITS];
+ union {
+ __u64 dmar_unit_base[JAILHOUSE_MAX_DMAR_UNITS];
+ __u64 iommu_base[JAILHOUSE_MAX_IOMMU_UNITS];
+ };
} __attribute__((packed)) x86;
} __attribute__((packed)) platform_info;
__u32 device_limit;
--
2.3.0

Valentine Sinitsyn

unread,

Feb 25, 2015, 2:25:33 PM2/25/15

to jailho...@googlegroups.com, Valentine Sinitsyn

Add parse_ivrs() function that extracts relevant bits of information
from ACPI IVRS table which describes AMD IOMMU units found on the system.
Configuration file template was also adjusted to accommodate AMD IOMMU data.

Signed-off-by: Valentine Sinitsyn <valentine...@gmail.com>
---

tools/jailhouse-config-create | 202 ++++++++++++++++++++++++++++++++++++------
tools/root-cell-config.c.tmpl | 9 ++
2 files changed, 186 insertions(+), 25 deletions(-)

diff --git a/tools/jailhouse-config-create b/tools/jailhouse-config-create
index c6abe63..d420e71 100755
--- a/tools/jailhouse-config-create
+++ b/tools/jailhouse-config-create
@@ -69,7 +69,7 @@ parser.add_argument('file', metavar='FILE',

options = parser.parse_args()

-inputs = {'files': set(), 'files_opt': set(), 'files_intel': set()}
+inputs = {'files': set(), 'files_opt': set(), 'files_intel': set(), 'files_amd': set()}

# required files
inputs['files'].add('/proc/iomem')
@@ -86,6 +86,7 @@ inputs['files_opt'].add('/sys/class/dmi/id/sys_vendor')
inputs['files_opt'].add('/sys/devices/jailhouse/enabled')
# platform specific files
inputs['files_intel'].add('/sys/firmware/acpi/tables/DMAR')
+inputs['files_amd'].add('/sys/firmware/acpi/tables/IVRS')

def kmg_multiply(value, kmg):
@@ -112,6 +113,8 @@ def check_input_listed(name, optional=False):
global cpuvendor
if cpuvendor == 'GenuineIntel':
set = set.union(inputs['files_intel'])
+ elif cpuvendor == 'AuthenticAMD':
+ set = set.union(inputs['files_amd'])

for file in set:
if fnmatch.fnmatch(name, file):
@@ -306,7 +309,8 @@ class MemRegion:
self.typestr == 'System RAM' or
self.typestr == 'Kernel' or
self.typestr == 'RAM buffer' or
- self.typestr == 'ACPI DMAR RMRR'
+ self.typestr == 'ACPI DMAR RMRR' or
+ self.typestr == 'ACPI IVRS'
):
s = 'JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |\n'
s += p + '\t\tJAILHOUSE_MEM_EXECUTE | JAILHOUSE_MEM_DMA'
@@ -711,6 +715,148 @@ def parse_dmar(pcidevices, ioapics):
return units, regions

+def parse_ivrs(pcidevices, ioapics):
+ def format_bdf(bdf):
+ return '%02x:%02x.%x' % ((bdf >> 8) & 0xff, (bdf >> 3) & 0x1f, bdf & 0x7)
+
+ f = input_open('/sys/firmware/acpi/tables/IVRS', 'rb')
+ signature = f.read(4)
+ if signature != b'IVRS':
+ raise RuntimeError('IVRS: incorrect input file format %s' % signature)
+
+ (length, revision) = struct.unpack('<IB', f.read(5))
+ if revision > 2:
+ raise RuntimeError('IVRS: unsupported Revision %02x' % revision)
+
+ f.seek(48, os.SEEK_SET)
+ length -= 48
+
+ units = []
+ regions = []
+ # BDF of devices that are permitted outside IOMMU: root complex
+ iommu_skiplist = set([0x0])
+ while length > 0:
+ (block_type, block_length) = struct.unpack('<BxH', f.read(4))
+ if block_type in [0x10, 0x11]:
+ # IVHD block
+ (iommu_id, base_addr, pci_seg) = \
+ struct.unpack('<HxxQH', f.read(14))
+ length -= block_length
+ block_length -= 18
+
+ # IOMMU EFR image and reserved area
+ skip_bytes = 6 if block_type == 0x10 else 22
+ f.seek(skip_bytes, os.SEEK_CUR)
+ block_length -= skip_bytes
+
+ if pci_seg != 0:
+ raise RuntimeError('We do not support multiple PCI segments')
+
+ if len(units) > 8:
+ raise RuntimeError('Too many IOMMU units. '
+ 'Raise JAILHOUSE_MAX_IOMMU_UNITS.')
+
+ # We shouldn't map IOMMU to the cells
+ for i, d in enumerate(pcidevices):
+ if d.bdf() == iommu_id:
+ del pcidevices[i]
+
+ units.append(base_addr)
+
+ bdf_start_range = None
+ while block_length > 0:
+ (entry_type, device_id, dte_setting) = struct.unpack('<BHB', f.read(4))
+ block_length -= 4
+
+ if entry_type == 0x01:
+ # All

+ for d in pcidevices:

+ d.iommu = len(units) - 1
+ elif entry_type == 0x02:
+ # Select

+ for d in pcidevices:

+ if d.bdf() == device_id:
+ d.iommu = len(units) - 1
+ elif entry_type == 0x03:
+ # Start of range
+ bdf_start_range = device_id
+ elif entry_type == 0x04:
+ # End of range
+ if bdf_start_range is None:
+ continue

+ for d in pcidevices:

+ if d.bdf() >= bdf_start_range and d.bdf() <= device_id:
+ d.iommu = len(units) - 1
+ bdf_start_range = None
+ elif entry_type == 0x42:
+ # Alias select
+ (device_id_b,) = struct.unpack('<xHx', f.read(4))
+ block_length -= 4

+ for d in pcidevices:

+ if d.bdf() == device_id_b:
+ d.iommu = len(units) - 1
+ elif entry_type == 0x43:
+ # Alias start of range
+ (device_id_b,) = struct.unpack('<xHx', f.read(4))
+ block_length -= 4
+ bdf_start_range = device_id_b
+ elif entry_type == 0x48:
+ # Special device
+ (handle, device_id_b, variety) = struct.unpack('<BHB', f.read(4))
+ block_length -= 4
+ if variety == 0x01: # IOAPIC
+ for chip in ioapics:
+ if chip.id == handle:
+ chip.bdf = device_id
+ chip.iommu = len(units) - 1
+ else:
+ # Reserved or ignored entries
+ if entry_type >= 0x40:
+ f.seek(4, os.SEEK_CUR)
+ block_length -= 4
+
+ elif type in [0x20, 0x21, 0x22]:
+ # IVMD block
+ (block_type, block_flags, block_length, device_id, aux_data, mem_addr, mem_len) = \
+ struct.unpack('<BBHHHxxxxxxxxQQ')
+ length -= block_length
+
+ format_bdf = lambda bdf: '%02x:%02x.%x' % ((bdf >> 8) & 0xff, (bdf >> 3) & 0x1f, bdf & 0x7)
+
+ if int(block_flags):
+ print('WARNING: Jailhouse doesn\'t support configurable (eg. read-only) '
+ 'device memory. Device %s may not work properly, especially '
+ 'in non-root cell.' % format_bdf(device_id))
+
+ if block_type == 0x20:
+ # All devices
+ comment = None
+ elif block_type == 0x21:
+ # Selected device
+ comment = 'PCI Device: %s' % format_bdf(device_id)
+ elif block_type == 0x22:
+ # Device range
+ comment = 'PCI Device: %s - %s' % (format_bdf(device_id), format_bdf(aux_data))
+
+ if comment:
+ print('WARNING: Jailhouse doesn\'t support per-device memory regions.'
+ 'The memory at 0x%x will be mapped accessible to all devices.' % mem_addr)
+
+ regions.append(MemRegion(mem_addr, mem_len, 'ACPI IVRS', comment))
+ elif type == 0x40:
+ raise RuntimeError('You board uses IVRS Rev. 2 feature Jailhouse doesn\'t support yet. '
+ 'Please report this to jailho...@googlegroups.com.')
+ else:
+ print('WARNING: Skipping unknown IVRS block type 0x%02x' % block_type)

+
+ for d in pcidevices:

+ if d.bdf() not in iommu_skiplist and d.iommu is None:

+ raise RuntimeError('PCI device %02x:%02x.%x outside the scope of an '
+ 'IOMMU' % (d.bus, d.dev, d.fn))
+

+ f.close()
+ return units, regions
+
def parse_ioports():
pm_timer_base = None
f = input_open('/proc/ioports')
@@ -803,18 +949,12 @@ mmconfig = MMConfig.parse()

ioapics = parse_madt()

-if get_cpu_vendor() == 'GenuineIntel':
- (dmar_units, rmrr_regs) = parse_dmar(pcidevices, ioapics)
+vendor = get_cpu_vendor()
+if vendor == 'GenuineIntel':
+ (iommu_units, extra_memregs) = parse_dmar(pcidevices, ioapics)
else:
- (dmar_units, rmrr_regs) = [], []
-regions += rmrr_regs
-
-for d in pcidevices:
- if get_cpu_vendor() == 'AuthenticAMD':
- d.iommu = 0 # temporary workaround
- if d.iommu is None:
- raise RuntimeError('PCI device %02x:%02x.%x outside the scope of an '
- 'IOMMU' % (d.bus, d.dev, d.fn))
+ (iommu_units, extra_memregs) = parse_ivrs(pcidevices, ioapics)
+regions += extra_memregs

# kernel does not have memmap region, pick one
if ourmem is None:
@@ -838,17 +978,29 @@ pm_timer_base = parse_ioports()
f = open(options.file, 'w')
tmpl = Template(filename=os.path.join(options.template_dir,
'root-cell-config.c.tmpl'))
-f.write(tmpl.render(regions=regions,
- ourmem=ourmem,
- argstr=' '.join(sys.argv),
- hvmem=hvmem,
- product=product,
- pcidevices=pcidevices,
- pcicaps=pcicaps,
- cpucount=cpucount,
- irqchips=ioapics,
- pm_timer_base=pm_timer_base,
- mmconfig=mmconfig,
- dmar_units=dmar_units))
+kwargs = {
+ 'regions': regions,
+ 'ourmem': ourmem,
+ 'argstr': ' '.join(sys.argv),
+ 'hvmem': hvmem,
+ 'product': product,
+ 'pcidevices': pcidevices,
+ 'pcicaps': pcicaps,
+ 'cpucount': cpucount,
+ 'irqchips': ioapics,
+ 'pm_timer_base': pm_timer_base,
+ 'mmconfig': mmconfig,
+}
+
+if vendor == 'GenuineIntel':
+ kwargs.update({
+ 'dmar_units': iommu_units
+ })
+else:
+ kwargs.update({
+ 'iommu_units': iommu_units
+ })
+
+f.write(tmpl.render(**kwargs))

f.close()
diff --git a/tools/root-cell-config.c.tmpl b/tools/root-cell-config.c.tmpl
index a0ca787..20270fa 100644
--- a/tools/root-cell-config.c.tmpl
+++ b/tools/root-cell-config.c.tmpl
@@ -43,6 +43,13 @@ struct {
% endfor
},
% endif
+ % if iommu_units:
+ .iommu_base = {
+ % for d in iommu_units:
+ ${hex(d)},
+ % endfor
+ },
+ % endif
},
.device_limit = 128,
.interrupt_limit = 256,
@@ -110,7 +117,9 @@ struct {
/* ${str(d)} */
{
.type = ${d.type},
+ % if d.iommu is not None:
.iommu = ${d.iommu},
+ % endif
.domain = ${hex(d.domain)},
.bdf = ${hex(d.bdf())},
.caps_start = ${d.caps_start},
--
2.3.0

Valentine Sinitsyn

unread,

Feb 25, 2015, 2:25:34 PM2/25/15

to jailho...@googlegroups.com, Valentine Sinitsyn

Add single IOMMU entry that covers all builtin peripherals.

Signed-off-by: Valentine Sinitsyn <valentine...@gmail.com>
---

configs/f2a88xm-hd3.c | 35 ++++++++++++++++++++++++++---------
1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/configs/f2a88xm-hd3.c b/configs/f2a88xm-hd3.c
index b570c58..0a492c0 100644
--- a/configs/f2a88xm-hd3.c
+++ b/configs/f2a88xm-hd3.c
@@ -28,7 +28,7 @@ struct {
struct jailhouse_memory mem_regions[33];
struct jailhouse_irqchip irqchips[2];
__u8 pio_bitmap[0x2000];
- struct jailhouse_pci_device pci_devices[24];
+ struct jailhouse_pci_device pci_devices[23];
struct jailhouse_pci_capability pci_caps[26];
} __attribute__((packed)) config = {
.header = {
@@ -40,6 +40,9 @@ struct {
.mmconfig_base = 0xe0000000,
.mmconfig_end_bus = 0xff,
.pm_timer_address = 0x808,
+ .iommu_base = {
+ 0xfeb80000,
+ },
},
.root_cell = {
.name = "F2A88XM-HD3",
@@ -340,17 +343,10 @@ struct {
.caps_start = 0,
.num_caps = 0,
},
- /* PCIDevice: 00:00.2 */
- {
- .type = JAILHOUSE_PCI_TYPE_DEVICE,
- .domain = 0x0,
- .bdf = 0x2,
- .caps_start = 0,
- .num_caps = 3,
- },
/* PCIDevice: 00:01.0 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0x8,
.caps_start = 3,
@@ -359,6 +355,7 @@ struct {
/* PCIDevice: 00:01.1 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0x9,
.caps_start = 3,
@@ -367,6 +364,7 @@ struct {
/* PCIDevice: 00:04.0 */
{
.type = JAILHOUSE_PCI_TYPE_BRIDGE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0x20,
.caps_start = 6,
@@ -375,6 +373,7 @@ struct {
/* PCIDevice: 00:10.0 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0x80,
.caps_start = 11,
@@ -383,6 +382,7 @@ struct {
/* PCIDevice: 00:10.1 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0x81,
.caps_start = 11,
@@ -391,6 +391,7 @@ struct {
/* PCIDevice: 00:11.0 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0x88,
.caps_start = 15,
@@ -399,6 +400,7 @@ struct {
/* PCIDevice: 00:12.0 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0x90,
.caps_start = 0,
@@ -407,6 +409,7 @@ struct {
/* PCIDevice: 00:12.2 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0x92,
.caps_start = 17,
@@ -415,6 +418,7 @@ struct {
/* PCIDevice: 00:13.0 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0x98,
.caps_start = 0,
@@ -423,6 +427,7 @@ struct {
/* PCIDevice: 00:13.2 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0x9a,
.caps_start = 17,
@@ -431,6 +436,7 @@ struct {
/* PCIDevice: 00:14.0 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0xa0,
.caps_start = 0,
@@ -439,6 +445,7 @@ struct {
/* PCIDevice: 00:14.2 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0xa2,
.caps_start = 19,
@@ -447,6 +454,7 @@ struct {
/* PCIDevice: 00:14.3 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0xa3,
.caps_start = 0,
@@ -455,6 +463,7 @@ struct {
/* PCIDevice: 00:14.4 */
{
.type = JAILHOUSE_PCI_TYPE_BRIDGE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0xa4,
.caps_start = 0,
@@ -463,6 +472,7 @@ struct {
/* PCIDevice: 00:14.5 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0xa5,
.caps_start = 0,
@@ -471,6 +481,7 @@ struct {
/* PCIDevice: 00:18.0 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0xc0,
.caps_start = 0,
@@ -479,6 +490,7 @@ struct {
/* PCIDevice: 00:18.1 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0xc1,
.caps_start = 0,
@@ -487,6 +499,7 @@ struct {
/* PCIDevice: 00:18.2 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0xc2,
.caps_start = 0,
@@ -495,6 +508,7 @@ struct {
/* PCIDevice: 00:18.3 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0xc3,
.caps_start = 20,
@@ -503,6 +517,7 @@ struct {
/* PCIDevice: 00:18.4 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0xc4,
.caps_start = 0,
@@ -511,6 +526,7 @@ struct {
/* PCIDevice: 00:18.5 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0xc5,
.caps_start = 0,
@@ -519,6 +535,7 @@ struct {
/* PCIDevice: 01:00.0 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0x100,
.caps_start = 21,
--
2.3.0

Valentine Sinitsyn

unread,

Feb 25, 2015, 2:27:38 PM2/25/15

to Jailhouse

I've just noted I've forgotten to include corresponding line deletions
in this commit (and also commit message has a typo). Sorry for that,
will fix in v2.

Valentine

Jan Kiszka

unread,

Feb 26, 2015, 6:24:40 AM2/26/15

to Valentine Sinitsyn, jailho...@googlegroups.com

On 2015-02-25 20:25, Valentine Sinitsyn wrote:
> Adapt platform_info to store either VT-d (dmar_units_base) or AMD IOMMU
> (currently, iommu_base) data.
>
> Signed-off-by: Valentine Sinitsyn <valentine...@gmail.com>
> ---
> It is likely more AMD-specific flags will be needed for IOMMU support,
> that's why we introduce a union and not simply rename dmar_unit_base.
>
> hypervisor/include/jailhouse/cell-config.h | 6 +++++-
> 1 file changed, 5 insertions(+), 1 deletion(-)
>
> diff --git a/hypervisor/include/jailhouse/cell-config.h b/hypervisor/include/jailhouse/cell-config.h
> index e533127..d9f0606 100644
> --- a/hypervisor/include/jailhouse/cell-config.h
> +++ b/hypervisor/include/jailhouse/cell-config.h
> @@ -90,6 +90,7 @@ struct jailhouse_pci_capability {
> } __attribute__((packed));
>
> #define JAILHOUSE_MAX_DMAR_UNITS 8
> +#define JAILHOUSE_MAX_IOMMU_UNITS 1

There can really only be one on AMD systems?

>
> struct jailhouse_system {
> struct jailhouse_memory hypervisor_memory;
> @@ -100,7 +101,10 @@ struct jailhouse_system {
> __u8 mmconfig_end_bus;
> __u8 padding[5];
> __u16 pm_timer_address;
> - __u64 dmar_unit_base[JAILHOUSE_MAX_DMAR_UNITS];
> + union {
> + __u64 dmar_unit_base[JAILHOUSE_MAX_DMAR_UNITS];
> + __u64 iommu_base[JAILHOUSE_MAX_IOMMU_UNITS];

Event if AMD-specific flags should be added, I would prefer to
generalize "dmar unit" to "iommu" for now.

Jan

> + };
> } __attribute__((packed)) x86;
> } __attribute__((packed)) platform_info;
> __u32 device_limit;
>

--

Siemens AG, Corporate Technology, CT RTC ITP SES-DE
Corporate Competence Center Embedded Linux

Henning Schild

unread,

Feb 26, 2015, 6:42:31 AM2/26/15

to Valentine Sinitsyn, jailho...@googlegroups.com

Please update the config-collect code path as well.

Henning

Valentine Sinitsyn

unread,

Feb 26, 2015, 11:54:51 PM2/26/15

to Jan Kiszka, jailho...@googlegroups.com

Hi Jan,

On 26.02.2015 16:24, Jan Kiszka wrote:
>> diff --git a/hypervisor/include/jailhouse/cell-config.h b/hypervisor/include/jailhouse/cell-config.h
>> index e533127..d9f0606 100644
>> --- a/hypervisor/include/jailhouse/cell-config.h
>> +++ b/hypervisor/include/jailhouse/cell-config.h
>> @@ -90,6 +90,7 @@ struct jailhouse_pci_capability {
>> } __attribute__((packed));
>>
>> #define JAILHOUSE_MAX_DMAR_UNITS 8
>> +#define JAILHOUSE_MAX_IOMMU_UNITS 1
>
> There can really only be one on AMD systems?

No, but that's the only configuration I can test. If we can assume risks
of letting more IOMMUs we've actually tried out, this can probably be
set to the same value as JAILHOUSE_MAX_DMAR_UNITS (and reworked into
single definition).

>> struct jailhouse_system {
>> struct jailhouse_memory hypervisor_memory;
>> @@ -100,7 +101,10 @@ struct jailhouse_system {
>> __u8 mmconfig_end_bus;
>> __u8 padding[5];
>> __u16 pm_timer_address;
>> - __u64 dmar_unit_base[JAILHOUSE_MAX_DMAR_UNITS];
>> + union {
>> + __u64 dmar_unit_base[JAILHOUSE_MAX_DMAR_UNITS];
>> + __u64 iommu_base[JAILHOUSE_MAX_IOMMU_UNITS];
>
> Event if AMD-specific flags should be added, I would prefer to
> generalize "dmar unit" to "iommu" for now.

Okay, we'll set to iommu_base here and JAILHOUSE_MAX_IOMMU_UNITS (eight)
then.

Valentine

Valentine Sinitsyn

unread,

Feb 26, 2015, 11:55:51 PM2/26/15

to Henning Schild, jailho...@googlegroups.com

Hi Henning,

On 26.02.2015 16:43, Henning Schild wrote:
> Please update the config-collect code path as well.

Yup, I always forget this thing. Sorry.

Valentine

Valentine Sinitsyn

unread,

Feb 28, 2015, 2:40:34 AM2/28/15

to jailho...@googlegroups.com, Valentine Sinitsyn

Hi all,

This is an updated version of the previous patch series that accounts for the
comments I received on the list.

Valentine

Valentine Sinitsyn (3):

core: Introduce AMD IOMMU config data structures
tools: Implement ACPI IVRS table parser

configs: Update F2A88XM-HD3 config

configs/f2a88xm-hd3.c | 274 +++++++++++++++++++++++++----
configs/h87i.c | 2 +-
configs/qemu-vm.c | 2 +-
hypervisor/arch/x86/vtd.c | 8 +-
hypervisor/include/jailhouse/cell-config.h | 4 +-
tools/jailhouse-config-collect.tmpl | 5 +
tools/jailhouse-config-create | 203 ++++++++++++++++++---
tools/root-cell-config.c.tmpl | 8 +-
8 files changed, 431 insertions(+), 75 deletions(-)

--
2.3.1

Valentine Sinitsyn

unread,

Feb 28, 2015, 2:40:40 AM2/28/15

to jailho...@googlegroups.com, Valentine Sinitsyn

Adapt platform_info to store either VT-d (dmar_units_base) or AMD IOMMU
(currently, iommu_base) data.

Signed-off-by: Valentine Sinitsyn <valentine...@gmail.com>
---

configs/h87i.c | 2 +-
configs/qemu-vm.c | 2 +-

hypervisor/arch/x86/vtd.c | 8 ++++----
hypervisor/include/jailhouse/cell-config.h | 4 ++--
4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/configs/h87i.c b/configs/h87i.c
index 92b1989..c5473ac 100644
--- a/configs/h87i.c
+++ b/configs/h87i.c
@@ -35,7 +35,7 @@ struct {
.mmconfig_base = 0xf8000000,
.mmconfig_end_bus = 0x3f,
.pm_timer_address = 0x1808,
- .dmar_unit_base = {
+ .iommu_base = {
0xfed90000,
0xfed91000,
},
diff --git a/configs/qemu-vm.c b/configs/qemu-vm.c
index b11bc9e..3a9f309 100644
--- a/configs/qemu-vm.c
+++ b/configs/qemu-vm.c
@@ -47,7 +47,7 @@ struct {
.mmconfig_base = 0xb0000000,
.mmconfig_end_bus = 0xff,
.pm_timer_address = 0x608,
- .dmar_unit_base = {
+ .iommu_base = {
0xfed90000,
},
},
diff --git a/hypervisor/arch/x86/vtd.c b/hypervisor/arch/x86/vtd.c
index e1aad59..16fc1ae 100644
--- a/hypervisor/arch/x86/vtd.c
+++ b/hypervisor/arch/x86/vtd.c
@@ -69,7 +69,7 @@ static unsigned int dmar_pt_levels;
static unsigned int dmar_num_did = ~0U;
static unsigned int fault_reporting_cpu_id;
static DEFINE_SPINLOCK(inv_queue_lock);
-static struct vtd_emulation root_cell_units[JAILHOUSE_MAX_DMAR_UNITS];
+static struct vtd_emulation root_cell_units[JAILHOUSE_MAX_IOMMU_UNITS];
static bool dmar_units_initialized;

static unsigned int inv_queue_write(void *inv_queue, unsigned int index,
@@ -355,7 +355,7 @@ int iommu_mmio_access_handler(bool is_write, u64 addr, u32 *value)
return 0;

for (n = 0; n < dmar_units; n++) {
- base_addr = system_config->platform_info.x86.dmar_unit_base[n];
+ base_addr = system_config->platform_info.x86.iommu_base[n];
if (addr >= base_addr && addr < base_addr + PAGE_SIZE)
return vtd_unit_access_handler(n, is_write,
addr - base_addr,
@@ -465,7 +465,7 @@ int iommu_init(void)

int_remap_table_size_log2 = n;

- while (system_config->platform_info.x86.dmar_unit_base[units])
+ while (system_config->platform_info.x86.iommu_base[units])
units++;
if (units == 0)
return -EINVAL;
@@ -476,7 +476,7 @@ int iommu_init(void)
return -ENOMEM;

for (n = 0; n < units; n++) {
- base_addr = system_config->platform_info.x86.dmar_unit_base[n];
+ base_addr = system_config->platform_info.x86.iommu_base[n];

reg_base = dmar_reg_base + n * PAGE_SIZE;

diff --git a/hypervisor/include/jailhouse/cell-config.h b/hypervisor/include/jailhouse/cell-config.h
index e533127..da3d0c4 100644
--- a/hypervisor/include/jailhouse/cell-config.h
+++ b/hypervisor/include/jailhouse/cell-config.h
@@ -89,7 +89,7 @@ struct jailhouse_pci_capability {
__u16 flags;
} __attribute__((packed));

-#define JAILHOUSE_MAX_DMAR_UNITS 8
+#define JAILHOUSE_MAX_IOMMU_UNITS 8

struct jailhouse_system {
struct jailhouse_memory hypervisor_memory;

@@ -100,7 +100,7 @@ struct jailhouse_system {

__u8 mmconfig_end_bus;
__u8 padding[5];
__u16 pm_timer_address;
- __u64 dmar_unit_base[JAILHOUSE_MAX_DMAR_UNITS];

+ __u64 iommu_base[JAILHOUSE_MAX_IOMMU_UNITS];

} __attribute__((packed)) x86;
} __attribute__((packed)) platform_info;
__u32 device_limit;
--

2.3.1

Valentine Sinitsyn

unread,

Feb 28, 2015, 2:40:43 AM2/28/15

to jailho...@googlegroups.com, Valentine Sinitsyn

Add parse_ivrs() function that extracts relevant bits of information

from ACPI IVRS table which describes AMD IOMMU units found in the system.

As VT-d and AMD-Vi impose slightly different requirements on PCI devices
configuration (eg PCI root complex), move sanity checks to corresponding
functions to account for these discrepancies.

Configuration file template was also updated to account for new cell
definition parameters names.

Signed-off-by: Valentine Sinitsyn <valentine...@gmail.com>
---

tools/jailhouse-config-collect.tmpl | 5 +
tools/jailhouse-config-create | 203 +++++++++++++++++++++++++++++++-----
tools/root-cell-config.c.tmpl | 8 +-
3 files changed, 187 insertions(+), 29 deletions(-)

diff --git a/tools/jailhouse-config-collect.tmpl b/tools/jailhouse-config-collect.tmpl
index d155b18..6cdf806 100644
--- a/tools/jailhouse-config-collect.tmpl
+++ b/tools/jailhouse-config-collect.tmpl
@@ -37,6 +37,7 @@ fi
filelist="${filelist}"
filelist_opt="${filelist_opt}"
filelist_intel="${filelist_intel}"
+filelist_amd="${filelist_amd}"

tmpdir=/tmp/jailhouse-config-collect.$$

@@ -60,6 +61,10 @@ grep GenuineIntel /proc/cpuinfo > /dev/null &&
for f in $filelist_intel; do
copy_file $f
done
+grep AuthenticAMD /proc/cpuino > /dev/null &&
+ for f in $filelist_amd; do
+ copy_file $f
+ done
for f in $filelist_opt; do
if [ -f $f ]; then
copy_file $f
diff --git a/tools/jailhouse-config-create b/tools/jailhouse-config-create
index c3eebbe..bfe31aa 100755
--- a/tools/jailhouse-config-create
+++ b/tools/jailhouse-config-create

@@ -702,9 +706,157 @@ def parse_dmar(pcidevices, ioapics):

f.seek(struct_len - offset, os.SEEK_CUR)

f.close()
+
+ for d in pcidevices:
+ if d.iommu is None:
+ raise RuntimeError('PCI device %02x:%02x.%x outside the scope of an '
+ 'IOMMU' % (d.bus, d.dev, d.fn))
+

+ for d in pcidevices:

+ d.iommu = len(units) - 1
+ elif entry_type == 0x02:
+ # Select

+ for d in pcidevices:

+ if d.bdf() == device_id:
+ d.iommu = len(units) - 1
+ elif entry_type == 0x03:
+ # Start of range
+ bdf_start_range = device_id
+ elif entry_type == 0x04:
+ # End of range
+ if bdf_start_range is None:
+ continue

+ for d in pcidevices:

+ if d.bdf() >= bdf_start_range and d.bdf() <= device_id:
+ d.iommu = len(units) - 1
+ bdf_start_range = None
+ elif entry_type == 0x42:
+ # Alias select
+ (device_id_b,) = struct.unpack('<xHx', f.read(4))
+ block_length -= 4

+ for d in pcidevices:

+
+ for d in pcidevices:

+ if d.bdf() not in iommu_skiplist and d.iommu is None:

+ raise RuntimeError('PCI device %02x:%02x.%x outside the scope of an '
+ 'IOMMU' % (d.bus, d.dev, d.fn))
+

+ f.close()
+ return units, regions
+
def parse_ioports():
pm_timer_base = None
f = input_open('/proc/ioports')

@@ -759,11 +911,12 @@ if options.generate_collector:
filelist = ' '.join(inputs['files'])
filelist_opt = ' '.join(inputs['files_opt'])
filelist_intel = ' '.join(inputs['files_intel'])
+ filelist_amd = ' '.join(inputs['files_amd'])

tmpl = Template(filename=os.path.join(options.template_dir,
'jailhouse-config-collect.tmpl'))
f.write(tmpl.render(filelist=filelist, filelist_opt=filelist_opt,
- filelist_intel=filelist_intel))
+ filelist_intel=filelist_intel, filelist_amd=filelist_amd))
f.close()
sys.exit(0)

@@ -797,18 +950,12 @@ mmconfig = MMConfig.parse()

ioapics = parse_madt()

-if get_cpu_vendor() == 'GenuineIntel':
- (dmar_units, rmrr_regs) = parse_dmar(pcidevices, ioapics)
+vendor = get_cpu_vendor()
+if vendor == 'GenuineIntel':
+ (iommu_units, extra_memregs) = parse_dmar(pcidevices, ioapics)
else:
- (dmar_units, rmrr_regs) = [], []
-regions += rmrr_regs
-
-for d in pcidevices:
- if get_cpu_vendor() == 'AuthenticAMD':
- d.iommu = 0 # temporary workaround
- if d.iommu is None:
- raise RuntimeError('PCI device %02x:%02x.%x outside the scope of an '
- 'IOMMU' % (d.bus, d.dev, d.fn))
+ (iommu_units, extra_memregs) = parse_ivrs(pcidevices, ioapics)
+regions += extra_memregs

# kernel does not have memmap region, pick one
if ourmem is None:

@@ -832,17 +979,21 @@ pm_timer_base = parse_ioports()

+ 'iommu_units': iommu_units
+}

+
+f.write(tmpl.render(**kwargs))

f.close()
diff --git a/tools/root-cell-config.c.tmpl b/tools/root-cell-config.c.tmpl

index a0ca787..004836a 100644
--- a/tools/root-cell-config.c.tmpl
+++ b/tools/root-cell-config.c.tmpl
@@ -36,9 +36,9 @@ struct {
.mmconfig_base = ${hex(mmconfig.base)},
.mmconfig_end_bus = ${hex(mmconfig.end_bus)},
.pm_timer_address = ${hex(pm_timer_base)},
- % if dmar_units:
- .dmar_unit_base = {
- % for d in dmar_units:

+ % if iommu_units:
+ .iommu_base = {
+ % for d in iommu_units:

${hex(d)},
% endfor
},
@@ -110,7 +110,9 @@ struct {

/* ${str(d)} */
{
.type = ${d.type},
+ % if d.iommu is not None:
.iommu = ${d.iommu},
+ % endif
.domain = ${hex(d.domain)},
.bdf = ${hex(d.bdf())},
.caps_start = ${d.caps_start},
--

2.3.1

Valentine Sinitsyn

unread,

Feb 28, 2015, 2:40:45 AM2/28/15

to jailho...@googlegroups.com, Valentine Sinitsyn

Add single IOMMU entry that covers all built-in peripherals.
Also, include all missing bits of configuration data (mostly
PCI-related anyway).

Signed-off-by: Valentine Sinitsyn <valentine...@gmail.com>
---

configs/f2a88xm-hd3.c | 274 +++++++++++++++++++++++++++++++++++++++++++-------
1 file changed, 236 insertions(+), 38 deletions(-)

diff --git a/configs/f2a88xm-hd3.c b/configs/f2a88xm-hd3.c
index b570c58..2736496 100644
--- a/configs/f2a88xm-hd3.c
+++ b/configs/f2a88xm-hd3.c
@@ -20,16 +20,16 @@
#include <linux/types.h>
#include <jailhouse/cell-config.h>

-#define ARRAY_SIZE(a) sizeof(a) / sizeof(a[0])
+#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))

struct {
struct jailhouse_system header;
__u64 cpus[1];
- struct jailhouse_memory mem_regions[33];
+ struct jailhouse_memory mem_regions[35];

struct jailhouse_irqchip irqchips[2];
__u8 pio_bitmap[0x2000];
- struct jailhouse_pci_device pci_devices[24];

- struct jailhouse_pci_capability pci_caps[26];
+ struct jailhouse_pci_device pci_devices[26];
+ struct jailhouse_pci_capability pci_caps[27];

} __attribute__((packed)) config = {
.header = {

.hypervisor_memory = {
@@ -40,7 +40,12 @@ struct {
.mmconfig_base = 0xe0000000,
.mmconfig_end_bus = 0xff,

.pm_timer_address = 0x808,
+ .iommu_base = {
+ 0xfeb80000,
+ },
},

+ .device_limit = 128,
+ .interrupt_limit = 256,

.root_cell = {
.name = "F2A88XM-HD3",

.cpu_set_size = sizeof(config.cpus),
@@ -86,34 +91,50 @@ struct {
.size = 0x20000,
.flags = JAILHOUSE_MEM_READ,
},
- /* MemRegion: 00100000-3affffff : System RAM */
+ /* MemRegion: 00100000-00ffffff : System RAM */
{
.phys_start = 0x100000,
.virt_start = 0x100000,
- .size = 0x3af00000,
+ .size = 0xf00000,
.flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
JAILHOUSE_MEM_EXECUTE | JAILHOUSE_MEM_DMA,
},
- /* MemRegion: 3f200000-6c046fff : System RAM */
+ /* MemRegion: 01000000-01ffffff : Kernel */
+ {
+ .phys_start = 0x1000000,
+ .virt_start = 0x1000000,
+ .size = 0x1000000,
+ .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
+ JAILHOUSE_MEM_EXECUTE | JAILHOUSE_MEM_DMA,
+ },
+ /* MemRegion: 02000000-3affffff : System RAM */
+ {
+ .phys_start = 0x2000000,
+ .virt_start = 0x2000000,
+ .size = 0x39000000,
+ .flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
+ JAILHOUSE_MEM_EXECUTE | JAILHOUSE_MEM_DMA,
+ },
+ /* MemRegion: 3f200000-6b8ecfff : System RAM */
{
.phys_start = 0x3f200000,
.virt_start = 0x3f200000,
- .size = 0x2ce47000,
+ .size = 0x2c6ed000,
.flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
JAILHOUSE_MEM_EXECUTE | JAILHOUSE_MEM_DMA,
},
- /* MemRegion: 6c077000-6c339fff : System RAM */
+ /* MemRegion: 6b91d000-6bbdffff : System RAM */
{
- .phys_start = 0x6c077000,
- .virt_start = 0x6c077000,
+ .phys_start = 0x6b91d000,
+ .virt_start = 0x6b91d000,
.size = 0x2c3000,
.flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE |
JAILHOUSE_MEM_EXECUTE | JAILHOUSE_MEM_DMA,
},
- /* MemRegion: 6c33a000-6c407fff : ACPI Non-volatile Storage */
+ /* MemRegion: 6bbe0000-6bcadfff : ACPI Non-volatile Storage */
{
- .phys_start = 0x6c33a000,
- .virt_start = 0x6c33a000,
+ .phys_start = 0x6bbe0000,
+ .virt_start = 0x6bbe0000,
.size = 0xce000,
.flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
},
@@ -268,7 +289,7 @@ struct {
.size = 0x1000,
.flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
},
- /* MemRegion: feb80000-febfffff : amd_iommu */
+ /* MemRegion: feb80000-febfffff : pnp 00:02 */
{
.phys_start = 0xfeb80000,
.virt_start = 0xfeb80000,
@@ -282,7 +303,7 @@ struct {
.size = 0x1000,
.flags = JAILHOUSE_MEM_READ | JAILHOUSE_MEM_WRITE,
},
- /* MemRegion: fed61000-fed70fff : pnp 00:0a */
+ /* MemRegion: fed61000-fed70fff : pnp 00:09 */
{
.phys_start = 0xfed61000,
.virt_start = 0xfed61000,
@@ -339,190 +360,361 @@ struct {
.bdf = 0x0,

.caps_start = 0,
.num_caps = 0,

- },

- /* PCIDevice: 00:00.2 */
- {
- .type = JAILHOUSE_PCI_TYPE_DEVICE,
- .domain = 0x0,
- .bdf = 0x2,
- .caps_start = 0,
- .num_caps = 3,

+ .num_msi_vectors = 0,
+ .msi_64bits = 0,
+ .num_msix_vectors = 0,
+ .msix_region_size = 0x0,
+ .msix_address = 0x0,

},
/* PCIDevice: 00:01.0 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0x8,
.caps_start = 3,

- .num_caps = 3,
+ .num_caps = 4,
+ .num_msi_vectors = 1,
+ .msi_64bits = 1,
+ .num_msix_vectors = 0,
+ .msix_region_size = 0x0,
+ .msix_address = 0x0,
},

/* PCIDevice: 00:01.1 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0x9,
.caps_start = 3,

- .num_caps = 3,
+ .num_caps = 4,
+ .num_msi_vectors = 1,
+ .msi_64bits = 1,
+ .num_msix_vectors = 0,
+ .msix_region_size = 0x0,
+ .msix_address = 0x0,
},
- /* PCIDevice: 00:04.0 */
+ /* PCIDevice: 00:02.0 */
+ {
+ .type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
+ .domain = 0x0,
+ .bdf = 0x10,
+ .caps_start = 0,
+ .num_caps = 0,
+ .num_msi_vectors = 0,
+ .msi_64bits = 0,
+ .num_msix_vectors = 0,
+ .msix_region_size = 0x0,
+ .msix_address = 0x0,
+ },
+ /* PCIDevice: 00:03.0 */
+ {
+ .type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
+ .domain = 0x0,
+ .bdf = 0x18,
+ .caps_start = 0,
+ .num_caps = 0,
+ .num_msi_vectors = 0,
+ .msi_64bits = 0,
+ .num_msix_vectors = 0,
+ .msix_region_size = 0x0,
+ .msix_address = 0x0,
+ },
+ /* PCIDevice: 00:03.1 */

{
.type = JAILHOUSE_PCI_TYPE_BRIDGE,
+ .iommu = 0,
.domain = 0x0,

- .bdf = 0x20,
- .caps_start = 6,
+ .bdf = 0x19,
+ .caps_start = 7,
.num_caps = 5,
+ .num_msi_vectors = 1,
+ .msi_64bits = 1,
+ .num_msix_vectors = 0,
+ .msix_region_size = 0x0,
+ .msix_address = 0x0,
+ },
+ /* PCIDevice: 00:04.0 */
+ {
+ .type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
+ .domain = 0x0,
+ .bdf = 0x20,
+ .caps_start = 0,
+ .num_caps = 0,
+ .num_msi_vectors = 0,
+ .msi_64bits = 0,
+ .num_msix_vectors = 0,
+ .msix_region_size = 0x0,
+ .msix_address = 0x0,
},

/* PCIDevice: 00:10.0 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0x80,

- .caps_start = 11,
+ .caps_start = 12,
.num_caps = 4,
+ .num_msi_vectors = 8,
+ .msi_64bits = 1,
+ .num_msix_vectors = 8,
+ .msix_region_size = 0x1000,
+ .msix_address = 0xfeb6b000,
},

/* PCIDevice: 00:10.1 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0x81,

- .caps_start = 11,
+ .caps_start = 12,
.num_caps = 4,
+ .num_msi_vectors = 8,
+ .msi_64bits = 1,
+ .num_msix_vectors = 8,
+ .msix_region_size = 0x1000,
+ .msix_address = 0xfeb69000,
},

/* PCIDevice: 00:11.0 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0x88,

- .caps_start = 15,
+ .caps_start = 16,
.num_caps = 2,
+ .num_msi_vectors = 8,
+ .msi_64bits = 1,
+ .num_msix_vectors = 0,
+ .msix_region_size = 0x0,
+ .msix_address = 0x0,
},

/* PCIDevice: 00:12.0 */
{
.type = JAILHOUSE_PCI_TYPE_DEVICE,
+ .iommu = 0,
.domain = 0x0,
.bdf = 0x90,
.caps_start = 0,