This massively simplifies the code and reduces the memory usage in
struct per_cpu. However, adding interrupt priorities later on may
require another rework.
hypervisor/arch/arm/gic-v2.c | 10 +-
hypervisor/arch/arm/gic-v3.c | 10 +-
hypervisor/arch/arm/include/asm/irqchip.h | 18 +---
hypervisor/arch/arm/include/asm/percpu.h | 13 +--
hypervisor/arch/arm/irqchip.c | 151 +++++-------------------------
5 files changed, 46 insertions(+), 156 deletions(-)
diff --git a/hypervisor/arch/arm/gic-v2.c b/hypervisor/arch/arm/gic-v2.c
index 6e939fa..aae5c0e 100644
--- a/hypervisor/arch/arm/gic-v2.c
+++ b/hypervisor/arch/arm/gic-v2.c
@@ -228,7 +228,7 @@ static int gic_send_sgi(struct sgi *sgi)
return 0;
}
-static int gic_inject_irq(struct per_cpu *cpu_data, struct pending_irq *irq)
+static int gic_inject_irq(struct per_cpu *cpu_data, u16 irq_id)
{
int i;
int first_free = -1;
@@ -247,7 +247,7 @@ static int gic_inject_irq(struct per_cpu *cpu_data, struct pending_irq *irq)
/* Check that there is no overlapping */
lr = gic_read_lr(i);
- if ((lr & GICH_LR_VIRT_ID_MASK) == irq->virt_id)
+ if ((lr & GICH_LR_VIRT_ID_MASK) == irq_id)
return -EEXIST;
}
@@ -255,12 +255,12 @@ static int gic_inject_irq(struct per_cpu *cpu_data, struct pending_irq *irq)
return -EBUSY;
/* Inject group 0 interrupt (seen as IRQ by the guest) */
- lr = irq->virt_id;
+ lr = irq_id;
lr |= GICH_LR_PENDING_BIT;
- if (!is_sgi(irq->virt_id)) {
+ if (!is_sgi(irq_id)) {
lr |= GICH_LR_HW_BIT;
- lr |= irq->virt_id << GICH_LR_PHYS_ID_SHIFT;
+ lr |= (u32)irq_id << GICH_LR_PHYS_ID_SHIFT;
}
gic_write_lr(first_free, lr);
diff --git a/hypervisor/arch/arm/gic-v3.c b/hypervisor/arch/arm/gic-v3.c
index 40526ea..65f326c 100644
--- a/hypervisor/arch/arm/gic-v3.c
+++ b/hypervisor/arch/arm/gic-v3.c
@@ -340,7 +340,7 @@ static void gic_eoi_irq(u32 irq_id, bool deactivate)
arm_write_sysreg(ICC_DIR_EL1, irq_id);
}
-static int gic_inject_irq(struct per_cpu *cpu_data, struct pending_irq *irq)
+static int gic_inject_irq(struct per_cpu *cpu_data, u16 irq_id)
{
int i;
int free_lr = -1;
@@ -366,7 +366,7 @@ static int gic_inject_irq(struct per_cpu *cpu_data, struct pending_irq *irq)
* A strict phys->virt id mapping is used for SPIs, so this test
* should be sufficient.
*/
- if ((u32)lr == irq->virt_id)
+ if ((u32)lr == irq_id)
return -EEXIST;
}
@@ -374,13 +374,13 @@ static int gic_inject_irq(struct per_cpu *cpu_data, struct pending_irq *irq)
/* All list registers are in use */
return -EBUSY;
- lr = irq->virt_id;
+ lr = irq_id;
/* Only group 1 interrupts */
lr |= ICH_LR_GROUP_BIT;
lr |= ICH_LR_PENDING;
- if (!is_sgi(irq->virt_id)) {
+ if (!is_sgi(irq_id)) {
lr |= ICH_LR_HW_BIT;
- lr |= (u64)irq->virt_id << ICH_LR_PHYS_ID_SHIFT;
+ lr |= (u64)irq_id << ICH_LR_PHYS_ID_SHIFT;
}
gic_write_lr(free_lr, lr);
diff --git a/hypervisor/arch/arm/include/asm/irqchip.h b/hypervisor/arch/arm/include/asm/irqchip.h
index 538b432..f0f0e7a 100644
--- a/hypervisor/arch/arm/include/asm/irqchip.h
+++ b/hypervisor/arch/arm/include/asm/irqchip.h
@@ -13,20 +13,15 @@
#ifndef _JAILHOUSE_ASM_IRQCHIP_H
#define _JAILHOUSE_ASM_IRQCHIP_H
-/*
- * Since there is no finer-grained allocation than page-alloc for the moment,
- * and it is very complicated to predict the total size needed at
- * initialisation, each cpu is allocated one page of pending irqs.
- * This allows for 256 pending IRQs, which should be sufficient.
- */
-#define MAX_PENDING_IRQS (PAGE_SIZE / sizeof(struct pending_irq))
+#define MAX_PENDING_IRQS 256
#include <jailhouse/cell.h>
#include <jailhouse/mmio.h>
-#include <asm/percpu.h>
#ifndef __ASSEMBLY__
+struct per_cpu;
+
struct sgi {
/*
* Routing mode values:
@@ -54,8 +49,7 @@ struct irqchip_ops {
int (*send_sgi)(struct sgi *sgi);
void (*handle_irq)(struct per_cpu *cpu_data);
void (*eoi_irq)(u32 irqn, bool deactivate);
- int (*inject_irq)(struct per_cpu *cpu_data,
- struct pending_irq *irq);
+ int (*inject_irq)(struct per_cpu *cpu_data, u16 irq_id);
void (*enable_maint_irq)(bool enable);
int (*mmio_access)(struct mmio_access *access);
@@ -85,9 +79,7 @@ void irqchip_handle_irq(struct per_cpu *cpu_data);
void irqchip_eoi_irq(u32 irqn, bool deactivate);
void irqchip_inject_pending(struct per_cpu *cpu_data);
-int irqchip_insert_pending(struct per_cpu *cpu_data, struct pending_irq *irq);
-int irqchip_remove_pending(struct per_cpu *cpu_data, struct pending_irq *irq);
-void irqchip_set_pending(struct per_cpu *cpu_data, u32 irq_id, bool try_inject);
+void irqchip_set_pending(struct per_cpu *cpu_data, u16 irq_id, bool try_inject);
bool spi_in_cell(struct cell *cell, unsigned int spi);
diff --git a/hypervisor/arch/arm/include/asm/percpu.h b/hypervisor/arch/arm/include/asm/percpu.h
index e6562da..ae026c4 100644
--- a/hypervisor/arch/arm/include/asm/percpu.h
+++ b/hypervisor/arch/arm/include/asm/percpu.h
@@ -21,6 +21,7 @@
#ifndef __ASSEMBLY__
#include <jailhouse/cell.h>
+#include <asm/irqchip.h>
#include <asm/psci.h>
#include <asm/spinlock.h>
#include <asm/sysregs.h>
@@ -29,8 +30,6 @@
#define PERCPU_SIZE_SHIFT \
(BITS_PER_LONG - __builtin_clzl(sizeof(struct per_cpu) - 1))
-struct pending_irq;
-
struct per_cpu {
u8 stack[PAGE_SIZE];
unsigned long linux_sp;
@@ -41,10 +40,12 @@ struct per_cpu {
unsigned int cpu_id;
unsigned int virt_id;
- /* Other CPUs can insert sgis into the pending array */
- spinlock_t gic_lock;
- struct pending_irq *pending_irqs;
- struct pending_irq *first_pending;
+ /* synchronizes parallel insertions of SGIs into the pending ring */
+ spinlock_t pending_irqs_lock;
+ u16 pending_irqs[MAX_PENDING_IRQS];
+ unsigned int pending_irqs_head;
+ /* removal from the ring happens lockless, thus tail is volatile */
+ volatile unsigned int pending_irqs_tail;
/* Only GICv3: redistributor base */
void *gicr_base;
diff --git a/hypervisor/arch/arm/irqchip.c b/hypervisor/arch/arm/irqchip.c
index fb25744..538bbc0 100644
--- a/hypervisor/arch/arm/irqchip.c
+++ b/hypervisor/arch/arm/irqchip.c
@@ -49,142 +49,49 @@ bool spi_in_cell(struct cell *cell, unsigned int spi)
return spi_mask & (1 << (spi & 31));
}
-static int irqchip_init_pending(struct per_cpu *cpu_data)
+void irqchip_set_pending(struct per_cpu *cpu_data, u16 irq_id, bool try_inject)
{
- struct pending_irq *pend_array;
-
- if (cpu_data->pending_irqs == NULL) {
- cpu_data->pending_irqs = pend_array = page_alloc(&mem_pool, 1);
- if (pend_array == NULL)
- return -ENOMEM;
- } else {
- pend_array = cpu_data->pending_irqs;
- }
-
- memset(pend_array, 0, PAGE_SIZE);
-
- cpu_data->pending_irqs = pend_array;
- cpu_data->first_pending = NULL;
-
- return 0;
-}
-
-/*
- * Find the first available pending struct for insertion. The `prev' pointer is
- * set to the previous pending interrupt, if any, to help inserting the new one
- * into the list.
- * Returns NULL when no slot is available
- */
-static struct pending_irq* get_pending_slot(struct per_cpu *cpu_data,
- struct pending_irq **prev)
-{
- u32 i, pending_idx;
- struct pending_irq *pending = cpu_data->first_pending;
-
- *prev = NULL;
-
- for (i = 0; i < MAX_PENDING_IRQS; i++) {
- pending_idx = pending - cpu_data->pending_irqs;
- if (pending == NULL || i < pending_idx)
- return cpu_data->pending_irqs + i;
+ unsigned int new_tail;
- *prev = pending;
- pending = pending->next;
- }
-
- return NULL;
-}
+ if (try_inject && irqchip.inject_irq(cpu_data, irq_id) != -EBUSY)
+ return;
-int irqchip_insert_pending(struct per_cpu *cpu_data, struct pending_irq *irq)
-{
- struct pending_irq *prev = NULL;
- struct pending_irq *slot;
+ spin_lock(&cpu_data->pending_irqs_lock);
- spin_lock(&cpu_data->gic_lock);
+ new_tail = (cpu_data->pending_irqs_tail + 1) % MAX_PENDING_IRQS;
- slot = get_pending_slot(cpu_data, &prev);
- if (slot == NULL) {
- spin_unlock(&cpu_data->gic_lock);
- return -ENOMEM;
+ /* Queue space available? */
+ if (new_tail != cpu_data->pending_irqs_head) {
+ cpu_data->pending_irqs[cpu_data->pending_irqs_tail] = irq_id;
+ cpu_data->pending_irqs_tail = new_tail;
+ /*
+ * Make the change to pending_irqs_tail visible before the
+ * caller sends SGI_INJECT.
+ */
+ memory_barrier();
}
- /*
- * Don't override the pointers yet, they may be read by the injection
- * loop. Odds are astronomically low, but hey.
- */
- memcpy(slot, irq, sizeof(struct pending_irq) - 2 * sizeof(void *));
- slot->prev = prev;
- if (prev) {
- slot->next = prev->next;
- prev->next = slot;
- } else {
- slot->next = cpu_data->first_pending;
- cpu_data->first_pending = slot;
- }
- if (slot->next)
- slot->next->prev = slot;
-
- spin_unlock(&cpu_data->gic_lock);
-
- return 0;
-}
-
-void irqchip_set_pending(struct per_cpu *cpu_data, u32 irq_id, bool try_inject)
-{
- struct pending_irq pending;
-
- pending.virt_id = irq_id;
-
- if (!try_inject || irqchip.inject_irq(cpu_data, &pending) == -EBUSY)
- irqchip_insert_pending(cpu_data, &pending);
-}
-
-/*
- * Only executed by `irqchip_inject_pending' on a CPU to inject its own stuff.
- */
-int irqchip_remove_pending(struct per_cpu *cpu_data, struct pending_irq *irq)
-{
- spin_lock(&cpu_data->gic_lock);
-
- if (cpu_data->first_pending == irq)
- cpu_data->first_pending = irq->next;
- if (irq->prev)
- irq->prev->next = irq->next;
- if (irq->next)
- irq->next->prev = irq->prev;
-
- spin_unlock(&cpu_data->gic_lock);
-
- return 0;
+ spin_unlock(&cpu_data->pending_irqs_lock);
}
void irqchip_inject_pending(struct per_cpu *cpu_data)
{
- int err;
- struct pending_irq *pending = cpu_data->first_pending;
+ u16 irq_id;
- while (pending != NULL) {
- err = irqchip.inject_irq(cpu_data, pending);
- if (err == -EBUSY) {
+ while (cpu_data->pending_irqs_head != cpu_data->pending_irqs_tail) {
+ irq_id = cpu_data->pending_irqs[cpu_data->pending_irqs_head];
+
+ if (irqchip.inject_irq(cpu_data, irq_id) == -EBUSY) {
/*
* The list registers are full, trigger maintenance
* interrupt and leave.
*/
irqchip.enable_maint_irq(true);
return;
- } else {
- /*
- * Removal only changes the pointers, but does not
- * deallocate anything.
- * Concurrent accesses are avoided with the spinlock,
- * but the `next' pointer of the current pending object
- * may be rewritten by an external insert before or
- * after this removal, which isn't an issue.
- */
- irqchip_remove_pending(cpu_data, pending);
}
- pending = pending->next;
+ cpu_data->pending_irqs_head =
+ (cpu_data->pending_irqs_head + 1) % MAX_PENDING_IRQS;
}
/*
@@ -211,12 +118,6 @@ int irqchip_send_sgi(struct sgi *sgi)
int irqchip_cpu_init(struct per_cpu *cpu_data)
{
- int err;
-
- err = irqchip_init_pending(cpu_data);
- if (err)
- return err;
-
if (irqchip.cpu_init)
return irqchip.cpu_init(cpu_data);
@@ -225,11 +126,7 @@ int irqchip_cpu_init(struct per_cpu *cpu_data)
int irqchip_cpu_reset(struct per_cpu *cpu_data)
{
- int err;
-
- err = irqchip_init_pending(cpu_data);
- if (err)
- return err;
+ cpu_data->pending_irqs_head = cpu_data->pending_irqs_tail = 0;
if (irqchip.cpu_reset)
return irqchip.cpu_reset(cpu_data, false);
--
2.1.4