[PATCH v2 00/18] Increase maximum number of Infiniband HCAs per system
Alex Chiang
IB HCAs supported per system.
The original mail thread is here:
http://lkml.org/lkml/2010/1/29/346
One note, I decided to "copy/paste" since factoring out the overflow
code in the three drivers seemed like overkill. If so desired, I could
factor those three separate functions into something provided by the
core, but that seemed like more trouble than it was worth at the time.
As before, I still don't have access to a giant system, so what I did
to test was to stick 4 cards into a small system, and then modify the
limits with debug patches similar to this:
diff --git a/drivers/infiniband/core/ucm.c b/drivers/infiniband/core/ucm.c
index 7bf0a82..8581e64 100644
--- a/drivers/infiniband/core/ucm.c
+++ b/drivers/infiniband/core/ucm.c
@@ -102,7 +102,7 @@ struct ib_ucm_event {
enum {
IB_UCM_MAJOR = 231,
IB_UCM_BASE_MINOR = 224,
- IB_UCM_MAX_DEVICES = 32
+ IB_UCM_MAX_DEVICES = 2
};
I tested all 3 drivers this way (uverbs, umad, ucm). I verified that
we're not leaking device numbers on multiple modprobe/rmmod cycles,
that there aren't any funny interactions when various combinations of
the drivers are loaded.
I did not test the rest of the OFED stack. I did write some trivial
programs to open the devices in /dev and close them again.
Here's an example of some of the testing:
dl585g2:~ # modprobe ib_uverbs
dl585g2:~ # modprobe ib_umad
dl585g2:~ # modprobe ib_ucm
dl585g2:~ # ls -l /dev/uverb*
crw-rw---- 1 root root 231, 192 Feb 2 05:55 /dev/uverbs0
crw-rw---- 1 root root 231, 193 Feb 2 05:55 /dev/uverbs1
crw-rw---- 1 root root 249, 0 Feb 2 05:55 /dev/uverbs2
crw-rw---- 1 root root 249, 1 Feb 2 05:55 /dev/uverbs3
dl585g2:~ # ls -l /dev/umad*
crw-rw---- 1 root root 231, 0 Feb 2 05:55 /dev/umad0
crw-rw---- 1 root root 231, 1 Feb 2 05:55 /dev/umad1
crw-rw---- 1 root root 231, 2 Feb 2 05:55 /dev/umad2
crw-rw---- 1 root root 231, 3 Feb 2 05:55 /dev/umad3
crw-rw---- 1 root root 248, 0 Feb 2 05:55 /dev/umad4
crw-rw---- 1 root root 248, 1 Feb 2 05:55 /dev/umad5
crw-rw---- 1 root root 248, 2 Feb 2 05:55 /dev/umad6
crw-rw---- 1 root root 248, 3 Feb 2 05:55 /dev/umad7
dl585g2:~ # ls -l /dev/issm*
crw-rw---- 1 root root 231, 4 Feb 2 05:55 /dev/issm0
crw-rw---- 1 root root 231, 5 Feb 2 05:55 /dev/issm1
crw-rw---- 1 root root 231, 6 Feb 2 05:55 /dev/issm2
crw-rw---- 1 root root 231, 7 Feb 2 05:55 /dev/issm3
crw-rw---- 1 root root 248, 4 Feb 2 05:55 /dev/issm4
crw-rw---- 1 root root 248, 5 Feb 2 05:55 /dev/issm5
crw-rw---- 1 root root 248, 6 Feb 2 05:55 /dev/issm6
crw-rw---- 1 root root 248, 7 Feb 2 05:55 /dev/issm7
dl585g2:~ # ls -l /dev/ucm*
crw-rw---- 1 root root 231, 224 Feb 2 05:55 /dev/ucm0
crw-rw---- 1 root root 231, 225 Feb 2 05:55 /dev/ucm1
crw-rw---- 1 root root 247, 0 Feb 2 05:55 /dev/ucm2
crw-rw---- 1 root root 247, 1 Feb 2 05:55 /dev/ucm3
Note that the major and minor numbers are behaving rather sanely.
dl585g2:~ # rmmod ib_ucm
dl585g2:~ # rmmod ib_uverbs
dl585g2:~ # rmmod ib_umad
Reset.
dl585g2:~ # modprobe ib_ucm
dl585g2:~ # ls -l /dev/ucm*
crw-rw---- 1 root root 231, 224 Feb 2 05:57 /dev/ucm0
crw-rw---- 1 root root 231, 225 Feb 2 05:57 /dev/ucm1
crw-rw---- 1 root root 248, 0 Feb 2 05:57 /dev/ucm2
crw-rw---- 1 root root 248, 1 Feb 2 05:57 /dev/ucm3
See that /dev/ucm* devices now have a different major number
compared to last time(248 vs 247), since we loaded that driver first.
But wait, why is it 248 and not 249? Is there a leak somewhere?
dl585g2:~ # ls -l /dev/uverb*
crw-rw---- 1 root root 231, 192 Feb 2 05:57 /dev/uverbs0
crw-rw---- 1 root root 231, 193 Feb 2 05:57 /dev/uverbs1
crw-rw---- 1 root root 249, 0 Feb 2 05:57 /dev/uverbs2
crw-rw---- 1 root root 249, 1 Feb 2 05:57 /dev/uverbs3
dl585g2:~ # rmmod ib_uverbs
ERROR: Module ib_uverbs is in use by ib_ucm
Ah, ib_ucm is dependent on ib_uverbs, so when we modprobed ib_ucm,
in reality ib_uverbs got loaded first. See how it has a higher
major number.
dl585g2:~ # rmmod ib_ucm
dl585g2:~ # rmmod ib_uverbs
dl585g2:~ # modprobe ib_umad
dl585g2:~ # ls -l /dev/umad*
crw-rw---- 1 root root 231, 0 Feb 2 05:58 /dev/umad0
crw-rw---- 1 root root 231, 1 Feb 2 05:58 /dev/umad1
crw-rw---- 1 root root 231, 2 Feb 2 05:58 /dev/umad2
crw-rw---- 1 root root 231, 3 Feb 2 05:58 /dev/umad3
crw-rw---- 1 root root 249, 0 Feb 2 05:58 /dev/umad4
crw-rw---- 1 root root 249, 1 Feb 2 05:58 /dev/umad5
crw-rw---- 1 root root 249, 2 Feb 2 05:58 /dev/umad6
crw-rw---- 1 root root 249, 3 Feb 2 05:58 /dev/umad7
dl585g2:~ # ls -l /dev/issm*
crw-rw---- 1 root root 231, 4 Feb 2 05:58 /dev/issm0
crw-rw---- 1 root root 231, 5 Feb 2 05:58 /dev/issm1
crw-rw---- 1 root root 231, 6 Feb 2 05:58 /dev/issm2
crw-rw---- 1 root root 231, 7 Feb 2 05:58 /dev/issm3
crw-rw---- 1 root root 249, 4 Feb 2 05:58 /dev/issm4
crw-rw---- 1 root root 249, 5 Feb 2 05:58 /dev/issm5
crw-rw---- 1 root root 249, 6 Feb 2 05:58 /dev/issm6
crw-rw---- 1 root root 249, 7 Feb 2 05:58 /dev/issm7
Finally, after one more reset, we see ib_umad loaded first and
obtaining the major number of 249.
v1 -> v2:
- update umad and ucm drivers too
---
Alex Chiang (18):
IB/uverbs: convert *cdev to cdev in struct ib_uverbs_device
IB/uverbs: remove dev_table
IB/uverbs: use stack variable 'devnum' in ib_uverbs_add_one
IB/uverbs: use stack variable 'base' in ib_uverbs_add_one
IB/uverbs: increase maximum devices supported
IB/uverbs: pack struct ib_uverbs_event_file tighter
IB/uverbs: whitespace cleanup
IB/umad: convert cdev pointers to embedded structs in struct ib_umad_port
IB/umad: remove port_table[]
IB/umad: use stack variable 'devnum' in ib_umad_init_port
IB/umad: use stack variable 'base' in ib_umad_init_port
IB/umad: increase maximum devices supported
IB/umad: clean whitespace
IB/ucm: use stack variable 'devnum' in ib_ucm_add_one
IB/ucm: use stack variable 'base' in ib_ucm_add_one
IB/ucm: increase maximum devices supported
IB/ucm: clean whitespace errors
IB/core: pack struct ib_device a little tighter
drivers/infiniband/core/ucm.c | 63 ++++++++++--
drivers/infiniband/core/user_mad.c | 173 +++++++++++++++++----------------
drivers/infiniband/core/uverbs.h | 11 +-
drivers/infiniband/core/uverbs_main.c | 175 +++++++++++++++++++--------------
include/rdma/ib_verbs.h | 4 -
5 files changed, 255 insertions(+), 171 deletions(-)
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majo...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Alex Chiang
Signed-off-by: Alex Chiang <ach...@hp.com>
---
drivers/infiniband/core/uverbs_main.c | 68 +++++++++++++++++----------------
1 files changed, 34 insertions(+), 34 deletions(-)
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index 38b9085..aff706e 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -79,34 +79,34 @@ static DECLARE_BITMAP(dev_map, IB_UVERBS_MAX_DEVICES);
static ssize_t (*uverbs_cmd_table[])(struct ib_uverbs_file *file,
const char __user *buf, int in_len,
int out_len) = {
- [IB_USER_VERBS_CMD_GET_CONTEXT] = ib_uverbs_get_context,
- [IB_USER_VERBS_CMD_QUERY_DEVICE] = ib_uverbs_query_device,
- [IB_USER_VERBS_CMD_QUERY_PORT] = ib_uverbs_query_port,
- [IB_USER_VERBS_CMD_ALLOC_PD] = ib_uverbs_alloc_pd,
- [IB_USER_VERBS_CMD_DEALLOC_PD] = ib_uverbs_dealloc_pd,
- [IB_USER_VERBS_CMD_REG_MR] = ib_uverbs_reg_mr,
- [IB_USER_VERBS_CMD_DEREG_MR] = ib_uverbs_dereg_mr,
+ [IB_USER_VERBS_CMD_GET_CONTEXT] = ib_uverbs_get_context,
+ [IB_USER_VERBS_CMD_QUERY_DEVICE] = ib_uverbs_query_device,
+ [IB_USER_VERBS_CMD_QUERY_PORT] = ib_uverbs_query_port,
+ [IB_USER_VERBS_CMD_ALLOC_PD] = ib_uverbs_alloc_pd,
+ [IB_USER_VERBS_CMD_DEALLOC_PD] = ib_uverbs_dealloc_pd,
+ [IB_USER_VERBS_CMD_REG_MR] = ib_uverbs_reg_mr,
+ [IB_USER_VERBS_CMD_DEREG_MR] = ib_uverbs_dereg_mr,
[IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL] = ib_uverbs_create_comp_channel,
- [IB_USER_VERBS_CMD_CREATE_CQ] = ib_uverbs_create_cq,
- [IB_USER_VERBS_CMD_RESIZE_CQ] = ib_uverbs_resize_cq,
- [IB_USER_VERBS_CMD_POLL_CQ] = ib_uverbs_poll_cq,
- [IB_USER_VERBS_CMD_REQ_NOTIFY_CQ] = ib_uverbs_req_notify_cq,
- [IB_USER_VERBS_CMD_DESTROY_CQ] = ib_uverbs_destroy_cq,
- [IB_USER_VERBS_CMD_CREATE_QP] = ib_uverbs_create_qp,
- [IB_USER_VERBS_CMD_QUERY_QP] = ib_uverbs_query_qp,
- [IB_USER_VERBS_CMD_MODIFY_QP] = ib_uverbs_modify_qp,
- [IB_USER_VERBS_CMD_DESTROY_QP] = ib_uverbs_destroy_qp,
- [IB_USER_VERBS_CMD_POST_SEND] = ib_uverbs_post_send,
- [IB_USER_VERBS_CMD_POST_RECV] = ib_uverbs_post_recv,
- [IB_USER_VERBS_CMD_POST_SRQ_RECV] = ib_uverbs_post_srq_recv,
- [IB_USER_VERBS_CMD_CREATE_AH] = ib_uverbs_create_ah,
- [IB_USER_VERBS_CMD_DESTROY_AH] = ib_uverbs_destroy_ah,
- [IB_USER_VERBS_CMD_ATTACH_MCAST] = ib_uverbs_attach_mcast,
- [IB_USER_VERBS_CMD_DETACH_MCAST] = ib_uverbs_detach_mcast,
- [IB_USER_VERBS_CMD_CREATE_SRQ] = ib_uverbs_create_srq,
- [IB_USER_VERBS_CMD_MODIFY_SRQ] = ib_uverbs_modify_srq,
- [IB_USER_VERBS_CMD_QUERY_SRQ] = ib_uverbs_query_srq,
- [IB_USER_VERBS_CMD_DESTROY_SRQ] = ib_uverbs_destroy_srq,
+ [IB_USER_VERBS_CMD_CREATE_CQ] = ib_uverbs_create_cq,
+ [IB_USER_VERBS_CMD_RESIZE_CQ] = ib_uverbs_resize_cq,
+ [IB_USER_VERBS_CMD_POLL_CQ] = ib_uverbs_poll_cq,
+ [IB_USER_VERBS_CMD_REQ_NOTIFY_CQ] = ib_uverbs_req_notify_cq,
+ [IB_USER_VERBS_CMD_DESTROY_CQ] = ib_uverbs_destroy_cq,
+ [IB_USER_VERBS_CMD_CREATE_QP] = ib_uverbs_create_qp,
+ [IB_USER_VERBS_CMD_QUERY_QP] = ib_uverbs_query_qp,
+ [IB_USER_VERBS_CMD_MODIFY_QP] = ib_uverbs_modify_qp,
+ [IB_USER_VERBS_CMD_DESTROY_QP] = ib_uverbs_destroy_qp,
+ [IB_USER_VERBS_CMD_POST_SEND] = ib_uverbs_post_send,
+ [IB_USER_VERBS_CMD_POST_RECV] = ib_uverbs_post_recv,
+ [IB_USER_VERBS_CMD_POST_SRQ_RECV] = ib_uverbs_post_srq_recv,
+ [IB_USER_VERBS_CMD_CREATE_AH] = ib_uverbs_create_ah,
+ [IB_USER_VERBS_CMD_DESTROY_AH] = ib_uverbs_destroy_ah,
+ [IB_USER_VERBS_CMD_ATTACH_MCAST] = ib_uverbs_attach_mcast,
+ [IB_USER_VERBS_CMD_DETACH_MCAST] = ib_uverbs_detach_mcast,
+ [IB_USER_VERBS_CMD_CREATE_SRQ] = ib_uverbs_create_srq,
+ [IB_USER_VERBS_CMD_MODIFY_SRQ] = ib_uverbs_modify_srq,
+ [IB_USER_VERBS_CMD_QUERY_SRQ] = ib_uverbs_query_srq,
+ [IB_USER_VERBS_CMD_DESTROY_SRQ] = ib_uverbs_destroy_srq,
};
static struct vfsmount *uverbs_event_mnt;
@@ -368,7 +368,7 @@ static int ib_uverbs_event_close(struct inode *inode, struct file *filp)
static const struct file_operations uverbs_event_fops = {
.owner = THIS_MODULE,
- .read = ib_uverbs_event_read,
+ .read = ib_uverbs_event_read,
.poll = ib_uverbs_event_poll,
.release = ib_uverbs_event_close,
.fasync = ib_uverbs_event_fasync
@@ -678,17 +678,17 @@ static int ib_uverbs_close(struct inode *inode, struct file *filp)
}
static const struct file_operations uverbs_fops = {
- .owner = THIS_MODULE,
- .write = ib_uverbs_write,
- .open = ib_uverbs_open,
+ .owner = THIS_MODULE,
+ .write = ib_uverbs_write,
+ .open = ib_uverbs_open,
.release = ib_uverbs_close
};
static const struct file_operations uverbs_mmap_fops = {
- .owner = THIS_MODULE,
- .write = ib_uverbs_write,
+ .owner = THIS_MODULE,
+ .write = ib_uverbs_write,
.mmap = ib_uverbs_mmap,
- .open = ib_uverbs_open,
+ .open = ib_uverbs_open,
.release = ib_uverbs_close
};
Alex Chiang
change that allows us to dynamically allocate device numbers
in case we have more than IB_UMAD_MAX_PORTS in the system.
Signed-off-by: Alex Chiang <ach...@hp.com>
---
drivers/infiniband/core/user_mad.c | 15 +++++++++------
1 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c
index 46c8879..df403ec 100644
--- a/drivers/infiniband/core/user_mad.c
+++ b/drivers/infiniband/core/user_mad.c
@@ -974,13 +974,16 @@ static CLASS_ATTR(abi_version, S_IRUGO, show_abi_version, NULL);
static int ib_umad_init_port(struct ib_device *device, int port_num,
struct ib_umad_port *port)
{
+ int devnum;
+
spin_lock(&port_lock);
- port->dev_num = find_first_zero_bit(dev_map, IB_UMAD_MAX_PORTS);
- if (port->dev_num >= IB_UMAD_MAX_PORTS) {
+ devnum = find_first_zero_bit(dev_map, IB_UMAD_MAX_PORTS);
+ if (devnum >= IB_UMAD_MAX_PORTS) {
spin_unlock(&port_lock);
return -1;
}
- set_bit(port->dev_num, dev_map);
+ port->dev_num = devnum;
+ set_bit(devnum, dev_map);
spin_unlock(&port_lock);
port->ib_dev = device;
@@ -992,7 +995,7 @@ static int ib_umad_init_port(struct ib_device *device, int port_num,
cdev_init(&port->cdev, &umad_fops);
port->cdev.owner = THIS_MODULE;
kobject_set_name(&port->cdev.kobj, "umad%d", port->dev_num);
- if (cdev_add(&port->cdev, base_dev + port->dev_num, 1))
+ if (cdev_add(&port->cdev, base_dev + devnum, 1))
goto err_cdev;
port->dev = device_create(umad_class, device->dma_device,
@@ -1009,7 +1012,7 @@ static int ib_umad_init_port(struct ib_device *device, int port_num,
cdev_init(&port->sm_cdev, &umad_sm_fops);
port->sm_cdev.owner = THIS_MODULE;
kobject_set_name(&port->sm_cdev.kobj, "issm%d", port->dev_num);
- if (cdev_add(&port->sm_cdev, base_dev + port->dev_num + IB_UMAD_MAX_PORTS, 1))
+ if (cdev_add(&port->sm_cdev, base_dev + devnum + IB_UMAD_MAX_PORTS, 1))
goto err_sm_cdev;
port->sm_dev = device_create(umad_class, device->dma_device,
@@ -1036,7 +1039,7 @@ err_dev:
err_cdev:
cdev_del(&port->cdev);
- clear_bit(port->dev_num, dev_map);
+ clear_bit(devnum, dev_map);
return -1;
Alex Chiang
sizeof(struct ib_uverbs_event_file) is now 72 bytes (from 80) and
more members now fit in the first cacheline.
Signed-off-by: Alex Chiang <ach...@hp.com>
---
drivers/infiniband/core/uverbs.h | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h
index e695f65..e54d9ac 100644
--- a/drivers/infiniband/core/uverbs.h
+++ b/drivers/infiniband/core/uverbs.h
@@ -80,13 +80,13 @@ struct ib_uverbs_device {
struct ib_uverbs_event_file {
struct kref ref;
+ int is_async;
struct ib_uverbs_file *uverbs_file;
spinlock_t lock;
+ int is_closed;
wait_queue_head_t poll_wait;
struct fasync_struct *async_queue;
struct list_head event_list;
- int is_async;
- int is_closed;
};
struct ib_uverbs_file {
Alex Chiang
an inode back to a struct ib_umad_port during ->open().
But now that we're embedding a struct cdev in struct ib_umad_port,
we can use the container_of() macro to go from the inode back to
the device instead.
Signed-off-by: Alex Chiang <ach...@hp.com>
---
drivers/infiniband/core/user_mad.c | 45 +++++++-----------------------------
1 files changed, 9 insertions(+), 36 deletions(-)
diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c
index 40440ef..46c8879 100644
--- a/drivers/infiniband/core/user_mad.c
+++ b/drivers/infiniband/core/user_mad.c
@@ -65,12 +65,9 @@ enum {
};
/*
- * Our lifetime rules for these structs are the following: each time a
- * device special file is opened, we look up the corresponding struct
- * ib_umad_port by minor in the umad_port[] table while holding the
- * port_lock. If this lookup succeeds, we take a reference on the
- * ib_umad_port's struct ib_umad_device while still holding the
- * port_lock; if the lookup fails, we fail the open(). We drop these
+ * Our lifetime rules for these structs are the following:
+ * device special file is opened, we take a reference on the
+ * ib_umad_port's struct ib_umad_device. We drop these
* references in the corresponding close().
*
* In addition to references coming from open character devices, there
@@ -78,12 +75,7 @@ enum {
* module's reference taken when allocating the ib_umad_device in
* ib_umad_add_one().
*
- * When destroying an ib_umad_device, we clear all of its
- * ib_umad_ports from umad_port[] while holding port_lock before
- * dropping the module's reference to the ib_umad_device. This is
- * always safe because any open() calls will either succeed and obtain
- * a reference before we clear the umad_port[] entries, or fail after
- * we clear the umad_port[] entries.
+ * When destroying an ib_umad_device, we drop the module's reference.
*/
struct ib_umad_port {
@@ -136,7 +128,6 @@ static struct class *umad_class;
static const dev_t base_dev = MKDEV(IB_UMAD_MAJOR, IB_UMAD_MINOR_BASE);
static DEFINE_SPINLOCK(port_lock);
-static struct ib_umad_port *umad_port[IB_UMAD_MAX_PORTS];
static DECLARE_BITMAP(dev_map, IB_UMAD_MAX_PORTS);
static void ib_umad_add_one(struct ib_device *device);
@@ -779,15 +770,11 @@ static long ib_umad_compat_ioctl(struct file *filp, unsigned int cmd,
/*
* ib_umad_open() does not need the BKL:
*
- * - umad_port[] accesses are protected by port_lock, the
- * ib_umad_port structures are properly reference counted, and
+ * - the ib_umad_port structures are properly reference counted, and
* everything else is purely local to the file being created, so
* races against other open calls are not a problem;
* - the ioctl method does not affect any global state outside of the
* file structure being operated on;
- * - the port is added to umad_port[] as the last part of module
- * initialization so the open method will either immediately run
- * -ENXIO, or all required initialization will be done.
*/
static int ib_umad_open(struct inode *inode, struct file *filp)
{
@@ -795,13 +782,10 @@ static int ib_umad_open(struct inode *inode, struct file *filp)
struct ib_umad_file *file;
int ret = 0;
- spin_lock(&port_lock);
- port = umad_port[iminor(inode) - IB_UMAD_MINOR_BASE];
+ port = container_of(inode->i_cdev, struct ib_umad_port, cdev);
if (port)
kref_get(&port->umad_dev->ref);
- spin_unlock(&port_lock);
-
- if (!port)
+ else
return -ENXIO;
mutex_lock(&port->file_mutex);
@@ -892,13 +876,10 @@ static int ib_umad_sm_open(struct inode *inode, struct file *filp)
};
int ret;
- spin_lock(&port_lock);
- port = umad_port[iminor(inode) - IB_UMAD_MINOR_BASE - IB_UMAD_MAX_PORTS];
+ port = container_of(inode->i_cdev, struct ib_umad_port, sm_cdev);
if (port)
kref_get(&port->umad_dev->ref);
- spin_unlock(&port_lock);
-
- if (!port)
+ else
return -ENXIO;
if (filp->f_flags & O_NONBLOCK) {
@@ -1042,10 +1023,6 @@ static int ib_umad_init_port(struct ib_device *device, int port_num,
if (device_create_file(port->sm_dev, &dev_attr_port))
goto err_sm_dev;
- spin_lock(&port_lock);
- umad_port[port->dev_num] = port;
- spin_unlock(&port_lock);
-
return 0;
err_sm_dev:
@@ -1079,10 +1056,6 @@ static void ib_umad_kill_port(struct ib_umad_port *port)
cdev_del(&port->cdev);
cdev_del(&port->sm_cdev);
- spin_lock(&port_lock);
- umad_port[port->dev_num] = NULL;
- spin_unlock(&port_lock);
-
mutex_lock(&port->file_mutex);
port->ib_dev = NULL;
Alex Chiang
(from 1128).
Signed-off-by: Alex Chiang <ach...@hp.com>
---
include/rdma/ib_verbs.h | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 09509ed..a585e0f 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -984,9 +984,9 @@ struct ib_device {
struct list_head event_handler_list;
spinlock_t event_handler_lock;
+ spinlock_t client_data_lock;
struct list_head core_list;
struct list_head client_data_list;
- spinlock_t client_data_lock;
struct ib_cache cache;
int *pkey_tbl_len;
@@ -1144,8 +1144,8 @@ struct ib_device {
IB_DEV_UNREGISTERED
} reg_state;
- u64 uverbs_cmd_mask;
int uverbs_abi_ver;
+ u64 uverbs_cmd_mask;
char node_desc[64];
__be64 node_guid;
Alex Chiang
Signed-off-by: Alex Chiang <ach...@hp.com>
---
drivers/infiniband/core/ucm.c | 4 +++-
1 files changed, 3 insertions(+), 1 deletions(-)
diff --git a/drivers/infiniband/core/ucm.c b/drivers/infiniband/core/ucm.c
index 7ff3300..06c50d8 100644
--- a/drivers/infiniband/core/ucm.c
+++ b/drivers/infiniband/core/ucm.c
@@ -1240,6 +1240,7 @@ static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
static void ib_ucm_add_one(struct ib_device *device)
{
int devnum;
+ dev_t base;
struct ib_ucm_device *ucm_dev;
if (!device->alloc_ucontext ||
@@ -1257,12 +1258,13 @@ static void ib_ucm_add_one(struct ib_device *device)
goto err;
ucm_dev->devnum = devnum;
+ base = devnum + IB_UCM_BASE_DEV;
set_bit(devnum, dev_map);
cdev_init(&ucm_dev->cdev, &ucm_fops);
ucm_dev->cdev.owner = THIS_MODULE;
kobject_set_name(&ucm_dev->cdev.kobj, "ucm%d", ucm_dev->devnum);
- if (cdev_add(&ucm_dev->cdev, IB_UCM_BASE_DEV + devnum, 1))
+ if (cdev_add(&ucm_dev->cdev, base, 1))
goto err;
ucm_dev->dev.class = &cm_class;
Alex Chiang
Signed-off-by: Alex Chiang <ach...@hp.com>
---
drivers/infiniband/core/user_mad.c | 7 +++++--
1 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c
index df403ec..5a66bd0 100644
--- a/drivers/infiniband/core/user_mad.c
+++ b/drivers/infiniband/core/user_mad.c
@@ -975,6 +975,7 @@ static int ib_umad_init_port(struct ib_device *device, int port_num,
struct ib_umad_port *port)
{
int devnum;
+ dev_t base;
spin_lock(&port_lock);
devnum = find_first_zero_bit(dev_map, IB_UMAD_MAX_PORTS);
@@ -983,6 +984,7 @@ static int ib_umad_init_port(struct ib_device *device, int port_num,
return -1;
}
port->dev_num = devnum;
+ base = devnum + base_dev;
set_bit(devnum, dev_map);
spin_unlock(&port_lock);
@@ -995,7 +997,7 @@ static int ib_umad_init_port(struct ib_device *device, int port_num,
cdev_init(&port->cdev, &umad_fops);
port->cdev.owner = THIS_MODULE;
kobject_set_name(&port->cdev.kobj, "umad%d", port->dev_num);
- if (cdev_add(&port->cdev, base_dev + devnum, 1))
+ if (cdev_add(&port->cdev, base, 1))
goto err_cdev;
port->dev = device_create(umad_class, device->dma_device,
@@ -1009,10 +1011,11 @@ static int ib_umad_init_port(struct ib_device *device, int port_num,
if (device_create_file(port->dev, &dev_attr_port))
goto err_dev;
+ base += IB_UMAD_MAX_PORTS;
cdev_init(&port->sm_cdev, &umad_sm_fops);
port->sm_cdev.owner = THIS_MODULE;
kobject_set_name(&port->sm_cdev.kobj, "issm%d", port->dev_num);
- if (cdev_add(&port->sm_cdev, base_dev + devnum + IB_UMAD_MAX_PORTS, 1))
+ if (cdev_add(&port->sm_cdev, base, 1))
goto err_sm_cdev;
port->sm_dev = device_create(umad_class, device->dma_device,
--
Alex Chiang
Signed-off-by: Alex Chiang <ach...@hp.com>
---
drivers/infiniband/core/ucm.c | 12 +++++++-----
1 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/drivers/infiniband/core/ucm.c b/drivers/infiniband/core/ucm.c
index f504c9b..7ff3300 100644
--- a/drivers/infiniband/core/ucm.c
+++ b/drivers/infiniband/core/ucm.c
@@ -1239,6 +1239,7 @@ static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
static void ib_ucm_add_one(struct ib_device *device)
{
+ int devnum;
struct ib_ucm_device *ucm_dev;
if (!device->alloc_ucontext ||
@@ -1251,16 +1252,17 @@ static void ib_ucm_add_one(struct ib_device *device)
ucm_dev->ib_dev = device;
- ucm_dev->devnum = find_first_zero_bit(dev_map, IB_UCM_MAX_DEVICES);
- if (ucm_dev->devnum >= IB_UCM_MAX_DEVICES)
+ devnum = find_first_zero_bit(dev_map, IB_UCM_MAX_DEVICES);
+ if (devnum >= IB_UCM_MAX_DEVICES)
goto err;
- set_bit(ucm_dev->devnum, dev_map);
+ ucm_dev->devnum = devnum;
+ set_bit(devnum, dev_map);
cdev_init(&ucm_dev->cdev, &ucm_fops);
ucm_dev->cdev.owner = THIS_MODULE;
kobject_set_name(&ucm_dev->cdev.kobj, "ucm%d", ucm_dev->devnum);
- if (cdev_add(&ucm_dev->cdev, IB_UCM_BASE_DEV + ucm_dev->devnum, 1))
+ if (cdev_add(&ucm_dev->cdev, IB_UCM_BASE_DEV + devnum, 1))
goto err;
ucm_dev->dev.class = &cm_class;
@@ -1281,7 +1283,7 @@ err_dev:
device_unregister(&ucm_dev->dev);
err_cdev:
cdev_del(&ucm_dev->cdev);
- clear_bit(ucm_dev->devnum, dev_map);
+ clear_bit(devnum, dev_map);
err:
kfree(ucm_dev);
return;
Alex Chiang
Signed-off-by: Alex Chiang <ach...@hp.com>
---
drivers/infiniband/core/ucm.c | 6 +++---
1 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/drivers/infiniband/core/ucm.c b/drivers/infiniband/core/ucm.c
index 1203ca4..7bf0a82 100644
--- a/drivers/infiniband/core/ucm.c
+++ b/drivers/infiniband/core/ucm.c
@@ -1223,10 +1223,10 @@ static void ib_ucm_release_dev(struct device *dev)
}
static const struct file_operations ucm_fops = {
- .owner = THIS_MODULE,
- .open = ib_ucm_open,
+ .owner = THIS_MODULE,
+ .open = ib_ucm_open,
.release = ib_ucm_close,
- .write = ib_ucm_write,
+ .write = ib_ucm_write,
.poll = ib_ucm_poll,
};
Alex Chiang
(currently 32).
This change allows us to support more devices in a backwards-compatible
manner. the first IB_UCM_MAX_DEVICES keep the same major/minor device
numbers they've always had.
If there are more than IB_UCM_MAX_DEVICES, then we dynamically request
a new major device number (new minors start at 0).
Signed-off-by: Alex Chiang <ach...@hp.com>
---
drivers/infiniband/core/ucm.c | 53 +++++++++++++++++++++++++++++++++++------
1 files changed, 45 insertions(+), 8 deletions(-)
diff --git a/drivers/infiniband/core/ucm.c b/drivers/infiniband/core/ucm.c
index 06c50d8..1203ca4 100644
--- a/drivers/infiniband/core/ucm.c
+++ b/drivers/infiniband/core/ucm.c
@@ -1215,7 +1215,10 @@ static void ib_ucm_release_dev(struct device *dev)
ucm_dev = container_of(dev, struct ib_ucm_device, dev);
cdev_del(&ucm_dev->cdev);
- clear_bit(ucm_dev->devnum, dev_map);
+ if (ucm_dev->devnum < IB_UCM_MAX_DEVICES)
+ clear_bit(ucm_dev->devnum, dev_map);
+ else
+ clear_bit(ucm_dev->devnum - IB_UCM_MAX_DEVICES, dev_map);
kfree(ucm_dev);
}
@@ -1237,6 +1240,28 @@ static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
}
static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
+static dev_t oflo_maj;
+static DECLARE_BITMAP(oflo_map, IB_UCM_MAX_DEVICES);
+static int find_oflo_devnum(void)
+{
+ int ret;
+
+ if (!oflo_maj) {
+ ret = alloc_chrdev_region(&oflo_maj, 0, IB_UCM_MAX_DEVICES,
+ "infiniband_cm");
+ if (ret) {
+ printk(KERN_ERR "ucm: couldn't register dynamic device number\n");
+ return ret;
+ }
+ }
+
+ ret = find_first_zero_bit(oflo_map, IB_UCM_MAX_DEVICES);
+ if (ret >= IB_UCM_MAX_DEVICES)
+ return -1;
+
+ return ret;
+}
+
static void ib_ucm_add_one(struct ib_device *device)
{
int devnum;
@@ -1254,12 +1279,19 @@ static void ib_ucm_add_one(struct ib_device *device)
ucm_dev->ib_dev = device;
devnum = find_first_zero_bit(dev_map, IB_UCM_MAX_DEVICES);
- if (devnum >= IB_UCM_MAX_DEVICES)
- goto err;
-
- ucm_dev->devnum = devnum;
- base = devnum + IB_UCM_BASE_DEV;
- set_bit(devnum, dev_map);
+ if (devnum >= IB_UCM_MAX_DEVICES) {
+ devnum = find_oflo_devnum();
+ if (devnum < 0)
+ goto err;
+
+ ucm_dev->devnum = devnum + IB_UCM_MAX_DEVICES;
+ base = devnum + oflo_maj;
+ set_bit(devnum, oflo_map);
+ } else {
+ ucm_dev->devnum = devnum;
+ base = devnum + IB_UCM_BASE_DEV;
+ set_bit(devnum, dev_map);
+ }
cdev_init(&ucm_dev->cdev, &ucm_fops);
ucm_dev->cdev.owner = THIS_MODULE;
@@ -1285,7 +1317,10 @@ err_dev:
device_unregister(&ucm_dev->dev);
err_cdev:
cdev_del(&ucm_dev->cdev);
- clear_bit(devnum, dev_map);
+ if (ucm_dev->devnum < IB_UCM_MAX_DEVICES)
+ clear_bit(devnum, dev_map);
+ else
+ clear_bit(devnum, oflo_map);
err:
kfree(ucm_dev);
return;
@@ -1344,6 +1379,8 @@ static void __exit ib_ucm_cleanup(void)
ib_unregister_client(&ucm_client);
class_remove_file(&cm_class, &class_attr_abi_version);
unregister_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_MAX_DEVICES);
+ if (oflo_maj)
+ unregister_chrdev_region(oflo_maj, IB_UCM_MAX_DEVICES);
idr_destroy(&ctx_id_table);
Alex Chiang
(currently 32).
This change allows us to support more devices in a backwards-compatible
manner. The first IB_UVERBS_MAX_DEVICES keep the same major/minor
device numbers that they've always had.
If there are more than IB_UVERBS_MAX_DEVICES, we then dynamically
request a new major device number (new minors start at 0).
This change increases the maximum number of HCAs to 64 (from 32).
Signed-off-by: Alex Chiang <ach...@hp.com>
---
drivers/infiniband/core/uverbs_main.c | 56 +++++++++++++++++++++++++++++----
1 files changed, 50 insertions(+), 6 deletions(-)
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index 8c594cd..38b9085 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -728,6 +728,34 @@ static ssize_t show_abi_version(struct class *class, char *buf)
}
static CLASS_ATTR(abi_version, S_IRUGO, show_abi_version, NULL);
+static dev_t oflo_maj;
+static DECLARE_BITMAP(oflo_map, IB_UVERBS_MAX_DEVICES);
+
+/*
+ * If we have more than IB_UVERBS_MAX_DEVICES, dynamically overflow by
+ * requesting a new major number and doubling the number of max devices we
+ * support. It's stupid, but simple.
+ */
+static int find_oflo_devnum(void)
+{
+ int ret;
+
+ if (!oflo_maj) {
+ ret = alloc_chrdev_region(&oflo_maj, 0, IB_UVERBS_MAX_DEVICES,
+ "infiniband_verbs");
+ if (ret) {
+ printk(KERN_ERR "user_verbs: couldn't register dynamic device number\n");
+ return ret;
+ }
+ }
+
+ ret = find_first_zero_bit(oflo_map, IB_UVERBS_MAX_DEVICES);
+ if (ret >= IB_UVERBS_MAX_DEVICES)
+ return -1;
+
+ return ret;
+}
+
static void ib_uverbs_add_one(struct ib_device *device)
{
int devnum;
@@ -748,11 +776,19 @@ static void ib_uverbs_add_one(struct ib_device *device)
devnum = find_first_zero_bit(dev_map, IB_UVERBS_MAX_DEVICES);
if (devnum >= IB_UVERBS_MAX_DEVICES) {
spin_unlock(&map_lock);
- goto err;
+ devnum = find_oflo_devnum();
+ if (devnum < 0)
+ goto err;
+
+ spin_lock(&map_lock);
+ uverbs_dev->devnum = devnum + IB_UVERBS_MAX_DEVICES;
+ base = devnum + oflo_maj;
+ set_bit(devnum, oflo_map);
+ } else {
+ uverbs_dev->devnum = devnum;
+ base = devnum + IB_UVERBS_BASE_DEV;
+ set_bit(devnum, dev_map);
}
- uverbs_dev->devnum = devnum;
- base = devnum + IB_UVERBS_BASE_DEV;
- set_bit(devnum, dev_map);
spin_unlock(&map_lock);
uverbs_dev->ib_dev = device;
@@ -785,7 +821,10 @@ err_class:
err_cdev:
cdev_del(&uverbs_dev->cdev);
- clear_bit(devnum, dev_map);
+ if (uverbs_dev->devnum < IB_UVERBS_MAX_DEVICES)
+ clear_bit(devnum, dev_map);
+ else
+ clear_bit(devnum, oflo_map);
err:
kref_put(&uverbs_dev->ref, ib_uverbs_release_dev);
@@ -805,7 +844,10 @@ static void ib_uverbs_remove_one(struct ib_device *device)
device_destroy(uverbs_class, uverbs_dev->cdev.dev);
cdev_del(&uverbs_dev->cdev);
- clear_bit(uverbs_dev->devnum, dev_map);
+ if (uverbs_dev->devnum < IB_UVERBS_MAX_DEVICES)
+ clear_bit(uverbs_dev->devnum, dev_map);
+ else
+ clear_bit(uverbs_dev->devnum - IB_UVERBS_MAX_DEVICES, oflo_map);
kref_put(&uverbs_dev->ref, ib_uverbs_release_dev);
wait_for_completion(&uverbs_dev->comp);
@@ -895,6 +937,8 @@ static void __exit ib_uverbs_cleanup(void)
unregister_filesystem(&uverbs_event_fs);
class_destroy(uverbs_class);
unregister_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES);
+ if (oflo_maj)
+ unregister_chrdev_region(oflo_maj, IB_UVERBS_MAX_DEVICES);
idr_destroy(&ib_uverbs_pd_idr);
idr_destroy(&ib_uverbs_mr_idr);
idr_destroy(&ib_uverbs_mw_idr);
Alex Chiang
Signed-off-by: Alex Chiang <ach...@hp.com>
---
drivers/infiniband/core/uverbs_main.c | 4 +++-
1 files changed, 3 insertions(+), 1 deletions(-)
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index acae9ed..8c594cd 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -731,6 +731,7 @@ static CLASS_ATTR(abi_version, S_IRUGO, show_abi_version, NULL);
static void ib_uverbs_add_one(struct ib_device *device)
{
int devnum;
+ dev_t base;
struct ib_uverbs_device *uverbs_dev;
if (!device->alloc_ucontext)
@@ -750,6 +751,7 @@ static void ib_uverbs_add_one(struct ib_device *device)
goto err;
}
uverbs_dev->devnum = devnum;
+ base = devnum + IB_UVERBS_BASE_DEV;
set_bit(devnum, dev_map);
spin_unlock(&map_lock);
@@ -760,7 +762,7 @@ static void ib_uverbs_add_one(struct ib_device *device)
uverbs_dev->cdev.owner = THIS_MODULE;
uverbs_dev->cdev.ops = device->mmap ? &uverbs_mmap_fops : &uverbs_fops;
kobject_set_name(&uverbs_dev->cdev.kobj, "uverbs%d", uverbs_dev->devnum);
- if (cdev_add(&uverbs_dev->cdev, IB_UVERBS_BASE_DEV + devnum, 1))
+ if (cdev_add(&uverbs_dev->cdev, base, 1))
goto err_cdev;
uverbs_dev->dev = device_create(uverbs_class, device->dma_device,
Alex Chiang
Roland Dreier
splitting this up nicely...
- R.
--
Roland Dreier <rol...@cisco.com>
For corporate legal information go to:
http://www.cisco.com/web/about/doing_business/legal/cri/index.html