[PATCH] IB/iSER: Release connection resources when receiving a RDMA_CM_EVENT_DEVICE_REMOVAL event

[Erez Zilber]

When a RDMA_CM_EVENT_DEVICE_REMOVAL event is raised, iSER should release the
connection resources except for the rdma cm id (which will be released by the
cma itself).

This behavior is necessary if IB modules are unloaded while open-iscsi is still
running. Currently, iSER just initiates a BUG() call.

Signed-off-by: Erez Zilber <er...@voltaire.com>
---
drivers/infiniband/ulp/iser/iscsi_iser.h | 2 ++
drivers/infiniband/ulp/iser/iser_verbs.c | 18 ++++++++++++++----
2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h
index 1ee867b..9fe0b3f 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.h
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.h
@@ -249,6 +249,8 @@ struct iser_conn {
struct iser_page_vec *page_vec; /* represents SG to fmr maps*
* maps serialized as tx is*/
struct list_head conn_list; /* entry in ig conn list */
+ wait_queue_head_t rem_wait;
+ int dev_removed;
};

struct iscsi_iser_conn {
diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index 993f0a8..9beddb9 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -219,7 +219,8 @@ static int iser_free_ib_conn_res(struct iser_conn *ib_conn)
if (ib_conn->qp != NULL)
rdma_destroy_qp(ib_conn->cma_id);

- if (ib_conn->cma_id != NULL)
+ /* if the device was removed, the cma will call rdma_destroy_id itself */
+ if (ib_conn->cma_id != NULL && !ib_conn->dev_removed)
rdma_destroy_id(ib_conn->cma_id);

ib_conn->fmr_pool = NULL;
@@ -325,7 +326,10 @@ static void iser_conn_release(struct iser_conn *ib_conn)
iser_device_try_release(device);
if (ib_conn->iser_conn)
ib_conn->iser_conn->ib_conn = NULL;
- kfree(ib_conn);
+ if (ib_conn->dev_removed)
+ wake_up_interruptible(&ib_conn->rem_wait);
+ else
+ kfree(ib_conn);
}

/**
@@ -451,6 +455,7 @@ static void iser_disconnected_handler(struct rdma_cm_id *cma_id)
static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
{
int ret = 0;
+ struct iser_conn *ib_conn;

iser_err("event %d conn %p id %p\n",event->event,cma_id->context,cma_id);

@@ -476,8 +481,12 @@ static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *eve
iser_disconnected_handler(cma_id);
break;
case RDMA_CM_EVENT_DEVICE_REMOVAL:
- iser_err("Device removal is currently unsupported\n");
- BUG();
+ ib_conn = (struct iser_conn *)cma_id->context;
+ ib_conn->dev_removed = 1;
+ iser_disconnected_handler(cma_id);
+ wait_event_interruptible(ib_conn->rem_wait, ib_conn->state == ISER_CONN_DOWN);
+ kfree(ib_conn);
+ ret = 1;
break;
default:
iser_err("Unexpected RDMA CM event (%d)\n", event->event);
@@ -497,6 +506,7 @@ int iser_conn_init(struct iser_conn **ibconn)
}
ib_conn->state = ISER_CONN_INIT;
init_waitqueue_head(&ib_conn->wait);
+ init_waitqueue_head(&ib_conn->rem_wait);
atomic_set(&ib_conn->post_recv_buf_count, 0);
atomic_set(&ib_conn->post_send_buf_count, 0);
INIT_LIST_HEAD(&ib_conn->conn_list);
--
1.5.3.6

Roland,

This patch was built against your 2.6.26 branch. Can you add it to your list?

Thanks,
Erez

[Erez Zilber]

When a RDMA_CM_EVENT_DEVICE_REMOVAL event is raised, iSER should release the

connection resources.

This is necessary when the IB HCA module is unloaded while open-iscsi is still

running. Currently, iSER just initiates a BUG() call.

Signed-off-by: Erez Zilber <er...@voltaire.com>
---

drivers/infiniband/ulp/iser/iser_verbs.c | 5 +----
1 files changed, 1 insertions(+), 4 deletions(-)

diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index 993f0a8..d19cfe6 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -473,11 +473,8 @@ static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *eve
iser_connect_error(cma_id);
break;
case RDMA_CM_EVENT_DISCONNECTED:
- iser_disconnected_handler(cma_id);
- break;

case RDMA_CM_EVENT_DEVICE_REMOVAL:
- iser_err("Device removal is currently unsupported\n");
- BUG();

+ iser_disconnected_handler(cma_id);

break;
default:
iser_err("Unexpected RDMA CM event (%d)\n", event->event);

--
1.5.3.6

[Erez Zilber]

Roland Dreier wrote:

> thanks, applied... I assume this much simpler patch replaces the earlier
> one completely?
>

Yes (that's why I added "v2" in the subject).

Thanks,
Erez