[portals4] r2279 committed - Patch to allow for the disabling of the IB memory registration cache...

[port...@googlecode.com]

Revision: 2279
Author: regra...@gmail.com
Date: Wed May 28 20:03:11 2014 UTC
Log: Patch to allow for the disabling of the IB memory registration
cache

Set envvar PTL_DISABLE_MEM_REG_CACHE=1 to disable caching.

Disabling the cache removes the need for ummu_notify for prpoer operation.


http://code.google.com/p/portals4/source/detail?r=2279

Modified:
/trunk/README
/trunk/src/ib/ptl_init.c
/trunk/src/ib/ptl_misc.c
/trunk/src/ib/ptl_mr.c
/trunk/src/ib/ptl_param.c
/trunk/src/ib/ptl_param.h
/trunk/src/ib/ptl_tgt.c

=======================================
--- /trunk/README Sat Nov 23 04:26:38 2013 UTC
+++ /trunk/README Wed May 28 20:03:11 2014 UTC
@@ -119,6 +119,9 @@
* PTL_LOG_LEVEL=[0|1|2|3] will set the trace level.
* PTL_IFACE_NAME allows for the explicit naming of the network
interface
for example ib0, eno1, etc.
+ * PTL_DISABLE_MEM_REG_CACHE=[0|1] deactivates/activates the IB memory
+ registration cache. Disabling it no longer requires ummunotify, and
+ the implementation does not keep a registered memory cache.

For instance:
PTL_LOG_LEVEL=3 PTL_DEBUG=1 yod -n 1 ./spam
=======================================
--- /trunk/src/ib/ptl_init.c Fri Nov 15 16:59:59 2013 UTC
+++ /trunk/src/ib/ptl_init.c Wed May 28 20:03:11 2014 UTC
@@ -1108,6 +1108,21 @@
static void cleanup(buf_t *buf)
{

+#if WITH_TRANSPORT_IB && !IS_PPE
+ /* In the case where ummunotify is not installed */
+ ni_t *ni = obj_to_ni(buf);
+ if (ni->umn_fd == -1) {
+ if (buf->conn->transport.type == CONN_TYPE_RDMA) {
+ if (buf->get_md)
+ if (buf->get_md->mr_list)
+ mr_put(buf->get_md->mr_list[0]);
+ if (buf->put_md)
+ if (buf->put_md->mr_list)
+ mr_put(buf->put_md->mr_list[0]);
+ }
+ }
+#endif
+
if (buf->get_md) {
md_put(buf->get_md);
buf->get_md = NULL;
=======================================
--- /trunk/src/ib/ptl_misc.c Mon Oct 21 22:40:12 2013 UTC
+++ /trunk/src/ib/ptl_misc.c Wed May 28 20:03:11 2014 UTC
@@ -10,6 +10,7 @@
int debug;
int ptl_log_level;
char *ptl_iface_name;
+int ptl_disable_ummu;

unsigned long pagesize;
unsigned int linesize;
@@ -27,6 +28,7 @@
debug = get_param(PTL_DEBUG);
ptl_log_level = get_param(PTL_LOG_LEVEL);
ptl_iface_name = getenv("PTL_IFACE_NAME");
+ ptl_disable_ummu = get_param(PTL_DISABLE_MEM_REG_CACHE);
pagesize = sysconf(_SC_PAGESIZE);
#ifdef _SC_LEVEL1_DCACHE_LINESIZE
linesize = sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
@@ -66,6 +68,7 @@
debug = get_param(PTL_DEBUG);
ptl_log_level = get_param(PTL_LOG_LEVEL);
ptl_iface_name = getenv("PTL_IFACE_NAME");
+ ptl_disable_ummu = get_param(PTL_DISABLE_MEM_REG_CACHE);
pagesize = sysconf(_SC_PAGESIZE);
#ifdef _SC_LEVEL1_DCACHE_LINESIZE
linesize = sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
=======================================
--- /trunk/src/ib/ptl_mr.c Sat Nov 23 04:26:38 2013 UTC
+++ /trunk/src/ib/ptl_mr.c Wed May 28 20:03:11 2014 UTC
@@ -474,6 +474,7 @@
*/
void mr_init(ni_t *ni)
{
+ if (get_param(PTL_DISABLE_MEM_REG_CACHE) != 1) {
ni->umn_fd = open("/dev/ummunotify", O_RDONLY | O_NONBLOCK);
if (ni->umn_fd == -1) {
fprintf(stderr,
@@ -493,6 +494,10 @@
ni->umn_watcher.data = ni;
ev_io_init(&ni->umn_watcher, process_ummunotify, ni->umn_fd, EV_READ);
EVL_WATCH(ev_io_start(evl.loop, &ni->umn_watcher));
+ }
+ else {
+ fprintf(stderr, "NOTE: Ummunotify and IB registered mem cache
disabled, set PTL_DISABLE_MEM_REG_CACHE=0 to re-enable.\n");
+ }
}
#endif

@@ -511,6 +516,8 @@
for (mr = RB_MIN(the_root, &tree->tree); mr != NULL; mr = next_mr) {
next_mr = RB_NEXT(the_root, &tree->tree, mr);
RB_REMOVE(the_root, &tree->tree, mr);
+ //account for the case where no active mrs are on the list
+ if (atomic_read(&mr->obj.obj_ref.ref_cnt) > 1)
mr_put(mr);
}

=======================================
--- /trunk/src/ib/ptl_param.c Tue Apr 1 14:37:21 2014 UTC
+++ /trunk/src/ib/ptl_param.c Wed May 28 20:03:11 2014 UTC
@@ -273,6 +273,12 @@
.max = 10000000,
.val = 8 * 4096,
},
+ [PTL_DISABLE_MEM_REG_CACHE] = {
+ .name = "PTL_DISABLE_MEM_REG_CACHE",
+ .min = 0,
+ .max = 1,
+ .val = 0,
+ },
};

/**
=======================================
--- /trunk/src/ib/ptl_param.h Tue Apr 1 14:37:21 2014 UTC
+++ /trunk/src/ib/ptl_param.h Wed May 28 20:03:11 2014 UTC
@@ -53,7 +53,7 @@
PTL_ENABLE_MEM,
PTL_BOUNCE_NUM_BUFS,
PTL_BOUNCE_BUF_SIZE,
-
+ PTL_DISABLE_MEM_REG_CACHE,
PTL_PARAM_LAST, /* keep me last */
};

=======================================
--- /trunk/src/ib/ptl_tgt.c Fri Mar 28 15:20:24 2014 UTC
+++ /trunk/src/ib/ptl_tgt.c Wed May 28 20:03:11 2014 UTC
@@ -1934,6 +1934,21 @@
atomic_set(&buf->matching.le->busy, 0);
buf->matching.le = NULL;
}
+
+#if WITH_TRANSPORT_IB && !IS_PPE
+ if(buf->conn->transport.type == CONN_TYPE_RDMA){
+ ni_t *ni = obj_to_ni(buf);
+ if(buf->mr_list[0] != NULL && ni->umn_fd == -1){
+ int i = 0;
+ while (buf->mr_list[i] != NULL){
+ mr_put(buf->mr_list[i]);
+ i++;
+ if (i == 2)
+ abort();
+ }
+ }
+ }
+#endif

if (buf->conn) {
conn_put(buf->conn);