[portals4] r2278 committed - Allow IB transport to set the max message size based on device capabil...

[port...@googlecode.com]

Revision: 2278
Author: regra...@gmail.com
Date: Tue Apr 1 14:37:21 2014 UTC
Log: Allow IB transport to set the max message size based on device
capability
http://code.google.com/p/portals4/source/detail?r=2278

Modified:
/trunk/src/ib/ptl_iface_ib.c
/trunk/src/ib/ptl_ni.c
/trunk/src/ib/ptl_param.c
/trunk/src/ib/ptl_param.h
/trunk/test/basic/test_amo.c

=======================================
--- /trunk/src/ib/ptl_iface_ib.c Thu Nov 14 20:20:35 2013 UTC
+++ /trunk/src/ib/ptl_iface_ib.c Tue Apr 1 14:37:21 2014 UTC
@@ -14,6 +14,7 @@
static int query_rdma_interface(iface_t *iface)
{
int ret;
+ struct ibv_port_attr port_attr;

ret = ibv_query_device(iface->ibv_context, &iface->cap.device_attr);
if (ret)
@@ -37,6 +38,14 @@
min(get_param(PTL_MAX_SRQ_RECV_WR),
iface->cap.device_attr.max_srq_wr);

+ ret = ibv_query_port(iface->ibv_context, iface->listen_id->port_num,
+ &port_attr);
+ if (ret)
+ return ret;
+ set_param_max(PTL_LIM_MAX_MSG_SIZE, port_attr.max_msg_sz);
+
+ ptl_info("max msg size is: %u",port_attr.max_msg_sz);
+
return 0;
}

=======================================
--- /trunk/src/ib/ptl_ni.c Wed Oct 2 21:26:45 2013 UTC
+++ /trunk/src/ib/ptl_ni.c Tue Apr 1 14:37:21 2014 UTC
@@ -310,7 +310,7 @@
atomic_set(&ni->ref_cnt, 1);
ni->options = options;
ni->last_pt = -1;
- set_limits(ni, desired);
+
#ifndef HAVE_KITTEN
ni->uid = geteuid();
#endif
@@ -343,14 +343,6 @@
}

mr_init(ni);
-
- /* Note: pt range is [0..max_pt_index]. */
- ni->pt = calloc(ni->limits.max_pt_index + 1, sizeof(*ni->pt));
- if (unlikely(!ni->pt)) {
- WARN();
- err = PTL_NO_SPACE;
- goto err3;
- }

err = init_pools(ni);
if (unlikely(err))
@@ -378,6 +370,18 @@
goto err3;
}
}
+
+ /* Set limits now that we know the transports limits */
+ set_limits(ni, desired);
+
+ /* Note: pt range is [0..max_pt_index]. */
+ ni->pt = calloc(ni->limits.max_pt_index + 1, sizeof(*ni->pt));
+ if (unlikely(!ni->pt)) {
+ WARN();
+ err = PTL_NO_SPACE;
+ goto err3;
+ }
+

/* Add a progress thread. */
err = start_progress_thread(ni);
=======================================
--- /trunk/src/ib/ptl_param.c Wed Oct 2 21:26:45 2013 UTC
+++ /trunk/src/ib/ptl_param.c Tue Apr 1 14:37:21 2014 UTC
@@ -374,3 +374,24 @@

return p->val;
}
+
+/**
+ * @brief Set the max value given a requested value.
+ *
+ * This is currently used to feed back device maximum
+ * values to cap portals default maximums in the case
+ * that a device does not support the default max.
+ *
+ * @param[in] The parameter ID.
+ * @param[in] The requested value.
+ *
+ * @return The current value.
+ */
+unsigned long set_param_max(int parm, long val)
+{
+ param_t *p;
+ assert(parm < PTL_PARAM_LAST);
+ p = &param[parm];
+ p->max = val;
+ return p->max;
+}
=======================================
--- /trunk/src/ib/ptl_param.h Wed Oct 2 21:26:45 2013 UTC
+++ /trunk/src/ib/ptl_param.h Tue Apr 1 14:37:21 2014 UTC
@@ -116,5 +116,18 @@
{
return param[parm].val;
}
+
+/**
+ * @brief Set the maximum allowable value of a parameter.
+ *
+ * This call sets the parameter max value.
+ *
+ * @param[in] the index of the parameter.
+ *
+ * @param[in] the value to set the param max to.
+ *
+ * @return the max value set
+ */
+unsigned long set_param_max(int parm, long val);

#endif /* PTL_PARAM_H */
=======================================
--- /trunk/test/basic/test_amo.c Thu Oct 3 22:56:24 2013 UTC
+++ /trunk/test/basic/test_amo.c Tue Apr 1 14:37:21 2014 UTC
@@ -10,7 +10,7 @@

#include "testing.h"

-const int tries = 1000000;
+const int tries = 100000;
typedef int32_t locktype;