/drivers/infiniband/core/cm.c

https://bitbucket.org/ndreys/linux-sunxi · C · 3875 lines · 3297 code · 482 blank · 96 comment · 442 complexity · 9ba594543d5101f200f1c1152ab8cdaf MD5 · raw file

Large files are truncated click here to view the full file

  1. /*
  2. * Copyright (c) 2004-2007 Intel Corporation. All rights reserved.
  3. * Copyright (c) 2004 Topspin Corporation. All rights reserved.
  4. * Copyright (c) 2004, 2005 Voltaire Corporation. All rights reserved.
  5. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
  6. *
  7. * This software is available to you under a choice of one of two
  8. * licenses. You may choose to be licensed under the terms of the GNU
  9. * General Public License (GPL) Version 2, available from the file
  10. * COPYING in the main directory of this source tree, or the
  11. * OpenIB.org BSD license below:
  12. *
  13. * Redistribution and use in source and binary forms, with or
  14. * without modification, are permitted provided that the following
  15. * conditions are met:
  16. *
  17. * - Redistributions of source code must retain the above
  18. * copyright notice, this list of conditions and the following
  19. * disclaimer.
  20. *
  21. * - Redistributions in binary form must reproduce the above
  22. * copyright notice, this list of conditions and the following
  23. * disclaimer in the documentation and/or other materials
  24. * provided with the distribution.
  25. *
  26. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33. * SOFTWARE.
  34. */
  35. #include <linux/completion.h>
  36. #include <linux/dma-mapping.h>
  37. #include <linux/device.h>
  38. #include <linux/err.h>
  39. #include <linux/idr.h>
  40. #include <linux/interrupt.h>
  41. #include <linux/random.h>
  42. #include <linux/rbtree.h>
  43. #include <linux/spinlock.h>
  44. #include <linux/slab.h>
  45. #include <linux/sysfs.h>
  46. #include <linux/workqueue.h>
  47. #include <linux/kdev_t.h>
  48. #include <rdma/ib_cache.h>
  49. #include <rdma/ib_cm.h>
  50. #include "cm_msgs.h"
  51. MODULE_AUTHOR("Sean Hefty");
  52. MODULE_DESCRIPTION("InfiniBand CM");
  53. MODULE_LICENSE("Dual BSD/GPL");
  54. static void cm_add_one(struct ib_device *device);
  55. static void cm_remove_one(struct ib_device *device);
  56. static struct ib_client cm_client = {
  57. .name = "cm",
  58. .add = cm_add_one,
  59. .remove = cm_remove_one
  60. };
  61. static struct ib_cm {
  62. spinlock_t lock;
  63. struct list_head device_list;
  64. rwlock_t device_lock;
  65. struct rb_root listen_service_table;
  66. u64 listen_service_id;
  67. /* struct rb_root peer_service_table; todo: fix peer to peer */
  68. struct rb_root remote_qp_table;
  69. struct rb_root remote_id_table;
  70. struct rb_root remote_sidr_table;
  71. struct idr local_id_table;
  72. __be32 random_id_operand;
  73. struct list_head timewait_list;
  74. struct workqueue_struct *wq;
  75. } cm;
  76. /* Counter indexes ordered by attribute ID */
  77. enum {
  78. CM_REQ_COUNTER,
  79. CM_MRA_COUNTER,
  80. CM_REJ_COUNTER,
  81. CM_REP_COUNTER,
  82. CM_RTU_COUNTER,
  83. CM_DREQ_COUNTER,
  84. CM_DREP_COUNTER,
  85. CM_SIDR_REQ_COUNTER,
  86. CM_SIDR_REP_COUNTER,
  87. CM_LAP_COUNTER,
  88. CM_APR_COUNTER,
  89. CM_ATTR_COUNT,
  90. CM_ATTR_ID_OFFSET = 0x0010,
  91. };
  92. enum {
  93. CM_XMIT,
  94. CM_XMIT_RETRIES,
  95. CM_RECV,
  96. CM_RECV_DUPLICATES,
  97. CM_COUNTER_GROUPS
  98. };
  99. static char const counter_group_names[CM_COUNTER_GROUPS]
  100. [sizeof("cm_rx_duplicates")] = {
  101. "cm_tx_msgs", "cm_tx_retries",
  102. "cm_rx_msgs", "cm_rx_duplicates"
  103. };
  104. struct cm_counter_group {
  105. struct kobject obj;
  106. atomic_long_t counter[CM_ATTR_COUNT];
  107. };
  108. struct cm_counter_attribute {
  109. struct attribute attr;
  110. int index;
  111. };
  112. #define CM_COUNTER_ATTR(_name, _index) \
  113. struct cm_counter_attribute cm_##_name##_counter_attr = { \
  114. .attr = { .name = __stringify(_name), .mode = 0444 }, \
  115. .index = _index \
  116. }
  117. static CM_COUNTER_ATTR(req, CM_REQ_COUNTER);
  118. static CM_COUNTER_ATTR(mra, CM_MRA_COUNTER);
  119. static CM_COUNTER_ATTR(rej, CM_REJ_COUNTER);
  120. static CM_COUNTER_ATTR(rep, CM_REP_COUNTER);
  121. static CM_COUNTER_ATTR(rtu, CM_RTU_COUNTER);
  122. static CM_COUNTER_ATTR(dreq, CM_DREQ_COUNTER);
  123. static CM_COUNTER_ATTR(drep, CM_DREP_COUNTER);
  124. static CM_COUNTER_ATTR(sidr_req, CM_SIDR_REQ_COUNTER);
  125. static CM_COUNTER_ATTR(sidr_rep, CM_SIDR_REP_COUNTER);
  126. static CM_COUNTER_ATTR(lap, CM_LAP_COUNTER);
  127. static CM_COUNTER_ATTR(apr, CM_APR_COUNTER);
  128. static struct attribute *cm_counter_default_attrs[] = {
  129. &cm_req_counter_attr.attr,
  130. &cm_mra_counter_attr.attr,
  131. &cm_rej_counter_attr.attr,
  132. &cm_rep_counter_attr.attr,
  133. &cm_rtu_counter_attr.attr,
  134. &cm_dreq_counter_attr.attr,
  135. &cm_drep_counter_attr.attr,
  136. &cm_sidr_req_counter_attr.attr,
  137. &cm_sidr_rep_counter_attr.attr,
  138. &cm_lap_counter_attr.attr,
  139. &cm_apr_counter_attr.attr,
  140. NULL
  141. };
  142. struct cm_port {
  143. struct cm_device *cm_dev;
  144. struct ib_mad_agent *mad_agent;
  145. struct kobject port_obj;
  146. u8 port_num;
  147. struct cm_counter_group counter_group[CM_COUNTER_GROUPS];
  148. };
  149. struct cm_device {
  150. struct list_head list;
  151. struct ib_device *ib_device;
  152. struct device *device;
  153. u8 ack_delay;
  154. struct cm_port *port[0];
  155. };
  156. struct cm_av {
  157. struct cm_port *port;
  158. union ib_gid dgid;
  159. struct ib_ah_attr ah_attr;
  160. u16 pkey_index;
  161. u8 timeout;
  162. };
  163. struct cm_work {
  164. struct delayed_work work;
  165. struct list_head list;
  166. struct cm_port *port;
  167. struct ib_mad_recv_wc *mad_recv_wc; /* Received MADs */
  168. __be32 local_id; /* Established / timewait */
  169. __be32 remote_id;
  170. struct ib_cm_event cm_event;
  171. struct ib_sa_path_rec path[0];
  172. };
  173. struct cm_timewait_info {
  174. struct cm_work work; /* Must be first. */
  175. struct list_head list;
  176. struct rb_node remote_qp_node;
  177. struct rb_node remote_id_node;
  178. __be64 remote_ca_guid;
  179. __be32 remote_qpn;
  180. u8 inserted_remote_qp;
  181. u8 inserted_remote_id;
  182. };
  183. struct cm_id_private {
  184. struct ib_cm_id id;
  185. struct rb_node service_node;
  186. struct rb_node sidr_id_node;
  187. spinlock_t lock; /* Do not acquire inside cm.lock */
  188. struct completion comp;
  189. atomic_t refcount;
  190. struct ib_mad_send_buf *msg;
  191. struct cm_timewait_info *timewait_info;
  192. /* todo: use alternate port on send failure */
  193. struct cm_av av;
  194. struct cm_av alt_av;
  195. struct ib_cm_compare_data *compare_data;
  196. void *private_data;
  197. __be64 tid;
  198. __be32 local_qpn;
  199. __be32 remote_qpn;
  200. enum ib_qp_type qp_type;
  201. __be32 sq_psn;
  202. __be32 rq_psn;
  203. int timeout_ms;
  204. enum ib_mtu path_mtu;
  205. __be16 pkey;
  206. u8 private_data_len;
  207. u8 max_cm_retries;
  208. u8 peer_to_peer;
  209. u8 responder_resources;
  210. u8 initiator_depth;
  211. u8 retry_count;
  212. u8 rnr_retry_count;
  213. u8 service_timeout;
  214. u8 target_ack_delay;
  215. struct list_head work_list;
  216. atomic_t work_count;
  217. };
  218. static void cm_work_handler(struct work_struct *work);
  219. static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
  220. {
  221. if (atomic_dec_and_test(&cm_id_priv->refcount))
  222. complete(&cm_id_priv->comp);
  223. }
  224. static int cm_alloc_msg(struct cm_id_private *cm_id_priv,
  225. struct ib_mad_send_buf **msg)
  226. {
  227. struct ib_mad_agent *mad_agent;
  228. struct ib_mad_send_buf *m;
  229. struct ib_ah *ah;
  230. mad_agent = cm_id_priv->av.port->mad_agent;
  231. ah = ib_create_ah(mad_agent->qp->pd, &cm_id_priv->av.ah_attr);
  232. if (IS_ERR(ah))
  233. return PTR_ERR(ah);
  234. m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
  235. cm_id_priv->av.pkey_index,
  236. 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
  237. GFP_ATOMIC);
  238. if (IS_ERR(m)) {
  239. ib_destroy_ah(ah);
  240. return PTR_ERR(m);
  241. }
  242. /* Timeout set by caller if response is expected. */
  243. m->ah = ah;
  244. m->retries = cm_id_priv->max_cm_retries;
  245. atomic_inc(&cm_id_priv->refcount);
  246. m->context[0] = cm_id_priv;
  247. *msg = m;
  248. return 0;
  249. }
  250. static int cm_alloc_response_msg(struct cm_port *port,
  251. struct ib_mad_recv_wc *mad_recv_wc,
  252. struct ib_mad_send_buf **msg)
  253. {
  254. struct ib_mad_send_buf *m;
  255. struct ib_ah *ah;
  256. ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc,
  257. mad_recv_wc->recv_buf.grh, port->port_num);
  258. if (IS_ERR(ah))
  259. return PTR_ERR(ah);
  260. m = ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
  261. 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
  262. GFP_ATOMIC);
  263. if (IS_ERR(m)) {
  264. ib_destroy_ah(ah);
  265. return PTR_ERR(m);
  266. }
  267. m->ah = ah;
  268. *msg = m;
  269. return 0;
  270. }
  271. static void cm_free_msg(struct ib_mad_send_buf *msg)
  272. {
  273. ib_destroy_ah(msg->ah);
  274. if (msg->context[0])
  275. cm_deref_id(msg->context[0]);
  276. ib_free_send_mad(msg);
  277. }
  278. static void * cm_copy_private_data(const void *private_data,
  279. u8 private_data_len)
  280. {
  281. void *data;
  282. if (!private_data || !private_data_len)
  283. return NULL;
  284. data = kmemdup(private_data, private_data_len, GFP_KERNEL);
  285. if (!data)
  286. return ERR_PTR(-ENOMEM);
  287. return data;
  288. }
  289. static void cm_set_private_data(struct cm_id_private *cm_id_priv,
  290. void *private_data, u8 private_data_len)
  291. {
  292. if (cm_id_priv->private_data && cm_id_priv->private_data_len)
  293. kfree(cm_id_priv->private_data);
  294. cm_id_priv->private_data = private_data;
  295. cm_id_priv->private_data_len = private_data_len;
  296. }
  297. static void cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
  298. struct ib_grh *grh, struct cm_av *av)
  299. {
  300. av->port = port;
  301. av->pkey_index = wc->pkey_index;
  302. ib_init_ah_from_wc(port->cm_dev->ib_device, port->port_num, wc,
  303. grh, &av->ah_attr);
  304. }
  305. static int cm_init_av_by_path(struct ib_sa_path_rec *path, struct cm_av *av)
  306. {
  307. struct cm_device *cm_dev;
  308. struct cm_port *port = NULL;
  309. unsigned long flags;
  310. int ret;
  311. u8 p;
  312. read_lock_irqsave(&cm.device_lock, flags);
  313. list_for_each_entry(cm_dev, &cm.device_list, list) {
  314. if (!ib_find_cached_gid(cm_dev->ib_device, &path->sgid,
  315. &p, NULL)) {
  316. port = cm_dev->port[p-1];
  317. break;
  318. }
  319. }
  320. read_unlock_irqrestore(&cm.device_lock, flags);
  321. if (!port)
  322. return -EINVAL;
  323. ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num,
  324. be16_to_cpu(path->pkey), &av->pkey_index);
  325. if (ret)
  326. return ret;
  327. av->port = port;
  328. ib_init_ah_from_path(cm_dev->ib_device, port->port_num, path,
  329. &av->ah_attr);
  330. av->timeout = path->packet_life_time + 1;
  331. return 0;
  332. }
  333. static int cm_alloc_id(struct cm_id_private *cm_id_priv)
  334. {
  335. unsigned long flags;
  336. int ret, id;
  337. static int next_id;
  338. do {
  339. spin_lock_irqsave(&cm.lock, flags);
  340. ret = idr_get_new_above(&cm.local_id_table, cm_id_priv,
  341. next_id, &id);
  342. if (!ret)
  343. next_id = ((unsigned) id + 1) & MAX_ID_MASK;
  344. spin_unlock_irqrestore(&cm.lock, flags);
  345. } while( (ret == -EAGAIN) && idr_pre_get(&cm.local_id_table, GFP_KERNEL) );
  346. cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
  347. return ret;
  348. }
  349. static void cm_free_id(__be32 local_id)
  350. {
  351. spin_lock_irq(&cm.lock);
  352. idr_remove(&cm.local_id_table,
  353. (__force int) (local_id ^ cm.random_id_operand));
  354. spin_unlock_irq(&cm.lock);
  355. }
  356. static struct cm_id_private * cm_get_id(__be32 local_id, __be32 remote_id)
  357. {
  358. struct cm_id_private *cm_id_priv;
  359. cm_id_priv = idr_find(&cm.local_id_table,
  360. (__force int) (local_id ^ cm.random_id_operand));
  361. if (cm_id_priv) {
  362. if (cm_id_priv->id.remote_id == remote_id)
  363. atomic_inc(&cm_id_priv->refcount);
  364. else
  365. cm_id_priv = NULL;
  366. }
  367. return cm_id_priv;
  368. }
  369. static struct cm_id_private * cm_acquire_id(__be32 local_id, __be32 remote_id)
  370. {
  371. struct cm_id_private *cm_id_priv;
  372. spin_lock_irq(&cm.lock);
  373. cm_id_priv = cm_get_id(local_id, remote_id);
  374. spin_unlock_irq(&cm.lock);
  375. return cm_id_priv;
  376. }
  377. static void cm_mask_copy(u8 *dst, u8 *src, u8 *mask)
  378. {
  379. int i;
  380. for (i = 0; i < IB_CM_COMPARE_SIZE / sizeof(unsigned long); i++)
  381. ((unsigned long *) dst)[i] = ((unsigned long *) src)[i] &
  382. ((unsigned long *) mask)[i];
  383. }
  384. static int cm_compare_data(struct ib_cm_compare_data *src_data,
  385. struct ib_cm_compare_data *dst_data)
  386. {
  387. u8 src[IB_CM_COMPARE_SIZE];
  388. u8 dst[IB_CM_COMPARE_SIZE];
  389. if (!src_data || !dst_data)
  390. return 0;
  391. cm_mask_copy(src, src_data->data, dst_data->mask);
  392. cm_mask_copy(dst, dst_data->data, src_data->mask);
  393. return memcmp(src, dst, IB_CM_COMPARE_SIZE);
  394. }
  395. static int cm_compare_private_data(u8 *private_data,
  396. struct ib_cm_compare_data *dst_data)
  397. {
  398. u8 src[IB_CM_COMPARE_SIZE];
  399. if (!dst_data)
  400. return 0;
  401. cm_mask_copy(src, private_data, dst_data->mask);
  402. return memcmp(src, dst_data->data, IB_CM_COMPARE_SIZE);
  403. }
  404. /*
  405. * Trivial helpers to strip endian annotation and compare; the
  406. * endianness doesn't actually matter since we just need a stable
  407. * order for the RB tree.
  408. */
  409. static int be32_lt(__be32 a, __be32 b)
  410. {
  411. return (__force u32) a < (__force u32) b;
  412. }
  413. static int be32_gt(__be32 a, __be32 b)
  414. {
  415. return (__force u32) a > (__force u32) b;
  416. }
  417. static int be64_lt(__be64 a, __be64 b)
  418. {
  419. return (__force u64) a < (__force u64) b;
  420. }
  421. static int be64_gt(__be64 a, __be64 b)
  422. {
  423. return (__force u64) a > (__force u64) b;
  424. }
  425. static struct cm_id_private * cm_insert_listen(struct cm_id_private *cm_id_priv)
  426. {
  427. struct rb_node **link = &cm.listen_service_table.rb_node;
  428. struct rb_node *parent = NULL;
  429. struct cm_id_private *cur_cm_id_priv;
  430. __be64 service_id = cm_id_priv->id.service_id;
  431. __be64 service_mask = cm_id_priv->id.service_mask;
  432. int data_cmp;
  433. while (*link) {
  434. parent = *link;
  435. cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
  436. service_node);
  437. data_cmp = cm_compare_data(cm_id_priv->compare_data,
  438. cur_cm_id_priv->compare_data);
  439. if ((cur_cm_id_priv->id.service_mask & service_id) ==
  440. (service_mask & cur_cm_id_priv->id.service_id) &&
  441. (cm_id_priv->id.device == cur_cm_id_priv->id.device) &&
  442. !data_cmp)
  443. return cur_cm_id_priv;
  444. if (cm_id_priv->id.device < cur_cm_id_priv->id.device)
  445. link = &(*link)->rb_left;
  446. else if (cm_id_priv->id.device > cur_cm_id_priv->id.device)
  447. link = &(*link)->rb_right;
  448. else if (be64_lt(service_id, cur_cm_id_priv->id.service_id))
  449. link = &(*link)->rb_left;
  450. else if (be64_gt(service_id, cur_cm_id_priv->id.service_id))
  451. link = &(*link)->rb_right;
  452. else if (data_cmp < 0)
  453. link = &(*link)->rb_left;
  454. else
  455. link = &(*link)->rb_right;
  456. }
  457. rb_link_node(&cm_id_priv->service_node, parent, link);
  458. rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table);
  459. return NULL;
  460. }
  461. static struct cm_id_private * cm_find_listen(struct ib_device *device,
  462. __be64 service_id,
  463. u8 *private_data)
  464. {
  465. struct rb_node *node = cm.listen_service_table.rb_node;
  466. struct cm_id_private *cm_id_priv;
  467. int data_cmp;
  468. while (node) {
  469. cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
  470. data_cmp = cm_compare_private_data(private_data,
  471. cm_id_priv->compare_data);
  472. if ((cm_id_priv->id.service_mask & service_id) ==
  473. cm_id_priv->id.service_id &&
  474. (cm_id_priv->id.device == device) && !data_cmp)
  475. return cm_id_priv;
  476. if (device < cm_id_priv->id.device)
  477. node = node->rb_left;
  478. else if (device > cm_id_priv->id.device)
  479. node = node->rb_right;
  480. else if (be64_lt(service_id, cm_id_priv->id.service_id))
  481. node = node->rb_left;
  482. else if (be64_gt(service_id, cm_id_priv->id.service_id))
  483. node = node->rb_right;
  484. else if (data_cmp < 0)
  485. node = node->rb_left;
  486. else
  487. node = node->rb_right;
  488. }
  489. return NULL;
  490. }
  491. static struct cm_timewait_info * cm_insert_remote_id(struct cm_timewait_info
  492. *timewait_info)
  493. {
  494. struct rb_node **link = &cm.remote_id_table.rb_node;
  495. struct rb_node *parent = NULL;
  496. struct cm_timewait_info *cur_timewait_info;
  497. __be64 remote_ca_guid = timewait_info->remote_ca_guid;
  498. __be32 remote_id = timewait_info->work.remote_id;
  499. while (*link) {
  500. parent = *link;
  501. cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
  502. remote_id_node);
  503. if (be32_lt(remote_id, cur_timewait_info->work.remote_id))
  504. link = &(*link)->rb_left;
  505. else if (be32_gt(remote_id, cur_timewait_info->work.remote_id))
  506. link = &(*link)->rb_right;
  507. else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
  508. link = &(*link)->rb_left;
  509. else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
  510. link = &(*link)->rb_right;
  511. else
  512. return cur_timewait_info;
  513. }
  514. timewait_info->inserted_remote_id = 1;
  515. rb_link_node(&timewait_info->remote_id_node, parent, link);
  516. rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table);
  517. return NULL;
  518. }
  519. static struct cm_timewait_info * cm_find_remote_id(__be64 remote_ca_guid,
  520. __be32 remote_id)
  521. {
  522. struct rb_node *node = cm.remote_id_table.rb_node;
  523. struct cm_timewait_info *timewait_info;
  524. while (node) {
  525. timewait_info = rb_entry(node, struct cm_timewait_info,
  526. remote_id_node);
  527. if (be32_lt(remote_id, timewait_info->work.remote_id))
  528. node = node->rb_left;
  529. else if (be32_gt(remote_id, timewait_info->work.remote_id))
  530. node = node->rb_right;
  531. else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid))
  532. node = node->rb_left;
  533. else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid))
  534. node = node->rb_right;
  535. else
  536. return timewait_info;
  537. }
  538. return NULL;
  539. }
  540. static struct cm_timewait_info * cm_insert_remote_qpn(struct cm_timewait_info
  541. *timewait_info)
  542. {
  543. struct rb_node **link = &cm.remote_qp_table.rb_node;
  544. struct rb_node *parent = NULL;
  545. struct cm_timewait_info *cur_timewait_info;
  546. __be64 remote_ca_guid = timewait_info->remote_ca_guid;
  547. __be32 remote_qpn = timewait_info->remote_qpn;
  548. while (*link) {
  549. parent = *link;
  550. cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
  551. remote_qp_node);
  552. if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn))
  553. link = &(*link)->rb_left;
  554. else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn))
  555. link = &(*link)->rb_right;
  556. else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
  557. link = &(*link)->rb_left;
  558. else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
  559. link = &(*link)->rb_right;
  560. else
  561. return cur_timewait_info;
  562. }
  563. timewait_info->inserted_remote_qp = 1;
  564. rb_link_node(&timewait_info->remote_qp_node, parent, link);
  565. rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table);
  566. return NULL;
  567. }
  568. static struct cm_id_private * cm_insert_remote_sidr(struct cm_id_private
  569. *cm_id_priv)
  570. {
  571. struct rb_node **link = &cm.remote_sidr_table.rb_node;
  572. struct rb_node *parent = NULL;
  573. struct cm_id_private *cur_cm_id_priv;
  574. union ib_gid *port_gid = &cm_id_priv->av.dgid;
  575. __be32 remote_id = cm_id_priv->id.remote_id;
  576. while (*link) {
  577. parent = *link;
  578. cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
  579. sidr_id_node);
  580. if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id))
  581. link = &(*link)->rb_left;
  582. else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id))
  583. link = &(*link)->rb_right;
  584. else {
  585. int cmp;
  586. cmp = memcmp(port_gid, &cur_cm_id_priv->av.dgid,
  587. sizeof *port_gid);
  588. if (cmp < 0)
  589. link = &(*link)->rb_left;
  590. else if (cmp > 0)
  591. link = &(*link)->rb_right;
  592. else
  593. return cur_cm_id_priv;
  594. }
  595. }
  596. rb_link_node(&cm_id_priv->sidr_id_node, parent, link);
  597. rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
  598. return NULL;
  599. }
  600. static void cm_reject_sidr_req(struct cm_id_private *cm_id_priv,
  601. enum ib_cm_sidr_status status)
  602. {
  603. struct ib_cm_sidr_rep_param param;
  604. memset(&param, 0, sizeof param);
  605. param.status = status;
  606. ib_send_cm_sidr_rep(&cm_id_priv->id, &param);
  607. }
  608. struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
  609. ib_cm_handler cm_handler,
  610. void *context)
  611. {
  612. struct cm_id_private *cm_id_priv;
  613. int ret;
  614. cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL);
  615. if (!cm_id_priv)
  616. return ERR_PTR(-ENOMEM);
  617. cm_id_priv->id.state = IB_CM_IDLE;
  618. cm_id_priv->id.device = device;
  619. cm_id_priv->id.cm_handler = cm_handler;
  620. cm_id_priv->id.context = context;
  621. cm_id_priv->id.remote_cm_qpn = 1;
  622. ret = cm_alloc_id(cm_id_priv);
  623. if (ret)
  624. goto error;
  625. spin_lock_init(&cm_id_priv->lock);
  626. init_completion(&cm_id_priv->comp);
  627. INIT_LIST_HEAD(&cm_id_priv->work_list);
  628. atomic_set(&cm_id_priv->work_count, -1);
  629. atomic_set(&cm_id_priv->refcount, 1);
  630. return &cm_id_priv->id;
  631. error:
  632. kfree(cm_id_priv);
  633. return ERR_PTR(-ENOMEM);
  634. }
  635. EXPORT_SYMBOL(ib_create_cm_id);
  636. static struct cm_work * cm_dequeue_work(struct cm_id_private *cm_id_priv)
  637. {
  638. struct cm_work *work;
  639. if (list_empty(&cm_id_priv->work_list))
  640. return NULL;
  641. work = list_entry(cm_id_priv->work_list.next, struct cm_work, list);
  642. list_del(&work->list);
  643. return work;
  644. }
  645. static void cm_free_work(struct cm_work *work)
  646. {
  647. if (work->mad_recv_wc)
  648. ib_free_recv_mad(work->mad_recv_wc);
  649. kfree(work);
  650. }
  651. static inline int cm_convert_to_ms(int iba_time)
  652. {
  653. /* approximate conversion to ms from 4.096us x 2^iba_time */
  654. return 1 << max(iba_time - 8, 0);
  655. }
  656. /*
  657. * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time
  658. * Because of how ack_timeout is stored, adding one doubles the timeout.
  659. * To avoid large timeouts, select the max(ack_delay, life_time + 1), and
  660. * increment it (round up) only if the other is within 50%.
  661. */
  662. static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
  663. {
  664. int ack_timeout = packet_life_time + 1;
  665. if (ack_timeout >= ca_ack_delay)
  666. ack_timeout += (ca_ack_delay >= (ack_timeout - 1));
  667. else
  668. ack_timeout = ca_ack_delay +
  669. (ack_timeout >= (ca_ack_delay - 1));
  670. return min(31, ack_timeout);
  671. }
  672. static void cm_cleanup_timewait(struct cm_timewait_info *timewait_info)
  673. {
  674. if (timewait_info->inserted_remote_id) {
  675. rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
  676. timewait_info->inserted_remote_id = 0;
  677. }
  678. if (timewait_info->inserted_remote_qp) {
  679. rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table);
  680. timewait_info->inserted_remote_qp = 0;
  681. }
  682. }
  683. static struct cm_timewait_info * cm_create_timewait_info(__be32 local_id)
  684. {
  685. struct cm_timewait_info *timewait_info;
  686. timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL);
  687. if (!timewait_info)
  688. return ERR_PTR(-ENOMEM);
  689. timewait_info->work.local_id = local_id;
  690. INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler);
  691. timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT;
  692. return timewait_info;
  693. }
  694. static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
  695. {
  696. int wait_time;
  697. unsigned long flags;
  698. spin_lock_irqsave(&cm.lock, flags);
  699. cm_cleanup_timewait(cm_id_priv->timewait_info);
  700. list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list);
  701. spin_unlock_irqrestore(&cm.lock, flags);
  702. /*
  703. * The cm_id could be destroyed by the user before we exit timewait.
  704. * To protect against this, we search for the cm_id after exiting
  705. * timewait before notifying the user that we've exited timewait.
  706. */
  707. cm_id_priv->id.state = IB_CM_TIMEWAIT;
  708. wait_time = cm_convert_to_ms(cm_id_priv->av.timeout);
  709. queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work,
  710. msecs_to_jiffies(wait_time));
  711. cm_id_priv->timewait_info = NULL;
  712. }
  713. static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
  714. {
  715. unsigned long flags;
  716. cm_id_priv->id.state = IB_CM_IDLE;
  717. if (cm_id_priv->timewait_info) {
  718. spin_lock_irqsave(&cm.lock, flags);
  719. cm_cleanup_timewait(cm_id_priv->timewait_info);
  720. spin_unlock_irqrestore(&cm.lock, flags);
  721. kfree(cm_id_priv->timewait_info);
  722. cm_id_priv->timewait_info = NULL;
  723. }
  724. }
  725. static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
  726. {
  727. struct cm_id_private *cm_id_priv;
  728. struct cm_work *work;
  729. cm_id_priv = container_of(cm_id, struct cm_id_private, id);
  730. retest:
  731. spin_lock_irq(&cm_id_priv->lock);
  732. switch (cm_id->state) {
  733. case IB_CM_LISTEN:
  734. cm_id->state = IB_CM_IDLE;
  735. spin_unlock_irq(&cm_id_priv->lock);
  736. spin_lock_irq(&cm.lock);
  737. rb_erase(&cm_id_priv->service_node, &cm.listen_service_table);
  738. spin_unlock_irq(&cm.lock);
  739. break;
  740. case IB_CM_SIDR_REQ_SENT:
  741. cm_id->state = IB_CM_IDLE;
  742. ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
  743. spin_unlock_irq(&cm_id_priv->lock);
  744. break;
  745. case IB_CM_SIDR_REQ_RCVD:
  746. spin_unlock_irq(&cm_id_priv->lock);
  747. cm_reject_sidr_req(cm_id_priv, IB_SIDR_REJECT);
  748. break;
  749. case IB_CM_REQ_SENT:
  750. ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
  751. spin_unlock_irq(&cm_id_priv->lock);
  752. ib_send_cm_rej(cm_id, IB_CM_REJ_TIMEOUT,
  753. &cm_id_priv->id.device->node_guid,
  754. sizeof cm_id_priv->id.device->node_guid,
  755. NULL, 0);
  756. break;
  757. case IB_CM_REQ_RCVD:
  758. if (err == -ENOMEM) {
  759. /* Do not reject to allow future retries. */
  760. cm_reset_to_idle(cm_id_priv);
  761. spin_unlock_irq(&cm_id_priv->lock);
  762. } else {
  763. spin_unlock_irq(&cm_id_priv->lock);
  764. ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,
  765. NULL, 0, NULL, 0);
  766. }
  767. break;
  768. case IB_CM_MRA_REQ_RCVD:
  769. case IB_CM_REP_SENT:
  770. case IB_CM_MRA_REP_RCVD:
  771. ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
  772. /* Fall through */
  773. case IB_CM_MRA_REQ_SENT:
  774. case IB_CM_REP_RCVD:
  775. case IB_CM_MRA_REP_SENT:
  776. spin_unlock_irq(&cm_id_priv->lock);
  777. ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,
  778. NULL, 0, NULL, 0);
  779. break;
  780. case IB_CM_ESTABLISHED:
  781. spin_unlock_irq(&cm_id_priv->lock);
  782. ib_send_cm_dreq(cm_id, NULL, 0);
  783. goto retest;
  784. case IB_CM_DREQ_SENT:
  785. ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
  786. cm_enter_timewait(cm_id_priv);
  787. spin_unlock_irq(&cm_id_priv->lock);
  788. break;
  789. case IB_CM_DREQ_RCVD:
  790. spin_unlock_irq(&cm_id_priv->lock);
  791. ib_send_cm_drep(cm_id, NULL, 0);
  792. break;
  793. default:
  794. spin_unlock_irq(&cm_id_priv->lock);
  795. break;
  796. }
  797. cm_free_id(cm_id->local_id);
  798. cm_deref_id(cm_id_priv);
  799. wait_for_completion(&cm_id_priv->comp);
  800. while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
  801. cm_free_work(work);
  802. kfree(cm_id_priv->compare_data);
  803. kfree(cm_id_priv->private_data);
  804. kfree(cm_id_priv);
  805. }
  806. void ib_destroy_cm_id(struct ib_cm_id *cm_id)
  807. {
  808. cm_destroy_id(cm_id, 0);
  809. }
  810. EXPORT_SYMBOL(ib_destroy_cm_id);
  811. int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, __be64 service_mask,
  812. struct ib_cm_compare_data *compare_data)
  813. {
  814. struct cm_id_private *cm_id_priv, *cur_cm_id_priv;
  815. unsigned long flags;
  816. int ret = 0;
  817. service_mask = service_mask ? service_mask : ~cpu_to_be64(0);
  818. service_id &= service_mask;
  819. if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &&
  820. (service_id != IB_CM_ASSIGN_SERVICE_ID))
  821. return -EINVAL;
  822. cm_id_priv = container_of(cm_id, struct cm_id_private, id);
  823. if (cm_id->state != IB_CM_IDLE)
  824. return -EINVAL;
  825. if (compare_data) {
  826. cm_id_priv->compare_data = kzalloc(sizeof *compare_data,
  827. GFP_KERNEL);
  828. if (!cm_id_priv->compare_data)
  829. return -ENOMEM;
  830. cm_mask_copy(cm_id_priv->compare_data->data,
  831. compare_data->data, compare_data->mask);
  832. memcpy(cm_id_priv->compare_data->mask, compare_data->mask,
  833. IB_CM_COMPARE_SIZE);
  834. }
  835. cm_id->state = IB_CM_LISTEN;
  836. spin_lock_irqsave(&cm.lock, flags);
  837. if (service_id == IB_CM_ASSIGN_SERVICE_ID) {
  838. cm_id->service_id = cpu_to_be64(cm.listen_service_id++);
  839. cm_id->service_mask = ~cpu_to_be64(0);
  840. } else {
  841. cm_id->service_id = service_id;
  842. cm_id->service_mask = service_mask;
  843. }
  844. cur_cm_id_priv = cm_insert_listen(cm_id_priv);
  845. spin_unlock_irqrestore(&cm.lock, flags);
  846. if (cur_cm_id_priv) {
  847. cm_id->state = IB_CM_IDLE;
  848. kfree(cm_id_priv->compare_data);
  849. cm_id_priv->compare_data = NULL;
  850. ret = -EBUSY;
  851. }
  852. return ret;
  853. }
  854. EXPORT_SYMBOL(ib_cm_listen);
  855. static __be64 cm_form_tid(struct cm_id_private *cm_id_priv,
  856. enum cm_msg_sequence msg_seq)
  857. {
  858. u64 hi_tid, low_tid;
  859. hi_tid = ((u64) cm_id_priv->av.port->mad_agent->hi_tid) << 32;
  860. low_tid = (u64) ((__force u32)cm_id_priv->id.local_id |
  861. (msg_seq << 30));
  862. return cpu_to_be64(hi_tid | low_tid);
  863. }
  864. static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
  865. __be16 attr_id, __be64 tid)
  866. {
  867. hdr->base_version = IB_MGMT_BASE_VERSION;
  868. hdr->mgmt_class = IB_MGMT_CLASS_CM;
  869. hdr->class_version = IB_CM_CLASS_VERSION;
  870. hdr->method = IB_MGMT_METHOD_SEND;
  871. hdr->attr_id = attr_id;
  872. hdr->tid = tid;
  873. }
  874. static void cm_format_req(struct cm_req_msg *req_msg,
  875. struct cm_id_private *cm_id_priv,
  876. struct ib_cm_req_param *param)
  877. {
  878. struct ib_sa_path_rec *pri_path = param->primary_path;
  879. struct ib_sa_path_rec *alt_path = param->alternate_path;
  880. cm_format_mad_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
  881. cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_REQ));
  882. req_msg->local_comm_id = cm_id_priv->id.local_id;
  883. req_msg->service_id = param->service_id;
  884. req_msg->local_ca_guid = cm_id_priv->id.device->node_guid;
  885. cm_req_set_local_qpn(req_msg, cpu_to_be32(param->qp_num));
  886. cm_req_set_resp_res(req_msg, param->responder_resources);
  887. cm_req_set_init_depth(req_msg, param->initiator_depth);
  888. cm_req_set_remote_resp_timeout(req_msg,
  889. param->remote_cm_response_timeout);
  890. cm_req_set_qp_type(req_msg, param->qp_type);
  891. cm_req_set_flow_ctrl(req_msg, param->flow_control);
  892. cm_req_set_starting_psn(req_msg, cpu_to_be32(param->starting_psn));
  893. cm_req_set_local_resp_timeout(req_msg,
  894. param->local_cm_response_timeout);
  895. cm_req_set_retry_count(req_msg, param->retry_count);
  896. req_msg->pkey = param->primary_path->pkey;
  897. cm_req_set_path_mtu(req_msg, param->primary_path->mtu);
  898. cm_req_set_rnr_retry_count(req_msg, param->rnr_retry_count);
  899. cm_req_set_max_cm_retries(req_msg, param->max_cm_retries);
  900. cm_req_set_srq(req_msg, param->srq);
  901. if (pri_path->hop_limit <= 1) {
  902. req_msg->primary_local_lid = pri_path->slid;
  903. req_msg->primary_remote_lid = pri_path->dlid;
  904. } else {
  905. /* Work-around until there's a way to obtain remote LID info */
  906. req_msg->primary_local_lid = IB_LID_PERMISSIVE;
  907. req_msg->primary_remote_lid = IB_LID_PERMISSIVE;
  908. }
  909. req_msg->primary_local_gid = pri_path->sgid;
  910. req_msg->primary_remote_gid = pri_path->dgid;
  911. cm_req_set_primary_flow_label(req_msg, pri_path->flow_label);
  912. cm_req_set_primary_packet_rate(req_msg, pri_path->rate);
  913. req_msg->primary_traffic_class = pri_path->traffic_class;
  914. req_msg->primary_hop_limit = pri_path->hop_limit;
  915. cm_req_set_primary_sl(req_msg, pri_path->sl);
  916. cm_req_set_primary_subnet_local(req_msg, (pri_path->hop_limit <= 1));
  917. cm_req_set_primary_local_ack_timeout(req_msg,
  918. cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
  919. pri_path->packet_life_time));
  920. if (alt_path) {
  921. if (alt_path->hop_limit <= 1) {
  922. req_msg->alt_local_lid = alt_path->slid;
  923. req_msg->alt_remote_lid = alt_path->dlid;
  924. } else {
  925. req_msg->alt_local_lid = IB_LID_PERMISSIVE;
  926. req_msg->alt_remote_lid = IB_LID_PERMISSIVE;
  927. }
  928. req_msg->alt_local_gid = alt_path->sgid;
  929. req_msg->alt_remote_gid = alt_path->dgid;
  930. cm_req_set_alt_flow_label(req_msg,
  931. alt_path->flow_label);
  932. cm_req_set_alt_packet_rate(req_msg, alt_path->rate);
  933. req_msg->alt_traffic_class = alt_path->traffic_class;
  934. req_msg->alt_hop_limit = alt_path->hop_limit;
  935. cm_req_set_alt_sl(req_msg, alt_path->sl);
  936. cm_req_set_alt_subnet_local(req_msg, (alt_path->hop_limit <= 1));
  937. cm_req_set_alt_local_ack_timeout(req_msg,
  938. cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
  939. alt_path->packet_life_time));
  940. }
  941. if (param->private_data && param->private_data_len)
  942. memcpy(req_msg->private_data, param->private_data,
  943. param->private_data_len);
  944. }
  945. static int cm_validate_req_param(struct ib_cm_req_param *param)
  946. {
  947. /* peer-to-peer not supported */
  948. if (param->peer_to_peer)
  949. return -EINVAL;
  950. if (!param->primary_path)
  951. return -EINVAL;
  952. if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC)
  953. return -EINVAL;
  954. if (param->private_data &&
  955. param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE)
  956. return -EINVAL;
  957. if (param->alternate_path &&
  958. (param->alternate_path->pkey != param->primary_path->pkey ||
  959. param->alternate_path->mtu != param->primary_path->mtu))
  960. return -EINVAL;
  961. return 0;
  962. }
  963. int ib_send_cm_req(struct ib_cm_id *cm_id,
  964. struct ib_cm_req_param *param)
  965. {
  966. struct cm_id_private *cm_id_priv;
  967. struct cm_req_msg *req_msg;
  968. unsigned long flags;
  969. int ret;
  970. ret = cm_validate_req_param(param);
  971. if (ret)
  972. return ret;
  973. /* Verify that we're not in timewait. */
  974. cm_id_priv = container_of(cm_id, struct cm_id_private, id);
  975. spin_lock_irqsave(&cm_id_priv->lock, flags);
  976. if (cm_id->state != IB_CM_IDLE) {
  977. spin_unlock_irqrestore(&cm_id_priv->lock, flags);
  978. ret = -EINVAL;
  979. goto out;
  980. }
  981. spin_unlock_irqrestore(&cm_id_priv->lock, flags);
  982. cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
  983. id.local_id);
  984. if (IS_ERR(cm_id_priv->timewait_info)) {
  985. ret = PTR_ERR(cm_id_priv->timewait_info);
  986. goto out;
  987. }
  988. ret = cm_init_av_by_path(param->primary_path, &cm_id_priv->av);
  989. if (ret)
  990. goto error1;
  991. if (param->alternate_path) {
  992. ret = cm_init_av_by_path(param->alternate_path,
  993. &cm_id_priv->alt_av);
  994. if (ret)
  995. goto error1;
  996. }
  997. cm_id->service_id = param->service_id;
  998. cm_id->service_mask = ~cpu_to_be64(0);
  999. cm_id_priv->timeout_ms = cm_convert_to_ms(
  1000. param->primary_path->packet_life_time) * 2 +
  1001. cm_convert_to_ms(
  1002. param->remote_cm_response_timeout);
  1003. cm_id_priv->max_cm_retries = param->max_cm_retries;
  1004. cm_id_priv->initiator_depth = param->initiator_depth;
  1005. cm_id_priv->responder_resources = param->responder_resources;
  1006. cm_id_priv->retry_count = param->retry_count;
  1007. cm_id_priv->path_mtu = param->primary_path->mtu;
  1008. cm_id_priv->pkey = param->primary_path->pkey;
  1009. cm_id_priv->qp_type = param->qp_type;
  1010. ret = cm_alloc_msg(cm_id_priv, &cm_id_priv->msg);
  1011. if (ret)
  1012. goto error1;
  1013. req_msg = (struct cm_req_msg *) cm_id_priv->msg->mad;
  1014. cm_format_req(req_msg, cm_id_priv, param);
  1015. cm_id_priv->tid = req_msg->hdr.tid;
  1016. cm_id_priv->msg->timeout_ms = cm_id_priv->timeout_ms;
  1017. cm_id_priv->msg->context[1] = (void *) (unsigned long) IB_CM_REQ_SENT;
  1018. cm_id_priv->local_qpn = cm_req_get_local_qpn(req_msg);
  1019. cm_id_priv->rq_psn = cm_req_get_starting_psn(req_msg);
  1020. spin_lock_irqsave(&cm_id_priv->lock, flags);
  1021. ret = ib_post_send_mad(cm_id_priv->msg, NULL);
  1022. if (ret) {
  1023. spin_unlock_irqrestore(&cm_id_priv->lock, flags);
  1024. goto error2;
  1025. }
  1026. BUG_ON(cm_id->state != IB_CM_IDLE);
  1027. cm_id->state = IB_CM_REQ_SENT;
  1028. spin_unlock_irqrestore(&cm_id_priv->lock, flags);
  1029. return 0;
  1030. error2: cm_free_msg(cm_id_priv->msg);
  1031. error1: kfree(cm_id_priv->timewait_info);
  1032. out: return ret;
  1033. }
  1034. EXPORT_SYMBOL(ib_send_cm_req);
  1035. static int cm_issue_rej(struct cm_port *port,
  1036. struct ib_mad_recv_wc *mad_recv_wc,
  1037. enum ib_cm_rej_reason reason,
  1038. enum cm_msg_response msg_rejected,
  1039. void *ari, u8 ari_length)
  1040. {
  1041. struct ib_mad_send_buf *msg = NULL;
  1042. struct cm_rej_msg *rej_msg, *rcv_msg;
  1043. int ret;
  1044. ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
  1045. if (ret)
  1046. return ret;
  1047. /* We just need common CM header information. Cast to any message. */
  1048. rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad;
  1049. rej_msg = (struct cm_rej_msg *) msg->mad;
  1050. cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid);
  1051. rej_msg->remote_comm_id = rcv_msg->local_comm_id;
  1052. rej_msg->local_comm_id = rcv_msg->remote_comm_id;
  1053. cm_rej_set_msg_rejected(rej_msg, msg_rejected);
  1054. rej_msg->reason = cpu_to_be16(reason);
  1055. if (ari && ari_length) {
  1056. cm_rej_set_reject_info_len(rej_msg, ari_length);
  1057. memcpy(rej_msg->ari, ari, ari_length);
  1058. }
  1059. ret = ib_post_send_mad(msg, NULL);
  1060. if (ret)
  1061. cm_free_msg(msg);
  1062. return ret;
  1063. }
  1064. static inline int cm_is_active_peer(__be64 local_ca_guid, __be64 remote_ca_guid,
  1065. __be32 local_qpn, __be32 remote_qpn)
  1066. {
  1067. return (be64_to_cpu(local_ca_guid) > be64_to_cpu(remote_ca_guid) ||
  1068. ((local_ca_guid == remote_ca_guid) &&
  1069. (be32_to_cpu(local_qpn) > be32_to_cpu(remote_qpn))));
  1070. }
  1071. static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
  1072. struct ib_sa_path_rec *primary_path,
  1073. struct ib_sa_path_rec *alt_path)
  1074. {
  1075. memset(primary_path, 0, sizeof *primary_path);
  1076. primary_path->dgid = req_msg->primary_local_gid;
  1077. primary_path->sgid = req_msg->primary_remote_gid;
  1078. primary_path->dlid = req_msg->primary_local_lid;
  1079. primary_path->slid = req_msg->primary_remote_lid;
  1080. primary_path->flow_label = cm_req_get_primary_flow_label(req_msg);
  1081. primary_path->hop_limit = req_msg->primary_hop_limit;
  1082. primary_path->traffic_class = req_msg->primary_traffic_class;
  1083. primary_path->reversible = 1;
  1084. primary_path->pkey = req_msg->pkey;
  1085. primary_path->sl = cm_req_get_primary_sl(req_msg);
  1086. primary_path->mtu_selector = IB_SA_EQ;
  1087. primary_path->mtu = cm_req_get_path_mtu(req_msg);
  1088. primary_path->rate_selector = IB_SA_EQ;
  1089. primary_path->rate = cm_req_get_primary_packet_rate(req_msg);
  1090. primary_path->packet_life_time_selector = IB_SA_EQ;
  1091. primary_path->packet_life_time =
  1092. cm_req_get_primary_local_ack_timeout(req_msg);
  1093. primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
  1094. if (req_msg->alt_local_lid) {
  1095. memset(alt_path, 0, sizeof *alt_path);
  1096. alt_path->dgid = req_msg->alt_local_gid;
  1097. alt_path->sgid = req_msg->alt_remote_gid;
  1098. alt_path->dlid = req_msg->alt_local_lid;
  1099. alt_path->slid = req_msg->alt_remote_lid;
  1100. alt_path->flow_label = cm_req_get_alt_flow_label(req_msg);
  1101. alt_path->hop_limit = req_msg->alt_hop_limit;
  1102. alt_path->traffic_class = req_msg->alt_traffic_class;
  1103. alt_path->reversible = 1;
  1104. alt_path->pkey = req_msg->pkey;
  1105. alt_path->sl = cm_req_get_alt_sl(req_msg);
  1106. alt_path->mtu_selector = IB_SA_EQ;
  1107. alt_path->mtu = cm_req_get_path_mtu(req_msg);
  1108. alt_path->rate_selector = IB_SA_EQ;
  1109. alt_path->rate = cm_req_get_alt_packet_rate(req_msg);
  1110. alt_path->packet_life_time_selector = IB_SA_EQ;
  1111. alt_path->packet_life_time =
  1112. cm_req_get_alt_local_ack_timeout(req_msg);
  1113. alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
  1114. }
  1115. }
  1116. static void cm_format_req_event(struct cm_work *work,
  1117. struct cm_id_private *cm_id_priv,
  1118. struct ib_cm_id *listen_id)
  1119. {
  1120. struct cm_req_msg *req_msg;
  1121. struct ib_cm_req_event_param *param;
  1122. req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
  1123. param = &work->cm_event.param.req_rcvd;
  1124. param->listen_id = listen_id;
  1125. param->port = cm_id_priv->av.port->port_num;
  1126. param->primary_path = &work->path[0];
  1127. if (req_msg->alt_local_lid)
  1128. param->alternate_path = &work->path[1];
  1129. else
  1130. param->alternate_path = NULL;
  1131. param->remote_ca_guid = req_msg->local_ca_guid;
  1132. param->remote_qkey = be32_to_cpu(req_msg->local_qkey);
  1133. param->remote_qpn = be32_to_cpu(cm_req_get_local_qpn(req_msg));
  1134. param->qp_type = cm_req_get_qp_type(req_msg);
  1135. param->starting_psn = be32_to_cpu(cm_req_get_starting_psn(req_msg));
  1136. param->responder_resources = cm_req_get_init_depth(req_msg);
  1137. param->initiator_depth = cm_req_get_resp_res(req_msg);
  1138. param->local_cm_response_timeout =
  1139. cm_req_get_remote_resp_timeout(req_msg);
  1140. param->flow_control = cm_req_get_flow_ctrl(req_msg);
  1141. param->remote_cm_response_timeout =
  1142. cm_req_get_local_resp_timeout(req_msg);
  1143. param->retry_count = cm_req_get_retry_count(req_msg);
  1144. param->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg);
  1145. param->srq = cm_req_get_srq(req_msg);
  1146. work->cm_event.private_data = &req_msg->private_data;
  1147. }
  1148. static void cm_process_work(struct cm_id_private *cm_id_priv,
  1149. struct cm_work *work)
  1150. {
  1151. int ret;
  1152. /* We will typically only have the current event to report. */
  1153. ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
  1154. cm_free_work(work);
  1155. while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) {
  1156. spin_lock_irq(&cm_id_priv->lock);
  1157. work = cm_dequeue_work(cm_id_priv);
  1158. spin_unlock_irq(&cm_id_priv->lock);
  1159. BUG_ON(!work);
  1160. ret = cm_id_priv->id.cm_handler(&cm_id_priv->id,
  1161. &work->cm_event);
  1162. cm_free_work(work);
  1163. }
  1164. cm_deref_id(cm_id_priv);
  1165. if (ret)
  1166. cm_destroy_id(&cm_id_priv->id, ret);
  1167. }
  1168. static void cm_format_mra(struct cm_mra_msg *mra_msg,
  1169. struct cm_id_private *cm_id_priv,
  1170. enum cm_msg_response msg_mraed, u8 service_timeout,
  1171. const void *private_data, u8 private_data_len)
  1172. {
  1173. cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid);
  1174. cm_mra_set_msg_mraed(mra_msg, msg_mraed);
  1175. mra_msg->local_comm_id = cm_id_priv->id.local_id;
  1176. mra_msg->remote_comm_id = cm_id_priv->id.remote_id;
  1177. cm_mra_set_service_timeout(mra_msg, service_timeout);
  1178. if (private_data && private_data_len)
  1179. memcpy(mra_msg->private_data, private_data, private_data_len);
  1180. }
  1181. static void cm_format_rej(struct cm_rej_msg *rej_msg,
  1182. struct cm_id_private *cm_id_priv,
  1183. enum ib_cm_rej_reason reason,
  1184. void *ari,
  1185. u8 ari_length,
  1186. const void *private_data,
  1187. u8 private_data_len)
  1188. {
  1189. cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid);
  1190. rej_msg->remote_comm_id = cm_id_priv->id.remote_id;
  1191. switch(cm_id_priv->id.state) {
  1192. case IB_CM_REQ_RCVD:
  1193. rej_msg->local_comm_id = 0;
  1194. cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ);
  1195. break;
  1196. case IB_CM_MRA_REQ_SENT:
  1197. rej_msg->local_comm_id = cm_id_priv->id.local_id;
  1198. cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ);
  1199. break;
  1200. case IB_CM_REP_RCVD:
  1201. case IB_CM_MRA_REP_SENT:
  1202. rej_msg->local_comm_id = cm_id_priv->id.local_id;
  1203. cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REP);
  1204. break;
  1205. default:
  1206. rej_msg->local_comm_id = cm_id_priv->id.local_id;
  1207. cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_OTHER);
  1208. break;
  1209. }
  1210. rej_msg->reason = cpu_to_be16(reason);
  1211. if (ari && ari_length) {
  1212. cm_rej_set_reject_info_len(rej_msg, ari_length);
  1213. memcpy(rej_msg->ari, ari, ari_length);
  1214. }
  1215. if (private_data && private_data_len)
  1216. memcpy(rej_msg->private_data, private_data, private_data_len);
  1217. }
  1218. static void cm_dup_req_handler(struct cm_work *work,
  1219. struct cm_id_private *cm_id_priv)
  1220. {
  1221. struct ib_mad_send_buf *msg = NULL;
  1222. int ret;
  1223. atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
  1224. counter[CM_REQ_COUNTER]);
  1225. /* Quick state check to discard duplicate REQs. */
  1226. if (cm_id_priv->id.state == IB_CM_REQ_RCVD)
  1227. return;
  1228. ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
  1229. if (ret)
  1230. return;
  1231. spin_lock_irq(&cm_id_priv->lock);
  1232. switch (cm_id_priv->id.state) {
  1233. case IB_CM_MRA_REQ_SENT:
  1234. cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
  1235. CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout,
  1236. cm_id_priv->private_data,
  1237. cm_id_priv->private_data_len);
  1238. break;
  1239. case IB_CM_TIMEWAIT:
  1240. cm_format_rej((struct cm_rej_msg *) msg->mad, cm_id_priv,
  1241. IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0);
  1242. break;
  1243. default:
  1244. goto unlock;
  1245. }
  1246. spin_unlock_irq(&cm_id_priv->lock);
  1247. ret = ib_post_send_mad(msg, NULL);
  1248. if (ret)
  1249. goto free;
  1250. return;
  1251. unlock: spin_unlock_irq(&cm_id_priv->lock);
  1252. free: cm_free_msg(msg);
  1253. }
  1254. static struct cm_id_private * cm_match_req(struct cm_work *work,
  1255. struct cm_id_private *cm_id_priv)
  1256. {
  1257. struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
  1258. struct cm_timewait_info *timewait_info;
  1259. struct cm_req_msg *req_msg;
  1260. req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
  1261. /* Check for possible duplicate REQ. */
  1262. spin_lock_irq(&cm.lock);
  1263. timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info);
  1264. if (timewait_info) {
  1265. cur_cm_id_priv = cm_get_id(timewait_info->work.local_id,
  1266. timewait_info->work.remote_id);
  1267. spin_unlock_irq(&cm.lock);
  1268. if (cur_cm_id_priv) {
  1269. cm_dup_req_handler(work, cur_cm_id_priv);
  1270. cm_deref_id(cur_cm_id_priv);
  1271. }
  1272. return NULL;
  1273. }
  1274. /* Check for stale connections. */
  1275. timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
  1276. if (timewait_info) {
  1277. cm_cleanup_timewait(cm_id_priv->timewait_info);
  1278. spin_unlock_irq(&cm.lock);
  1279. cm_issue_rej(work->port, work->mad_recv_wc,
  1280. IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
  1281. NULL, 0);
  1282. return NULL;
  1283. }
  1284. /* Find matching listen request. */
  1285. listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device,
  1286. req_msg->service_id,
  1287. req_msg->private_data);
  1288. if (!listen_cm_id_priv) {
  1289. cm_cleanup_timewait(cm_id_priv->timewait_info);
  1290. spin_unlock_irq(&cm.lock);
  1291. cm_issue_rej(work->port, work->mad_recv_wc,
  1292. IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,
  1293. NULL, 0);
  1294. goto out;
  1295. }
  1296. atomic_inc(&listen_cm_id_priv->refcount);
  1297. atomic_inc(&cm_id_priv->refcount);
  1298. cm_id_priv->id.state = IB_CM_REQ_RCVD;
  1299. atomic_inc(&cm_id_priv->work_count);
  1300. spin_unlock_irq(&cm.lock);
  1301. out:
  1302. return listen_cm_id_priv;
  1303. }
  1304. /*
  1305. * Work-around for inter-subnet connections. If the LIDs are permissive,
  1306. * we need to override the LID/SL data in the REQ with the LID information
  1307. * in the work completion.
  1308. */
  1309. static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
  1310. {
  1311. if (!cm_req_get_primary_subnet_local(req_msg)) {
  1312. if (req_msg->primary_local_lid == IB_LID_PERMISSIVE) {
  1313. req_msg->primary_local_lid = cpu_to_be16(wc->slid);
  1314. cm_req_set_primary_sl(req_msg, wc->sl);
  1315. }
  1316. if (req_msg->primary_remote_lid == IB_LID_PERMISSIVE)
  1317. req_msg->primary_remote_lid = cpu_to_be16(wc->dlid_path_bits);
  1318. }
  1319. if (!cm_req_get_alt_subnet_local(req_msg)) {
  1320. if (req_msg->alt_local_lid == IB_LID_PERMISSIVE) {
  1321. req_msg->alt_local_lid = cpu_to_be16(wc->slid);
  1322. cm_req_set_alt_sl(req_msg, wc->sl);
  1323. }
  1324. if (req_msg->alt_remote_lid == IB_LID_PERMISSIVE)
  1325. req_msg->alt_remote_lid = cpu_to_be16(wc->dlid_path_bits);
  1326. }
  1327. }
  1328. static int cm_req_handler(struct cm_work *work)
  1329. {
  1330. struct ib_cm_id *cm_id;
  1331. struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
  1332. struct cm_req_msg *req_msg;
  1333. int ret;
  1334. req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
  1335. cm_id = ib_create_cm_id(work->port->cm_dev->ib_device, NULL, NULL);
  1336. if (IS_ERR(cm_id))
  1337. return PTR_ERR(cm_id);
  1338. cm_id_priv = container_of(cm_id, struct cm_id_private, id);
  1339. cm_id_priv->id.remote_id = req_msg->local_comm_id;
  1340. cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
  1341. work->mad_recv_wc->recv_buf.grh,
  1342. &cm_id_priv->av);
  1343. cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
  1344. id.local_id);
  1345. if (IS_ERR(cm_id_priv->timewait_info)) {
  1346. ret = PTR_ERR(cm_id_priv->timewait_info);
  1347. goto destroy;
  1348. }
  1349. cm_id_priv->timewait_info->work.remote_id = req_msg->local_comm_id;
  1350. cm_id_priv->timewait_info->remote_ca_guid = req_msg->local_ca_guid;
  1351. cm_id_priv->timewait_info->remote_qpn = cm_req_get_local_qpn(req_msg);
  1352. listen_cm_id_priv = cm_match_req(work, cm_id_priv);
  1353. if (!listen_cm_id_priv) {
  1354. ret = -EINVAL;
  1355. kfree(cm_id_priv->timewait_info);
  1356. goto destroy;
  1357. }
  1358. cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
  1359. cm_id_priv->id.context = listen_cm_id_priv->id.context;
  1360. cm_id_priv->id.service_id = req_msg->service_id;
  1361. cm_id_priv->id.service_mask = ~cpu_to_be64(0);
  1362. cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
  1363. cm_format_paths_from_req(req_msg, &work->path[0], &work->path[1]);
  1364. ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av);
  1365. if (ret) {
  1366. ib_get_cached_gid(work->port->cm_dev->ib_device,
  1367. work->port->port_num, 0, &work->path[0].sgid);
  1368. ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,
  1369. &work->path[0].sgid, sizeof work->path[0].sgid,
  1370. NULL, 0);
  1371. goto rejected;
  1372. }
  1373. if (req_msg->alt_local_lid) {
  1374. ret = cm_init_av_by_path(&work->path[1], &cm_id_priv->alt_av);
  1375. if (ret) {
  1376. ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_ALT_GID,
  1377. &work->path[0].sgid,
  1378. sizeof work->path[0].sgid, NULL, 0);
  1379. goto rejected;
  1380. }
  1381. }
  1382. cm_id_priv->tid = req_msg->hdr.tid;
  1383. cm_id_priv->timeout_ms = cm_convert_to_ms(
  1384. cm_req_get_local_resp_timeout(req_msg));
  1385. cm_id_priv->max_cm_retries = cm_req_get_max_cm_retries(req_msg);
  1386. cm_id_priv->remote_qpn = cm_req_get_local_qpn(req_msg);
  1387. cm_id_priv->initiator_depth = cm_req_get_resp_res(req_msg);
  1388. cm_id_priv->responder_resources = cm_req_get_init_depth(req_msg);
  1389. cm_id_priv->path_mtu = cm_req_get_path_mtu(req_msg);
  1390. cm_id_priv->pkey = req_msg->pkey;
  1391. cm_id_priv->sq_psn = cm_req_get_starting_psn(req_msg);
  1392. cm_id_priv->retry_count = cm_req_get_retry_count(req_msg);
  1393. cm_id_priv->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg);
  1394. cm_id_priv->qp_type = cm_req_get_qp_type(req_msg);
  1395. cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
  1396. cm_process_work(cm_id_priv, work);
  1397. cm_deref_id(listen_cm_id_priv);
  1398. return 0;
  1399. rejected:
  1400. atomic_dec(&cm_id_priv->refcount);
  1401. cm_deref_id(listen_cm_id_priv);
  1402. destroy:
  1403. ib_destroy_cm_id(cm_id);
  1404. return ret;
  1405. }
  1406. static void cm_format_rep(struct cm_rep_msg *rep_msg,
  1407. struct cm_id_private *cm_id_priv,
  1408. struct ib_cm_rep_param *param)
  1409. {
  1410. cm_format_mad_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid);
  1411. rep_msg->local_comm_id = cm_id_priv->id.local_id;
  1412. rep_msg->remote_comm_id = cm_id_priv->id.remote_id;
  1413. cm_rep_set_local_qpn(rep_msg, cpu_to_be32(param->qp_num));
  1414. cm_rep_set_starting_psn(rep_msg, cpu_to_be32(param->starting_psn));
  1415. rep_msg->resp_resources = param->responder_resources;
  1416. rep_msg->initiator_depth = param->initiator_depth;
  1417. cm_rep_set_target_ack_delay(rep_msg,
  1418. cm_id_priv->av.port->cm_dev->ack_delay);
  1419. cm_rep_set_failover(rep_msg, param->failover_accepted);
  1420. cm_rep_set_flow_ctrl(rep_msg, param->flow_control);
  1421. cm_rep_set_rnr_retry_count(rep_msg, param->rnr_retry_count);
  1422. cm_rep_set_srq(rep_msg, param->srq);
  1423. rep_msg->local_ca_guid = cm_id_priv->id.device->node_guid;
  1424. if (param->private_data && param->private_data_len)
  1425. memcpy(rep_msg->private_data, param->private_data,
  1426. param->private_data_len);
  1427. }
  1428. int ib_send_cm_rep(struct ib_cm_id *cm_id,
  1429. struct ib_cm_rep_param *param)
  1430. {
  1431. struct cm_id_private *cm_id_priv;
  1432. struct ib_mad_send_buf *msg;
  1433. struct cm_rep_msg *rep_msg;
  1434. unsigned long flags;
  1435. int ret;
  1436. if (param->private_data &&
  1437. param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
  1438. return -EINVAL;
  1439. cm_id_priv = container_of(cm_id, struct cm_id_private, id);
  1440. spin_lock_irqsave(&cm_id_priv->lock, flags);
  1441. if (cm_id->state != IB_CM_REQ_RCVD &&
  1442. cm_id->state != IB_CM_MRA_REQ_SENT) {
  1443. ret = -EINVAL;
  1444. goto out;
  1445. }
  1446. ret = cm_alloc_msg(cm_id_priv, &msg);
  1447. if (ret)
  1448. goto out;
  1449. rep_msg = (struct cm_rep_msg *) msg->mad;
  1450. cm_format_rep(rep_msg, cm_id_priv, param);
  1451. msg->timeout_ms = cm_id_priv->timeout_ms;
  1452. msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT;
  1453. ret = ib_post_send_mad(msg, NULL);
  1454. if (ret) {
  1455. spin_unlock_irqrestore(&cm_id_priv->lock, flags);
  1456. cm_free_msg(msg);
  1457. return ret;
  1458. }
  1459. cm_id->state = IB_CM_REP_SENT;
  1460. cm_id_priv->msg = msg;
  1461. cm_id_priv->initiator_depth = param->initiator_depth;
  1462. cm_id_priv->responder_resources = param->responder_resources;
  1463. cm_id_priv->rq_psn = cm_rep_get_starting_psn(rep_msg);
  1464. cm_id_priv->local_qpn = cm_rep_get_local_qpn(rep_msg);
  1465. out: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
  1466. return ret;
  1467. }
  1468. EXPORT_SYMBOL(ib_send_cm_rep);
  1469. static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,
  1470. struct cm_id_private *cm_id_priv,
  1471. const void *private_data,
  1472. u8 private_data_len)
  1473. {
  1474. cm_format_m