PageRenderTime 94ms CodeModel.GetById 28ms RepoModel.GetById 0ms app.codeStats 1ms

/libvendor/osm_vendor_mlx_sim.c

https://gitlab.com/domke/osm-routing-dev
C | 439 lines | 262 code | 62 blank | 115 comment | 23 complexity | 3aee867dcb18b1814f406425fb5203f5 MD5 | raw file
  1. /*
  2. * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
  3. * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
  4. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
  5. *
  6. * This software is available to you under a choice of one of two
  7. * licenses. You may choose to be licensed under the terms of the GNU
  8. * General Public License (GPL) Version 2, available from the file
  9. * COPYING in the main directory of this source tree, or the
  10. * OpenIB.org BSD license below:
  11. *
  12. * Redistribution and use in source and binary forms, with or
  13. * without modification, are permitted provided that the following
  14. * conditions are met:
  15. *
  16. * - Redistributions of source code must retain the above
  17. * copyright notice, this list of conditions and the following
  18. * disclaimer.
  19. *
  20. * - Redistributions in binary form must reproduce the above
  21. * copyright notice, this list of conditions and the following
  22. * disclaimer in the documentation and/or other materials
  23. * provided with the distribution.
  24. *
  25. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32. * SOFTWARE.
  33. *
  34. */
  35. /* AUTHOR Eitan Zahavi
  36. *
  37. * DESCRIPTION
  38. * The lower-level MAD transport interface implementation
  39. * that allows sending a single MAD/receiving a callback
  40. * when a single MAD is received.
  41. */
  42. #if HAVE_CONFIG_H
  43. # include <config.h>
  44. #endif /* HAVE_CONFIG_H */
  45. #include <sys/types.h>
  46. #include <sys/stat.h>
  47. #include <sys/ioctl.h>
  48. #include <fcntl.h>
  49. #include <errno.h>
  50. #include <stdlib.h>
  51. #include <string.h>
  52. #include <vendor/osm_vendor_api.h>
  53. #include <vendor/osm_vendor_mlx_transport.h>
  54. #include <vendor/osm_vendor_mlx_dispatcher.h>
  55. #include <vendor/osm_vendor_mlx_svc.h>
  56. #include <complib/cl_thread.h>
  57. /* the simulator messages definition */
  58. #include <ibmgtsim/ibms_client_api.h>
  59. typedef struct _osmv_ibms_transport_mgr {
  60. ibms_conn_handle_t conHdl; /* the connection handle we talk to */
  61. ibms_bind_msg_t filter; /* the bind message defining the filtering */
  62. cl_thread_t receiver; /* the thread waiting for incomming messages */
  63. } osmv_ibms_transport_mgr_t;
  64. static void
  65. __osmv_ibms_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
  66. IN struct _ibms_mad_addr *p_ibms_addr,
  67. IN uint8_t is_smi,
  68. OUT osm_mad_addr_t * p_osm_addr);
  69. static void
  70. __osmv_ibms_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_osm_addr,
  71. IN uint8_t is_smi,
  72. OUT struct _ibms_mad_addr *p_ibms_addr);
  73. /* this is the callback function the "server" will call on incoming
  74. messages */
  75. void __osmv_ibms_receiver_callback(void *p_ctx, ibms_mad_msg_t * p_mad)
  76. {
  77. osm_mad_addr_t mad_addr;
  78. osmv_bind_obj_t *const p_bo = (osmv_bind_obj_t *) p_ctx;
  79. ib_api_status_t status = IB_SUCCESS;
  80. /* Make sure the p_bo object is still relevant */
  81. if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
  82. return;
  83. {
  84. OSM_LOG_ENTER(p_bo->p_vendor->p_log);
  85. /* some logging */
  86. osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
  87. "__osmv_ibms_receiver_callback: "
  88. "MAD QPN:%d SLID:0x%04x class:0x%02x "
  89. "method:0x%02x attr:0x%04x status:0x%04x "
  90. "tid:0x%016" PRIx64 "\n",
  91. p_mad->addr.dqpn,
  92. cl_ntoh16(p_mad->addr.slid),
  93. p_mad->header.mgmt_class,
  94. p_mad->header.method,
  95. cl_ntoh16(p_mad->header.attr_id),
  96. cl_ntoh16(p_mad->header.status),
  97. cl_ntoh64(p_mad->header.trans_id));
  98. /* first arrange an address */
  99. __osmv_ibms_mad_addr_to_osm_addr(p_bo->p_vendor,
  100. &p_mad->addr,
  101. (((ib_mad_t *) & p_mad->
  102. header)->mgmt_class ==
  103. IB_MCLASS_SUBN_LID)
  104. ||
  105. (((ib_mad_t *) & p_mad->
  106. header)->mgmt_class ==
  107. IB_MCLASS_SUBN_DIR),
  108. &mad_addr);
  109. /* call the receiver callback */
  110. status =
  111. osmv_dispatch_mad((osm_bind_handle_t) p_bo,
  112. (void *)&p_mad->header, &mad_addr);
  113. OSM_LOG_EXIT(p_bo->p_vendor->p_log);
  114. }
  115. }
  116. ib_api_status_t
  117. osm_vendor_get_guid_by_ca_and_port(IN osm_vendor_t * const p_vend,
  118. IN char *hca_id,
  119. IN uint32_t port_num,
  120. OUT uint64_t * p_port_guid);
  121. /*
  122. * NAME
  123. * osmv_transport_init
  124. *
  125. * DESCRIPTION
  126. * Setup the MAD transport infrastructure (filters, callbacks etc).
  127. */
  128. ib_api_status_t
  129. osmv_transport_init(IN osm_bind_info_t * p_info,
  130. IN char hca_id[VENDOR_HCA_MAXNAMES],
  131. IN uint8_t hca_idx, IN osmv_bind_obj_t * p_bo)
  132. {
  133. ibms_conn_handle_t conHdl; /* the connection we talk to the simulator through */
  134. osmv_ibms_transport_mgr_t *p_mgr =
  135. malloc(sizeof(osmv_ibms_transport_mgr_t));
  136. int qpn;
  137. int ibms_status;
  138. uint64_t port_guid;
  139. if (!p_mgr) {
  140. return IB_INSUFFICIENT_MEMORY;
  141. }
  142. memset(p_mgr, 0, sizeof(osmv_ibms_transport_mgr_t));
  143. /* create the client socket connected to the simulator */
  144. /* also perform the "connect" message - such that we
  145. validate the target guid */
  146. if (osm_vendor_get_guid_by_ca_and_port
  147. (p_bo->p_vendor, hca_id, p_bo->port_num, &port_guid)) {
  148. return IB_INVALID_GUID;
  149. }
  150. conHdl =
  151. ibms_connect(port_guid, __osmv_ibms_receiver_callback,
  152. (void *)p_bo);
  153. if (!conHdl) {
  154. printf("fail to connect to the server.\n");
  155. exit(1);
  156. }
  157. /*
  158. * Create the MAD filter on this file handle.
  159. */
  160. p_mgr->filter.port = p_bo->port_num;
  161. p_mgr->filter.only_input = 1;
  162. p_mgr->filter.mask =
  163. IBMS_BIND_MASK_PORT |
  164. IBMS_BIND_MASK_INPUT | IBMS_BIND_MASK_QP | IBMS_BIND_MASK_CLASS;
  165. switch (p_info->mad_class) {
  166. case IB_MCLASS_SUBN_LID:
  167. case IB_MCLASS_SUBN_DIR:
  168. qpn = 0;
  169. p_mgr->filter.qpn = qpn;
  170. p_mgr->filter.mgt_class = IB_MCLASS_SUBN_LID;
  171. ibms_status = ibms_bind(conHdl, &p_mgr->filter);
  172. if (ibms_status) {
  173. return IB_ERROR;
  174. }
  175. p_mgr->filter.mgt_class = IB_MCLASS_SUBN_DIR;
  176. ibms_status = ibms_bind(conHdl, &p_mgr->filter);
  177. if (ibms_status) {
  178. return IB_ERROR;
  179. }
  180. break;
  181. case IB_MCLASS_SUBN_ADM:
  182. default:
  183. qpn = 1;
  184. p_mgr->filter.qpn = qpn;
  185. p_mgr->filter.mgt_class = p_info->mad_class;
  186. ibms_status = ibms_bind(conHdl, &p_mgr->filter);
  187. if (ibms_status) {
  188. return IB_ERROR;
  189. }
  190. break;
  191. }
  192. p_mgr->conHdl = conHdl;
  193. p_bo->p_transp_mgr = p_mgr;
  194. /* Initialize the magic_ptr to the pointer of the p_bo info.
  195. This will be used to signal when the object is being destroyed, so no
  196. real action will be done then. */
  197. p_bo->magic_ptr = p_bo;
  198. return IB_SUCCESS;
  199. }
  200. /*
  201. * NAME
  202. * osmv_transport_send_mad
  203. *
  204. * DESCRIPTION
  205. * Send a single MAD (256 byte)
  206. */
  207. ib_api_status_t
  208. osmv_transport_mad_send(IN const osm_bind_handle_t h_bind,
  209. IN void *p_mad, IN const osm_mad_addr_t * p_mad_addr)
  210. {
  211. osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
  212. osm_vendor_t const *p_vend = p_bo->p_vendor;
  213. int ret;
  214. ibms_mad_msg_t mad_msg;
  215. ib_api_status_t status;
  216. const ib_mad_t *p_mad_hdr = p_mad;
  217. OSM_LOG_ENTER(p_vend->p_log);
  218. memset(&mad_msg, 0, sizeof(mad_msg));
  219. /* Make sure the p_bo object is still relevant */
  220. if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
  221. return IB_INVALID_CALLBACK;
  222. /*
  223. * Copy the MAD over to the sent mad
  224. */
  225. memcpy(&mad_msg.header, p_mad_hdr, MAD_BLOCK_SIZE);
  226. /*
  227. * For all sends other than directed route SM MADs,
  228. * acquire an address vector for the destination.
  229. */
  230. if (p_mad_hdr->mgmt_class != IB_MCLASS_SUBN_DIR) {
  231. __osmv_ibms_osm_addr_to_mad_addr(p_mad_addr,
  232. p_mad_hdr->mgmt_class ==
  233. IB_MCLASS_SUBN_LID,
  234. &mad_msg.addr);
  235. } else {
  236. /* is a directed route - we need to construct a permissive address */
  237. /* we do not need port number since it is part of the mad_hndl */
  238. mad_msg.addr.dlid = IB_LID_PERMISSIVE;
  239. mad_msg.addr.slid = IB_LID_PERMISSIVE;
  240. mad_msg.addr.sqpn = 0;
  241. mad_msg.addr.dqpn = 0;
  242. }
  243. osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
  244. "osmv_transport_mad_send: "
  245. "Sending QPN:%d DLID:0x%04x class:0x%02x "
  246. "method:0x%02x attr:0x%04x status:0x%04x "
  247. "tid:0x%016" PRIx64 "\n",
  248. mad_msg.addr.dqpn,
  249. cl_ntoh16(mad_msg.addr.dlid),
  250. mad_msg.header.mgmt_class,
  251. mad_msg.header.method,
  252. cl_ntoh16(mad_msg.header.attr_id),
  253. cl_ntoh16(mad_msg.header.status),
  254. cl_ntoh64(mad_msg.header.trans_id)
  255. );
  256. /* send it */
  257. ret =
  258. ibms_send(((osmv_ibms_transport_mgr_t *) (p_bo->p_transp_mgr))->
  259. conHdl, &mad_msg);
  260. if (ret) {
  261. osm_log(p_vend->p_log, OSM_LOG_ERROR,
  262. "osmv_transport_mad_send: ERR 5304: "
  263. "Error sending mad (%d).\n", ret);
  264. status = IB_ERROR;
  265. goto Exit;
  266. }
  267. status = IB_SUCCESS;
  268. Exit:
  269. OSM_LOG_EXIT(p_vend->p_log);
  270. return (status);
  271. }
  272. void osmv_transport_done(IN const osm_bind_handle_t h_bind)
  273. {
  274. osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
  275. osmv_ibms_transport_mgr_t *p_tpot_mgr =
  276. (osmv_ibms_transport_mgr_t *) (p_bo->p_transp_mgr);
  277. CL_ASSERT(p_bo);
  278. /* First of all - zero out the magic_ptr, so if a callback is called -
  279. it'll know that we are currently closing down, and will not handle the
  280. mad. */
  281. p_bo->magic_ptr = 0;
  282. /* usleep(3000000); */
  283. ibms_disconnect(p_tpot_mgr->conHdl);
  284. /* seems the only way to abort a blocking read is to make it read something */
  285. free(p_tpot_mgr);
  286. }
  287. static void
  288. __osmv_ibms_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_osm_addr,
  289. IN uint8_t is_smi,
  290. OUT struct _ibms_mad_addr *p_ibms_addr)
  291. {
  292. /* For global destination or Multicast address: */
  293. p_ibms_addr->dlid = cl_ntoh16(p_osm_addr->dest_lid);
  294. p_ibms_addr->sl = p_osm_addr->addr_type.gsi.service_level;
  295. if (is_smi) {
  296. p_ibms_addr->sqpn = 0;
  297. p_ibms_addr->dqpn = 0;
  298. } else {
  299. p_ibms_addr->sqpn = 1;
  300. p_ibms_addr->dqpn =
  301. cl_ntoh32(p_osm_addr->addr_type.gsi.remote_qp);
  302. }
  303. /*
  304. HACK we limit to the first PKey Index assuming it will
  305. always be the default PKey
  306. */
  307. p_ibms_addr->pkey_index = 0;
  308. }
  309. static void
  310. __osmv_ibms_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
  311. IN struct _ibms_mad_addr *p_ibms_addr,
  312. IN uint8_t is_smi,
  313. OUT osm_mad_addr_t * p_osm_addr)
  314. {
  315. memset(p_osm_addr, 0, sizeof(osm_mad_addr_t));
  316. p_osm_addr->dest_lid = cl_hton16(p_ibms_addr->slid);
  317. p_osm_addr->static_rate = 0;
  318. p_osm_addr->path_bits = 0;
  319. if (is_smi) {
  320. /* SMI */
  321. p_osm_addr->addr_type.smi.source_lid =
  322. cl_hton16(p_ibms_addr->slid);
  323. p_osm_addr->addr_type.smi.port_num = 1; /* TODO add if required p_ibms_addr->port; */
  324. } else {
  325. /* GSI */
  326. p_osm_addr->addr_type.gsi.remote_qp =
  327. cl_ntoh32(p_ibms_addr->sqpn);
  328. p_osm_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
  329. p_osm_addr->addr_type.gsi.pkey_ix = p_ibms_addr->pkey_index;
  330. p_osm_addr->addr_type.gsi.service_level = p_ibms_addr->sl;
  331. p_osm_addr->addr_type.gsi.global_route = FALSE;
  332. /* copy the GRH data if relevant - TopSpin imp doesnt relate to GRH!!! */
  333. /*
  334. if (p_osm_addr->addr_type.gsi.global_route)
  335. {
  336. p_osm_addr->addr_type.gsi.grh_info.ver_class_flow =
  337. ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version,
  338. p_rcv_desc->grh.traffic_class,
  339. p_rcv_desc->grh.flow_label);
  340. p_osm_addr->addr_type.gsi.grh_info.hop_limit = p_rcv_desc->grh.hop_limit;
  341. memcpy(&p_osm_addr->addr_type.gsi.grh_info.src_gid.raw,
  342. &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
  343. memcpy(&p_osm_addr->addr_type.gsi.grh_info.dest_gid.raw,
  344. p_rcv_desc->grh.dgid, sizeof(ib_net64_t));
  345. }
  346. */
  347. }
  348. }
  349. /*
  350. * NAME osm_vendor_set_sm
  351. *
  352. * DESCRIPTION Modifies the port info for the bound port to set the "IS_SM" bit
  353. * according to the value given (TRUE or FALSE).
  354. */
  355. void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
  356. {
  357. osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
  358. osm_vendor_t const *p_vend = p_bo->p_vendor;
  359. int ret;
  360. ibms_cap_msg_t cap_msg;
  361. OSM_LOG_ENTER(p_vend->p_log);
  362. cap_msg.mask = IB_PORT_CAP_IS_SM;
  363. if (is_sm_val)
  364. cap_msg.capabilities = IB_PORT_CAP_IS_SM;
  365. else
  366. cap_msg.capabilities = 0;
  367. ret = ibms_set_cap(((osmv_ibms_transport_mgr_t *) (p_bo->
  368. p_transp_mgr))->
  369. conHdl, &cap_msg);
  370. if (ret) {
  371. osm_log(p_vend->p_log, OSM_LOG_ERROR,
  372. "osm_vendor_set_sm: ERR 5312: "
  373. "Unable set 'IS_SM' bit to:%u in port attributes.\n",
  374. is_sm_val);
  375. }
  376. OSM_LOG_EXIT(p_vend->p_log);
  377. }