/libvendor/osm_vendor_mlx_sim.c
C | 439 lines | 262 code | 62 blank | 115 comment | 23 complexity | 3aee867dcb18b1814f406425fb5203f5 MD5 | raw file
- /*
- * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
- * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
- * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- */
- /* AUTHOR Eitan Zahavi
- *
- * DESCRIPTION
- * The lower-level MAD transport interface implementation
- * that allows sending a single MAD/receiving a callback
- * when a single MAD is received.
- */
- #if HAVE_CONFIG_H
- # include <config.h>
- #endif /* HAVE_CONFIG_H */
- #include <sys/types.h>
- #include <sys/stat.h>
- #include <sys/ioctl.h>
- #include <fcntl.h>
- #include <errno.h>
- #include <stdlib.h>
- #include <string.h>
- #include <vendor/osm_vendor_api.h>
- #include <vendor/osm_vendor_mlx_transport.h>
- #include <vendor/osm_vendor_mlx_dispatcher.h>
- #include <vendor/osm_vendor_mlx_svc.h>
- #include <complib/cl_thread.h>
- /* the simulator messages definition */
- #include <ibmgtsim/ibms_client_api.h>
- typedef struct _osmv_ibms_transport_mgr {
- ibms_conn_handle_t conHdl; /* the connection handle we talk to */
- ibms_bind_msg_t filter; /* the bind message defining the filtering */
- cl_thread_t receiver; /* the thread waiting for incomming messages */
- } osmv_ibms_transport_mgr_t;
- static void
- __osmv_ibms_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
- IN struct _ibms_mad_addr *p_ibms_addr,
- IN uint8_t is_smi,
- OUT osm_mad_addr_t * p_osm_addr);
- static void
- __osmv_ibms_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_osm_addr,
- IN uint8_t is_smi,
- OUT struct _ibms_mad_addr *p_ibms_addr);
- /* this is the callback function the "server" will call on incoming
- messages */
- void __osmv_ibms_receiver_callback(void *p_ctx, ibms_mad_msg_t * p_mad)
- {
- osm_mad_addr_t mad_addr;
- osmv_bind_obj_t *const p_bo = (osmv_bind_obj_t *) p_ctx;
- ib_api_status_t status = IB_SUCCESS;
- /* Make sure the p_bo object is still relevant */
- if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
- return;
- {
- OSM_LOG_ENTER(p_bo->p_vendor->p_log);
- /* some logging */
- osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
- "__osmv_ibms_receiver_callback: "
- "MAD QPN:%d SLID:0x%04x class:0x%02x "
- "method:0x%02x attr:0x%04x status:0x%04x "
- "tid:0x%016" PRIx64 "\n",
- p_mad->addr.dqpn,
- cl_ntoh16(p_mad->addr.slid),
- p_mad->header.mgmt_class,
- p_mad->header.method,
- cl_ntoh16(p_mad->header.attr_id),
- cl_ntoh16(p_mad->header.status),
- cl_ntoh64(p_mad->header.trans_id));
- /* first arrange an address */
- __osmv_ibms_mad_addr_to_osm_addr(p_bo->p_vendor,
- &p_mad->addr,
- (((ib_mad_t *) & p_mad->
- header)->mgmt_class ==
- IB_MCLASS_SUBN_LID)
- ||
- (((ib_mad_t *) & p_mad->
- header)->mgmt_class ==
- IB_MCLASS_SUBN_DIR),
- &mad_addr);
- /* call the receiver callback */
- status =
- osmv_dispatch_mad((osm_bind_handle_t) p_bo,
- (void *)&p_mad->header, &mad_addr);
- OSM_LOG_EXIT(p_bo->p_vendor->p_log);
- }
- }
- ib_api_status_t
- osm_vendor_get_guid_by_ca_and_port(IN osm_vendor_t * const p_vend,
- IN char *hca_id,
- IN uint32_t port_num,
- OUT uint64_t * p_port_guid);
- /*
- * NAME
- * osmv_transport_init
- *
- * DESCRIPTION
- * Setup the MAD transport infrastructure (filters, callbacks etc).
- */
- ib_api_status_t
- osmv_transport_init(IN osm_bind_info_t * p_info,
- IN char hca_id[VENDOR_HCA_MAXNAMES],
- IN uint8_t hca_idx, IN osmv_bind_obj_t * p_bo)
- {
- ibms_conn_handle_t conHdl; /* the connection we talk to the simulator through */
- osmv_ibms_transport_mgr_t *p_mgr =
- malloc(sizeof(osmv_ibms_transport_mgr_t));
- int qpn;
- int ibms_status;
- uint64_t port_guid;
- if (!p_mgr) {
- return IB_INSUFFICIENT_MEMORY;
- }
- memset(p_mgr, 0, sizeof(osmv_ibms_transport_mgr_t));
- /* create the client socket connected to the simulator */
- /* also perform the "connect" message - such that we
- validate the target guid */
- if (osm_vendor_get_guid_by_ca_and_port
- (p_bo->p_vendor, hca_id, p_bo->port_num, &port_guid)) {
- return IB_INVALID_GUID;
- }
- conHdl =
- ibms_connect(port_guid, __osmv_ibms_receiver_callback,
- (void *)p_bo);
- if (!conHdl) {
- printf("fail to connect to the server.\n");
- exit(1);
- }
- /*
- * Create the MAD filter on this file handle.
- */
- p_mgr->filter.port = p_bo->port_num;
- p_mgr->filter.only_input = 1;
- p_mgr->filter.mask =
- IBMS_BIND_MASK_PORT |
- IBMS_BIND_MASK_INPUT | IBMS_BIND_MASK_QP | IBMS_BIND_MASK_CLASS;
- switch (p_info->mad_class) {
- case IB_MCLASS_SUBN_LID:
- case IB_MCLASS_SUBN_DIR:
- qpn = 0;
- p_mgr->filter.qpn = qpn;
- p_mgr->filter.mgt_class = IB_MCLASS_SUBN_LID;
- ibms_status = ibms_bind(conHdl, &p_mgr->filter);
- if (ibms_status) {
- return IB_ERROR;
- }
- p_mgr->filter.mgt_class = IB_MCLASS_SUBN_DIR;
- ibms_status = ibms_bind(conHdl, &p_mgr->filter);
- if (ibms_status) {
- return IB_ERROR;
- }
- break;
- case IB_MCLASS_SUBN_ADM:
- default:
- qpn = 1;
- p_mgr->filter.qpn = qpn;
- p_mgr->filter.mgt_class = p_info->mad_class;
- ibms_status = ibms_bind(conHdl, &p_mgr->filter);
- if (ibms_status) {
- return IB_ERROR;
- }
- break;
- }
- p_mgr->conHdl = conHdl;
- p_bo->p_transp_mgr = p_mgr;
- /* Initialize the magic_ptr to the pointer of the p_bo info.
- This will be used to signal when the object is being destroyed, so no
- real action will be done then. */
- p_bo->magic_ptr = p_bo;
- return IB_SUCCESS;
- }
- /*
- * NAME
- * osmv_transport_send_mad
- *
- * DESCRIPTION
- * Send a single MAD (256 byte)
- */
- ib_api_status_t
- osmv_transport_mad_send(IN const osm_bind_handle_t h_bind,
- IN void *p_mad, IN const osm_mad_addr_t * p_mad_addr)
- {
- osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
- osm_vendor_t const *p_vend = p_bo->p_vendor;
- int ret;
- ibms_mad_msg_t mad_msg;
- ib_api_status_t status;
- const ib_mad_t *p_mad_hdr = p_mad;
- OSM_LOG_ENTER(p_vend->p_log);
- memset(&mad_msg, 0, sizeof(mad_msg));
- /* Make sure the p_bo object is still relevant */
- if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
- return IB_INVALID_CALLBACK;
- /*
- * Copy the MAD over to the sent mad
- */
- memcpy(&mad_msg.header, p_mad_hdr, MAD_BLOCK_SIZE);
- /*
- * For all sends other than directed route SM MADs,
- * acquire an address vector for the destination.
- */
- if (p_mad_hdr->mgmt_class != IB_MCLASS_SUBN_DIR) {
- __osmv_ibms_osm_addr_to_mad_addr(p_mad_addr,
- p_mad_hdr->mgmt_class ==
- IB_MCLASS_SUBN_LID,
- &mad_msg.addr);
- } else {
- /* is a directed route - we need to construct a permissive address */
- /* we do not need port number since it is part of the mad_hndl */
- mad_msg.addr.dlid = IB_LID_PERMISSIVE;
- mad_msg.addr.slid = IB_LID_PERMISSIVE;
- mad_msg.addr.sqpn = 0;
- mad_msg.addr.dqpn = 0;
- }
- osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
- "osmv_transport_mad_send: "
- "Sending QPN:%d DLID:0x%04x class:0x%02x "
- "method:0x%02x attr:0x%04x status:0x%04x "
- "tid:0x%016" PRIx64 "\n",
- mad_msg.addr.dqpn,
- cl_ntoh16(mad_msg.addr.dlid),
- mad_msg.header.mgmt_class,
- mad_msg.header.method,
- cl_ntoh16(mad_msg.header.attr_id),
- cl_ntoh16(mad_msg.header.status),
- cl_ntoh64(mad_msg.header.trans_id)
- );
- /* send it */
- ret =
- ibms_send(((osmv_ibms_transport_mgr_t *) (p_bo->p_transp_mgr))->
- conHdl, &mad_msg);
- if (ret) {
- osm_log(p_vend->p_log, OSM_LOG_ERROR,
- "osmv_transport_mad_send: ERR 5304: "
- "Error sending mad (%d).\n", ret);
- status = IB_ERROR;
- goto Exit;
- }
- status = IB_SUCCESS;
- Exit:
- OSM_LOG_EXIT(p_vend->p_log);
- return (status);
- }
- void osmv_transport_done(IN const osm_bind_handle_t h_bind)
- {
- osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
- osmv_ibms_transport_mgr_t *p_tpot_mgr =
- (osmv_ibms_transport_mgr_t *) (p_bo->p_transp_mgr);
- CL_ASSERT(p_bo);
- /* First of all - zero out the magic_ptr, so if a callback is called -
- it'll know that we are currently closing down, and will not handle the
- mad. */
- p_bo->magic_ptr = 0;
- /* usleep(3000000); */
- ibms_disconnect(p_tpot_mgr->conHdl);
- /* seems the only way to abort a blocking read is to make it read something */
- free(p_tpot_mgr);
- }
- static void
- __osmv_ibms_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_osm_addr,
- IN uint8_t is_smi,
- OUT struct _ibms_mad_addr *p_ibms_addr)
- {
- /* For global destination or Multicast address: */
- p_ibms_addr->dlid = cl_ntoh16(p_osm_addr->dest_lid);
- p_ibms_addr->sl = p_osm_addr->addr_type.gsi.service_level;
- if (is_smi) {
- p_ibms_addr->sqpn = 0;
- p_ibms_addr->dqpn = 0;
- } else {
- p_ibms_addr->sqpn = 1;
- p_ibms_addr->dqpn =
- cl_ntoh32(p_osm_addr->addr_type.gsi.remote_qp);
- }
- /*
- HACK we limit to the first PKey Index assuming it will
- always be the default PKey
- */
- p_ibms_addr->pkey_index = 0;
- }
- static void
- __osmv_ibms_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
- IN struct _ibms_mad_addr *p_ibms_addr,
- IN uint8_t is_smi,
- OUT osm_mad_addr_t * p_osm_addr)
- {
- memset(p_osm_addr, 0, sizeof(osm_mad_addr_t));
- p_osm_addr->dest_lid = cl_hton16(p_ibms_addr->slid);
- p_osm_addr->static_rate = 0;
- p_osm_addr->path_bits = 0;
- if (is_smi) {
- /* SMI */
- p_osm_addr->addr_type.smi.source_lid =
- cl_hton16(p_ibms_addr->slid);
- p_osm_addr->addr_type.smi.port_num = 1; /* TODO add if required p_ibms_addr->port; */
- } else {
- /* GSI */
- p_osm_addr->addr_type.gsi.remote_qp =
- cl_ntoh32(p_ibms_addr->sqpn);
- p_osm_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
- p_osm_addr->addr_type.gsi.pkey_ix = p_ibms_addr->pkey_index;
- p_osm_addr->addr_type.gsi.service_level = p_ibms_addr->sl;
- p_osm_addr->addr_type.gsi.global_route = FALSE;
- /* copy the GRH data if relevant - TopSpin imp doesnt relate to GRH!!! */
- /*
- if (p_osm_addr->addr_type.gsi.global_route)
- {
- p_osm_addr->addr_type.gsi.grh_info.ver_class_flow =
- ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version,
- p_rcv_desc->grh.traffic_class,
- p_rcv_desc->grh.flow_label);
- p_osm_addr->addr_type.gsi.grh_info.hop_limit = p_rcv_desc->grh.hop_limit;
- memcpy(&p_osm_addr->addr_type.gsi.grh_info.src_gid.raw,
- &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
- memcpy(&p_osm_addr->addr_type.gsi.grh_info.dest_gid.raw,
- p_rcv_desc->grh.dgid, sizeof(ib_net64_t));
- }
- */
- }
- }
- /*
- * NAME osm_vendor_set_sm
- *
- * DESCRIPTION Modifies the port info for the bound port to set the "IS_SM" bit
- * according to the value given (TRUE or FALSE).
- */
- void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
- {
- osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
- osm_vendor_t const *p_vend = p_bo->p_vendor;
- int ret;
- ibms_cap_msg_t cap_msg;
- OSM_LOG_ENTER(p_vend->p_log);
- cap_msg.mask = IB_PORT_CAP_IS_SM;
- if (is_sm_val)
- cap_msg.capabilities = IB_PORT_CAP_IS_SM;
- else
- cap_msg.capabilities = 0;
- ret = ibms_set_cap(((osmv_ibms_transport_mgr_t *) (p_bo->
- p_transp_mgr))->
- conHdl, &cap_msg);
- if (ret) {
- osm_log(p_vend->p_log, OSM_LOG_ERROR,
- "osm_vendor_set_sm: ERR 5312: "
- "Unable set 'IS_SM' bit to:%u in port attributes.\n",
- is_sm_val);
- }
- OSM_LOG_EXIT(p_vend->p_log);
- }