/drivers/infiniband/core/cm.c
https://bitbucket.org/ndreys/linux-sunxi · C · 3875 lines · 3297 code · 482 blank · 96 comment · 442 complexity · 9ba594543d5101f200f1c1152ab8cdaf MD5 · raw file
Large files are truncated click here to view the full file
- /*
- * Copyright (c) 2004-2007 Intel Corporation. All rights reserved.
- * Copyright (c) 2004 Topspin Corporation. All rights reserved.
- * Copyright (c) 2004, 2005 Voltaire Corporation. All rights reserved.
- * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
- #include <linux/completion.h>
- #include <linux/dma-mapping.h>
- #include <linux/device.h>
- #include <linux/err.h>
- #include <linux/idr.h>
- #include <linux/interrupt.h>
- #include <linux/random.h>
- #include <linux/rbtree.h>
- #include <linux/spinlock.h>
- #include <linux/slab.h>
- #include <linux/sysfs.h>
- #include <linux/workqueue.h>
- #include <linux/kdev_t.h>
- #include <rdma/ib_cache.h>
- #include <rdma/ib_cm.h>
- #include "cm_msgs.h"
- MODULE_AUTHOR("Sean Hefty");
- MODULE_DESCRIPTION("InfiniBand CM");
- MODULE_LICENSE("Dual BSD/GPL");
- static void cm_add_one(struct ib_device *device);
- static void cm_remove_one(struct ib_device *device);
- static struct ib_client cm_client = {
- .name = "cm",
- .add = cm_add_one,
- .remove = cm_remove_one
- };
- static struct ib_cm {
- spinlock_t lock;
- struct list_head device_list;
- rwlock_t device_lock;
- struct rb_root listen_service_table;
- u64 listen_service_id;
- /* struct rb_root peer_service_table; todo: fix peer to peer */
- struct rb_root remote_qp_table;
- struct rb_root remote_id_table;
- struct rb_root remote_sidr_table;
- struct idr local_id_table;
- __be32 random_id_operand;
- struct list_head timewait_list;
- struct workqueue_struct *wq;
- } cm;
- /* Counter indexes ordered by attribute ID */
- enum {
- CM_REQ_COUNTER,
- CM_MRA_COUNTER,
- CM_REJ_COUNTER,
- CM_REP_COUNTER,
- CM_RTU_COUNTER,
- CM_DREQ_COUNTER,
- CM_DREP_COUNTER,
- CM_SIDR_REQ_COUNTER,
- CM_SIDR_REP_COUNTER,
- CM_LAP_COUNTER,
- CM_APR_COUNTER,
- CM_ATTR_COUNT,
- CM_ATTR_ID_OFFSET = 0x0010,
- };
- enum {
- CM_XMIT,
- CM_XMIT_RETRIES,
- CM_RECV,
- CM_RECV_DUPLICATES,
- CM_COUNTER_GROUPS
- };
- static char const counter_group_names[CM_COUNTER_GROUPS]
- [sizeof("cm_rx_duplicates")] = {
- "cm_tx_msgs", "cm_tx_retries",
- "cm_rx_msgs", "cm_rx_duplicates"
- };
- struct cm_counter_group {
- struct kobject obj;
- atomic_long_t counter[CM_ATTR_COUNT];
- };
- struct cm_counter_attribute {
- struct attribute attr;
- int index;
- };
- #define CM_COUNTER_ATTR(_name, _index) \
- struct cm_counter_attribute cm_##_name##_counter_attr = { \
- .attr = { .name = __stringify(_name), .mode = 0444 }, \
- .index = _index \
- }
- static CM_COUNTER_ATTR(req, CM_REQ_COUNTER);
- static CM_COUNTER_ATTR(mra, CM_MRA_COUNTER);
- static CM_COUNTER_ATTR(rej, CM_REJ_COUNTER);
- static CM_COUNTER_ATTR(rep, CM_REP_COUNTER);
- static CM_COUNTER_ATTR(rtu, CM_RTU_COUNTER);
- static CM_COUNTER_ATTR(dreq, CM_DREQ_COUNTER);
- static CM_COUNTER_ATTR(drep, CM_DREP_COUNTER);
- static CM_COUNTER_ATTR(sidr_req, CM_SIDR_REQ_COUNTER);
- static CM_COUNTER_ATTR(sidr_rep, CM_SIDR_REP_COUNTER);
- static CM_COUNTER_ATTR(lap, CM_LAP_COUNTER);
- static CM_COUNTER_ATTR(apr, CM_APR_COUNTER);
- static struct attribute *cm_counter_default_attrs[] = {
- &cm_req_counter_attr.attr,
- &cm_mra_counter_attr.attr,
- &cm_rej_counter_attr.attr,
- &cm_rep_counter_attr.attr,
- &cm_rtu_counter_attr.attr,
- &cm_dreq_counter_attr.attr,
- &cm_drep_counter_attr.attr,
- &cm_sidr_req_counter_attr.attr,
- &cm_sidr_rep_counter_attr.attr,
- &cm_lap_counter_attr.attr,
- &cm_apr_counter_attr.attr,
- NULL
- };
- struct cm_port {
- struct cm_device *cm_dev;
- struct ib_mad_agent *mad_agent;
- struct kobject port_obj;
- u8 port_num;
- struct cm_counter_group counter_group[CM_COUNTER_GROUPS];
- };
- struct cm_device {
- struct list_head list;
- struct ib_device *ib_device;
- struct device *device;
- u8 ack_delay;
- struct cm_port *port[0];
- };
- struct cm_av {
- struct cm_port *port;
- union ib_gid dgid;
- struct ib_ah_attr ah_attr;
- u16 pkey_index;
- u8 timeout;
- };
- struct cm_work {
- struct delayed_work work;
- struct list_head list;
- struct cm_port *port;
- struct ib_mad_recv_wc *mad_recv_wc; /* Received MADs */
- __be32 local_id; /* Established / timewait */
- __be32 remote_id;
- struct ib_cm_event cm_event;
- struct ib_sa_path_rec path[0];
- };
- struct cm_timewait_info {
- struct cm_work work; /* Must be first. */
- struct list_head list;
- struct rb_node remote_qp_node;
- struct rb_node remote_id_node;
- __be64 remote_ca_guid;
- __be32 remote_qpn;
- u8 inserted_remote_qp;
- u8 inserted_remote_id;
- };
- struct cm_id_private {
- struct ib_cm_id id;
- struct rb_node service_node;
- struct rb_node sidr_id_node;
- spinlock_t lock; /* Do not acquire inside cm.lock */
- struct completion comp;
- atomic_t refcount;
- struct ib_mad_send_buf *msg;
- struct cm_timewait_info *timewait_info;
- /* todo: use alternate port on send failure */
- struct cm_av av;
- struct cm_av alt_av;
- struct ib_cm_compare_data *compare_data;
- void *private_data;
- __be64 tid;
- __be32 local_qpn;
- __be32 remote_qpn;
- enum ib_qp_type qp_type;
- __be32 sq_psn;
- __be32 rq_psn;
- int timeout_ms;
- enum ib_mtu path_mtu;
- __be16 pkey;
- u8 private_data_len;
- u8 max_cm_retries;
- u8 peer_to_peer;
- u8 responder_resources;
- u8 initiator_depth;
- u8 retry_count;
- u8 rnr_retry_count;
- u8 service_timeout;
- u8 target_ack_delay;
- struct list_head work_list;
- atomic_t work_count;
- };
- static void cm_work_handler(struct work_struct *work);
- static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
- {
- if (atomic_dec_and_test(&cm_id_priv->refcount))
- complete(&cm_id_priv->comp);
- }
- static int cm_alloc_msg(struct cm_id_private *cm_id_priv,
- struct ib_mad_send_buf **msg)
- {
- struct ib_mad_agent *mad_agent;
- struct ib_mad_send_buf *m;
- struct ib_ah *ah;
- mad_agent = cm_id_priv->av.port->mad_agent;
- ah = ib_create_ah(mad_agent->qp->pd, &cm_id_priv->av.ah_attr);
- if (IS_ERR(ah))
- return PTR_ERR(ah);
- m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
- cm_id_priv->av.pkey_index,
- 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
- GFP_ATOMIC);
- if (IS_ERR(m)) {
- ib_destroy_ah(ah);
- return PTR_ERR(m);
- }
- /* Timeout set by caller if response is expected. */
- m->ah = ah;
- m->retries = cm_id_priv->max_cm_retries;
- atomic_inc(&cm_id_priv->refcount);
- m->context[0] = cm_id_priv;
- *msg = m;
- return 0;
- }
- static int cm_alloc_response_msg(struct cm_port *port,
- struct ib_mad_recv_wc *mad_recv_wc,
- struct ib_mad_send_buf **msg)
- {
- struct ib_mad_send_buf *m;
- struct ib_ah *ah;
- ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc,
- mad_recv_wc->recv_buf.grh, port->port_num);
- if (IS_ERR(ah))
- return PTR_ERR(ah);
- m = ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
- 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
- GFP_ATOMIC);
- if (IS_ERR(m)) {
- ib_destroy_ah(ah);
- return PTR_ERR(m);
- }
- m->ah = ah;
- *msg = m;
- return 0;
- }
- static void cm_free_msg(struct ib_mad_send_buf *msg)
- {
- ib_destroy_ah(msg->ah);
- if (msg->context[0])
- cm_deref_id(msg->context[0]);
- ib_free_send_mad(msg);
- }
- static void * cm_copy_private_data(const void *private_data,
- u8 private_data_len)
- {
- void *data;
- if (!private_data || !private_data_len)
- return NULL;
- data = kmemdup(private_data, private_data_len, GFP_KERNEL);
- if (!data)
- return ERR_PTR(-ENOMEM);
- return data;
- }
- static void cm_set_private_data(struct cm_id_private *cm_id_priv,
- void *private_data, u8 private_data_len)
- {
- if (cm_id_priv->private_data && cm_id_priv->private_data_len)
- kfree(cm_id_priv->private_data);
- cm_id_priv->private_data = private_data;
- cm_id_priv->private_data_len = private_data_len;
- }
- static void cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
- struct ib_grh *grh, struct cm_av *av)
- {
- av->port = port;
- av->pkey_index = wc->pkey_index;
- ib_init_ah_from_wc(port->cm_dev->ib_device, port->port_num, wc,
- grh, &av->ah_attr);
- }
- static int cm_init_av_by_path(struct ib_sa_path_rec *path, struct cm_av *av)
- {
- struct cm_device *cm_dev;
- struct cm_port *port = NULL;
- unsigned long flags;
- int ret;
- u8 p;
- read_lock_irqsave(&cm.device_lock, flags);
- list_for_each_entry(cm_dev, &cm.device_list, list) {
- if (!ib_find_cached_gid(cm_dev->ib_device, &path->sgid,
- &p, NULL)) {
- port = cm_dev->port[p-1];
- break;
- }
- }
- read_unlock_irqrestore(&cm.device_lock, flags);
- if (!port)
- return -EINVAL;
- ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num,
- be16_to_cpu(path->pkey), &av->pkey_index);
- if (ret)
- return ret;
- av->port = port;
- ib_init_ah_from_path(cm_dev->ib_device, port->port_num, path,
- &av->ah_attr);
- av->timeout = path->packet_life_time + 1;
- return 0;
- }
- static int cm_alloc_id(struct cm_id_private *cm_id_priv)
- {
- unsigned long flags;
- int ret, id;
- static int next_id;
- do {
- spin_lock_irqsave(&cm.lock, flags);
- ret = idr_get_new_above(&cm.local_id_table, cm_id_priv,
- next_id, &id);
- if (!ret)
- next_id = ((unsigned) id + 1) & MAX_ID_MASK;
- spin_unlock_irqrestore(&cm.lock, flags);
- } while( (ret == -EAGAIN) && idr_pre_get(&cm.local_id_table, GFP_KERNEL) );
- cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
- return ret;
- }
- static void cm_free_id(__be32 local_id)
- {
- spin_lock_irq(&cm.lock);
- idr_remove(&cm.local_id_table,
- (__force int) (local_id ^ cm.random_id_operand));
- spin_unlock_irq(&cm.lock);
- }
- static struct cm_id_private * cm_get_id(__be32 local_id, __be32 remote_id)
- {
- struct cm_id_private *cm_id_priv;
- cm_id_priv = idr_find(&cm.local_id_table,
- (__force int) (local_id ^ cm.random_id_operand));
- if (cm_id_priv) {
- if (cm_id_priv->id.remote_id == remote_id)
- atomic_inc(&cm_id_priv->refcount);
- else
- cm_id_priv = NULL;
- }
- return cm_id_priv;
- }
- static struct cm_id_private * cm_acquire_id(__be32 local_id, __be32 remote_id)
- {
- struct cm_id_private *cm_id_priv;
- spin_lock_irq(&cm.lock);
- cm_id_priv = cm_get_id(local_id, remote_id);
- spin_unlock_irq(&cm.lock);
- return cm_id_priv;
- }
- static void cm_mask_copy(u8 *dst, u8 *src, u8 *mask)
- {
- int i;
- for (i = 0; i < IB_CM_COMPARE_SIZE / sizeof(unsigned long); i++)
- ((unsigned long *) dst)[i] = ((unsigned long *) src)[i] &
- ((unsigned long *) mask)[i];
- }
- static int cm_compare_data(struct ib_cm_compare_data *src_data,
- struct ib_cm_compare_data *dst_data)
- {
- u8 src[IB_CM_COMPARE_SIZE];
- u8 dst[IB_CM_COMPARE_SIZE];
- if (!src_data || !dst_data)
- return 0;
- cm_mask_copy(src, src_data->data, dst_data->mask);
- cm_mask_copy(dst, dst_data->data, src_data->mask);
- return memcmp(src, dst, IB_CM_COMPARE_SIZE);
- }
- static int cm_compare_private_data(u8 *private_data,
- struct ib_cm_compare_data *dst_data)
- {
- u8 src[IB_CM_COMPARE_SIZE];
- if (!dst_data)
- return 0;
- cm_mask_copy(src, private_data, dst_data->mask);
- return memcmp(src, dst_data->data, IB_CM_COMPARE_SIZE);
- }
- /*
- * Trivial helpers to strip endian annotation and compare; the
- * endianness doesn't actually matter since we just need a stable
- * order for the RB tree.
- */
- static int be32_lt(__be32 a, __be32 b)
- {
- return (__force u32) a < (__force u32) b;
- }
- static int be32_gt(__be32 a, __be32 b)
- {
- return (__force u32) a > (__force u32) b;
- }
- static int be64_lt(__be64 a, __be64 b)
- {
- return (__force u64) a < (__force u64) b;
- }
- static int be64_gt(__be64 a, __be64 b)
- {
- return (__force u64) a > (__force u64) b;
- }
- static struct cm_id_private * cm_insert_listen(struct cm_id_private *cm_id_priv)
- {
- struct rb_node **link = &cm.listen_service_table.rb_node;
- struct rb_node *parent = NULL;
- struct cm_id_private *cur_cm_id_priv;
- __be64 service_id = cm_id_priv->id.service_id;
- __be64 service_mask = cm_id_priv->id.service_mask;
- int data_cmp;
- while (*link) {
- parent = *link;
- cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
- service_node);
- data_cmp = cm_compare_data(cm_id_priv->compare_data,
- cur_cm_id_priv->compare_data);
- if ((cur_cm_id_priv->id.service_mask & service_id) ==
- (service_mask & cur_cm_id_priv->id.service_id) &&
- (cm_id_priv->id.device == cur_cm_id_priv->id.device) &&
- !data_cmp)
- return cur_cm_id_priv;
- if (cm_id_priv->id.device < cur_cm_id_priv->id.device)
- link = &(*link)->rb_left;
- else if (cm_id_priv->id.device > cur_cm_id_priv->id.device)
- link = &(*link)->rb_right;
- else if (be64_lt(service_id, cur_cm_id_priv->id.service_id))
- link = &(*link)->rb_left;
- else if (be64_gt(service_id, cur_cm_id_priv->id.service_id))
- link = &(*link)->rb_right;
- else if (data_cmp < 0)
- link = &(*link)->rb_left;
- else
- link = &(*link)->rb_right;
- }
- rb_link_node(&cm_id_priv->service_node, parent, link);
- rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table);
- return NULL;
- }
- static struct cm_id_private * cm_find_listen(struct ib_device *device,
- __be64 service_id,
- u8 *private_data)
- {
- struct rb_node *node = cm.listen_service_table.rb_node;
- struct cm_id_private *cm_id_priv;
- int data_cmp;
- while (node) {
- cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
- data_cmp = cm_compare_private_data(private_data,
- cm_id_priv->compare_data);
- if ((cm_id_priv->id.service_mask & service_id) ==
- cm_id_priv->id.service_id &&
- (cm_id_priv->id.device == device) && !data_cmp)
- return cm_id_priv;
- if (device < cm_id_priv->id.device)
- node = node->rb_left;
- else if (device > cm_id_priv->id.device)
- node = node->rb_right;
- else if (be64_lt(service_id, cm_id_priv->id.service_id))
- node = node->rb_left;
- else if (be64_gt(service_id, cm_id_priv->id.service_id))
- node = node->rb_right;
- else if (data_cmp < 0)
- node = node->rb_left;
- else
- node = node->rb_right;
- }
- return NULL;
- }
- static struct cm_timewait_info * cm_insert_remote_id(struct cm_timewait_info
- *timewait_info)
- {
- struct rb_node **link = &cm.remote_id_table.rb_node;
- struct rb_node *parent = NULL;
- struct cm_timewait_info *cur_timewait_info;
- __be64 remote_ca_guid = timewait_info->remote_ca_guid;
- __be32 remote_id = timewait_info->work.remote_id;
- while (*link) {
- parent = *link;
- cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
- remote_id_node);
- if (be32_lt(remote_id, cur_timewait_info->work.remote_id))
- link = &(*link)->rb_left;
- else if (be32_gt(remote_id, cur_timewait_info->work.remote_id))
- link = &(*link)->rb_right;
- else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
- link = &(*link)->rb_left;
- else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
- link = &(*link)->rb_right;
- else
- return cur_timewait_info;
- }
- timewait_info->inserted_remote_id = 1;
- rb_link_node(&timewait_info->remote_id_node, parent, link);
- rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table);
- return NULL;
- }
- static struct cm_timewait_info * cm_find_remote_id(__be64 remote_ca_guid,
- __be32 remote_id)
- {
- struct rb_node *node = cm.remote_id_table.rb_node;
- struct cm_timewait_info *timewait_info;
- while (node) {
- timewait_info = rb_entry(node, struct cm_timewait_info,
- remote_id_node);
- if (be32_lt(remote_id, timewait_info->work.remote_id))
- node = node->rb_left;
- else if (be32_gt(remote_id, timewait_info->work.remote_id))
- node = node->rb_right;
- else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid))
- node = node->rb_left;
- else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid))
- node = node->rb_right;
- else
- return timewait_info;
- }
- return NULL;
- }
- static struct cm_timewait_info * cm_insert_remote_qpn(struct cm_timewait_info
- *timewait_info)
- {
- struct rb_node **link = &cm.remote_qp_table.rb_node;
- struct rb_node *parent = NULL;
- struct cm_timewait_info *cur_timewait_info;
- __be64 remote_ca_guid = timewait_info->remote_ca_guid;
- __be32 remote_qpn = timewait_info->remote_qpn;
- while (*link) {
- parent = *link;
- cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
- remote_qp_node);
- if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn))
- link = &(*link)->rb_left;
- else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn))
- link = &(*link)->rb_right;
- else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
- link = &(*link)->rb_left;
- else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
- link = &(*link)->rb_right;
- else
- return cur_timewait_info;
- }
- timewait_info->inserted_remote_qp = 1;
- rb_link_node(&timewait_info->remote_qp_node, parent, link);
- rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table);
- return NULL;
- }
- static struct cm_id_private * cm_insert_remote_sidr(struct cm_id_private
- *cm_id_priv)
- {
- struct rb_node **link = &cm.remote_sidr_table.rb_node;
- struct rb_node *parent = NULL;
- struct cm_id_private *cur_cm_id_priv;
- union ib_gid *port_gid = &cm_id_priv->av.dgid;
- __be32 remote_id = cm_id_priv->id.remote_id;
- while (*link) {
- parent = *link;
- cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
- sidr_id_node);
- if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id))
- link = &(*link)->rb_left;
- else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id))
- link = &(*link)->rb_right;
- else {
- int cmp;
- cmp = memcmp(port_gid, &cur_cm_id_priv->av.dgid,
- sizeof *port_gid);
- if (cmp < 0)
- link = &(*link)->rb_left;
- else if (cmp > 0)
- link = &(*link)->rb_right;
- else
- return cur_cm_id_priv;
- }
- }
- rb_link_node(&cm_id_priv->sidr_id_node, parent, link);
- rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
- return NULL;
- }
- static void cm_reject_sidr_req(struct cm_id_private *cm_id_priv,
- enum ib_cm_sidr_status status)
- {
- struct ib_cm_sidr_rep_param param;
- memset(¶m, 0, sizeof param);
- param.status = status;
- ib_send_cm_sidr_rep(&cm_id_priv->id, ¶m);
- }
- struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
- ib_cm_handler cm_handler,
- void *context)
- {
- struct cm_id_private *cm_id_priv;
- int ret;
- cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL);
- if (!cm_id_priv)
- return ERR_PTR(-ENOMEM);
- cm_id_priv->id.state = IB_CM_IDLE;
- cm_id_priv->id.device = device;
- cm_id_priv->id.cm_handler = cm_handler;
- cm_id_priv->id.context = context;
- cm_id_priv->id.remote_cm_qpn = 1;
- ret = cm_alloc_id(cm_id_priv);
- if (ret)
- goto error;
- spin_lock_init(&cm_id_priv->lock);
- init_completion(&cm_id_priv->comp);
- INIT_LIST_HEAD(&cm_id_priv->work_list);
- atomic_set(&cm_id_priv->work_count, -1);
- atomic_set(&cm_id_priv->refcount, 1);
- return &cm_id_priv->id;
- error:
- kfree(cm_id_priv);
- return ERR_PTR(-ENOMEM);
- }
- EXPORT_SYMBOL(ib_create_cm_id);
- static struct cm_work * cm_dequeue_work(struct cm_id_private *cm_id_priv)
- {
- struct cm_work *work;
- if (list_empty(&cm_id_priv->work_list))
- return NULL;
- work = list_entry(cm_id_priv->work_list.next, struct cm_work, list);
- list_del(&work->list);
- return work;
- }
- static void cm_free_work(struct cm_work *work)
- {
- if (work->mad_recv_wc)
- ib_free_recv_mad(work->mad_recv_wc);
- kfree(work);
- }
- static inline int cm_convert_to_ms(int iba_time)
- {
- /* approximate conversion to ms from 4.096us x 2^iba_time */
- return 1 << max(iba_time - 8, 0);
- }
- /*
- * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time
- * Because of how ack_timeout is stored, adding one doubles the timeout.
- * To avoid large timeouts, select the max(ack_delay, life_time + 1), and
- * increment it (round up) only if the other is within 50%.
- */
- static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
- {
- int ack_timeout = packet_life_time + 1;
- if (ack_timeout >= ca_ack_delay)
- ack_timeout += (ca_ack_delay >= (ack_timeout - 1));
- else
- ack_timeout = ca_ack_delay +
- (ack_timeout >= (ca_ack_delay - 1));
- return min(31, ack_timeout);
- }
- static void cm_cleanup_timewait(struct cm_timewait_info *timewait_info)
- {
- if (timewait_info->inserted_remote_id) {
- rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
- timewait_info->inserted_remote_id = 0;
- }
- if (timewait_info->inserted_remote_qp) {
- rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table);
- timewait_info->inserted_remote_qp = 0;
- }
- }
- static struct cm_timewait_info * cm_create_timewait_info(__be32 local_id)
- {
- struct cm_timewait_info *timewait_info;
- timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL);
- if (!timewait_info)
- return ERR_PTR(-ENOMEM);
- timewait_info->work.local_id = local_id;
- INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler);
- timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT;
- return timewait_info;
- }
- static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
- {
- int wait_time;
- unsigned long flags;
- spin_lock_irqsave(&cm.lock, flags);
- cm_cleanup_timewait(cm_id_priv->timewait_info);
- list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list);
- spin_unlock_irqrestore(&cm.lock, flags);
- /*
- * The cm_id could be destroyed by the user before we exit timewait.
- * To protect against this, we search for the cm_id after exiting
- * timewait before notifying the user that we've exited timewait.
- */
- cm_id_priv->id.state = IB_CM_TIMEWAIT;
- wait_time = cm_convert_to_ms(cm_id_priv->av.timeout);
- queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work,
- msecs_to_jiffies(wait_time));
- cm_id_priv->timewait_info = NULL;
- }
- static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
- {
- unsigned long flags;
- cm_id_priv->id.state = IB_CM_IDLE;
- if (cm_id_priv->timewait_info) {
- spin_lock_irqsave(&cm.lock, flags);
- cm_cleanup_timewait(cm_id_priv->timewait_info);
- spin_unlock_irqrestore(&cm.lock, flags);
- kfree(cm_id_priv->timewait_info);
- cm_id_priv->timewait_info = NULL;
- }
- }
- static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
- {
- struct cm_id_private *cm_id_priv;
- struct cm_work *work;
- cm_id_priv = container_of(cm_id, struct cm_id_private, id);
- retest:
- spin_lock_irq(&cm_id_priv->lock);
- switch (cm_id->state) {
- case IB_CM_LISTEN:
- cm_id->state = IB_CM_IDLE;
- spin_unlock_irq(&cm_id_priv->lock);
- spin_lock_irq(&cm.lock);
- rb_erase(&cm_id_priv->service_node, &cm.listen_service_table);
- spin_unlock_irq(&cm.lock);
- break;
- case IB_CM_SIDR_REQ_SENT:
- cm_id->state = IB_CM_IDLE;
- ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
- spin_unlock_irq(&cm_id_priv->lock);
- break;
- case IB_CM_SIDR_REQ_RCVD:
- spin_unlock_irq(&cm_id_priv->lock);
- cm_reject_sidr_req(cm_id_priv, IB_SIDR_REJECT);
- break;
- case IB_CM_REQ_SENT:
- ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
- spin_unlock_irq(&cm_id_priv->lock);
- ib_send_cm_rej(cm_id, IB_CM_REJ_TIMEOUT,
- &cm_id_priv->id.device->node_guid,
- sizeof cm_id_priv->id.device->node_guid,
- NULL, 0);
- break;
- case IB_CM_REQ_RCVD:
- if (err == -ENOMEM) {
- /* Do not reject to allow future retries. */
- cm_reset_to_idle(cm_id_priv);
- spin_unlock_irq(&cm_id_priv->lock);
- } else {
- spin_unlock_irq(&cm_id_priv->lock);
- ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,
- NULL, 0, NULL, 0);
- }
- break;
- case IB_CM_MRA_REQ_RCVD:
- case IB_CM_REP_SENT:
- case IB_CM_MRA_REP_RCVD:
- ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
- /* Fall through */
- case IB_CM_MRA_REQ_SENT:
- case IB_CM_REP_RCVD:
- case IB_CM_MRA_REP_SENT:
- spin_unlock_irq(&cm_id_priv->lock);
- ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,
- NULL, 0, NULL, 0);
- break;
- case IB_CM_ESTABLISHED:
- spin_unlock_irq(&cm_id_priv->lock);
- ib_send_cm_dreq(cm_id, NULL, 0);
- goto retest;
- case IB_CM_DREQ_SENT:
- ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
- cm_enter_timewait(cm_id_priv);
- spin_unlock_irq(&cm_id_priv->lock);
- break;
- case IB_CM_DREQ_RCVD:
- spin_unlock_irq(&cm_id_priv->lock);
- ib_send_cm_drep(cm_id, NULL, 0);
- break;
- default:
- spin_unlock_irq(&cm_id_priv->lock);
- break;
- }
- cm_free_id(cm_id->local_id);
- cm_deref_id(cm_id_priv);
- wait_for_completion(&cm_id_priv->comp);
- while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
- cm_free_work(work);
- kfree(cm_id_priv->compare_data);
- kfree(cm_id_priv->private_data);
- kfree(cm_id_priv);
- }
- void ib_destroy_cm_id(struct ib_cm_id *cm_id)
- {
- cm_destroy_id(cm_id, 0);
- }
- EXPORT_SYMBOL(ib_destroy_cm_id);
- int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, __be64 service_mask,
- struct ib_cm_compare_data *compare_data)
- {
- struct cm_id_private *cm_id_priv, *cur_cm_id_priv;
- unsigned long flags;
- int ret = 0;
- service_mask = service_mask ? service_mask : ~cpu_to_be64(0);
- service_id &= service_mask;
- if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &&
- (service_id != IB_CM_ASSIGN_SERVICE_ID))
- return -EINVAL;
- cm_id_priv = container_of(cm_id, struct cm_id_private, id);
- if (cm_id->state != IB_CM_IDLE)
- return -EINVAL;
- if (compare_data) {
- cm_id_priv->compare_data = kzalloc(sizeof *compare_data,
- GFP_KERNEL);
- if (!cm_id_priv->compare_data)
- return -ENOMEM;
- cm_mask_copy(cm_id_priv->compare_data->data,
- compare_data->data, compare_data->mask);
- memcpy(cm_id_priv->compare_data->mask, compare_data->mask,
- IB_CM_COMPARE_SIZE);
- }
- cm_id->state = IB_CM_LISTEN;
- spin_lock_irqsave(&cm.lock, flags);
- if (service_id == IB_CM_ASSIGN_SERVICE_ID) {
- cm_id->service_id = cpu_to_be64(cm.listen_service_id++);
- cm_id->service_mask = ~cpu_to_be64(0);
- } else {
- cm_id->service_id = service_id;
- cm_id->service_mask = service_mask;
- }
- cur_cm_id_priv = cm_insert_listen(cm_id_priv);
- spin_unlock_irqrestore(&cm.lock, flags);
- if (cur_cm_id_priv) {
- cm_id->state = IB_CM_IDLE;
- kfree(cm_id_priv->compare_data);
- cm_id_priv->compare_data = NULL;
- ret = -EBUSY;
- }
- return ret;
- }
- EXPORT_SYMBOL(ib_cm_listen);
- static __be64 cm_form_tid(struct cm_id_private *cm_id_priv,
- enum cm_msg_sequence msg_seq)
- {
- u64 hi_tid, low_tid;
- hi_tid = ((u64) cm_id_priv->av.port->mad_agent->hi_tid) << 32;
- low_tid = (u64) ((__force u32)cm_id_priv->id.local_id |
- (msg_seq << 30));
- return cpu_to_be64(hi_tid | low_tid);
- }
- static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
- __be16 attr_id, __be64 tid)
- {
- hdr->base_version = IB_MGMT_BASE_VERSION;
- hdr->mgmt_class = IB_MGMT_CLASS_CM;
- hdr->class_version = IB_CM_CLASS_VERSION;
- hdr->method = IB_MGMT_METHOD_SEND;
- hdr->attr_id = attr_id;
- hdr->tid = tid;
- }
- static void cm_format_req(struct cm_req_msg *req_msg,
- struct cm_id_private *cm_id_priv,
- struct ib_cm_req_param *param)
- {
- struct ib_sa_path_rec *pri_path = param->primary_path;
- struct ib_sa_path_rec *alt_path = param->alternate_path;
- cm_format_mad_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
- cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_REQ));
- req_msg->local_comm_id = cm_id_priv->id.local_id;
- req_msg->service_id = param->service_id;
- req_msg->local_ca_guid = cm_id_priv->id.device->node_guid;
- cm_req_set_local_qpn(req_msg, cpu_to_be32(param->qp_num));
- cm_req_set_resp_res(req_msg, param->responder_resources);
- cm_req_set_init_depth(req_msg, param->initiator_depth);
- cm_req_set_remote_resp_timeout(req_msg,
- param->remote_cm_response_timeout);
- cm_req_set_qp_type(req_msg, param->qp_type);
- cm_req_set_flow_ctrl(req_msg, param->flow_control);
- cm_req_set_starting_psn(req_msg, cpu_to_be32(param->starting_psn));
- cm_req_set_local_resp_timeout(req_msg,
- param->local_cm_response_timeout);
- cm_req_set_retry_count(req_msg, param->retry_count);
- req_msg->pkey = param->primary_path->pkey;
- cm_req_set_path_mtu(req_msg, param->primary_path->mtu);
- cm_req_set_rnr_retry_count(req_msg, param->rnr_retry_count);
- cm_req_set_max_cm_retries(req_msg, param->max_cm_retries);
- cm_req_set_srq(req_msg, param->srq);
- if (pri_path->hop_limit <= 1) {
- req_msg->primary_local_lid = pri_path->slid;
- req_msg->primary_remote_lid = pri_path->dlid;
- } else {
- /* Work-around until there's a way to obtain remote LID info */
- req_msg->primary_local_lid = IB_LID_PERMISSIVE;
- req_msg->primary_remote_lid = IB_LID_PERMISSIVE;
- }
- req_msg->primary_local_gid = pri_path->sgid;
- req_msg->primary_remote_gid = pri_path->dgid;
- cm_req_set_primary_flow_label(req_msg, pri_path->flow_label);
- cm_req_set_primary_packet_rate(req_msg, pri_path->rate);
- req_msg->primary_traffic_class = pri_path->traffic_class;
- req_msg->primary_hop_limit = pri_path->hop_limit;
- cm_req_set_primary_sl(req_msg, pri_path->sl);
- cm_req_set_primary_subnet_local(req_msg, (pri_path->hop_limit <= 1));
- cm_req_set_primary_local_ack_timeout(req_msg,
- cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
- pri_path->packet_life_time));
- if (alt_path) {
- if (alt_path->hop_limit <= 1) {
- req_msg->alt_local_lid = alt_path->slid;
- req_msg->alt_remote_lid = alt_path->dlid;
- } else {
- req_msg->alt_local_lid = IB_LID_PERMISSIVE;
- req_msg->alt_remote_lid = IB_LID_PERMISSIVE;
- }
- req_msg->alt_local_gid = alt_path->sgid;
- req_msg->alt_remote_gid = alt_path->dgid;
- cm_req_set_alt_flow_label(req_msg,
- alt_path->flow_label);
- cm_req_set_alt_packet_rate(req_msg, alt_path->rate);
- req_msg->alt_traffic_class = alt_path->traffic_class;
- req_msg->alt_hop_limit = alt_path->hop_limit;
- cm_req_set_alt_sl(req_msg, alt_path->sl);
- cm_req_set_alt_subnet_local(req_msg, (alt_path->hop_limit <= 1));
- cm_req_set_alt_local_ack_timeout(req_msg,
- cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
- alt_path->packet_life_time));
- }
- if (param->private_data && param->private_data_len)
- memcpy(req_msg->private_data, param->private_data,
- param->private_data_len);
- }
- static int cm_validate_req_param(struct ib_cm_req_param *param)
- {
- /* peer-to-peer not supported */
- if (param->peer_to_peer)
- return -EINVAL;
- if (!param->primary_path)
- return -EINVAL;
- if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC)
- return -EINVAL;
- if (param->private_data &&
- param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE)
- return -EINVAL;
- if (param->alternate_path &&
- (param->alternate_path->pkey != param->primary_path->pkey ||
- param->alternate_path->mtu != param->primary_path->mtu))
- return -EINVAL;
- return 0;
- }
- int ib_send_cm_req(struct ib_cm_id *cm_id,
- struct ib_cm_req_param *param)
- {
- struct cm_id_private *cm_id_priv;
- struct cm_req_msg *req_msg;
- unsigned long flags;
- int ret;
- ret = cm_validate_req_param(param);
- if (ret)
- return ret;
- /* Verify that we're not in timewait. */
- cm_id_priv = container_of(cm_id, struct cm_id_private, id);
- spin_lock_irqsave(&cm_id_priv->lock, flags);
- if (cm_id->state != IB_CM_IDLE) {
- spin_unlock_irqrestore(&cm_id_priv->lock, flags);
- ret = -EINVAL;
- goto out;
- }
- spin_unlock_irqrestore(&cm_id_priv->lock, flags);
- cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
- id.local_id);
- if (IS_ERR(cm_id_priv->timewait_info)) {
- ret = PTR_ERR(cm_id_priv->timewait_info);
- goto out;
- }
- ret = cm_init_av_by_path(param->primary_path, &cm_id_priv->av);
- if (ret)
- goto error1;
- if (param->alternate_path) {
- ret = cm_init_av_by_path(param->alternate_path,
- &cm_id_priv->alt_av);
- if (ret)
- goto error1;
- }
- cm_id->service_id = param->service_id;
- cm_id->service_mask = ~cpu_to_be64(0);
- cm_id_priv->timeout_ms = cm_convert_to_ms(
- param->primary_path->packet_life_time) * 2 +
- cm_convert_to_ms(
- param->remote_cm_response_timeout);
- cm_id_priv->max_cm_retries = param->max_cm_retries;
- cm_id_priv->initiator_depth = param->initiator_depth;
- cm_id_priv->responder_resources = param->responder_resources;
- cm_id_priv->retry_count = param->retry_count;
- cm_id_priv->path_mtu = param->primary_path->mtu;
- cm_id_priv->pkey = param->primary_path->pkey;
- cm_id_priv->qp_type = param->qp_type;
- ret = cm_alloc_msg(cm_id_priv, &cm_id_priv->msg);
- if (ret)
- goto error1;
- req_msg = (struct cm_req_msg *) cm_id_priv->msg->mad;
- cm_format_req(req_msg, cm_id_priv, param);
- cm_id_priv->tid = req_msg->hdr.tid;
- cm_id_priv->msg->timeout_ms = cm_id_priv->timeout_ms;
- cm_id_priv->msg->context[1] = (void *) (unsigned long) IB_CM_REQ_SENT;
- cm_id_priv->local_qpn = cm_req_get_local_qpn(req_msg);
- cm_id_priv->rq_psn = cm_req_get_starting_psn(req_msg);
- spin_lock_irqsave(&cm_id_priv->lock, flags);
- ret = ib_post_send_mad(cm_id_priv->msg, NULL);
- if (ret) {
- spin_unlock_irqrestore(&cm_id_priv->lock, flags);
- goto error2;
- }
- BUG_ON(cm_id->state != IB_CM_IDLE);
- cm_id->state = IB_CM_REQ_SENT;
- spin_unlock_irqrestore(&cm_id_priv->lock, flags);
- return 0;
- error2: cm_free_msg(cm_id_priv->msg);
- error1: kfree(cm_id_priv->timewait_info);
- out: return ret;
- }
- EXPORT_SYMBOL(ib_send_cm_req);
- static int cm_issue_rej(struct cm_port *port,
- struct ib_mad_recv_wc *mad_recv_wc,
- enum ib_cm_rej_reason reason,
- enum cm_msg_response msg_rejected,
- void *ari, u8 ari_length)
- {
- struct ib_mad_send_buf *msg = NULL;
- struct cm_rej_msg *rej_msg, *rcv_msg;
- int ret;
- ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
- if (ret)
- return ret;
- /* We just need common CM header information. Cast to any message. */
- rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad;
- rej_msg = (struct cm_rej_msg *) msg->mad;
- cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid);
- rej_msg->remote_comm_id = rcv_msg->local_comm_id;
- rej_msg->local_comm_id = rcv_msg->remote_comm_id;
- cm_rej_set_msg_rejected(rej_msg, msg_rejected);
- rej_msg->reason = cpu_to_be16(reason);
- if (ari && ari_length) {
- cm_rej_set_reject_info_len(rej_msg, ari_length);
- memcpy(rej_msg->ari, ari, ari_length);
- }
- ret = ib_post_send_mad(msg, NULL);
- if (ret)
- cm_free_msg(msg);
- return ret;
- }
- static inline int cm_is_active_peer(__be64 local_ca_guid, __be64 remote_ca_guid,
- __be32 local_qpn, __be32 remote_qpn)
- {
- return (be64_to_cpu(local_ca_guid) > be64_to_cpu(remote_ca_guid) ||
- ((local_ca_guid == remote_ca_guid) &&
- (be32_to_cpu(local_qpn) > be32_to_cpu(remote_qpn))));
- }
- static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
- struct ib_sa_path_rec *primary_path,
- struct ib_sa_path_rec *alt_path)
- {
- memset(primary_path, 0, sizeof *primary_path);
- primary_path->dgid = req_msg->primary_local_gid;
- primary_path->sgid = req_msg->primary_remote_gid;
- primary_path->dlid = req_msg->primary_local_lid;
- primary_path->slid = req_msg->primary_remote_lid;
- primary_path->flow_label = cm_req_get_primary_flow_label(req_msg);
- primary_path->hop_limit = req_msg->primary_hop_limit;
- primary_path->traffic_class = req_msg->primary_traffic_class;
- primary_path->reversible = 1;
- primary_path->pkey = req_msg->pkey;
- primary_path->sl = cm_req_get_primary_sl(req_msg);
- primary_path->mtu_selector = IB_SA_EQ;
- primary_path->mtu = cm_req_get_path_mtu(req_msg);
- primary_path->rate_selector = IB_SA_EQ;
- primary_path->rate = cm_req_get_primary_packet_rate(req_msg);
- primary_path->packet_life_time_selector = IB_SA_EQ;
- primary_path->packet_life_time =
- cm_req_get_primary_local_ack_timeout(req_msg);
- primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
- if (req_msg->alt_local_lid) {
- memset(alt_path, 0, sizeof *alt_path);
- alt_path->dgid = req_msg->alt_local_gid;
- alt_path->sgid = req_msg->alt_remote_gid;
- alt_path->dlid = req_msg->alt_local_lid;
- alt_path->slid = req_msg->alt_remote_lid;
- alt_path->flow_label = cm_req_get_alt_flow_label(req_msg);
- alt_path->hop_limit = req_msg->alt_hop_limit;
- alt_path->traffic_class = req_msg->alt_traffic_class;
- alt_path->reversible = 1;
- alt_path->pkey = req_msg->pkey;
- alt_path->sl = cm_req_get_alt_sl(req_msg);
- alt_path->mtu_selector = IB_SA_EQ;
- alt_path->mtu = cm_req_get_path_mtu(req_msg);
- alt_path->rate_selector = IB_SA_EQ;
- alt_path->rate = cm_req_get_alt_packet_rate(req_msg);
- alt_path->packet_life_time_selector = IB_SA_EQ;
- alt_path->packet_life_time =
- cm_req_get_alt_local_ack_timeout(req_msg);
- alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
- }
- }
- static void cm_format_req_event(struct cm_work *work,
- struct cm_id_private *cm_id_priv,
- struct ib_cm_id *listen_id)
- {
- struct cm_req_msg *req_msg;
- struct ib_cm_req_event_param *param;
- req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
- param = &work->cm_event.param.req_rcvd;
- param->listen_id = listen_id;
- param->port = cm_id_priv->av.port->port_num;
- param->primary_path = &work->path[0];
- if (req_msg->alt_local_lid)
- param->alternate_path = &work->path[1];
- else
- param->alternate_path = NULL;
- param->remote_ca_guid = req_msg->local_ca_guid;
- param->remote_qkey = be32_to_cpu(req_msg->local_qkey);
- param->remote_qpn = be32_to_cpu(cm_req_get_local_qpn(req_msg));
- param->qp_type = cm_req_get_qp_type(req_msg);
- param->starting_psn = be32_to_cpu(cm_req_get_starting_psn(req_msg));
- param->responder_resources = cm_req_get_init_depth(req_msg);
- param->initiator_depth = cm_req_get_resp_res(req_msg);
- param->local_cm_response_timeout =
- cm_req_get_remote_resp_timeout(req_msg);
- param->flow_control = cm_req_get_flow_ctrl(req_msg);
- param->remote_cm_response_timeout =
- cm_req_get_local_resp_timeout(req_msg);
- param->retry_count = cm_req_get_retry_count(req_msg);
- param->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg);
- param->srq = cm_req_get_srq(req_msg);
- work->cm_event.private_data = &req_msg->private_data;
- }
- static void cm_process_work(struct cm_id_private *cm_id_priv,
- struct cm_work *work)
- {
- int ret;
- /* We will typically only have the current event to report. */
- ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
- cm_free_work(work);
- while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) {
- spin_lock_irq(&cm_id_priv->lock);
- work = cm_dequeue_work(cm_id_priv);
- spin_unlock_irq(&cm_id_priv->lock);
- BUG_ON(!work);
- ret = cm_id_priv->id.cm_handler(&cm_id_priv->id,
- &work->cm_event);
- cm_free_work(work);
- }
- cm_deref_id(cm_id_priv);
- if (ret)
- cm_destroy_id(&cm_id_priv->id, ret);
- }
- static void cm_format_mra(struct cm_mra_msg *mra_msg,
- struct cm_id_private *cm_id_priv,
- enum cm_msg_response msg_mraed, u8 service_timeout,
- const void *private_data, u8 private_data_len)
- {
- cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid);
- cm_mra_set_msg_mraed(mra_msg, msg_mraed);
- mra_msg->local_comm_id = cm_id_priv->id.local_id;
- mra_msg->remote_comm_id = cm_id_priv->id.remote_id;
- cm_mra_set_service_timeout(mra_msg, service_timeout);
- if (private_data && private_data_len)
- memcpy(mra_msg->private_data, private_data, private_data_len);
- }
- static void cm_format_rej(struct cm_rej_msg *rej_msg,
- struct cm_id_private *cm_id_priv,
- enum ib_cm_rej_reason reason,
- void *ari,
- u8 ari_length,
- const void *private_data,
- u8 private_data_len)
- {
- cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid);
- rej_msg->remote_comm_id = cm_id_priv->id.remote_id;
- switch(cm_id_priv->id.state) {
- case IB_CM_REQ_RCVD:
- rej_msg->local_comm_id = 0;
- cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ);
- break;
- case IB_CM_MRA_REQ_SENT:
- rej_msg->local_comm_id = cm_id_priv->id.local_id;
- cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ);
- break;
- case IB_CM_REP_RCVD:
- case IB_CM_MRA_REP_SENT:
- rej_msg->local_comm_id = cm_id_priv->id.local_id;
- cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REP);
- break;
- default:
- rej_msg->local_comm_id = cm_id_priv->id.local_id;
- cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_OTHER);
- break;
- }
- rej_msg->reason = cpu_to_be16(reason);
- if (ari && ari_length) {
- cm_rej_set_reject_info_len(rej_msg, ari_length);
- memcpy(rej_msg->ari, ari, ari_length);
- }
- if (private_data && private_data_len)
- memcpy(rej_msg->private_data, private_data, private_data_len);
- }
- static void cm_dup_req_handler(struct cm_work *work,
- struct cm_id_private *cm_id_priv)
- {
- struct ib_mad_send_buf *msg = NULL;
- int ret;
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
- counter[CM_REQ_COUNTER]);
- /* Quick state check to discard duplicate REQs. */
- if (cm_id_priv->id.state == IB_CM_REQ_RCVD)
- return;
- ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
- if (ret)
- return;
- spin_lock_irq(&cm_id_priv->lock);
- switch (cm_id_priv->id.state) {
- case IB_CM_MRA_REQ_SENT:
- cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
- CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout,
- cm_id_priv->private_data,
- cm_id_priv->private_data_len);
- break;
- case IB_CM_TIMEWAIT:
- cm_format_rej((struct cm_rej_msg *) msg->mad, cm_id_priv,
- IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0);
- break;
- default:
- goto unlock;
- }
- spin_unlock_irq(&cm_id_priv->lock);
- ret = ib_post_send_mad(msg, NULL);
- if (ret)
- goto free;
- return;
- unlock: spin_unlock_irq(&cm_id_priv->lock);
- free: cm_free_msg(msg);
- }
- static struct cm_id_private * cm_match_req(struct cm_work *work,
- struct cm_id_private *cm_id_priv)
- {
- struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
- struct cm_timewait_info *timewait_info;
- struct cm_req_msg *req_msg;
- req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
- /* Check for possible duplicate REQ. */
- spin_lock_irq(&cm.lock);
- timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info);
- if (timewait_info) {
- cur_cm_id_priv = cm_get_id(timewait_info->work.local_id,
- timewait_info->work.remote_id);
- spin_unlock_irq(&cm.lock);
- if (cur_cm_id_priv) {
- cm_dup_req_handler(work, cur_cm_id_priv);
- cm_deref_id(cur_cm_id_priv);
- }
- return NULL;
- }
- /* Check for stale connections. */
- timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
- if (timewait_info) {
- cm_cleanup_timewait(cm_id_priv->timewait_info);
- spin_unlock_irq(&cm.lock);
- cm_issue_rej(work->port, work->mad_recv_wc,
- IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
- NULL, 0);
- return NULL;
- }
- /* Find matching listen request. */
- listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device,
- req_msg->service_id,
- req_msg->private_data);
- if (!listen_cm_id_priv) {
- cm_cleanup_timewait(cm_id_priv->timewait_info);
- spin_unlock_irq(&cm.lock);
- cm_issue_rej(work->port, work->mad_recv_wc,
- IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,
- NULL, 0);
- goto out;
- }
- atomic_inc(&listen_cm_id_priv->refcount);
- atomic_inc(&cm_id_priv->refcount);
- cm_id_priv->id.state = IB_CM_REQ_RCVD;
- atomic_inc(&cm_id_priv->work_count);
- spin_unlock_irq(&cm.lock);
- out:
- return listen_cm_id_priv;
- }
- /*
- * Work-around for inter-subnet connections. If the LIDs are permissive,
- * we need to override the LID/SL data in the REQ with the LID information
- * in the work completion.
- */
- static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
- {
- if (!cm_req_get_primary_subnet_local(req_msg)) {
- if (req_msg->primary_local_lid == IB_LID_PERMISSIVE) {
- req_msg->primary_local_lid = cpu_to_be16(wc->slid);
- cm_req_set_primary_sl(req_msg, wc->sl);
- }
- if (req_msg->primary_remote_lid == IB_LID_PERMISSIVE)
- req_msg->primary_remote_lid = cpu_to_be16(wc->dlid_path_bits);
- }
- if (!cm_req_get_alt_subnet_local(req_msg)) {
- if (req_msg->alt_local_lid == IB_LID_PERMISSIVE) {
- req_msg->alt_local_lid = cpu_to_be16(wc->slid);
- cm_req_set_alt_sl(req_msg, wc->sl);
- }
- if (req_msg->alt_remote_lid == IB_LID_PERMISSIVE)
- req_msg->alt_remote_lid = cpu_to_be16(wc->dlid_path_bits);
- }
- }
- static int cm_req_handler(struct cm_work *work)
- {
- struct ib_cm_id *cm_id;
- struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
- struct cm_req_msg *req_msg;
- int ret;
- req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
- cm_id = ib_create_cm_id(work->port->cm_dev->ib_device, NULL, NULL);
- if (IS_ERR(cm_id))
- return PTR_ERR(cm_id);
- cm_id_priv = container_of(cm_id, struct cm_id_private, id);
- cm_id_priv->id.remote_id = req_msg->local_comm_id;
- cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
- work->mad_recv_wc->recv_buf.grh,
- &cm_id_priv->av);
- cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
- id.local_id);
- if (IS_ERR(cm_id_priv->timewait_info)) {
- ret = PTR_ERR(cm_id_priv->timewait_info);
- goto destroy;
- }
- cm_id_priv->timewait_info->work.remote_id = req_msg->local_comm_id;
- cm_id_priv->timewait_info->remote_ca_guid = req_msg->local_ca_guid;
- cm_id_priv->timewait_info->remote_qpn = cm_req_get_local_qpn(req_msg);
- listen_cm_id_priv = cm_match_req(work, cm_id_priv);
- if (!listen_cm_id_priv) {
- ret = -EINVAL;
- kfree(cm_id_priv->timewait_info);
- goto destroy;
- }
- cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
- cm_id_priv->id.context = listen_cm_id_priv->id.context;
- cm_id_priv->id.service_id = req_msg->service_id;
- cm_id_priv->id.service_mask = ~cpu_to_be64(0);
- cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
- cm_format_paths_from_req(req_msg, &work->path[0], &work->path[1]);
- ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av);
- if (ret) {
- ib_get_cached_gid(work->port->cm_dev->ib_device,
- work->port->port_num, 0, &work->path[0].sgid);
- ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,
- &work->path[0].sgid, sizeof work->path[0].sgid,
- NULL, 0);
- goto rejected;
- }
- if (req_msg->alt_local_lid) {
- ret = cm_init_av_by_path(&work->path[1], &cm_id_priv->alt_av);
- if (ret) {
- ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_ALT_GID,
- &work->path[0].sgid,
- sizeof work->path[0].sgid, NULL, 0);
- goto rejected;
- }
- }
- cm_id_priv->tid = req_msg->hdr.tid;
- cm_id_priv->timeout_ms = cm_convert_to_ms(
- cm_req_get_local_resp_timeout(req_msg));
- cm_id_priv->max_cm_retries = cm_req_get_max_cm_retries(req_msg);
- cm_id_priv->remote_qpn = cm_req_get_local_qpn(req_msg);
- cm_id_priv->initiator_depth = cm_req_get_resp_res(req_msg);
- cm_id_priv->responder_resources = cm_req_get_init_depth(req_msg);
- cm_id_priv->path_mtu = cm_req_get_path_mtu(req_msg);
- cm_id_priv->pkey = req_msg->pkey;
- cm_id_priv->sq_psn = cm_req_get_starting_psn(req_msg);
- cm_id_priv->retry_count = cm_req_get_retry_count(req_msg);
- cm_id_priv->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg);
- cm_id_priv->qp_type = cm_req_get_qp_type(req_msg);
- cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
- cm_process_work(cm_id_priv, work);
- cm_deref_id(listen_cm_id_priv);
- return 0;
- rejected:
- atomic_dec(&cm_id_priv->refcount);
- cm_deref_id(listen_cm_id_priv);
- destroy:
- ib_destroy_cm_id(cm_id);
- return ret;
- }
- static void cm_format_rep(struct cm_rep_msg *rep_msg,
- struct cm_id_private *cm_id_priv,
- struct ib_cm_rep_param *param)
- {
- cm_format_mad_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid);
- rep_msg->local_comm_id = cm_id_priv->id.local_id;
- rep_msg->remote_comm_id = cm_id_priv->id.remote_id;
- cm_rep_set_local_qpn(rep_msg, cpu_to_be32(param->qp_num));
- cm_rep_set_starting_psn(rep_msg, cpu_to_be32(param->starting_psn));
- rep_msg->resp_resources = param->responder_resources;
- rep_msg->initiator_depth = param->initiator_depth;
- cm_rep_set_target_ack_delay(rep_msg,
- cm_id_priv->av.port->cm_dev->ack_delay);
- cm_rep_set_failover(rep_msg, param->failover_accepted);
- cm_rep_set_flow_ctrl(rep_msg, param->flow_control);
- cm_rep_set_rnr_retry_count(rep_msg, param->rnr_retry_count);
- cm_rep_set_srq(rep_msg, param->srq);
- rep_msg->local_ca_guid = cm_id_priv->id.device->node_guid;
- if (param->private_data && param->private_data_len)
- memcpy(rep_msg->private_data, param->private_data,
- param->private_data_len);
- }
- int ib_send_cm_rep(struct ib_cm_id *cm_id,
- struct ib_cm_rep_param *param)
- {
- struct cm_id_private *cm_id_priv;
- struct ib_mad_send_buf *msg;
- struct cm_rep_msg *rep_msg;
- unsigned long flags;
- int ret;
- if (param->private_data &&
- param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
- return -EINVAL;
- cm_id_priv = container_of(cm_id, struct cm_id_private, id);
- spin_lock_irqsave(&cm_id_priv->lock, flags);
- if (cm_id->state != IB_CM_REQ_RCVD &&
- cm_id->state != IB_CM_MRA_REQ_SENT) {
- ret = -EINVAL;
- goto out;
- }
- ret = cm_alloc_msg(cm_id_priv, &msg);
- if (ret)
- goto out;
- rep_msg = (struct cm_rep_msg *) msg->mad;
- cm_format_rep(rep_msg, cm_id_priv, param);
- msg->timeout_ms = cm_id_priv->timeout_ms;
- msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT;
- ret = ib_post_send_mad(msg, NULL);
- if (ret) {
- spin_unlock_irqrestore(&cm_id_priv->lock, flags);
- cm_free_msg(msg);
- return ret;
- }
- cm_id->state = IB_CM_REP_SENT;
- cm_id_priv->msg = msg;
- cm_id_priv->initiator_depth = param->initiator_depth;
- cm_id_priv->responder_resources = param->responder_resources;
- cm_id_priv->rq_psn = cm_rep_get_starting_psn(rep_msg);
- cm_id_priv->local_qpn = cm_rep_get_local_qpn(rep_msg);
- out: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
- return ret;
- }
- EXPORT_SYMBOL(ib_send_cm_rep);
- static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,
- struct cm_id_private *cm_id_priv,
- const void *private_data,
- u8 private_data_len)
- {
- cm_format_m…