PageRenderTime 48ms CodeModel.GetById 16ms app.highlight 27ms RepoModel.GetById 1ms app.codeStats 1ms

/net/rds/ib.h

http://github.com/mirrors/linux
C Header | 467 lines | 352 code | 63 blank | 52 comment | 0 complexity | 064a8fa1b813e5f34b7242dd9a69f3b0 MD5 | raw file
  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef _RDS_IB_H
  3#define _RDS_IB_H
  4
  5#include <rdma/ib_verbs.h>
  6#include <rdma/rdma_cm.h>
  7#include <linux/interrupt.h>
  8#include <linux/pci.h>
  9#include <linux/slab.h>
 10#include "rds.h"
 11#include "rdma_transport.h"
 12
 13#define RDS_IB_MAX_SGE			8
 14#define RDS_IB_RECV_SGE 		2
 15
 16#define RDS_IB_DEFAULT_RECV_WR		1024
 17#define RDS_IB_DEFAULT_SEND_WR		256
 18#define RDS_IB_DEFAULT_FR_WR		512
 19
 20#define RDS_IB_DEFAULT_RETRY_COUNT	1
 21
 22#define RDS_IB_SUPPORTED_PROTOCOLS	0x00000003	/* minor versions supported */
 23
 24#define RDS_IB_RECYCLE_BATCH_COUNT	32
 25
 26#define RDS_IB_WC_MAX			32
 27
 28extern struct rw_semaphore rds_ib_devices_lock;
 29extern struct list_head rds_ib_devices;
 30
 31/*
 32 * IB posts RDS_FRAG_SIZE fragments of pages to the receive queues to
 33 * try and minimize the amount of memory tied up both the device and
 34 * socket receive queues.
 35 */
 36struct rds_page_frag {
 37	struct list_head	f_item;
 38	struct list_head	f_cache_entry;
 39	struct scatterlist	f_sg;
 40};
 41
 42struct rds_ib_incoming {
 43	struct list_head	ii_frags;
 44	struct list_head	ii_cache_entry;
 45	struct rds_incoming	ii_inc;
 46};
 47
 48struct rds_ib_cache_head {
 49	struct list_head *first;
 50	unsigned long count;
 51};
 52
 53struct rds_ib_refill_cache {
 54	struct rds_ib_cache_head __percpu *percpu;
 55	struct list_head	 *xfer;
 56	struct list_head	 *ready;
 57};
 58
 59/* This is the common structure for the IB private data exchange in setting up
 60 * an RDS connection.  The exchange is different for IPv4 and IPv6 connections.
 61 * The reason is that the address size is different and the addresses
 62 * exchanged are in the beginning of the structure.  Hence it is not possible
 63 * for interoperability if same structure is used.
 64 */
 65struct rds_ib_conn_priv_cmn {
 66	u8			ricpc_protocol_major;
 67	u8			ricpc_protocol_minor;
 68	__be16			ricpc_protocol_minor_mask;	/* bitmask */
 69	u8			ricpc_dp_toss;
 70	u8			ripc_reserved1;
 71	__be16			ripc_reserved2;
 72	__be64			ricpc_ack_seq;
 73	__be32			ricpc_credit;	/* non-zero enables flow ctl */
 74};
 75
 76struct rds_ib_connect_private {
 77	/* Add new fields at the end, and don't permute existing fields. */
 78	__be32				dp_saddr;
 79	__be32				dp_daddr;
 80	struct rds_ib_conn_priv_cmn	dp_cmn;
 81};
 82
 83struct rds6_ib_connect_private {
 84	/* Add new fields at the end, and don't permute existing fields. */
 85	struct in6_addr			dp_saddr;
 86	struct in6_addr			dp_daddr;
 87	struct rds_ib_conn_priv_cmn	dp_cmn;
 88};
 89
 90#define dp_protocol_major	dp_cmn.ricpc_protocol_major
 91#define dp_protocol_minor	dp_cmn.ricpc_protocol_minor
 92#define dp_protocol_minor_mask	dp_cmn.ricpc_protocol_minor_mask
 93#define dp_ack_seq		dp_cmn.ricpc_ack_seq
 94#define dp_credit		dp_cmn.ricpc_credit
 95
 96union rds_ib_conn_priv {
 97	struct rds_ib_connect_private	ricp_v4;
 98	struct rds6_ib_connect_private	ricp_v6;
 99};
100
101struct rds_ib_send_work {
102	void			*s_op;
103	union {
104		struct ib_send_wr	s_wr;
105		struct ib_rdma_wr	s_rdma_wr;
106		struct ib_atomic_wr	s_atomic_wr;
107	};
108	struct ib_sge		s_sge[RDS_IB_MAX_SGE];
109	unsigned long		s_queued;
110};
111
112struct rds_ib_recv_work {
113	struct rds_ib_incoming 	*r_ibinc;
114	struct rds_page_frag	*r_frag;
115	struct ib_recv_wr	r_wr;
116	struct ib_sge		r_sge[2];
117};
118
119struct rds_ib_work_ring {
120	u32		w_nr;
121	u32		w_alloc_ptr;
122	u32		w_alloc_ctr;
123	u32		w_free_ptr;
124	atomic_t	w_free_ctr;
125};
126
127/* Rings are posted with all the allocations they'll need to queue the
128 * incoming message to the receiving socket so this can't fail.
129 * All fragments start with a header, so we can make sure we're not receiving
130 * garbage, and we can tell a small 8 byte fragment from an ACK frame.
131 */
132struct rds_ib_ack_state {
133	u64		ack_next;
134	u64		ack_recv;
135	unsigned int	ack_required:1;
136	unsigned int	ack_next_valid:1;
137	unsigned int	ack_recv_valid:1;
138};
139
140
141struct rds_ib_device;
142
143struct rds_ib_connection {
144
145	struct list_head	ib_node;
146	struct rds_ib_device	*rds_ibdev;
147	struct rds_connection	*conn;
148
149	/* alphabet soup, IBTA style */
150	struct rdma_cm_id	*i_cm_id;
151	struct ib_pd		*i_pd;
152	struct ib_cq		*i_send_cq;
153	struct ib_cq		*i_recv_cq;
154	struct ib_wc		i_send_wc[RDS_IB_WC_MAX];
155	struct ib_wc		i_recv_wc[RDS_IB_WC_MAX];
156
157	/* To control the number of wrs from fastreg */
158	atomic_t		i_fastreg_wrs;
159	atomic_t		i_fastreg_inuse_count;
160
161	/* interrupt handling */
162	struct tasklet_struct	i_send_tasklet;
163	struct tasklet_struct	i_recv_tasklet;
164
165	/* tx */
166	struct rds_ib_work_ring	i_send_ring;
167	struct rm_data_op	*i_data_op;
168	struct rds_header	**i_send_hdrs;
169	dma_addr_t		*i_send_hdrs_dma;
170	struct rds_ib_send_work *i_sends;
171	atomic_t		i_signaled_sends;
172
173	/* rx */
174	struct mutex		i_recv_mutex;
175	struct rds_ib_work_ring	i_recv_ring;
176	struct rds_ib_incoming	*i_ibinc;
177	u32			i_recv_data_rem;
178	struct rds_header	**i_recv_hdrs;
179	dma_addr_t		*i_recv_hdrs_dma;
180	struct rds_ib_recv_work *i_recvs;
181	u64			i_ack_recv;	/* last ACK received */
182	struct rds_ib_refill_cache i_cache_incs;
183	struct rds_ib_refill_cache i_cache_frags;
184	atomic_t		i_cache_allocs;
185
186	/* sending acks */
187	unsigned long		i_ack_flags;
188#ifdef KERNEL_HAS_ATOMIC64
189	atomic64_t		i_ack_next;	/* next ACK to send */
190#else
191	spinlock_t		i_ack_lock;	/* protect i_ack_next */
192	u64			i_ack_next;	/* next ACK to send */
193#endif
194	struct rds_header	*i_ack;
195	struct ib_send_wr	i_ack_wr;
196	struct ib_sge		i_ack_sge;
197	dma_addr_t		i_ack_dma;
198	unsigned long		i_ack_queued;
199
200	/* Flow control related information
201	 *
202	 * Our algorithm uses a pair variables that we need to access
203	 * atomically - one for the send credits, and one posted
204	 * recv credits we need to transfer to remote.
205	 * Rather than protect them using a slow spinlock, we put both into
206	 * a single atomic_t and update it using cmpxchg
207	 */
208	atomic_t		i_credits;
209
210	/* Protocol version specific information */
211	unsigned int		i_flowctl:1;	/* enable/disable flow ctl */
212
213	/* Batched completions */
214	unsigned int		i_unsignaled_wrs;
215
216	/* Endpoint role in connection */
217	bool			i_active_side;
218	atomic_t		i_cq_quiesce;
219
220	/* Send/Recv vectors */
221	int			i_scq_vector;
222	int			i_rcq_vector;
223	u8			i_sl;
224};
225
226/* This assumes that atomic_t is at least 32 bits */
227#define IB_GET_SEND_CREDITS(v)	((v) & 0xffff)
228#define IB_GET_POST_CREDITS(v)	((v) >> 16)
229#define IB_SET_SEND_CREDITS(v)	((v) & 0xffff)
230#define IB_SET_POST_CREDITS(v)	((v) << 16)
231
232struct rds_ib_ipaddr {
233	struct list_head	list;
234	__be32			ipaddr;
235	struct rcu_head		rcu;
236};
237
238enum {
239	RDS_IB_MR_8K_POOL,
240	RDS_IB_MR_1M_POOL,
241};
242
243struct rds_ib_device {
244	struct list_head	list;
245	struct list_head	ipaddr_list;
246	struct list_head	conn_list;
247	struct ib_device	*dev;
248	struct ib_pd		*pd;
249	struct dma_pool		*rid_hdrs_pool; /* RDS headers DMA pool */
250	u8			use_fastreg:1;
251	u8			odp_capable:1;
252
253	unsigned int		max_mrs;
254	struct rds_ib_mr_pool	*mr_1m_pool;
255	struct rds_ib_mr_pool   *mr_8k_pool;
256	unsigned int		fmr_max_remaps;
257	unsigned int		max_8k_mrs;
258	unsigned int		max_1m_mrs;
259	int			max_sge;
260	unsigned int		max_wrs;
261	unsigned int		max_initiator_depth;
262	unsigned int		max_responder_resources;
263	spinlock_t		spinlock;	/* protect the above */
264	refcount_t		refcount;
265	struct work_struct	free_work;
266	int			*vector_load;
267};
268
269#define ibdev_to_node(ibdev) dev_to_node((ibdev)->dev.parent)
270#define rdsibdev_to_node(rdsibdev) ibdev_to_node(rdsibdev->dev)
271
272/* bits for i_ack_flags */
273#define IB_ACK_IN_FLIGHT	0
274#define IB_ACK_REQUESTED	1
275
276/* Magic WR_ID for ACKs */
277#define RDS_IB_ACK_WR_ID	(~(u64) 0)
278
279struct rds_ib_statistics {
280	uint64_t	s_ib_connect_raced;
281	uint64_t	s_ib_listen_closed_stale;
282	uint64_t	s_ib_evt_handler_call;
283	uint64_t	s_ib_tasklet_call;
284	uint64_t	s_ib_tx_cq_event;
285	uint64_t	s_ib_tx_ring_full;
286	uint64_t	s_ib_tx_throttle;
287	uint64_t	s_ib_tx_sg_mapping_failure;
288	uint64_t	s_ib_tx_stalled;
289	uint64_t	s_ib_tx_credit_updates;
290	uint64_t	s_ib_rx_cq_event;
291	uint64_t	s_ib_rx_ring_empty;
292	uint64_t	s_ib_rx_refill_from_cq;
293	uint64_t	s_ib_rx_refill_from_thread;
294	uint64_t	s_ib_rx_alloc_limit;
295	uint64_t	s_ib_rx_total_frags;
296	uint64_t	s_ib_rx_total_incs;
297	uint64_t	s_ib_rx_credit_updates;
298	uint64_t	s_ib_ack_sent;
299	uint64_t	s_ib_ack_send_failure;
300	uint64_t	s_ib_ack_send_delayed;
301	uint64_t	s_ib_ack_send_piggybacked;
302	uint64_t	s_ib_ack_received;
303	uint64_t	s_ib_rdma_mr_8k_alloc;
304	uint64_t	s_ib_rdma_mr_8k_free;
305	uint64_t	s_ib_rdma_mr_8k_used;
306	uint64_t	s_ib_rdma_mr_8k_pool_flush;
307	uint64_t	s_ib_rdma_mr_8k_pool_wait;
308	uint64_t	s_ib_rdma_mr_8k_pool_depleted;
309	uint64_t	s_ib_rdma_mr_1m_alloc;
310	uint64_t	s_ib_rdma_mr_1m_free;
311	uint64_t	s_ib_rdma_mr_1m_used;
312	uint64_t	s_ib_rdma_mr_1m_pool_flush;
313	uint64_t	s_ib_rdma_mr_1m_pool_wait;
314	uint64_t	s_ib_rdma_mr_1m_pool_depleted;
315	uint64_t	s_ib_rdma_mr_8k_reused;
316	uint64_t	s_ib_rdma_mr_1m_reused;
317	uint64_t	s_ib_atomic_cswp;
318	uint64_t	s_ib_atomic_fadd;
319	uint64_t	s_ib_recv_added_to_cache;
320	uint64_t	s_ib_recv_removed_from_cache;
321};
322
323extern struct workqueue_struct *rds_ib_wq;
324
325/*
326 * Fake ib_dma_sync_sg_for_{cpu,device} as long as ib_verbs.h
327 * doesn't define it.
328 */
329static inline void rds_ib_dma_sync_sg_for_cpu(struct ib_device *dev,
330					      struct scatterlist *sglist,
331					      unsigned int sg_dma_len,
332					      int direction)
333{
334	struct scatterlist *sg;
335	unsigned int i;
336
337	for_each_sg(sglist, sg, sg_dma_len, i) {
338		ib_dma_sync_single_for_cpu(dev, sg_dma_address(sg),
339					   sg_dma_len(sg), direction);
340	}
341}
342#define ib_dma_sync_sg_for_cpu	rds_ib_dma_sync_sg_for_cpu
343
344static inline void rds_ib_dma_sync_sg_for_device(struct ib_device *dev,
345						 struct scatterlist *sglist,
346						 unsigned int sg_dma_len,
347						 int direction)
348{
349	struct scatterlist *sg;
350	unsigned int i;
351
352	for_each_sg(sglist, sg, sg_dma_len, i) {
353		ib_dma_sync_single_for_device(dev, sg_dma_address(sg),
354					      sg_dma_len(sg), direction);
355	}
356}
357#define ib_dma_sync_sg_for_device	rds_ib_dma_sync_sg_for_device
358
359
360/* ib.c */
361extern struct rds_transport rds_ib_transport;
362struct rds_ib_device *rds_ib_get_client_data(struct ib_device *device);
363void rds_ib_dev_put(struct rds_ib_device *rds_ibdev);
364extern struct ib_client rds_ib_client;
365
366extern unsigned int rds_ib_retry_count;
367
368extern spinlock_t ib_nodev_conns_lock;
369extern struct list_head ib_nodev_conns;
370
371/* ib_cm.c */
372int rds_ib_conn_alloc(struct rds_connection *conn, gfp_t gfp);
373void rds_ib_conn_free(void *arg);
374int rds_ib_conn_path_connect(struct rds_conn_path *cp);
375void rds_ib_conn_path_shutdown(struct rds_conn_path *cp);
376void rds_ib_state_change(struct sock *sk);
377int rds_ib_listen_init(void);
378void rds_ib_listen_stop(void);
379__printf(2, 3)
380void __rds_ib_conn_error(struct rds_connection *conn, const char *, ...);
381int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
382			     struct rdma_cm_event *event, bool isv6);
383int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id, bool isv6);
384void rds_ib_cm_connect_complete(struct rds_connection *conn,
385				struct rdma_cm_event *event);
386struct rds_header **rds_dma_hdrs_alloc(struct ib_device *ibdev,
387				       struct dma_pool *pool,
388				       dma_addr_t **dma_addrs, u32 num_hdrs);
389void rds_dma_hdrs_free(struct dma_pool *pool, struct rds_header **hdrs,
390		       dma_addr_t *dma_addrs, u32 num_hdrs);
391
392#define rds_ib_conn_error(conn, fmt...) \
393	__rds_ib_conn_error(conn, KERN_WARNING "RDS/IB: " fmt)
394
395/* ib_rdma.c */
396int rds_ib_update_ipaddr(struct rds_ib_device *rds_ibdev,
397			 struct in6_addr *ipaddr);
398void rds_ib_add_conn(struct rds_ib_device *rds_ibdev, struct rds_connection *conn);
399void rds_ib_remove_conn(struct rds_ib_device *rds_ibdev, struct rds_connection *conn);
400void rds_ib_destroy_nodev_conns(void);
401void rds_ib_mr_cqe_handler(struct rds_ib_connection *ic, struct ib_wc *wc);
402
403/* ib_recv.c */
404int rds_ib_recv_init(void);
405void rds_ib_recv_exit(void);
406int rds_ib_recv_path(struct rds_conn_path *conn);
407int rds_ib_recv_alloc_caches(struct rds_ib_connection *ic, gfp_t gfp);
408void rds_ib_recv_free_caches(struct rds_ib_connection *ic);
409void rds_ib_recv_refill(struct rds_connection *conn, int prefill, gfp_t gfp);
410void rds_ib_inc_free(struct rds_incoming *inc);
411int rds_ib_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to);
412void rds_ib_recv_cqe_handler(struct rds_ib_connection *ic, struct ib_wc *wc,
413			     struct rds_ib_ack_state *state);
414void rds_ib_recv_tasklet_fn(unsigned long data);
415void rds_ib_recv_init_ring(struct rds_ib_connection *ic);
416void rds_ib_recv_clear_ring(struct rds_ib_connection *ic);
417void rds_ib_recv_init_ack(struct rds_ib_connection *ic);
418void rds_ib_attempt_ack(struct rds_ib_connection *ic);
419void rds_ib_ack_send_complete(struct rds_ib_connection *ic);
420u64 rds_ib_piggyb_ack(struct rds_ib_connection *ic);
421void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq, int ack_required);
422
423/* ib_ring.c */
424void rds_ib_ring_init(struct rds_ib_work_ring *ring, u32 nr);
425void rds_ib_ring_resize(struct rds_ib_work_ring *ring, u32 nr);
426u32 rds_ib_ring_alloc(struct rds_ib_work_ring *ring, u32 val, u32 *pos);
427void rds_ib_ring_free(struct rds_ib_work_ring *ring, u32 val);
428void rds_ib_ring_unalloc(struct rds_ib_work_ring *ring, u32 val);
429int rds_ib_ring_empty(struct rds_ib_work_ring *ring);
430int rds_ib_ring_low(struct rds_ib_work_ring *ring);
431u32 rds_ib_ring_oldest(struct rds_ib_work_ring *ring);
432u32 rds_ib_ring_completed(struct rds_ib_work_ring *ring, u32 wr_id, u32 oldest);
433extern wait_queue_head_t rds_ib_ring_empty_wait;
434
435/* ib_send.c */
436void rds_ib_xmit_path_complete(struct rds_conn_path *cp);
437int rds_ib_xmit(struct rds_connection *conn, struct rds_message *rm,
438		unsigned int hdr_off, unsigned int sg, unsigned int off);
439void rds_ib_send_cqe_handler(struct rds_ib_connection *ic, struct ib_wc *wc);
440void rds_ib_send_init_ring(struct rds_ib_connection *ic);
441void rds_ib_send_clear_ring(struct rds_ib_connection *ic);
442int rds_ib_xmit_rdma(struct rds_connection *conn, struct rm_rdma_op *op);
443void rds_ib_send_add_credits(struct rds_connection *conn, unsigned int credits);
444void rds_ib_advertise_credits(struct rds_connection *conn, unsigned int posted);
445int rds_ib_send_grab_credits(struct rds_ib_connection *ic, u32 wanted,
446			     u32 *adv_credits, int need_posted, int max_posted);
447int rds_ib_xmit_atomic(struct rds_connection *conn, struct rm_atomic_op *op);
448
449/* ib_stats.c */
450DECLARE_PER_CPU_SHARED_ALIGNED(struct rds_ib_statistics, rds_ib_stats);
451#define rds_ib_stats_inc(member) rds_stats_inc_which(rds_ib_stats, member)
452#define rds_ib_stats_add(member, count) \
453		rds_stats_add_which(rds_ib_stats, member, count)
454unsigned int rds_ib_stats_info_copy(struct rds_info_iterator *iter,
455				    unsigned int avail);
456
457/* ib_sysctl.c */
458int rds_ib_sysctl_init(void);
459void rds_ib_sysctl_exit(void);
460extern unsigned long rds_ib_sysctl_max_send_wr;
461extern unsigned long rds_ib_sysctl_max_recv_wr;
462extern unsigned long rds_ib_sysctl_max_unsig_wrs;
463extern unsigned long rds_ib_sysctl_max_unsig_bytes;
464extern unsigned long rds_ib_sysctl_max_recv_allocation;
465extern unsigned int rds_ib_sysctl_flow_control;
466
467#endif