PageRenderTime 65ms CodeModel.GetById 19ms app.highlight 38ms RepoModel.GetById 1ms app.codeStats 1ms

/drivers/net/sfc/filter.c

https://bitbucket.org/slukk/jb-tsm-kernel-4.2
C | 720 lines | 531 code | 106 blank | 83 comment | 86 complexity | c8801d99245644a0c21dd0cca6689b04 MD5 | raw file
Possible License(s): GPL-2.0, LGPL-2.0, AGPL-1.0
  1/****************************************************************************
  2 * Driver for Solarflare Solarstorm network controllers and boards
  3 * Copyright 2005-2010 Solarflare Communications Inc.
  4 *
  5 * This program is free software; you can redistribute it and/or modify it
  6 * under the terms of the GNU General Public License version 2 as published
  7 * by the Free Software Foundation, incorporated herein by reference.
  8 */
  9
 10#include <linux/in.h>
 11#include <net/ip.h>
 12#include "efx.h"
 13#include "filter.h"
 14#include "io.h"
 15#include "nic.h"
 16#include "regs.h"
 17
 18/* "Fudge factors" - difference between programmed value and actual depth.
 19 * Due to pipelined implementation we need to program H/W with a value that
 20 * is larger than the hop limit we want.
 21 */
 22#define FILTER_CTL_SRCH_FUDGE_WILD 3
 23#define FILTER_CTL_SRCH_FUDGE_FULL 1
 24
 25/* Hard maximum hop limit.  Hardware will time-out beyond 200-something.
 26 * We also need to avoid infinite loops in efx_filter_search() when the
 27 * table is full.
 28 */
 29#define FILTER_CTL_SRCH_MAX 200
 30
 31/* Don't try very hard to find space for performance hints, as this is
 32 * counter-productive. */
 33#define FILTER_CTL_SRCH_HINT_MAX 5
 34
 35enum efx_filter_table_id {
 36	EFX_FILTER_TABLE_RX_IP = 0,
 37	EFX_FILTER_TABLE_RX_MAC,
 38	EFX_FILTER_TABLE_COUNT,
 39};
 40
 41struct efx_filter_table {
 42	enum efx_filter_table_id id;
 43	u32		offset;		/* address of table relative to BAR */
 44	unsigned	size;		/* number of entries */
 45	unsigned	step;		/* step between entries */
 46	unsigned	used;		/* number currently used */
 47	unsigned long	*used_bitmap;
 48	struct efx_filter_spec *spec;
 49	unsigned	search_depth[EFX_FILTER_TYPE_COUNT];
 50};
 51
 52struct efx_filter_state {
 53	spinlock_t	lock;
 54	struct efx_filter_table table[EFX_FILTER_TABLE_COUNT];
 55#ifdef CONFIG_RFS_ACCEL
 56	u32		*rps_flow_id;
 57	unsigned	rps_expire_index;
 58#endif
 59};
 60
 61/* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
 62 * key derived from the n-tuple.  The initial LFSR state is 0xffff. */
 63static u16 efx_filter_hash(u32 key)
 64{
 65	u16 tmp;
 66
 67	/* First 16 rounds */
 68	tmp = 0x1fff ^ key >> 16;
 69	tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 70	tmp = tmp ^ tmp >> 9;
 71	/* Last 16 rounds */
 72	tmp = tmp ^ tmp << 13 ^ key;
 73	tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 74	return tmp ^ tmp >> 9;
 75}
 76
 77/* To allow for hash collisions, filter search continues at these
 78 * increments from the first possible entry selected by the hash. */
 79static u16 efx_filter_increment(u32 key)
 80{
 81	return key * 2 - 1;
 82}
 83
 84static enum efx_filter_table_id
 85efx_filter_spec_table_id(const struct efx_filter_spec *spec)
 86{
 87	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_FULL >> 2));
 88	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_WILD >> 2));
 89	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_FULL >> 2));
 90	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_WILD >> 2));
 91	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_FULL >> 2));
 92	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_WILD >> 2));
 93	EFX_BUG_ON_PARANOID(spec->type == EFX_FILTER_UNSPEC);
 94	return spec->type >> 2;
 95}
 96
 97static struct efx_filter_table *
 98efx_filter_spec_table(struct efx_filter_state *state,
 99		      const struct efx_filter_spec *spec)
100{
101	if (spec->type == EFX_FILTER_UNSPEC)
102		return NULL;
103	else
104		return &state->table[efx_filter_spec_table_id(spec)];
105}
106
107static void efx_filter_table_reset_search_depth(struct efx_filter_table *table)
108{
109	memset(table->search_depth, 0, sizeof(table->search_depth));
110}
111
112static void efx_filter_push_rx_limits(struct efx_nic *efx)
113{
114	struct efx_filter_state *state = efx->filter_state;
115	struct efx_filter_table *table;
116	efx_oword_t filter_ctl;
117
118	efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
119
120	table = &state->table[EFX_FILTER_TABLE_RX_IP];
121	EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT,
122			    table->search_depth[EFX_FILTER_TCP_FULL] +
123			    FILTER_CTL_SRCH_FUDGE_FULL);
124	EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT,
125			    table->search_depth[EFX_FILTER_TCP_WILD] +
126			    FILTER_CTL_SRCH_FUDGE_WILD);
127	EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT,
128			    table->search_depth[EFX_FILTER_UDP_FULL] +
129			    FILTER_CTL_SRCH_FUDGE_FULL);
130	EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT,
131			    table->search_depth[EFX_FILTER_UDP_WILD] +
132			    FILTER_CTL_SRCH_FUDGE_WILD);
133
134	table = &state->table[EFX_FILTER_TABLE_RX_MAC];
135	if (table->size) {
136		EFX_SET_OWORD_FIELD(
137			filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
138			table->search_depth[EFX_FILTER_MAC_FULL] +
139			FILTER_CTL_SRCH_FUDGE_FULL);
140		EFX_SET_OWORD_FIELD(
141			filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
142			table->search_depth[EFX_FILTER_MAC_WILD] +
143			FILTER_CTL_SRCH_FUDGE_WILD);
144	}
145
146	efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
147}
148
149static inline void __efx_filter_set_ipv4(struct efx_filter_spec *spec,
150					 __be32 host1, __be16 port1,
151					 __be32 host2, __be16 port2)
152{
153	spec->data[0] = ntohl(host1) << 16 | ntohs(port1);
154	spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16;
155	spec->data[2] = ntohl(host2);
156}
157
158/**
159 * efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port
160 * @spec: Specification to initialise
161 * @proto: Transport layer protocol number
162 * @host: Local host address (network byte order)
163 * @port: Local port (network byte order)
164 */
165int efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto,
166			      __be32 host, __be16 port)
167{
168	__be32 host1;
169	__be16 port1;
170
171	EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
172
173	/* This cannot currently be combined with other filtering */
174	if (spec->type != EFX_FILTER_UNSPEC)
175		return -EPROTONOSUPPORT;
176
177	if (port == 0)
178		return -EINVAL;
179
180	switch (proto) {
181	case IPPROTO_TCP:
182		spec->type = EFX_FILTER_TCP_WILD;
183		break;
184	case IPPROTO_UDP:
185		spec->type = EFX_FILTER_UDP_WILD;
186		break;
187	default:
188		return -EPROTONOSUPPORT;
189	}
190
191	/* Filter is constructed in terms of source and destination,
192	 * with the odd wrinkle that the ports are swapped in a UDP
193	 * wildcard filter.  We need to convert from local and remote
194	 * (= zero for wildcard) addresses.
195	 */
196	host1 = 0;
197	if (proto != IPPROTO_UDP) {
198		port1 = 0;
199	} else {
200		port1 = port;
201		port = 0;
202	}
203
204	__efx_filter_set_ipv4(spec, host1, port1, host, port);
205	return 0;
206}
207
208/**
209 * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports
210 * @spec: Specification to initialise
211 * @proto: Transport layer protocol number
212 * @host: Local host address (network byte order)
213 * @port: Local port (network byte order)
214 * @rhost: Remote host address (network byte order)
215 * @rport: Remote port (network byte order)
216 */
217int efx_filter_set_ipv4_full(struct efx_filter_spec *spec, u8 proto,
218			     __be32 host, __be16 port,
219			     __be32 rhost, __be16 rport)
220{
221	EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
222
223	/* This cannot currently be combined with other filtering */
224	if (spec->type != EFX_FILTER_UNSPEC)
225		return -EPROTONOSUPPORT;
226
227	if (port == 0 || rport == 0)
228		return -EINVAL;
229
230	switch (proto) {
231	case IPPROTO_TCP:
232		spec->type = EFX_FILTER_TCP_FULL;
233		break;
234	case IPPROTO_UDP:
235		spec->type = EFX_FILTER_UDP_FULL;
236		break;
237	default:
238		return -EPROTONOSUPPORT;
239	}
240
241	__efx_filter_set_ipv4(spec, rhost, rport, host, port);
242	return 0;
243}
244
245/**
246 * efx_filter_set_eth_local - specify local Ethernet address and optional VID
247 * @spec: Specification to initialise
248 * @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC
249 * @addr: Local Ethernet MAC address
250 */
251int efx_filter_set_eth_local(struct efx_filter_spec *spec,
252			     u16 vid, const u8 *addr)
253{
254	EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
255
256	/* This cannot currently be combined with other filtering */
257	if (spec->type != EFX_FILTER_UNSPEC)
258		return -EPROTONOSUPPORT;
259
260	if (vid == EFX_FILTER_VID_UNSPEC) {
261		spec->type = EFX_FILTER_MAC_WILD;
262		spec->data[0] = 0;
263	} else {
264		spec->type = EFX_FILTER_MAC_FULL;
265		spec->data[0] = vid;
266	}
267
268	spec->data[1] = addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | addr[5];
269	spec->data[2] = addr[0] << 8 | addr[1];
270	return 0;
271}
272
273/* Build a filter entry and return its n-tuple key. */
274static u32 efx_filter_build(efx_oword_t *filter, struct efx_filter_spec *spec)
275{
276	u32 data3;
277
278	switch (efx_filter_spec_table_id(spec)) {
279	case EFX_FILTER_TABLE_RX_IP: {
280		bool is_udp = (spec->type == EFX_FILTER_UDP_FULL ||
281			       spec->type == EFX_FILTER_UDP_WILD);
282		EFX_POPULATE_OWORD_7(
283			*filter,
284			FRF_BZ_RSS_EN,
285			!!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
286			FRF_BZ_SCATTER_EN,
287			!!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
288			FRF_BZ_TCP_UDP, is_udp,
289			FRF_BZ_RXQ_ID, spec->dmaq_id,
290			EFX_DWORD_2, spec->data[2],
291			EFX_DWORD_1, spec->data[1],
292			EFX_DWORD_0, spec->data[0]);
293		data3 = is_udp;
294		break;
295	}
296
297	case EFX_FILTER_TABLE_RX_MAC: {
298		bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
299		EFX_POPULATE_OWORD_8(
300			*filter,
301			FRF_CZ_RMFT_RSS_EN,
302			!!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
303			FRF_CZ_RMFT_SCATTER_EN,
304			!!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
305			FRF_CZ_RMFT_IP_OVERRIDE,
306			!!(spec->flags & EFX_FILTER_FLAG_RX_OVERRIDE_IP),
307			FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id,
308			FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
309			FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2],
310			FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1],
311			FRF_CZ_RMFT_VLAN_ID, spec->data[0]);
312		data3 = is_wild;
313		break;
314	}
315
316	default:
317		BUG();
318	}
319
320	return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3;
321}
322
323static bool efx_filter_equal(const struct efx_filter_spec *left,
324			     const struct efx_filter_spec *right)
325{
326	if (left->type != right->type ||
327	    memcmp(left->data, right->data, sizeof(left->data)))
328		return false;
329
330	return true;
331}
332
333static int efx_filter_search(struct efx_filter_table *table,
334			     struct efx_filter_spec *spec, u32 key,
335			     bool for_insert, int *depth_required)
336{
337	unsigned hash, incr, filter_idx, depth, depth_max;
338	struct efx_filter_spec *cmp;
339
340	hash = efx_filter_hash(key);
341	incr = efx_filter_increment(key);
342	depth_max = (spec->priority <= EFX_FILTER_PRI_HINT ?
343		     FILTER_CTL_SRCH_HINT_MAX : FILTER_CTL_SRCH_MAX);
344
345	for (depth = 1, filter_idx = hash & (table->size - 1);
346	     depth <= depth_max && test_bit(filter_idx, table->used_bitmap);
347	     ++depth) {
348		cmp = &table->spec[filter_idx];
349		if (efx_filter_equal(spec, cmp))
350			goto found;
351		filter_idx = (filter_idx + incr) & (table->size - 1);
352	}
353	if (!for_insert)
354		return -ENOENT;
355	if (depth > depth_max)
356		return -EBUSY;
357found:
358	*depth_required = depth;
359	return filter_idx;
360}
361
362/* Construct/deconstruct external filter IDs */
363
364static inline int
365efx_filter_make_id(enum efx_filter_table_id table_id, unsigned index)
366{
367	return table_id << 16 | index;
368}
369
370/**
371 * efx_filter_insert_filter - add or replace a filter
372 * @efx: NIC in which to insert the filter
373 * @spec: Specification for the filter
374 * @replace: Flag for whether the specified filter may replace a filter
375 *	with an identical match expression and equal or lower priority
376 *
377 * On success, return the filter ID.
378 * On failure, return a negative error code.
379 */
380int efx_filter_insert_filter(struct efx_nic *efx, struct efx_filter_spec *spec,
381			     bool replace)
382{
383	struct efx_filter_state *state = efx->filter_state;
384	struct efx_filter_table *table = efx_filter_spec_table(state, spec);
385	struct efx_filter_spec *saved_spec;
386	efx_oword_t filter;
387	int filter_idx, depth;
388	u32 key;
389	int rc;
390
391	if (!table || table->size == 0)
392		return -EINVAL;
393
394	key = efx_filter_build(&filter, spec);
395
396	netif_vdbg(efx, hw, efx->net_dev,
397		   "%s: type %d search_depth=%d", __func__, spec->type,
398		   table->search_depth[spec->type]);
399
400	spin_lock_bh(&state->lock);
401
402	rc = efx_filter_search(table, spec, key, true, &depth);
403	if (rc < 0)
404		goto out;
405	filter_idx = rc;
406	BUG_ON(filter_idx >= table->size);
407	saved_spec = &table->spec[filter_idx];
408
409	if (test_bit(filter_idx, table->used_bitmap)) {
410		/* Should we replace the existing filter? */
411		if (!replace) {
412			rc = -EEXIST;
413			goto out;
414		}
415		if (spec->priority < saved_spec->priority) {
416			rc = -EPERM;
417			goto out;
418		}
419	} else {
420		__set_bit(filter_idx, table->used_bitmap);
421		++table->used;
422	}
423	*saved_spec = *spec;
424
425	if (table->search_depth[spec->type] < depth) {
426		table->search_depth[spec->type] = depth;
427		efx_filter_push_rx_limits(efx);
428	}
429
430	efx_writeo(efx, &filter, table->offset + table->step * filter_idx);
431
432	netif_vdbg(efx, hw, efx->net_dev,
433		   "%s: filter type %d index %d rxq %u set",
434		   __func__, spec->type, filter_idx, spec->dmaq_id);
435	rc = efx_filter_make_id(table->id, filter_idx);
436
437out:
438	spin_unlock_bh(&state->lock);
439	return rc;
440}
441
442static void efx_filter_table_clear_entry(struct efx_nic *efx,
443					 struct efx_filter_table *table,
444					 int filter_idx)
445{
446	static efx_oword_t filter;
447
448	if (test_bit(filter_idx, table->used_bitmap)) {
449		__clear_bit(filter_idx, table->used_bitmap);
450		--table->used;
451		memset(&table->spec[filter_idx], 0, sizeof(table->spec[0]));
452
453		efx_writeo(efx, &filter,
454			   table->offset + table->step * filter_idx);
455	}
456}
457
458/**
459 * efx_filter_remove_filter - remove a filter by specification
460 * @efx: NIC from which to remove the filter
461 * @spec: Specification for the filter
462 *
463 * On success, return zero.
464 * On failure, return a negative error code.
465 */
466int efx_filter_remove_filter(struct efx_nic *efx, struct efx_filter_spec *spec)
467{
468	struct efx_filter_state *state = efx->filter_state;
469	struct efx_filter_table *table = efx_filter_spec_table(state, spec);
470	struct efx_filter_spec *saved_spec;
471	efx_oword_t filter;
472	int filter_idx, depth;
473	u32 key;
474	int rc;
475
476	if (!table)
477		return -EINVAL;
478
479	key = efx_filter_build(&filter, spec);
480
481	spin_lock_bh(&state->lock);
482
483	rc = efx_filter_search(table, spec, key, false, &depth);
484	if (rc < 0)
485		goto out;
486	filter_idx = rc;
487	saved_spec = &table->spec[filter_idx];
488
489	if (spec->priority < saved_spec->priority) {
490		rc = -EPERM;
491		goto out;
492	}
493
494	efx_filter_table_clear_entry(efx, table, filter_idx);
495	if (table->used == 0)
496		efx_filter_table_reset_search_depth(table);
497	rc = 0;
498
499out:
500	spin_unlock_bh(&state->lock);
501	return rc;
502}
503
504static void efx_filter_table_clear(struct efx_nic *efx,
505				   enum efx_filter_table_id table_id,
506				   enum efx_filter_priority priority)
507{
508	struct efx_filter_state *state = efx->filter_state;
509	struct efx_filter_table *table = &state->table[table_id];
510	int filter_idx;
511
512	spin_lock_bh(&state->lock);
513
514	for (filter_idx = 0; filter_idx < table->size; ++filter_idx)
515		if (table->spec[filter_idx].priority <= priority)
516			efx_filter_table_clear_entry(efx, table, filter_idx);
517	if (table->used == 0)
518		efx_filter_table_reset_search_depth(table);
519
520	spin_unlock_bh(&state->lock);
521}
522
523/**
524 * efx_filter_clear_rx - remove RX filters by priority
525 * @efx: NIC from which to remove the filters
526 * @priority: Maximum priority to remove
527 */
528void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority)
529{
530	efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_IP, priority);
531	efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_MAC, priority);
532}
533
534/* Restore filter stater after reset */
535void efx_restore_filters(struct efx_nic *efx)
536{
537	struct efx_filter_state *state = efx->filter_state;
538	enum efx_filter_table_id table_id;
539	struct efx_filter_table *table;
540	efx_oword_t filter;
541	int filter_idx;
542
543	spin_lock_bh(&state->lock);
544
545	for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
546		table = &state->table[table_id];
547		for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
548			if (!test_bit(filter_idx, table->used_bitmap))
549				continue;
550			efx_filter_build(&filter, &table->spec[filter_idx]);
551			efx_writeo(efx, &filter,
552				   table->offset + table->step * filter_idx);
553		}
554	}
555
556	efx_filter_push_rx_limits(efx);
557
558	spin_unlock_bh(&state->lock);
559}
560
561int efx_probe_filters(struct efx_nic *efx)
562{
563	struct efx_filter_state *state;
564	struct efx_filter_table *table;
565	unsigned table_id;
566
567	state = kzalloc(sizeof(*efx->filter_state), GFP_KERNEL);
568	if (!state)
569		return -ENOMEM;
570	efx->filter_state = state;
571
572	spin_lock_init(&state->lock);
573
574	if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
575#ifdef CONFIG_RFS_ACCEL
576		state->rps_flow_id = kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS,
577					     sizeof(*state->rps_flow_id),
578					     GFP_KERNEL);
579		if (!state->rps_flow_id)
580			goto fail;
581#endif
582		table = &state->table[EFX_FILTER_TABLE_RX_IP];
583		table->id = EFX_FILTER_TABLE_RX_IP;
584		table->offset = FR_BZ_RX_FILTER_TBL0;
585		table->size = FR_BZ_RX_FILTER_TBL0_ROWS;
586		table->step = FR_BZ_RX_FILTER_TBL0_STEP;
587	}
588
589	if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
590		table = &state->table[EFX_FILTER_TABLE_RX_MAC];
591		table->id = EFX_FILTER_TABLE_RX_MAC;
592		table->offset = FR_CZ_RX_MAC_FILTER_TBL0;
593		table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
594		table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP;
595	}
596
597	for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
598		table = &state->table[table_id];
599		if (table->size == 0)
600			continue;
601		table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size),
602					     sizeof(unsigned long),
603					     GFP_KERNEL);
604		if (!table->used_bitmap)
605			goto fail;
606		table->spec = vzalloc(table->size * sizeof(*table->spec));
607		if (!table->spec)
608			goto fail;
609	}
610
611	return 0;
612
613fail:
614	efx_remove_filters(efx);
615	return -ENOMEM;
616}
617
618void efx_remove_filters(struct efx_nic *efx)
619{
620	struct efx_filter_state *state = efx->filter_state;
621	enum efx_filter_table_id table_id;
622
623	for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
624		kfree(state->table[table_id].used_bitmap);
625		vfree(state->table[table_id].spec);
626	}
627#ifdef CONFIG_RFS_ACCEL
628	kfree(state->rps_flow_id);
629#endif
630	kfree(state);
631}
632
633#ifdef CONFIG_RFS_ACCEL
634
635int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
636		   u16 rxq_index, u32 flow_id)
637{
638	struct efx_nic *efx = netdev_priv(net_dev);
639	struct efx_channel *channel;
640	struct efx_filter_state *state = efx->filter_state;
641	struct efx_filter_spec spec;
642	const struct iphdr *ip;
643	const __be16 *ports;
644	int nhoff;
645	int rc;
646
647	nhoff = skb_network_offset(skb);
648
649	if (skb->protocol != htons(ETH_P_IP))
650		return -EPROTONOSUPPORT;
651
652	/* RFS must validate the IP header length before calling us */
653	EFX_BUG_ON_PARANOID(!pskb_may_pull(skb, nhoff + sizeof(*ip)));
654	ip = (const struct iphdr *)(skb->data + nhoff);
655	if (ip->frag_off & htons(IP_MF | IP_OFFSET))
656		return -EPROTONOSUPPORT;
657	EFX_BUG_ON_PARANOID(!pskb_may_pull(skb, nhoff + 4 * ip->ihl + 4));
658	ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl);
659
660	efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, 0, rxq_index);
661	rc = efx_filter_set_ipv4_full(&spec, ip->protocol,
662				      ip->daddr, ports[1], ip->saddr, ports[0]);
663	if (rc)
664		return rc;
665
666	rc = efx_filter_insert_filter(efx, &spec, true);
667	if (rc < 0)
668		return rc;
669
670	/* Remember this so we can check whether to expire the filter later */
671	state->rps_flow_id[rc] = flow_id;
672	channel = efx_get_channel(efx, skb_get_rx_queue(skb));
673	++channel->rfs_filters_added;
674
675	netif_info(efx, rx_status, efx->net_dev,
676		   "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
677		   (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP",
678		   &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]),
679		   rxq_index, flow_id, rc);
680
681	return rc;
682}
683
684bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota)
685{
686	struct efx_filter_state *state = efx->filter_state;
687	struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_IP];
688	unsigned mask = table->size - 1;
689	unsigned index;
690	unsigned stop;
691
692	if (!spin_trylock_bh(&state->lock))
693		return false;
694
695	index = state->rps_expire_index;
696	stop = (index + quota) & mask;
697
698	while (index != stop) {
699		if (test_bit(index, table->used_bitmap) &&
700		    table->spec[index].priority == EFX_FILTER_PRI_HINT &&
701		    rps_may_expire_flow(efx->net_dev,
702					table->spec[index].dmaq_id,
703					state->rps_flow_id[index], index)) {
704			netif_info(efx, rx_status, efx->net_dev,
705				   "expiring filter %d [flow %u]\n",
706				   index, state->rps_flow_id[index]);
707			efx_filter_table_clear_entry(efx, table, index);
708		}
709		index = (index + 1) & mask;
710	}
711
712	state->rps_expire_index = stop;
713	if (table->used == 0)
714		efx_filter_table_reset_search_depth(table);
715
716	spin_unlock_bh(&state->lock);
717	return true;
718}
719
720#endif /* CONFIG_RFS_ACCEL */