/drivers/net/sfc/filter.c

https://bitbucket.org/slukk/jb-tsm-kernel-4.2 · C · 720 lines · 531 code · 106 blank · 83 comment · 86 complexity · c8801d99245644a0c21dd0cca6689b04 MD5 · raw file

  1. /****************************************************************************
  2. * Driver for Solarflare Solarstorm network controllers and boards
  3. * Copyright 2005-2010 Solarflare Communications Inc.
  4. *
  5. * This program is free software; you can redistribute it and/or modify it
  6. * under the terms of the GNU General Public License version 2 as published
  7. * by the Free Software Foundation, incorporated herein by reference.
  8. */
  9. #include <linux/in.h>
  10. #include <net/ip.h>
  11. #include "efx.h"
  12. #include "filter.h"
  13. #include "io.h"
  14. #include "nic.h"
  15. #include "regs.h"
  16. /* "Fudge factors" - difference between programmed value and actual depth.
  17. * Due to pipelined implementation we need to program H/W with a value that
  18. * is larger than the hop limit we want.
  19. */
  20. #define FILTER_CTL_SRCH_FUDGE_WILD 3
  21. #define FILTER_CTL_SRCH_FUDGE_FULL 1
  22. /* Hard maximum hop limit. Hardware will time-out beyond 200-something.
  23. * We also need to avoid infinite loops in efx_filter_search() when the
  24. * table is full.
  25. */
  26. #define FILTER_CTL_SRCH_MAX 200
  27. /* Don't try very hard to find space for performance hints, as this is
  28. * counter-productive. */
  29. #define FILTER_CTL_SRCH_HINT_MAX 5
  30. enum efx_filter_table_id {
  31. EFX_FILTER_TABLE_RX_IP = 0,
  32. EFX_FILTER_TABLE_RX_MAC,
  33. EFX_FILTER_TABLE_COUNT,
  34. };
  35. struct efx_filter_table {
  36. enum efx_filter_table_id id;
  37. u32 offset; /* address of table relative to BAR */
  38. unsigned size; /* number of entries */
  39. unsigned step; /* step between entries */
  40. unsigned used; /* number currently used */
  41. unsigned long *used_bitmap;
  42. struct efx_filter_spec *spec;
  43. unsigned search_depth[EFX_FILTER_TYPE_COUNT];
  44. };
  45. struct efx_filter_state {
  46. spinlock_t lock;
  47. struct efx_filter_table table[EFX_FILTER_TABLE_COUNT];
  48. #ifdef CONFIG_RFS_ACCEL
  49. u32 *rps_flow_id;
  50. unsigned rps_expire_index;
  51. #endif
  52. };
  53. /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
  54. * key derived from the n-tuple. The initial LFSR state is 0xffff. */
  55. static u16 efx_filter_hash(u32 key)
  56. {
  57. u16 tmp;
  58. /* First 16 rounds */
  59. tmp = 0x1fff ^ key >> 16;
  60. tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
  61. tmp = tmp ^ tmp >> 9;
  62. /* Last 16 rounds */
  63. tmp = tmp ^ tmp << 13 ^ key;
  64. tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
  65. return tmp ^ tmp >> 9;
  66. }
  67. /* To allow for hash collisions, filter search continues at these
  68. * increments from the first possible entry selected by the hash. */
  69. static u16 efx_filter_increment(u32 key)
  70. {
  71. return key * 2 - 1;
  72. }
  73. static enum efx_filter_table_id
  74. efx_filter_spec_table_id(const struct efx_filter_spec *spec)
  75. {
  76. BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_FULL >> 2));
  77. BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_WILD >> 2));
  78. BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_FULL >> 2));
  79. BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_WILD >> 2));
  80. BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_FULL >> 2));
  81. BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_WILD >> 2));
  82. EFX_BUG_ON_PARANOID(spec->type == EFX_FILTER_UNSPEC);
  83. return spec->type >> 2;
  84. }
  85. static struct efx_filter_table *
  86. efx_filter_spec_table(struct efx_filter_state *state,
  87. const struct efx_filter_spec *spec)
  88. {
  89. if (spec->type == EFX_FILTER_UNSPEC)
  90. return NULL;
  91. else
  92. return &state->table[efx_filter_spec_table_id(spec)];
  93. }
  94. static void efx_filter_table_reset_search_depth(struct efx_filter_table *table)
  95. {
  96. memset(table->search_depth, 0, sizeof(table->search_depth));
  97. }
  98. static void efx_filter_push_rx_limits(struct efx_nic *efx)
  99. {
  100. struct efx_filter_state *state = efx->filter_state;
  101. struct efx_filter_table *table;
  102. efx_oword_t filter_ctl;
  103. efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
  104. table = &state->table[EFX_FILTER_TABLE_RX_IP];
  105. EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT,
  106. table->search_depth[EFX_FILTER_TCP_FULL] +
  107. FILTER_CTL_SRCH_FUDGE_FULL);
  108. EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT,
  109. table->search_depth[EFX_FILTER_TCP_WILD] +
  110. FILTER_CTL_SRCH_FUDGE_WILD);
  111. EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT,
  112. table->search_depth[EFX_FILTER_UDP_FULL] +
  113. FILTER_CTL_SRCH_FUDGE_FULL);
  114. EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT,
  115. table->search_depth[EFX_FILTER_UDP_WILD] +
  116. FILTER_CTL_SRCH_FUDGE_WILD);
  117. table = &state->table[EFX_FILTER_TABLE_RX_MAC];
  118. if (table->size) {
  119. EFX_SET_OWORD_FIELD(
  120. filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
  121. table->search_depth[EFX_FILTER_MAC_FULL] +
  122. FILTER_CTL_SRCH_FUDGE_FULL);
  123. EFX_SET_OWORD_FIELD(
  124. filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
  125. table->search_depth[EFX_FILTER_MAC_WILD] +
  126. FILTER_CTL_SRCH_FUDGE_WILD);
  127. }
  128. efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
  129. }
  130. static inline void __efx_filter_set_ipv4(struct efx_filter_spec *spec,
  131. __be32 host1, __be16 port1,
  132. __be32 host2, __be16 port2)
  133. {
  134. spec->data[0] = ntohl(host1) << 16 | ntohs(port1);
  135. spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16;
  136. spec->data[2] = ntohl(host2);
  137. }
  138. /**
  139. * efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port
  140. * @spec: Specification to initialise
  141. * @proto: Transport layer protocol number
  142. * @host: Local host address (network byte order)
  143. * @port: Local port (network byte order)
  144. */
  145. int efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto,
  146. __be32 host, __be16 port)
  147. {
  148. __be32 host1;
  149. __be16 port1;
  150. EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
  151. /* This cannot currently be combined with other filtering */
  152. if (spec->type != EFX_FILTER_UNSPEC)
  153. return -EPROTONOSUPPORT;
  154. if (port == 0)
  155. return -EINVAL;
  156. switch (proto) {
  157. case IPPROTO_TCP:
  158. spec->type = EFX_FILTER_TCP_WILD;
  159. break;
  160. case IPPROTO_UDP:
  161. spec->type = EFX_FILTER_UDP_WILD;
  162. break;
  163. default:
  164. return -EPROTONOSUPPORT;
  165. }
  166. /* Filter is constructed in terms of source and destination,
  167. * with the odd wrinkle that the ports are swapped in a UDP
  168. * wildcard filter. We need to convert from local and remote
  169. * (= zero for wildcard) addresses.
  170. */
  171. host1 = 0;
  172. if (proto != IPPROTO_UDP) {
  173. port1 = 0;
  174. } else {
  175. port1 = port;
  176. port = 0;
  177. }
  178. __efx_filter_set_ipv4(spec, host1, port1, host, port);
  179. return 0;
  180. }
  181. /**
  182. * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports
  183. * @spec: Specification to initialise
  184. * @proto: Transport layer protocol number
  185. * @host: Local host address (network byte order)
  186. * @port: Local port (network byte order)
  187. * @rhost: Remote host address (network byte order)
  188. * @rport: Remote port (network byte order)
  189. */
  190. int efx_filter_set_ipv4_full(struct efx_filter_spec *spec, u8 proto,
  191. __be32 host, __be16 port,
  192. __be32 rhost, __be16 rport)
  193. {
  194. EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
  195. /* This cannot currently be combined with other filtering */
  196. if (spec->type != EFX_FILTER_UNSPEC)
  197. return -EPROTONOSUPPORT;
  198. if (port == 0 || rport == 0)
  199. return -EINVAL;
  200. switch (proto) {
  201. case IPPROTO_TCP:
  202. spec->type = EFX_FILTER_TCP_FULL;
  203. break;
  204. case IPPROTO_UDP:
  205. spec->type = EFX_FILTER_UDP_FULL;
  206. break;
  207. default:
  208. return -EPROTONOSUPPORT;
  209. }
  210. __efx_filter_set_ipv4(spec, rhost, rport, host, port);
  211. return 0;
  212. }
  213. /**
  214. * efx_filter_set_eth_local - specify local Ethernet address and optional VID
  215. * @spec: Specification to initialise
  216. * @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC
  217. * @addr: Local Ethernet MAC address
  218. */
  219. int efx_filter_set_eth_local(struct efx_filter_spec *spec,
  220. u16 vid, const u8 *addr)
  221. {
  222. EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
  223. /* This cannot currently be combined with other filtering */
  224. if (spec->type != EFX_FILTER_UNSPEC)
  225. return -EPROTONOSUPPORT;
  226. if (vid == EFX_FILTER_VID_UNSPEC) {
  227. spec->type = EFX_FILTER_MAC_WILD;
  228. spec->data[0] = 0;
  229. } else {
  230. spec->type = EFX_FILTER_MAC_FULL;
  231. spec->data[0] = vid;
  232. }
  233. spec->data[1] = addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | addr[5];
  234. spec->data[2] = addr[0] << 8 | addr[1];
  235. return 0;
  236. }
  237. /* Build a filter entry and return its n-tuple key. */
  238. static u32 efx_filter_build(efx_oword_t *filter, struct efx_filter_spec *spec)
  239. {
  240. u32 data3;
  241. switch (efx_filter_spec_table_id(spec)) {
  242. case EFX_FILTER_TABLE_RX_IP: {
  243. bool is_udp = (spec->type == EFX_FILTER_UDP_FULL ||
  244. spec->type == EFX_FILTER_UDP_WILD);
  245. EFX_POPULATE_OWORD_7(
  246. *filter,
  247. FRF_BZ_RSS_EN,
  248. !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
  249. FRF_BZ_SCATTER_EN,
  250. !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
  251. FRF_BZ_TCP_UDP, is_udp,
  252. FRF_BZ_RXQ_ID, spec->dmaq_id,
  253. EFX_DWORD_2, spec->data[2],
  254. EFX_DWORD_1, spec->data[1],
  255. EFX_DWORD_0, spec->data[0]);
  256. data3 = is_udp;
  257. break;
  258. }
  259. case EFX_FILTER_TABLE_RX_MAC: {
  260. bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
  261. EFX_POPULATE_OWORD_8(
  262. *filter,
  263. FRF_CZ_RMFT_RSS_EN,
  264. !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
  265. FRF_CZ_RMFT_SCATTER_EN,
  266. !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
  267. FRF_CZ_RMFT_IP_OVERRIDE,
  268. !!(spec->flags & EFX_FILTER_FLAG_RX_OVERRIDE_IP),
  269. FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id,
  270. FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
  271. FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2],
  272. FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1],
  273. FRF_CZ_RMFT_VLAN_ID, spec->data[0]);
  274. data3 = is_wild;
  275. break;
  276. }
  277. default:
  278. BUG();
  279. }
  280. return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3;
  281. }
  282. static bool efx_filter_equal(const struct efx_filter_spec *left,
  283. const struct efx_filter_spec *right)
  284. {
  285. if (left->type != right->type ||
  286. memcmp(left->data, right->data, sizeof(left->data)))
  287. return false;
  288. return true;
  289. }
  290. static int efx_filter_search(struct efx_filter_table *table,
  291. struct efx_filter_spec *spec, u32 key,
  292. bool for_insert, int *depth_required)
  293. {
  294. unsigned hash, incr, filter_idx, depth, depth_max;
  295. struct efx_filter_spec *cmp;
  296. hash = efx_filter_hash(key);
  297. incr = efx_filter_increment(key);
  298. depth_max = (spec->priority <= EFX_FILTER_PRI_HINT ?
  299. FILTER_CTL_SRCH_HINT_MAX : FILTER_CTL_SRCH_MAX);
  300. for (depth = 1, filter_idx = hash & (table->size - 1);
  301. depth <= depth_max && test_bit(filter_idx, table->used_bitmap);
  302. ++depth) {
  303. cmp = &table->spec[filter_idx];
  304. if (efx_filter_equal(spec, cmp))
  305. goto found;
  306. filter_idx = (filter_idx + incr) & (table->size - 1);
  307. }
  308. if (!for_insert)
  309. return -ENOENT;
  310. if (depth > depth_max)
  311. return -EBUSY;
  312. found:
  313. *depth_required = depth;
  314. return filter_idx;
  315. }
  316. /* Construct/deconstruct external filter IDs */
  317. static inline int
  318. efx_filter_make_id(enum efx_filter_table_id table_id, unsigned index)
  319. {
  320. return table_id << 16 | index;
  321. }
  322. /**
  323. * efx_filter_insert_filter - add or replace a filter
  324. * @efx: NIC in which to insert the filter
  325. * @spec: Specification for the filter
  326. * @replace: Flag for whether the specified filter may replace a filter
  327. * with an identical match expression and equal or lower priority
  328. *
  329. * On success, return the filter ID.
  330. * On failure, return a negative error code.
  331. */
  332. int efx_filter_insert_filter(struct efx_nic *efx, struct efx_filter_spec *spec,
  333. bool replace)
  334. {
  335. struct efx_filter_state *state = efx->filter_state;
  336. struct efx_filter_table *table = efx_filter_spec_table(state, spec);
  337. struct efx_filter_spec *saved_spec;
  338. efx_oword_t filter;
  339. int filter_idx, depth;
  340. u32 key;
  341. int rc;
  342. if (!table || table->size == 0)
  343. return -EINVAL;
  344. key = efx_filter_build(&filter, spec);
  345. netif_vdbg(efx, hw, efx->net_dev,
  346. "%s: type %d search_depth=%d", __func__, spec->type,
  347. table->search_depth[spec->type]);
  348. spin_lock_bh(&state->lock);
  349. rc = efx_filter_search(table, spec, key, true, &depth);
  350. if (rc < 0)
  351. goto out;
  352. filter_idx = rc;
  353. BUG_ON(filter_idx >= table->size);
  354. saved_spec = &table->spec[filter_idx];
  355. if (test_bit(filter_idx, table->used_bitmap)) {
  356. /* Should we replace the existing filter? */
  357. if (!replace) {
  358. rc = -EEXIST;
  359. goto out;
  360. }
  361. if (spec->priority < saved_spec->priority) {
  362. rc = -EPERM;
  363. goto out;
  364. }
  365. } else {
  366. __set_bit(filter_idx, table->used_bitmap);
  367. ++table->used;
  368. }
  369. *saved_spec = *spec;
  370. if (table->search_depth[spec->type] < depth) {
  371. table->search_depth[spec->type] = depth;
  372. efx_filter_push_rx_limits(efx);
  373. }
  374. efx_writeo(efx, &filter, table->offset + table->step * filter_idx);
  375. netif_vdbg(efx, hw, efx->net_dev,
  376. "%s: filter type %d index %d rxq %u set",
  377. __func__, spec->type, filter_idx, spec->dmaq_id);
  378. rc = efx_filter_make_id(table->id, filter_idx);
  379. out:
  380. spin_unlock_bh(&state->lock);
  381. return rc;
  382. }
  383. static void efx_filter_table_clear_entry(struct efx_nic *efx,
  384. struct efx_filter_table *table,
  385. int filter_idx)
  386. {
  387. static efx_oword_t filter;
  388. if (test_bit(filter_idx, table->used_bitmap)) {
  389. __clear_bit(filter_idx, table->used_bitmap);
  390. --table->used;
  391. memset(&table->spec[filter_idx], 0, sizeof(table->spec[0]));
  392. efx_writeo(efx, &filter,
  393. table->offset + table->step * filter_idx);
  394. }
  395. }
  396. /**
  397. * efx_filter_remove_filter - remove a filter by specification
  398. * @efx: NIC from which to remove the filter
  399. * @spec: Specification for the filter
  400. *
  401. * On success, return zero.
  402. * On failure, return a negative error code.
  403. */
  404. int efx_filter_remove_filter(struct efx_nic *efx, struct efx_filter_spec *spec)
  405. {
  406. struct efx_filter_state *state = efx->filter_state;
  407. struct efx_filter_table *table = efx_filter_spec_table(state, spec);
  408. struct efx_filter_spec *saved_spec;
  409. efx_oword_t filter;
  410. int filter_idx, depth;
  411. u32 key;
  412. int rc;
  413. if (!table)
  414. return -EINVAL;
  415. key = efx_filter_build(&filter, spec);
  416. spin_lock_bh(&state->lock);
  417. rc = efx_filter_search(table, spec, key, false, &depth);
  418. if (rc < 0)
  419. goto out;
  420. filter_idx = rc;
  421. saved_spec = &table->spec[filter_idx];
  422. if (spec->priority < saved_spec->priority) {
  423. rc = -EPERM;
  424. goto out;
  425. }
  426. efx_filter_table_clear_entry(efx, table, filter_idx);
  427. if (table->used == 0)
  428. efx_filter_table_reset_search_depth(table);
  429. rc = 0;
  430. out:
  431. spin_unlock_bh(&state->lock);
  432. return rc;
  433. }
  434. static void efx_filter_table_clear(struct efx_nic *efx,
  435. enum efx_filter_table_id table_id,
  436. enum efx_filter_priority priority)
  437. {
  438. struct efx_filter_state *state = efx->filter_state;
  439. struct efx_filter_table *table = &state->table[table_id];
  440. int filter_idx;
  441. spin_lock_bh(&state->lock);
  442. for (filter_idx = 0; filter_idx < table->size; ++filter_idx)
  443. if (table->spec[filter_idx].priority <= priority)
  444. efx_filter_table_clear_entry(efx, table, filter_idx);
  445. if (table->used == 0)
  446. efx_filter_table_reset_search_depth(table);
  447. spin_unlock_bh(&state->lock);
  448. }
  449. /**
  450. * efx_filter_clear_rx - remove RX filters by priority
  451. * @efx: NIC from which to remove the filters
  452. * @priority: Maximum priority to remove
  453. */
  454. void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority)
  455. {
  456. efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_IP, priority);
  457. efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_MAC, priority);
  458. }
  459. /* Restore filter stater after reset */
  460. void efx_restore_filters(struct efx_nic *efx)
  461. {
  462. struct efx_filter_state *state = efx->filter_state;
  463. enum efx_filter_table_id table_id;
  464. struct efx_filter_table *table;
  465. efx_oword_t filter;
  466. int filter_idx;
  467. spin_lock_bh(&state->lock);
  468. for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
  469. table = &state->table[table_id];
  470. for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
  471. if (!test_bit(filter_idx, table->used_bitmap))
  472. continue;
  473. efx_filter_build(&filter, &table->spec[filter_idx]);
  474. efx_writeo(efx, &filter,
  475. table->offset + table->step * filter_idx);
  476. }
  477. }
  478. efx_filter_push_rx_limits(efx);
  479. spin_unlock_bh(&state->lock);
  480. }
  481. int efx_probe_filters(struct efx_nic *efx)
  482. {
  483. struct efx_filter_state *state;
  484. struct efx_filter_table *table;
  485. unsigned table_id;
  486. state = kzalloc(sizeof(*efx->filter_state), GFP_KERNEL);
  487. if (!state)
  488. return -ENOMEM;
  489. efx->filter_state = state;
  490. spin_lock_init(&state->lock);
  491. if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
  492. #ifdef CONFIG_RFS_ACCEL
  493. state->rps_flow_id = kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS,
  494. sizeof(*state->rps_flow_id),
  495. GFP_KERNEL);
  496. if (!state->rps_flow_id)
  497. goto fail;
  498. #endif
  499. table = &state->table[EFX_FILTER_TABLE_RX_IP];
  500. table->id = EFX_FILTER_TABLE_RX_IP;
  501. table->offset = FR_BZ_RX_FILTER_TBL0;
  502. table->size = FR_BZ_RX_FILTER_TBL0_ROWS;
  503. table->step = FR_BZ_RX_FILTER_TBL0_STEP;
  504. }
  505. if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
  506. table = &state->table[EFX_FILTER_TABLE_RX_MAC];
  507. table->id = EFX_FILTER_TABLE_RX_MAC;
  508. table->offset = FR_CZ_RX_MAC_FILTER_TBL0;
  509. table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
  510. table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP;
  511. }
  512. for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
  513. table = &state->table[table_id];
  514. if (table->size == 0)
  515. continue;
  516. table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size),
  517. sizeof(unsigned long),
  518. GFP_KERNEL);
  519. if (!table->used_bitmap)
  520. goto fail;
  521. table->spec = vzalloc(table->size * sizeof(*table->spec));
  522. if (!table->spec)
  523. goto fail;
  524. }
  525. return 0;
  526. fail:
  527. efx_remove_filters(efx);
  528. return -ENOMEM;
  529. }
  530. void efx_remove_filters(struct efx_nic *efx)
  531. {
  532. struct efx_filter_state *state = efx->filter_state;
  533. enum efx_filter_table_id table_id;
  534. for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
  535. kfree(state->table[table_id].used_bitmap);
  536. vfree(state->table[table_id].spec);
  537. }
  538. #ifdef CONFIG_RFS_ACCEL
  539. kfree(state->rps_flow_id);
  540. #endif
  541. kfree(state);
  542. }
  543. #ifdef CONFIG_RFS_ACCEL
  544. int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
  545. u16 rxq_index, u32 flow_id)
  546. {
  547. struct efx_nic *efx = netdev_priv(net_dev);
  548. struct efx_channel *channel;
  549. struct efx_filter_state *state = efx->filter_state;
  550. struct efx_filter_spec spec;
  551. const struct iphdr *ip;
  552. const __be16 *ports;
  553. int nhoff;
  554. int rc;
  555. nhoff = skb_network_offset(skb);
  556. if (skb->protocol != htons(ETH_P_IP))
  557. return -EPROTONOSUPPORT;
  558. /* RFS must validate the IP header length before calling us */
  559. EFX_BUG_ON_PARANOID(!pskb_may_pull(skb, nhoff + sizeof(*ip)));
  560. ip = (const struct iphdr *)(skb->data + nhoff);
  561. if (ip->frag_off & htons(IP_MF | IP_OFFSET))
  562. return -EPROTONOSUPPORT;
  563. EFX_BUG_ON_PARANOID(!pskb_may_pull(skb, nhoff + 4 * ip->ihl + 4));
  564. ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl);
  565. efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, 0, rxq_index);
  566. rc = efx_filter_set_ipv4_full(&spec, ip->protocol,
  567. ip->daddr, ports[1], ip->saddr, ports[0]);
  568. if (rc)
  569. return rc;
  570. rc = efx_filter_insert_filter(efx, &spec, true);
  571. if (rc < 0)
  572. return rc;
  573. /* Remember this so we can check whether to expire the filter later */
  574. state->rps_flow_id[rc] = flow_id;
  575. channel = efx_get_channel(efx, skb_get_rx_queue(skb));
  576. ++channel->rfs_filters_added;
  577. netif_info(efx, rx_status, efx->net_dev,
  578. "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
  579. (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP",
  580. &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]),
  581. rxq_index, flow_id, rc);
  582. return rc;
  583. }
  584. bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota)
  585. {
  586. struct efx_filter_state *state = efx->filter_state;
  587. struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_IP];
  588. unsigned mask = table->size - 1;
  589. unsigned index;
  590. unsigned stop;
  591. if (!spin_trylock_bh(&state->lock))
  592. return false;
  593. index = state->rps_expire_index;
  594. stop = (index + quota) & mask;
  595. while (index != stop) {
  596. if (test_bit(index, table->used_bitmap) &&
  597. table->spec[index].priority == EFX_FILTER_PRI_HINT &&
  598. rps_may_expire_flow(efx->net_dev,
  599. table->spec[index].dmaq_id,
  600. state->rps_flow_id[index], index)) {
  601. netif_info(efx, rx_status, efx->net_dev,
  602. "expiring filter %d [flow %u]\n",
  603. index, state->rps_flow_id[index]);
  604. efx_filter_table_clear_entry(efx, table, index);
  605. }
  606. index = (index + 1) & mask;
  607. }
  608. state->rps_expire_index = stop;
  609. if (table->used == 0)
  610. efx_filter_table_reset_search_depth(table);
  611. spin_unlock_bh(&state->lock);
  612. return true;
  613. }
  614. #endif /* CONFIG_RFS_ACCEL */