PageRenderTime 4987ms CodeModel.GetById 37ms RepoModel.GetById 1ms app.codeStats 0ms

/fs/cifs/smbdirect.c

https://gitlab.com/deepcypher/linux
C | 1714 lines | 1259 code | 266 blank | 189 comment | 148 complexity | 1f069b04ba7466779ccc659aa99ec839 MD5 | raw file
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright (C) 2017, Microsoft Corporation.
  4. *
  5. * Author(s): Long Li <longli@microsoft.com>
  6. */
  7. #include <linux/module.h>
  8. #include <linux/highmem.h>
  9. #include "smbdirect.h"
  10. #include "cifs_debug.h"
  11. #include "cifsproto.h"
  12. #include "smb2proto.h"
  13. static struct smbd_response *get_empty_queue_buffer(
  14. struct smbd_connection *info);
  15. static struct smbd_response *get_receive_buffer(
  16. struct smbd_connection *info);
  17. static void put_receive_buffer(
  18. struct smbd_connection *info,
  19. struct smbd_response *response);
  20. static int allocate_receive_buffers(struct smbd_connection *info, int num_buf);
  21. static void destroy_receive_buffers(struct smbd_connection *info);
  22. static void put_empty_packet(
  23. struct smbd_connection *info, struct smbd_response *response);
  24. static void enqueue_reassembly(
  25. struct smbd_connection *info,
  26. struct smbd_response *response, int data_length);
  27. static struct smbd_response *_get_first_reassembly(
  28. struct smbd_connection *info);
  29. static int smbd_post_recv(
  30. struct smbd_connection *info,
  31. struct smbd_response *response);
  32. static int smbd_post_send_empty(struct smbd_connection *info);
  33. static int smbd_post_send_data(
  34. struct smbd_connection *info,
  35. struct kvec *iov, int n_vec, int remaining_data_length);
  36. static int smbd_post_send_page(struct smbd_connection *info,
  37. struct page *page, unsigned long offset,
  38. size_t size, int remaining_data_length);
  39. static void destroy_mr_list(struct smbd_connection *info);
  40. static int allocate_mr_list(struct smbd_connection *info);
  41. /* SMBD version number */
  42. #define SMBD_V1 0x0100
  43. /* Port numbers for SMBD transport */
  44. #define SMB_PORT 445
  45. #define SMBD_PORT 5445
  46. /* Address lookup and resolve timeout in ms */
  47. #define RDMA_RESOLVE_TIMEOUT 5000
  48. /* SMBD negotiation timeout in seconds */
  49. #define SMBD_NEGOTIATE_TIMEOUT 120
  50. /* SMBD minimum receive size and fragmented sized defined in [MS-SMBD] */
  51. #define SMBD_MIN_RECEIVE_SIZE 128
  52. #define SMBD_MIN_FRAGMENTED_SIZE 131072
  53. /*
  54. * Default maximum number of RDMA read/write outstanding on this connection
  55. * This value is possibly decreased during QP creation on hardware limit
  56. */
  57. #define SMBD_CM_RESPONDER_RESOURCES 32
  58. /* Maximum number of retries on data transfer operations */
  59. #define SMBD_CM_RETRY 6
  60. /* No need to retry on Receiver Not Ready since SMBD manages credits */
  61. #define SMBD_CM_RNR_RETRY 0
  62. /*
  63. * User configurable initial values per SMBD transport connection
  64. * as defined in [MS-SMBD] 3.1.1.1
  65. * Those may change after a SMBD negotiation
  66. */
  67. /* The local peer's maximum number of credits to grant to the peer */
  68. int smbd_receive_credit_max = 255;
  69. /* The remote peer's credit request of local peer */
  70. int smbd_send_credit_target = 255;
  71. /* The maximum single message size can be sent to remote peer */
  72. int smbd_max_send_size = 1364;
  73. /* The maximum fragmented upper-layer payload receive size supported */
  74. int smbd_max_fragmented_recv_size = 1024 * 1024;
  75. /* The maximum single-message size which can be received */
  76. int smbd_max_receive_size = 8192;
  77. /* The timeout to initiate send of a keepalive message on idle */
  78. int smbd_keep_alive_interval = 120;
  79. /*
  80. * User configurable initial values for RDMA transport
  81. * The actual values used may be lower and are limited to hardware capabilities
  82. */
  83. /* Default maximum number of SGEs in a RDMA write/read */
  84. int smbd_max_frmr_depth = 2048;
  85. /* If payload is less than this byte, use RDMA send/recv not read/write */
  86. int rdma_readwrite_threshold = 4096;
  87. /* Transport logging functions
  88. * Logging are defined as classes. They can be OR'ed to define the actual
  89. * logging level via module parameter smbd_logging_class
  90. * e.g. cifs.smbd_logging_class=0xa0 will log all log_rdma_recv() and
  91. * log_rdma_event()
  92. */
  93. #define LOG_OUTGOING 0x1
  94. #define LOG_INCOMING 0x2
  95. #define LOG_READ 0x4
  96. #define LOG_WRITE 0x8
  97. #define LOG_RDMA_SEND 0x10
  98. #define LOG_RDMA_RECV 0x20
  99. #define LOG_KEEP_ALIVE 0x40
  100. #define LOG_RDMA_EVENT 0x80
  101. #define LOG_RDMA_MR 0x100
  102. static unsigned int smbd_logging_class;
  103. module_param(smbd_logging_class, uint, 0644);
  104. MODULE_PARM_DESC(smbd_logging_class,
  105. "Logging class for SMBD transport 0x0 to 0x100");
  106. #define ERR 0x0
  107. #define INFO 0x1
  108. static unsigned int smbd_logging_level = ERR;
  109. module_param(smbd_logging_level, uint, 0644);
  110. MODULE_PARM_DESC(smbd_logging_level,
  111. "Logging level for SMBD transport, 0 (default): error, 1: info");
  112. #define log_rdma(level, class, fmt, args...) \
  113. do { \
  114. if (level <= smbd_logging_level || class & smbd_logging_class) \
  115. cifs_dbg(VFS, "%s:%d " fmt, __func__, __LINE__, ##args);\
  116. } while (0)
  117. #define log_outgoing(level, fmt, args...) \
  118. log_rdma(level, LOG_OUTGOING, fmt, ##args)
  119. #define log_incoming(level, fmt, args...) \
  120. log_rdma(level, LOG_INCOMING, fmt, ##args)
  121. #define log_read(level, fmt, args...) log_rdma(level, LOG_READ, fmt, ##args)
  122. #define log_write(level, fmt, args...) log_rdma(level, LOG_WRITE, fmt, ##args)
  123. #define log_rdma_send(level, fmt, args...) \
  124. log_rdma(level, LOG_RDMA_SEND, fmt, ##args)
  125. #define log_rdma_recv(level, fmt, args...) \
  126. log_rdma(level, LOG_RDMA_RECV, fmt, ##args)
  127. #define log_keep_alive(level, fmt, args...) \
  128. log_rdma(level, LOG_KEEP_ALIVE, fmt, ##args)
  129. #define log_rdma_event(level, fmt, args...) \
  130. log_rdma(level, LOG_RDMA_EVENT, fmt, ##args)
  131. #define log_rdma_mr(level, fmt, args...) \
  132. log_rdma(level, LOG_RDMA_MR, fmt, ##args)
  133. static void smbd_disconnect_rdma_work(struct work_struct *work)
  134. {
  135. struct smbd_connection *info =
  136. container_of(work, struct smbd_connection, disconnect_work);
  137. if (info->transport_status == SMBD_CONNECTED) {
  138. info->transport_status = SMBD_DISCONNECTING;
  139. rdma_disconnect(info->id);
  140. }
  141. }
  142. static void smbd_disconnect_rdma_connection(struct smbd_connection *info)
  143. {
  144. queue_work(info->workqueue, &info->disconnect_work);
  145. }
  146. /* Upcall from RDMA CM */
  147. static int smbd_conn_upcall(
  148. struct rdma_cm_id *id, struct rdma_cm_event *event)
  149. {
  150. struct smbd_connection *info = id->context;
  151. log_rdma_event(INFO, "event=%d status=%d\n",
  152. event->event, event->status);
  153. switch (event->event) {
  154. case RDMA_CM_EVENT_ADDR_RESOLVED:
  155. case RDMA_CM_EVENT_ROUTE_RESOLVED:
  156. info->ri_rc = 0;
  157. complete(&info->ri_done);
  158. break;
  159. case RDMA_CM_EVENT_ADDR_ERROR:
  160. info->ri_rc = -EHOSTUNREACH;
  161. complete(&info->ri_done);
  162. break;
  163. case RDMA_CM_EVENT_ROUTE_ERROR:
  164. info->ri_rc = -ENETUNREACH;
  165. complete(&info->ri_done);
  166. break;
  167. case RDMA_CM_EVENT_ESTABLISHED:
  168. log_rdma_event(INFO, "connected event=%d\n", event->event);
  169. info->transport_status = SMBD_CONNECTED;
  170. wake_up_interruptible(&info->conn_wait);
  171. break;
  172. case RDMA_CM_EVENT_CONNECT_ERROR:
  173. case RDMA_CM_EVENT_UNREACHABLE:
  174. case RDMA_CM_EVENT_REJECTED:
  175. log_rdma_event(INFO, "connecting failed event=%d\n", event->event);
  176. info->transport_status = SMBD_DISCONNECTED;
  177. wake_up_interruptible(&info->conn_wait);
  178. break;
  179. case RDMA_CM_EVENT_DEVICE_REMOVAL:
  180. case RDMA_CM_EVENT_DISCONNECTED:
  181. /* This happenes when we fail the negotiation */
  182. if (info->transport_status == SMBD_NEGOTIATE_FAILED) {
  183. info->transport_status = SMBD_DISCONNECTED;
  184. wake_up(&info->conn_wait);
  185. break;
  186. }
  187. info->transport_status = SMBD_DISCONNECTED;
  188. wake_up_interruptible(&info->disconn_wait);
  189. wake_up_interruptible(&info->wait_reassembly_queue);
  190. wake_up_interruptible_all(&info->wait_send_queue);
  191. break;
  192. default:
  193. break;
  194. }
  195. return 0;
  196. }
  197. /* Upcall from RDMA QP */
  198. static void
  199. smbd_qp_async_error_upcall(struct ib_event *event, void *context)
  200. {
  201. struct smbd_connection *info = context;
  202. log_rdma_event(ERR, "%s on device %s info %p\n",
  203. ib_event_msg(event->event), event->device->name, info);
  204. switch (event->event) {
  205. case IB_EVENT_CQ_ERR:
  206. case IB_EVENT_QP_FATAL:
  207. smbd_disconnect_rdma_connection(info);
  208. break;
  209. default:
  210. break;
  211. }
  212. }
  213. static inline void *smbd_request_payload(struct smbd_request *request)
  214. {
  215. return (void *)request->packet;
  216. }
  217. static inline void *smbd_response_payload(struct smbd_response *response)
  218. {
  219. return (void *)response->packet;
  220. }
  221. /* Called when a RDMA send is done */
  222. static void send_done(struct ib_cq *cq, struct ib_wc *wc)
  223. {
  224. int i;
  225. struct smbd_request *request =
  226. container_of(wc->wr_cqe, struct smbd_request, cqe);
  227. log_rdma_send(INFO, "smbd_request %p completed wc->status=%d\n",
  228. request, wc->status);
  229. if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) {
  230. log_rdma_send(ERR, "wc->status=%d wc->opcode=%d\n",
  231. wc->status, wc->opcode);
  232. smbd_disconnect_rdma_connection(request->info);
  233. }
  234. for (i = 0; i < request->num_sge; i++)
  235. ib_dma_unmap_single(request->info->id->device,
  236. request->sge[i].addr,
  237. request->sge[i].length,
  238. DMA_TO_DEVICE);
  239. if (atomic_dec_and_test(&request->info->send_pending))
  240. wake_up(&request->info->wait_send_pending);
  241. wake_up(&request->info->wait_post_send);
  242. mempool_free(request, request->info->request_mempool);
  243. }
  244. static void dump_smbd_negotiate_resp(struct smbd_negotiate_resp *resp)
  245. {
  246. log_rdma_event(INFO, "resp message min_version %u max_version %u negotiated_version %u credits_requested %u credits_granted %u status %u max_readwrite_size %u preferred_send_size %u max_receive_size %u max_fragmented_size %u\n",
  247. resp->min_version, resp->max_version,
  248. resp->negotiated_version, resp->credits_requested,
  249. resp->credits_granted, resp->status,
  250. resp->max_readwrite_size, resp->preferred_send_size,
  251. resp->max_receive_size, resp->max_fragmented_size);
  252. }
  253. /*
  254. * Process a negotiation response message, according to [MS-SMBD]3.1.5.7
  255. * response, packet_length: the negotiation response message
  256. * return value: true if negotiation is a success, false if failed
  257. */
  258. static bool process_negotiation_response(
  259. struct smbd_response *response, int packet_length)
  260. {
  261. struct smbd_connection *info = response->info;
  262. struct smbd_negotiate_resp *packet = smbd_response_payload(response);
  263. if (packet_length < sizeof(struct smbd_negotiate_resp)) {
  264. log_rdma_event(ERR,
  265. "error: packet_length=%d\n", packet_length);
  266. return false;
  267. }
  268. if (le16_to_cpu(packet->negotiated_version) != SMBD_V1) {
  269. log_rdma_event(ERR, "error: negotiated_version=%x\n",
  270. le16_to_cpu(packet->negotiated_version));
  271. return false;
  272. }
  273. info->protocol = le16_to_cpu(packet->negotiated_version);
  274. if (packet->credits_requested == 0) {
  275. log_rdma_event(ERR, "error: credits_requested==0\n");
  276. return false;
  277. }
  278. info->receive_credit_target = le16_to_cpu(packet->credits_requested);
  279. if (packet->credits_granted == 0) {
  280. log_rdma_event(ERR, "error: credits_granted==0\n");
  281. return false;
  282. }
  283. atomic_set(&info->send_credits, le16_to_cpu(packet->credits_granted));
  284. atomic_set(&info->receive_credits, 0);
  285. if (le32_to_cpu(packet->preferred_send_size) > info->max_receive_size) {
  286. log_rdma_event(ERR, "error: preferred_send_size=%d\n",
  287. le32_to_cpu(packet->preferred_send_size));
  288. return false;
  289. }
  290. info->max_receive_size = le32_to_cpu(packet->preferred_send_size);
  291. if (le32_to_cpu(packet->max_receive_size) < SMBD_MIN_RECEIVE_SIZE) {
  292. log_rdma_event(ERR, "error: max_receive_size=%d\n",
  293. le32_to_cpu(packet->max_receive_size));
  294. return false;
  295. }
  296. info->max_send_size = min_t(int, info->max_send_size,
  297. le32_to_cpu(packet->max_receive_size));
  298. if (le32_to_cpu(packet->max_fragmented_size) <
  299. SMBD_MIN_FRAGMENTED_SIZE) {
  300. log_rdma_event(ERR, "error: max_fragmented_size=%d\n",
  301. le32_to_cpu(packet->max_fragmented_size));
  302. return false;
  303. }
  304. info->max_fragmented_send_size =
  305. le32_to_cpu(packet->max_fragmented_size);
  306. info->rdma_readwrite_threshold =
  307. rdma_readwrite_threshold > info->max_fragmented_send_size ?
  308. info->max_fragmented_send_size :
  309. rdma_readwrite_threshold;
  310. info->max_readwrite_size = min_t(u32,
  311. le32_to_cpu(packet->max_readwrite_size),
  312. info->max_frmr_depth * PAGE_SIZE);
  313. info->max_frmr_depth = info->max_readwrite_size / PAGE_SIZE;
  314. return true;
  315. }
  316. static void smbd_post_send_credits(struct work_struct *work)
  317. {
  318. int ret = 0;
  319. int use_receive_queue = 1;
  320. int rc;
  321. struct smbd_response *response;
  322. struct smbd_connection *info =
  323. container_of(work, struct smbd_connection,
  324. post_send_credits_work);
  325. if (info->transport_status != SMBD_CONNECTED) {
  326. wake_up(&info->wait_receive_queues);
  327. return;
  328. }
  329. if (info->receive_credit_target >
  330. atomic_read(&info->receive_credits)) {
  331. while (true) {
  332. if (use_receive_queue)
  333. response = get_receive_buffer(info);
  334. else
  335. response = get_empty_queue_buffer(info);
  336. if (!response) {
  337. /* now switch to emtpy packet queue */
  338. if (use_receive_queue) {
  339. use_receive_queue = 0;
  340. continue;
  341. } else
  342. break;
  343. }
  344. response->type = SMBD_TRANSFER_DATA;
  345. response->first_segment = false;
  346. rc = smbd_post_recv(info, response);
  347. if (rc) {
  348. log_rdma_recv(ERR,
  349. "post_recv failed rc=%d\n", rc);
  350. put_receive_buffer(info, response);
  351. break;
  352. }
  353. ret++;
  354. }
  355. }
  356. spin_lock(&info->lock_new_credits_offered);
  357. info->new_credits_offered += ret;
  358. spin_unlock(&info->lock_new_credits_offered);
  359. /* Promptly send an immediate packet as defined in [MS-SMBD] 3.1.1.1 */
  360. info->send_immediate = true;
  361. if (atomic_read(&info->receive_credits) <
  362. info->receive_credit_target - 1) {
  363. if (info->keep_alive_requested == KEEP_ALIVE_PENDING ||
  364. info->send_immediate) {
  365. log_keep_alive(INFO, "send an empty message\n");
  366. smbd_post_send_empty(info);
  367. }
  368. }
  369. }
  370. /* Called from softirq, when recv is done */
  371. static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
  372. {
  373. struct smbd_data_transfer *data_transfer;
  374. struct smbd_response *response =
  375. container_of(wc->wr_cqe, struct smbd_response, cqe);
  376. struct smbd_connection *info = response->info;
  377. int data_length = 0;
  378. log_rdma_recv(INFO, "response=%p type=%d wc status=%d wc opcode %d byte_len=%d pkey_index=%x\n",
  379. response, response->type, wc->status, wc->opcode,
  380. wc->byte_len, wc->pkey_index);
  381. if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) {
  382. log_rdma_recv(INFO, "wc->status=%d opcode=%d\n",
  383. wc->status, wc->opcode);
  384. smbd_disconnect_rdma_connection(info);
  385. goto error;
  386. }
  387. ib_dma_sync_single_for_cpu(
  388. wc->qp->device,
  389. response->sge.addr,
  390. response->sge.length,
  391. DMA_FROM_DEVICE);
  392. switch (response->type) {
  393. /* SMBD negotiation response */
  394. case SMBD_NEGOTIATE_RESP:
  395. dump_smbd_negotiate_resp(smbd_response_payload(response));
  396. info->full_packet_received = true;
  397. info->negotiate_done =
  398. process_negotiation_response(response, wc->byte_len);
  399. complete(&info->negotiate_completion);
  400. break;
  401. /* SMBD data transfer packet */
  402. case SMBD_TRANSFER_DATA:
  403. data_transfer = smbd_response_payload(response);
  404. data_length = le32_to_cpu(data_transfer->data_length);
  405. /*
  406. * If this is a packet with data playload place the data in
  407. * reassembly queue and wake up the reading thread
  408. */
  409. if (data_length) {
  410. if (info->full_packet_received)
  411. response->first_segment = true;
  412. if (le32_to_cpu(data_transfer->remaining_data_length))
  413. info->full_packet_received = false;
  414. else
  415. info->full_packet_received = true;
  416. enqueue_reassembly(
  417. info,
  418. response,
  419. data_length);
  420. } else
  421. put_empty_packet(info, response);
  422. if (data_length)
  423. wake_up_interruptible(&info->wait_reassembly_queue);
  424. atomic_dec(&info->receive_credits);
  425. info->receive_credit_target =
  426. le16_to_cpu(data_transfer->credits_requested);
  427. if (le16_to_cpu(data_transfer->credits_granted)) {
  428. atomic_add(le16_to_cpu(data_transfer->credits_granted),
  429. &info->send_credits);
  430. /*
  431. * We have new send credits granted from remote peer
  432. * If any sender is waiting for credits, unblock it
  433. */
  434. wake_up_interruptible(&info->wait_send_queue);
  435. }
  436. log_incoming(INFO, "data flags %d data_offset %d data_length %d remaining_data_length %d\n",
  437. le16_to_cpu(data_transfer->flags),
  438. le32_to_cpu(data_transfer->data_offset),
  439. le32_to_cpu(data_transfer->data_length),
  440. le32_to_cpu(data_transfer->remaining_data_length));
  441. /* Send a KEEP_ALIVE response right away if requested */
  442. info->keep_alive_requested = KEEP_ALIVE_NONE;
  443. if (le16_to_cpu(data_transfer->flags) &
  444. SMB_DIRECT_RESPONSE_REQUESTED) {
  445. info->keep_alive_requested = KEEP_ALIVE_PENDING;
  446. }
  447. return;
  448. default:
  449. log_rdma_recv(ERR,
  450. "unexpected response type=%d\n", response->type);
  451. }
  452. error:
  453. put_receive_buffer(info, response);
  454. }
  455. static struct rdma_cm_id *smbd_create_id(
  456. struct smbd_connection *info,
  457. struct sockaddr *dstaddr, int port)
  458. {
  459. struct rdma_cm_id *id;
  460. int rc;
  461. __be16 *sport;
  462. id = rdma_create_id(&init_net, smbd_conn_upcall, info,
  463. RDMA_PS_TCP, IB_QPT_RC);
  464. if (IS_ERR(id)) {
  465. rc = PTR_ERR(id);
  466. log_rdma_event(ERR, "rdma_create_id() failed %i\n", rc);
  467. return id;
  468. }
  469. if (dstaddr->sa_family == AF_INET6)
  470. sport = &((struct sockaddr_in6 *)dstaddr)->sin6_port;
  471. else
  472. sport = &((struct sockaddr_in *)dstaddr)->sin_port;
  473. *sport = htons(port);
  474. init_completion(&info->ri_done);
  475. info->ri_rc = -ETIMEDOUT;
  476. rc = rdma_resolve_addr(id, NULL, (struct sockaddr *)dstaddr,
  477. RDMA_RESOLVE_TIMEOUT);
  478. if (rc) {
  479. log_rdma_event(ERR, "rdma_resolve_addr() failed %i\n", rc);
  480. goto out;
  481. }
  482. rc = wait_for_completion_interruptible_timeout(
  483. &info->ri_done, msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT));
  484. /* e.g. if interrupted returns -ERESTARTSYS */
  485. if (rc < 0) {
  486. log_rdma_event(ERR, "rdma_resolve_addr timeout rc: %i\n", rc);
  487. goto out;
  488. }
  489. rc = info->ri_rc;
  490. if (rc) {
  491. log_rdma_event(ERR, "rdma_resolve_addr() completed %i\n", rc);
  492. goto out;
  493. }
  494. info->ri_rc = -ETIMEDOUT;
  495. rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT);
  496. if (rc) {
  497. log_rdma_event(ERR, "rdma_resolve_route() failed %i\n", rc);
  498. goto out;
  499. }
  500. rc = wait_for_completion_interruptible_timeout(
  501. &info->ri_done, msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT));
  502. /* e.g. if interrupted returns -ERESTARTSYS */
  503. if (rc < 0) {
  504. log_rdma_event(ERR, "rdma_resolve_addr timeout rc: %i\n", rc);
  505. goto out;
  506. }
  507. rc = info->ri_rc;
  508. if (rc) {
  509. log_rdma_event(ERR, "rdma_resolve_route() completed %i\n", rc);
  510. goto out;
  511. }
  512. return id;
  513. out:
  514. rdma_destroy_id(id);
  515. return ERR_PTR(rc);
  516. }
  517. /*
  518. * Test if FRWR (Fast Registration Work Requests) is supported on the device
  519. * This implementation requries FRWR on RDMA read/write
  520. * return value: true if it is supported
  521. */
  522. static bool frwr_is_supported(struct ib_device_attr *attrs)
  523. {
  524. if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS))
  525. return false;
  526. if (attrs->max_fast_reg_page_list_len == 0)
  527. return false;
  528. return true;
  529. }
  530. static int smbd_ia_open(
  531. struct smbd_connection *info,
  532. struct sockaddr *dstaddr, int port)
  533. {
  534. int rc;
  535. info->id = smbd_create_id(info, dstaddr, port);
  536. if (IS_ERR(info->id)) {
  537. rc = PTR_ERR(info->id);
  538. goto out1;
  539. }
  540. if (!frwr_is_supported(&info->id->device->attrs)) {
  541. log_rdma_event(ERR, "Fast Registration Work Requests (FRWR) is not supported\n");
  542. log_rdma_event(ERR, "Device capability flags = %llx max_fast_reg_page_list_len = %u\n",
  543. info->id->device->attrs.device_cap_flags,
  544. info->id->device->attrs.max_fast_reg_page_list_len);
  545. rc = -EPROTONOSUPPORT;
  546. goto out2;
  547. }
  548. info->max_frmr_depth = min_t(int,
  549. smbd_max_frmr_depth,
  550. info->id->device->attrs.max_fast_reg_page_list_len);
  551. info->mr_type = IB_MR_TYPE_MEM_REG;
  552. if (info->id->device->attrs.kernel_cap_flags & IBK_SG_GAPS_REG)
  553. info->mr_type = IB_MR_TYPE_SG_GAPS;
  554. info->pd = ib_alloc_pd(info->id->device, 0);
  555. if (IS_ERR(info->pd)) {
  556. rc = PTR_ERR(info->pd);
  557. log_rdma_event(ERR, "ib_alloc_pd() returned %d\n", rc);
  558. goto out2;
  559. }
  560. return 0;
  561. out2:
  562. rdma_destroy_id(info->id);
  563. info->id = NULL;
  564. out1:
  565. return rc;
  566. }
  567. /*
  568. * Send a negotiation request message to the peer
  569. * The negotiation procedure is in [MS-SMBD] 3.1.5.2 and 3.1.5.3
  570. * After negotiation, the transport is connected and ready for
  571. * carrying upper layer SMB payload
  572. */
  573. static int smbd_post_send_negotiate_req(struct smbd_connection *info)
  574. {
  575. struct ib_send_wr send_wr;
  576. int rc = -ENOMEM;
  577. struct smbd_request *request;
  578. struct smbd_negotiate_req *packet;
  579. request = mempool_alloc(info->request_mempool, GFP_KERNEL);
  580. if (!request)
  581. return rc;
  582. request->info = info;
  583. packet = smbd_request_payload(request);
  584. packet->min_version = cpu_to_le16(SMBD_V1);
  585. packet->max_version = cpu_to_le16(SMBD_V1);
  586. packet->reserved = 0;
  587. packet->credits_requested = cpu_to_le16(info->send_credit_target);
  588. packet->preferred_send_size = cpu_to_le32(info->max_send_size);
  589. packet->max_receive_size = cpu_to_le32(info->max_receive_size);
  590. packet->max_fragmented_size =
  591. cpu_to_le32(info->max_fragmented_recv_size);
  592. request->num_sge = 1;
  593. request->sge[0].addr = ib_dma_map_single(
  594. info->id->device, (void *)packet,
  595. sizeof(*packet), DMA_TO_DEVICE);
  596. if (ib_dma_mapping_error(info->id->device, request->sge[0].addr)) {
  597. rc = -EIO;
  598. goto dma_mapping_failed;
  599. }
  600. request->sge[0].length = sizeof(*packet);
  601. request->sge[0].lkey = info->pd->local_dma_lkey;
  602. ib_dma_sync_single_for_device(
  603. info->id->device, request->sge[0].addr,
  604. request->sge[0].length, DMA_TO_DEVICE);
  605. request->cqe.done = send_done;
  606. send_wr.next = NULL;
  607. send_wr.wr_cqe = &request->cqe;
  608. send_wr.sg_list = request->sge;
  609. send_wr.num_sge = request->num_sge;
  610. send_wr.opcode = IB_WR_SEND;
  611. send_wr.send_flags = IB_SEND_SIGNALED;
  612. log_rdma_send(INFO, "sge addr=%llx length=%x lkey=%x\n",
  613. request->sge[0].addr,
  614. request->sge[0].length, request->sge[0].lkey);
  615. atomic_inc(&info->send_pending);
  616. rc = ib_post_send(info->id->qp, &send_wr, NULL);
  617. if (!rc)
  618. return 0;
  619. /* if we reach here, post send failed */
  620. log_rdma_send(ERR, "ib_post_send failed rc=%d\n", rc);
  621. atomic_dec(&info->send_pending);
  622. ib_dma_unmap_single(info->id->device, request->sge[0].addr,
  623. request->sge[0].length, DMA_TO_DEVICE);
  624. smbd_disconnect_rdma_connection(info);
  625. dma_mapping_failed:
  626. mempool_free(request, info->request_mempool);
  627. return rc;
  628. }
  629. /*
  630. * Extend the credits to remote peer
  631. * This implements [MS-SMBD] 3.1.5.9
  632. * The idea is that we should extend credits to remote peer as quickly as
  633. * it's allowed, to maintain data flow. We allocate as much receive
  634. * buffer as possible, and extend the receive credits to remote peer
  635. * return value: the new credtis being granted.
  636. */
  637. static int manage_credits_prior_sending(struct smbd_connection *info)
  638. {
  639. int new_credits;
  640. spin_lock(&info->lock_new_credits_offered);
  641. new_credits = info->new_credits_offered;
  642. info->new_credits_offered = 0;
  643. spin_unlock(&info->lock_new_credits_offered);
  644. return new_credits;
  645. }
  646. /*
  647. * Check if we need to send a KEEP_ALIVE message
  648. * The idle connection timer triggers a KEEP_ALIVE message when expires
  649. * SMB_DIRECT_RESPONSE_REQUESTED is set in the message flag to have peer send
  650. * back a response.
  651. * return value:
  652. * 1 if SMB_DIRECT_RESPONSE_REQUESTED needs to be set
  653. * 0: otherwise
  654. */
  655. static int manage_keep_alive_before_sending(struct smbd_connection *info)
  656. {
  657. if (info->keep_alive_requested == KEEP_ALIVE_PENDING) {
  658. info->keep_alive_requested = KEEP_ALIVE_SENT;
  659. return 1;
  660. }
  661. return 0;
  662. }
  663. /* Post the send request */
  664. static int smbd_post_send(struct smbd_connection *info,
  665. struct smbd_request *request)
  666. {
  667. struct ib_send_wr send_wr;
  668. int rc, i;
  669. for (i = 0; i < request->num_sge; i++) {
  670. log_rdma_send(INFO,
  671. "rdma_request sge[%d] addr=%llu length=%u\n",
  672. i, request->sge[i].addr, request->sge[i].length);
  673. ib_dma_sync_single_for_device(
  674. info->id->device,
  675. request->sge[i].addr,
  676. request->sge[i].length,
  677. DMA_TO_DEVICE);
  678. }
  679. request->cqe.done = send_done;
  680. send_wr.next = NULL;
  681. send_wr.wr_cqe = &request->cqe;
  682. send_wr.sg_list = request->sge;
  683. send_wr.num_sge = request->num_sge;
  684. send_wr.opcode = IB_WR_SEND;
  685. send_wr.send_flags = IB_SEND_SIGNALED;
  686. rc = ib_post_send(info->id->qp, &send_wr, NULL);
  687. if (rc) {
  688. log_rdma_send(ERR, "ib_post_send failed rc=%d\n", rc);
  689. smbd_disconnect_rdma_connection(info);
  690. rc = -EAGAIN;
  691. } else
  692. /* Reset timer for idle connection after packet is sent */
  693. mod_delayed_work(info->workqueue, &info->idle_timer_work,
  694. info->keep_alive_interval*HZ);
  695. return rc;
  696. }
  697. static int smbd_post_send_sgl(struct smbd_connection *info,
  698. struct scatterlist *sgl, int data_length, int remaining_data_length)
  699. {
  700. int num_sgs;
  701. int i, rc;
  702. int header_length;
  703. struct smbd_request *request;
  704. struct smbd_data_transfer *packet;
  705. int new_credits;
  706. struct scatterlist *sg;
  707. wait_credit:
  708. /* Wait for send credits. A SMBD packet needs one credit */
  709. rc = wait_event_interruptible(info->wait_send_queue,
  710. atomic_read(&info->send_credits) > 0 ||
  711. info->transport_status != SMBD_CONNECTED);
  712. if (rc)
  713. goto err_wait_credit;
  714. if (info->transport_status != SMBD_CONNECTED) {
  715. log_outgoing(ERR, "disconnected not sending on wait_credit\n");
  716. rc = -EAGAIN;
  717. goto err_wait_credit;
  718. }
  719. if (unlikely(atomic_dec_return(&info->send_credits) < 0)) {
  720. atomic_inc(&info->send_credits);
  721. goto wait_credit;
  722. }
  723. wait_send_queue:
  724. wait_event(info->wait_post_send,
  725. atomic_read(&info->send_pending) < info->send_credit_target ||
  726. info->transport_status != SMBD_CONNECTED);
  727. if (info->transport_status != SMBD_CONNECTED) {
  728. log_outgoing(ERR, "disconnected not sending on wait_send_queue\n");
  729. rc = -EAGAIN;
  730. goto err_wait_send_queue;
  731. }
  732. if (unlikely(atomic_inc_return(&info->send_pending) >
  733. info->send_credit_target)) {
  734. atomic_dec(&info->send_pending);
  735. goto wait_send_queue;
  736. }
  737. request = mempool_alloc(info->request_mempool, GFP_KERNEL);
  738. if (!request) {
  739. rc = -ENOMEM;
  740. goto err_alloc;
  741. }
  742. request->info = info;
  743. /* Fill in the packet header */
  744. packet = smbd_request_payload(request);
  745. packet->credits_requested = cpu_to_le16(info->send_credit_target);
  746. new_credits = manage_credits_prior_sending(info);
  747. atomic_add(new_credits, &info->receive_credits);
  748. packet->credits_granted = cpu_to_le16(new_credits);
  749. info->send_immediate = false;
  750. packet->flags = 0;
  751. if (manage_keep_alive_before_sending(info))
  752. packet->flags |= cpu_to_le16(SMB_DIRECT_RESPONSE_REQUESTED);
  753. packet->reserved = 0;
  754. if (!data_length)
  755. packet->data_offset = 0;
  756. else
  757. packet->data_offset = cpu_to_le32(24);
  758. packet->data_length = cpu_to_le32(data_length);
  759. packet->remaining_data_length = cpu_to_le32(remaining_data_length);
  760. packet->padding = 0;
  761. log_outgoing(INFO, "credits_requested=%d credits_granted=%d data_offset=%d data_length=%d remaining_data_length=%d\n",
  762. le16_to_cpu(packet->credits_requested),
  763. le16_to_cpu(packet->credits_granted),
  764. le32_to_cpu(packet->data_offset),
  765. le32_to_cpu(packet->data_length),
  766. le32_to_cpu(packet->remaining_data_length));
  767. /* Map the packet to DMA */
  768. header_length = sizeof(struct smbd_data_transfer);
  769. /* If this is a packet without payload, don't send padding */
  770. if (!data_length)
  771. header_length = offsetof(struct smbd_data_transfer, padding);
  772. request->num_sge = 1;
  773. request->sge[0].addr = ib_dma_map_single(info->id->device,
  774. (void *)packet,
  775. header_length,
  776. DMA_TO_DEVICE);
  777. if (ib_dma_mapping_error(info->id->device, request->sge[0].addr)) {
  778. rc = -EIO;
  779. request->sge[0].addr = 0;
  780. goto err_dma;
  781. }
  782. request->sge[0].length = header_length;
  783. request->sge[0].lkey = info->pd->local_dma_lkey;
  784. /* Fill in the packet data payload */
  785. num_sgs = sgl ? sg_nents(sgl) : 0;
  786. for_each_sg(sgl, sg, num_sgs, i) {
  787. request->sge[i+1].addr =
  788. ib_dma_map_page(info->id->device, sg_page(sg),
  789. sg->offset, sg->length, DMA_TO_DEVICE);
  790. if (ib_dma_mapping_error(
  791. info->id->device, request->sge[i+1].addr)) {
  792. rc = -EIO;
  793. request->sge[i+1].addr = 0;
  794. goto err_dma;
  795. }
  796. request->sge[i+1].length = sg->length;
  797. request->sge[i+1].lkey = info->pd->local_dma_lkey;
  798. request->num_sge++;
  799. }
  800. rc = smbd_post_send(info, request);
  801. if (!rc)
  802. return 0;
  803. err_dma:
  804. for (i = 0; i < request->num_sge; i++)
  805. if (request->sge[i].addr)
  806. ib_dma_unmap_single(info->id->device,
  807. request->sge[i].addr,
  808. request->sge[i].length,
  809. DMA_TO_DEVICE);
  810. mempool_free(request, info->request_mempool);
  811. /* roll back receive credits and credits to be offered */
  812. spin_lock(&info->lock_new_credits_offered);
  813. info->new_credits_offered += new_credits;
  814. spin_unlock(&info->lock_new_credits_offered);
  815. atomic_sub(new_credits, &info->receive_credits);
  816. err_alloc:
  817. if (atomic_dec_and_test(&info->send_pending))
  818. wake_up(&info->wait_send_pending);
  819. err_wait_send_queue:
  820. /* roll back send credits and pending */
  821. atomic_inc(&info->send_credits);
  822. err_wait_credit:
  823. return rc;
  824. }
  825. /*
  826. * Send a page
  827. * page: the page to send
  828. * offset: offset in the page to send
  829. * size: length in the page to send
  830. * remaining_data_length: remaining data to send in this payload
  831. */
  832. static int smbd_post_send_page(struct smbd_connection *info, struct page *page,
  833. unsigned long offset, size_t size, int remaining_data_length)
  834. {
  835. struct scatterlist sgl;
  836. sg_init_table(&sgl, 1);
  837. sg_set_page(&sgl, page, size, offset);
  838. return smbd_post_send_sgl(info, &sgl, size, remaining_data_length);
  839. }
  840. /*
  841. * Send an empty message
  842. * Empty message is used to extend credits to peer to for keep live
  843. * while there is no upper layer payload to send at the time
  844. */
  845. static int smbd_post_send_empty(struct smbd_connection *info)
  846. {
  847. info->count_send_empty++;
  848. return smbd_post_send_sgl(info, NULL, 0, 0);
  849. }
  850. /*
  851. * Send a data buffer
  852. * iov: the iov array describing the data buffers
  853. * n_vec: number of iov array
  854. * remaining_data_length: remaining data to send following this packet
  855. * in segmented SMBD packet
  856. */
  857. static int smbd_post_send_data(
  858. struct smbd_connection *info, struct kvec *iov, int n_vec,
  859. int remaining_data_length)
  860. {
  861. int i;
  862. u32 data_length = 0;
  863. struct scatterlist sgl[SMBDIRECT_MAX_SGE];
  864. if (n_vec > SMBDIRECT_MAX_SGE) {
  865. cifs_dbg(VFS, "Can't fit data to SGL, n_vec=%d\n", n_vec);
  866. return -EINVAL;
  867. }
  868. sg_init_table(sgl, n_vec);
  869. for (i = 0; i < n_vec; i++) {
  870. data_length += iov[i].iov_len;
  871. sg_set_buf(&sgl[i], iov[i].iov_base, iov[i].iov_len);
  872. }
  873. return smbd_post_send_sgl(info, sgl, data_length, remaining_data_length);
  874. }
  875. /*
  876. * Post a receive request to the transport
  877. * The remote peer can only send data when a receive request is posted
  878. * The interaction is controlled by send/receive credit system
  879. */
  880. static int smbd_post_recv(
  881. struct smbd_connection *info, struct smbd_response *response)
  882. {
  883. struct ib_recv_wr recv_wr;
  884. int rc = -EIO;
  885. response->sge.addr = ib_dma_map_single(
  886. info->id->device, response->packet,
  887. info->max_receive_size, DMA_FROM_DEVICE);
  888. if (ib_dma_mapping_error(info->id->device, response->sge.addr))
  889. return rc;
  890. response->sge.length = info->max_receive_size;
  891. response->sge.lkey = info->pd->local_dma_lkey;
  892. response->cqe.done = recv_done;
  893. recv_wr.wr_cqe = &response->cqe;
  894. recv_wr.next = NULL;
  895. recv_wr.sg_list = &response->sge;
  896. recv_wr.num_sge = 1;
  897. rc = ib_post_recv(info->id->qp, &recv_wr, NULL);
  898. if (rc) {
  899. ib_dma_unmap_single(info->id->device, response->sge.addr,
  900. response->sge.length, DMA_FROM_DEVICE);
  901. smbd_disconnect_rdma_connection(info);
  902. log_rdma_recv(ERR, "ib_post_recv failed rc=%d\n", rc);
  903. }
  904. return rc;
  905. }
  906. /* Perform SMBD negotiate according to [MS-SMBD] 3.1.5.2 */
  907. static int smbd_negotiate(struct smbd_connection *info)
  908. {
  909. int rc;
  910. struct smbd_response *response = get_receive_buffer(info);
  911. response->type = SMBD_NEGOTIATE_RESP;
  912. rc = smbd_post_recv(info, response);
  913. log_rdma_event(INFO, "smbd_post_recv rc=%d iov.addr=%llx iov.length=%x iov.lkey=%x\n",
  914. rc, response->sge.addr,
  915. response->sge.length, response->sge.lkey);
  916. if (rc)
  917. return rc;
  918. init_completion(&info->negotiate_completion);
  919. info->negotiate_done = false;
  920. rc = smbd_post_send_negotiate_req(info);
  921. if (rc)
  922. return rc;
  923. rc = wait_for_completion_interruptible_timeout(
  924. &info->negotiate_completion, SMBD_NEGOTIATE_TIMEOUT * HZ);
  925. log_rdma_event(INFO, "wait_for_completion_timeout rc=%d\n", rc);
  926. if (info->negotiate_done)
  927. return 0;
  928. if (rc == 0)
  929. rc = -ETIMEDOUT;
  930. else if (rc == -ERESTARTSYS)
  931. rc = -EINTR;
  932. else
  933. rc = -ENOTCONN;
  934. return rc;
  935. }
  936. static void put_empty_packet(
  937. struct smbd_connection *info, struct smbd_response *response)
  938. {
  939. spin_lock(&info->empty_packet_queue_lock);
  940. list_add_tail(&response->list, &info->empty_packet_queue);
  941. info->count_empty_packet_queue++;
  942. spin_unlock(&info->empty_packet_queue_lock);
  943. queue_work(info->workqueue, &info->post_send_credits_work);
  944. }
  945. /*
  946. * Implement Connection.FragmentReassemblyBuffer defined in [MS-SMBD] 3.1.1.1
  947. * This is a queue for reassembling upper layer payload and present to upper
  948. * layer. All the inncoming payload go to the reassembly queue, regardless of
  949. * if reassembly is required. The uuper layer code reads from the queue for all
  950. * incoming payloads.
  951. * Put a received packet to the reassembly queue
  952. * response: the packet received
  953. * data_length: the size of payload in this packet
  954. */
  955. static void enqueue_reassembly(
  956. struct smbd_connection *info,
  957. struct smbd_response *response,
  958. int data_length)
  959. {
  960. spin_lock(&info->reassembly_queue_lock);
  961. list_add_tail(&response->list, &info->reassembly_queue);
  962. info->reassembly_queue_length++;
  963. /*
  964. * Make sure reassembly_data_length is updated after list and
  965. * reassembly_queue_length are updated. On the dequeue side
  966. * reassembly_data_length is checked without a lock to determine
  967. * if reassembly_queue_length and list is up to date
  968. */
  969. virt_wmb();
  970. info->reassembly_data_length += data_length;
  971. spin_unlock(&info->reassembly_queue_lock);
  972. info->count_reassembly_queue++;
  973. info->count_enqueue_reassembly_queue++;
  974. }
  975. /*
  976. * Get the first entry at the front of reassembly queue
  977. * Caller is responsible for locking
  978. * return value: the first entry if any, NULL if queue is empty
  979. */
  980. static struct smbd_response *_get_first_reassembly(struct smbd_connection *info)
  981. {
  982. struct smbd_response *ret = NULL;
  983. if (!list_empty(&info->reassembly_queue)) {
  984. ret = list_first_entry(
  985. &info->reassembly_queue,
  986. struct smbd_response, list);
  987. }
  988. return ret;
  989. }
  990. static struct smbd_response *get_empty_queue_buffer(
  991. struct smbd_connection *info)
  992. {
  993. struct smbd_response *ret = NULL;
  994. unsigned long flags;
  995. spin_lock_irqsave(&info->empty_packet_queue_lock, flags);
  996. if (!list_empty(&info->empty_packet_queue)) {
  997. ret = list_first_entry(
  998. &info->empty_packet_queue,
  999. struct smbd_response, list);
  1000. list_del(&ret->list);
  1001. info->count_empty_packet_queue--;
  1002. }
  1003. spin_unlock_irqrestore(&info->empty_packet_queue_lock, flags);
  1004. return ret;
  1005. }
  1006. /*
  1007. * Get a receive buffer
  1008. * For each remote send, we need to post a receive. The receive buffers are
  1009. * pre-allocated in advance.
  1010. * return value: the receive buffer, NULL if none is available
  1011. */
  1012. static struct smbd_response *get_receive_buffer(struct smbd_connection *info)
  1013. {
  1014. struct smbd_response *ret = NULL;
  1015. unsigned long flags;
  1016. spin_lock_irqsave(&info->receive_queue_lock, flags);
  1017. if (!list_empty(&info->receive_queue)) {
  1018. ret = list_first_entry(
  1019. &info->receive_queue,
  1020. struct smbd_response, list);
  1021. list_del(&ret->list);
  1022. info->count_receive_queue--;
  1023. info->count_get_receive_buffer++;
  1024. }
  1025. spin_unlock_irqrestore(&info->receive_queue_lock, flags);
  1026. return ret;
  1027. }
  1028. /*
  1029. * Return a receive buffer
  1030. * Upon returning of a receive buffer, we can post new receive and extend
  1031. * more receive credits to remote peer. This is done immediately after a
  1032. * receive buffer is returned.
  1033. */
  1034. static void put_receive_buffer(
  1035. struct smbd_connection *info, struct smbd_response *response)
  1036. {
  1037. unsigned long flags;
  1038. ib_dma_unmap_single(info->id->device, response->sge.addr,
  1039. response->sge.length, DMA_FROM_DEVICE);
  1040. spin_lock_irqsave(&info->receive_queue_lock, flags);
  1041. list_add_tail(&response->list, &info->receive_queue);
  1042. info->count_receive_queue++;
  1043. info->count_put_receive_buffer++;
  1044. spin_unlock_irqrestore(&info->receive_queue_lock, flags);
  1045. queue_work(info->workqueue, &info->post_send_credits_work);
  1046. }
  1047. /* Preallocate all receive buffer on transport establishment */
  1048. static int allocate_receive_buffers(struct smbd_connection *info, int num_buf)
  1049. {
  1050. int i;
  1051. struct smbd_response *response;
  1052. INIT_LIST_HEAD(&info->reassembly_queue);
  1053. spin_lock_init(&info->reassembly_queue_lock);
  1054. info->reassembly_data_length = 0;
  1055. info->reassembly_queue_length = 0;
  1056. INIT_LIST_HEAD(&info->receive_queue);
  1057. spin_lock_init(&info->receive_queue_lock);
  1058. info->count_receive_queue = 0;
  1059. INIT_LIST_HEAD(&info->empty_packet_queue);
  1060. spin_lock_init(&info->empty_packet_queue_lock);
  1061. info->count_empty_packet_queue = 0;
  1062. init_waitqueue_head(&info->wait_receive_queues);
  1063. for (i = 0; i < num_buf; i++) {
  1064. response = mempool_alloc(info->response_mempool, GFP_KERNEL);
  1065. if (!response)
  1066. goto allocate_failed;
  1067. response->info = info;
  1068. list_add_tail(&response->list, &info->receive_queue);
  1069. info->count_receive_queue++;
  1070. }
  1071. return 0;
  1072. allocate_failed:
  1073. while (!list_empty(&info->receive_queue)) {
  1074. response = list_first_entry(
  1075. &info->receive_queue,
  1076. struct smbd_response, list);
  1077. list_del(&response->list);
  1078. info->count_receive_queue--;
  1079. mempool_free(response, info->response_mempool);
  1080. }
  1081. return -ENOMEM;
  1082. }
  1083. static void destroy_receive_buffers(struct smbd_connection *info)
  1084. {
  1085. struct smbd_response *response;
  1086. while ((response = get_receive_buffer(info)))
  1087. mempool_free(response, info->response_mempool);
  1088. while ((response = get_empty_queue_buffer(info)))
  1089. mempool_free(response, info->response_mempool);
  1090. }
  1091. /* Implement idle connection timer [MS-SMBD] 3.1.6.2 */
  1092. static void idle_connection_timer(struct work_struct *work)
  1093. {
  1094. struct smbd_connection *info = container_of(
  1095. work, struct smbd_connection,
  1096. idle_timer_work.work);
  1097. if (info->keep_alive_requested != KEEP_ALIVE_NONE) {
  1098. log_keep_alive(ERR,
  1099. "error status info->keep_alive_requested=%d\n",
  1100. info->keep_alive_requested);
  1101. smbd_disconnect_rdma_connection(info);
  1102. return;
  1103. }
  1104. log_keep_alive(INFO, "about to send an empty idle message\n");
  1105. smbd_post_send_empty(info);
  1106. /* Setup the next idle timeout work */
  1107. queue_delayed_work(info->workqueue, &info->idle_timer_work,
  1108. info->keep_alive_interval*HZ);
  1109. }
  1110. /*
  1111. * Destroy the transport and related RDMA and memory resources
  1112. * Need to go through all the pending counters and make sure on one is using
  1113. * the transport while it is destroyed
  1114. */
  1115. void smbd_destroy(struct TCP_Server_Info *server)
  1116. {
  1117. struct smbd_connection *info = server->smbd_conn;
  1118. struct smbd_response *response;
  1119. unsigned long flags;
  1120. if (!info) {
  1121. log_rdma_event(INFO, "rdma session already destroyed\n");
  1122. return;
  1123. }
  1124. log_rdma_event(INFO, "destroying rdma session\n");
  1125. if (info->transport_status != SMBD_DISCONNECTED) {
  1126. rdma_disconnect(server->smbd_conn->id);
  1127. log_rdma_event(INFO, "wait for transport being disconnected\n");
  1128. wait_event_interruptible(
  1129. info->disconn_wait,
  1130. info->transport_status == SMBD_DISCONNECTED);
  1131. }
  1132. log_rdma_event(INFO, "destroying qp\n");
  1133. ib_drain_qp(info->id->qp);
  1134. rdma_destroy_qp(info->id);
  1135. log_rdma_event(INFO, "cancelling idle timer\n");
  1136. cancel_delayed_work_sync(&info->idle_timer_work);
  1137. log_rdma_event(INFO, "wait for all send posted to IB to finish\n");
  1138. wait_event(info->wait_send_pending,
  1139. atomic_read(&info->send_pending) == 0);
  1140. /* It's not possible for upper layer to get to reassembly */
  1141. log_rdma_event(INFO, "drain the reassembly queue\n");
  1142. do {
  1143. spin_lock_irqsave(&info->reassembly_queue_lock, flags);
  1144. response = _get_first_reassembly(info);
  1145. if (response) {
  1146. list_del(&response->list);
  1147. spin_unlock_irqrestore(
  1148. &info->reassembly_queue_lock, flags);
  1149. put_receive_buffer(info, response);
  1150. } else
  1151. spin_unlock_irqrestore(
  1152. &info->reassembly_queue_lock, flags);
  1153. } while (response);
  1154. info->reassembly_data_length = 0;
  1155. log_rdma_event(INFO, "free receive buffers\n");
  1156. wait_event(info->wait_receive_queues,
  1157. info->count_receive_queue + info->count_empty_packet_queue
  1158. == info->receive_credit_max);
  1159. destroy_receive_buffers(info);
  1160. /*
  1161. * For performance reasons, memory registration and deregistration
  1162. * are not locked by srv_mutex. It is possible some processes are
  1163. * blocked on transport srv_mutex while holding memory registration.
  1164. * Release the transport srv_mutex to allow them to hit the failure
  1165. * path when sending data, and then release memory registartions.
  1166. */
  1167. log_rdma_event(INFO, "freeing mr list\n");
  1168. wake_up_interruptible_all(&info->wait_mr);
  1169. while (atomic_read(&info->mr_used_count)) {
  1170. cifs_server_unlock(server);
  1171. msleep(1000);
  1172. cifs_server_lock(server);
  1173. }
  1174. destroy_mr_list(info);
  1175. ib_free_cq(info->send_cq);
  1176. ib_free_cq(info->recv_cq);
  1177. ib_dealloc_pd(info->pd);
  1178. rdma_destroy_id(info->id);
  1179. /* free mempools */
  1180. mempool_destroy(info->request_mempool);
  1181. kmem_cache_destroy(info->request_cache);
  1182. mempool_destroy(info->response_mempool);
  1183. kmem_cache_destroy(info->response_cache);
  1184. info->transport_status = SMBD_DESTROYED;
  1185. destroy_workqueue(info->workqueue);
  1186. log_rdma_event(INFO, "rdma session destroyed\n");
  1187. kfree(info);
  1188. }
  1189. /*
  1190. * Reconnect this SMBD connection, called from upper layer
  1191. * return value: 0 on success, or actual error code
  1192. */
  1193. int smbd_reconnect(struct TCP_Server_Info *server)
  1194. {
  1195. log_rdma_event(INFO, "reconnecting rdma session\n");
  1196. if (!server->smbd_conn) {
  1197. log_rdma_event(INFO, "rdma session already destroyed\n");
  1198. goto create_conn;
  1199. }
  1200. /*
  1201. * This is possible if transport is disconnected and we haven't received
  1202. * notification from RDMA, but upper layer has detected timeout
  1203. */
  1204. if (server->smbd_conn->transport_status == SMBD_CONNECTED) {
  1205. log_rdma_event(INFO, "disconnecting transport\n");
  1206. smbd_destroy(server);
  1207. }
  1208. create_conn:
  1209. log_rdma_event(INFO, "creating rdma session\n");
  1210. server->smbd_conn = smbd_get_connection(
  1211. server, (struct sockaddr *) &server->dstaddr);
  1212. if (server->smbd_conn)
  1213. cifs_dbg(VFS, "RDMA transport re-established\n");
  1214. return server->smbd_conn ? 0 : -ENOENT;
  1215. }
  1216. static void destroy_caches_and_workqueue(struct smbd_connection *info)
  1217. {
  1218. destroy_receive_buffers(info);
  1219. destroy_workqueue(info->workqueue);
  1220. mempool_destroy(info->response_mempool);
  1221. kmem_cache_destroy(info->response_cache);
  1222. mempool_destroy(info->request_mempool);
  1223. kmem_cache_destroy(info->request_cache);
  1224. }
  1225. #define MAX_NAME_LEN 80
  1226. static int allocate_caches_and_workqueue(struct smbd_connection *info)
  1227. {
  1228. char name[MAX_NAME_LEN];
  1229. int rc;
  1230. scnprintf(name, MAX_NAME_LEN, "smbd_request_%p", info);
  1231. info->request_cache =
  1232. kmem_cache_create(
  1233. name,
  1234. sizeof(struct smbd_request) +
  1235. sizeof(struct smbd_data_transfer),
  1236. 0, SLAB_HWCACHE_ALIGN, NULL);
  1237. if (!info->request_cache)
  1238. return -ENOMEM;
  1239. info->request_mempool =
  1240. mempool_create(info->send_credit_target, mempool_alloc_slab,
  1241. mempool_free_slab, info->request_cache);
  1242. if (!info->request_mempool)
  1243. goto out1;
  1244. scnprintf(name, MAX_NAME_LEN, "smbd_response_%p", info);
  1245. info->response_cache =
  1246. kmem_cache_create(
  1247. name,
  1248. sizeof(struct smbd_response) +
  1249. info->max_receive_size,
  1250. 0, SLAB_HWCACHE_ALIGN, NULL);
  1251. if (!info->response_cache)
  1252. goto out2;
  1253. info->response_mempool =
  1254. mempool_create(info->receive_credit_max, mempool_alloc_slab,
  1255. mempool_free_slab, info->response_cache);
  1256. if (!info->response_mempool)
  1257. goto out3;
  1258. scnprintf(name, MAX_NAME_LEN, "smbd_%p", info);
  1259. info->workqueue = create_workqueue(name);
  1260. if (!info->workqueue)
  1261. goto out4;
  1262. rc = allocate_receive_buffers(info, info->receive_credit_max);
  1263. if (rc) {
  1264. log_rdma_event(ERR, "failed to allocate receive buffers\n");
  1265. goto out5;
  1266. }
  1267. return 0;
  1268. out5:
  1269. destroy_workqueue(info->workqueue);
  1270. out4:
  1271. mempool_destroy(info->response_mempool);
  1272. out3:
  1273. kmem_cache_destroy(info->response_cache);
  1274. out2:
  1275. mempool_destroy(info->request_mempool);
  1276. out1:
  1277. kmem_cache_destroy(info->request_cache);
  1278. return -ENOMEM;
  1279. }
  1280. /* Create a SMBD connection, called by upper layer */
  1281. static struct smbd_connection *_smbd_get_connection(
  1282. struct TCP_Server_Info *server, struct sockaddr *dstaddr, int port)
  1283. {
  1284. int rc;
  1285. struct smbd_connection *info;
  1286. struct rdma_conn_param conn_param;
  1287. struct ib_qp_init_attr qp_attr;
  1288. struct sockaddr_in *addr_in = (struct sockaddr_in *) dstaddr;
  1289. struct ib_port_immutable port_immutable;
  1290. u32 ird_ord_hdr[2];
  1291. info = kzalloc(sizeof(struct smbd_connection), GFP_KERNEL);
  1292. if (!info)
  1293. return NULL;
  1294. info->transport_status = SMBD_CONNECTING;
  1295. rc = smbd_ia_open(info, dstaddr, port);
  1296. if (rc) {
  1297. log_rdma_event(INFO, "smbd_ia_open rc=%d\n", rc);
  1298. goto create_id_failed;
  1299. }
  1300. if (smbd_send_credit_target > info->id->device->attrs.max_cqe ||
  1301. smbd_send_credit_target > info->id->device->attrs.max_qp_wr) {
  1302. log_rdma_event(ERR, "consider lowering send_credit_target = %d. Possible CQE overrun, device reporting max_cpe %d max_qp_wr %d\n",
  1303. smbd_send_credit_target,
  1304. info->id->device->attrs.max_cqe,
  1305. info->id->device->attrs.max_qp_wr);
  1306. goto config_failed;
  1307. }
  1308. if (smbd_receive_credit_max > info->id->device->attrs.max_cqe ||
  1309. smbd_receive_credit_max > info->id->device->attrs.max_qp_wr) {
  1310. log_rdma_event(ERR, "consider lowering receive_credit_max = %d. Possible CQE overrun, device reporting max_cpe %d max_qp_wr %d\n",
  1311. smbd_receive_credit_max,
  1312. info->id->device->attrs.max_cqe,
  1313. info->id->device->attrs.max_qp_wr);
  1314. goto config_failed;
  1315. }
  1316. info->receive_credit_max = smbd_receive_credit_max;
  1317. info->send_credit_target = smbd_send_credit_target;
  1318. info->max_send_size = smbd_max_send_size;
  1319. info->max_fragmented_recv_size = smbd_max_fragmented_recv_size;
  1320. info->max_receive_size = smbd_max_receive_size;
  1321. info->keep_alive_interval = smbd_keep_alive_interval;
  1322. if (info->id->device->attrs.max_send_sge < SMBDIRECT_MAX_SGE) {
  1323. log_rdma_event(ERR,
  1324. "warning: device max_send_sge = %d too small\n",
  1325. info->id->device->attrs.max_send_sge);
  1326. log_rdma_event(ERR, "Queue Pair creation may fail\n");
  1327. }
  1328. if (info->id->device->attrs.max_recv_sge < SMBDIRECT_MAX_SGE) {
  1329. log_rdma_event(ERR,
  1330. "warning: device max_recv_sge = %d too small\n",
  1331. info->id->device->attrs.max_recv_sge);
  1332. log_rdma_event(ERR, "Queue Pair creation may fail\n");
  1333. }
  1334. info->send_cq = NULL;
  1335. info->recv_cq = NULL;
  1336. info->send_cq =
  1337. ib_alloc_cq_any(info->id->device, info,
  1338. info->send_credit_target, IB_POLL_SOFTIRQ);
  1339. if (IS_ERR(info->send_cq)) {
  1340. info->send_cq = NULL;
  1341. goto alloc_cq_failed;
  1342. }
  1343. info->recv_cq =
  1344. ib_alloc_cq_any(info->id->device, info,
  1345. info->receive_credit_max, IB_POLL_SOFTIRQ);
  1346. if (IS_ERR(info->recv_cq)) {
  1347. info->recv_cq = NULL;
  1348. goto alloc_cq_failed;
  1349. }
  1350. memset(&qp_attr, 0, sizeof(qp_attr));
  1351. qp_attr.event_handler = smbd_qp_async_error_upcall;
  1352. qp_attr.qp_context = info;
  1353. qp_attr.cap.max_send_wr = info->send_credit_target;
  1354. qp_attr.cap.max_recv_wr = info->receive_credit_max;
  1355. qp_attr.cap.max_send_sge = SMBDIRECT_MAX_SGE;
  1356. qp_attr.cap.max_recv_sge = SMBDIRECT_MAX_SGE;
  1357. qp_attr.cap.max_inline_data = 0;
  1358. qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
  1359. qp_attr.qp_type = IB_QPT_RC;
  1360. qp_attr.send_cq = info->send_cq;
  1361. qp_attr.recv_cq = info->recv_cq;
  1362. qp_attr.port_num = ~0;
  1363. rc = rdma_create_qp(info->id, info->pd, &qp_attr);
  1364. if (rc) {
  1365. log_rdma_event(ERR, "rdma_create_qp failed %i\n", rc);
  1366. goto create_qp_failed;
  1367. }
  1368. memset(&conn_param, 0, sizeof(conn_param));
  1369. conn_param.initiator_depth = 0;
  1370. conn_param.responder_resources =
  1371. info->id->device->attrs.max_qp_rd_atom
  1372. < SMBD_CM_RESPONDER_RESOURCES ?
  1373. info->id->device->attrs.max_qp_rd_atom :
  1374. SMBD_CM_RESPONDER_RESOURCES;
  1375. info->responder_resources = conn_param.responder_resources;
  1376. log_rdma_mr(INFO, "responder_resources=%d\n",
  1377. info->responder_resources);
  1378. /* Need to send IRD/ORD in private data for iWARP */
  1379. info->id->device->ops.get_port_immutable(
  1380. info->id->device, info->id->port_num, &port_immutable);
  1381. if (port_immutable.core_cap_flags & RDMA_CORE_PORT_IWARP) {
  1382. ird_ord_hdr[0] = info->responder_resources;
  1383. ird_ord_hdr[1] = 1;
  1384. conn_param.private_data = ird_ord_hdr;
  1385. conn_param.private_data_len = sizeof(ird_ord_hdr);
  1386. } else {
  1387. conn_param.private_data = NULL;
  1388. conn_param.private_data_len = 0;
  1389. }
  1390. conn_param.retry_count = SMBD_CM_RETRY;
  1391. conn_param.rnr_retry_count = SMBD_CM_RNR_RETRY;
  1392. conn_param.flow_control = 0;
  1393. log_rdma_event(INFO, "connecting to IP %pI4 port %d\n",
  1394. &addr_in->sin_addr, port);
  1395. init_waitqueue_head(&info->conn_wait);
  1396. init_waitqueue_head(&info->disconn_wait);
  1397. init_waitqueue_head(&info->wait_reassembly_queue);
  1398. rc = rdma_connect(info->id, &conn_param);
  1399. if (rc) {
  1400. log_rdma_event(ERR, "rdma_connect() failed with %i\n", rc);
  1401. goto rdma_connect_failed;
  1402. }
  1403. wait_event_interruptible(
  1404. info->conn_wait, info->transport_status != SMBD_CONNECTING);
  1405. if (info->transport_status != SMBD_CONNECTED) {
  1406. log_rdma_event(ERR, "rdma_connect failed port=%d\n", port);
  1407. goto rdma_connect_failed;
  1408. }
  1409. log_rdma_event(INFO, "rdma_connect connected\n");
  1410. rc = allocate_caches_and_workqueue(info);
  1411. if (rc) {
  1412. log_rdma_event(ERR, "cache allocation failed\n");
  1413. goto allocate_cache_failed;
  1414. }
  1415. init_waitqueue_head(&info->wait_send_queue);
  1416. INIT_DELAYED_WORK(&info->idle_timer_work, idle_connection_timer);
  1417. queue_delayed_work(info->workqueue, &info->idle_timer_work,
  1418. info->keep_alive_interval*HZ);
  1419. init_waitqueue_head(&info->wait_send_pending);
  1420. atomic_set(&info->send_pending, 0);
  1421. init_waitqueue_head(&info->wait_post_send);
  1422. INIT_WORK(&info->disconnect_work, smbd_disconnect_rdma_work);
  1423. INIT_WORK(&info->post_send_credits_work, smbd_post_send_credits);
  1424. info->new_credits_offered = 0;
  1425. spin_lock_init(&info->lock_new_credits_offered);
  1426. rc = smbd_negotiate(info);
  1427. if (rc) {
  1428. log_rdma_event(ERR, "smbd_negotiate rc=%d\n", rc);
  1429. goto negotiation_failed;
  1430. }
  1431. rc = allocate_mr_list(info);
  1432. if (rc) {
  1433. log_rdma_mr(ERR, "memory registration allocation failed\n");
  1434. goto allocate_mr_failed;
  1435. }
  1436. return info;
  1437. allocate_mr_failed:
  1438. /* At this point, need to a full transport shutdown */
  1439. smbd_destroy(server);
  1440. return NULL;
  1441. negotiation_failed:
  1442. cancel_delayed_work_sync(&info->idle_timer_work);
  1443. destroy_caches_and_workqueue(info);
  1444. info->transport_status = SMBD_NEGOTIATE_FAILED;
  1445. init_waitqueue_head(&info->conn_wait);
  1446. rdma_disconnect(info->id);
  1447. wait_event(info->conn_wait,
  1448. info->tran