PageRenderTime 31ms CodeModel.GetById 17ms RepoModel.GetById 1ms app.codeStats 1ms

/fs/ksmbd/transport_rdma.c

https://gitlab.com/deepcypher/linux
C | 1813 lines | 1448 code | 270 blank | 95 comment | 169 complexity | b6d6968cb2e333ef32d9341dfc0eae41 MD5 | raw file
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright (C) 2017, Microsoft Corporation.
  4. * Copyright (C) 2018, LG Electronics.
  5. *
  6. * Author(s): Long Li <longli@microsoft.com>,
  7. * Hyunchul Lee <hyc.lee@gmail.com>
  8. *
  9. * This program is free software; you can redistribute it and/or modify
  10. * it under the terms of the GNU General Public License as published by
  11. * the Free Software Foundation; either version 2 of the License, or
  12. * (at your option) any later version.
  13. *
  14. * This program is distributed in the hope that it will be useful,
  15. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See
  17. * the GNU General Public License for more details.
  18. */
  19. #define SUBMOD_NAME "smb_direct"
  20. #include <linux/kthread.h>
  21. #include <linux/list.h>
  22. #include <linux/mempool.h>
  23. #include <linux/highmem.h>
  24. #include <linux/scatterlist.h>
  25. #include <rdma/ib_verbs.h>
  26. #include <rdma/rdma_cm.h>
  27. #include <rdma/rw.h>
  28. #include "glob.h"
  29. #include "connection.h"
  30. #include "smb_common.h"
  31. #include "smbstatus.h"
  32. #include "transport_rdma.h"
  33. #define SMB_DIRECT_PORT_IWARP 5445
  34. #define SMB_DIRECT_PORT_INFINIBAND 445
  35. #define SMB_DIRECT_VERSION_LE cpu_to_le16(0x0100)
  36. /* SMB_DIRECT negotiation timeout in seconds */
  37. #define SMB_DIRECT_NEGOTIATE_TIMEOUT 120
  38. #define SMB_DIRECT_MAX_SEND_SGES 8
  39. #define SMB_DIRECT_MAX_RECV_SGES 1
  40. /*
  41. * Default maximum number of RDMA read/write outstanding on this connection
  42. * This value is possibly decreased during QP creation on hardware limit
  43. */
  44. #define SMB_DIRECT_CM_INITIATOR_DEPTH 8
  45. /* Maximum number of retries on data transfer operations */
  46. #define SMB_DIRECT_CM_RETRY 6
  47. /* No need to retry on Receiver Not Ready since SMB_DIRECT manages credits */
  48. #define SMB_DIRECT_CM_RNR_RETRY 0
  49. /*
  50. * User configurable initial values per SMB_DIRECT transport connection
  51. * as defined in [MS-SMBD] 3.1.1.1
  52. * Those may change after a SMB_DIRECT negotiation
  53. */
  54. /* Set 445 port to SMB Direct port by default */
  55. static int smb_direct_port = SMB_DIRECT_PORT_INFINIBAND;
  56. /* The local peer's maximum number of credits to grant to the peer */
  57. static int smb_direct_receive_credit_max = 255;
  58. /* The remote peer's credit request of local peer */
  59. static int smb_direct_send_credit_target = 255;
  60. /* The maximum single message size can be sent to remote peer */
  61. static int smb_direct_max_send_size = 8192;
  62. /* The maximum fragmented upper-layer payload receive size supported */
  63. static int smb_direct_max_fragmented_recv_size = 1024 * 1024;
  64. /* The maximum single-message size which can be received */
  65. static int smb_direct_max_receive_size = 8192;
  66. static int smb_direct_max_read_write_size = SMBD_DEFAULT_IOSIZE;
  67. static LIST_HEAD(smb_direct_device_list);
  68. static DEFINE_RWLOCK(smb_direct_device_lock);
  69. struct smb_direct_device {
  70. struct ib_device *ib_dev;
  71. struct list_head list;
  72. };
  73. static struct smb_direct_listener {
  74. struct rdma_cm_id *cm_id;
  75. } smb_direct_listener;
  76. static struct workqueue_struct *smb_direct_wq;
  77. enum smb_direct_status {
  78. SMB_DIRECT_CS_NEW = 0,
  79. SMB_DIRECT_CS_CONNECTED,
  80. SMB_DIRECT_CS_DISCONNECTING,
  81. SMB_DIRECT_CS_DISCONNECTED,
  82. };
  83. struct smb_direct_transport {
  84. struct ksmbd_transport transport;
  85. enum smb_direct_status status;
  86. bool full_packet_received;
  87. wait_queue_head_t wait_status;
  88. struct rdma_cm_id *cm_id;
  89. struct ib_cq *send_cq;
  90. struct ib_cq *recv_cq;
  91. struct ib_pd *pd;
  92. struct ib_qp *qp;
  93. int max_send_size;
  94. int max_recv_size;
  95. int max_fragmented_send_size;
  96. int max_fragmented_recv_size;
  97. int max_rdma_rw_size;
  98. spinlock_t reassembly_queue_lock;
  99. struct list_head reassembly_queue;
  100. int reassembly_data_length;
  101. int reassembly_queue_length;
  102. int first_entry_offset;
  103. wait_queue_head_t wait_reassembly_queue;
  104. spinlock_t receive_credit_lock;
  105. int recv_credits;
  106. int count_avail_recvmsg;
  107. int recv_credit_max;
  108. int recv_credit_target;
  109. spinlock_t recvmsg_queue_lock;
  110. struct list_head recvmsg_queue;
  111. spinlock_t empty_recvmsg_queue_lock;
  112. struct list_head empty_recvmsg_queue;
  113. int send_credit_target;
  114. atomic_t send_credits;
  115. spinlock_t lock_new_recv_credits;
  116. int new_recv_credits;
  117. int max_rw_credits;
  118. int pages_per_rw_credit;
  119. atomic_t rw_credits;
  120. wait_queue_head_t wait_send_credits;
  121. wait_queue_head_t wait_rw_credits;
  122. mempool_t *sendmsg_mempool;
  123. struct kmem_cache *sendmsg_cache;
  124. mempool_t *recvmsg_mempool;
  125. struct kmem_cache *recvmsg_cache;
  126. wait_queue_head_t wait_send_pending;
  127. atomic_t send_pending;
  128. struct delayed_work post_recv_credits_work;
  129. struct work_struct send_immediate_work;
  130. struct work_struct disconnect_work;
  131. bool negotiation_requested;
  132. };
  133. #define KSMBD_TRANS(t) ((struct ksmbd_transport *)&((t)->transport))
  134. enum {
  135. SMB_DIRECT_MSG_NEGOTIATE_REQ = 0,
  136. SMB_DIRECT_MSG_DATA_TRANSFER
  137. };
  138. static struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops;
  139. struct smb_direct_send_ctx {
  140. struct list_head msg_list;
  141. int wr_cnt;
  142. bool need_invalidate_rkey;
  143. unsigned int remote_key;
  144. };
  145. struct smb_direct_sendmsg {
  146. struct smb_direct_transport *transport;
  147. struct ib_send_wr wr;
  148. struct list_head list;
  149. int num_sge;
  150. struct ib_sge sge[SMB_DIRECT_MAX_SEND_SGES];
  151. struct ib_cqe cqe;
  152. u8 packet[];
  153. };
  154. struct smb_direct_recvmsg {
  155. struct smb_direct_transport *transport;
  156. struct list_head list;
  157. int type;
  158. struct ib_sge sge;
  159. struct ib_cqe cqe;
  160. bool first_segment;
  161. u8 packet[];
  162. };
  163. struct smb_direct_rdma_rw_msg {
  164. struct smb_direct_transport *t;
  165. struct ib_cqe cqe;
  166. int status;
  167. struct completion *completion;
  168. struct list_head list;
  169. struct rdma_rw_ctx rw_ctx;
  170. struct sg_table sgt;
  171. struct scatterlist sg_list[];
  172. };
  173. void init_smbd_max_io_size(unsigned int sz)
  174. {
  175. sz = clamp_val(sz, SMBD_MIN_IOSIZE, SMBD_MAX_IOSIZE);
  176. smb_direct_max_read_write_size = sz;
  177. }
  178. unsigned int get_smbd_max_read_write_size(void)
  179. {
  180. return smb_direct_max_read_write_size;
  181. }
  182. static inline int get_buf_page_count(void *buf, int size)
  183. {
  184. return DIV_ROUND_UP((uintptr_t)buf + size, PAGE_SIZE) -
  185. (uintptr_t)buf / PAGE_SIZE;
  186. }
  187. static void smb_direct_destroy_pools(struct smb_direct_transport *transport);
  188. static void smb_direct_post_recv_credits(struct work_struct *work);
  189. static int smb_direct_post_send_data(struct smb_direct_transport *t,
  190. struct smb_direct_send_ctx *send_ctx,
  191. struct kvec *iov, int niov,
  192. int remaining_data_length);
  193. static inline struct smb_direct_transport *
  194. smb_trans_direct_transfort(struct ksmbd_transport *t)
  195. {
  196. return container_of(t, struct smb_direct_transport, transport);
  197. }
  198. static inline void
  199. *smb_direct_recvmsg_payload(struct smb_direct_recvmsg *recvmsg)
  200. {
  201. return (void *)recvmsg->packet;
  202. }
  203. static inline bool is_receive_credit_post_required(int receive_credits,
  204. int avail_recvmsg_count)
  205. {
  206. return receive_credits <= (smb_direct_receive_credit_max >> 3) &&
  207. avail_recvmsg_count >= (receive_credits >> 2);
  208. }
  209. static struct
  210. smb_direct_recvmsg *get_free_recvmsg(struct smb_direct_transport *t)
  211. {
  212. struct smb_direct_recvmsg *recvmsg = NULL;
  213. spin_lock(&t->recvmsg_queue_lock);
  214. if (!list_empty(&t->recvmsg_queue)) {
  215. recvmsg = list_first_entry(&t->recvmsg_queue,
  216. struct smb_direct_recvmsg,
  217. list);
  218. list_del(&recvmsg->list);
  219. }
  220. spin_unlock(&t->recvmsg_queue_lock);
  221. return recvmsg;
  222. }
  223. static void put_recvmsg(struct smb_direct_transport *t,
  224. struct smb_direct_recvmsg *recvmsg)
  225. {
  226. ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr,
  227. recvmsg->sge.length, DMA_FROM_DEVICE);
  228. spin_lock(&t->recvmsg_queue_lock);
  229. list_add(&recvmsg->list, &t->recvmsg_queue);
  230. spin_unlock(&t->recvmsg_queue_lock);
  231. }
  232. static struct
  233. smb_direct_recvmsg *get_empty_recvmsg(struct smb_direct_transport *t)
  234. {
  235. struct smb_direct_recvmsg *recvmsg = NULL;
  236. spin_lock(&t->empty_recvmsg_queue_lock);
  237. if (!list_empty(&t->empty_recvmsg_queue)) {
  238. recvmsg = list_first_entry(&t->empty_recvmsg_queue,
  239. struct smb_direct_recvmsg, list);
  240. list_del(&recvmsg->list);
  241. }
  242. spin_unlock(&t->empty_recvmsg_queue_lock);
  243. return recvmsg;
  244. }
  245. static void put_empty_recvmsg(struct smb_direct_transport *t,
  246. struct smb_direct_recvmsg *recvmsg)
  247. {
  248. ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr,
  249. recvmsg->sge.length, DMA_FROM_DEVICE);
  250. spin_lock(&t->empty_recvmsg_queue_lock);
  251. list_add_tail(&recvmsg->list, &t->empty_recvmsg_queue);
  252. spin_unlock(&t->empty_recvmsg_queue_lock);
  253. }
  254. static void enqueue_reassembly(struct smb_direct_transport *t,
  255. struct smb_direct_recvmsg *recvmsg,
  256. int data_length)
  257. {
  258. spin_lock(&t->reassembly_queue_lock);
  259. list_add_tail(&recvmsg->list, &t->reassembly_queue);
  260. t->reassembly_queue_length++;
  261. /*
  262. * Make sure reassembly_data_length is updated after list and
  263. * reassembly_queue_length are updated. On the dequeue side
  264. * reassembly_data_length is checked without a lock to determine
  265. * if reassembly_queue_length and list is up to date
  266. */
  267. virt_wmb();
  268. t->reassembly_data_length += data_length;
  269. spin_unlock(&t->reassembly_queue_lock);
  270. }
  271. static struct smb_direct_recvmsg *get_first_reassembly(struct smb_direct_transport *t)
  272. {
  273. if (!list_empty(&t->reassembly_queue))
  274. return list_first_entry(&t->reassembly_queue,
  275. struct smb_direct_recvmsg, list);
  276. else
  277. return NULL;
  278. }
  279. static void smb_direct_disconnect_rdma_work(struct work_struct *work)
  280. {
  281. struct smb_direct_transport *t =
  282. container_of(work, struct smb_direct_transport,
  283. disconnect_work);
  284. if (t->status == SMB_DIRECT_CS_CONNECTED) {
  285. t->status = SMB_DIRECT_CS_DISCONNECTING;
  286. rdma_disconnect(t->cm_id);
  287. }
  288. }
  289. static void
  290. smb_direct_disconnect_rdma_connection(struct smb_direct_transport *t)
  291. {
  292. if (t->status == SMB_DIRECT_CS_CONNECTED)
  293. queue_work(smb_direct_wq, &t->disconnect_work);
  294. }
  295. static void smb_direct_send_immediate_work(struct work_struct *work)
  296. {
  297. struct smb_direct_transport *t = container_of(work,
  298. struct smb_direct_transport, send_immediate_work);
  299. if (t->status != SMB_DIRECT_CS_CONNECTED)
  300. return;
  301. smb_direct_post_send_data(t, NULL, NULL, 0, 0);
  302. }
  303. static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id)
  304. {
  305. struct smb_direct_transport *t;
  306. struct ksmbd_conn *conn;
  307. t = kzalloc(sizeof(*t), GFP_KERNEL);
  308. if (!t)
  309. return NULL;
  310. t->cm_id = cm_id;
  311. cm_id->context = t;
  312. t->status = SMB_DIRECT_CS_NEW;
  313. init_waitqueue_head(&t->wait_status);
  314. spin_lock_init(&t->reassembly_queue_lock);
  315. INIT_LIST_HEAD(&t->reassembly_queue);
  316. t->reassembly_data_length = 0;
  317. t->reassembly_queue_length = 0;
  318. init_waitqueue_head(&t->wait_reassembly_queue);
  319. init_waitqueue_head(&t->wait_send_credits);
  320. init_waitqueue_head(&t->wait_rw_credits);
  321. spin_lock_init(&t->receive_credit_lock);
  322. spin_lock_init(&t->recvmsg_queue_lock);
  323. INIT_LIST_HEAD(&t->recvmsg_queue);
  324. spin_lock_init(&t->empty_recvmsg_queue_lock);
  325. INIT_LIST_HEAD(&t->empty_recvmsg_queue);
  326. init_waitqueue_head(&t->wait_send_pending);
  327. atomic_set(&t->send_pending, 0);
  328. spin_lock_init(&t->lock_new_recv_credits);
  329. INIT_DELAYED_WORK(&t->post_recv_credits_work,
  330. smb_direct_post_recv_credits);
  331. INIT_WORK(&t->send_immediate_work, smb_direct_send_immediate_work);
  332. INIT_WORK(&t->disconnect_work, smb_direct_disconnect_rdma_work);
  333. conn = ksmbd_conn_alloc();
  334. if (!conn)
  335. goto err;
  336. conn->transport = KSMBD_TRANS(t);
  337. KSMBD_TRANS(t)->conn = conn;
  338. KSMBD_TRANS(t)->ops = &ksmbd_smb_direct_transport_ops;
  339. return t;
  340. err:
  341. kfree(t);
  342. return NULL;
  343. }
  344. static void free_transport(struct smb_direct_transport *t)
  345. {
  346. struct smb_direct_recvmsg *recvmsg;
  347. wake_up_interruptible(&t->wait_send_credits);
  348. ksmbd_debug(RDMA, "wait for all send posted to IB to finish\n");
  349. wait_event(t->wait_send_pending,
  350. atomic_read(&t->send_pending) == 0);
  351. cancel_work_sync(&t->disconnect_work);
  352. cancel_delayed_work_sync(&t->post_recv_credits_work);
  353. cancel_work_sync(&t->send_immediate_work);
  354. if (t->qp) {
  355. ib_drain_qp(t->qp);
  356. ib_mr_pool_destroy(t->qp, &t->qp->rdma_mrs);
  357. ib_destroy_qp(t->qp);
  358. }
  359. ksmbd_debug(RDMA, "drain the reassembly queue\n");
  360. do {
  361. spin_lock(&t->reassembly_queue_lock);
  362. recvmsg = get_first_reassembly(t);
  363. if (recvmsg) {
  364. list_del(&recvmsg->list);
  365. spin_unlock(&t->reassembly_queue_lock);
  366. put_recvmsg(t, recvmsg);
  367. } else {
  368. spin_unlock(&t->reassembly_queue_lock);
  369. }
  370. } while (recvmsg);
  371. t->reassembly_data_length = 0;
  372. if (t->send_cq)
  373. ib_free_cq(t->send_cq);
  374. if (t->recv_cq)
  375. ib_free_cq(t->recv_cq);
  376. if (t->pd)
  377. ib_dealloc_pd(t->pd);
  378. if (t->cm_id)
  379. rdma_destroy_id(t->cm_id);
  380. smb_direct_destroy_pools(t);
  381. ksmbd_conn_free(KSMBD_TRANS(t)->conn);
  382. kfree(t);
  383. }
  384. static struct smb_direct_sendmsg
  385. *smb_direct_alloc_sendmsg(struct smb_direct_transport *t)
  386. {
  387. struct smb_direct_sendmsg *msg;
  388. msg = mempool_alloc(t->sendmsg_mempool, GFP_KERNEL);
  389. if (!msg)
  390. return ERR_PTR(-ENOMEM);
  391. msg->transport = t;
  392. INIT_LIST_HEAD(&msg->list);
  393. msg->num_sge = 0;
  394. return msg;
  395. }
  396. static void smb_direct_free_sendmsg(struct smb_direct_transport *t,
  397. struct smb_direct_sendmsg *msg)
  398. {
  399. int i;
  400. if (msg->num_sge > 0) {
  401. ib_dma_unmap_single(t->cm_id->device,
  402. msg->sge[0].addr, msg->sge[0].length,
  403. DMA_TO_DEVICE);
  404. for (i = 1; i < msg->num_sge; i++)
  405. ib_dma_unmap_page(t->cm_id->device,
  406. msg->sge[i].addr, msg->sge[i].length,
  407. DMA_TO_DEVICE);
  408. }
  409. mempool_free(msg, t->sendmsg_mempool);
  410. }
  411. static int smb_direct_check_recvmsg(struct smb_direct_recvmsg *recvmsg)
  412. {
  413. switch (recvmsg->type) {
  414. case SMB_DIRECT_MSG_DATA_TRANSFER: {
  415. struct smb_direct_data_transfer *req =
  416. (struct smb_direct_data_transfer *)recvmsg->packet;
  417. struct smb2_hdr *hdr = (struct smb2_hdr *)(recvmsg->packet
  418. + le32_to_cpu(req->data_offset));
  419. ksmbd_debug(RDMA,
  420. "CreditGranted: %u, CreditRequested: %u, DataLength: %u, RemainingDataLength: %u, SMB: %x, Command: %u\n",
  421. le16_to_cpu(req->credits_granted),
  422. le16_to_cpu(req->credits_requested),
  423. req->data_length, req->remaining_data_length,
  424. hdr->ProtocolId, hdr->Command);
  425. break;
  426. }
  427. case SMB_DIRECT_MSG_NEGOTIATE_REQ: {
  428. struct smb_direct_negotiate_req *req =
  429. (struct smb_direct_negotiate_req *)recvmsg->packet;
  430. ksmbd_debug(RDMA,
  431. "MinVersion: %u, MaxVersion: %u, CreditRequested: %u, MaxSendSize: %u, MaxRecvSize: %u, MaxFragmentedSize: %u\n",
  432. le16_to_cpu(req->min_version),
  433. le16_to_cpu(req->max_version),
  434. le16_to_cpu(req->credits_requested),
  435. le32_to_cpu(req->preferred_send_size),
  436. le32_to_cpu(req->max_receive_size),
  437. le32_to_cpu(req->max_fragmented_size));
  438. if (le16_to_cpu(req->min_version) > 0x0100 ||
  439. le16_to_cpu(req->max_version) < 0x0100)
  440. return -EOPNOTSUPP;
  441. if (le16_to_cpu(req->credits_requested) <= 0 ||
  442. le32_to_cpu(req->max_receive_size) <= 128 ||
  443. le32_to_cpu(req->max_fragmented_size) <=
  444. 128 * 1024)
  445. return -ECONNABORTED;
  446. break;
  447. }
  448. default:
  449. return -EINVAL;
  450. }
  451. return 0;
  452. }
  453. static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
  454. {
  455. struct smb_direct_recvmsg *recvmsg;
  456. struct smb_direct_transport *t;
  457. recvmsg = container_of(wc->wr_cqe, struct smb_direct_recvmsg, cqe);
  458. t = recvmsg->transport;
  459. if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) {
  460. if (wc->status != IB_WC_WR_FLUSH_ERR) {
  461. pr_err("Recv error. status='%s (%d)' opcode=%d\n",
  462. ib_wc_status_msg(wc->status), wc->status,
  463. wc->opcode);
  464. smb_direct_disconnect_rdma_connection(t);
  465. }
  466. put_empty_recvmsg(t, recvmsg);
  467. return;
  468. }
  469. ksmbd_debug(RDMA, "Recv completed. status='%s (%d)', opcode=%d\n",
  470. ib_wc_status_msg(wc->status), wc->status,
  471. wc->opcode);
  472. ib_dma_sync_single_for_cpu(wc->qp->device, recvmsg->sge.addr,
  473. recvmsg->sge.length, DMA_FROM_DEVICE);
  474. switch (recvmsg->type) {
  475. case SMB_DIRECT_MSG_NEGOTIATE_REQ:
  476. if (wc->byte_len < sizeof(struct smb_direct_negotiate_req)) {
  477. put_empty_recvmsg(t, recvmsg);
  478. return;
  479. }
  480. t->negotiation_requested = true;
  481. t->full_packet_received = true;
  482. t->status = SMB_DIRECT_CS_CONNECTED;
  483. enqueue_reassembly(t, recvmsg, 0);
  484. wake_up_interruptible(&t->wait_status);
  485. break;
  486. case SMB_DIRECT_MSG_DATA_TRANSFER: {
  487. struct smb_direct_data_transfer *data_transfer =
  488. (struct smb_direct_data_transfer *)recvmsg->packet;
  489. unsigned int data_length;
  490. int avail_recvmsg_count, receive_credits;
  491. if (wc->byte_len <
  492. offsetof(struct smb_direct_data_transfer, padding)) {
  493. put_empty_recvmsg(t, recvmsg);
  494. return;
  495. }
  496. data_length = le32_to_cpu(data_transfer->data_length);
  497. if (data_length) {
  498. if (wc->byte_len < sizeof(struct smb_direct_data_transfer) +
  499. (u64)data_length) {
  500. put_empty_recvmsg(t, recvmsg);
  501. return;
  502. }
  503. if (t->full_packet_received)
  504. recvmsg->first_segment = true;
  505. if (le32_to_cpu(data_transfer->remaining_data_length))
  506. t->full_packet_received = false;
  507. else
  508. t->full_packet_received = true;
  509. enqueue_reassembly(t, recvmsg, (int)data_length);
  510. wake_up_interruptible(&t->wait_reassembly_queue);
  511. spin_lock(&t->receive_credit_lock);
  512. receive_credits = --(t->recv_credits);
  513. avail_recvmsg_count = t->count_avail_recvmsg;
  514. spin_unlock(&t->receive_credit_lock);
  515. } else {
  516. put_empty_recvmsg(t, recvmsg);
  517. spin_lock(&t->receive_credit_lock);
  518. receive_credits = --(t->recv_credits);
  519. avail_recvmsg_count = ++(t->count_avail_recvmsg);
  520. spin_unlock(&t->receive_credit_lock);
  521. }
  522. t->recv_credit_target =
  523. le16_to_cpu(data_transfer->credits_requested);
  524. atomic_add(le16_to_cpu(data_transfer->credits_granted),
  525. &t->send_credits);
  526. if (le16_to_cpu(data_transfer->flags) &
  527. SMB_DIRECT_RESPONSE_REQUESTED)
  528. queue_work(smb_direct_wq, &t->send_immediate_work);
  529. if (atomic_read(&t->send_credits) > 0)
  530. wake_up_interruptible(&t->wait_send_credits);
  531. if (is_receive_credit_post_required(receive_credits, avail_recvmsg_count))
  532. mod_delayed_work(smb_direct_wq,
  533. &t->post_recv_credits_work, 0);
  534. break;
  535. }
  536. default:
  537. break;
  538. }
  539. }
  540. static int smb_direct_post_recv(struct smb_direct_transport *t,
  541. struct smb_direct_recvmsg *recvmsg)
  542. {
  543. struct ib_recv_wr wr;
  544. int ret;
  545. recvmsg->sge.addr = ib_dma_map_single(t->cm_id->device,
  546. recvmsg->packet, t->max_recv_size,
  547. DMA_FROM_DEVICE);
  548. ret = ib_dma_mapping_error(t->cm_id->device, recvmsg->sge.addr);
  549. if (ret)
  550. return ret;
  551. recvmsg->sge.length = t->max_recv_size;
  552. recvmsg->sge.lkey = t->pd->local_dma_lkey;
  553. recvmsg->cqe.done = recv_done;
  554. wr.wr_cqe = &recvmsg->cqe;
  555. wr.next = NULL;
  556. wr.sg_list = &recvmsg->sge;
  557. wr.num_sge = 1;
  558. ret = ib_post_recv(t->qp, &wr, NULL);
  559. if (ret) {
  560. pr_err("Can't post recv: %d\n", ret);
  561. ib_dma_unmap_single(t->cm_id->device,
  562. recvmsg->sge.addr, recvmsg->sge.length,
  563. DMA_FROM_DEVICE);
  564. smb_direct_disconnect_rdma_connection(t);
  565. return ret;
  566. }
  567. return ret;
  568. }
  569. static int smb_direct_read(struct ksmbd_transport *t, char *buf,
  570. unsigned int size)
  571. {
  572. struct smb_direct_recvmsg *recvmsg;
  573. struct smb_direct_data_transfer *data_transfer;
  574. int to_copy, to_read, data_read, offset;
  575. u32 data_length, remaining_data_length, data_offset;
  576. int rc;
  577. struct smb_direct_transport *st = smb_trans_direct_transfort(t);
  578. again:
  579. if (st->status != SMB_DIRECT_CS_CONNECTED) {
  580. pr_err("disconnected\n");
  581. return -ENOTCONN;
  582. }
  583. /*
  584. * No need to hold the reassembly queue lock all the time as we are
  585. * the only one reading from the front of the queue. The transport
  586. * may add more entries to the back of the queue at the same time
  587. */
  588. if (st->reassembly_data_length >= size) {
  589. int queue_length;
  590. int queue_removed = 0;
  591. /*
  592. * Need to make sure reassembly_data_length is read before
  593. * reading reassembly_queue_length and calling
  594. * get_first_reassembly. This call is lock free
  595. * as we never read at the end of the queue which are being
  596. * updated in SOFTIRQ as more data is received
  597. */
  598. virt_rmb();
  599. queue_length = st->reassembly_queue_length;
  600. data_read = 0;
  601. to_read = size;
  602. offset = st->first_entry_offset;
  603. while (data_read < size) {
  604. recvmsg = get_first_reassembly(st);
  605. data_transfer = smb_direct_recvmsg_payload(recvmsg);
  606. data_length = le32_to_cpu(data_transfer->data_length);
  607. remaining_data_length =
  608. le32_to_cpu(data_transfer->remaining_data_length);
  609. data_offset = le32_to_cpu(data_transfer->data_offset);
  610. /*
  611. * The upper layer expects RFC1002 length at the
  612. * beginning of the payload. Return it to indicate
  613. * the total length of the packet. This minimize the
  614. * change to upper layer packet processing logic. This
  615. * will be eventually remove when an intermediate
  616. * transport layer is added
  617. */
  618. if (recvmsg->first_segment && size == 4) {
  619. unsigned int rfc1002_len =
  620. data_length + remaining_data_length;
  621. *((__be32 *)buf) = cpu_to_be32(rfc1002_len);
  622. data_read = 4;
  623. recvmsg->first_segment = false;
  624. ksmbd_debug(RDMA,
  625. "returning rfc1002 length %d\n",
  626. rfc1002_len);
  627. goto read_rfc1002_done;
  628. }
  629. to_copy = min_t(int, data_length - offset, to_read);
  630. memcpy(buf + data_read, (char *)data_transfer + data_offset + offset,
  631. to_copy);
  632. /* move on to the next buffer? */
  633. if (to_copy == data_length - offset) {
  634. queue_length--;
  635. /*
  636. * No need to lock if we are not at the
  637. * end of the queue
  638. */
  639. if (queue_length) {
  640. list_del(&recvmsg->list);
  641. } else {
  642. spin_lock_irq(&st->reassembly_queue_lock);
  643. list_del(&recvmsg->list);
  644. spin_unlock_irq(&st->reassembly_queue_lock);
  645. }
  646. queue_removed++;
  647. put_recvmsg(st, recvmsg);
  648. offset = 0;
  649. } else {
  650. offset += to_copy;
  651. }
  652. to_read -= to_copy;
  653. data_read += to_copy;
  654. }
  655. spin_lock_irq(&st->reassembly_queue_lock);
  656. st->reassembly_data_length -= data_read;
  657. st->reassembly_queue_length -= queue_removed;
  658. spin_unlock_irq(&st->reassembly_queue_lock);
  659. spin_lock(&st->receive_credit_lock);
  660. st->count_avail_recvmsg += queue_removed;
  661. if (is_receive_credit_post_required(st->recv_credits, st->count_avail_recvmsg)) {
  662. spin_unlock(&st->receive_credit_lock);
  663. mod_delayed_work(smb_direct_wq,
  664. &st->post_recv_credits_work, 0);
  665. } else {
  666. spin_unlock(&st->receive_credit_lock);
  667. }
  668. st->first_entry_offset = offset;
  669. ksmbd_debug(RDMA,
  670. "returning to thread data_read=%d reassembly_data_length=%d first_entry_offset=%d\n",
  671. data_read, st->reassembly_data_length,
  672. st->first_entry_offset);
  673. read_rfc1002_done:
  674. return data_read;
  675. }
  676. ksmbd_debug(RDMA, "wait_event on more data\n");
  677. rc = wait_event_interruptible(st->wait_reassembly_queue,
  678. st->reassembly_data_length >= size ||
  679. st->status != SMB_DIRECT_CS_CONNECTED);
  680. if (rc)
  681. return -EINTR;
  682. goto again;
  683. }
  684. static void smb_direct_post_recv_credits(struct work_struct *work)
  685. {
  686. struct smb_direct_transport *t = container_of(work,
  687. struct smb_direct_transport, post_recv_credits_work.work);
  688. struct smb_direct_recvmsg *recvmsg;
  689. int receive_credits, credits = 0;
  690. int ret;
  691. int use_free = 1;
  692. spin_lock(&t->receive_credit_lock);
  693. receive_credits = t->recv_credits;
  694. spin_unlock(&t->receive_credit_lock);
  695. if (receive_credits < t->recv_credit_target) {
  696. while (true) {
  697. if (use_free)
  698. recvmsg = get_free_recvmsg(t);
  699. else
  700. recvmsg = get_empty_recvmsg(t);
  701. if (!recvmsg) {
  702. if (use_free) {
  703. use_free = 0;
  704. continue;
  705. } else {
  706. break;
  707. }
  708. }
  709. recvmsg->type = SMB_DIRECT_MSG_DATA_TRANSFER;
  710. recvmsg->first_segment = false;
  711. ret = smb_direct_post_recv(t, recvmsg);
  712. if (ret) {
  713. pr_err("Can't post recv: %d\n", ret);
  714. put_recvmsg(t, recvmsg);
  715. break;
  716. }
  717. credits++;
  718. }
  719. }
  720. spin_lock(&t->receive_credit_lock);
  721. t->recv_credits += credits;
  722. t->count_avail_recvmsg -= credits;
  723. spin_unlock(&t->receive_credit_lock);
  724. spin_lock(&t->lock_new_recv_credits);
  725. t->new_recv_credits += credits;
  726. spin_unlock(&t->lock_new_recv_credits);
  727. if (credits)
  728. queue_work(smb_direct_wq, &t->send_immediate_work);
  729. }
  730. static void send_done(struct ib_cq *cq, struct ib_wc *wc)
  731. {
  732. struct smb_direct_sendmsg *sendmsg, *sibling;
  733. struct smb_direct_transport *t;
  734. struct list_head *pos, *prev, *end;
  735. sendmsg = container_of(wc->wr_cqe, struct smb_direct_sendmsg, cqe);
  736. t = sendmsg->transport;
  737. ksmbd_debug(RDMA, "Send completed. status='%s (%d)', opcode=%d\n",
  738. ib_wc_status_msg(wc->status), wc->status,
  739. wc->opcode);
  740. if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) {
  741. pr_err("Send error. status='%s (%d)', opcode=%d\n",
  742. ib_wc_status_msg(wc->status), wc->status,
  743. wc->opcode);
  744. smb_direct_disconnect_rdma_connection(t);
  745. }
  746. if (atomic_dec_and_test(&t->send_pending))
  747. wake_up(&t->wait_send_pending);
  748. /* iterate and free the list of messages in reverse. the list's head
  749. * is invalid.
  750. */
  751. for (pos = &sendmsg->list, prev = pos->prev, end = sendmsg->list.next;
  752. prev != end; pos = prev, prev = prev->prev) {
  753. sibling = container_of(pos, struct smb_direct_sendmsg, list);
  754. smb_direct_free_sendmsg(t, sibling);
  755. }
  756. sibling = container_of(pos, struct smb_direct_sendmsg, list);
  757. smb_direct_free_sendmsg(t, sibling);
  758. }
  759. static int manage_credits_prior_sending(struct smb_direct_transport *t)
  760. {
  761. int new_credits;
  762. spin_lock(&t->lock_new_recv_credits);
  763. new_credits = t->new_recv_credits;
  764. t->new_recv_credits = 0;
  765. spin_unlock(&t->lock_new_recv_credits);
  766. return new_credits;
  767. }
  768. static int smb_direct_post_send(struct smb_direct_transport *t,
  769. struct ib_send_wr *wr)
  770. {
  771. int ret;
  772. atomic_inc(&t->send_pending);
  773. ret = ib_post_send(t->qp, wr, NULL);
  774. if (ret) {
  775. pr_err("failed to post send: %d\n", ret);
  776. if (atomic_dec_and_test(&t->send_pending))
  777. wake_up(&t->wait_send_pending);
  778. smb_direct_disconnect_rdma_connection(t);
  779. }
  780. return ret;
  781. }
  782. static void smb_direct_send_ctx_init(struct smb_direct_transport *t,
  783. struct smb_direct_send_ctx *send_ctx,
  784. bool need_invalidate_rkey,
  785. unsigned int remote_key)
  786. {
  787. INIT_LIST_HEAD(&send_ctx->msg_list);
  788. send_ctx->wr_cnt = 0;
  789. send_ctx->need_invalidate_rkey = need_invalidate_rkey;
  790. send_ctx->remote_key = remote_key;
  791. }
  792. static int smb_direct_flush_send_list(struct smb_direct_transport *t,
  793. struct smb_direct_send_ctx *send_ctx,
  794. bool is_last)
  795. {
  796. struct smb_direct_sendmsg *first, *last;
  797. int ret;
  798. if (list_empty(&send_ctx->msg_list))
  799. return 0;
  800. first = list_first_entry(&send_ctx->msg_list,
  801. struct smb_direct_sendmsg,
  802. list);
  803. last = list_last_entry(&send_ctx->msg_list,
  804. struct smb_direct_sendmsg,
  805. list);
  806. last->wr.send_flags = IB_SEND_SIGNALED;
  807. last->wr.wr_cqe = &last->cqe;
  808. if (is_last && send_ctx->need_invalidate_rkey) {
  809. last->wr.opcode = IB_WR_SEND_WITH_INV;
  810. last->wr.ex.invalidate_rkey = send_ctx->remote_key;
  811. }
  812. ret = smb_direct_post_send(t, &first->wr);
  813. if (!ret) {
  814. smb_direct_send_ctx_init(t, send_ctx,
  815. send_ctx->need_invalidate_rkey,
  816. send_ctx->remote_key);
  817. } else {
  818. atomic_add(send_ctx->wr_cnt, &t->send_credits);
  819. wake_up(&t->wait_send_credits);
  820. list_for_each_entry_safe(first, last, &send_ctx->msg_list,
  821. list) {
  822. smb_direct_free_sendmsg(t, first);
  823. }
  824. }
  825. return ret;
  826. }
  827. static int wait_for_credits(struct smb_direct_transport *t,
  828. wait_queue_head_t *waitq, atomic_t *total_credits,
  829. int needed)
  830. {
  831. int ret;
  832. do {
  833. if (atomic_sub_return(needed, total_credits) >= 0)
  834. return 0;
  835. atomic_add(needed, total_credits);
  836. ret = wait_event_interruptible(*waitq,
  837. atomic_read(total_credits) >= needed ||
  838. t->status != SMB_DIRECT_CS_CONNECTED);
  839. if (t->status != SMB_DIRECT_CS_CONNECTED)
  840. return -ENOTCONN;
  841. else if (ret < 0)
  842. return ret;
  843. } while (true);
  844. }
  845. static int wait_for_send_credits(struct smb_direct_transport *t,
  846. struct smb_direct_send_ctx *send_ctx)
  847. {
  848. int ret;
  849. if (send_ctx &&
  850. (send_ctx->wr_cnt >= 16 || atomic_read(&t->send_credits) <= 1)) {
  851. ret = smb_direct_flush_send_list(t, send_ctx, false);
  852. if (ret)
  853. return ret;
  854. }
  855. return wait_for_credits(t, &t->wait_send_credits, &t->send_credits, 1);
  856. }
  857. static int wait_for_rw_credits(struct smb_direct_transport *t, int credits)
  858. {
  859. return wait_for_credits(t, &t->wait_rw_credits, &t->rw_credits, credits);
  860. }
  861. static int calc_rw_credits(struct smb_direct_transport *t,
  862. char *buf, unsigned int len)
  863. {
  864. return DIV_ROUND_UP(get_buf_page_count(buf, len),
  865. t->pages_per_rw_credit);
  866. }
  867. static int smb_direct_create_header(struct smb_direct_transport *t,
  868. int size, int remaining_data_length,
  869. struct smb_direct_sendmsg **sendmsg_out)
  870. {
  871. struct smb_direct_sendmsg *sendmsg;
  872. struct smb_direct_data_transfer *packet;
  873. int header_length;
  874. int ret;
  875. sendmsg = smb_direct_alloc_sendmsg(t);
  876. if (IS_ERR(sendmsg))
  877. return PTR_ERR(sendmsg);
  878. /* Fill in the packet header */
  879. packet = (struct smb_direct_data_transfer *)sendmsg->packet;
  880. packet->credits_requested = cpu_to_le16(t->send_credit_target);
  881. packet->credits_granted = cpu_to_le16(manage_credits_prior_sending(t));
  882. packet->flags = 0;
  883. packet->reserved = 0;
  884. if (!size)
  885. packet->data_offset = 0;
  886. else
  887. packet->data_offset = cpu_to_le32(24);
  888. packet->data_length = cpu_to_le32(size);
  889. packet->remaining_data_length = cpu_to_le32(remaining_data_length);
  890. packet->padding = 0;
  891. ksmbd_debug(RDMA,
  892. "credits_requested=%d credits_granted=%d data_offset=%d data_length=%d remaining_data_length=%d\n",
  893. le16_to_cpu(packet->credits_requested),
  894. le16_to_cpu(packet->credits_granted),
  895. le32_to_cpu(packet->data_offset),
  896. le32_to_cpu(packet->data_length),
  897. le32_to_cpu(packet->remaining_data_length));
  898. /* Map the packet to DMA */
  899. header_length = sizeof(struct smb_direct_data_transfer);
  900. /* If this is a packet without payload, don't send padding */
  901. if (!size)
  902. header_length =
  903. offsetof(struct smb_direct_data_transfer, padding);
  904. sendmsg->sge[0].addr = ib_dma_map_single(t->cm_id->device,
  905. (void *)packet,
  906. header_length,
  907. DMA_TO_DEVICE);
  908. ret = ib_dma_mapping_error(t->cm_id->device, sendmsg->sge[0].addr);
  909. if (ret) {
  910. smb_direct_free_sendmsg(t, sendmsg);
  911. return ret;
  912. }
  913. sendmsg->num_sge = 1;
  914. sendmsg->sge[0].length = header_length;
  915. sendmsg->sge[0].lkey = t->pd->local_dma_lkey;
  916. *sendmsg_out = sendmsg;
  917. return 0;
  918. }
  919. static int get_sg_list(void *buf, int size, struct scatterlist *sg_list, int nentries)
  920. {
  921. bool high = is_vmalloc_addr(buf);
  922. struct page *page;
  923. int offset, len;
  924. int i = 0;
  925. if (size <= 0 || nentries < get_buf_page_count(buf, size))
  926. return -EINVAL;
  927. offset = offset_in_page(buf);
  928. buf -= offset;
  929. while (size > 0) {
  930. len = min_t(int, PAGE_SIZE - offset, size);
  931. if (high)
  932. page = vmalloc_to_page(buf);
  933. else
  934. page = kmap_to_page(buf);
  935. if (!sg_list)
  936. return -EINVAL;
  937. sg_set_page(sg_list, page, len, offset);
  938. sg_list = sg_next(sg_list);
  939. buf += PAGE_SIZE;
  940. size -= len;
  941. offset = 0;
  942. i++;
  943. }
  944. return i;
  945. }
  946. static int get_mapped_sg_list(struct ib_device *device, void *buf, int size,
  947. struct scatterlist *sg_list, int nentries,
  948. enum dma_data_direction dir)
  949. {
  950. int npages;
  951. npages = get_sg_list(buf, size, sg_list, nentries);
  952. if (npages < 0)
  953. return -EINVAL;
  954. return ib_dma_map_sg(device, sg_list, npages, dir);
  955. }
  956. static int post_sendmsg(struct smb_direct_transport *t,
  957. struct smb_direct_send_ctx *send_ctx,
  958. struct smb_direct_sendmsg *msg)
  959. {
  960. int i;
  961. for (i = 0; i < msg->num_sge; i++)
  962. ib_dma_sync_single_for_device(t->cm_id->device,
  963. msg->sge[i].addr, msg->sge[i].length,
  964. DMA_TO_DEVICE);
  965. msg->cqe.done = send_done;
  966. msg->wr.opcode = IB_WR_SEND;
  967. msg->wr.sg_list = &msg->sge[0];
  968. msg->wr.num_sge = msg->num_sge;
  969. msg->wr.next = NULL;
  970. if (send_ctx) {
  971. msg->wr.wr_cqe = NULL;
  972. msg->wr.send_flags = 0;
  973. if (!list_empty(&send_ctx->msg_list)) {
  974. struct smb_direct_sendmsg *last;
  975. last = list_last_entry(&send_ctx->msg_list,
  976. struct smb_direct_sendmsg,
  977. list);
  978. last->wr.next = &msg->wr;
  979. }
  980. list_add_tail(&msg->list, &send_ctx->msg_list);
  981. send_ctx->wr_cnt++;
  982. return 0;
  983. }
  984. msg->wr.wr_cqe = &msg->cqe;
  985. msg->wr.send_flags = IB_SEND_SIGNALED;
  986. return smb_direct_post_send(t, &msg->wr);
  987. }
  988. static int smb_direct_post_send_data(struct smb_direct_transport *t,
  989. struct smb_direct_send_ctx *send_ctx,
  990. struct kvec *iov, int niov,
  991. int remaining_data_length)
  992. {
  993. int i, j, ret;
  994. struct smb_direct_sendmsg *msg;
  995. int data_length;
  996. struct scatterlist sg[SMB_DIRECT_MAX_SEND_SGES - 1];
  997. ret = wait_for_send_credits(t, send_ctx);
  998. if (ret)
  999. return ret;
  1000. data_length = 0;
  1001. for (i = 0; i < niov; i++)
  1002. data_length += iov[i].iov_len;
  1003. ret = smb_direct_create_header(t, data_length, remaining_data_length,
  1004. &msg);
  1005. if (ret) {
  1006. atomic_inc(&t->send_credits);
  1007. return ret;
  1008. }
  1009. for (i = 0; i < niov; i++) {
  1010. struct ib_sge *sge;
  1011. int sg_cnt;
  1012. sg_init_table(sg, SMB_DIRECT_MAX_SEND_SGES - 1);
  1013. sg_cnt = get_mapped_sg_list(t->cm_id->device,
  1014. iov[i].iov_base, iov[i].iov_len,
  1015. sg, SMB_DIRECT_MAX_SEND_SGES - 1,
  1016. DMA_TO_DEVICE);
  1017. if (sg_cnt <= 0) {
  1018. pr_err("failed to map buffer\n");
  1019. ret = -ENOMEM;
  1020. goto err;
  1021. } else if (sg_cnt + msg->num_sge > SMB_DIRECT_MAX_SEND_SGES) {
  1022. pr_err("buffer not fitted into sges\n");
  1023. ret = -E2BIG;
  1024. ib_dma_unmap_sg(t->cm_id->device, sg, sg_cnt,
  1025. DMA_TO_DEVICE);
  1026. goto err;
  1027. }
  1028. for (j = 0; j < sg_cnt; j++) {
  1029. sge = &msg->sge[msg->num_sge];
  1030. sge->addr = sg_dma_address(&sg[j]);
  1031. sge->length = sg_dma_len(&sg[j]);
  1032. sge->lkey = t->pd->local_dma_lkey;
  1033. msg->num_sge++;
  1034. }
  1035. }
  1036. ret = post_sendmsg(t, send_ctx, msg);
  1037. if (ret)
  1038. goto err;
  1039. return 0;
  1040. err:
  1041. smb_direct_free_sendmsg(t, msg);
  1042. atomic_inc(&t->send_credits);
  1043. return ret;
  1044. }
  1045. static int smb_direct_writev(struct ksmbd_transport *t,
  1046. struct kvec *iov, int niovs, int buflen,
  1047. bool need_invalidate, unsigned int remote_key)
  1048. {
  1049. struct smb_direct_transport *st = smb_trans_direct_transfort(t);
  1050. int remaining_data_length;
  1051. int start, i, j;
  1052. int max_iov_size = st->max_send_size -
  1053. sizeof(struct smb_direct_data_transfer);
  1054. int ret;
  1055. struct kvec vec;
  1056. struct smb_direct_send_ctx send_ctx;
  1057. if (st->status != SMB_DIRECT_CS_CONNECTED)
  1058. return -ENOTCONN;
  1059. //FIXME: skip RFC1002 header..
  1060. buflen -= 4;
  1061. iov[0].iov_base += 4;
  1062. iov[0].iov_len -= 4;
  1063. remaining_data_length = buflen;
  1064. ksmbd_debug(RDMA, "Sending smb (RDMA): smb_len=%u\n", buflen);
  1065. smb_direct_send_ctx_init(st, &send_ctx, need_invalidate, remote_key);
  1066. start = i = 0;
  1067. buflen = 0;
  1068. while (true) {
  1069. buflen += iov[i].iov_len;
  1070. if (buflen > max_iov_size) {
  1071. if (i > start) {
  1072. remaining_data_length -=
  1073. (buflen - iov[i].iov_len);
  1074. ret = smb_direct_post_send_data(st, &send_ctx,
  1075. &iov[start], i - start,
  1076. remaining_data_length);
  1077. if (ret)
  1078. goto done;
  1079. } else {
  1080. /* iov[start] is too big, break it */
  1081. int nvec = (buflen + max_iov_size - 1) /
  1082. max_iov_size;
  1083. for (j = 0; j < nvec; j++) {
  1084. vec.iov_base =
  1085. (char *)iov[start].iov_base +
  1086. j * max_iov_size;
  1087. vec.iov_len =
  1088. min_t(int, max_iov_size,
  1089. buflen - max_iov_size * j);
  1090. remaining_data_length -= vec.iov_len;
  1091. ret = smb_direct_post_send_data(st, &send_ctx, &vec, 1,
  1092. remaining_data_length);
  1093. if (ret)
  1094. goto done;
  1095. }
  1096. i++;
  1097. if (i == niovs)
  1098. break;
  1099. }
  1100. start = i;
  1101. buflen = 0;
  1102. } else {
  1103. i++;
  1104. if (i == niovs) {
  1105. /* send out all remaining vecs */
  1106. remaining_data_length -= buflen;
  1107. ret = smb_direct_post_send_data(st, &send_ctx,
  1108. &iov[start], i - start,
  1109. remaining_data_length);
  1110. if (ret)
  1111. goto done;
  1112. break;
  1113. }
  1114. }
  1115. }
  1116. done:
  1117. ret = smb_direct_flush_send_list(st, &send_ctx, true);
  1118. /*
  1119. * As an optimization, we don't wait for individual I/O to finish
  1120. * before sending the next one.
  1121. * Send them all and wait for pending send count to get to 0
  1122. * that means all the I/Os have been out and we are good to return
  1123. */
  1124. wait_event(st->wait_send_pending,
  1125. atomic_read(&st->send_pending) == 0);
  1126. return ret;
  1127. }
  1128. static void smb_direct_free_rdma_rw_msg(struct smb_direct_transport *t,
  1129. struct smb_direct_rdma_rw_msg *msg,
  1130. enum dma_data_direction dir)
  1131. {
  1132. rdma_rw_ctx_destroy(&msg->rw_ctx, t->qp, t->qp->port,
  1133. msg->sgt.sgl, msg->sgt.nents, dir);
  1134. sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);
  1135. kfree(msg);
  1136. }
  1137. static void read_write_done(struct ib_cq *cq, struct ib_wc *wc,
  1138. enum dma_data_direction dir)
  1139. {
  1140. struct smb_direct_rdma_rw_msg *msg = container_of(wc->wr_cqe,
  1141. struct smb_direct_rdma_rw_msg, cqe);
  1142. struct smb_direct_transport *t = msg->t;
  1143. if (wc->status != IB_WC_SUCCESS) {
  1144. msg->status = -EIO;
  1145. pr_err("read/write error. opcode = %d, status = %s(%d)\n",
  1146. wc->opcode, ib_wc_status_msg(wc->status), wc->status);
  1147. if (wc->status != IB_WC_WR_FLUSH_ERR)
  1148. smb_direct_disconnect_rdma_connection(t);
  1149. }
  1150. complete(msg->completion);
  1151. }
  1152. static void read_done(struct ib_cq *cq, struct ib_wc *wc)
  1153. {
  1154. read_write_done(cq, wc, DMA_FROM_DEVICE);
  1155. }
  1156. static void write_done(struct ib_cq *cq, struct ib_wc *wc)
  1157. {
  1158. read_write_done(cq, wc, DMA_TO_DEVICE);
  1159. }
  1160. static int smb_direct_rdma_xmit(struct smb_direct_transport *t,
  1161. void *buf, int buf_len,
  1162. struct smb2_buffer_desc_v1 *desc,
  1163. unsigned int desc_len,
  1164. bool is_read)
  1165. {
  1166. struct smb_direct_rdma_rw_msg *msg, *next_msg;
  1167. int i, ret;
  1168. DECLARE_COMPLETION_ONSTACK(completion);
  1169. struct ib_send_wr *first_wr;
  1170. LIST_HEAD(msg_list);
  1171. char *desc_buf;
  1172. int credits_needed;
  1173. unsigned int desc_buf_len;
  1174. size_t total_length = 0;
  1175. if (t->status != SMB_DIRECT_CS_CONNECTED)
  1176. return -ENOTCONN;
  1177. /* calculate needed credits */
  1178. credits_needed = 0;
  1179. desc_buf = buf;
  1180. for (i = 0; i < desc_len / sizeof(*desc); i++) {
  1181. desc_buf_len = le32_to_cpu(desc[i].length);
  1182. credits_needed += calc_rw_credits(t, desc_buf, desc_buf_len);
  1183. desc_buf += desc_buf_len;
  1184. total_length += desc_buf_len;
  1185. if (desc_buf_len == 0 || total_length > buf_len ||
  1186. total_length > t->max_rdma_rw_size)
  1187. return -EINVAL;
  1188. }
  1189. ksmbd_debug(RDMA, "RDMA %s, len %#x, needed credits %#x\n",
  1190. is_read ? "read" : "write", buf_len, credits_needed);
  1191. ret = wait_for_rw_credits(t, credits_needed);
  1192. if (ret < 0)
  1193. return ret;
  1194. /* build rdma_rw_ctx for each descriptor */
  1195. desc_buf = buf;
  1196. for (i = 0; i < desc_len / sizeof(*desc); i++) {
  1197. msg = kzalloc(offsetof(struct smb_direct_rdma_rw_msg, sg_list) +
  1198. sizeof(struct scatterlist) * SG_CHUNK_SIZE, GFP_KERNEL);
  1199. if (!msg) {
  1200. ret = -ENOMEM;
  1201. goto out;
  1202. }
  1203. desc_buf_len = le32_to_cpu(desc[i].length);
  1204. msg->t = t;
  1205. msg->cqe.done = is_read ? read_done : write_done;
  1206. msg->completion = &completion;
  1207. msg->sgt.sgl = &msg->sg_list[0];
  1208. ret = sg_alloc_table_chained(&msg->sgt,
  1209. get_buf_page_count(desc_buf, desc_buf_len),
  1210. msg->sg_list, SG_CHUNK_SIZE);
  1211. if (ret) {
  1212. kfree(msg);
  1213. ret = -ENOMEM;
  1214. goto out;
  1215. }
  1216. ret = get_sg_list(desc_buf, desc_buf_len,
  1217. msg->sgt.sgl, msg->sgt.orig_nents);
  1218. if (ret < 0) {
  1219. sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);
  1220. kfree(msg);
  1221. goto out;
  1222. }
  1223. ret = rdma_rw_ctx_init(&msg->rw_ctx, t->qp, t->qp->port,
  1224. msg->sgt.sgl,
  1225. get_buf_page_count(desc_buf, desc_buf_len),
  1226. 0,
  1227. le64_to_cpu(desc[i].offset),
  1228. le32_to_cpu(desc[i].token),
  1229. is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
  1230. if (ret < 0) {
  1231. pr_err("failed to init rdma_rw_ctx: %d\n", ret);
  1232. sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);
  1233. kfree(msg);
  1234. goto out;
  1235. }
  1236. list_add_tail(&msg->list, &msg_list);
  1237. desc_buf += desc_buf_len;
  1238. }
  1239. /* concatenate work requests of rdma_rw_ctxs */
  1240. first_wr = NULL;
  1241. list_for_each_entry_reverse(msg, &msg_list, list) {
  1242. first_wr = rdma_rw_ctx_wrs(&msg->rw_ctx, t->qp, t->qp->port,
  1243. &msg->cqe, first_wr);
  1244. }
  1245. ret = ib_post_send(t->qp, first_wr, NULL);
  1246. if (ret) {
  1247. pr_err("failed to post send wr for RDMA R/W: %d\n", ret);
  1248. goto out;
  1249. }
  1250. msg = list_last_entry(&msg_list, struct smb_direct_rdma_rw_msg, list);
  1251. wait_for_completion(&completion);
  1252. ret = msg->status;
  1253. out:
  1254. list_for_each_entry_safe(msg, next_msg, &msg_list, list) {
  1255. list_del(&msg->list);
  1256. smb_direct_free_rdma_rw_msg(t, msg,
  1257. is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
  1258. }
  1259. atomic_add(credits_needed, &t->rw_credits);
  1260. wake_up(&t->wait_rw_credits);
  1261. return ret;
  1262. }
  1263. static int smb_direct_rdma_write(struct ksmbd_transport *t,
  1264. void *buf, unsigned int buflen,
  1265. struct smb2_buffer_desc_v1 *desc,
  1266. unsigned int desc_len)
  1267. {
  1268. return smb_direct_rdma_xmit(smb_trans_direct_transfort(t), buf, buflen,
  1269. desc, desc_len, false);
  1270. }
  1271. static int smb_direct_rdma_read(struct ksmbd_transport *t,
  1272. void *buf, unsigned int buflen,
  1273. struct smb2_buffer_desc_v1 *desc,
  1274. unsigned int desc_len)
  1275. {
  1276. return smb_direct_rdma_xmit(smb_trans_direct_transfort(t), buf, buflen,
  1277. desc, desc_len, true);
  1278. }
  1279. static void smb_direct_disconnect(struct ksmbd_transport *t)
  1280. {
  1281. struct smb_direct_transport *st = smb_trans_direct_transfort(t);
  1282. ksmbd_debug(RDMA, "Disconnecting cm_id=%p\n", st->cm_id);
  1283. smb_direct_disconnect_rdma_work(&st->disconnect_work);
  1284. wait_event_interruptible(st->wait_status,
  1285. st->status == SMB_DIRECT_CS_DISCONNECTED);
  1286. free_transport(st);
  1287. }
  1288. static void smb_direct_shutdown(struct ksmbd_transport *t)
  1289. {
  1290. struct smb_direct_transport *st = smb_trans_direct_transfort(t);
  1291. ksmbd_debug(RDMA, "smb-direct shutdown cm_id=%p\n", st->cm_id);
  1292. smb_direct_disconnect_rdma_work(&st->disconnect_work);
  1293. }
  1294. static int smb_direct_cm_handler(struct rdma_cm_id *cm_id,
  1295. struct rdma_cm_event *event)
  1296. {
  1297. struct smb_direct_transport *t = cm_id->context;
  1298. ksmbd_debug(RDMA, "RDMA CM event. cm_id=%p event=%s (%d)\n",
  1299. cm_id, rdma_event_msg(event->event), event->event);
  1300. switch (event->event) {
  1301. case RDMA_CM_EVENT_ESTABLISHED: {
  1302. t->status = SMB_DIRECT_CS_CONNECTED;
  1303. wake_up_interruptible(&t->wait_status);
  1304. break;
  1305. }
  1306. case RDMA_CM_EVENT_DEVICE_REMOVAL:
  1307. case RDMA_CM_EVENT_DISCONNECTED: {
  1308. t->status = SMB_DIRECT_CS_DISCONNECTED;
  1309. wake_up_interruptible(&t->wait_status);
  1310. wake_up_interruptible(&t->wait_reassembly_queue);
  1311. wake_up(&t->wait_send_credits);
  1312. break;
  1313. }
  1314. case RDMA_CM_EVENT_CONNECT_ERROR: {
  1315. t->status = SMB_DIRECT_CS_DISCONNECTED;
  1316. wake_up_interruptible(&t->wait_status);
  1317. break;
  1318. }
  1319. default:
  1320. pr_err("Unexpected RDMA CM event. cm_id=%p, event=%s (%d)\n",
  1321. cm_id, rdma_event_msg(event->event),
  1322. event->event);
  1323. break;
  1324. }
  1325. return 0;
  1326. }
  1327. static void smb_direct_qpair_handler(struct ib_event *event, void *context)
  1328. {
  1329. struct smb_direct_transport *t = context;
  1330. ksmbd_debug(RDMA, "Received QP event. cm_id=%p, event=%s (%d)\n",
  1331. t->cm_id, ib_event_msg(event->event), event->event);
  1332. switch (event->event) {
  1333. case IB_EVENT_CQ_ERR:
  1334. case IB_EVENT_QP_FATAL:
  1335. smb_direct_disconnect_rdma_connection(t);
  1336. break;
  1337. default:
  1338. break;
  1339. }
  1340. }
  1341. static int smb_direct_send_negotiate_response(struct smb_direct_transport *t,
  1342. int failed)
  1343. {
  1344. struct smb_direct_sendmsg *sendmsg;
  1345. struct smb_direct_negotiate_resp *resp;
  1346. int ret;
  1347. sendmsg = smb_direct_alloc_sendmsg(t);
  1348. if (IS_ERR(sendmsg))
  1349. return -ENOMEM;
  1350. resp = (struct smb_direct_negotiate_resp *)sendmsg->packet;
  1351. if (failed) {
  1352. memset(resp, 0, sizeof(*resp));
  1353. resp->min_version = cpu_to_le16(0x0100);
  1354. resp->max_version = cpu_to_le16(0x0100);
  1355. resp->status = STATUS_NOT_SUPPORTED;
  1356. } else {
  1357. resp->status = STATUS_SUCCESS;
  1358. resp->min_version = SMB_DIRECT_VERSION_LE;
  1359. resp->max_version = SMB_DIRECT_VERSION_LE;
  1360. resp->negotiated_version = SMB_DIRECT_VERSION_LE;
  1361. resp->reserved = 0;
  1362. resp->credits_requested =
  1363. cpu_to_le16(t->send_credit_target);
  1364. resp->credits_granted = cpu_to_le16(manage_credits_prior_sending(t));
  1365. resp->max_readwrite_size = cpu_to_le32(t->max_rdma_rw_size);
  1366. resp->preferred_send_size = cpu_to_le32(t->max_send_size);
  1367. resp->max_receive_size = cpu_to_le32(t->max_recv_size);
  1368. resp->max_fragmented_size =
  1369. cpu_to_le32(t->max_fragmented_recv_size);
  1370. }
  1371. sendmsg->sge[0].addr = ib_dma_map_single(t->cm_id->device,
  1372. (void *)resp, sizeof(*resp),
  1373. DMA_TO_DEVICE);
  1374. ret = ib_dma_mapping_error(t->cm_id->device, sendmsg->sge[0].addr);
  1375. if (ret) {
  1376. smb_direct_free_sendmsg(t, sendmsg);
  1377. return ret;
  1378. }
  1379. sendmsg->num_sge = 1;
  1380. sendmsg->sge[0].length = sizeof(*resp);
  1381. sendmsg->sge[0].lkey = t->pd->local_dma_lkey;
  1382. ret = post_sendmsg(t, NULL, sendmsg);
  1383. if (ret) {
  1384. smb_direct_free_sendmsg(t, sendmsg);
  1385. return ret;
  1386. }
  1387. wait_event(t->wait_send_pending,
  1388. atomic_read(&t->send_pending) == 0);
  1389. return 0;
  1390. }
  1391. static int smb_direct_accept_client(struct smb_direct_transport *t)
  1392. {
  1393. struct rdma_conn_param conn_param;
  1394. struct ib_port_immutable port_immutable;
  1395. u32 ird_ord_hdr[2];
  1396. int ret;
  1397. memset(&conn_param, 0, sizeof(conn_param));
  1398. conn_param.initiator_depth = min_t(u8, t->cm_id->device->attrs.max_qp_rd_atom,
  1399. SMB_DIRECT_CM_INITIATOR_DEPTH);
  1400. conn_param.responder_resources = 0;
  1401. t->cm_id->device->ops.get_port_immutable(t->cm_id->device,
  1402. t->cm_id->port_num,
  1403. &port_immutable);
  1404. if (port_immutable.core_cap_flags & RDMA_CORE_PORT_IWARP) {
  1405. ird_ord_hdr[0] = conn_param.responder_resources;
  1406. ird_ord_hdr[1] = 1;
  1407. conn_param.private_data = ird_ord_hdr;
  1408. conn_param.private_data_len = sizeof(ird_ord_hdr);
  1409. } else {
  1410. conn_param.private_data = NULL;
  1411. conn_param.private_data_len = 0;
  1412. }
  1413. conn_param.retry_count = SMB_DIRECT_CM_RETRY;
  1414. conn_param.rnr_retry_count = SMB_DIRECT_CM_RNR_RETRY;
  1415. conn_param.flow_control = 0;
  1416. ret = rdma_accept(t->cm_id, &conn_param);
  1417. if (ret) {
  1418. pr_err("error at rdma_accept: %d\n", ret);
  1419. return ret;
  1420. }
  1421. return 0;
  1422. }
  1423. static int smb_direct_prepare_negotiation(struct smb_direct_transport *t)
  1424. {
  1425. int ret;
  1426. struct smb_direct_recvmsg *recvmsg;
  1427. recvmsg = get_free_recvmsg(t);
  1428. if (!recvmsg)
  1429. return -ENOMEM;
  1430. recvmsg->type = SMB_DIRECT_MSG_NEGOTIATE_REQ;
  1431. ret = smb_direct_post_recv(t, recvmsg);
  1432. if (ret) {
  1433. pr_err("Can't post recv: %d\n", ret);
  1434. goto out_err;
  1435. }
  1436. t->negotiation_requested = false;
  1437. ret = smb_direct_accept_client(t);
  1438. if (ret) {
  1439. pr_err("Can't accept client\n");
  1440. goto out_err;
  1441. }
  1442. smb_direct_post_recv_credits(&t->post_recv_credits_work.work);
  1443. return 0;
  1444. out_err:
  1445. put_recvmsg(t, recvmsg);
  1446. return ret;
  1447. }
  1448. static unsigned int smb_direct_get_max_fr_pages(struct smb_direct_transport *t)
  1449. {
  1450. return min_t(unsigned int,
  1451. t->cm_id->device->attrs.max_fast_reg_page_list_len,
  1452. 256);
  1453. }
  1454. static int smb_direct_init_params(struct smb_direct_transport *t,
  1455. struct ib_qp_cap *cap)
  1456. {
  1457. struct ib_device *device = t->cm_id->device;
  1458. int max_send_sges, max_rw_wrs, max_send_wrs;
  1459. unsigned int max_sge_per_wr, wrs_per_credit;
  1460. /* need 3 more sge. because a SMB_DIRECT header, SMB2 header,
  1461. * SMB2 response could be mapped.
  1462. */
  1463. t->max_send_size = smb_direct_max_send_size;
  1464. max_send_sges = DIV_ROUND_UP(t->max_send_size, PAGE_SIZE) + 3;
  1465. if (max_send_sges > SMB_DIRECT_MAX_SEND_SGES) {
  1466. pr_err("max_send_size %d is too large\n", t->max_send_size);
  1467. return -EINVAL;
  1468. }
  1469. /* Calculate the number of work requests for RDMA R/W.
  1470. * The maximum number of pages which can be registered
  1471. * with one Memory region can be transferred with one
  1472. * R/W credit. And at least 4 work requests for each credit
  1473. * are needed for MR registration, RDMA R/W, local & remote
  1474. * MR invalidation.
  1475. */
  1476. t->max_rdma_rw_size = smb_direct_max_read_write_size;
  1477. t->pages_per_rw_credit = smb_direct_get_max_fr_pages(t);
  1478. t->max_rw_credits = DIV_ROUND_UP(t->max_rdma_rw_size,
  1479. (t->pages_per_rw_credit - 1) *
  1480. PAGE_SIZE);
  1481. max_sge_per_wr = min_t(unsigned int, device->attrs.max_send_sge,
  1482. device->attrs.max_sge_rd);
  1483. max_sge_per_wr = max_t(unsigned int, max_sge_per_wr,
  1484. max_send_sges);
  1485. wrs_per_credit = max_t(unsigned int, 4,
  1486. DIV_ROUND_UP(t->pages_per_rw_credit,
  1487. max_sge_per_wr) + 1);
  1488. max_rw_wrs = t->max_rw_credits * wrs_per_credit;
  1489. max_send_wrs = smb_direct_send_credit_target + max_rw_wrs;
  1490. if (max_send_wrs > device->attrs.max_cqe ||
  1491. max_send_wrs > device->attrs.max_qp_wr) {
  1492. pr_err("consider lowering send_credit_target = %d\n",
  1493. smb_direct_send_credit_target);
  1494. pr_err("Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n",
  1495. device->attrs.max_cqe, device->attrs.max_qp_wr);
  1496. return -EINVAL;
  1497. }
  1498. if (smb_direct_receive_credit_max > device->attrs.max_cqe ||
  1499. smb_direct_receive_credit_max > device->attrs.max_qp_wr) {
  1500. pr_err("consider lowering receive_credit_max = %d\n",
  1501. smb_direct_receive_credit_max);
  1502. pr_err("Possible CQE overrun, device reporting max_cpe %d max_qp_wr %d\n",
  1503. device->attrs.max_cqe, device->attrs.max_qp_wr);
  1504. return -EINVAL;
  1505. }
  1506. if (device->attrs.max_recv_sge < SMB_DIRECT_MAX_RECV_SGES) {
  1507. pr_err("warning: device max_recv_sge = %d too small\n",
  1508. device->attrs.max_recv_sge);
  1509. return -EINVAL;
  1510. }
  1511. t->recv_credits = 0;
  1512. t->count_avail_recvmsg = 0;
  1513. t->recv_credit_max = smb_direct_receive_credit_max;
  1514. t->recv_credit_target = 10;
  1515. t->new_recv_credits = 0;
  1516. t->send_credit_target = smb_direct_send_credit_target;
  1517. atomic_set(&t->send_credits, 0);
  1518. atomic_set(&t->rw_credits, t->max_rw_credits);
  1519. t->max_send_size = smb_direct_max_send_size;
  1520. t->max_recv_size = smb_direct_max_receive_size;
  1521. t->max_fragmented_recv_size = smb_direct_max_fragmented_recv_size;
  1522. cap->max_send_wr = max_send_wrs;
  1523. cap->max_recv_wr = t->recv_credit_max;
  1524. cap->max_send_sge = max_sge_per_wr;
  1525. cap->max_recv_sge = SMB_DIRECT_MAX_RECV_SGES;
  1526. cap->max_inline_data = 0;
  1527. cap->max_rdma_ctxs = t->max_rw_credits;
  1528. return 0;
  1529. }
  1530. static void smb_direct_destroy_pools(struct smb_direct_transport *t)
  1531. {
  1532. struct smb_direct_recvmsg *recvmsg;
  1533. while ((recvmsg = get_free_recvmsg(t)))
  1534. mempool_free(recvmsg, t->recvmsg_mempool);
  1535. while ((recvmsg = get_empty_recvmsg(t)))
  1536. mempool_free(recvmsg, t->recvmsg_mempool);
  1537. mempool_destroy(t->recvmsg_mempool);
  1538. t->recvmsg_mempool = NULL;
  1539. kmem_cache_destroy(t->recvmsg_cache);
  1540. t->recvmsg_cache = NULL;
  1541. mempool_destroy(t->sendmsg_mempool);
  1542. t->sendmsg_mempool = NULL;
  1543. kmem_cache_destroy(t->se