/net/sunrpc/svc.c

http://github.com/mirrors/linux · C · 1738 lines · 1182 code · 247 blank · 309 comment · 195 complexity · 115d1705b6371656144d4c6adb2943e9 MD5 · raw file

  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/net/sunrpc/svc.c
  4. *
  5. * High-level RPC service routines
  6. *
  7. * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de>
  8. *
  9. * Multiple threads pools and NUMAisation
  10. * Copyright (c) 2006 Silicon Graphics, Inc.
  11. * by Greg Banks <gnb@melbourne.sgi.com>
  12. */
  13. #include <linux/linkage.h>
  14. #include <linux/sched/signal.h>
  15. #include <linux/errno.h>
  16. #include <linux/net.h>
  17. #include <linux/in.h>
  18. #include <linux/mm.h>
  19. #include <linux/interrupt.h>
  20. #include <linux/module.h>
  21. #include <linux/kthread.h>
  22. #include <linux/slab.h>
  23. #include <linux/sunrpc/types.h>
  24. #include <linux/sunrpc/xdr.h>
  25. #include <linux/sunrpc/stats.h>
  26. #include <linux/sunrpc/svcsock.h>
  27. #include <linux/sunrpc/clnt.h>
  28. #include <linux/sunrpc/bc_xprt.h>
  29. #include <trace/events/sunrpc.h>
  30. #define RPCDBG_FACILITY RPCDBG_SVCDSP
  31. static void svc_unregister(const struct svc_serv *serv, struct net *net);
  32. #define svc_serv_is_pooled(serv) ((serv)->sv_ops->svo_function)
  33. #define SVC_POOL_DEFAULT SVC_POOL_GLOBAL
  34. /*
  35. * Structure for mapping cpus to pools and vice versa.
  36. * Setup once during sunrpc initialisation.
  37. */
  38. struct svc_pool_map svc_pool_map = {
  39. .mode = SVC_POOL_DEFAULT
  40. };
  41. EXPORT_SYMBOL_GPL(svc_pool_map);
  42. static DEFINE_MUTEX(svc_pool_map_mutex);/* protects svc_pool_map.count only */
  43. static int
  44. param_set_pool_mode(const char *val, const struct kernel_param *kp)
  45. {
  46. int *ip = (int *)kp->arg;
  47. struct svc_pool_map *m = &svc_pool_map;
  48. int err;
  49. mutex_lock(&svc_pool_map_mutex);
  50. err = -EBUSY;
  51. if (m->count)
  52. goto out;
  53. err = 0;
  54. if (!strncmp(val, "auto", 4))
  55. *ip = SVC_POOL_AUTO;
  56. else if (!strncmp(val, "global", 6))
  57. *ip = SVC_POOL_GLOBAL;
  58. else if (!strncmp(val, "percpu", 6))
  59. *ip = SVC_POOL_PERCPU;
  60. else if (!strncmp(val, "pernode", 7))
  61. *ip = SVC_POOL_PERNODE;
  62. else
  63. err = -EINVAL;
  64. out:
  65. mutex_unlock(&svc_pool_map_mutex);
  66. return err;
  67. }
  68. static int
  69. param_get_pool_mode(char *buf, const struct kernel_param *kp)
  70. {
  71. int *ip = (int *)kp->arg;
  72. switch (*ip)
  73. {
  74. case SVC_POOL_AUTO:
  75. return strlcpy(buf, "auto", 20);
  76. case SVC_POOL_GLOBAL:
  77. return strlcpy(buf, "global", 20);
  78. case SVC_POOL_PERCPU:
  79. return strlcpy(buf, "percpu", 20);
  80. case SVC_POOL_PERNODE:
  81. return strlcpy(buf, "pernode", 20);
  82. default:
  83. return sprintf(buf, "%d", *ip);
  84. }
  85. }
  86. module_param_call(pool_mode, param_set_pool_mode, param_get_pool_mode,
  87. &svc_pool_map.mode, 0644);
  88. /*
  89. * Detect best pool mapping mode heuristically,
  90. * according to the machine's topology.
  91. */
  92. static int
  93. svc_pool_map_choose_mode(void)
  94. {
  95. unsigned int node;
  96. if (nr_online_nodes > 1) {
  97. /*
  98. * Actually have multiple NUMA nodes,
  99. * so split pools on NUMA node boundaries
  100. */
  101. return SVC_POOL_PERNODE;
  102. }
  103. node = first_online_node;
  104. if (nr_cpus_node(node) > 2) {
  105. /*
  106. * Non-trivial SMP, or CONFIG_NUMA on
  107. * non-NUMA hardware, e.g. with a generic
  108. * x86_64 kernel on Xeons. In this case we
  109. * want to divide the pools on cpu boundaries.
  110. */
  111. return SVC_POOL_PERCPU;
  112. }
  113. /* default: one global pool */
  114. return SVC_POOL_GLOBAL;
  115. }
  116. /*
  117. * Allocate the to_pool[] and pool_to[] arrays.
  118. * Returns 0 on success or an errno.
  119. */
  120. static int
  121. svc_pool_map_alloc_arrays(struct svc_pool_map *m, unsigned int maxpools)
  122. {
  123. m->to_pool = kcalloc(maxpools, sizeof(unsigned int), GFP_KERNEL);
  124. if (!m->to_pool)
  125. goto fail;
  126. m->pool_to = kcalloc(maxpools, sizeof(unsigned int), GFP_KERNEL);
  127. if (!m->pool_to)
  128. goto fail_free;
  129. return 0;
  130. fail_free:
  131. kfree(m->to_pool);
  132. m->to_pool = NULL;
  133. fail:
  134. return -ENOMEM;
  135. }
  136. /*
  137. * Initialise the pool map for SVC_POOL_PERCPU mode.
  138. * Returns number of pools or <0 on error.
  139. */
  140. static int
  141. svc_pool_map_init_percpu(struct svc_pool_map *m)
  142. {
  143. unsigned int maxpools = nr_cpu_ids;
  144. unsigned int pidx = 0;
  145. unsigned int cpu;
  146. int err;
  147. err = svc_pool_map_alloc_arrays(m, maxpools);
  148. if (err)
  149. return err;
  150. for_each_online_cpu(cpu) {
  151. BUG_ON(pidx >= maxpools);
  152. m->to_pool[cpu] = pidx;
  153. m->pool_to[pidx] = cpu;
  154. pidx++;
  155. }
  156. /* cpus brought online later all get mapped to pool0, sorry */
  157. return pidx;
  158. };
  159. /*
  160. * Initialise the pool map for SVC_POOL_PERNODE mode.
  161. * Returns number of pools or <0 on error.
  162. */
  163. static int
  164. svc_pool_map_init_pernode(struct svc_pool_map *m)
  165. {
  166. unsigned int maxpools = nr_node_ids;
  167. unsigned int pidx = 0;
  168. unsigned int node;
  169. int err;
  170. err = svc_pool_map_alloc_arrays(m, maxpools);
  171. if (err)
  172. return err;
  173. for_each_node_with_cpus(node) {
  174. /* some architectures (e.g. SN2) have cpuless nodes */
  175. BUG_ON(pidx > maxpools);
  176. m->to_pool[node] = pidx;
  177. m->pool_to[pidx] = node;
  178. pidx++;
  179. }
  180. /* nodes brought online later all get mapped to pool0, sorry */
  181. return pidx;
  182. }
  183. /*
  184. * Add a reference to the global map of cpus to pools (and
  185. * vice versa). Initialise the map if we're the first user.
  186. * Returns the number of pools.
  187. */
  188. unsigned int
  189. svc_pool_map_get(void)
  190. {
  191. struct svc_pool_map *m = &svc_pool_map;
  192. int npools = -1;
  193. mutex_lock(&svc_pool_map_mutex);
  194. if (m->count++) {
  195. mutex_unlock(&svc_pool_map_mutex);
  196. return m->npools;
  197. }
  198. if (m->mode == SVC_POOL_AUTO)
  199. m->mode = svc_pool_map_choose_mode();
  200. switch (m->mode) {
  201. case SVC_POOL_PERCPU:
  202. npools = svc_pool_map_init_percpu(m);
  203. break;
  204. case SVC_POOL_PERNODE:
  205. npools = svc_pool_map_init_pernode(m);
  206. break;
  207. }
  208. if (npools < 0) {
  209. /* default, or memory allocation failure */
  210. npools = 1;
  211. m->mode = SVC_POOL_GLOBAL;
  212. }
  213. m->npools = npools;
  214. mutex_unlock(&svc_pool_map_mutex);
  215. return m->npools;
  216. }
  217. EXPORT_SYMBOL_GPL(svc_pool_map_get);
  218. /*
  219. * Drop a reference to the global map of cpus to pools.
  220. * When the last reference is dropped, the map data is
  221. * freed; this allows the sysadmin to change the pool
  222. * mode using the pool_mode module option without
  223. * rebooting or re-loading sunrpc.ko.
  224. */
  225. void
  226. svc_pool_map_put(void)
  227. {
  228. struct svc_pool_map *m = &svc_pool_map;
  229. mutex_lock(&svc_pool_map_mutex);
  230. if (!--m->count) {
  231. kfree(m->to_pool);
  232. m->to_pool = NULL;
  233. kfree(m->pool_to);
  234. m->pool_to = NULL;
  235. m->npools = 0;
  236. }
  237. mutex_unlock(&svc_pool_map_mutex);
  238. }
  239. EXPORT_SYMBOL_GPL(svc_pool_map_put);
  240. static int svc_pool_map_get_node(unsigned int pidx)
  241. {
  242. const struct svc_pool_map *m = &svc_pool_map;
  243. if (m->count) {
  244. if (m->mode == SVC_POOL_PERCPU)
  245. return cpu_to_node(m->pool_to[pidx]);
  246. if (m->mode == SVC_POOL_PERNODE)
  247. return m->pool_to[pidx];
  248. }
  249. return NUMA_NO_NODE;
  250. }
  251. /*
  252. * Set the given thread's cpus_allowed mask so that it
  253. * will only run on cpus in the given pool.
  254. */
  255. static inline void
  256. svc_pool_map_set_cpumask(struct task_struct *task, unsigned int pidx)
  257. {
  258. struct svc_pool_map *m = &svc_pool_map;
  259. unsigned int node = m->pool_to[pidx];
  260. /*
  261. * The caller checks for sv_nrpools > 1, which
  262. * implies that we've been initialized.
  263. */
  264. WARN_ON_ONCE(m->count == 0);
  265. if (m->count == 0)
  266. return;
  267. switch (m->mode) {
  268. case SVC_POOL_PERCPU:
  269. {
  270. set_cpus_allowed_ptr(task, cpumask_of(node));
  271. break;
  272. }
  273. case SVC_POOL_PERNODE:
  274. {
  275. set_cpus_allowed_ptr(task, cpumask_of_node(node));
  276. break;
  277. }
  278. }
  279. }
  280. /*
  281. * Use the mapping mode to choose a pool for a given CPU.
  282. * Used when enqueueing an incoming RPC. Always returns
  283. * a non-NULL pool pointer.
  284. */
  285. struct svc_pool *
  286. svc_pool_for_cpu(struct svc_serv *serv, int cpu)
  287. {
  288. struct svc_pool_map *m = &svc_pool_map;
  289. unsigned int pidx = 0;
  290. /*
  291. * An uninitialised map happens in a pure client when
  292. * lockd is brought up, so silently treat it the
  293. * same as SVC_POOL_GLOBAL.
  294. */
  295. if (svc_serv_is_pooled(serv)) {
  296. switch (m->mode) {
  297. case SVC_POOL_PERCPU:
  298. pidx = m->to_pool[cpu];
  299. break;
  300. case SVC_POOL_PERNODE:
  301. pidx = m->to_pool[cpu_to_node(cpu)];
  302. break;
  303. }
  304. }
  305. return &serv->sv_pools[pidx % serv->sv_nrpools];
  306. }
  307. int svc_rpcb_setup(struct svc_serv *serv, struct net *net)
  308. {
  309. int err;
  310. err = rpcb_create_local(net);
  311. if (err)
  312. return err;
  313. /* Remove any stale portmap registrations */
  314. svc_unregister(serv, net);
  315. return 0;
  316. }
  317. EXPORT_SYMBOL_GPL(svc_rpcb_setup);
  318. void svc_rpcb_cleanup(struct svc_serv *serv, struct net *net)
  319. {
  320. svc_unregister(serv, net);
  321. rpcb_put_local(net);
  322. }
  323. EXPORT_SYMBOL_GPL(svc_rpcb_cleanup);
  324. static int svc_uses_rpcbind(struct svc_serv *serv)
  325. {
  326. struct svc_program *progp;
  327. unsigned int i;
  328. for (progp = serv->sv_program; progp; progp = progp->pg_next) {
  329. for (i = 0; i < progp->pg_nvers; i++) {
  330. if (progp->pg_vers[i] == NULL)
  331. continue;
  332. if (!progp->pg_vers[i]->vs_hidden)
  333. return 1;
  334. }
  335. }
  336. return 0;
  337. }
  338. int svc_bind(struct svc_serv *serv, struct net *net)
  339. {
  340. if (!svc_uses_rpcbind(serv))
  341. return 0;
  342. return svc_rpcb_setup(serv, net);
  343. }
  344. EXPORT_SYMBOL_GPL(svc_bind);
  345. #if defined(CONFIG_SUNRPC_BACKCHANNEL)
  346. static void
  347. __svc_init_bc(struct svc_serv *serv)
  348. {
  349. INIT_LIST_HEAD(&serv->sv_cb_list);
  350. spin_lock_init(&serv->sv_cb_lock);
  351. init_waitqueue_head(&serv->sv_cb_waitq);
  352. }
  353. #else
  354. static void
  355. __svc_init_bc(struct svc_serv *serv)
  356. {
  357. }
  358. #endif
  359. /*
  360. * Create an RPC service
  361. */
  362. static struct svc_serv *
  363. __svc_create(struct svc_program *prog, unsigned int bufsize, int npools,
  364. const struct svc_serv_ops *ops)
  365. {
  366. struct svc_serv *serv;
  367. unsigned int vers;
  368. unsigned int xdrsize;
  369. unsigned int i;
  370. if (!(serv = kzalloc(sizeof(*serv), GFP_KERNEL)))
  371. return NULL;
  372. serv->sv_name = prog->pg_name;
  373. serv->sv_program = prog;
  374. serv->sv_nrthreads = 1;
  375. serv->sv_stats = prog->pg_stats;
  376. if (bufsize > RPCSVC_MAXPAYLOAD)
  377. bufsize = RPCSVC_MAXPAYLOAD;
  378. serv->sv_max_payload = bufsize? bufsize : 4096;
  379. serv->sv_max_mesg = roundup(serv->sv_max_payload + PAGE_SIZE, PAGE_SIZE);
  380. serv->sv_ops = ops;
  381. xdrsize = 0;
  382. while (prog) {
  383. prog->pg_lovers = prog->pg_nvers-1;
  384. for (vers=0; vers<prog->pg_nvers ; vers++)
  385. if (prog->pg_vers[vers]) {
  386. prog->pg_hivers = vers;
  387. if (prog->pg_lovers > vers)
  388. prog->pg_lovers = vers;
  389. if (prog->pg_vers[vers]->vs_xdrsize > xdrsize)
  390. xdrsize = prog->pg_vers[vers]->vs_xdrsize;
  391. }
  392. prog = prog->pg_next;
  393. }
  394. serv->sv_xdrsize = xdrsize;
  395. INIT_LIST_HEAD(&serv->sv_tempsocks);
  396. INIT_LIST_HEAD(&serv->sv_permsocks);
  397. timer_setup(&serv->sv_temptimer, NULL, 0);
  398. spin_lock_init(&serv->sv_lock);
  399. __svc_init_bc(serv);
  400. serv->sv_nrpools = npools;
  401. serv->sv_pools =
  402. kcalloc(serv->sv_nrpools, sizeof(struct svc_pool),
  403. GFP_KERNEL);
  404. if (!serv->sv_pools) {
  405. kfree(serv);
  406. return NULL;
  407. }
  408. for (i = 0; i < serv->sv_nrpools; i++) {
  409. struct svc_pool *pool = &serv->sv_pools[i];
  410. dprintk("svc: initialising pool %u for %s\n",
  411. i, serv->sv_name);
  412. pool->sp_id = i;
  413. INIT_LIST_HEAD(&pool->sp_sockets);
  414. INIT_LIST_HEAD(&pool->sp_all_threads);
  415. spin_lock_init(&pool->sp_lock);
  416. }
  417. return serv;
  418. }
  419. struct svc_serv *
  420. svc_create(struct svc_program *prog, unsigned int bufsize,
  421. const struct svc_serv_ops *ops)
  422. {
  423. return __svc_create(prog, bufsize, /*npools*/1, ops);
  424. }
  425. EXPORT_SYMBOL_GPL(svc_create);
  426. struct svc_serv *
  427. svc_create_pooled(struct svc_program *prog, unsigned int bufsize,
  428. const struct svc_serv_ops *ops)
  429. {
  430. struct svc_serv *serv;
  431. unsigned int npools = svc_pool_map_get();
  432. serv = __svc_create(prog, bufsize, npools, ops);
  433. if (!serv)
  434. goto out_err;
  435. return serv;
  436. out_err:
  437. svc_pool_map_put();
  438. return NULL;
  439. }
  440. EXPORT_SYMBOL_GPL(svc_create_pooled);
  441. void svc_shutdown_net(struct svc_serv *serv, struct net *net)
  442. {
  443. svc_close_net(serv, net);
  444. if (serv->sv_ops->svo_shutdown)
  445. serv->sv_ops->svo_shutdown(serv, net);
  446. }
  447. EXPORT_SYMBOL_GPL(svc_shutdown_net);
  448. /*
  449. * Destroy an RPC service. Should be called with appropriate locking to
  450. * protect the sv_nrthreads, sv_permsocks and sv_tempsocks.
  451. */
  452. void
  453. svc_destroy(struct svc_serv *serv)
  454. {
  455. dprintk("svc: svc_destroy(%s, %d)\n",
  456. serv->sv_program->pg_name,
  457. serv->sv_nrthreads);
  458. if (serv->sv_nrthreads) {
  459. if (--(serv->sv_nrthreads) != 0) {
  460. svc_sock_update_bufs(serv);
  461. return;
  462. }
  463. } else
  464. printk("svc_destroy: no threads for serv=%p!\n", serv);
  465. del_timer_sync(&serv->sv_temptimer);
  466. /*
  467. * The last user is gone and thus all sockets have to be destroyed to
  468. * the point. Check this.
  469. */
  470. BUG_ON(!list_empty(&serv->sv_permsocks));
  471. BUG_ON(!list_empty(&serv->sv_tempsocks));
  472. cache_clean_deferred(serv);
  473. if (svc_serv_is_pooled(serv))
  474. svc_pool_map_put();
  475. kfree(serv->sv_pools);
  476. kfree(serv);
  477. }
  478. EXPORT_SYMBOL_GPL(svc_destroy);
  479. /*
  480. * Allocate an RPC server's buffer space.
  481. * We allocate pages and place them in rq_argpages.
  482. */
  483. static int
  484. svc_init_buffer(struct svc_rqst *rqstp, unsigned int size, int node)
  485. {
  486. unsigned int pages, arghi;
  487. /* bc_xprt uses fore channel allocated buffers */
  488. if (svc_is_backchannel(rqstp))
  489. return 1;
  490. pages = size / PAGE_SIZE + 1; /* extra page as we hold both request and reply.
  491. * We assume one is at most one page
  492. */
  493. arghi = 0;
  494. WARN_ON_ONCE(pages > RPCSVC_MAXPAGES);
  495. if (pages > RPCSVC_MAXPAGES)
  496. pages = RPCSVC_MAXPAGES;
  497. while (pages) {
  498. struct page *p = alloc_pages_node(node, GFP_KERNEL, 0);
  499. if (!p)
  500. break;
  501. rqstp->rq_pages[arghi++] = p;
  502. pages--;
  503. }
  504. return pages == 0;
  505. }
  506. /*
  507. * Release an RPC server buffer
  508. */
  509. static void
  510. svc_release_buffer(struct svc_rqst *rqstp)
  511. {
  512. unsigned int i;
  513. for (i = 0; i < ARRAY_SIZE(rqstp->rq_pages); i++)
  514. if (rqstp->rq_pages[i])
  515. put_page(rqstp->rq_pages[i]);
  516. }
  517. struct svc_rqst *
  518. svc_rqst_alloc(struct svc_serv *serv, struct svc_pool *pool, int node)
  519. {
  520. struct svc_rqst *rqstp;
  521. rqstp = kzalloc_node(sizeof(*rqstp), GFP_KERNEL, node);
  522. if (!rqstp)
  523. return rqstp;
  524. __set_bit(RQ_BUSY, &rqstp->rq_flags);
  525. spin_lock_init(&rqstp->rq_lock);
  526. rqstp->rq_server = serv;
  527. rqstp->rq_pool = pool;
  528. rqstp->rq_argp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
  529. if (!rqstp->rq_argp)
  530. goto out_enomem;
  531. rqstp->rq_resp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
  532. if (!rqstp->rq_resp)
  533. goto out_enomem;
  534. if (!svc_init_buffer(rqstp, serv->sv_max_mesg, node))
  535. goto out_enomem;
  536. return rqstp;
  537. out_enomem:
  538. svc_rqst_free(rqstp);
  539. return NULL;
  540. }
  541. EXPORT_SYMBOL_GPL(svc_rqst_alloc);
  542. struct svc_rqst *
  543. svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)
  544. {
  545. struct svc_rqst *rqstp;
  546. rqstp = svc_rqst_alloc(serv, pool, node);
  547. if (!rqstp)
  548. return ERR_PTR(-ENOMEM);
  549. serv->sv_nrthreads++;
  550. spin_lock_bh(&pool->sp_lock);
  551. pool->sp_nrthreads++;
  552. list_add_rcu(&rqstp->rq_all, &pool->sp_all_threads);
  553. spin_unlock_bh(&pool->sp_lock);
  554. return rqstp;
  555. }
  556. EXPORT_SYMBOL_GPL(svc_prepare_thread);
  557. /*
  558. * Choose a pool in which to create a new thread, for svc_set_num_threads
  559. */
  560. static inline struct svc_pool *
  561. choose_pool(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state)
  562. {
  563. if (pool != NULL)
  564. return pool;
  565. return &serv->sv_pools[(*state)++ % serv->sv_nrpools];
  566. }
  567. /*
  568. * Choose a thread to kill, for svc_set_num_threads
  569. */
  570. static inline struct task_struct *
  571. choose_victim(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state)
  572. {
  573. unsigned int i;
  574. struct task_struct *task = NULL;
  575. if (pool != NULL) {
  576. spin_lock_bh(&pool->sp_lock);
  577. } else {
  578. /* choose a pool in round-robin fashion */
  579. for (i = 0; i < serv->sv_nrpools; i++) {
  580. pool = &serv->sv_pools[--(*state) % serv->sv_nrpools];
  581. spin_lock_bh(&pool->sp_lock);
  582. if (!list_empty(&pool->sp_all_threads))
  583. goto found_pool;
  584. spin_unlock_bh(&pool->sp_lock);
  585. }
  586. return NULL;
  587. }
  588. found_pool:
  589. if (!list_empty(&pool->sp_all_threads)) {
  590. struct svc_rqst *rqstp;
  591. /*
  592. * Remove from the pool->sp_all_threads list
  593. * so we don't try to kill it again.
  594. */
  595. rqstp = list_entry(pool->sp_all_threads.next, struct svc_rqst, rq_all);
  596. set_bit(RQ_VICTIM, &rqstp->rq_flags);
  597. list_del_rcu(&rqstp->rq_all);
  598. task = rqstp->rq_task;
  599. }
  600. spin_unlock_bh(&pool->sp_lock);
  601. return task;
  602. }
  603. /* create new threads */
  604. static int
  605. svc_start_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
  606. {
  607. struct svc_rqst *rqstp;
  608. struct task_struct *task;
  609. struct svc_pool *chosen_pool;
  610. unsigned int state = serv->sv_nrthreads-1;
  611. int node;
  612. do {
  613. nrservs--;
  614. chosen_pool = choose_pool(serv, pool, &state);
  615. node = svc_pool_map_get_node(chosen_pool->sp_id);
  616. rqstp = svc_prepare_thread(serv, chosen_pool, node);
  617. if (IS_ERR(rqstp))
  618. return PTR_ERR(rqstp);
  619. __module_get(serv->sv_ops->svo_module);
  620. task = kthread_create_on_node(serv->sv_ops->svo_function, rqstp,
  621. node, "%s", serv->sv_name);
  622. if (IS_ERR(task)) {
  623. module_put(serv->sv_ops->svo_module);
  624. svc_exit_thread(rqstp);
  625. return PTR_ERR(task);
  626. }
  627. rqstp->rq_task = task;
  628. if (serv->sv_nrpools > 1)
  629. svc_pool_map_set_cpumask(task, chosen_pool->sp_id);
  630. svc_sock_update_bufs(serv);
  631. wake_up_process(task);
  632. } while (nrservs > 0);
  633. return 0;
  634. }
  635. /* destroy old threads */
  636. static int
  637. svc_signal_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
  638. {
  639. struct task_struct *task;
  640. unsigned int state = serv->sv_nrthreads-1;
  641. /* destroy old threads */
  642. do {
  643. task = choose_victim(serv, pool, &state);
  644. if (task == NULL)
  645. break;
  646. send_sig(SIGINT, task, 1);
  647. nrservs++;
  648. } while (nrservs < 0);
  649. return 0;
  650. }
  651. /*
  652. * Create or destroy enough new threads to make the number
  653. * of threads the given number. If `pool' is non-NULL, applies
  654. * only to threads in that pool, otherwise round-robins between
  655. * all pools. Caller must ensure that mutual exclusion between this and
  656. * server startup or shutdown.
  657. *
  658. * Destroying threads relies on the service threads filling in
  659. * rqstp->rq_task, which only the nfs ones do. Assumes the serv
  660. * has been created using svc_create_pooled().
  661. *
  662. * Based on code that used to be in nfsd_svc() but tweaked
  663. * to be pool-aware.
  664. */
  665. int
  666. svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
  667. {
  668. if (pool == NULL) {
  669. /* The -1 assumes caller has done a svc_get() */
  670. nrservs -= (serv->sv_nrthreads-1);
  671. } else {
  672. spin_lock_bh(&pool->sp_lock);
  673. nrservs -= pool->sp_nrthreads;
  674. spin_unlock_bh(&pool->sp_lock);
  675. }
  676. if (nrservs > 0)
  677. return svc_start_kthreads(serv, pool, nrservs);
  678. if (nrservs < 0)
  679. return svc_signal_kthreads(serv, pool, nrservs);
  680. return 0;
  681. }
  682. EXPORT_SYMBOL_GPL(svc_set_num_threads);
  683. /* destroy old threads */
  684. static int
  685. svc_stop_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
  686. {
  687. struct task_struct *task;
  688. unsigned int state = serv->sv_nrthreads-1;
  689. /* destroy old threads */
  690. do {
  691. task = choose_victim(serv, pool, &state);
  692. if (task == NULL)
  693. break;
  694. kthread_stop(task);
  695. nrservs++;
  696. } while (nrservs < 0);
  697. return 0;
  698. }
  699. int
  700. svc_set_num_threads_sync(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
  701. {
  702. if (pool == NULL) {
  703. /* The -1 assumes caller has done a svc_get() */
  704. nrservs -= (serv->sv_nrthreads-1);
  705. } else {
  706. spin_lock_bh(&pool->sp_lock);
  707. nrservs -= pool->sp_nrthreads;
  708. spin_unlock_bh(&pool->sp_lock);
  709. }
  710. if (nrservs > 0)
  711. return svc_start_kthreads(serv, pool, nrservs);
  712. if (nrservs < 0)
  713. return svc_stop_kthreads(serv, pool, nrservs);
  714. return 0;
  715. }
  716. EXPORT_SYMBOL_GPL(svc_set_num_threads_sync);
  717. /*
  718. * Called from a server thread as it's exiting. Caller must hold the "service
  719. * mutex" for the service.
  720. */
  721. void
  722. svc_rqst_free(struct svc_rqst *rqstp)
  723. {
  724. svc_release_buffer(rqstp);
  725. kfree(rqstp->rq_resp);
  726. kfree(rqstp->rq_argp);
  727. kfree(rqstp->rq_auth_data);
  728. kfree_rcu(rqstp, rq_rcu_head);
  729. }
  730. EXPORT_SYMBOL_GPL(svc_rqst_free);
  731. void
  732. svc_exit_thread(struct svc_rqst *rqstp)
  733. {
  734. struct svc_serv *serv = rqstp->rq_server;
  735. struct svc_pool *pool = rqstp->rq_pool;
  736. spin_lock_bh(&pool->sp_lock);
  737. pool->sp_nrthreads--;
  738. if (!test_and_set_bit(RQ_VICTIM, &rqstp->rq_flags))
  739. list_del_rcu(&rqstp->rq_all);
  740. spin_unlock_bh(&pool->sp_lock);
  741. svc_rqst_free(rqstp);
  742. /* Release the server */
  743. if (serv)
  744. svc_destroy(serv);
  745. }
  746. EXPORT_SYMBOL_GPL(svc_exit_thread);
  747. /*
  748. * Register an "inet" protocol family netid with the local
  749. * rpcbind daemon via an rpcbind v4 SET request.
  750. *
  751. * No netconfig infrastructure is available in the kernel, so
  752. * we map IP_ protocol numbers to netids by hand.
  753. *
  754. * Returns zero on success; a negative errno value is returned
  755. * if any error occurs.
  756. */
  757. static int __svc_rpcb_register4(struct net *net, const u32 program,
  758. const u32 version,
  759. const unsigned short protocol,
  760. const unsigned short port)
  761. {
  762. const struct sockaddr_in sin = {
  763. .sin_family = AF_INET,
  764. .sin_addr.s_addr = htonl(INADDR_ANY),
  765. .sin_port = htons(port),
  766. };
  767. const char *netid;
  768. int error;
  769. switch (protocol) {
  770. case IPPROTO_UDP:
  771. netid = RPCBIND_NETID_UDP;
  772. break;
  773. case IPPROTO_TCP:
  774. netid = RPCBIND_NETID_TCP;
  775. break;
  776. default:
  777. return -ENOPROTOOPT;
  778. }
  779. error = rpcb_v4_register(net, program, version,
  780. (const struct sockaddr *)&sin, netid);
  781. /*
  782. * User space didn't support rpcbind v4, so retry this
  783. * registration request with the legacy rpcbind v2 protocol.
  784. */
  785. if (error == -EPROTONOSUPPORT)
  786. error = rpcb_register(net, program, version, protocol, port);
  787. return error;
  788. }
  789. #if IS_ENABLED(CONFIG_IPV6)
  790. /*
  791. * Register an "inet6" protocol family netid with the local
  792. * rpcbind daemon via an rpcbind v4 SET request.
  793. *
  794. * No netconfig infrastructure is available in the kernel, so
  795. * we map IP_ protocol numbers to netids by hand.
  796. *
  797. * Returns zero on success; a negative errno value is returned
  798. * if any error occurs.
  799. */
  800. static int __svc_rpcb_register6(struct net *net, const u32 program,
  801. const u32 version,
  802. const unsigned short protocol,
  803. const unsigned short port)
  804. {
  805. const struct sockaddr_in6 sin6 = {
  806. .sin6_family = AF_INET6,
  807. .sin6_addr = IN6ADDR_ANY_INIT,
  808. .sin6_port = htons(port),
  809. };
  810. const char *netid;
  811. int error;
  812. switch (protocol) {
  813. case IPPROTO_UDP:
  814. netid = RPCBIND_NETID_UDP6;
  815. break;
  816. case IPPROTO_TCP:
  817. netid = RPCBIND_NETID_TCP6;
  818. break;
  819. default:
  820. return -ENOPROTOOPT;
  821. }
  822. error = rpcb_v4_register(net, program, version,
  823. (const struct sockaddr *)&sin6, netid);
  824. /*
  825. * User space didn't support rpcbind version 4, so we won't
  826. * use a PF_INET6 listener.
  827. */
  828. if (error == -EPROTONOSUPPORT)
  829. error = -EAFNOSUPPORT;
  830. return error;
  831. }
  832. #endif /* IS_ENABLED(CONFIG_IPV6) */
  833. /*
  834. * Register a kernel RPC service via rpcbind version 4.
  835. *
  836. * Returns zero on success; a negative errno value is returned
  837. * if any error occurs.
  838. */
  839. static int __svc_register(struct net *net, const char *progname,
  840. const u32 program, const u32 version,
  841. const int family,
  842. const unsigned short protocol,
  843. const unsigned short port)
  844. {
  845. int error = -EAFNOSUPPORT;
  846. switch (family) {
  847. case PF_INET:
  848. error = __svc_rpcb_register4(net, program, version,
  849. protocol, port);
  850. break;
  851. #if IS_ENABLED(CONFIG_IPV6)
  852. case PF_INET6:
  853. error = __svc_rpcb_register6(net, program, version,
  854. protocol, port);
  855. #endif
  856. }
  857. return error;
  858. }
  859. int svc_rpcbind_set_version(struct net *net,
  860. const struct svc_program *progp,
  861. u32 version, int family,
  862. unsigned short proto,
  863. unsigned short port)
  864. {
  865. dprintk("svc: svc_register(%sv%d, %s, %u, %u)\n",
  866. progp->pg_name, version,
  867. proto == IPPROTO_UDP? "udp" : "tcp",
  868. port, family);
  869. return __svc_register(net, progp->pg_name, progp->pg_prog,
  870. version, family, proto, port);
  871. }
  872. EXPORT_SYMBOL_GPL(svc_rpcbind_set_version);
  873. int svc_generic_rpcbind_set(struct net *net,
  874. const struct svc_program *progp,
  875. u32 version, int family,
  876. unsigned short proto,
  877. unsigned short port)
  878. {
  879. const struct svc_version *vers = progp->pg_vers[version];
  880. int error;
  881. if (vers == NULL)
  882. return 0;
  883. if (vers->vs_hidden) {
  884. dprintk("svc: svc_register(%sv%d, %s, %u, %u)"
  885. " (but not telling portmap)\n",
  886. progp->pg_name, version,
  887. proto == IPPROTO_UDP? "udp" : "tcp",
  888. port, family);
  889. return 0;
  890. }
  891. /*
  892. * Don't register a UDP port if we need congestion
  893. * control.
  894. */
  895. if (vers->vs_need_cong_ctrl && proto == IPPROTO_UDP)
  896. return 0;
  897. error = svc_rpcbind_set_version(net, progp, version,
  898. family, proto, port);
  899. return (vers->vs_rpcb_optnl) ? 0 : error;
  900. }
  901. EXPORT_SYMBOL_GPL(svc_generic_rpcbind_set);
  902. /**
  903. * svc_register - register an RPC service with the local portmapper
  904. * @serv: svc_serv struct for the service to register
  905. * @net: net namespace for the service to register
  906. * @family: protocol family of service's listener socket
  907. * @proto: transport protocol number to advertise
  908. * @port: port to advertise
  909. *
  910. * Service is registered for any address in the passed-in protocol family
  911. */
  912. int svc_register(const struct svc_serv *serv, struct net *net,
  913. const int family, const unsigned short proto,
  914. const unsigned short port)
  915. {
  916. struct svc_program *progp;
  917. unsigned int i;
  918. int error = 0;
  919. WARN_ON_ONCE(proto == 0 && port == 0);
  920. if (proto == 0 && port == 0)
  921. return -EINVAL;
  922. for (progp = serv->sv_program; progp; progp = progp->pg_next) {
  923. for (i = 0; i < progp->pg_nvers; i++) {
  924. error = progp->pg_rpcbind_set(net, progp, i,
  925. family, proto, port);
  926. if (error < 0) {
  927. printk(KERN_WARNING "svc: failed to register "
  928. "%sv%u RPC service (errno %d).\n",
  929. progp->pg_name, i, -error);
  930. break;
  931. }
  932. }
  933. }
  934. return error;
  935. }
  936. /*
  937. * If user space is running rpcbind, it should take the v4 UNSET
  938. * and clear everything for this [program, version]. If user space
  939. * is running portmap, it will reject the v4 UNSET, but won't have
  940. * any "inet6" entries anyway. So a PMAP_UNSET should be sufficient
  941. * in this case to clear all existing entries for [program, version].
  942. */
  943. static void __svc_unregister(struct net *net, const u32 program, const u32 version,
  944. const char *progname)
  945. {
  946. int error;
  947. error = rpcb_v4_register(net, program, version, NULL, "");
  948. /*
  949. * User space didn't support rpcbind v4, so retry this
  950. * request with the legacy rpcbind v2 protocol.
  951. */
  952. if (error == -EPROTONOSUPPORT)
  953. error = rpcb_register(net, program, version, 0, 0);
  954. dprintk("svc: %s(%sv%u), error %d\n",
  955. __func__, progname, version, error);
  956. }
  957. /*
  958. * All netids, bind addresses and ports registered for [program, version]
  959. * are removed from the local rpcbind database (if the service is not
  960. * hidden) to make way for a new instance of the service.
  961. *
  962. * The result of unregistration is reported via dprintk for those who want
  963. * verification of the result, but is otherwise not important.
  964. */
  965. static void svc_unregister(const struct svc_serv *serv, struct net *net)
  966. {
  967. struct svc_program *progp;
  968. unsigned long flags;
  969. unsigned int i;
  970. clear_thread_flag(TIF_SIGPENDING);
  971. for (progp = serv->sv_program; progp; progp = progp->pg_next) {
  972. for (i = 0; i < progp->pg_nvers; i++) {
  973. if (progp->pg_vers[i] == NULL)
  974. continue;
  975. if (progp->pg_vers[i]->vs_hidden)
  976. continue;
  977. dprintk("svc: attempting to unregister %sv%u\n",
  978. progp->pg_name, i);
  979. __svc_unregister(net, progp->pg_prog, i, progp->pg_name);
  980. }
  981. }
  982. spin_lock_irqsave(&current->sighand->siglock, flags);
  983. recalc_sigpending();
  984. spin_unlock_irqrestore(&current->sighand->siglock, flags);
  985. }
  986. /*
  987. * dprintk the given error with the address of the client that caused it.
  988. */
  989. #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
  990. static __printf(2, 3)
  991. void svc_printk(struct svc_rqst *rqstp, const char *fmt, ...)
  992. {
  993. struct va_format vaf;
  994. va_list args;
  995. char buf[RPC_MAX_ADDRBUFLEN];
  996. va_start(args, fmt);
  997. vaf.fmt = fmt;
  998. vaf.va = &args;
  999. dprintk("svc: %s: %pV", svc_print_addr(rqstp, buf, sizeof(buf)), &vaf);
  1000. va_end(args);
  1001. }
  1002. #else
  1003. static __printf(2,3) void svc_printk(struct svc_rqst *rqstp, const char *fmt, ...) {}
  1004. #endif
  1005. __be32
  1006. svc_return_autherr(struct svc_rqst *rqstp, __be32 auth_err)
  1007. {
  1008. set_bit(RQ_AUTHERR, &rqstp->rq_flags);
  1009. return auth_err;
  1010. }
  1011. EXPORT_SYMBOL_GPL(svc_return_autherr);
  1012. static __be32
  1013. svc_get_autherr(struct svc_rqst *rqstp, __be32 *statp)
  1014. {
  1015. if (test_and_clear_bit(RQ_AUTHERR, &rqstp->rq_flags))
  1016. return *statp;
  1017. return rpc_auth_ok;
  1018. }
  1019. static int
  1020. svc_generic_dispatch(struct svc_rqst *rqstp, __be32 *statp)
  1021. {
  1022. struct kvec *argv = &rqstp->rq_arg.head[0];
  1023. struct kvec *resv = &rqstp->rq_res.head[0];
  1024. const struct svc_procedure *procp = rqstp->rq_procinfo;
  1025. /*
  1026. * Decode arguments
  1027. * XXX: why do we ignore the return value?
  1028. */
  1029. if (procp->pc_decode &&
  1030. !procp->pc_decode(rqstp, argv->iov_base)) {
  1031. *statp = rpc_garbage_args;
  1032. return 1;
  1033. }
  1034. *statp = procp->pc_func(rqstp);
  1035. if (*statp == rpc_drop_reply ||
  1036. test_bit(RQ_DROPME, &rqstp->rq_flags))
  1037. return 0;
  1038. if (test_bit(RQ_AUTHERR, &rqstp->rq_flags))
  1039. return 1;
  1040. if (*statp != rpc_success)
  1041. return 1;
  1042. /* Encode reply */
  1043. if (procp->pc_encode &&
  1044. !procp->pc_encode(rqstp, resv->iov_base + resv->iov_len)) {
  1045. dprintk("svc: failed to encode reply\n");
  1046. /* serv->sv_stats->rpcsystemerr++; */
  1047. *statp = rpc_system_err;
  1048. }
  1049. return 1;
  1050. }
  1051. __be32
  1052. svc_generic_init_request(struct svc_rqst *rqstp,
  1053. const struct svc_program *progp,
  1054. struct svc_process_info *ret)
  1055. {
  1056. const struct svc_version *versp = NULL; /* compiler food */
  1057. const struct svc_procedure *procp = NULL;
  1058. if (rqstp->rq_vers >= progp->pg_nvers )
  1059. goto err_bad_vers;
  1060. versp = progp->pg_vers[rqstp->rq_vers];
  1061. if (!versp)
  1062. goto err_bad_vers;
  1063. /*
  1064. * Some protocol versions (namely NFSv4) require some form of
  1065. * congestion control. (See RFC 7530 section 3.1 paragraph 2)
  1066. * In other words, UDP is not allowed. We mark those when setting
  1067. * up the svc_xprt, and verify that here.
  1068. *
  1069. * The spec is not very clear about what error should be returned
  1070. * when someone tries to access a server that is listening on UDP
  1071. * for lower versions. RPC_PROG_MISMATCH seems to be the closest
  1072. * fit.
  1073. */
  1074. if (versp->vs_need_cong_ctrl && rqstp->rq_xprt &&
  1075. !test_bit(XPT_CONG_CTRL, &rqstp->rq_xprt->xpt_flags))
  1076. goto err_bad_vers;
  1077. if (rqstp->rq_proc >= versp->vs_nproc)
  1078. goto err_bad_proc;
  1079. rqstp->rq_procinfo = procp = &versp->vs_proc[rqstp->rq_proc];
  1080. if (!procp)
  1081. goto err_bad_proc;
  1082. /* Initialize storage for argp and resp */
  1083. memset(rqstp->rq_argp, 0, procp->pc_argsize);
  1084. memset(rqstp->rq_resp, 0, procp->pc_ressize);
  1085. /* Bump per-procedure stats counter */
  1086. versp->vs_count[rqstp->rq_proc]++;
  1087. ret->dispatch = versp->vs_dispatch;
  1088. return rpc_success;
  1089. err_bad_vers:
  1090. ret->mismatch.lovers = progp->pg_lovers;
  1091. ret->mismatch.hivers = progp->pg_hivers;
  1092. return rpc_prog_mismatch;
  1093. err_bad_proc:
  1094. return rpc_proc_unavail;
  1095. }
  1096. EXPORT_SYMBOL_GPL(svc_generic_init_request);
  1097. /*
  1098. * Common routine for processing the RPC request.
  1099. */
  1100. static int
  1101. svc_process_common(struct svc_rqst *rqstp, struct kvec *argv, struct kvec *resv)
  1102. {
  1103. struct svc_program *progp;
  1104. const struct svc_procedure *procp = NULL;
  1105. struct svc_serv *serv = rqstp->rq_server;
  1106. struct svc_process_info process;
  1107. __be32 *statp;
  1108. u32 prog, vers;
  1109. __be32 auth_stat, rpc_stat;
  1110. int auth_res;
  1111. __be32 *reply_statp;
  1112. rpc_stat = rpc_success;
  1113. if (argv->iov_len < 6*4)
  1114. goto err_short_len;
  1115. /* Will be turned off by GSS integrity and privacy services */
  1116. set_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
  1117. /* Will be turned off only when NFSv4 Sessions are used */
  1118. set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
  1119. clear_bit(RQ_DROPME, &rqstp->rq_flags);
  1120. svc_putu32(resv, rqstp->rq_xid);
  1121. vers = svc_getnl(argv);
  1122. /* First words of reply: */
  1123. svc_putnl(resv, 1); /* REPLY */
  1124. if (vers != 2) /* RPC version number */
  1125. goto err_bad_rpc;
  1126. /* Save position in case we later decide to reject: */
  1127. reply_statp = resv->iov_base + resv->iov_len;
  1128. svc_putnl(resv, 0); /* ACCEPT */
  1129. rqstp->rq_prog = prog = svc_getnl(argv); /* program number */
  1130. rqstp->rq_vers = svc_getnl(argv); /* version number */
  1131. rqstp->rq_proc = svc_getnl(argv); /* procedure number */
  1132. for (progp = serv->sv_program; progp; progp = progp->pg_next)
  1133. if (prog == progp->pg_prog)
  1134. break;
  1135. /*
  1136. * Decode auth data, and add verifier to reply buffer.
  1137. * We do this before anything else in order to get a decent
  1138. * auth verifier.
  1139. */
  1140. auth_res = svc_authenticate(rqstp, &auth_stat);
  1141. /* Also give the program a chance to reject this call: */
  1142. if (auth_res == SVC_OK && progp) {
  1143. auth_stat = rpc_autherr_badcred;
  1144. auth_res = progp->pg_authenticate(rqstp);
  1145. }
  1146. if (auth_res != SVC_OK)
  1147. trace_svc_authenticate(rqstp, auth_res, auth_stat);
  1148. switch (auth_res) {
  1149. case SVC_OK:
  1150. break;
  1151. case SVC_GARBAGE:
  1152. goto err_garbage;
  1153. case SVC_SYSERR:
  1154. rpc_stat = rpc_system_err;
  1155. goto err_bad;
  1156. case SVC_DENIED:
  1157. goto err_bad_auth;
  1158. case SVC_CLOSE:
  1159. goto close;
  1160. case SVC_DROP:
  1161. goto dropit;
  1162. case SVC_COMPLETE:
  1163. goto sendit;
  1164. }
  1165. if (progp == NULL)
  1166. goto err_bad_prog;
  1167. rpc_stat = progp->pg_init_request(rqstp, progp, &process);
  1168. switch (rpc_stat) {
  1169. case rpc_success:
  1170. break;
  1171. case rpc_prog_unavail:
  1172. goto err_bad_prog;
  1173. case rpc_prog_mismatch:
  1174. goto err_bad_vers;
  1175. case rpc_proc_unavail:
  1176. goto err_bad_proc;
  1177. }
  1178. procp = rqstp->rq_procinfo;
  1179. /* Should this check go into the dispatcher? */
  1180. if (!procp || !procp->pc_func)
  1181. goto err_bad_proc;
  1182. /* Syntactic check complete */
  1183. serv->sv_stats->rpccnt++;
  1184. trace_svc_process(rqstp, progp->pg_name);
  1185. /* Build the reply header. */
  1186. statp = resv->iov_base +resv->iov_len;
  1187. svc_putnl(resv, RPC_SUCCESS);
  1188. /* un-reserve some of the out-queue now that we have a
  1189. * better idea of reply size
  1190. */
  1191. if (procp->pc_xdrressize)
  1192. svc_reserve_auth(rqstp, procp->pc_xdrressize<<2);
  1193. /* Call the function that processes the request. */
  1194. if (!process.dispatch) {
  1195. if (!svc_generic_dispatch(rqstp, statp))
  1196. goto release_dropit;
  1197. if (*statp == rpc_garbage_args)
  1198. goto err_garbage;
  1199. auth_stat = svc_get_autherr(rqstp, statp);
  1200. if (auth_stat != rpc_auth_ok)
  1201. goto err_release_bad_auth;
  1202. } else {
  1203. dprintk("svc: calling dispatcher\n");
  1204. if (!process.dispatch(rqstp, statp))
  1205. goto release_dropit; /* Release reply info */
  1206. }
  1207. /* Check RPC status result */
  1208. if (*statp != rpc_success)
  1209. resv->iov_len = ((void*)statp) - resv->iov_base + 4;
  1210. /* Release reply info */
  1211. if (procp->pc_release)
  1212. procp->pc_release(rqstp);
  1213. if (procp->pc_encode == NULL)
  1214. goto dropit;
  1215. sendit:
  1216. if (svc_authorise(rqstp))
  1217. goto close;
  1218. return 1; /* Caller can now send it */
  1219. release_dropit:
  1220. if (procp->pc_release)
  1221. procp->pc_release(rqstp);
  1222. dropit:
  1223. svc_authorise(rqstp); /* doesn't hurt to call this twice */
  1224. dprintk("svc: svc_process dropit\n");
  1225. return 0;
  1226. close:
  1227. if (rqstp->rq_xprt && test_bit(XPT_TEMP, &rqstp->rq_xprt->xpt_flags))
  1228. svc_close_xprt(rqstp->rq_xprt);
  1229. dprintk("svc: svc_process close\n");
  1230. return 0;
  1231. err_short_len:
  1232. svc_printk(rqstp, "short len %zd, dropping request\n",
  1233. argv->iov_len);
  1234. goto close;
  1235. err_bad_rpc:
  1236. serv->sv_stats->rpcbadfmt++;
  1237. svc_putnl(resv, 1); /* REJECT */
  1238. svc_putnl(resv, 0); /* RPC_MISMATCH */
  1239. svc_putnl(resv, 2); /* Only RPCv2 supported */
  1240. svc_putnl(resv, 2);
  1241. goto sendit;
  1242. err_release_bad_auth:
  1243. if (procp->pc_release)
  1244. procp->pc_release(rqstp);
  1245. err_bad_auth:
  1246. dprintk("svc: authentication failed (%d)\n", ntohl(auth_stat));
  1247. serv->sv_stats->rpcbadauth++;
  1248. /* Restore write pointer to location of accept status: */
  1249. xdr_ressize_check(rqstp, reply_statp);
  1250. svc_putnl(resv, 1); /* REJECT */
  1251. svc_putnl(resv, 1); /* AUTH_ERROR */
  1252. svc_putnl(resv, ntohl(auth_stat)); /* status */
  1253. goto sendit;
  1254. err_bad_prog:
  1255. dprintk("svc: unknown program %d\n", prog);
  1256. serv->sv_stats->rpcbadfmt++;
  1257. svc_putnl(resv, RPC_PROG_UNAVAIL);
  1258. goto sendit;
  1259. err_bad_vers:
  1260. svc_printk(rqstp, "unknown version (%d for prog %d, %s)\n",
  1261. rqstp->rq_vers, rqstp->rq_prog, progp->pg_name);
  1262. serv->sv_stats->rpcbadfmt++;
  1263. svc_putnl(resv, RPC_PROG_MISMATCH);
  1264. svc_putnl(resv, process.mismatch.lovers);
  1265. svc_putnl(resv, process.mismatch.hivers);
  1266. goto sendit;
  1267. err_bad_proc:
  1268. svc_printk(rqstp, "unknown procedure (%d)\n", rqstp->rq_proc);
  1269. serv->sv_stats->rpcbadfmt++;
  1270. svc_putnl(resv, RPC_PROC_UNAVAIL);
  1271. goto sendit;
  1272. err_garbage:
  1273. svc_printk(rqstp, "failed to decode args\n");
  1274. rpc_stat = rpc_garbage_args;
  1275. err_bad:
  1276. serv->sv_stats->rpcbadfmt++;
  1277. svc_putnl(resv, ntohl(rpc_stat));
  1278. goto sendit;
  1279. }
  1280. /*
  1281. * Process the RPC request.
  1282. */
  1283. int
  1284. svc_process(struct svc_rqst *rqstp)
  1285. {
  1286. struct kvec *argv = &rqstp->rq_arg.head[0];
  1287. struct kvec *resv = &rqstp->rq_res.head[0];
  1288. struct svc_serv *serv = rqstp->rq_server;
  1289. u32 dir;
  1290. /*
  1291. * Setup response xdr_buf.
  1292. * Initially it has just one page
  1293. */
  1294. rqstp->rq_next_page = &rqstp->rq_respages[1];
  1295. resv->iov_base = page_address(rqstp->rq_respages[0]);
  1296. resv->iov_len = 0;
  1297. rqstp->rq_res.pages = rqstp->rq_respages + 1;
  1298. rqstp->rq_res.len = 0;
  1299. rqstp->rq_res.page_base = 0;
  1300. rqstp->rq_res.page_len = 0;
  1301. rqstp->rq_res.buflen = PAGE_SIZE;
  1302. rqstp->rq_res.tail[0].iov_base = NULL;
  1303. rqstp->rq_res.tail[0].iov_len = 0;
  1304. dir = svc_getnl(argv);
  1305. if (dir != 0) {
  1306. /* direction != CALL */
  1307. svc_printk(rqstp, "bad direction %d, dropping request\n", dir);
  1308. serv->sv_stats->rpcbadfmt++;
  1309. goto out_drop;
  1310. }
  1311. /* Returns 1 for send, 0 for drop */
  1312. if (likely(svc_process_common(rqstp, argv, resv)))
  1313. return svc_send(rqstp);
  1314. out_drop:
  1315. svc_drop(rqstp);
  1316. return 0;
  1317. }
  1318. EXPORT_SYMBOL_GPL(svc_process);
  1319. #if defined(CONFIG_SUNRPC_BACKCHANNEL)
  1320. /*
  1321. * Process a backchannel RPC request that arrived over an existing
  1322. * outbound connection
  1323. */
  1324. int
  1325. bc_svc_process(struct svc_serv *serv, struct rpc_rqst *req,
  1326. struct svc_rqst *rqstp)
  1327. {
  1328. struct kvec *argv = &rqstp->rq_arg.head[0];
  1329. struct kvec *resv = &rqstp->rq_res.head[0];
  1330. struct rpc_task *task;
  1331. int proc_error;
  1332. int error;
  1333. dprintk("svc: %s(%p)\n", __func__, req);
  1334. /* Build the svc_rqst used by the common processing routine */
  1335. rqstp->rq_xid = req->rq_xid;
  1336. rqstp->rq_prot = req->rq_xprt->prot;
  1337. rqstp->rq_server = serv;
  1338. rqstp->rq_bc_net = req->rq_xprt->xprt_net;
  1339. rqstp->rq_addrlen = sizeof(req->rq_xprt->addr);
  1340. memcpy(&rqstp->rq_addr, &req->rq_xprt->addr, rqstp->rq_addrlen);
  1341. memcpy(&rqstp->rq_arg, &req->rq_rcv_buf, sizeof(rqstp->rq_arg));
  1342. memcpy(&rqstp->rq_res, &req->rq_snd_buf, sizeof(rqstp->rq_res));
  1343. /* Adjust the argument buffer length */
  1344. rqstp->rq_arg.len = req->rq_private_buf.len;
  1345. if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len) {
  1346. rqstp->rq_arg.head[0].iov_len = rqstp->rq_arg.len;
  1347. rqstp->rq_arg.page_len = 0;
  1348. } else if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len +
  1349. rqstp->rq_arg.page_len)
  1350. rqstp->rq_arg.page_len = rqstp->rq_arg.len -
  1351. rqstp->rq_arg.head[0].iov_len;
  1352. else
  1353. rqstp->rq_arg.len = rqstp->rq_arg.head[0].iov_len +
  1354. rqstp->rq_arg.page_len;
  1355. /* reset result send buffer "put" position */
  1356. resv->iov_len = 0;
  1357. /*
  1358. * Skip the next two words because they've already been
  1359. * processed in the transport
  1360. */
  1361. svc_getu32(argv); /* XID */
  1362. svc_getnl(argv); /* CALLDIR */
  1363. /* Parse and execute the bc call */
  1364. proc_error = svc_process_common(rqstp, argv, resv);
  1365. atomic_dec(&req->rq_xprt->bc_slot_count);
  1366. if (!proc_error) {
  1367. /* Processing error: drop the request */
  1368. xprt_free_bc_request(req);
  1369. error = -EINVAL;
  1370. goto out;
  1371. }
  1372. /* Finally, send the reply synchronously */
  1373. memcpy(&req->rq_snd_buf, &rqstp->rq_res, sizeof(req->rq_snd_buf));
  1374. task = rpc_run_bc_task(req);
  1375. if (IS_ERR(task)) {
  1376. error = PTR_ERR(task);
  1377. goto out;
  1378. }
  1379. WARN_ON_ONCE(atomic_read(&task->tk_count) != 1);
  1380. error = task->tk_status;
  1381. rpc_put_task(task);
  1382. out:
  1383. dprintk("svc: %s(), error=%d\n", __func__, error);
  1384. return error;
  1385. }
  1386. EXPORT_SYMBOL_GPL(bc_svc_process);
  1387. #endif /* CONFIG_SUNRPC_BACKCHANNEL */
  1388. /*
  1389. * Return (transport-specific) limit on the rpc payload.
  1390. */
  1391. u32 svc_max_payload(const struct svc_rqst *rqstp)
  1392. {
  1393. u32 max = rqstp->rq_xprt->xpt_class->xcl_max_payload;
  1394. if (rqstp->rq_server->sv_max_payload < max)
  1395. max = rqstp->rq_server->sv_max_payload;
  1396. return max;
  1397. }
  1398. EXPORT_SYMBOL_GPL(svc_max_payload);
  1399. /**
  1400. * svc_encode_read_payload - mark a range of bytes as a READ payload
  1401. * @rqstp: svc_rqst to operate on
  1402. * @offset: payload's byte offset in rqstp->rq_res
  1403. * @length: size of payload, in bytes
  1404. *
  1405. * Returns zero on success, or a negative errno if a permanent
  1406. * error occurred.
  1407. */
  1408. int svc_encode_read_payload(struct svc_rqst *rqstp, unsigned int offset,
  1409. unsigned int length)
  1410. {
  1411. return rqstp->rq_xprt->xpt_ops->xpo_read_payload(rqstp, offset, length);
  1412. }
  1413. EXPORT_SYMBOL_GPL(svc_encode_read_payload);
  1414. /**
  1415. * svc_fill_write_vector - Construct data argument for VFS write call
  1416. * @rqstp: svc_rqst to operate on
  1417. * @pages: list of pages containing data payload
  1418. * @first: buffer containing first section of write payload
  1419. * @total: total number of bytes of write payload
  1420. *
  1421. * Fills in rqstp::rq_vec, and returns the number of elements.
  1422. */
  1423. unsigned int svc_fill_write_vector(struct svc_rqst *rqstp, struct page **pages,
  1424. struct kvec *first, size_t total)
  1425. {
  1426. struct kvec *vec = rqstp->rq_vec;
  1427. unsigned int i;
  1428. /* Some types of transport can present the write payload
  1429. * entirely in rq_arg.pages. In this case, @first is empty.
  1430. */
  1431. i = 0;
  1432. if (first->iov_len) {
  1433. vec[i].iov_base = first->iov_base;
  1434. vec[i].iov_len = min_t(size_t, total, first->iov_len);
  1435. total -= vec[i].iov_len;
  1436. ++i;
  1437. }
  1438. while (total) {
  1439. vec[i].iov_base = page_address(*pages);
  1440. vec[i].iov_len = min_t(size_t, total, PAGE_SIZE);
  1441. total -= vec[i].iov_len;
  1442. ++i;
  1443. ++pages;
  1444. }
  1445. WARN_ON_ONCE(i > ARRAY_SIZE(rqstp->rq_vec));
  1446. return i;
  1447. }
  1448. EXPORT_SYMBOL_GPL(svc_fill_write_vector);
  1449. /**
  1450. * svc_fill_symlink_pathname - Construct pathname argument for VFS symlink call
  1451. * @rqstp: svc_rqst to operate on
  1452. * @first: buffer containing first section of pathname
  1453. * @p: buffer containing remaining section of pathname
  1454. * @total: total length of the pathname argument
  1455. *
  1456. * The VFS symlink API demands a NUL-terminated pathname in mapped memory.
  1457. * Returns pointer to a NUL-terminated string, or an ERR_PTR. Caller must free
  1458. * the returned string.
  1459. */
  1460. char *svc_fill_symlink_pathname(struct svc_rqst *rqstp, struct kvec *first,
  1461. void *p, size_t total)
  1462. {
  1463. size_t len, remaining;
  1464. char *result, *dst;
  1465. result = kmalloc(total + 1, GFP_KERNEL);
  1466. if (!result)
  1467. return ERR_PTR(-ESERVERFAULT);
  1468. dst = result;
  1469. remaining = total;
  1470. len = min_t(size_t, total, first->iov_len);
  1471. if (len) {
  1472. memcpy(dst, first->iov_base, len);
  1473. dst += len;
  1474. remaining -= len;
  1475. }
  1476. if (remaining) {
  1477. len = min_t(size_t, remaining, PAGE_SIZE);
  1478. memcpy(dst, p, len);
  1479. dst += len;
  1480. }
  1481. *dst = '\0';
  1482. /* Sanity check: Linux doesn't allow the pathname argument to
  1483. * contain a NUL byte.
  1484. */
  1485. if (strlen(result) != total) {
  1486. kfree(result);
  1487. return ERR_PTR(-EINVAL);
  1488. }
  1489. return result;
  1490. }
  1491. EXPORT_SYMBOL_GPL(svc_fill_symlink_pathname);