/drivers/scsi/scsi_lib.c

https://bitbucket.org/slukk/jb-tsm-kernel-4.2 · C · 2585 lines · 1551 code · 314 blank · 720 comment · 292 complexity · 8f8fde3f909bc78f876112220ae4335a MD5 · raw file

Large files are truncated click here to view the full file

  1. /*
  2. * scsi_lib.c Copyright (C) 1999 Eric Youngdale
  3. *
  4. * SCSI queueing library.
  5. * Initial versions: Eric Youngdale (eric@andante.org).
  6. * Based upon conversations with large numbers
  7. * of people at Linux Expo.
  8. */
  9. #include <linux/bio.h>
  10. #include <linux/bitops.h>
  11. #include <linux/blkdev.h>
  12. #include <linux/completion.h>
  13. #include <linux/kernel.h>
  14. #include <linux/mempool.h>
  15. #include <linux/slab.h>
  16. #include <linux/init.h>
  17. #include <linux/pci.h>
  18. #include <linux/delay.h>
  19. #include <linux/hardirq.h>
  20. #include <linux/scatterlist.h>
  21. #include <scsi/scsi.h>
  22. #include <scsi/scsi_cmnd.h>
  23. #include <scsi/scsi_dbg.h>
  24. #include <scsi/scsi_device.h>
  25. #include <scsi/scsi_driver.h>
  26. #include <scsi/scsi_eh.h>
  27. #include <scsi/scsi_host.h>
  28. #include "scsi_priv.h"
  29. #include "scsi_logging.h"
  30. #define SG_MEMPOOL_NR ARRAY_SIZE(scsi_sg_pools)
  31. #define SG_MEMPOOL_SIZE 2
  32. struct scsi_host_sg_pool {
  33. size_t size;
  34. char *name;
  35. struct kmem_cache *slab;
  36. mempool_t *pool;
  37. };
  38. #define SP(x) { x, "sgpool-" __stringify(x) }
  39. #if (SCSI_MAX_SG_SEGMENTS < 32)
  40. #error SCSI_MAX_SG_SEGMENTS is too small (must be 32 or greater)
  41. #endif
  42. static struct scsi_host_sg_pool scsi_sg_pools[] = {
  43. SP(8),
  44. SP(16),
  45. #if (SCSI_MAX_SG_SEGMENTS > 32)
  46. SP(32),
  47. #if (SCSI_MAX_SG_SEGMENTS > 64)
  48. SP(64),
  49. #if (SCSI_MAX_SG_SEGMENTS > 128)
  50. SP(128),
  51. #if (SCSI_MAX_SG_SEGMENTS > 256)
  52. #error SCSI_MAX_SG_SEGMENTS is too large (256 MAX)
  53. #endif
  54. #endif
  55. #endif
  56. #endif
  57. SP(SCSI_MAX_SG_SEGMENTS)
  58. };
  59. #undef SP
  60. struct kmem_cache *scsi_sdb_cache;
  61. /*
  62. * When to reinvoke queueing after a resource shortage. It's 3 msecs to
  63. * not change behaviour from the previous unplug mechanism, experimentation
  64. * may prove this needs changing.
  65. */
  66. #define SCSI_QUEUE_DELAY 3
  67. /*
  68. * Function: scsi_unprep_request()
  69. *
  70. * Purpose: Remove all preparation done for a request, including its
  71. * associated scsi_cmnd, so that it can be requeued.
  72. *
  73. * Arguments: req - request to unprepare
  74. *
  75. * Lock status: Assumed that no locks are held upon entry.
  76. *
  77. * Returns: Nothing.
  78. */
  79. static void scsi_unprep_request(struct request *req)
  80. {
  81. struct scsi_cmnd *cmd = req->special;
  82. blk_unprep_request(req);
  83. req->special = NULL;
  84. scsi_put_command(cmd);
  85. }
  86. /**
  87. * __scsi_queue_insert - private queue insertion
  88. * @cmd: The SCSI command being requeued
  89. * @reason: The reason for the requeue
  90. * @unbusy: Whether the queue should be unbusied
  91. *
  92. * This is a private queue insertion. The public interface
  93. * scsi_queue_insert() always assumes the queue should be unbusied
  94. * because it's always called before the completion. This function is
  95. * for a requeue after completion, which should only occur in this
  96. * file.
  97. */
  98. static int __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy)
  99. {
  100. struct Scsi_Host *host = cmd->device->host;
  101. struct scsi_device *device = cmd->device;
  102. struct scsi_target *starget = scsi_target(device);
  103. struct request_queue *q = device->request_queue;
  104. unsigned long flags;
  105. SCSI_LOG_MLQUEUE(1,
  106. printk("Inserting command %p into mlqueue\n", cmd));
  107. /*
  108. * Set the appropriate busy bit for the device/host.
  109. *
  110. * If the host/device isn't busy, assume that something actually
  111. * completed, and that we should be able to queue a command now.
  112. *
  113. * Note that the prior mid-layer assumption that any host could
  114. * always queue at least one command is now broken. The mid-layer
  115. * will implement a user specifiable stall (see
  116. * scsi_host.max_host_blocked and scsi_device.max_device_blocked)
  117. * if a command is requeued with no other commands outstanding
  118. * either for the device or for the host.
  119. */
  120. switch (reason) {
  121. case SCSI_MLQUEUE_HOST_BUSY:
  122. host->host_blocked = host->max_host_blocked;
  123. break;
  124. case SCSI_MLQUEUE_DEVICE_BUSY:
  125. device->device_blocked = device->max_device_blocked;
  126. break;
  127. case SCSI_MLQUEUE_TARGET_BUSY:
  128. starget->target_blocked = starget->max_target_blocked;
  129. break;
  130. }
  131. /*
  132. * Decrement the counters, since these commands are no longer
  133. * active on the host/device.
  134. */
  135. if (unbusy)
  136. scsi_device_unbusy(device);
  137. /*
  138. * Requeue this command. It will go before all other commands
  139. * that are already in the queue.
  140. */
  141. spin_lock_irqsave(q->queue_lock, flags);
  142. blk_requeue_request(q, cmd->request);
  143. spin_unlock_irqrestore(q->queue_lock, flags);
  144. kblockd_schedule_work(q, &device->requeue_work);
  145. return 0;
  146. }
  147. /*
  148. * Function: scsi_queue_insert()
  149. *
  150. * Purpose: Insert a command in the midlevel queue.
  151. *
  152. * Arguments: cmd - command that we are adding to queue.
  153. * reason - why we are inserting command to queue.
  154. *
  155. * Lock status: Assumed that lock is not held upon entry.
  156. *
  157. * Returns: Nothing.
  158. *
  159. * Notes: We do this for one of two cases. Either the host is busy
  160. * and it cannot accept any more commands for the time being,
  161. * or the device returned QUEUE_FULL and can accept no more
  162. * commands.
  163. * Notes: This could be called either from an interrupt context or a
  164. * normal process context.
  165. */
  166. int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
  167. {
  168. return __scsi_queue_insert(cmd, reason, 1);
  169. }
  170. /**
  171. * scsi_execute - insert request and wait for the result
  172. * @sdev: scsi device
  173. * @cmd: scsi command
  174. * @data_direction: data direction
  175. * @buffer: data buffer
  176. * @bufflen: len of buffer
  177. * @sense: optional sense buffer
  178. * @timeout: request timeout in seconds
  179. * @retries: number of times to retry request
  180. * @flags: or into request flags;
  181. * @resid: optional residual length
  182. *
  183. * returns the req->errors value which is the scsi_cmnd result
  184. * field.
  185. */
  186. int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
  187. int data_direction, void *buffer, unsigned bufflen,
  188. unsigned char *sense, int timeout, int retries, int flags,
  189. int *resid)
  190. {
  191. struct request *req;
  192. int write = (data_direction == DMA_TO_DEVICE);
  193. int ret = DRIVER_ERROR << 24;
  194. req = blk_get_request(sdev->request_queue, write, __GFP_WAIT);
  195. if (!req)
  196. return ret;
  197. if (bufflen && blk_rq_map_kern(sdev->request_queue, req,
  198. buffer, bufflen, __GFP_WAIT))
  199. goto out;
  200. req->cmd_len = COMMAND_SIZE(cmd[0]);
  201. memcpy(req->cmd, cmd, req->cmd_len);
  202. req->sense = sense;
  203. req->sense_len = 0;
  204. req->retries = retries;
  205. req->timeout = timeout;
  206. req->cmd_type = REQ_TYPE_BLOCK_PC;
  207. req->cmd_flags |= flags | REQ_QUIET | REQ_PREEMPT;
  208. /*
  209. * head injection *required* here otherwise quiesce won't work
  210. */
  211. blk_execute_rq(req->q, NULL, req, 1);
  212. /*
  213. * Some devices (USB mass-storage in particular) may transfer
  214. * garbage data together with a residue indicating that the data
  215. * is invalid. Prevent the garbage from being misinterpreted
  216. * and prevent security leaks by zeroing out the excess data.
  217. */
  218. if (unlikely(req->resid_len > 0 && req->resid_len <= bufflen))
  219. memset(buffer + (bufflen - req->resid_len), 0, req->resid_len);
  220. if (resid)
  221. *resid = req->resid_len;
  222. ret = req->errors;
  223. out:
  224. blk_put_request(req);
  225. return ret;
  226. }
  227. EXPORT_SYMBOL(scsi_execute);
  228. int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd,
  229. int data_direction, void *buffer, unsigned bufflen,
  230. struct scsi_sense_hdr *sshdr, int timeout, int retries,
  231. int *resid)
  232. {
  233. char *sense = NULL;
  234. int result;
  235. if (sshdr) {
  236. sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO);
  237. if (!sense)
  238. return DRIVER_ERROR << 24;
  239. }
  240. result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen,
  241. sense, timeout, retries, 0, resid);
  242. if (sshdr)
  243. scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr);
  244. kfree(sense);
  245. return result;
  246. }
  247. EXPORT_SYMBOL(scsi_execute_req);
  248. /*
  249. * Function: scsi_init_cmd_errh()
  250. *
  251. * Purpose: Initialize cmd fields related to error handling.
  252. *
  253. * Arguments: cmd - command that is ready to be queued.
  254. *
  255. * Notes: This function has the job of initializing a number of
  256. * fields related to error handling. Typically this will
  257. * be called once for each command, as required.
  258. */
  259. static void scsi_init_cmd_errh(struct scsi_cmnd *cmd)
  260. {
  261. cmd->serial_number = 0;
  262. scsi_set_resid(cmd, 0);
  263. memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
  264. if (cmd->cmd_len == 0)
  265. cmd->cmd_len = scsi_command_size(cmd->cmnd);
  266. }
  267. void scsi_device_unbusy(struct scsi_device *sdev)
  268. {
  269. struct Scsi_Host *shost = sdev->host;
  270. struct scsi_target *starget = scsi_target(sdev);
  271. unsigned long flags;
  272. spin_lock_irqsave(shost->host_lock, flags);
  273. shost->host_busy--;
  274. starget->target_busy--;
  275. if (unlikely(scsi_host_in_recovery(shost) &&
  276. (shost->host_failed || shost->host_eh_scheduled)))
  277. scsi_eh_wakeup(shost);
  278. spin_unlock(shost->host_lock);
  279. spin_lock(sdev->request_queue->queue_lock);
  280. sdev->device_busy--;
  281. spin_unlock_irqrestore(sdev->request_queue->queue_lock, flags);
  282. }
  283. /*
  284. * Called for single_lun devices on IO completion. Clear starget_sdev_user,
  285. * and call blk_run_queue for all the scsi_devices on the target -
  286. * including current_sdev first.
  287. *
  288. * Called with *no* scsi locks held.
  289. */
  290. static void scsi_single_lun_run(struct scsi_device *current_sdev)
  291. {
  292. struct Scsi_Host *shost = current_sdev->host;
  293. struct scsi_device *sdev, *tmp;
  294. struct scsi_target *starget = scsi_target(current_sdev);
  295. unsigned long flags;
  296. spin_lock_irqsave(shost->host_lock, flags);
  297. starget->starget_sdev_user = NULL;
  298. spin_unlock_irqrestore(shost->host_lock, flags);
  299. /*
  300. * Call blk_run_queue for all LUNs on the target, starting with
  301. * current_sdev. We race with others (to set starget_sdev_user),
  302. * but in most cases, we will be first. Ideally, each LU on the
  303. * target would get some limited time or requests on the target.
  304. */
  305. blk_run_queue(current_sdev->request_queue);
  306. spin_lock_irqsave(shost->host_lock, flags);
  307. if (starget->starget_sdev_user)
  308. goto out;
  309. list_for_each_entry_safe(sdev, tmp, &starget->devices,
  310. same_target_siblings) {
  311. if (sdev == current_sdev)
  312. continue;
  313. if (scsi_device_get(sdev))
  314. continue;
  315. spin_unlock_irqrestore(shost->host_lock, flags);
  316. blk_run_queue(sdev->request_queue);
  317. spin_lock_irqsave(shost->host_lock, flags);
  318. scsi_device_put(sdev);
  319. }
  320. out:
  321. spin_unlock_irqrestore(shost->host_lock, flags);
  322. }
  323. static inline int scsi_device_is_busy(struct scsi_device *sdev)
  324. {
  325. if (sdev->device_busy >= sdev->queue_depth || sdev->device_blocked)
  326. return 1;
  327. return 0;
  328. }
  329. static inline int scsi_target_is_busy(struct scsi_target *starget)
  330. {
  331. return ((starget->can_queue > 0 &&
  332. starget->target_busy >= starget->can_queue) ||
  333. starget->target_blocked);
  334. }
  335. static inline int scsi_host_is_busy(struct Scsi_Host *shost)
  336. {
  337. if ((shost->can_queue > 0 && shost->host_busy >= shost->can_queue) ||
  338. shost->host_blocked || shost->host_self_blocked)
  339. return 1;
  340. return 0;
  341. }
  342. /*
  343. * Function: scsi_run_queue()
  344. *
  345. * Purpose: Select a proper request queue to serve next
  346. *
  347. * Arguments: q - last request's queue
  348. *
  349. * Returns: Nothing
  350. *
  351. * Notes: The previous command was completely finished, start
  352. * a new one if possible.
  353. */
  354. static void scsi_run_queue(struct request_queue *q)
  355. {
  356. struct scsi_device *sdev = q->queuedata;
  357. struct Scsi_Host *shost;
  358. LIST_HEAD(starved_list);
  359. unsigned long flags;
  360. /* if the device is dead, sdev will be NULL, so no queue to run */
  361. if (!sdev)
  362. return;
  363. shost = sdev->host;
  364. if (scsi_target(sdev)->single_lun)
  365. scsi_single_lun_run(sdev);
  366. spin_lock_irqsave(shost->host_lock, flags);
  367. list_splice_init(&shost->starved_list, &starved_list);
  368. while (!list_empty(&starved_list)) {
  369. /*
  370. * As long as shost is accepting commands and we have
  371. * starved queues, call blk_run_queue. scsi_request_fn
  372. * drops the queue_lock and can add us back to the
  373. * starved_list.
  374. *
  375. * host_lock protects the starved_list and starved_entry.
  376. * scsi_request_fn must get the host_lock before checking
  377. * or modifying starved_list or starved_entry.
  378. */
  379. if (scsi_host_is_busy(shost))
  380. break;
  381. sdev = list_entry(starved_list.next,
  382. struct scsi_device, starved_entry);
  383. list_del_init(&sdev->starved_entry);
  384. if (scsi_target_is_busy(scsi_target(sdev))) {
  385. list_move_tail(&sdev->starved_entry,
  386. &shost->starved_list);
  387. continue;
  388. }
  389. spin_unlock(shost->host_lock);
  390. spin_lock(sdev->request_queue->queue_lock);
  391. __blk_run_queue(sdev->request_queue);
  392. spin_unlock(sdev->request_queue->queue_lock);
  393. spin_lock(shost->host_lock);
  394. }
  395. /* put any unprocessed entries back */
  396. list_splice(&starved_list, &shost->starved_list);
  397. spin_unlock_irqrestore(shost->host_lock, flags);
  398. blk_run_queue(q);
  399. }
  400. void scsi_requeue_run_queue(struct work_struct *work)
  401. {
  402. struct scsi_device *sdev;
  403. struct request_queue *q;
  404. sdev = container_of(work, struct scsi_device, requeue_work);
  405. q = sdev->request_queue;
  406. scsi_run_queue(q);
  407. }
  408. /*
  409. * Function: scsi_requeue_command()
  410. *
  411. * Purpose: Handle post-processing of completed commands.
  412. *
  413. * Arguments: q - queue to operate on
  414. * cmd - command that may need to be requeued.
  415. *
  416. * Returns: Nothing
  417. *
  418. * Notes: After command completion, there may be blocks left
  419. * over which weren't finished by the previous command
  420. * this can be for a number of reasons - the main one is
  421. * I/O errors in the middle of the request, in which case
  422. * we need to request the blocks that come after the bad
  423. * sector.
  424. * Notes: Upon return, cmd is a stale pointer.
  425. */
  426. static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd)
  427. {
  428. struct request *req = cmd->request;
  429. unsigned long flags;
  430. spin_lock_irqsave(q->queue_lock, flags);
  431. scsi_unprep_request(req);
  432. blk_requeue_request(q, req);
  433. spin_unlock_irqrestore(q->queue_lock, flags);
  434. scsi_run_queue(q);
  435. }
  436. void scsi_next_command(struct scsi_cmnd *cmd)
  437. {
  438. struct scsi_device *sdev = cmd->device;
  439. struct request_queue *q = sdev->request_queue;
  440. /* need to hold a reference on the device before we let go of the cmd */
  441. get_device(&sdev->sdev_gendev);
  442. scsi_put_command(cmd);
  443. scsi_run_queue(q);
  444. /* ok to remove device now */
  445. put_device(&sdev->sdev_gendev);
  446. }
  447. void scsi_run_host_queues(struct Scsi_Host *shost)
  448. {
  449. struct scsi_device *sdev;
  450. shost_for_each_device(sdev, shost)
  451. scsi_run_queue(sdev->request_queue);
  452. }
  453. static void __scsi_release_buffers(struct scsi_cmnd *, int);
  454. /*
  455. * Function: scsi_end_request()
  456. *
  457. * Purpose: Post-processing of completed commands (usually invoked at end
  458. * of upper level post-processing and scsi_io_completion).
  459. *
  460. * Arguments: cmd - command that is complete.
  461. * error - 0 if I/O indicates success, < 0 for I/O error.
  462. * bytes - number of bytes of completed I/O
  463. * requeue - indicates whether we should requeue leftovers.
  464. *
  465. * Lock status: Assumed that lock is not held upon entry.
  466. *
  467. * Returns: cmd if requeue required, NULL otherwise.
  468. *
  469. * Notes: This is called for block device requests in order to
  470. * mark some number of sectors as complete.
  471. *
  472. * We are guaranteeing that the request queue will be goosed
  473. * at some point during this call.
  474. * Notes: If cmd was requeued, upon return it will be a stale pointer.
  475. */
  476. static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int error,
  477. int bytes, int requeue)
  478. {
  479. struct request_queue *q = cmd->device->request_queue;
  480. struct request *req = cmd->request;
  481. /*
  482. * If there are blocks left over at the end, set up the command
  483. * to queue the remainder of them.
  484. */
  485. if (blk_end_request(req, error, bytes)) {
  486. /* kill remainder if no retrys */
  487. if (error && scsi_noretry_cmd(cmd))
  488. blk_end_request_all(req, error);
  489. else {
  490. if (requeue) {
  491. /*
  492. * Bleah. Leftovers again. Stick the
  493. * leftovers in the front of the
  494. * queue, and goose the queue again.
  495. */
  496. scsi_release_buffers(cmd);
  497. scsi_requeue_command(q, cmd);
  498. cmd = NULL;
  499. }
  500. return cmd;
  501. }
  502. }
  503. /*
  504. * This will goose the queue request function at the end, so we don't
  505. * need to worry about launching another command.
  506. */
  507. __scsi_release_buffers(cmd, 0);
  508. scsi_next_command(cmd);
  509. return NULL;
  510. }
  511. static inline unsigned int scsi_sgtable_index(unsigned short nents)
  512. {
  513. unsigned int index;
  514. BUG_ON(nents > SCSI_MAX_SG_SEGMENTS);
  515. if (nents <= 8)
  516. index = 0;
  517. else
  518. index = get_count_order(nents) - 3;
  519. return index;
  520. }
  521. static void scsi_sg_free(struct scatterlist *sgl, unsigned int nents)
  522. {
  523. struct scsi_host_sg_pool *sgp;
  524. sgp = scsi_sg_pools + scsi_sgtable_index(nents);
  525. mempool_free(sgl, sgp->pool);
  526. }
  527. static struct scatterlist *scsi_sg_alloc(unsigned int nents, gfp_t gfp_mask)
  528. {
  529. struct scsi_host_sg_pool *sgp;
  530. sgp = scsi_sg_pools + scsi_sgtable_index(nents);
  531. return mempool_alloc(sgp->pool, gfp_mask);
  532. }
  533. static int scsi_alloc_sgtable(struct scsi_data_buffer *sdb, int nents,
  534. gfp_t gfp_mask)
  535. {
  536. int ret;
  537. BUG_ON(!nents);
  538. ret = __sg_alloc_table(&sdb->table, nents, SCSI_MAX_SG_SEGMENTS,
  539. gfp_mask, scsi_sg_alloc);
  540. if (unlikely(ret))
  541. __sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS,
  542. scsi_sg_free);
  543. return ret;
  544. }
  545. static void scsi_free_sgtable(struct scsi_data_buffer *sdb)
  546. {
  547. __sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS, scsi_sg_free);
  548. }
  549. static void __scsi_release_buffers(struct scsi_cmnd *cmd, int do_bidi_check)
  550. {
  551. if (cmd->sdb.table.nents)
  552. scsi_free_sgtable(&cmd->sdb);
  553. memset(&cmd->sdb, 0, sizeof(cmd->sdb));
  554. if (do_bidi_check && scsi_bidi_cmnd(cmd)) {
  555. struct scsi_data_buffer *bidi_sdb =
  556. cmd->request->next_rq->special;
  557. scsi_free_sgtable(bidi_sdb);
  558. kmem_cache_free(scsi_sdb_cache, bidi_sdb);
  559. cmd->request->next_rq->special = NULL;
  560. }
  561. if (scsi_prot_sg_count(cmd))
  562. scsi_free_sgtable(cmd->prot_sdb);
  563. }
  564. /*
  565. * Function: scsi_release_buffers()
  566. *
  567. * Purpose: Completion processing for block device I/O requests.
  568. *
  569. * Arguments: cmd - command that we are bailing.
  570. *
  571. * Lock status: Assumed that no lock is held upon entry.
  572. *
  573. * Returns: Nothing
  574. *
  575. * Notes: In the event that an upper level driver rejects a
  576. * command, we must release resources allocated during
  577. * the __init_io() function. Primarily this would involve
  578. * the scatter-gather table, and potentially any bounce
  579. * buffers.
  580. */
  581. void scsi_release_buffers(struct scsi_cmnd *cmd)
  582. {
  583. __scsi_release_buffers(cmd, 1);
  584. }
  585. EXPORT_SYMBOL(scsi_release_buffers);
  586. static int __scsi_error_from_host_byte(struct scsi_cmnd *cmd, int result)
  587. {
  588. int error = 0;
  589. switch(host_byte(result)) {
  590. case DID_TRANSPORT_FAILFAST:
  591. error = -ENOLINK;
  592. break;
  593. case DID_TARGET_FAILURE:
  594. cmd->result |= (DID_OK << 16);
  595. error = -EREMOTEIO;
  596. break;
  597. case DID_NEXUS_FAILURE:
  598. cmd->result |= (DID_OK << 16);
  599. error = -EBADE;
  600. break;
  601. default:
  602. error = -EIO;
  603. break;
  604. }
  605. return error;
  606. }
  607. /*
  608. * Function: scsi_io_completion()
  609. *
  610. * Purpose: Completion processing for block device I/O requests.
  611. *
  612. * Arguments: cmd - command that is finished.
  613. *
  614. * Lock status: Assumed that no lock is held upon entry.
  615. *
  616. * Returns: Nothing
  617. *
  618. * Notes: This function is matched in terms of capabilities to
  619. * the function that created the scatter-gather list.
  620. * In other words, if there are no bounce buffers
  621. * (the normal case for most drivers), we don't need
  622. * the logic to deal with cleaning up afterwards.
  623. *
  624. * We must call scsi_end_request(). This will finish off
  625. * the specified number of sectors. If we are done, the
  626. * command block will be released and the queue function
  627. * will be goosed. If we are not done then we have to
  628. * figure out what to do next:
  629. *
  630. * a) We can call scsi_requeue_command(). The request
  631. * will be unprepared and put back on the queue. Then
  632. * a new command will be created for it. This should
  633. * be used if we made forward progress, or if we want
  634. * to switch from READ(10) to READ(6) for example.
  635. *
  636. * b) We can call scsi_queue_insert(). The request will
  637. * be put back on the queue and retried using the same
  638. * command as before, possibly after a delay.
  639. *
  640. * c) We can call blk_end_request() with -EIO to fail
  641. * the remainder of the request.
  642. */
  643. void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
  644. {
  645. int result = cmd->result;
  646. struct request_queue *q = cmd->device->request_queue;
  647. struct request *req = cmd->request;
  648. int error = 0;
  649. struct scsi_sense_hdr sshdr;
  650. int sense_valid = 0;
  651. int sense_deferred = 0;
  652. enum {ACTION_FAIL, ACTION_REPREP, ACTION_RETRY,
  653. ACTION_DELAYED_RETRY} action;
  654. char *description = NULL;
  655. if (result) {
  656. sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
  657. if (sense_valid)
  658. sense_deferred = scsi_sense_is_deferred(&sshdr);
  659. }
  660. if (req->cmd_type == REQ_TYPE_BLOCK_PC) { /* SG_IO ioctl from block level */
  661. req->errors = result;
  662. if (result) {
  663. if (sense_valid && req->sense) {
  664. /*
  665. * SG_IO wants current and deferred errors
  666. */
  667. int len = 8 + cmd->sense_buffer[7];
  668. if (len > SCSI_SENSE_BUFFERSIZE)
  669. len = SCSI_SENSE_BUFFERSIZE;
  670. memcpy(req->sense, cmd->sense_buffer, len);
  671. req->sense_len = len;
  672. }
  673. if (!sense_deferred)
  674. error = __scsi_error_from_host_byte(cmd, result);
  675. }
  676. req->resid_len = scsi_get_resid(cmd);
  677. if (scsi_bidi_cmnd(cmd)) {
  678. /*
  679. * Bidi commands Must be complete as a whole,
  680. * both sides at once.
  681. */
  682. req->next_rq->resid_len = scsi_in(cmd)->resid;
  683. scsi_release_buffers(cmd);
  684. blk_end_request_all(req, 0);
  685. scsi_next_command(cmd);
  686. return;
  687. }
  688. }
  689. /* no bidi support for !REQ_TYPE_BLOCK_PC yet */
  690. BUG_ON(blk_bidi_rq(req));
  691. /*
  692. * Next deal with any sectors which we were able to correctly
  693. * handle.
  694. */
  695. SCSI_LOG_HLCOMPLETE(1, printk("%u sectors total, "
  696. "%d bytes done.\n",
  697. blk_rq_sectors(req), good_bytes));
  698. /*
  699. * Recovered errors need reporting, but they're always treated
  700. * as success, so fiddle the result code here. For BLOCK_PC
  701. * we already took a copy of the original into rq->errors which
  702. * is what gets returned to the user
  703. */
  704. if (sense_valid && (sshdr.sense_key == RECOVERED_ERROR)) {
  705. /* if ATA PASS-THROUGH INFORMATION AVAILABLE skip
  706. * print since caller wants ATA registers. Only occurs on
  707. * SCSI ATA PASS_THROUGH commands when CK_COND=1
  708. */
  709. if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d))
  710. ;
  711. else if (!(req->cmd_flags & REQ_QUIET))
  712. scsi_print_sense("", cmd);
  713. result = 0;
  714. /* BLOCK_PC may have set error */
  715. error = 0;
  716. }
  717. /*
  718. * A number of bytes were successfully read. If there
  719. * are leftovers and there is some kind of error
  720. * (result != 0), retry the rest.
  721. */
  722. if (scsi_end_request(cmd, error, good_bytes, result == 0) == NULL)
  723. return;
  724. error = __scsi_error_from_host_byte(cmd, result);
  725. if (host_byte(result) == DID_RESET) {
  726. /* Third party bus reset or reset for error recovery
  727. * reasons. Just retry the command and see what
  728. * happens.
  729. */
  730. action = ACTION_RETRY;
  731. } else if (sense_valid && !sense_deferred) {
  732. switch (sshdr.sense_key) {
  733. case UNIT_ATTENTION:
  734. if (cmd->device->removable) {
  735. /* Detected disc change. Set a bit
  736. * and quietly refuse further access.
  737. */
  738. cmd->device->changed = 1;
  739. description = "Media Changed";
  740. action = ACTION_FAIL;
  741. } else {
  742. /* Must have been a power glitch, or a
  743. * bus reset. Could not have been a
  744. * media change, so we just retry the
  745. * command and see what happens.
  746. */
  747. action = ACTION_RETRY;
  748. }
  749. break;
  750. case ILLEGAL_REQUEST:
  751. /* If we had an ILLEGAL REQUEST returned, then
  752. * we may have performed an unsupported
  753. * command. The only thing this should be
  754. * would be a ten byte read where only a six
  755. * byte read was supported. Also, on a system
  756. * where READ CAPACITY failed, we may have
  757. * read past the end of the disk.
  758. */
  759. if ((cmd->device->use_10_for_rw &&
  760. sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&
  761. (cmd->cmnd[0] == READ_10 ||
  762. cmd->cmnd[0] == WRITE_10)) {
  763. /* This will issue a new 6-byte command. */
  764. cmd->device->use_10_for_rw = 0;
  765. action = ACTION_REPREP;
  766. } else if (sshdr.asc == 0x10) /* DIX */ {
  767. description = "Host Data Integrity Failure";
  768. action = ACTION_FAIL;
  769. error = -EILSEQ;
  770. /* INVALID COMMAND OPCODE or INVALID FIELD IN CDB */
  771. } else if ((sshdr.asc == 0x20 || sshdr.asc == 0x24) &&
  772. (cmd->cmnd[0] == UNMAP ||
  773. cmd->cmnd[0] == WRITE_SAME_16 ||
  774. cmd->cmnd[0] == WRITE_SAME)) {
  775. description = "Discard failure";
  776. action = ACTION_FAIL;
  777. } else
  778. action = ACTION_FAIL;
  779. break;
  780. case ABORTED_COMMAND:
  781. action = ACTION_FAIL;
  782. if (sshdr.asc == 0x10) { /* DIF */
  783. description = "Target Data Integrity Failure";
  784. error = -EILSEQ;
  785. }
  786. break;
  787. case NOT_READY:
  788. /* If the device is in the process of becoming
  789. * ready, or has a temporary blockage, retry.
  790. */
  791. if (sshdr.asc == 0x04) {
  792. switch (sshdr.ascq) {
  793. case 0x01: /* becoming ready */
  794. case 0x04: /* format in progress */
  795. case 0x05: /* rebuild in progress */
  796. case 0x06: /* recalculation in progress */
  797. case 0x07: /* operation in progress */
  798. case 0x08: /* Long write in progress */
  799. case 0x09: /* self test in progress */
  800. case 0x14: /* space allocation in progress */
  801. action = ACTION_DELAYED_RETRY;
  802. break;
  803. default:
  804. description = "Device not ready";
  805. action = ACTION_FAIL;
  806. break;
  807. }
  808. } else {
  809. description = "Device not ready";
  810. action = ACTION_FAIL;
  811. }
  812. break;
  813. case VOLUME_OVERFLOW:
  814. /* See SSC3rXX or current. */
  815. action = ACTION_FAIL;
  816. break;
  817. default:
  818. description = "Unhandled sense code";
  819. action = ACTION_FAIL;
  820. break;
  821. }
  822. } else {
  823. description = "Unhandled error code";
  824. action = ACTION_FAIL;
  825. }
  826. switch (action) {
  827. case ACTION_FAIL:
  828. /* Give up and fail the remainder of the request */
  829. scsi_release_buffers(cmd);
  830. if (!(req->cmd_flags & REQ_QUIET)) {
  831. if (description)
  832. scmd_printk(KERN_INFO, cmd, "%s\n",
  833. description);
  834. scsi_print_result(cmd);
  835. if (driver_byte(result) & DRIVER_SENSE)
  836. scsi_print_sense("", cmd);
  837. scsi_print_command(cmd);
  838. }
  839. if (blk_end_request_err(req, error))
  840. scsi_requeue_command(q, cmd);
  841. else
  842. scsi_next_command(cmd);
  843. break;
  844. case ACTION_REPREP:
  845. /* Unprep the request and put it back at the head of the queue.
  846. * A new command will be prepared and issued.
  847. */
  848. scsi_release_buffers(cmd);
  849. scsi_requeue_command(q, cmd);
  850. break;
  851. case ACTION_RETRY:
  852. /* Retry the same command immediately */
  853. __scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, 0);
  854. break;
  855. case ACTION_DELAYED_RETRY:
  856. /* Retry the same command after a delay */
  857. __scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, 0);
  858. break;
  859. }
  860. }
  861. static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb,
  862. gfp_t gfp_mask)
  863. {
  864. int count;
  865. /*
  866. * If sg table allocation fails, requeue request later.
  867. */
  868. if (unlikely(scsi_alloc_sgtable(sdb, req->nr_phys_segments,
  869. gfp_mask))) {
  870. return BLKPREP_DEFER;
  871. }
  872. req->buffer = NULL;
  873. /*
  874. * Next, walk the list, and fill in the addresses and sizes of
  875. * each segment.
  876. */
  877. count = blk_rq_map_sg(req->q, req, sdb->table.sgl);
  878. BUG_ON(count > sdb->table.nents);
  879. sdb->table.nents = count;
  880. sdb->length = blk_rq_bytes(req);
  881. return BLKPREP_OK;
  882. }
  883. /*
  884. * Function: scsi_init_io()
  885. *
  886. * Purpose: SCSI I/O initialize function.
  887. *
  888. * Arguments: cmd - Command descriptor we wish to initialize
  889. *
  890. * Returns: 0 on success
  891. * BLKPREP_DEFER if the failure is retryable
  892. * BLKPREP_KILL if the failure is fatal
  893. */
  894. int scsi_init_io(struct scsi_cmnd *cmd, gfp_t gfp_mask)
  895. {
  896. struct request *rq = cmd->request;
  897. int error = scsi_init_sgtable(rq, &cmd->sdb, gfp_mask);
  898. if (error)
  899. goto err_exit;
  900. if (blk_bidi_rq(rq)) {
  901. struct scsi_data_buffer *bidi_sdb = kmem_cache_zalloc(
  902. scsi_sdb_cache, GFP_ATOMIC);
  903. if (!bidi_sdb) {
  904. error = BLKPREP_DEFER;
  905. goto err_exit;
  906. }
  907. rq->next_rq->special = bidi_sdb;
  908. error = scsi_init_sgtable(rq->next_rq, bidi_sdb, GFP_ATOMIC);
  909. if (error)
  910. goto err_exit;
  911. }
  912. if (blk_integrity_rq(rq)) {
  913. struct scsi_data_buffer *prot_sdb = cmd->prot_sdb;
  914. int ivecs, count;
  915. BUG_ON(prot_sdb == NULL);
  916. ivecs = blk_rq_count_integrity_sg(rq->q, rq->bio);
  917. if (scsi_alloc_sgtable(prot_sdb, ivecs, gfp_mask)) {
  918. error = BLKPREP_DEFER;
  919. goto err_exit;
  920. }
  921. count = blk_rq_map_integrity_sg(rq->q, rq->bio,
  922. prot_sdb->table.sgl);
  923. BUG_ON(unlikely(count > ivecs));
  924. BUG_ON(unlikely(count > queue_max_integrity_segments(rq->q)));
  925. cmd->prot_sdb = prot_sdb;
  926. cmd->prot_sdb->table.nents = count;
  927. }
  928. return BLKPREP_OK ;
  929. err_exit:
  930. scsi_release_buffers(cmd);
  931. cmd->request->special = NULL;
  932. scsi_put_command(cmd);
  933. return error;
  934. }
  935. EXPORT_SYMBOL(scsi_init_io);
  936. static struct scsi_cmnd *scsi_get_cmd_from_req(struct scsi_device *sdev,
  937. struct request *req)
  938. {
  939. struct scsi_cmnd *cmd;
  940. if (!req->special) {
  941. cmd = scsi_get_command(sdev, GFP_ATOMIC);
  942. if (unlikely(!cmd))
  943. return NULL;
  944. req->special = cmd;
  945. } else {
  946. cmd = req->special;
  947. }
  948. /* pull a tag out of the request if we have one */
  949. cmd->tag = req->tag;
  950. cmd->request = req;
  951. cmd->cmnd = req->cmd;
  952. cmd->prot_op = SCSI_PROT_NORMAL;
  953. return cmd;
  954. }
  955. int scsi_setup_blk_pc_cmnd(struct scsi_device *sdev, struct request *req)
  956. {
  957. struct scsi_cmnd *cmd;
  958. int ret = scsi_prep_state_check(sdev, req);
  959. if (ret != BLKPREP_OK)
  960. return ret;
  961. cmd = scsi_get_cmd_from_req(sdev, req);
  962. if (unlikely(!cmd))
  963. return BLKPREP_DEFER;
  964. /*
  965. * BLOCK_PC requests may transfer data, in which case they must
  966. * a bio attached to them. Or they might contain a SCSI command
  967. * that does not transfer data, in which case they may optionally
  968. * submit a request without an attached bio.
  969. */
  970. if (req->bio) {
  971. int ret;
  972. BUG_ON(!req->nr_phys_segments);
  973. ret = scsi_init_io(cmd, GFP_ATOMIC);
  974. if (unlikely(ret))
  975. return ret;
  976. } else {
  977. BUG_ON(blk_rq_bytes(req));
  978. memset(&cmd->sdb, 0, sizeof(cmd->sdb));
  979. req->buffer = NULL;
  980. }
  981. cmd->cmd_len = req->cmd_len;
  982. if (!blk_rq_bytes(req))
  983. cmd->sc_data_direction = DMA_NONE;
  984. else if (rq_data_dir(req) == WRITE)
  985. cmd->sc_data_direction = DMA_TO_DEVICE;
  986. else
  987. cmd->sc_data_direction = DMA_FROM_DEVICE;
  988. cmd->transfersize = blk_rq_bytes(req);
  989. cmd->allowed = req->retries;
  990. return BLKPREP_OK;
  991. }
  992. EXPORT_SYMBOL(scsi_setup_blk_pc_cmnd);
  993. /*
  994. * Setup a REQ_TYPE_FS command. These are simple read/write request
  995. * from filesystems that still need to be translated to SCSI CDBs from
  996. * the ULD.
  997. */
  998. int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req)
  999. {
  1000. struct scsi_cmnd *cmd;
  1001. int ret = scsi_prep_state_check(sdev, req);
  1002. if (ret != BLKPREP_OK)
  1003. return ret;
  1004. if (unlikely(sdev->scsi_dh_data && sdev->scsi_dh_data->scsi_dh
  1005. && sdev->scsi_dh_data->scsi_dh->prep_fn)) {
  1006. ret = sdev->scsi_dh_data->scsi_dh->prep_fn(sdev, req);
  1007. if (ret != BLKPREP_OK)
  1008. return ret;
  1009. }
  1010. /*
  1011. * Filesystem requests must transfer data.
  1012. */
  1013. BUG_ON(!req->nr_phys_segments);
  1014. cmd = scsi_get_cmd_from_req(sdev, req);
  1015. if (unlikely(!cmd))
  1016. return BLKPREP_DEFER;
  1017. memset(cmd->cmnd, 0, BLK_MAX_CDB);
  1018. return scsi_init_io(cmd, GFP_ATOMIC);
  1019. }
  1020. EXPORT_SYMBOL(scsi_setup_fs_cmnd);
  1021. int scsi_prep_state_check(struct scsi_device *sdev, struct request *req)
  1022. {
  1023. int ret = BLKPREP_OK;
  1024. /*
  1025. * If the device is not in running state we will reject some
  1026. * or all commands.
  1027. */
  1028. if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
  1029. switch (sdev->sdev_state) {
  1030. case SDEV_OFFLINE:
  1031. /*
  1032. * If the device is offline we refuse to process any
  1033. * commands. The device must be brought online
  1034. * before trying any recovery commands.
  1035. */
  1036. sdev_printk(KERN_ERR, sdev,
  1037. "rejecting I/O to offline device\n");
  1038. ret = BLKPREP_KILL;
  1039. break;
  1040. case SDEV_DEL:
  1041. /*
  1042. * If the device is fully deleted, we refuse to
  1043. * process any commands as well.
  1044. */
  1045. sdev_printk(KERN_ERR, sdev,
  1046. "rejecting I/O to dead device\n");
  1047. ret = BLKPREP_KILL;
  1048. break;
  1049. case SDEV_QUIESCE:
  1050. case SDEV_BLOCK:
  1051. case SDEV_CREATED_BLOCK:
  1052. /*
  1053. * If the devices is blocked we defer normal commands.
  1054. */
  1055. if (!(req->cmd_flags & REQ_PREEMPT))
  1056. ret = BLKPREP_DEFER;
  1057. break;
  1058. default:
  1059. /*
  1060. * For any other not fully online state we only allow
  1061. * special commands. In particular any user initiated
  1062. * command is not allowed.
  1063. */
  1064. if (!(req->cmd_flags & REQ_PREEMPT))
  1065. ret = BLKPREP_KILL;
  1066. break;
  1067. }
  1068. }
  1069. return ret;
  1070. }
  1071. EXPORT_SYMBOL(scsi_prep_state_check);
  1072. int scsi_prep_return(struct request_queue *q, struct request *req, int ret)
  1073. {
  1074. struct scsi_device *sdev = q->queuedata;
  1075. switch (ret) {
  1076. case BLKPREP_KILL:
  1077. req->errors = DID_NO_CONNECT << 16;
  1078. /* release the command and kill it */
  1079. if (req->special) {
  1080. struct scsi_cmnd *cmd = req->special;
  1081. scsi_release_buffers(cmd);
  1082. scsi_put_command(cmd);
  1083. req->special = NULL;
  1084. }
  1085. break;
  1086. case BLKPREP_DEFER:
  1087. /*
  1088. * If we defer, the blk_peek_request() returns NULL, but the
  1089. * queue must be restarted, so we schedule a callback to happen
  1090. * shortly.
  1091. */
  1092. if (sdev->device_busy == 0)
  1093. blk_delay_queue(q, SCSI_QUEUE_DELAY);
  1094. break;
  1095. default:
  1096. req->cmd_flags |= REQ_DONTPREP;
  1097. }
  1098. return ret;
  1099. }
  1100. EXPORT_SYMBOL(scsi_prep_return);
  1101. int scsi_prep_fn(struct request_queue *q, struct request *req)
  1102. {
  1103. struct scsi_device *sdev = q->queuedata;
  1104. int ret = BLKPREP_KILL;
  1105. if (req->cmd_type == REQ_TYPE_BLOCK_PC)
  1106. ret = scsi_setup_blk_pc_cmnd(sdev, req);
  1107. return scsi_prep_return(q, req, ret);
  1108. }
  1109. EXPORT_SYMBOL(scsi_prep_fn);
  1110. /*
  1111. * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
  1112. * return 0.
  1113. *
  1114. * Called with the queue_lock held.
  1115. */
  1116. static inline int scsi_dev_queue_ready(struct request_queue *q,
  1117. struct scsi_device *sdev)
  1118. {
  1119. if (sdev->device_busy == 0 && sdev->device_blocked) {
  1120. /*
  1121. * unblock after device_blocked iterates to zero
  1122. */
  1123. if (--sdev->device_blocked == 0) {
  1124. SCSI_LOG_MLQUEUE(3,
  1125. sdev_printk(KERN_INFO, sdev,
  1126. "unblocking device at zero depth\n"));
  1127. } else {
  1128. blk_delay_queue(q, SCSI_QUEUE_DELAY);
  1129. return 0;
  1130. }
  1131. }
  1132. if (scsi_device_is_busy(sdev))
  1133. return 0;
  1134. return 1;
  1135. }
  1136. /*
  1137. * scsi_target_queue_ready: checks if there we can send commands to target
  1138. * @sdev: scsi device on starget to check.
  1139. *
  1140. * Called with the host lock held.
  1141. */
  1142. static inline int scsi_target_queue_ready(struct Scsi_Host *shost,
  1143. struct scsi_device *sdev)
  1144. {
  1145. struct scsi_target *starget = scsi_target(sdev);
  1146. if (starget->single_lun) {
  1147. if (starget->starget_sdev_user &&
  1148. starget->starget_sdev_user != sdev)
  1149. return 0;
  1150. starget->starget_sdev_user = sdev;
  1151. }
  1152. if (starget->target_busy == 0 && starget->target_blocked) {
  1153. /*
  1154. * unblock after target_blocked iterates to zero
  1155. */
  1156. if (--starget->target_blocked == 0) {
  1157. SCSI_LOG_MLQUEUE(3, starget_printk(KERN_INFO, starget,
  1158. "unblocking target at zero depth\n"));
  1159. } else
  1160. return 0;
  1161. }
  1162. if (scsi_target_is_busy(starget)) {
  1163. if (list_empty(&sdev->starved_entry))
  1164. list_add_tail(&sdev->starved_entry,
  1165. &shost->starved_list);
  1166. return 0;
  1167. }
  1168. /* We're OK to process the command, so we can't be starved */
  1169. if (!list_empty(&sdev->starved_entry))
  1170. list_del_init(&sdev->starved_entry);
  1171. return 1;
  1172. }
  1173. /*
  1174. * scsi_host_queue_ready: if we can send requests to shost, return 1 else
  1175. * return 0. We must end up running the queue again whenever 0 is
  1176. * returned, else IO can hang.
  1177. *
  1178. * Called with host_lock held.
  1179. */
  1180. static inline int scsi_host_queue_ready(struct request_queue *q,
  1181. struct Scsi_Host *shost,
  1182. struct scsi_device *sdev)
  1183. {
  1184. if (scsi_host_in_recovery(shost))
  1185. return 0;
  1186. if (shost->host_busy == 0 && shost->host_blocked) {
  1187. /*
  1188. * unblock after host_blocked iterates to zero
  1189. */
  1190. if (--shost->host_blocked == 0) {
  1191. SCSI_LOG_MLQUEUE(3,
  1192. printk("scsi%d unblocking host at zero depth\n",
  1193. shost->host_no));
  1194. } else {
  1195. return 0;
  1196. }
  1197. }
  1198. if (scsi_host_is_busy(shost)) {
  1199. if (list_empty(&sdev->starved_entry))
  1200. list_add_tail(&sdev->starved_entry, &shost->starved_list);
  1201. return 0;
  1202. }
  1203. /* We're OK to process the command, so we can't be starved */
  1204. if (!list_empty(&sdev->starved_entry))
  1205. list_del_init(&sdev->starved_entry);
  1206. return 1;
  1207. }
  1208. /*
  1209. * Busy state exporting function for request stacking drivers.
  1210. *
  1211. * For efficiency, no lock is taken to check the busy state of
  1212. * shost/starget/sdev, since the returned value is not guaranteed and
  1213. * may be changed after request stacking drivers call the function,
  1214. * regardless of taking lock or not.
  1215. *
  1216. * When scsi can't dispatch I/Os anymore and needs to kill I/Os
  1217. * (e.g. !sdev), scsi needs to return 'not busy'.
  1218. * Otherwise, request stacking drivers may hold requests forever.
  1219. */
  1220. static int scsi_lld_busy(struct request_queue *q)
  1221. {
  1222. struct scsi_device *sdev = q->queuedata;
  1223. struct Scsi_Host *shost;
  1224. struct scsi_target *starget;
  1225. if (!sdev)
  1226. return 0;
  1227. shost = sdev->host;
  1228. starget = scsi_target(sdev);
  1229. if (scsi_host_in_recovery(shost) || scsi_host_is_busy(shost) ||
  1230. scsi_target_is_busy(starget) || scsi_device_is_busy(sdev))
  1231. return 1;
  1232. return 0;
  1233. }
  1234. /*
  1235. * Kill a request for a dead device
  1236. */
  1237. static void scsi_kill_request(struct request *req, struct request_queue *q)
  1238. {
  1239. struct scsi_cmnd *cmd = req->special;
  1240. struct scsi_device *sdev;
  1241. struct scsi_target *starget;
  1242. struct Scsi_Host *shost;
  1243. blk_start_request(req);
  1244. scmd_printk(KERN_INFO, cmd, "killing request\n");
  1245. sdev = cmd->device;
  1246. starget = scsi_target(sdev);
  1247. shost = sdev->host;
  1248. scsi_init_cmd_errh(cmd);
  1249. cmd->result = DID_NO_CONNECT << 16;
  1250. atomic_inc(&cmd->device->iorequest_cnt);
  1251. /*
  1252. * SCSI request completion path will do scsi_device_unbusy(),
  1253. * bump busy counts. To bump the counters, we need to dance
  1254. * with the locks as normal issue path does.
  1255. */
  1256. sdev->device_busy++;
  1257. spin_unlock(sdev->request_queue->queue_lock);
  1258. spin_lock(shost->host_lock);
  1259. shost->host_busy++;
  1260. starget->target_busy++;
  1261. spin_unlock(shost->host_lock);
  1262. spin_lock(sdev->request_queue->queue_lock);
  1263. blk_complete_request(req);
  1264. }
  1265. static void scsi_softirq_done(struct request *rq)
  1266. {
  1267. struct scsi_cmnd *cmd = rq->special;
  1268. unsigned long wait_for = (cmd->allowed + 1) * rq->timeout;
  1269. int disposition;
  1270. INIT_LIST_HEAD(&cmd->eh_entry);
  1271. atomic_inc(&cmd->device->iodone_cnt);
  1272. if (cmd->result)
  1273. atomic_inc(&cmd->device->ioerr_cnt);
  1274. disposition = scsi_decide_disposition(cmd);
  1275. if (disposition != SUCCESS &&
  1276. time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
  1277. sdev_printk(KERN_ERR, cmd->device,
  1278. "timing out command, waited %lus\n",
  1279. wait_for/HZ);
  1280. disposition = SUCCESS;
  1281. }
  1282. scsi_log_completion(cmd, disposition);
  1283. switch (disposition) {
  1284. case SUCCESS:
  1285. scsi_finish_command(cmd);
  1286. break;
  1287. case NEEDS_RETRY:
  1288. scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY);
  1289. break;
  1290. case ADD_TO_MLQUEUE:
  1291. scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
  1292. break;
  1293. default:
  1294. if (!scsi_eh_scmd_add(cmd, 0))
  1295. scsi_finish_command(cmd);
  1296. }
  1297. }
  1298. /*
  1299. * Function: scsi_request_fn()
  1300. *
  1301. * Purpose: Main strategy routine for SCSI.
  1302. *
  1303. * Arguments: q - Pointer to actual queue.
  1304. *
  1305. * Returns: Nothing
  1306. *
  1307. * Lock status: IO request lock assumed to be held when called.
  1308. */
  1309. static void scsi_request_fn(struct request_queue *q)
  1310. {
  1311. struct scsi_device *sdev = q->queuedata;
  1312. struct Scsi_Host *shost;
  1313. struct scsi_cmnd *cmd;
  1314. struct request *req;
  1315. if (!sdev) {
  1316. while ((req = blk_peek_request(q)) != NULL)
  1317. scsi_kill_request(req, q);
  1318. return;
  1319. }
  1320. if(!get_device(&sdev->sdev_gendev))
  1321. /* We must be tearing the block queue down already */
  1322. return;
  1323. /*
  1324. * To start with, we keep looping until the queue is empty, or until
  1325. * the host is no longer able to accept any more requests.
  1326. */
  1327. shost = sdev->host;
  1328. for (;;) {
  1329. int rtn;
  1330. /*
  1331. * get next queueable request. We do this early to make sure
  1332. * that the request is fully prepared even if we cannot
  1333. * accept it.
  1334. */
  1335. req = blk_peek_request(q);
  1336. if (!req || !scsi_dev_queue_ready(q, sdev))
  1337. break;
  1338. if (unlikely(!scsi_device_online(sdev))) {
  1339. sdev_printk(KERN_ERR, sdev,
  1340. "rejecting I/O to offline device\n");
  1341. scsi_kill_request(req, q);
  1342. continue;
  1343. }
  1344. /*
  1345. * Remove the request from the request list.
  1346. */
  1347. if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))
  1348. blk_start_request(req);
  1349. sdev->device_busy++;
  1350. spin_unlock(q->queue_lock);
  1351. cmd = req->special;
  1352. if (unlikely(cmd == NULL)) {
  1353. printk(KERN_CRIT "impossible request in %s.\n"
  1354. "please mail a stack trace to "
  1355. "linux-scsi@vger.kernel.org\n",
  1356. __func__);
  1357. blk_dump_rq_flags(req, "foo");
  1358. BUG();
  1359. }
  1360. spin_lock(shost->host_lock);
  1361. /*
  1362. * We hit this when the driver is using a host wide
  1363. * tag map. For device level tag maps the queue_depth check
  1364. * in the device ready fn would prevent us from trying
  1365. * to allocate a tag. Since the map is a shared host resource
  1366. * we add the dev to the starved list so it eventually gets
  1367. * a run when a tag is freed.
  1368. */
  1369. if (blk_queue_tagged(q) && !blk_rq_tagged(req)) {
  1370. if (list_empty(&sdev->starved_entry))
  1371. list_add_tail(&sdev->starved_entry,
  1372. &shost->starved_list);
  1373. goto not_ready;
  1374. }
  1375. if (!scsi_target_queue_ready(shost, sdev))
  1376. goto not_ready;
  1377. if (!scsi_host_queue_ready(q, shost, sdev))
  1378. goto not_ready;
  1379. scsi_target(sdev)->target_busy++;
  1380. shost->host_busy++;
  1381. /*
  1382. * XXX(hch): This is rather suboptimal, scsi_dispatch_cmd will
  1383. * take the lock again.
  1384. */
  1385. spin_unlock_irq(shost->host_lock);
  1386. /*
  1387. * Finally, initialize any error handling parameters, and set up
  1388. * the timers for timeouts.
  1389. */
  1390. scsi_init_cmd_errh(cmd);
  1391. /*
  1392. * Dispatch the command to the low-level driver.
  1393. */
  1394. rtn = scsi_dispatch_cmd(cmd);
  1395. spin_lock_irq(q->queue_lock);
  1396. if (rtn)
  1397. goto out_delay;
  1398. }
  1399. goto out;
  1400. not_ready:
  1401. spin_unlock_irq(shost->host_lock);
  1402. /*
  1403. * lock q, handle tag, requeue req, and decrement device_busy. We
  1404. * must return with queue_lock held.
  1405. *
  1406. * Decrementing device_busy without checking it is OK, as all such
  1407. * cases (host limits or settings) should run the queue at some
  1408. * later time.
  1409. */
  1410. spin_lock_irq(q->queue_lock);
  1411. blk_requeue_request(q, req);
  1412. sdev->device_busy--;
  1413. out_delay:
  1414. if (sdev->device_busy == 0)
  1415. blk_delay_queue(q, SCSI_QUEUE_DELAY);
  1416. out:
  1417. /* must be careful here...if we trigger the ->remove() function
  1418. * we cannot be holding the q lock */
  1419. spin_unlock_irq(q->queue_lock);
  1420. put_device(&sdev->sdev_gendev);
  1421. spin_lock_irq(q->queue_lock);
  1422. }
  1423. u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost)
  1424. {
  1425. struct device *host_dev;
  1426. u64 bounce_limit = 0xffffffff;
  1427. if (shost->unchecked_isa_dma)
  1428. return BLK_BOUNCE_ISA;
  1429. /*
  1430. * Platforms with virtual-DMA translation
  1431. * hardware have no practical limit.
  1432. */
  1433. if (!PCI_DMA_BUS_IS_PHYS)
  1434. return BLK_BOUNCE_ANY;
  1435. host_dev = scsi_get_device(shost);
  1436. if (host_dev && host_dev->dma_mask)
  1437. bounce_limit = *host_dev->dma_mask;
  1438. return bounce_limit;
  1439. }
  1440. EXPORT_SYMBOL(scsi_calculate_bounce_limit);
  1441. struct request_queue *__scsi_alloc_queue(struct Scsi_Host *shost,
  1442. request_fn_proc *request_fn)
  1443. {
  1444. struct request_queue *q;
  1445. struct device *dev = shost->shost_gendev.parent;
  1446. q = blk_init_queue(request_fn, NULL);
  1447. if (!q)
  1448. return NULL;
  1449. /*
  1450. * this limit is imposed by hardware restrictions
  1451. */
  1452. blk_queue_max_segments(q, min_t(unsigned short, shost->sg_tablesize,
  1453. SCSI_MAX_SG_CHAIN_SEGMENTS));
  1454. if (scsi_host_prot_dma(shost)) {
  1455. shost->sg_prot_tablesize =
  1456. min_not_zero(shost->sg_prot_tablesize,
  1457. (unsigned short)SCSI_MAX_PROT_SG_SEGMENTS);
  1458. BUG_ON(shost->sg_prot_tablesize < shost->sg_tablesize);
  1459. blk_queue_max_integrity_segments(q, shost->sg_prot_tablesize);
  1460. }
  1461. blk_queue_max_hw_sectors(q, shost->max_sectors);
  1462. blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
  1463. blk_queue_segment_boundary(q, shost->dma_boundary);
  1464. dma_set_seg_boundary(dev, shost->dma_boundary);
  1465. blk_queue_max_segment_size(q, dma_get_max_seg_size(dev));
  1466. if (!shost->use_clustering)
  1467. q->limits.cluster = 0;
  1468. /*
  1469. * set a reasonable default alignment on word boundaries: the
  1470. * host and device may alter it using
  1471. * blk_queue_update_dma_alignment() later.
  1472. */
  1473. blk_queue_dma_alignment(q, 0x03);
  1474. return q;
  1475. }
  1476. EXPORT_SYMBOL(__scsi_alloc_queue);
  1477. struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
  1478. {
  1479. struct request_queue *q;
  1480. q = __scsi_alloc_queue(sdev->host, scsi_request_fn);
  1481. if (!q)
  1482. return NULL;
  1483. blk_queue_prep_rq(q, scsi_prep_fn);
  1484. blk_queue_softirq_done(q, scsi_softirq_done);
  1485. blk_queue_rq_timed_out(q, scsi_times_out);
  1486. blk_queue_lld_busy(q, scsi_lld_busy);
  1487. return q;
  1488. }
  1489. void scsi_free_queue(struct request_queue *q)
  1490. {
  1491. unsigned long flags;
  1492. WARN_ON(q->queuedata);
  1493. /* cause scsi_request_fn() to kill all non-finished requests */
  1494. spin_lock_irqsave(q->queue_lock, flags);
  1495. q->request_fn(q);
  1496. spin_unlock_irqrestore(q->queue_lock, flags);
  1497. blk_cleanup_queue(q);
  1498. }
  1499. /*
  1500. * Function: scsi_block_requests()
  1501. *
  1502. * Purpose: Utility function used by low-level drivers to prevent further
  1503. * commands from being queued to the device.
  1504. *
  1505. * Arguments: shost - Host in question
  1506. *
  1507. * Returns: Nothing
  1508. *
  1509. * Lock status: No locks are assumed held.
  1510. *
  1511. * Notes: There is no timer nor any other means by which the requests
  1512. * get unblocked other than the low-level driver calling
  1513. * scsi_unblock_requests().
  1514. */
  1515. void scsi_block_requests(struct Scsi_Host *shost)
  1516. {
  1517. shost->host_self_blocked = 1;
  1518. }
  1519. EXPORT_SYMBOL(scsi_block_requests);
  1520. /*
  1521. * Function: scsi_unblock_requests()
  1522. *
  1523. * Purpose: Utility function used by low-level drivers to allow further
  1524. * commands from being queued to the device.
  1525. *
  1526. * Arguments: shost - Host in question
  1527. *
  1528. * Returns: Nothing
  1529. *
  1530. * Lock status: No locks are assumed held.
  1531. *
  1532. * Notes: There is no timer nor any other means by which the requests
  1533. * get unblocked other than the low-level driver calling
  1534. * scsi_unblock_requests().
  1535. *
  1536. * This is done as an API function so that changes to the
  1537. * internals of the scsi mid-layer won't require wholesale
  1538. * changes to drivers that use this feature.
  1539. */
  1540. void scsi_unblock_requests(struct Scsi_Host *shost)
  1541. {
  1542. shost->host_self_blocked = 0;
  1543. scsi_run_host_queues(shost);
  1544. }
  1545. EXPORT_SYMBOL(scsi_unblock_requests);
  1546. int __init scsi_init_queue(void)
  1547. {
  1548. int i;
  1549. scsi_sdb_cache = kmem_cache_create("scsi_data_buffer",
  1550. sizeof(struct scsi_data_buffer),
  1551. 0, 0, NULL);
  1552. if (!scsi_sdb_cache) {
  1553. printk(KERN_ERR "SCSI: can't init scsi sdb cache\n");
  1554. return -ENOMEM;
  1555. }
  1556. for (i = 0; i < SG_MEMPOOL_NR; i++) {
  1557. struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
  1558. int size = sgp->size * sizeof(struct scatterlist);
  1559. sgp->slab = kmem_cache_create(sgp->name, size, 0,
  1560. SLAB_HWCACHE_ALIGN, NULL);
  1561. if (!sgp->slab) {
  1562. printk(KERN_ERR "SCSI: can't init sg slab %s\n",
  1563. sgp->name);
  1564. goto cleanup_sdb;
  1565. }
  1566. sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
  1567. sgp->slab);
  1568. if (!sgp->pool) {
  1569. printk(KERN_ERR "SCSI: can't init sg mempool %s\n",
  1570. sgp->name);
  1571. goto cleanup_sdb;
  1572. }
  1573. }
  1574. return 0;
  1575. cleanup_sdb:
  1576. for (i = 0; i < SG_MEMPOOL_NR; i++) {
  1577. struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
  1578. if (sgp->pool)
  1579. mempool_destroy(sgp->pool);
  1580. if (sgp->slab)
  1581. kmem_cache_destroy(sgp->slab);
  1582. }
  1583. kmem_cache_destroy(scsi_sdb_cache);
  1584. return -ENOMEM;
  1585. }
  1586. void scsi_exit_queue(void)
  1587. {
  1588. int i;
  1589. kmem_cache_destroy(scsi_sdb_cache);
  1590. for (i = 0; i < SG_MEMPOOL_NR; i++) {
  1591. struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
  1592. mempool_destroy(sgp->pool);
  1593. kmem_cache_destroy(sgp->slab);
  1594. }
  1595. }
  1596. /**
  1597. * scsi_mode_select - issue a mode select
  1598. * @sdev: SCSI device to be queried
  1599. * @pf: Page format bit (1 == standard, 0 == vendor specific)
  1600. * @sp: Save page bit (0 == don't save, 1 == save)
  1601. * @modepage: mode page being requested
  1602. * @buffer: request buffer (may not be smaller than eight bytes)
  1603. * @len: length of request buffer.
  1604. * @timeout: command timeout
  1605. * @retries: number of retries before failing
  1606. * @data: returns a structure abstracting the mode header data
  1607. * @sshdr: place to put sense data (or NULL if no sense to be collected).
  1608. * must be SCSI_SENSE_BUFFERSIZE big.
  1609. *
  1610. * Returns zero if successful; negative error number or scsi
  1611. * status on error
  1612. *
  1613. */
  1614. int
  1615. scsi_mode_select(struct scsi_device *sdev, int pf, int sp, int modepage,
  1616. unsigned char *buffer, int len, int timeout, int retries,
  1617. struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
  1618. {
  1619. unsigned char cmd[10];
  1620. unsigned char *real_buffer;
  1621. int ret;
  1622. memset(cmd, 0, sizeof(cmd));
  1623. cmd[1] = (pf ? 0x10 : 0) | (sp ? 0x01 : 0);
  1624. if (sdev->use_10_for_ms) {
  1625. if (len > 65535)
  1626. return -EINVAL;
  1627. real_buffer = kmalloc(8 + len, GFP_KERNEL);
  1628. if (!real_buffer)
  1629. return -ENOMEM;
  1630. memcpy(real_buffer + 8, buffer, len);
  1631. len += 8;
  1632. real_buffer[0] = 0;
  1633. real_buffer[1] = 0;
  1634. real_buffer[2] = data->medium_type;
  1635. real_buffer[3] = data->device_specific;
  1636. real_buffer[4] = data->longlba ? 0x01 : 0;
  1637. real_buffer[5] = 0;
  1638. real_buffer[6] = data->block_descriptor_length >> 8;
  1639. real_buffer[7] = data->block_descriptor_length;
  1640. cmd[0] = MODE_SELECT_10;
  1641. cmd[7] = len >> 8;
  1642. cmd[8] = len;
  1643. } else {
  1644. if (len > 255 || data->block_descriptor_length > 255 ||
  1645. data->longlba)
  1646. return -EINVAL;
  1647. real_buffer = kmalloc(4 + len, GFP_KERNEL);
  1648. if (!real_buffer)
  1649. return -ENOMEM;
  1650. memcpy(real_buffer + 4, buffer, len);
  1651. len += 4;
  1652. real_buffer[0] = 0;
  1653. real_buffer[1] = data->medium_type;
  1654. real_buffer[2] = data->device_specific;
  1655. real_buffer[3] = data->block_descriptor_length;
  1656. cmd[0] = MODE_SELECT;
  1657. cmd[4] = len;
  1658. }
  1659. ret = scsi_execute_req(sdev, cmd, DMA_TO_DEVICE, real_buffer, len,
  1660. sshdr, timeout, retries, NULL);
  1661. kfree(real_buffer);
  1662. return ret;
  1663. }
  1664. EXPORT_SYMBOL_GPL(scsi_mode_select);
  1665. /**
  1666. * scsi_mode_sense - issue a mode sense, falling back from 10 to six bytes if necessary.
  1667. * @sdev: SCSI device to be queried
  1668. * @dbd: set if mode sense will allow block descriptors to be returned
  1669. * @modepage: mode page being requested
  1670. * @bu…