/drivers/scsi/scsi_lib.c

https://bitbucket.org/abioy/linux · C · 2560 lines · 1523 code · 310 blank · 727 comment · 289 complexity · a2587118637162b49f349d2c4f4766d6 MD5 · raw file

Large files are truncated click here to view the full file

  1. /*
  2. * scsi_lib.c Copyright (C) 1999 Eric Youngdale
  3. *
  4. * SCSI queueing library.
  5. * Initial versions: Eric Youngdale (eric@andante.org).
  6. * Based upon conversations with large numbers
  7. * of people at Linux Expo.
  8. */
  9. #include <linux/bio.h>
  10. #include <linux/bitops.h>
  11. #include <linux/blkdev.h>
  12. #include <linux/completion.h>
  13. #include <linux/kernel.h>
  14. #include <linux/mempool.h>
  15. #include <linux/slab.h>
  16. #include <linux/init.h>
  17. #include <linux/pci.h>
  18. #include <linux/delay.h>
  19. #include <linux/hardirq.h>
  20. #include <linux/scatterlist.h>
  21. #include <scsi/scsi.h>
  22. #include <scsi/scsi_cmnd.h>
  23. #include <scsi/scsi_dbg.h>
  24. #include <scsi/scsi_device.h>
  25. #include <scsi/scsi_driver.h>
  26. #include <scsi/scsi_eh.h>
  27. #include <scsi/scsi_host.h>
  28. #include "scsi_priv.h"
  29. #include "scsi_logging.h"
  30. #define SG_MEMPOOL_NR ARRAY_SIZE(scsi_sg_pools)
  31. #define SG_MEMPOOL_SIZE 2
  32. struct scsi_host_sg_pool {
  33. size_t size;
  34. char *name;
  35. struct kmem_cache *slab;
  36. mempool_t *pool;
  37. };
  38. #define SP(x) { x, "sgpool-" __stringify(x) }
  39. #if (SCSI_MAX_SG_SEGMENTS < 32)
  40. #error SCSI_MAX_SG_SEGMENTS is too small (must be 32 or greater)
  41. #endif
  42. static struct scsi_host_sg_pool scsi_sg_pools[] = {
  43. SP(8),
  44. SP(16),
  45. #if (SCSI_MAX_SG_SEGMENTS > 32)
  46. SP(32),
  47. #if (SCSI_MAX_SG_SEGMENTS > 64)
  48. SP(64),
  49. #if (SCSI_MAX_SG_SEGMENTS > 128)
  50. SP(128),
  51. #if (SCSI_MAX_SG_SEGMENTS > 256)
  52. #error SCSI_MAX_SG_SEGMENTS is too large (256 MAX)
  53. #endif
  54. #endif
  55. #endif
  56. #endif
  57. SP(SCSI_MAX_SG_SEGMENTS)
  58. };
  59. #undef SP
  60. struct kmem_cache *scsi_sdb_cache;
  61. static void scsi_run_queue(struct request_queue *q);
  62. /*
  63. * Function: scsi_unprep_request()
  64. *
  65. * Purpose: Remove all preparation done for a request, including its
  66. * associated scsi_cmnd, so that it can be requeued.
  67. *
  68. * Arguments: req - request to unprepare
  69. *
  70. * Lock status: Assumed that no locks are held upon entry.
  71. *
  72. * Returns: Nothing.
  73. */
  74. static void scsi_unprep_request(struct request *req)
  75. {
  76. struct scsi_cmnd *cmd = req->special;
  77. req->cmd_flags &= ~REQ_DONTPREP;
  78. req->special = NULL;
  79. scsi_put_command(cmd);
  80. }
  81. /**
  82. * __scsi_queue_insert - private queue insertion
  83. * @cmd: The SCSI command being requeued
  84. * @reason: The reason for the requeue
  85. * @unbusy: Whether the queue should be unbusied
  86. *
  87. * This is a private queue insertion. The public interface
  88. * scsi_queue_insert() always assumes the queue should be unbusied
  89. * because it's always called before the completion. This function is
  90. * for a requeue after completion, which should only occur in this
  91. * file.
  92. */
  93. static int __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy)
  94. {
  95. struct Scsi_Host *host = cmd->device->host;
  96. struct scsi_device *device = cmd->device;
  97. struct scsi_target *starget = scsi_target(device);
  98. struct request_queue *q = device->request_queue;
  99. unsigned long flags;
  100. SCSI_LOG_MLQUEUE(1,
  101. printk("Inserting command %p into mlqueue\n", cmd));
  102. /*
  103. * Set the appropriate busy bit for the device/host.
  104. *
  105. * If the host/device isn't busy, assume that something actually
  106. * completed, and that we should be able to queue a command now.
  107. *
  108. * Note that the prior mid-layer assumption that any host could
  109. * always queue at least one command is now broken. The mid-layer
  110. * will implement a user specifiable stall (see
  111. * scsi_host.max_host_blocked and scsi_device.max_device_blocked)
  112. * if a command is requeued with no other commands outstanding
  113. * either for the device or for the host.
  114. */
  115. switch (reason) {
  116. case SCSI_MLQUEUE_HOST_BUSY:
  117. host->host_blocked = host->max_host_blocked;
  118. break;
  119. case SCSI_MLQUEUE_DEVICE_BUSY:
  120. device->device_blocked = device->max_device_blocked;
  121. break;
  122. case SCSI_MLQUEUE_TARGET_BUSY:
  123. starget->target_blocked = starget->max_target_blocked;
  124. break;
  125. }
  126. /*
  127. * Decrement the counters, since these commands are no longer
  128. * active on the host/device.
  129. */
  130. if (unbusy)
  131. scsi_device_unbusy(device);
  132. /*
  133. * Requeue this command. It will go before all other commands
  134. * that are already in the queue.
  135. *
  136. * NOTE: there is magic here about the way the queue is plugged if
  137. * we have no outstanding commands.
  138. *
  139. * Although we *don't* plug the queue, we call the request
  140. * function. The SCSI request function detects the blocked condition
  141. * and plugs the queue appropriately.
  142. */
  143. spin_lock_irqsave(q->queue_lock, flags);
  144. blk_requeue_request(q, cmd->request);
  145. spin_unlock_irqrestore(q->queue_lock, flags);
  146. scsi_run_queue(q);
  147. return 0;
  148. }
  149. /*
  150. * Function: scsi_queue_insert()
  151. *
  152. * Purpose: Insert a command in the midlevel queue.
  153. *
  154. * Arguments: cmd - command that we are adding to queue.
  155. * reason - why we are inserting command to queue.
  156. *
  157. * Lock status: Assumed that lock is not held upon entry.
  158. *
  159. * Returns: Nothing.
  160. *
  161. * Notes: We do this for one of two cases. Either the host is busy
  162. * and it cannot accept any more commands for the time being,
  163. * or the device returned QUEUE_FULL and can accept no more
  164. * commands.
  165. * Notes: This could be called either from an interrupt context or a
  166. * normal process context.
  167. */
  168. int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
  169. {
  170. return __scsi_queue_insert(cmd, reason, 1);
  171. }
  172. /**
  173. * scsi_execute - insert request and wait for the result
  174. * @sdev: scsi device
  175. * @cmd: scsi command
  176. * @data_direction: data direction
  177. * @buffer: data buffer
  178. * @bufflen: len of buffer
  179. * @sense: optional sense buffer
  180. * @timeout: request timeout in seconds
  181. * @retries: number of times to retry request
  182. * @flags: or into request flags;
  183. * @resid: optional residual length
  184. *
  185. * returns the req->errors value which is the scsi_cmnd result
  186. * field.
  187. */
  188. int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
  189. int data_direction, void *buffer, unsigned bufflen,
  190. unsigned char *sense, int timeout, int retries, int flags,
  191. int *resid)
  192. {
  193. struct request *req;
  194. int write = (data_direction == DMA_TO_DEVICE);
  195. int ret = DRIVER_ERROR << 24;
  196. req = blk_get_request(sdev->request_queue, write, __GFP_WAIT);
  197. if (bufflen && blk_rq_map_kern(sdev->request_queue, req,
  198. buffer, bufflen, __GFP_WAIT))
  199. goto out;
  200. req->cmd_len = COMMAND_SIZE(cmd[0]);
  201. memcpy(req->cmd, cmd, req->cmd_len);
  202. req->sense = sense;
  203. req->sense_len = 0;
  204. req->retries = retries;
  205. req->timeout = timeout;
  206. req->cmd_type = REQ_TYPE_BLOCK_PC;
  207. req->cmd_flags |= flags | REQ_QUIET | REQ_PREEMPT;
  208. /*
  209. * head injection *required* here otherwise quiesce won't work
  210. */
  211. blk_execute_rq(req->q, NULL, req, 1);
  212. /*
  213. * Some devices (USB mass-storage in particular) may transfer
  214. * garbage data together with a residue indicating that the data
  215. * is invalid. Prevent the garbage from being misinterpreted
  216. * and prevent security leaks by zeroing out the excess data.
  217. */
  218. if (unlikely(req->resid_len > 0 && req->resid_len <= bufflen))
  219. memset(buffer + (bufflen - req->resid_len), 0, req->resid_len);
  220. if (resid)
  221. *resid = req->resid_len;
  222. ret = req->errors;
  223. out:
  224. blk_put_request(req);
  225. return ret;
  226. }
  227. EXPORT_SYMBOL(scsi_execute);
  228. int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd,
  229. int data_direction, void *buffer, unsigned bufflen,
  230. struct scsi_sense_hdr *sshdr, int timeout, int retries,
  231. int *resid)
  232. {
  233. char *sense = NULL;
  234. int result;
  235. if (sshdr) {
  236. sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO);
  237. if (!sense)
  238. return DRIVER_ERROR << 24;
  239. }
  240. result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen,
  241. sense, timeout, retries, 0, resid);
  242. if (sshdr)
  243. scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr);
  244. kfree(sense);
  245. return result;
  246. }
  247. EXPORT_SYMBOL(scsi_execute_req);
  248. /*
  249. * Function: scsi_init_cmd_errh()
  250. *
  251. * Purpose: Initialize cmd fields related to error handling.
  252. *
  253. * Arguments: cmd - command that is ready to be queued.
  254. *
  255. * Notes: This function has the job of initializing a number of
  256. * fields related to error handling. Typically this will
  257. * be called once for each command, as required.
  258. */
  259. static void scsi_init_cmd_errh(struct scsi_cmnd *cmd)
  260. {
  261. cmd->serial_number = 0;
  262. scsi_set_resid(cmd, 0);
  263. memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
  264. if (cmd->cmd_len == 0)
  265. cmd->cmd_len = scsi_command_size(cmd->cmnd);
  266. }
  267. void scsi_device_unbusy(struct scsi_device *sdev)
  268. {
  269. struct Scsi_Host *shost = sdev->host;
  270. struct scsi_target *starget = scsi_target(sdev);
  271. unsigned long flags;
  272. spin_lock_irqsave(shost->host_lock, flags);
  273. shost->host_busy--;
  274. starget->target_busy--;
  275. if (unlikely(scsi_host_in_recovery(shost) &&
  276. (shost->host_failed || shost->host_eh_scheduled)))
  277. scsi_eh_wakeup(shost);
  278. spin_unlock(shost->host_lock);
  279. spin_lock(sdev->request_queue->queue_lock);
  280. sdev->device_busy--;
  281. spin_unlock_irqrestore(sdev->request_queue->queue_lock, flags);
  282. }
  283. /*
  284. * Called for single_lun devices on IO completion. Clear starget_sdev_user,
  285. * and call blk_run_queue for all the scsi_devices on the target -
  286. * including current_sdev first.
  287. *
  288. * Called with *no* scsi locks held.
  289. */
  290. static void scsi_single_lun_run(struct scsi_device *current_sdev)
  291. {
  292. struct Scsi_Host *shost = current_sdev->host;
  293. struct scsi_device *sdev, *tmp;
  294. struct scsi_target *starget = scsi_target(current_sdev);
  295. unsigned long flags;
  296. spin_lock_irqsave(shost->host_lock, flags);
  297. starget->starget_sdev_user = NULL;
  298. spin_unlock_irqrestore(shost->host_lock, flags);
  299. /*
  300. * Call blk_run_queue for all LUNs on the target, starting with
  301. * current_sdev. We race with others (to set starget_sdev_user),
  302. * but in most cases, we will be first. Ideally, each LU on the
  303. * target would get some limited time or requests on the target.
  304. */
  305. blk_run_queue(current_sdev->request_queue);
  306. spin_lock_irqsave(shost->host_lock, flags);
  307. if (starget->starget_sdev_user)
  308. goto out;
  309. list_for_each_entry_safe(sdev, tmp, &starget->devices,
  310. same_target_siblings) {
  311. if (sdev == current_sdev)
  312. continue;
  313. if (scsi_device_get(sdev))
  314. continue;
  315. spin_unlock_irqrestore(shost->host_lock, flags);
  316. blk_run_queue(sdev->request_queue);
  317. spin_lock_irqsave(shost->host_lock, flags);
  318. scsi_device_put(sdev);
  319. }
  320. out:
  321. spin_unlock_irqrestore(shost->host_lock, flags);
  322. }
  323. static inline int scsi_device_is_busy(struct scsi_device *sdev)
  324. {
  325. if (sdev->device_busy >= sdev->queue_depth || sdev->device_blocked)
  326. return 1;
  327. return 0;
  328. }
  329. static inline int scsi_target_is_busy(struct scsi_target *starget)
  330. {
  331. return ((starget->can_queue > 0 &&
  332. starget->target_busy >= starget->can_queue) ||
  333. starget->target_blocked);
  334. }
  335. static inline int scsi_host_is_busy(struct Scsi_Host *shost)
  336. {
  337. if ((shost->can_queue > 0 && shost->host_busy >= shost->can_queue) ||
  338. shost->host_blocked || shost->host_self_blocked)
  339. return 1;
  340. return 0;
  341. }
  342. /*
  343. * Function: scsi_run_queue()
  344. *
  345. * Purpose: Select a proper request queue to serve next
  346. *
  347. * Arguments: q - last request's queue
  348. *
  349. * Returns: Nothing
  350. *
  351. * Notes: The previous command was completely finished, start
  352. * a new one if possible.
  353. */
  354. static void scsi_run_queue(struct request_queue *q)
  355. {
  356. struct scsi_device *sdev = q->queuedata;
  357. struct Scsi_Host *shost = sdev->host;
  358. LIST_HEAD(starved_list);
  359. unsigned long flags;
  360. if (scsi_target(sdev)->single_lun)
  361. scsi_single_lun_run(sdev);
  362. spin_lock_irqsave(shost->host_lock, flags);
  363. list_splice_init(&shost->starved_list, &starved_list);
  364. while (!list_empty(&starved_list)) {
  365. int flagset;
  366. /*
  367. * As long as shost is accepting commands and we have
  368. * starved queues, call blk_run_queue. scsi_request_fn
  369. * drops the queue_lock and can add us back to the
  370. * starved_list.
  371. *
  372. * host_lock protects the starved_list and starved_entry.
  373. * scsi_request_fn must get the host_lock before checking
  374. * or modifying starved_list or starved_entry.
  375. */
  376. if (scsi_host_is_busy(shost))
  377. break;
  378. sdev = list_entry(starved_list.next,
  379. struct scsi_device, starved_entry);
  380. list_del_init(&sdev->starved_entry);
  381. if (scsi_target_is_busy(scsi_target(sdev))) {
  382. list_move_tail(&sdev->starved_entry,
  383. &shost->starved_list);
  384. continue;
  385. }
  386. spin_unlock(shost->host_lock);
  387. spin_lock(sdev->request_queue->queue_lock);
  388. flagset = test_bit(QUEUE_FLAG_REENTER, &q->queue_flags) &&
  389. !test_bit(QUEUE_FLAG_REENTER,
  390. &sdev->request_queue->queue_flags);
  391. if (flagset)
  392. queue_flag_set(QUEUE_FLAG_REENTER, sdev->request_queue);
  393. __blk_run_queue(sdev->request_queue);
  394. if (flagset)
  395. queue_flag_clear(QUEUE_FLAG_REENTER, sdev->request_queue);
  396. spin_unlock(sdev->request_queue->queue_lock);
  397. spin_lock(shost->host_lock);
  398. }
  399. /* put any unprocessed entries back */
  400. list_splice(&starved_list, &shost->starved_list);
  401. spin_unlock_irqrestore(shost->host_lock, flags);
  402. blk_run_queue(q);
  403. }
  404. /*
  405. * Function: scsi_requeue_command()
  406. *
  407. * Purpose: Handle post-processing of completed commands.
  408. *
  409. * Arguments: q - queue to operate on
  410. * cmd - command that may need to be requeued.
  411. *
  412. * Returns: Nothing
  413. *
  414. * Notes: After command completion, there may be blocks left
  415. * over which weren't finished by the previous command
  416. * this can be for a number of reasons - the main one is
  417. * I/O errors in the middle of the request, in which case
  418. * we need to request the blocks that come after the bad
  419. * sector.
  420. * Notes: Upon return, cmd is a stale pointer.
  421. */
  422. static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd)
  423. {
  424. struct request *req = cmd->request;
  425. unsigned long flags;
  426. spin_lock_irqsave(q->queue_lock, flags);
  427. scsi_unprep_request(req);
  428. blk_requeue_request(q, req);
  429. spin_unlock_irqrestore(q->queue_lock, flags);
  430. scsi_run_queue(q);
  431. }
  432. void scsi_next_command(struct scsi_cmnd *cmd)
  433. {
  434. struct scsi_device *sdev = cmd->device;
  435. struct request_queue *q = sdev->request_queue;
  436. /* need to hold a reference on the device before we let go of the cmd */
  437. get_device(&sdev->sdev_gendev);
  438. scsi_put_command(cmd);
  439. scsi_run_queue(q);
  440. /* ok to remove device now */
  441. put_device(&sdev->sdev_gendev);
  442. }
  443. void scsi_run_host_queues(struct Scsi_Host *shost)
  444. {
  445. struct scsi_device *sdev;
  446. shost_for_each_device(sdev, shost)
  447. scsi_run_queue(sdev->request_queue);
  448. }
  449. static void __scsi_release_buffers(struct scsi_cmnd *, int);
  450. /*
  451. * Function: scsi_end_request()
  452. *
  453. * Purpose: Post-processing of completed commands (usually invoked at end
  454. * of upper level post-processing and scsi_io_completion).
  455. *
  456. * Arguments: cmd - command that is complete.
  457. * error - 0 if I/O indicates success, < 0 for I/O error.
  458. * bytes - number of bytes of completed I/O
  459. * requeue - indicates whether we should requeue leftovers.
  460. *
  461. * Lock status: Assumed that lock is not held upon entry.
  462. *
  463. * Returns: cmd if requeue required, NULL otherwise.
  464. *
  465. * Notes: This is called for block device requests in order to
  466. * mark some number of sectors as complete.
  467. *
  468. * We are guaranteeing that the request queue will be goosed
  469. * at some point during this call.
  470. * Notes: If cmd was requeued, upon return it will be a stale pointer.
  471. */
  472. static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int error,
  473. int bytes, int requeue)
  474. {
  475. struct request_queue *q = cmd->device->request_queue;
  476. struct request *req = cmd->request;
  477. /*
  478. * If there are blocks left over at the end, set up the command
  479. * to queue the remainder of them.
  480. */
  481. if (blk_end_request(req, error, bytes)) {
  482. /* kill remainder if no retrys */
  483. if (error && scsi_noretry_cmd(cmd))
  484. blk_end_request_all(req, error);
  485. else {
  486. if (requeue) {
  487. /*
  488. * Bleah. Leftovers again. Stick the
  489. * leftovers in the front of the
  490. * queue, and goose the queue again.
  491. */
  492. scsi_release_buffers(cmd);
  493. scsi_requeue_command(q, cmd);
  494. cmd = NULL;
  495. }
  496. return cmd;
  497. }
  498. }
  499. /*
  500. * This will goose the queue request function at the end, so we don't
  501. * need to worry about launching another command.
  502. */
  503. __scsi_release_buffers(cmd, 0);
  504. scsi_next_command(cmd);
  505. return NULL;
  506. }
  507. static inline unsigned int scsi_sgtable_index(unsigned short nents)
  508. {
  509. unsigned int index;
  510. BUG_ON(nents > SCSI_MAX_SG_SEGMENTS);
  511. if (nents <= 8)
  512. index = 0;
  513. else
  514. index = get_count_order(nents) - 3;
  515. return index;
  516. }
  517. static void scsi_sg_free(struct scatterlist *sgl, unsigned int nents)
  518. {
  519. struct scsi_host_sg_pool *sgp;
  520. sgp = scsi_sg_pools + scsi_sgtable_index(nents);
  521. mempool_free(sgl, sgp->pool);
  522. }
  523. static struct scatterlist *scsi_sg_alloc(unsigned int nents, gfp_t gfp_mask)
  524. {
  525. struct scsi_host_sg_pool *sgp;
  526. sgp = scsi_sg_pools + scsi_sgtable_index(nents);
  527. return mempool_alloc(sgp->pool, gfp_mask);
  528. }
  529. static int scsi_alloc_sgtable(struct scsi_data_buffer *sdb, int nents,
  530. gfp_t gfp_mask)
  531. {
  532. int ret;
  533. BUG_ON(!nents);
  534. ret = __sg_alloc_table(&sdb->table, nents, SCSI_MAX_SG_SEGMENTS,
  535. gfp_mask, scsi_sg_alloc);
  536. if (unlikely(ret))
  537. __sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS,
  538. scsi_sg_free);
  539. return ret;
  540. }
  541. static void scsi_free_sgtable(struct scsi_data_buffer *sdb)
  542. {
  543. __sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS, scsi_sg_free);
  544. }
  545. static void __scsi_release_buffers(struct scsi_cmnd *cmd, int do_bidi_check)
  546. {
  547. if (cmd->sdb.table.nents)
  548. scsi_free_sgtable(&cmd->sdb);
  549. memset(&cmd->sdb, 0, sizeof(cmd->sdb));
  550. if (do_bidi_check && scsi_bidi_cmnd(cmd)) {
  551. struct scsi_data_buffer *bidi_sdb =
  552. cmd->request->next_rq->special;
  553. scsi_free_sgtable(bidi_sdb);
  554. kmem_cache_free(scsi_sdb_cache, bidi_sdb);
  555. cmd->request->next_rq->special = NULL;
  556. }
  557. if (scsi_prot_sg_count(cmd))
  558. scsi_free_sgtable(cmd->prot_sdb);
  559. }
  560. /*
  561. * Function: scsi_release_buffers()
  562. *
  563. * Purpose: Completion processing for block device I/O requests.
  564. *
  565. * Arguments: cmd - command that we are bailing.
  566. *
  567. * Lock status: Assumed that no lock is held upon entry.
  568. *
  569. * Returns: Nothing
  570. *
  571. * Notes: In the event that an upper level driver rejects a
  572. * command, we must release resources allocated during
  573. * the __init_io() function. Primarily this would involve
  574. * the scatter-gather table, and potentially any bounce
  575. * buffers.
  576. */
  577. void scsi_release_buffers(struct scsi_cmnd *cmd)
  578. {
  579. __scsi_release_buffers(cmd, 1);
  580. }
  581. EXPORT_SYMBOL(scsi_release_buffers);
  582. /*
  583. * Function: scsi_io_completion()
  584. *
  585. * Purpose: Completion processing for block device I/O requests.
  586. *
  587. * Arguments: cmd - command that is finished.
  588. *
  589. * Lock status: Assumed that no lock is held upon entry.
  590. *
  591. * Returns: Nothing
  592. *
  593. * Notes: This function is matched in terms of capabilities to
  594. * the function that created the scatter-gather list.
  595. * In other words, if there are no bounce buffers
  596. * (the normal case for most drivers), we don't need
  597. * the logic to deal with cleaning up afterwards.
  598. *
  599. * We must call scsi_end_request(). This will finish off
  600. * the specified number of sectors. If we are done, the
  601. * command block will be released and the queue function
  602. * will be goosed. If we are not done then we have to
  603. * figure out what to do next:
  604. *
  605. * a) We can call scsi_requeue_command(). The request
  606. * will be unprepared and put back on the queue. Then
  607. * a new command will be created for it. This should
  608. * be used if we made forward progress, or if we want
  609. * to switch from READ(10) to READ(6) for example.
  610. *
  611. * b) We can call scsi_queue_insert(). The request will
  612. * be put back on the queue and retried using the same
  613. * command as before, possibly after a delay.
  614. *
  615. * c) We can call blk_end_request() with -EIO to fail
  616. * the remainder of the request.
  617. */
  618. void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
  619. {
  620. int result = cmd->result;
  621. struct request_queue *q = cmd->device->request_queue;
  622. struct request *req = cmd->request;
  623. int error = 0;
  624. struct scsi_sense_hdr sshdr;
  625. int sense_valid = 0;
  626. int sense_deferred = 0;
  627. enum {ACTION_FAIL, ACTION_REPREP, ACTION_RETRY,
  628. ACTION_DELAYED_RETRY} action;
  629. char *description = NULL;
  630. if (result) {
  631. sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
  632. if (sense_valid)
  633. sense_deferred = scsi_sense_is_deferred(&sshdr);
  634. }
  635. if (blk_pc_request(req)) { /* SG_IO ioctl from block level */
  636. req->errors = result;
  637. if (result) {
  638. if (sense_valid && req->sense) {
  639. /*
  640. * SG_IO wants current and deferred errors
  641. */
  642. int len = 8 + cmd->sense_buffer[7];
  643. if (len > SCSI_SENSE_BUFFERSIZE)
  644. len = SCSI_SENSE_BUFFERSIZE;
  645. memcpy(req->sense, cmd->sense_buffer, len);
  646. req->sense_len = len;
  647. }
  648. if (!sense_deferred)
  649. error = -EIO;
  650. }
  651. req->resid_len = scsi_get_resid(cmd);
  652. if (scsi_bidi_cmnd(cmd)) {
  653. /*
  654. * Bidi commands Must be complete as a whole,
  655. * both sides at once.
  656. */
  657. req->next_rq->resid_len = scsi_in(cmd)->resid;
  658. scsi_release_buffers(cmd);
  659. blk_end_request_all(req, 0);
  660. scsi_next_command(cmd);
  661. return;
  662. }
  663. }
  664. BUG_ON(blk_bidi_rq(req)); /* bidi not support for !blk_pc_request yet */
  665. /*
  666. * Next deal with any sectors which we were able to correctly
  667. * handle.
  668. */
  669. SCSI_LOG_HLCOMPLETE(1, printk("%u sectors total, "
  670. "%d bytes done.\n",
  671. blk_rq_sectors(req), good_bytes));
  672. /*
  673. * Recovered errors need reporting, but they're always treated
  674. * as success, so fiddle the result code here. For BLOCK_PC
  675. * we already took a copy of the original into rq->errors which
  676. * is what gets returned to the user
  677. */
  678. if (sense_valid && (sshdr.sense_key == RECOVERED_ERROR)) {
  679. /* if ATA PASS-THROUGH INFORMATION AVAILABLE skip
  680. * print since caller wants ATA registers. Only occurs on
  681. * SCSI ATA PASS_THROUGH commands when CK_COND=1
  682. */
  683. if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d))
  684. ;
  685. else if (!(req->cmd_flags & REQ_QUIET))
  686. scsi_print_sense("", cmd);
  687. result = 0;
  688. /* BLOCK_PC may have set error */
  689. error = 0;
  690. }
  691. /*
  692. * A number of bytes were successfully read. If there
  693. * are leftovers and there is some kind of error
  694. * (result != 0), retry the rest.
  695. */
  696. if (scsi_end_request(cmd, error, good_bytes, result == 0) == NULL)
  697. return;
  698. error = -EIO;
  699. if (host_byte(result) == DID_RESET) {
  700. /* Third party bus reset or reset for error recovery
  701. * reasons. Just retry the command and see what
  702. * happens.
  703. */
  704. action = ACTION_RETRY;
  705. } else if (sense_valid && !sense_deferred) {
  706. switch (sshdr.sense_key) {
  707. case UNIT_ATTENTION:
  708. if (cmd->device->removable) {
  709. /* Detected disc change. Set a bit
  710. * and quietly refuse further access.
  711. */
  712. cmd->device->changed = 1;
  713. description = "Media Changed";
  714. action = ACTION_FAIL;
  715. } else {
  716. /* Must have been a power glitch, or a
  717. * bus reset. Could not have been a
  718. * media change, so we just retry the
  719. * command and see what happens.
  720. */
  721. action = ACTION_RETRY;
  722. }
  723. break;
  724. case ILLEGAL_REQUEST:
  725. /* If we had an ILLEGAL REQUEST returned, then
  726. * we may have performed an unsupported
  727. * command. The only thing this should be
  728. * would be a ten byte read where only a six
  729. * byte read was supported. Also, on a system
  730. * where READ CAPACITY failed, we may have
  731. * read past the end of the disk.
  732. */
  733. if ((cmd->device->use_10_for_rw &&
  734. sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&
  735. (cmd->cmnd[0] == READ_10 ||
  736. cmd->cmnd[0] == WRITE_10)) {
  737. /* This will issue a new 6-byte command. */
  738. cmd->device->use_10_for_rw = 0;
  739. action = ACTION_REPREP;
  740. } else if (sshdr.asc == 0x10) /* DIX */ {
  741. description = "Host Data Integrity Failure";
  742. action = ACTION_FAIL;
  743. error = -EILSEQ;
  744. } else
  745. action = ACTION_FAIL;
  746. break;
  747. case ABORTED_COMMAND:
  748. action = ACTION_FAIL;
  749. if (sshdr.asc == 0x10) { /* DIF */
  750. description = "Target Data Integrity Failure";
  751. error = -EILSEQ;
  752. }
  753. break;
  754. case NOT_READY:
  755. /* If the device is in the process of becoming
  756. * ready, or has a temporary blockage, retry.
  757. */
  758. if (sshdr.asc == 0x04) {
  759. switch (sshdr.ascq) {
  760. case 0x01: /* becoming ready */
  761. case 0x04: /* format in progress */
  762. case 0x05: /* rebuild in progress */
  763. case 0x06: /* recalculation in progress */
  764. case 0x07: /* operation in progress */
  765. case 0x08: /* Long write in progress */
  766. case 0x09: /* self test in progress */
  767. case 0x14: /* space allocation in progress */
  768. action = ACTION_DELAYED_RETRY;
  769. break;
  770. default:
  771. description = "Device not ready";
  772. action = ACTION_FAIL;
  773. break;
  774. }
  775. } else {
  776. description = "Device not ready";
  777. action = ACTION_FAIL;
  778. }
  779. break;
  780. case VOLUME_OVERFLOW:
  781. /* See SSC3rXX or current. */
  782. action = ACTION_FAIL;
  783. break;
  784. default:
  785. description = "Unhandled sense code";
  786. action = ACTION_FAIL;
  787. break;
  788. }
  789. } else {
  790. description = "Unhandled error code";
  791. action = ACTION_FAIL;
  792. }
  793. switch (action) {
  794. case ACTION_FAIL:
  795. /* Give up and fail the remainder of the request */
  796. scsi_release_buffers(cmd);
  797. if (!(req->cmd_flags & REQ_QUIET)) {
  798. if (description)
  799. scmd_printk(KERN_INFO, cmd, "%s\n",
  800. description);
  801. scsi_print_result(cmd);
  802. if (driver_byte(result) & DRIVER_SENSE)
  803. scsi_print_sense("", cmd);
  804. scsi_print_command(cmd);
  805. }
  806. if (blk_end_request_err(req, error))
  807. scsi_requeue_command(q, cmd);
  808. else
  809. scsi_next_command(cmd);
  810. break;
  811. case ACTION_REPREP:
  812. /* Unprep the request and put it back at the head of the queue.
  813. * A new command will be prepared and issued.
  814. */
  815. scsi_release_buffers(cmd);
  816. scsi_requeue_command(q, cmd);
  817. break;
  818. case ACTION_RETRY:
  819. /* Retry the same command immediately */
  820. __scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, 0);
  821. break;
  822. case ACTION_DELAYED_RETRY:
  823. /* Retry the same command after a delay */
  824. __scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, 0);
  825. break;
  826. }
  827. }
  828. static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb,
  829. gfp_t gfp_mask)
  830. {
  831. int count;
  832. /*
  833. * If sg table allocation fails, requeue request later.
  834. */
  835. if (unlikely(scsi_alloc_sgtable(sdb, req->nr_phys_segments,
  836. gfp_mask))) {
  837. return BLKPREP_DEFER;
  838. }
  839. req->buffer = NULL;
  840. /*
  841. * Next, walk the list, and fill in the addresses and sizes of
  842. * each segment.
  843. */
  844. count = blk_rq_map_sg(req->q, req, sdb->table.sgl);
  845. BUG_ON(count > sdb->table.nents);
  846. sdb->table.nents = count;
  847. sdb->length = blk_rq_bytes(req);
  848. return BLKPREP_OK;
  849. }
  850. /*
  851. * Function: scsi_init_io()
  852. *
  853. * Purpose: SCSI I/O initialize function.
  854. *
  855. * Arguments: cmd - Command descriptor we wish to initialize
  856. *
  857. * Returns: 0 on success
  858. * BLKPREP_DEFER if the failure is retryable
  859. * BLKPREP_KILL if the failure is fatal
  860. */
  861. int scsi_init_io(struct scsi_cmnd *cmd, gfp_t gfp_mask)
  862. {
  863. int error = scsi_init_sgtable(cmd->request, &cmd->sdb, gfp_mask);
  864. if (error)
  865. goto err_exit;
  866. if (blk_bidi_rq(cmd->request)) {
  867. struct scsi_data_buffer *bidi_sdb = kmem_cache_zalloc(
  868. scsi_sdb_cache, GFP_ATOMIC);
  869. if (!bidi_sdb) {
  870. error = BLKPREP_DEFER;
  871. goto err_exit;
  872. }
  873. cmd->request->next_rq->special = bidi_sdb;
  874. error = scsi_init_sgtable(cmd->request->next_rq, bidi_sdb,
  875. GFP_ATOMIC);
  876. if (error)
  877. goto err_exit;
  878. }
  879. if (blk_integrity_rq(cmd->request)) {
  880. struct scsi_data_buffer *prot_sdb = cmd->prot_sdb;
  881. int ivecs, count;
  882. BUG_ON(prot_sdb == NULL);
  883. ivecs = blk_rq_count_integrity_sg(cmd->request);
  884. if (scsi_alloc_sgtable(prot_sdb, ivecs, gfp_mask)) {
  885. error = BLKPREP_DEFER;
  886. goto err_exit;
  887. }
  888. count = blk_rq_map_integrity_sg(cmd->request,
  889. prot_sdb->table.sgl);
  890. BUG_ON(unlikely(count > ivecs));
  891. cmd->prot_sdb = prot_sdb;
  892. cmd->prot_sdb->table.nents = count;
  893. }
  894. return BLKPREP_OK ;
  895. err_exit:
  896. scsi_release_buffers(cmd);
  897. if (error == BLKPREP_KILL)
  898. scsi_put_command(cmd);
  899. else /* BLKPREP_DEFER */
  900. scsi_unprep_request(cmd->request);
  901. return error;
  902. }
  903. EXPORT_SYMBOL(scsi_init_io);
  904. static struct scsi_cmnd *scsi_get_cmd_from_req(struct scsi_device *sdev,
  905. struct request *req)
  906. {
  907. struct scsi_cmnd *cmd;
  908. if (!req->special) {
  909. cmd = scsi_get_command(sdev, GFP_ATOMIC);
  910. if (unlikely(!cmd))
  911. return NULL;
  912. req->special = cmd;
  913. } else {
  914. cmd = req->special;
  915. }
  916. /* pull a tag out of the request if we have one */
  917. cmd->tag = req->tag;
  918. cmd->request = req;
  919. cmd->cmnd = req->cmd;
  920. return cmd;
  921. }
  922. int scsi_setup_blk_pc_cmnd(struct scsi_device *sdev, struct request *req)
  923. {
  924. struct scsi_cmnd *cmd;
  925. int ret = scsi_prep_state_check(sdev, req);
  926. if (ret != BLKPREP_OK)
  927. return ret;
  928. cmd = scsi_get_cmd_from_req(sdev, req);
  929. if (unlikely(!cmd))
  930. return BLKPREP_DEFER;
  931. /*
  932. * BLOCK_PC requests may transfer data, in which case they must
  933. * a bio attached to them. Or they might contain a SCSI command
  934. * that does not transfer data, in which case they may optionally
  935. * submit a request without an attached bio.
  936. */
  937. if (req->bio) {
  938. int ret;
  939. BUG_ON(!req->nr_phys_segments);
  940. ret = scsi_init_io(cmd, GFP_ATOMIC);
  941. if (unlikely(ret))
  942. return ret;
  943. } else {
  944. BUG_ON(blk_rq_bytes(req));
  945. memset(&cmd->sdb, 0, sizeof(cmd->sdb));
  946. req->buffer = NULL;
  947. }
  948. cmd->cmd_len = req->cmd_len;
  949. if (!blk_rq_bytes(req))
  950. cmd->sc_data_direction = DMA_NONE;
  951. else if (rq_data_dir(req) == WRITE)
  952. cmd->sc_data_direction = DMA_TO_DEVICE;
  953. else
  954. cmd->sc_data_direction = DMA_FROM_DEVICE;
  955. cmd->transfersize = blk_rq_bytes(req);
  956. cmd->allowed = req->retries;
  957. return BLKPREP_OK;
  958. }
  959. EXPORT_SYMBOL(scsi_setup_blk_pc_cmnd);
  960. /*
  961. * Setup a REQ_TYPE_FS command. These are simple read/write request
  962. * from filesystems that still need to be translated to SCSI CDBs from
  963. * the ULD.
  964. */
  965. int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req)
  966. {
  967. struct scsi_cmnd *cmd;
  968. int ret = scsi_prep_state_check(sdev, req);
  969. if (ret != BLKPREP_OK)
  970. return ret;
  971. if (unlikely(sdev->scsi_dh_data && sdev->scsi_dh_data->scsi_dh
  972. && sdev->scsi_dh_data->scsi_dh->prep_fn)) {
  973. ret = sdev->scsi_dh_data->scsi_dh->prep_fn(sdev, req);
  974. if (ret != BLKPREP_OK)
  975. return ret;
  976. }
  977. /*
  978. * Filesystem requests must transfer data.
  979. */
  980. BUG_ON(!req->nr_phys_segments);
  981. cmd = scsi_get_cmd_from_req(sdev, req);
  982. if (unlikely(!cmd))
  983. return BLKPREP_DEFER;
  984. memset(cmd->cmnd, 0, BLK_MAX_CDB);
  985. return scsi_init_io(cmd, GFP_ATOMIC);
  986. }
  987. EXPORT_SYMBOL(scsi_setup_fs_cmnd);
  988. int scsi_prep_state_check(struct scsi_device *sdev, struct request *req)
  989. {
  990. int ret = BLKPREP_OK;
  991. /*
  992. * If the device is not in running state we will reject some
  993. * or all commands.
  994. */
  995. if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
  996. switch (sdev->sdev_state) {
  997. case SDEV_OFFLINE:
  998. /*
  999. * If the device is offline we refuse to process any
  1000. * commands. The device must be brought online
  1001. * before trying any recovery commands.
  1002. */
  1003. sdev_printk(KERN_ERR, sdev,
  1004. "rejecting I/O to offline device\n");
  1005. ret = BLKPREP_KILL;
  1006. break;
  1007. case SDEV_DEL:
  1008. /*
  1009. * If the device is fully deleted, we refuse to
  1010. * process any commands as well.
  1011. */
  1012. sdev_printk(KERN_ERR, sdev,
  1013. "rejecting I/O to dead device\n");
  1014. ret = BLKPREP_KILL;
  1015. break;
  1016. case SDEV_QUIESCE:
  1017. case SDEV_BLOCK:
  1018. case SDEV_CREATED_BLOCK:
  1019. /*
  1020. * If the devices is blocked we defer normal commands.
  1021. */
  1022. if (!(req->cmd_flags & REQ_PREEMPT))
  1023. ret = BLKPREP_DEFER;
  1024. break;
  1025. default:
  1026. /*
  1027. * For any other not fully online state we only allow
  1028. * special commands. In particular any user initiated
  1029. * command is not allowed.
  1030. */
  1031. if (!(req->cmd_flags & REQ_PREEMPT))
  1032. ret = BLKPREP_KILL;
  1033. break;
  1034. }
  1035. }
  1036. return ret;
  1037. }
  1038. EXPORT_SYMBOL(scsi_prep_state_check);
  1039. int scsi_prep_return(struct request_queue *q, struct request *req, int ret)
  1040. {
  1041. struct scsi_device *sdev = q->queuedata;
  1042. switch (ret) {
  1043. case BLKPREP_KILL:
  1044. req->errors = DID_NO_CONNECT << 16;
  1045. /* release the command and kill it */
  1046. if (req->special) {
  1047. struct scsi_cmnd *cmd = req->special;
  1048. scsi_release_buffers(cmd);
  1049. scsi_put_command(cmd);
  1050. req->special = NULL;
  1051. }
  1052. break;
  1053. case BLKPREP_DEFER:
  1054. /*
  1055. * If we defer, the blk_peek_request() returns NULL, but the
  1056. * queue must be restarted, so we plug here if no returning
  1057. * command will automatically do that.
  1058. */
  1059. if (sdev->device_busy == 0)
  1060. blk_plug_device(q);
  1061. break;
  1062. default:
  1063. req->cmd_flags |= REQ_DONTPREP;
  1064. }
  1065. return ret;
  1066. }
  1067. EXPORT_SYMBOL(scsi_prep_return);
  1068. int scsi_prep_fn(struct request_queue *q, struct request *req)
  1069. {
  1070. struct scsi_device *sdev = q->queuedata;
  1071. int ret = BLKPREP_KILL;
  1072. if (req->cmd_type == REQ_TYPE_BLOCK_PC)
  1073. ret = scsi_setup_blk_pc_cmnd(sdev, req);
  1074. return scsi_prep_return(q, req, ret);
  1075. }
  1076. EXPORT_SYMBOL(scsi_prep_fn);
  1077. /*
  1078. * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
  1079. * return 0.
  1080. *
  1081. * Called with the queue_lock held.
  1082. */
  1083. static inline int scsi_dev_queue_ready(struct request_queue *q,
  1084. struct scsi_device *sdev)
  1085. {
  1086. if (sdev->device_busy == 0 && sdev->device_blocked) {
  1087. /*
  1088. * unblock after device_blocked iterates to zero
  1089. */
  1090. if (--sdev->device_blocked == 0) {
  1091. SCSI_LOG_MLQUEUE(3,
  1092. sdev_printk(KERN_INFO, sdev,
  1093. "unblocking device at zero depth\n"));
  1094. } else {
  1095. blk_plug_device(q);
  1096. return 0;
  1097. }
  1098. }
  1099. if (scsi_device_is_busy(sdev))
  1100. return 0;
  1101. return 1;
  1102. }
  1103. /*
  1104. * scsi_target_queue_ready: checks if there we can send commands to target
  1105. * @sdev: scsi device on starget to check.
  1106. *
  1107. * Called with the host lock held.
  1108. */
  1109. static inline int scsi_target_queue_ready(struct Scsi_Host *shost,
  1110. struct scsi_device *sdev)
  1111. {
  1112. struct scsi_target *starget = scsi_target(sdev);
  1113. if (starget->single_lun) {
  1114. if (starget->starget_sdev_user &&
  1115. starget->starget_sdev_user != sdev)
  1116. return 0;
  1117. starget->starget_sdev_user = sdev;
  1118. }
  1119. if (starget->target_busy == 0 && starget->target_blocked) {
  1120. /*
  1121. * unblock after target_blocked iterates to zero
  1122. */
  1123. if (--starget->target_blocked == 0) {
  1124. SCSI_LOG_MLQUEUE(3, starget_printk(KERN_INFO, starget,
  1125. "unblocking target at zero depth\n"));
  1126. } else
  1127. return 0;
  1128. }
  1129. if (scsi_target_is_busy(starget)) {
  1130. if (list_empty(&sdev->starved_entry)) {
  1131. list_add_tail(&sdev->starved_entry,
  1132. &shost->starved_list);
  1133. return 0;
  1134. }
  1135. }
  1136. /* We're OK to process the command, so we can't be starved */
  1137. if (!list_empty(&sdev->starved_entry))
  1138. list_del_init(&sdev->starved_entry);
  1139. return 1;
  1140. }
  1141. /*
  1142. * scsi_host_queue_ready: if we can send requests to shost, return 1 else
  1143. * return 0. We must end up running the queue again whenever 0 is
  1144. * returned, else IO can hang.
  1145. *
  1146. * Called with host_lock held.
  1147. */
  1148. static inline int scsi_host_queue_ready(struct request_queue *q,
  1149. struct Scsi_Host *shost,
  1150. struct scsi_device *sdev)
  1151. {
  1152. if (scsi_host_in_recovery(shost))
  1153. return 0;
  1154. if (shost->host_busy == 0 && shost->host_blocked) {
  1155. /*
  1156. * unblock after host_blocked iterates to zero
  1157. */
  1158. if (--shost->host_blocked == 0) {
  1159. SCSI_LOG_MLQUEUE(3,
  1160. printk("scsi%d unblocking host at zero depth\n",
  1161. shost->host_no));
  1162. } else {
  1163. return 0;
  1164. }
  1165. }
  1166. if (scsi_host_is_busy(shost)) {
  1167. if (list_empty(&sdev->starved_entry))
  1168. list_add_tail(&sdev->starved_entry, &shost->starved_list);
  1169. return 0;
  1170. }
  1171. /* We're OK to process the command, so we can't be starved */
  1172. if (!list_empty(&sdev->starved_entry))
  1173. list_del_init(&sdev->starved_entry);
  1174. return 1;
  1175. }
  1176. /*
  1177. * Busy state exporting function for request stacking drivers.
  1178. *
  1179. * For efficiency, no lock is taken to check the busy state of
  1180. * shost/starget/sdev, since the returned value is not guaranteed and
  1181. * may be changed after request stacking drivers call the function,
  1182. * regardless of taking lock or not.
  1183. *
  1184. * When scsi can't dispatch I/Os anymore and needs to kill I/Os
  1185. * (e.g. !sdev), scsi needs to return 'not busy'.
  1186. * Otherwise, request stacking drivers may hold requests forever.
  1187. */
  1188. static int scsi_lld_busy(struct request_queue *q)
  1189. {
  1190. struct scsi_device *sdev = q->queuedata;
  1191. struct Scsi_Host *shost;
  1192. struct scsi_target *starget;
  1193. if (!sdev)
  1194. return 0;
  1195. shost = sdev->host;
  1196. starget = scsi_target(sdev);
  1197. if (scsi_host_in_recovery(shost) || scsi_host_is_busy(shost) ||
  1198. scsi_target_is_busy(starget) || scsi_device_is_busy(sdev))
  1199. return 1;
  1200. return 0;
  1201. }
  1202. /*
  1203. * Kill a request for a dead device
  1204. */
  1205. static void scsi_kill_request(struct request *req, struct request_queue *q)
  1206. {
  1207. struct scsi_cmnd *cmd = req->special;
  1208. struct scsi_device *sdev;
  1209. struct scsi_target *starget;
  1210. struct Scsi_Host *shost;
  1211. blk_start_request(req);
  1212. if (unlikely(cmd == NULL)) {
  1213. printk(KERN_CRIT "impossible request in %s.\n",
  1214. __func__);
  1215. BUG();
  1216. }
  1217. sdev = cmd->device;
  1218. starget = scsi_target(sdev);
  1219. shost = sdev->host;
  1220. scsi_init_cmd_errh(cmd);
  1221. cmd->result = DID_NO_CONNECT << 16;
  1222. atomic_inc(&cmd->device->iorequest_cnt);
  1223. /*
  1224. * SCSI request completion path will do scsi_device_unbusy(),
  1225. * bump busy counts. To bump the counters, we need to dance
  1226. * with the locks as normal issue path does.
  1227. */
  1228. sdev->device_busy++;
  1229. spin_unlock(sdev->request_queue->queue_lock);
  1230. spin_lock(shost->host_lock);
  1231. shost->host_busy++;
  1232. starget->target_busy++;
  1233. spin_unlock(shost->host_lock);
  1234. spin_lock(sdev->request_queue->queue_lock);
  1235. blk_complete_request(req);
  1236. }
  1237. static void scsi_softirq_done(struct request *rq)
  1238. {
  1239. struct scsi_cmnd *cmd = rq->special;
  1240. unsigned long wait_for = (cmd->allowed + 1) * rq->timeout;
  1241. int disposition;
  1242. INIT_LIST_HEAD(&cmd->eh_entry);
  1243. /*
  1244. * Set the serial numbers back to zero
  1245. */
  1246. cmd->serial_number = 0;
  1247. atomic_inc(&cmd->device->iodone_cnt);
  1248. if (cmd->result)
  1249. atomic_inc(&cmd->device->ioerr_cnt);
  1250. disposition = scsi_decide_disposition(cmd);
  1251. if (disposition != SUCCESS &&
  1252. time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
  1253. sdev_printk(KERN_ERR, cmd->device,
  1254. "timing out command, waited %lus\n",
  1255. wait_for/HZ);
  1256. disposition = SUCCESS;
  1257. }
  1258. scsi_log_completion(cmd, disposition);
  1259. switch (disposition) {
  1260. case SUCCESS:
  1261. scsi_finish_command(cmd);
  1262. break;
  1263. case NEEDS_RETRY:
  1264. scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY);
  1265. break;
  1266. case ADD_TO_MLQUEUE:
  1267. scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
  1268. break;
  1269. default:
  1270. if (!scsi_eh_scmd_add(cmd, 0))
  1271. scsi_finish_command(cmd);
  1272. }
  1273. }
  1274. /*
  1275. * Function: scsi_request_fn()
  1276. *
  1277. * Purpose: Main strategy routine for SCSI.
  1278. *
  1279. * Arguments: q - Pointer to actual queue.
  1280. *
  1281. * Returns: Nothing
  1282. *
  1283. * Lock status: IO request lock assumed to be held when called.
  1284. */
  1285. static void scsi_request_fn(struct request_queue *q)
  1286. {
  1287. struct scsi_device *sdev = q->queuedata;
  1288. struct Scsi_Host *shost;
  1289. struct scsi_cmnd *cmd;
  1290. struct request *req;
  1291. if (!sdev) {
  1292. printk("scsi: killing requests for dead queue\n");
  1293. while ((req = blk_peek_request(q)) != NULL)
  1294. scsi_kill_request(req, q);
  1295. return;
  1296. }
  1297. if(!get_device(&sdev->sdev_gendev))
  1298. /* We must be tearing the block queue down already */
  1299. return;
  1300. /*
  1301. * To start with, we keep looping until the queue is empty, or until
  1302. * the host is no longer able to accept any more requests.
  1303. */
  1304. shost = sdev->host;
  1305. while (!blk_queue_plugged(q)) {
  1306. int rtn;
  1307. /*
  1308. * get next queueable request. We do this early to make sure
  1309. * that the request is fully prepared even if we cannot
  1310. * accept it.
  1311. */
  1312. req = blk_peek_request(q);
  1313. if (!req || !scsi_dev_queue_ready(q, sdev))
  1314. break;
  1315. if (unlikely(!scsi_device_online(sdev))) {
  1316. sdev_printk(KERN_ERR, sdev,
  1317. "rejecting I/O to offline device\n");
  1318. scsi_kill_request(req, q);
  1319. continue;
  1320. }
  1321. /*
  1322. * Remove the request from the request list.
  1323. */
  1324. if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))
  1325. blk_start_request(req);
  1326. sdev->device_busy++;
  1327. spin_unlock(q->queue_lock);
  1328. cmd = req->special;
  1329. if (unlikely(cmd == NULL)) {
  1330. printk(KERN_CRIT "impossible request in %s.\n"
  1331. "please mail a stack trace to "
  1332. "linux-scsi@vger.kernel.org\n",
  1333. __func__);
  1334. blk_dump_rq_flags(req, "foo");
  1335. BUG();
  1336. }
  1337. spin_lock(shost->host_lock);
  1338. /*
  1339. * We hit this when the driver is using a host wide
  1340. * tag map. For device level tag maps the queue_depth check
  1341. * in the device ready fn would prevent us from trying
  1342. * to allocate a tag. Since the map is a shared host resource
  1343. * we add the dev to the starved list so it eventually gets
  1344. * a run when a tag is freed.
  1345. */
  1346. if (blk_queue_tagged(q) && !blk_rq_tagged(req)) {
  1347. if (list_empty(&sdev->starved_entry))
  1348. list_add_tail(&sdev->starved_entry,
  1349. &shost->starved_list);
  1350. goto not_ready;
  1351. }
  1352. if (!scsi_target_queue_ready(shost, sdev))
  1353. goto not_ready;
  1354. if (!scsi_host_queue_ready(q, shost, sdev))
  1355. goto not_ready;
  1356. scsi_target(sdev)->target_busy++;
  1357. shost->host_busy++;
  1358. /*
  1359. * XXX(hch): This is rather suboptimal, scsi_dispatch_cmd will
  1360. * take the lock again.
  1361. */
  1362. spin_unlock_irq(shost->host_lock);
  1363. /*
  1364. * Finally, initialize any error handling parameters, and set up
  1365. * the timers for timeouts.
  1366. */
  1367. scsi_init_cmd_errh(cmd);
  1368. /*
  1369. * Dispatch the command to the low-level driver.
  1370. */
  1371. rtn = scsi_dispatch_cmd(cmd);
  1372. spin_lock_irq(q->queue_lock);
  1373. if(rtn) {
  1374. /* we're refusing the command; because of
  1375. * the way locks get dropped, we need to
  1376. * check here if plugging is required */
  1377. if(sdev->device_busy == 0)
  1378. blk_plug_device(q);
  1379. break;
  1380. }
  1381. }
  1382. goto out;
  1383. not_ready:
  1384. spin_unlock_irq(shost->host_lock);
  1385. /*
  1386. * lock q, handle tag, requeue req, and decrement device_busy. We
  1387. * must return with queue_lock held.
  1388. *
  1389. * Decrementing device_busy without checking it is OK, as all such
  1390. * cases (host limits or settings) should run the queue at some
  1391. * later time.
  1392. */
  1393. spin_lock_irq(q->queue_lock);
  1394. blk_requeue_request(q, req);
  1395. sdev->device_busy--;
  1396. if(sdev->device_busy == 0)
  1397. blk_plug_device(q);
  1398. out:
  1399. /* must be careful here...if we trigger the ->remove() function
  1400. * we cannot be holding the q lock */
  1401. spin_unlock_irq(q->queue_lock);
  1402. put_device(&sdev->sdev_gendev);
  1403. spin_lock_irq(q->queue_lock);
  1404. }
  1405. u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost)
  1406. {
  1407. struct device *host_dev;
  1408. u64 bounce_limit = 0xffffffff;
  1409. if (shost->unchecked_isa_dma)
  1410. return BLK_BOUNCE_ISA;
  1411. /*
  1412. * Platforms with virtual-DMA translation
  1413. * hardware have no practical limit.
  1414. */
  1415. if (!PCI_DMA_BUS_IS_PHYS)
  1416. return BLK_BOUNCE_ANY;
  1417. host_dev = scsi_get_device(shost);
  1418. if (host_dev && host_dev->dma_mask)
  1419. bounce_limit = *host_dev->dma_mask;
  1420. return bounce_limit;
  1421. }
  1422. EXPORT_SYMBOL(scsi_calculate_bounce_limit);
  1423. struct request_queue *__scsi_alloc_queue(struct Scsi_Host *shost,
  1424. request_fn_proc *request_fn)
  1425. {
  1426. struct request_queue *q;
  1427. struct device *dev = shost->shost_gendev.parent;
  1428. q = blk_init_queue(request_fn, NULL);
  1429. if (!q)
  1430. return NULL;
  1431. /*
  1432. * this limit is imposed by hardware restrictions
  1433. */
  1434. blk_queue_max_segments(q, min_t(unsigned short, shost->sg_tablesize,
  1435. SCSI_MAX_SG_CHAIN_SEGMENTS));
  1436. blk_queue_max_hw_sectors(q, shost->max_sectors);
  1437. blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
  1438. blk_queue_segment_boundary(q, shost->dma_boundary);
  1439. dma_set_seg_boundary(dev, shost->dma_boundary);
  1440. blk_queue_max_segment_size(q, dma_get_max_seg_size(dev));
  1441. /* New queue, no concurrency on queue_flags */
  1442. if (!shost->use_clustering)
  1443. queue_flag_clear_unlocked(QUEUE_FLAG_CLUSTER, q);
  1444. /*
  1445. * set a reasonable default alignment on word boundaries: the
  1446. * host and device may alter it using
  1447. * blk_queue_update_dma_alignment() later.
  1448. */
  1449. blk_queue_dma_alignment(q, 0x03);
  1450. return q;
  1451. }
  1452. EXPORT_SYMBOL(__scsi_alloc_queue);
  1453. struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
  1454. {
  1455. struct request_queue *q;
  1456. q = __scsi_alloc_queue(sdev->host, scsi_request_fn);
  1457. if (!q)
  1458. return NULL;
  1459. blk_queue_prep_rq(q, scsi_prep_fn);
  1460. blk_queue_softirq_done(q, scsi_softirq_done);
  1461. blk_queue_rq_timed_out(q, scsi_times_out);
  1462. blk_queue_lld_busy(q, scsi_lld_busy);
  1463. return q;
  1464. }
  1465. void scsi_free_queue(struct request_queue *q)
  1466. {
  1467. blk_cleanup_queue(q);
  1468. }
  1469. /*
  1470. * Function: scsi_block_requests()
  1471. *
  1472. * Purpose: Utility function used by low-level drivers to prevent further
  1473. * commands from being queued to the device.
  1474. *
  1475. * Arguments: shost - Host in question
  1476. *
  1477. * Returns: Nothing
  1478. *
  1479. * Lock status: No locks are assumed held.
  1480. *
  1481. * Notes: There is no timer nor any other means by which the requests
  1482. * get unblocked other than the low-level driver calling
  1483. * scsi_unblock_requests().
  1484. */
  1485. void scsi_block_requests(struct Scsi_Host *shost)
  1486. {
  1487. shost->host_self_blocked = 1;
  1488. }
  1489. EXPORT_SYMBOL(scsi_block_requests);
  1490. /*
  1491. * Function: scsi_unblock_requests()
  1492. *
  1493. * Purpose: Utility function used by low-level drivers to allow further
  1494. * commands from being queued to the device.
  1495. *
  1496. * Arguments: shost - Host in question
  1497. *
  1498. * Returns: Nothing
  1499. *
  1500. * Lock status: No locks are assumed held.
  1501. *
  1502. * Notes: There is no timer nor any other means by which the requests
  1503. * get unblocked other than the low-level driver calling
  1504. * scsi_unblock_requests().
  1505. *
  1506. * This is done as an API function so that changes to the
  1507. * internals of the scsi mid-layer won't require wholesale
  1508. * changes to drivers that use this feature.
  1509. */
  1510. void scsi_unblock_requests(struct Scsi_Host *shost)
  1511. {
  1512. shost->host_self_blocked = 0;
  1513. scsi_run_host_queues(shost);
  1514. }
  1515. EXPORT_SYMBOL(scsi_unblock_requests);
  1516. int __init scsi_init_queue(void)
  1517. {
  1518. int i;
  1519. scsi_sdb_cache = kmem_cache_create("scsi_data_buffer",
  1520. sizeof(struct scsi_data_buffer),
  1521. 0, 0, NULL);
  1522. if (!scsi_sdb_cache) {
  1523. printk(KERN_ERR "SCSI: can't init scsi sdb cache\n");
  1524. return -ENOMEM;
  1525. }
  1526. for (i = 0; i < SG_MEMPOOL_NR; i++) {
  1527. struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
  1528. int size = sgp->size * sizeof(struct scatterlist);
  1529. sgp->slab = kmem_cache_create(sgp->name, size, 0,
  1530. SLAB_HWCACHE_ALIGN, NULL);
  1531. if (!sgp->slab) {
  1532. printk(KERN_ERR "SCSI: can't init sg slab %s\n",
  1533. sgp->name);
  1534. goto cleanup_sdb;
  1535. }
  1536. sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
  1537. sgp->slab);
  1538. if (!sgp->pool) {
  1539. printk(KERN_ERR "SCSI: can't init sg mempool %s\n",
  1540. sgp->name);
  1541. goto cleanup_sdb;
  1542. }
  1543. }
  1544. return 0;
  1545. cleanup_sdb:
  1546. for (i = 0; i < SG_MEMPOOL_NR; i++) {
  1547. struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
  1548. if (sgp->pool)
  1549. mempool_destroy(sgp->pool);
  1550. if (sgp->slab)
  1551. kmem_cache_destroy(sgp->slab);
  1552. }
  1553. kmem_cache_destroy(scsi_sdb_cache);
  1554. return -ENOMEM;
  1555. }
  1556. void scsi_exit_queue(void)
  1557. {
  1558. int i;
  1559. kmem_cache_destroy(scsi_sdb_cache);
  1560. for (i = 0; i < SG_MEMPOOL_NR; i++) {
  1561. struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
  1562. mempool_destroy(sgp->pool);
  1563. kmem_cache_destroy(sgp->slab);
  1564. }
  1565. }
  1566. /**
  1567. * scsi_mode_select - issue a mode select
  1568. * @sdev: SCSI device to be queried
  1569. * @pf: Page format bit (1 == standard, 0 == vendor specific)
  1570. * @sp: Save page bit (0 == don't save, 1 == save)
  1571. * @modepage: mode page being requested
  1572. * @buffer: request buffer (may not be smaller than eight bytes)
  1573. * @len: length of request buffer.
  1574. * @timeout: command timeout
  1575. * @retries: number of retries before failing
  1576. * @data: returns a structure abstracting the mode header data
  1577. * @sshdr: place to put sense data (or NULL if no sense to be collected).
  1578. * must be SCSI_SENSE_BUFFERSIZE big.
  1579. *
  1580. * Returns zero if successful; negative error number or scsi
  1581. * status on error
  1582. *
  1583. */
  1584. int
  1585. scsi_mode_select(struct scsi_device *sdev, int pf, int sp, int modepage,
  1586. unsigned char *buffer, int len, int timeout, int retries,
  1587. struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
  1588. {
  1589. unsigned char cmd[10];
  1590. unsigned char *real_buffer;
  1591. int ret;
  1592. memset(cmd, 0, sizeof(cmd));
  1593. cmd[1] = (pf ? 0x10 : 0) | (sp ? 0x01 : 0);
  1594. if (sdev->use_10_for_ms) {
  1595. if (len > 65535)
  1596. return -EINVAL;
  1597. real_buffer = kmalloc(8 + len, GFP_KERNEL);
  1598. if (!real_buffer)
  1599. return -ENOMEM;
  1600. memcpy(real_buffer + 8, buffer, len);
  1601. len += 8;
  1602. real_buffer[0] = 0;
  1603. real_buffer[1] = 0;
  1604. real_buffer[2] = data->medium_type;
  1605. real_buffer[3] = data->device_specific;
  1606. real_buffer[4] = data->longlba ? 0x01 : 0;
  1607. real_buffer[5] = 0;
  1608. real_buffer[6] = data->block_descriptor_length >> 8;
  1609. real_buffer[7] = data->block_descriptor_length;
  1610. cmd[0] = MODE_SELECT_10;
  1611. cmd[7] = len >> 8;
  1612. cmd[8] = len;
  1613. } else {
  1614. if (len > 255 || data->block_descriptor_length > 255 ||
  1615. data->longlba)
  1616. return -EINVAL;
  1617. real_buffer = kmalloc(4 + len, GFP_KERNEL);
  1618. if (!real_buffer)
  1619. return -ENOMEM;
  1620. memcpy(real_buffer + 4, buffer, len);
  1621. len += 4;
  1622. real_buffer[0] = 0;
  1623. real_buffer[1] = data->medium_type;
  1624. real_buffer[2] = data->device_specific;
  1625. real_buffer[3] = data->block_descriptor_length;
  1626. cmd[0] = MODE_SELECT;
  1627. cmd[4] = len;
  1628. }
  1629. ret = scsi_execute_req(sdev, cmd, DMA_TO_DEVICE, real_buffer, len,
  1630. sshdr, timeout, retries, NULL);
  1631. kfree(real_buffer);
  1632. return ret;
  1633. }
  1634. EXPORT_SYMBOL_GPL(scsi_mode_select);
  1635. /**
  1636. * scsi_mode_sense - issue a mode sense, falling back from 10 to six bytes if necessary.
  1637. * @sdev: SCSI device to be queried
  1638. * @dbd: set if mode sense will allow block descriptors to be returned
  1639. * @modepage: mode page being requested
  1640. * @buffer: request buffer (may not be smaller than eight bytes)
  1641. * @len: length of request buffer.
  1642. * @timeout: command timeout
  1643. * @retries: number of retries before failing
  1644. * @data: returns a structure abstracting the mode header data
  1645. * @sshdr: place to put sense data (or NULL if no sense to be collected).
  1646. * must be SCSI_SENSE_BUFFERSIZE big.
  1647. *
  1648. * Returns zero if unsuccessful, or the header offset (either 4
  1649. * or 8 depending on whether a six or ten byte command was
  1650. * issued) if successful.
  1651. */
  1652. int
  1653. scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
  1654. unsigned char *buffer, int len, int timeout, int retries,
  1655. struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
  1656. {
  1657. unsigned char cmd[12];
  1658. int use_10_for_ms;
  1659. int header_length;
  1660. int result;
  1661. struct scsi_sense_hdr my_sshdr;
  1662. memset(da