/drivers/dma/at_hdmac.c

https://bitbucket.org/ndreys/linux-sunxi · C · 1436 lines · 918 code · 242 blank · 276 comment · 110 complexity · 1af492729464b12600cb162f56519358 MD5 · raw file

  1. /*
  2. * Driver for the Atmel AHB DMA Controller (aka HDMA or DMAC on AT91 systems)
  3. *
  4. * Copyright (C) 2008 Atmel Corporation
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License as published by
  8. * the Free Software Foundation; either version 2 of the License, or
  9. * (at your option) any later version.
  10. *
  11. *
  12. * This supports the Atmel AHB DMA Controller,
  13. *
  14. * The driver has currently been tested with the Atmel AT91SAM9RL
  15. * and AT91SAM9G45 series.
  16. */
  17. #include <linux/clk.h>
  18. #include <linux/dmaengine.h>
  19. #include <linux/dma-mapping.h>
  20. #include <linux/dmapool.h>
  21. #include <linux/interrupt.h>
  22. #include <linux/module.h>
  23. #include <linux/platform_device.h>
  24. #include <linux/slab.h>
  25. #include "at_hdmac_regs.h"
  26. /*
  27. * Glossary
  28. * --------
  29. *
  30. * at_hdmac : Name of the ATmel AHB DMA Controller
  31. * at_dma_ / atdma : ATmel DMA controller entity related
  32. * atc_ / atchan : ATmel DMA Channel entity related
  33. */
  34. #define ATC_DEFAULT_CFG (ATC_FIFOCFG_HALFFIFO)
  35. #define ATC_DEFAULT_CTRLA (0)
  36. #define ATC_DEFAULT_CTRLB (ATC_SIF(AT_DMA_MEM_IF) \
  37. |ATC_DIF(AT_DMA_MEM_IF))
  38. /*
  39. * Initial number of descriptors to allocate for each channel. This could
  40. * be increased during dma usage.
  41. */
  42. static unsigned int init_nr_desc_per_channel = 64;
  43. module_param(init_nr_desc_per_channel, uint, 0644);
  44. MODULE_PARM_DESC(init_nr_desc_per_channel,
  45. "initial descriptors per channel (default: 64)");
  46. /* prototypes */
  47. static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx);
  48. /*----------------------------------------------------------------------*/
  49. static struct at_desc *atc_first_active(struct at_dma_chan *atchan)
  50. {
  51. return list_first_entry(&atchan->active_list,
  52. struct at_desc, desc_node);
  53. }
  54. static struct at_desc *atc_first_queued(struct at_dma_chan *atchan)
  55. {
  56. return list_first_entry(&atchan->queue,
  57. struct at_desc, desc_node);
  58. }
  59. /**
  60. * atc_alloc_descriptor - allocate and return an initialized descriptor
  61. * @chan: the channel to allocate descriptors for
  62. * @gfp_flags: GFP allocation flags
  63. *
  64. * Note: The ack-bit is positioned in the descriptor flag at creation time
  65. * to make initial allocation more convenient. This bit will be cleared
  66. * and control will be given to client at usage time (during
  67. * preparation functions).
  68. */
  69. static struct at_desc *atc_alloc_descriptor(struct dma_chan *chan,
  70. gfp_t gfp_flags)
  71. {
  72. struct at_desc *desc = NULL;
  73. struct at_dma *atdma = to_at_dma(chan->device);
  74. dma_addr_t phys;
  75. desc = dma_pool_alloc(atdma->dma_desc_pool, gfp_flags, &phys);
  76. if (desc) {
  77. memset(desc, 0, sizeof(struct at_desc));
  78. INIT_LIST_HEAD(&desc->tx_list);
  79. dma_async_tx_descriptor_init(&desc->txd, chan);
  80. /* txd.flags will be overwritten in prep functions */
  81. desc->txd.flags = DMA_CTRL_ACK;
  82. desc->txd.tx_submit = atc_tx_submit;
  83. desc->txd.phys = phys;
  84. }
  85. return desc;
  86. }
  87. /**
  88. * atc_desc_get - get an unused descriptor from free_list
  89. * @atchan: channel we want a new descriptor for
  90. */
  91. static struct at_desc *atc_desc_get(struct at_dma_chan *atchan)
  92. {
  93. struct at_desc *desc, *_desc;
  94. struct at_desc *ret = NULL;
  95. unsigned int i = 0;
  96. LIST_HEAD(tmp_list);
  97. spin_lock_bh(&atchan->lock);
  98. list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) {
  99. i++;
  100. if (async_tx_test_ack(&desc->txd)) {
  101. list_del(&desc->desc_node);
  102. ret = desc;
  103. break;
  104. }
  105. dev_dbg(chan2dev(&atchan->chan_common),
  106. "desc %p not ACKed\n", desc);
  107. }
  108. spin_unlock_bh(&atchan->lock);
  109. dev_vdbg(chan2dev(&atchan->chan_common),
  110. "scanned %u descriptors on freelist\n", i);
  111. /* no more descriptor available in initial pool: create one more */
  112. if (!ret) {
  113. ret = atc_alloc_descriptor(&atchan->chan_common, GFP_ATOMIC);
  114. if (ret) {
  115. spin_lock_bh(&atchan->lock);
  116. atchan->descs_allocated++;
  117. spin_unlock_bh(&atchan->lock);
  118. } else {
  119. dev_err(chan2dev(&atchan->chan_common),
  120. "not enough descriptors available\n");
  121. }
  122. }
  123. return ret;
  124. }
  125. /**
  126. * atc_desc_put - move a descriptor, including any children, to the free list
  127. * @atchan: channel we work on
  128. * @desc: descriptor, at the head of a chain, to move to free list
  129. */
  130. static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc)
  131. {
  132. if (desc) {
  133. struct at_desc *child;
  134. spin_lock_bh(&atchan->lock);
  135. list_for_each_entry(child, &desc->tx_list, desc_node)
  136. dev_vdbg(chan2dev(&atchan->chan_common),
  137. "moving child desc %p to freelist\n",
  138. child);
  139. list_splice_init(&desc->tx_list, &atchan->free_list);
  140. dev_vdbg(chan2dev(&atchan->chan_common),
  141. "moving desc %p to freelist\n", desc);
  142. list_add(&desc->desc_node, &atchan->free_list);
  143. spin_unlock_bh(&atchan->lock);
  144. }
  145. }
  146. /**
  147. * atc_desc_chain - build chain adding a descripor
  148. * @first: address of first descripor of the chain
  149. * @prev: address of previous descripor of the chain
  150. * @desc: descriptor to queue
  151. *
  152. * Called from prep_* functions
  153. */
  154. static void atc_desc_chain(struct at_desc **first, struct at_desc **prev,
  155. struct at_desc *desc)
  156. {
  157. if (!(*first)) {
  158. *first = desc;
  159. } else {
  160. /* inform the HW lli about chaining */
  161. (*prev)->lli.dscr = desc->txd.phys;
  162. /* insert the link descriptor to the LD ring */
  163. list_add_tail(&desc->desc_node,
  164. &(*first)->tx_list);
  165. }
  166. *prev = desc;
  167. }
  168. /**
  169. * atc_assign_cookie - compute and assign new cookie
  170. * @atchan: channel we work on
  171. * @desc: descriptor to assign cookie for
  172. *
  173. * Called with atchan->lock held and bh disabled
  174. */
  175. static dma_cookie_t
  176. atc_assign_cookie(struct at_dma_chan *atchan, struct at_desc *desc)
  177. {
  178. dma_cookie_t cookie = atchan->chan_common.cookie;
  179. if (++cookie < 0)
  180. cookie = 1;
  181. atchan->chan_common.cookie = cookie;
  182. desc->txd.cookie = cookie;
  183. return cookie;
  184. }
  185. /**
  186. * atc_dostart - starts the DMA engine for real
  187. * @atchan: the channel we want to start
  188. * @first: first descriptor in the list we want to begin with
  189. *
  190. * Called with atchan->lock held and bh disabled
  191. */
  192. static void atc_dostart(struct at_dma_chan *atchan, struct at_desc *first)
  193. {
  194. struct at_dma *atdma = to_at_dma(atchan->chan_common.device);
  195. /* ASSERT: channel is idle */
  196. if (atc_chan_is_enabled(atchan)) {
  197. dev_err(chan2dev(&atchan->chan_common),
  198. "BUG: Attempted to start non-idle channel\n");
  199. dev_err(chan2dev(&atchan->chan_common),
  200. " channel: s0x%x d0x%x ctrl0x%x:0x%x l0x%x\n",
  201. channel_readl(atchan, SADDR),
  202. channel_readl(atchan, DADDR),
  203. channel_readl(atchan, CTRLA),
  204. channel_readl(atchan, CTRLB),
  205. channel_readl(atchan, DSCR));
  206. /* The tasklet will hopefully advance the queue... */
  207. return;
  208. }
  209. vdbg_dump_regs(atchan);
  210. channel_writel(atchan, SADDR, 0);
  211. channel_writel(atchan, DADDR, 0);
  212. channel_writel(atchan, CTRLA, 0);
  213. channel_writel(atchan, CTRLB, 0);
  214. channel_writel(atchan, DSCR, first->txd.phys);
  215. dma_writel(atdma, CHER, atchan->mask);
  216. vdbg_dump_regs(atchan);
  217. }
  218. /**
  219. * atc_chain_complete - finish work for one transaction chain
  220. * @atchan: channel we work on
  221. * @desc: descriptor at the head of the chain we want do complete
  222. *
  223. * Called with atchan->lock held and bh disabled */
  224. static void
  225. atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc)
  226. {
  227. struct dma_async_tx_descriptor *txd = &desc->txd;
  228. dev_vdbg(chan2dev(&atchan->chan_common),
  229. "descriptor %u complete\n", txd->cookie);
  230. atchan->completed_cookie = txd->cookie;
  231. /* move children to free_list */
  232. list_splice_init(&desc->tx_list, &atchan->free_list);
  233. /* move myself to free_list */
  234. list_move(&desc->desc_node, &atchan->free_list);
  235. /* unmap dma addresses (not on slave channels) */
  236. if (!atchan->chan_common.private) {
  237. struct device *parent = chan2parent(&atchan->chan_common);
  238. if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
  239. if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
  240. dma_unmap_single(parent,
  241. desc->lli.daddr,
  242. desc->len, DMA_FROM_DEVICE);
  243. else
  244. dma_unmap_page(parent,
  245. desc->lli.daddr,
  246. desc->len, DMA_FROM_DEVICE);
  247. }
  248. if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
  249. if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
  250. dma_unmap_single(parent,
  251. desc->lli.saddr,
  252. desc->len, DMA_TO_DEVICE);
  253. else
  254. dma_unmap_page(parent,
  255. desc->lli.saddr,
  256. desc->len, DMA_TO_DEVICE);
  257. }
  258. }
  259. /* for cyclic transfers,
  260. * no need to replay callback function while stopping */
  261. if (!test_bit(ATC_IS_CYCLIC, &atchan->status)) {
  262. dma_async_tx_callback callback = txd->callback;
  263. void *param = txd->callback_param;
  264. /*
  265. * The API requires that no submissions are done from a
  266. * callback, so we don't need to drop the lock here
  267. */
  268. if (callback)
  269. callback(param);
  270. }
  271. dma_run_dependencies(txd);
  272. }
  273. /**
  274. * atc_complete_all - finish work for all transactions
  275. * @atchan: channel to complete transactions for
  276. *
  277. * Eventually submit queued descriptors if any
  278. *
  279. * Assume channel is idle while calling this function
  280. * Called with atchan->lock held and bh disabled
  281. */
  282. static void atc_complete_all(struct at_dma_chan *atchan)
  283. {
  284. struct at_desc *desc, *_desc;
  285. LIST_HEAD(list);
  286. dev_vdbg(chan2dev(&atchan->chan_common), "complete all\n");
  287. BUG_ON(atc_chan_is_enabled(atchan));
  288. /*
  289. * Submit queued descriptors ASAP, i.e. before we go through
  290. * the completed ones.
  291. */
  292. if (!list_empty(&atchan->queue))
  293. atc_dostart(atchan, atc_first_queued(atchan));
  294. /* empty active_list now it is completed */
  295. list_splice_init(&atchan->active_list, &list);
  296. /* empty queue list by moving descriptors (if any) to active_list */
  297. list_splice_init(&atchan->queue, &atchan->active_list);
  298. list_for_each_entry_safe(desc, _desc, &list, desc_node)
  299. atc_chain_complete(atchan, desc);
  300. }
  301. /**
  302. * atc_cleanup_descriptors - cleanup up finished descriptors in active_list
  303. * @atchan: channel to be cleaned up
  304. *
  305. * Called with atchan->lock held and bh disabled
  306. */
  307. static void atc_cleanup_descriptors(struct at_dma_chan *atchan)
  308. {
  309. struct at_desc *desc, *_desc;
  310. struct at_desc *child;
  311. dev_vdbg(chan2dev(&atchan->chan_common), "cleanup descriptors\n");
  312. list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) {
  313. if (!(desc->lli.ctrla & ATC_DONE))
  314. /* This one is currently in progress */
  315. return;
  316. list_for_each_entry(child, &desc->tx_list, desc_node)
  317. if (!(child->lli.ctrla & ATC_DONE))
  318. /* Currently in progress */
  319. return;
  320. /*
  321. * No descriptors so far seem to be in progress, i.e.
  322. * this chain must be done.
  323. */
  324. atc_chain_complete(atchan, desc);
  325. }
  326. }
  327. /**
  328. * atc_advance_work - at the end of a transaction, move forward
  329. * @atchan: channel where the transaction ended
  330. *
  331. * Called with atchan->lock held and bh disabled
  332. */
  333. static void atc_advance_work(struct at_dma_chan *atchan)
  334. {
  335. dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n");
  336. if (list_empty(&atchan->active_list) ||
  337. list_is_singular(&atchan->active_list)) {
  338. atc_complete_all(atchan);
  339. } else {
  340. atc_chain_complete(atchan, atc_first_active(atchan));
  341. /* advance work */
  342. atc_dostart(atchan, atc_first_active(atchan));
  343. }
  344. }
  345. /**
  346. * atc_handle_error - handle errors reported by DMA controller
  347. * @atchan: channel where error occurs
  348. *
  349. * Called with atchan->lock held and bh disabled
  350. */
  351. static void atc_handle_error(struct at_dma_chan *atchan)
  352. {
  353. struct at_desc *bad_desc;
  354. struct at_desc *child;
  355. /*
  356. * The descriptor currently at the head of the active list is
  357. * broked. Since we don't have any way to report errors, we'll
  358. * just have to scream loudly and try to carry on.
  359. */
  360. bad_desc = atc_first_active(atchan);
  361. list_del_init(&bad_desc->desc_node);
  362. /* As we are stopped, take advantage to push queued descriptors
  363. * in active_list */
  364. list_splice_init(&atchan->queue, atchan->active_list.prev);
  365. /* Try to restart the controller */
  366. if (!list_empty(&atchan->active_list))
  367. atc_dostart(atchan, atc_first_active(atchan));
  368. /*
  369. * KERN_CRITICAL may seem harsh, but since this only happens
  370. * when someone submits a bad physical address in a
  371. * descriptor, we should consider ourselves lucky that the
  372. * controller flagged an error instead of scribbling over
  373. * random memory locations.
  374. */
  375. dev_crit(chan2dev(&atchan->chan_common),
  376. "Bad descriptor submitted for DMA!\n");
  377. dev_crit(chan2dev(&atchan->chan_common),
  378. " cookie: %d\n", bad_desc->txd.cookie);
  379. atc_dump_lli(atchan, &bad_desc->lli);
  380. list_for_each_entry(child, &bad_desc->tx_list, desc_node)
  381. atc_dump_lli(atchan, &child->lli);
  382. /* Pretend the descriptor completed successfully */
  383. atc_chain_complete(atchan, bad_desc);
  384. }
  385. /**
  386. * atc_handle_cyclic - at the end of a period, run callback function
  387. * @atchan: channel used for cyclic operations
  388. *
  389. * Called with atchan->lock held and bh disabled
  390. */
  391. static void atc_handle_cyclic(struct at_dma_chan *atchan)
  392. {
  393. struct at_desc *first = atc_first_active(atchan);
  394. struct dma_async_tx_descriptor *txd = &first->txd;
  395. dma_async_tx_callback callback = txd->callback;
  396. void *param = txd->callback_param;
  397. dev_vdbg(chan2dev(&atchan->chan_common),
  398. "new cyclic period llp 0x%08x\n",
  399. channel_readl(atchan, DSCR));
  400. if (callback)
  401. callback(param);
  402. }
  403. /*-- IRQ & Tasklet ---------------------------------------------------*/
  404. static void atc_tasklet(unsigned long data)
  405. {
  406. struct at_dma_chan *atchan = (struct at_dma_chan *)data;
  407. spin_lock(&atchan->lock);
  408. if (test_and_clear_bit(ATC_IS_ERROR, &atchan->status))
  409. atc_handle_error(atchan);
  410. else if (test_bit(ATC_IS_CYCLIC, &atchan->status))
  411. atc_handle_cyclic(atchan);
  412. else
  413. atc_advance_work(atchan);
  414. spin_unlock(&atchan->lock);
  415. }
  416. static irqreturn_t at_dma_interrupt(int irq, void *dev_id)
  417. {
  418. struct at_dma *atdma = (struct at_dma *)dev_id;
  419. struct at_dma_chan *atchan;
  420. int i;
  421. u32 status, pending, imr;
  422. int ret = IRQ_NONE;
  423. do {
  424. imr = dma_readl(atdma, EBCIMR);
  425. status = dma_readl(atdma, EBCISR);
  426. pending = status & imr;
  427. if (!pending)
  428. break;
  429. dev_vdbg(atdma->dma_common.dev,
  430. "interrupt: status = 0x%08x, 0x%08x, 0x%08x\n",
  431. status, imr, pending);
  432. for (i = 0; i < atdma->dma_common.chancnt; i++) {
  433. atchan = &atdma->chan[i];
  434. if (pending & (AT_DMA_BTC(i) | AT_DMA_ERR(i))) {
  435. if (pending & AT_DMA_ERR(i)) {
  436. /* Disable channel on AHB error */
  437. dma_writel(atdma, CHDR,
  438. AT_DMA_RES(i) | atchan->mask);
  439. /* Give information to tasklet */
  440. set_bit(ATC_IS_ERROR, &atchan->status);
  441. }
  442. tasklet_schedule(&atchan->tasklet);
  443. ret = IRQ_HANDLED;
  444. }
  445. }
  446. } while (pending);
  447. return ret;
  448. }
  449. /*-- DMA Engine API --------------------------------------------------*/
  450. /**
  451. * atc_tx_submit - set the prepared descriptor(s) to be executed by the engine
  452. * @desc: descriptor at the head of the transaction chain
  453. *
  454. * Queue chain if DMA engine is working already
  455. *
  456. * Cookie increment and adding to active_list or queue must be atomic
  457. */
  458. static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx)
  459. {
  460. struct at_desc *desc = txd_to_at_desc(tx);
  461. struct at_dma_chan *atchan = to_at_dma_chan(tx->chan);
  462. dma_cookie_t cookie;
  463. spin_lock_bh(&atchan->lock);
  464. cookie = atc_assign_cookie(atchan, desc);
  465. if (list_empty(&atchan->active_list)) {
  466. dev_vdbg(chan2dev(tx->chan), "tx_submit: started %u\n",
  467. desc->txd.cookie);
  468. atc_dostart(atchan, desc);
  469. list_add_tail(&desc->desc_node, &atchan->active_list);
  470. } else {
  471. dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u\n",
  472. desc->txd.cookie);
  473. list_add_tail(&desc->desc_node, &atchan->queue);
  474. }
  475. spin_unlock_bh(&atchan->lock);
  476. return cookie;
  477. }
  478. /**
  479. * atc_prep_dma_memcpy - prepare a memcpy operation
  480. * @chan: the channel to prepare operation on
  481. * @dest: operation virtual destination address
  482. * @src: operation virtual source address
  483. * @len: operation length
  484. * @flags: tx descriptor status flags
  485. */
  486. static struct dma_async_tx_descriptor *
  487. atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
  488. size_t len, unsigned long flags)
  489. {
  490. struct at_dma_chan *atchan = to_at_dma_chan(chan);
  491. struct at_desc *desc = NULL;
  492. struct at_desc *first = NULL;
  493. struct at_desc *prev = NULL;
  494. size_t xfer_count;
  495. size_t offset;
  496. unsigned int src_width;
  497. unsigned int dst_width;
  498. u32 ctrla;
  499. u32 ctrlb;
  500. dev_vdbg(chan2dev(chan), "prep_dma_memcpy: d0x%x s0x%x l0x%zx f0x%lx\n",
  501. dest, src, len, flags);
  502. if (unlikely(!len)) {
  503. dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
  504. return NULL;
  505. }
  506. ctrla = ATC_DEFAULT_CTRLA;
  507. ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN
  508. | ATC_SRC_ADDR_MODE_INCR
  509. | ATC_DST_ADDR_MODE_INCR
  510. | ATC_FC_MEM2MEM;
  511. /*
  512. * We can be a lot more clever here, but this should take care
  513. * of the most common optimization.
  514. */
  515. if (!((src | dest | len) & 3)) {
  516. ctrla |= ATC_SRC_WIDTH_WORD | ATC_DST_WIDTH_WORD;
  517. src_width = dst_width = 2;
  518. } else if (!((src | dest | len) & 1)) {
  519. ctrla |= ATC_SRC_WIDTH_HALFWORD | ATC_DST_WIDTH_HALFWORD;
  520. src_width = dst_width = 1;
  521. } else {
  522. ctrla |= ATC_SRC_WIDTH_BYTE | ATC_DST_WIDTH_BYTE;
  523. src_width = dst_width = 0;
  524. }
  525. for (offset = 0; offset < len; offset += xfer_count << src_width) {
  526. xfer_count = min_t(size_t, (len - offset) >> src_width,
  527. ATC_BTSIZE_MAX);
  528. desc = atc_desc_get(atchan);
  529. if (!desc)
  530. goto err_desc_get;
  531. desc->lli.saddr = src + offset;
  532. desc->lli.daddr = dest + offset;
  533. desc->lli.ctrla = ctrla | xfer_count;
  534. desc->lli.ctrlb = ctrlb;
  535. desc->txd.cookie = 0;
  536. atc_desc_chain(&first, &prev, desc);
  537. }
  538. /* First descriptor of the chain embedds additional information */
  539. first->txd.cookie = -EBUSY;
  540. first->len = len;
  541. /* set end-of-link to the last link descriptor of list*/
  542. set_desc_eol(desc);
  543. first->txd.flags = flags; /* client is in control of this ack */
  544. return &first->txd;
  545. err_desc_get:
  546. atc_desc_put(atchan, first);
  547. return NULL;
  548. }
  549. /**
  550. * atc_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
  551. * @chan: DMA channel
  552. * @sgl: scatterlist to transfer to/from
  553. * @sg_len: number of entries in @scatterlist
  554. * @direction: DMA direction
  555. * @flags: tx descriptor status flags
  556. */
  557. static struct dma_async_tx_descriptor *
  558. atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
  559. unsigned int sg_len, enum dma_data_direction direction,
  560. unsigned long flags)
  561. {
  562. struct at_dma_chan *atchan = to_at_dma_chan(chan);
  563. struct at_dma_slave *atslave = chan->private;
  564. struct at_desc *first = NULL;
  565. struct at_desc *prev = NULL;
  566. u32 ctrla;
  567. u32 ctrlb;
  568. dma_addr_t reg;
  569. unsigned int reg_width;
  570. unsigned int mem_width;
  571. unsigned int i;
  572. struct scatterlist *sg;
  573. size_t total_len = 0;
  574. dev_vdbg(chan2dev(chan), "prep_slave_sg (%d): %s f0x%lx\n",
  575. sg_len,
  576. direction == DMA_TO_DEVICE ? "TO DEVICE" : "FROM DEVICE",
  577. flags);
  578. if (unlikely(!atslave || !sg_len)) {
  579. dev_dbg(chan2dev(chan), "prep_slave_sg: sg length is zero!\n");
  580. return NULL;
  581. }
  582. reg_width = atslave->reg_width;
  583. ctrla = ATC_DEFAULT_CTRLA | atslave->ctrla;
  584. ctrlb = ATC_IEN;
  585. switch (direction) {
  586. case DMA_TO_DEVICE:
  587. ctrla |= ATC_DST_WIDTH(reg_width);
  588. ctrlb |= ATC_DST_ADDR_MODE_FIXED
  589. | ATC_SRC_ADDR_MODE_INCR
  590. | ATC_FC_MEM2PER
  591. | ATC_SIF(AT_DMA_MEM_IF) | ATC_DIF(AT_DMA_PER_IF);
  592. reg = atslave->tx_reg;
  593. for_each_sg(sgl, sg, sg_len, i) {
  594. struct at_desc *desc;
  595. u32 len;
  596. u32 mem;
  597. desc = atc_desc_get(atchan);
  598. if (!desc)
  599. goto err_desc_get;
  600. mem = sg_dma_address(sg);
  601. len = sg_dma_len(sg);
  602. if (unlikely(!len)) {
  603. dev_dbg(chan2dev(chan),
  604. "prep_slave_sg: sg(%d) data length is zero\n", i);
  605. goto err;
  606. }
  607. mem_width = 2;
  608. if (unlikely(mem & 3 || len & 3))
  609. mem_width = 0;
  610. desc->lli.saddr = mem;
  611. desc->lli.daddr = reg;
  612. desc->lli.ctrla = ctrla
  613. | ATC_SRC_WIDTH(mem_width)
  614. | len >> mem_width;
  615. desc->lli.ctrlb = ctrlb;
  616. atc_desc_chain(&first, &prev, desc);
  617. total_len += len;
  618. }
  619. break;
  620. case DMA_FROM_DEVICE:
  621. ctrla |= ATC_SRC_WIDTH(reg_width);
  622. ctrlb |= ATC_DST_ADDR_MODE_INCR
  623. | ATC_SRC_ADDR_MODE_FIXED
  624. | ATC_FC_PER2MEM
  625. | ATC_SIF(AT_DMA_PER_IF) | ATC_DIF(AT_DMA_MEM_IF);
  626. reg = atslave->rx_reg;
  627. for_each_sg(sgl, sg, sg_len, i) {
  628. struct at_desc *desc;
  629. u32 len;
  630. u32 mem;
  631. desc = atc_desc_get(atchan);
  632. if (!desc)
  633. goto err_desc_get;
  634. mem = sg_dma_address(sg);
  635. len = sg_dma_len(sg);
  636. if (unlikely(!len)) {
  637. dev_dbg(chan2dev(chan),
  638. "prep_slave_sg: sg(%d) data length is zero\n", i);
  639. goto err;
  640. }
  641. mem_width = 2;
  642. if (unlikely(mem & 3 || len & 3))
  643. mem_width = 0;
  644. desc->lli.saddr = reg;
  645. desc->lli.daddr = mem;
  646. desc->lli.ctrla = ctrla
  647. | ATC_DST_WIDTH(mem_width)
  648. | len >> reg_width;
  649. desc->lli.ctrlb = ctrlb;
  650. atc_desc_chain(&first, &prev, desc);
  651. total_len += len;
  652. }
  653. break;
  654. default:
  655. return NULL;
  656. }
  657. /* set end-of-link to the last link descriptor of list*/
  658. set_desc_eol(prev);
  659. /* First descriptor of the chain embedds additional information */
  660. first->txd.cookie = -EBUSY;
  661. first->len = total_len;
  662. /* first link descriptor of list is responsible of flags */
  663. first->txd.flags = flags; /* client is in control of this ack */
  664. return &first->txd;
  665. err_desc_get:
  666. dev_err(chan2dev(chan), "not enough descriptors available\n");
  667. err:
  668. atc_desc_put(atchan, first);
  669. return NULL;
  670. }
  671. /**
  672. * atc_dma_cyclic_check_values
  673. * Check for too big/unaligned periods and unaligned DMA buffer
  674. */
  675. static int
  676. atc_dma_cyclic_check_values(unsigned int reg_width, dma_addr_t buf_addr,
  677. size_t period_len, enum dma_data_direction direction)
  678. {
  679. if (period_len > (ATC_BTSIZE_MAX << reg_width))
  680. goto err_out;
  681. if (unlikely(period_len & ((1 << reg_width) - 1)))
  682. goto err_out;
  683. if (unlikely(buf_addr & ((1 << reg_width) - 1)))
  684. goto err_out;
  685. if (unlikely(!(direction & (DMA_TO_DEVICE | DMA_FROM_DEVICE))))
  686. goto err_out;
  687. return 0;
  688. err_out:
  689. return -EINVAL;
  690. }
  691. /**
  692. * atc_dma_cyclic_fill_desc - Fill one period decriptor
  693. */
  694. static int
  695. atc_dma_cyclic_fill_desc(struct at_dma_slave *atslave, struct at_desc *desc,
  696. unsigned int period_index, dma_addr_t buf_addr,
  697. size_t period_len, enum dma_data_direction direction)
  698. {
  699. u32 ctrla;
  700. unsigned int reg_width = atslave->reg_width;
  701. /* prepare common CRTLA value */
  702. ctrla = ATC_DEFAULT_CTRLA | atslave->ctrla
  703. | ATC_DST_WIDTH(reg_width)
  704. | ATC_SRC_WIDTH(reg_width)
  705. | period_len >> reg_width;
  706. switch (direction) {
  707. case DMA_TO_DEVICE:
  708. desc->lli.saddr = buf_addr + (period_len * period_index);
  709. desc->lli.daddr = atslave->tx_reg;
  710. desc->lli.ctrla = ctrla;
  711. desc->lli.ctrlb = ATC_DST_ADDR_MODE_FIXED
  712. | ATC_SRC_ADDR_MODE_INCR
  713. | ATC_FC_MEM2PER
  714. | ATC_SIF(AT_DMA_MEM_IF)
  715. | ATC_DIF(AT_DMA_PER_IF);
  716. break;
  717. case DMA_FROM_DEVICE:
  718. desc->lli.saddr = atslave->rx_reg;
  719. desc->lli.daddr = buf_addr + (period_len * period_index);
  720. desc->lli.ctrla = ctrla;
  721. desc->lli.ctrlb = ATC_DST_ADDR_MODE_INCR
  722. | ATC_SRC_ADDR_MODE_FIXED
  723. | ATC_FC_PER2MEM
  724. | ATC_SIF(AT_DMA_PER_IF)
  725. | ATC_DIF(AT_DMA_MEM_IF);
  726. break;
  727. default:
  728. return -EINVAL;
  729. }
  730. return 0;
  731. }
  732. /**
  733. * atc_prep_dma_cyclic - prepare the cyclic DMA transfer
  734. * @chan: the DMA channel to prepare
  735. * @buf_addr: physical DMA address where the buffer starts
  736. * @buf_len: total number of bytes for the entire buffer
  737. * @period_len: number of bytes for each period
  738. * @direction: transfer direction, to or from device
  739. */
  740. static struct dma_async_tx_descriptor *
  741. atc_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
  742. size_t period_len, enum dma_data_direction direction)
  743. {
  744. struct at_dma_chan *atchan = to_at_dma_chan(chan);
  745. struct at_dma_slave *atslave = chan->private;
  746. struct at_desc *first = NULL;
  747. struct at_desc *prev = NULL;
  748. unsigned long was_cyclic;
  749. unsigned int periods = buf_len / period_len;
  750. unsigned int i;
  751. dev_vdbg(chan2dev(chan), "prep_dma_cyclic: %s buf@0x%08x - %d (%d/%d)\n",
  752. direction == DMA_TO_DEVICE ? "TO DEVICE" : "FROM DEVICE",
  753. buf_addr,
  754. periods, buf_len, period_len);
  755. if (unlikely(!atslave || !buf_len || !period_len)) {
  756. dev_dbg(chan2dev(chan), "prep_dma_cyclic: length is zero!\n");
  757. return NULL;
  758. }
  759. was_cyclic = test_and_set_bit(ATC_IS_CYCLIC, &atchan->status);
  760. if (was_cyclic) {
  761. dev_dbg(chan2dev(chan), "prep_dma_cyclic: channel in use!\n");
  762. return NULL;
  763. }
  764. /* Check for too big/unaligned periods and unaligned DMA buffer */
  765. if (atc_dma_cyclic_check_values(atslave->reg_width, buf_addr,
  766. period_len, direction))
  767. goto err_out;
  768. /* build cyclic linked list */
  769. for (i = 0; i < periods; i++) {
  770. struct at_desc *desc;
  771. desc = atc_desc_get(atchan);
  772. if (!desc)
  773. goto err_desc_get;
  774. if (atc_dma_cyclic_fill_desc(atslave, desc, i, buf_addr,
  775. period_len, direction))
  776. goto err_desc_get;
  777. atc_desc_chain(&first, &prev, desc);
  778. }
  779. /* lets make a cyclic list */
  780. prev->lli.dscr = first->txd.phys;
  781. /* First descriptor of the chain embedds additional information */
  782. first->txd.cookie = -EBUSY;
  783. first->len = buf_len;
  784. return &first->txd;
  785. err_desc_get:
  786. dev_err(chan2dev(chan), "not enough descriptors available\n");
  787. atc_desc_put(atchan, first);
  788. err_out:
  789. clear_bit(ATC_IS_CYCLIC, &atchan->status);
  790. return NULL;
  791. }
  792. static int atc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
  793. unsigned long arg)
  794. {
  795. struct at_dma_chan *atchan = to_at_dma_chan(chan);
  796. struct at_dma *atdma = to_at_dma(chan->device);
  797. int chan_id = atchan->chan_common.chan_id;
  798. LIST_HEAD(list);
  799. dev_vdbg(chan2dev(chan), "atc_control (%d)\n", cmd);
  800. if (cmd == DMA_PAUSE) {
  801. spin_lock_bh(&atchan->lock);
  802. dma_writel(atdma, CHER, AT_DMA_SUSP(chan_id));
  803. set_bit(ATC_IS_PAUSED, &atchan->status);
  804. spin_unlock_bh(&atchan->lock);
  805. } else if (cmd == DMA_RESUME) {
  806. if (!test_bit(ATC_IS_PAUSED, &atchan->status))
  807. return 0;
  808. spin_lock_bh(&atchan->lock);
  809. dma_writel(atdma, CHDR, AT_DMA_RES(chan_id));
  810. clear_bit(ATC_IS_PAUSED, &atchan->status);
  811. spin_unlock_bh(&atchan->lock);
  812. } else if (cmd == DMA_TERMINATE_ALL) {
  813. struct at_desc *desc, *_desc;
  814. /*
  815. * This is only called when something went wrong elsewhere, so
  816. * we don't really care about the data. Just disable the
  817. * channel. We still have to poll the channel enable bit due
  818. * to AHB/HSB limitations.
  819. */
  820. spin_lock_bh(&atchan->lock);
  821. /* disabling channel: must also remove suspend state */
  822. dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask);
  823. /* confirm that this channel is disabled */
  824. while (dma_readl(atdma, CHSR) & atchan->mask)
  825. cpu_relax();
  826. /* active_list entries will end up before queued entries */
  827. list_splice_init(&atchan->queue, &list);
  828. list_splice_init(&atchan->active_list, &list);
  829. /* Flush all pending and queued descriptors */
  830. list_for_each_entry_safe(desc, _desc, &list, desc_node)
  831. atc_chain_complete(atchan, desc);
  832. clear_bit(ATC_IS_PAUSED, &atchan->status);
  833. /* if channel dedicated to cyclic operations, free it */
  834. clear_bit(ATC_IS_CYCLIC, &atchan->status);
  835. spin_unlock_bh(&atchan->lock);
  836. } else {
  837. return -ENXIO;
  838. }
  839. return 0;
  840. }
  841. /**
  842. * atc_tx_status - poll for transaction completion
  843. * @chan: DMA channel
  844. * @cookie: transaction identifier to check status of
  845. * @txstate: if not %NULL updated with transaction state
  846. *
  847. * If @txstate is passed in, upon return it reflect the driver
  848. * internal state and can be used with dma_async_is_complete() to check
  849. * the status of multiple cookies without re-checking hardware state.
  850. */
  851. static enum dma_status
  852. atc_tx_status(struct dma_chan *chan,
  853. dma_cookie_t cookie,
  854. struct dma_tx_state *txstate)
  855. {
  856. struct at_dma_chan *atchan = to_at_dma_chan(chan);
  857. dma_cookie_t last_used;
  858. dma_cookie_t last_complete;
  859. enum dma_status ret;
  860. spin_lock_bh(&atchan->lock);
  861. last_complete = atchan->completed_cookie;
  862. last_used = chan->cookie;
  863. ret = dma_async_is_complete(cookie, last_complete, last_used);
  864. if (ret != DMA_SUCCESS) {
  865. atc_cleanup_descriptors(atchan);
  866. last_complete = atchan->completed_cookie;
  867. last_used = chan->cookie;
  868. ret = dma_async_is_complete(cookie, last_complete, last_used);
  869. }
  870. spin_unlock_bh(&atchan->lock);
  871. if (ret != DMA_SUCCESS)
  872. dma_set_tx_state(txstate, last_complete, last_used,
  873. atc_first_active(atchan)->len);
  874. else
  875. dma_set_tx_state(txstate, last_complete, last_used, 0);
  876. if (test_bit(ATC_IS_PAUSED, &atchan->status))
  877. ret = DMA_PAUSED;
  878. dev_vdbg(chan2dev(chan), "tx_status %d: cookie = %d (d%d, u%d)\n",
  879. ret, cookie, last_complete ? last_complete : 0,
  880. last_used ? last_used : 0);
  881. return ret;
  882. }
  883. /**
  884. * atc_issue_pending - try to finish work
  885. * @chan: target DMA channel
  886. */
  887. static void atc_issue_pending(struct dma_chan *chan)
  888. {
  889. struct at_dma_chan *atchan = to_at_dma_chan(chan);
  890. dev_vdbg(chan2dev(chan), "issue_pending\n");
  891. /* Not needed for cyclic transfers */
  892. if (test_bit(ATC_IS_CYCLIC, &atchan->status))
  893. return;
  894. spin_lock_bh(&atchan->lock);
  895. if (!atc_chan_is_enabled(atchan)) {
  896. atc_advance_work(atchan);
  897. }
  898. spin_unlock_bh(&atchan->lock);
  899. }
  900. /**
  901. * atc_alloc_chan_resources - allocate resources for DMA channel
  902. * @chan: allocate descriptor resources for this channel
  903. * @client: current client requesting the channel be ready for requests
  904. *
  905. * return - the number of allocated descriptors
  906. */
  907. static int atc_alloc_chan_resources(struct dma_chan *chan)
  908. {
  909. struct at_dma_chan *atchan = to_at_dma_chan(chan);
  910. struct at_dma *atdma = to_at_dma(chan->device);
  911. struct at_desc *desc;
  912. struct at_dma_slave *atslave;
  913. int i;
  914. u32 cfg;
  915. LIST_HEAD(tmp_list);
  916. dev_vdbg(chan2dev(chan), "alloc_chan_resources\n");
  917. /* ASSERT: channel is idle */
  918. if (atc_chan_is_enabled(atchan)) {
  919. dev_dbg(chan2dev(chan), "DMA channel not idle ?\n");
  920. return -EIO;
  921. }
  922. cfg = ATC_DEFAULT_CFG;
  923. atslave = chan->private;
  924. if (atslave) {
  925. /*
  926. * We need controller-specific data to set up slave
  927. * transfers.
  928. */
  929. BUG_ON(!atslave->dma_dev || atslave->dma_dev != atdma->dma_common.dev);
  930. /* if cfg configuration specified take it instad of default */
  931. if (atslave->cfg)
  932. cfg = atslave->cfg;
  933. }
  934. /* have we already been set up?
  935. * reconfigure channel but no need to reallocate descriptors */
  936. if (!list_empty(&atchan->free_list))
  937. return atchan->descs_allocated;
  938. /* Allocate initial pool of descriptors */
  939. for (i = 0; i < init_nr_desc_per_channel; i++) {
  940. desc = atc_alloc_descriptor(chan, GFP_KERNEL);
  941. if (!desc) {
  942. dev_err(atdma->dma_common.dev,
  943. "Only %d initial descriptors\n", i);
  944. break;
  945. }
  946. list_add_tail(&desc->desc_node, &tmp_list);
  947. }
  948. spin_lock_bh(&atchan->lock);
  949. atchan->descs_allocated = i;
  950. list_splice(&tmp_list, &atchan->free_list);
  951. atchan->completed_cookie = chan->cookie = 1;
  952. spin_unlock_bh(&atchan->lock);
  953. /* channel parameters */
  954. channel_writel(atchan, CFG, cfg);
  955. dev_dbg(chan2dev(chan),
  956. "alloc_chan_resources: allocated %d descriptors\n",
  957. atchan->descs_allocated);
  958. return atchan->descs_allocated;
  959. }
  960. /**
  961. * atc_free_chan_resources - free all channel resources
  962. * @chan: DMA channel
  963. */
  964. static void atc_free_chan_resources(struct dma_chan *chan)
  965. {
  966. struct at_dma_chan *atchan = to_at_dma_chan(chan);
  967. struct at_dma *atdma = to_at_dma(chan->device);
  968. struct at_desc *desc, *_desc;
  969. LIST_HEAD(list);
  970. dev_dbg(chan2dev(chan), "free_chan_resources: (descs allocated=%u)\n",
  971. atchan->descs_allocated);
  972. /* ASSERT: channel is idle */
  973. BUG_ON(!list_empty(&atchan->active_list));
  974. BUG_ON(!list_empty(&atchan->queue));
  975. BUG_ON(atc_chan_is_enabled(atchan));
  976. list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) {
  977. dev_vdbg(chan2dev(chan), " freeing descriptor %p\n", desc);
  978. list_del(&desc->desc_node);
  979. /* free link descriptor */
  980. dma_pool_free(atdma->dma_desc_pool, desc, desc->txd.phys);
  981. }
  982. list_splice_init(&atchan->free_list, &list);
  983. atchan->descs_allocated = 0;
  984. atchan->status = 0;
  985. dev_vdbg(chan2dev(chan), "free_chan_resources: done\n");
  986. }
  987. /*-- Module Management -----------------------------------------------*/
  988. /**
  989. * at_dma_off - disable DMA controller
  990. * @atdma: the Atmel HDAMC device
  991. */
  992. static void at_dma_off(struct at_dma *atdma)
  993. {
  994. dma_writel(atdma, EN, 0);
  995. /* disable all interrupts */
  996. dma_writel(atdma, EBCIDR, -1L);
  997. /* confirm that all channels are disabled */
  998. while (dma_readl(atdma, CHSR) & atdma->all_chan_mask)
  999. cpu_relax();
  1000. }
  1001. static int __init at_dma_probe(struct platform_device *pdev)
  1002. {
  1003. struct at_dma_platform_data *pdata;
  1004. struct resource *io;
  1005. struct at_dma *atdma;
  1006. size_t size;
  1007. int irq;
  1008. int err;
  1009. int i;
  1010. /* get DMA Controller parameters from platform */
  1011. pdata = pdev->dev.platform_data;
  1012. if (!pdata || pdata->nr_channels > AT_DMA_MAX_NR_CHANNELS)
  1013. return -EINVAL;
  1014. io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  1015. if (!io)
  1016. return -EINVAL;
  1017. irq = platform_get_irq(pdev, 0);
  1018. if (irq < 0)
  1019. return irq;
  1020. size = sizeof(struct at_dma);
  1021. size += pdata->nr_channels * sizeof(struct at_dma_chan);
  1022. atdma = kzalloc(size, GFP_KERNEL);
  1023. if (!atdma)
  1024. return -ENOMEM;
  1025. /* discover transaction capabilites from the platform data */
  1026. atdma->dma_common.cap_mask = pdata->cap_mask;
  1027. atdma->all_chan_mask = (1 << pdata->nr_channels) - 1;
  1028. size = io->end - io->start + 1;
  1029. if (!request_mem_region(io->start, size, pdev->dev.driver->name)) {
  1030. err = -EBUSY;
  1031. goto err_kfree;
  1032. }
  1033. atdma->regs = ioremap(io->start, size);
  1034. if (!atdma->regs) {
  1035. err = -ENOMEM;
  1036. goto err_release_r;
  1037. }
  1038. atdma->clk = clk_get(&pdev->dev, "dma_clk");
  1039. if (IS_ERR(atdma->clk)) {
  1040. err = PTR_ERR(atdma->clk);
  1041. goto err_clk;
  1042. }
  1043. clk_enable(atdma->clk);
  1044. /* force dma off, just in case */
  1045. at_dma_off(atdma);
  1046. err = request_irq(irq, at_dma_interrupt, 0, "at_hdmac", atdma);
  1047. if (err)
  1048. goto err_irq;
  1049. platform_set_drvdata(pdev, atdma);
  1050. /* create a pool of consistent memory blocks for hardware descriptors */
  1051. atdma->dma_desc_pool = dma_pool_create("at_hdmac_desc_pool",
  1052. &pdev->dev, sizeof(struct at_desc),
  1053. 4 /* word alignment */, 0);
  1054. if (!atdma->dma_desc_pool) {
  1055. dev_err(&pdev->dev, "No memory for descriptors dma pool\n");
  1056. err = -ENOMEM;
  1057. goto err_pool_create;
  1058. }
  1059. /* clear any pending interrupt */
  1060. while (dma_readl(atdma, EBCISR))
  1061. cpu_relax();
  1062. /* initialize channels related values */
  1063. INIT_LIST_HEAD(&atdma->dma_common.channels);
  1064. for (i = 0; i < pdata->nr_channels; i++, atdma->dma_common.chancnt++) {
  1065. struct at_dma_chan *atchan = &atdma->chan[i];
  1066. atchan->chan_common.device = &atdma->dma_common;
  1067. atchan->chan_common.cookie = atchan->completed_cookie = 1;
  1068. atchan->chan_common.chan_id = i;
  1069. list_add_tail(&atchan->chan_common.device_node,
  1070. &atdma->dma_common.channels);
  1071. atchan->ch_regs = atdma->regs + ch_regs(i);
  1072. spin_lock_init(&atchan->lock);
  1073. atchan->mask = 1 << i;
  1074. INIT_LIST_HEAD(&atchan->active_list);
  1075. INIT_LIST_HEAD(&atchan->queue);
  1076. INIT_LIST_HEAD(&atchan->free_list);
  1077. tasklet_init(&atchan->tasklet, atc_tasklet,
  1078. (unsigned long)atchan);
  1079. atc_enable_chan_irq(atdma, i);
  1080. }
  1081. /* set base routines */
  1082. atdma->dma_common.device_alloc_chan_resources = atc_alloc_chan_resources;
  1083. atdma->dma_common.device_free_chan_resources = atc_free_chan_resources;
  1084. atdma->dma_common.device_tx_status = atc_tx_status;
  1085. atdma->dma_common.device_issue_pending = atc_issue_pending;
  1086. atdma->dma_common.dev = &pdev->dev;
  1087. /* set prep routines based on capability */
  1088. if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask))
  1089. atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy;
  1090. if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask))
  1091. atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg;
  1092. if (dma_has_cap(DMA_CYCLIC, atdma->dma_common.cap_mask))
  1093. atdma->dma_common.device_prep_dma_cyclic = atc_prep_dma_cyclic;
  1094. if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask) ||
  1095. dma_has_cap(DMA_CYCLIC, atdma->dma_common.cap_mask))
  1096. atdma->dma_common.device_control = atc_control;
  1097. dma_writel(atdma, EN, AT_DMA_ENABLE);
  1098. dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s), %d channels\n",
  1099. dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "",
  1100. dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask) ? "slave " : "",
  1101. atdma->dma_common.chancnt);
  1102. dma_async_device_register(&atdma->dma_common);
  1103. return 0;
  1104. err_pool_create:
  1105. platform_set_drvdata(pdev, NULL);
  1106. free_irq(platform_get_irq(pdev, 0), atdma);
  1107. err_irq:
  1108. clk_disable(atdma->clk);
  1109. clk_put(atdma->clk);
  1110. err_clk:
  1111. iounmap(atdma->regs);
  1112. atdma->regs = NULL;
  1113. err_release_r:
  1114. release_mem_region(io->start, size);
  1115. err_kfree:
  1116. kfree(atdma);
  1117. return err;
  1118. }
  1119. static int __exit at_dma_remove(struct platform_device *pdev)
  1120. {
  1121. struct at_dma *atdma = platform_get_drvdata(pdev);
  1122. struct dma_chan *chan, *_chan;
  1123. struct resource *io;
  1124. at_dma_off(atdma);
  1125. dma_async_device_unregister(&atdma->dma_common);
  1126. dma_pool_destroy(atdma->dma_desc_pool);
  1127. platform_set_drvdata(pdev, NULL);
  1128. free_irq(platform_get_irq(pdev, 0), atdma);
  1129. list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,
  1130. device_node) {
  1131. struct at_dma_chan *atchan = to_at_dma_chan(chan);
  1132. /* Disable interrupts */
  1133. atc_disable_chan_irq(atdma, chan->chan_id);
  1134. tasklet_disable(&atchan->tasklet);
  1135. tasklet_kill(&atchan->tasklet);
  1136. list_del(&chan->device_node);
  1137. }
  1138. clk_disable(atdma->clk);
  1139. clk_put(atdma->clk);
  1140. iounmap(atdma->regs);
  1141. atdma->regs = NULL;
  1142. io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  1143. release_mem_region(io->start, io->end - io->start + 1);
  1144. kfree(atdma);
  1145. return 0;
  1146. }
  1147. static void at_dma_shutdown(struct platform_device *pdev)
  1148. {
  1149. struct at_dma *atdma = platform_get_drvdata(pdev);
  1150. at_dma_off(platform_get_drvdata(pdev));
  1151. clk_disable(atdma->clk);
  1152. }
  1153. static int at_dma_suspend_noirq(struct device *dev)
  1154. {
  1155. struct platform_device *pdev = to_platform_device(dev);
  1156. struct at_dma *atdma = platform_get_drvdata(pdev);
  1157. at_dma_off(platform_get_drvdata(pdev));
  1158. clk_disable(atdma->clk);
  1159. return 0;
  1160. }
  1161. static int at_dma_resume_noirq(struct device *dev)
  1162. {
  1163. struct platform_device *pdev = to_platform_device(dev);
  1164. struct at_dma *atdma = platform_get_drvdata(pdev);
  1165. clk_enable(atdma->clk);
  1166. dma_writel(atdma, EN, AT_DMA_ENABLE);
  1167. return 0;
  1168. }
  1169. static const struct dev_pm_ops at_dma_dev_pm_ops = {
  1170. .suspend_noirq = at_dma_suspend_noirq,
  1171. .resume_noirq = at_dma_resume_noirq,
  1172. };
  1173. static struct platform_driver at_dma_driver = {
  1174. .remove = __exit_p(at_dma_remove),
  1175. .shutdown = at_dma_shutdown,
  1176. .driver = {
  1177. .name = "at_hdmac",
  1178. .pm = &at_dma_dev_pm_ops,
  1179. },
  1180. };
  1181. static int __init at_dma_init(void)
  1182. {
  1183. return platform_driver_probe(&at_dma_driver, at_dma_probe);
  1184. }
  1185. subsys_initcall(at_dma_init);
  1186. static void __exit at_dma_exit(void)
  1187. {
  1188. platform_driver_unregister(&at_dma_driver);
  1189. }
  1190. module_exit(at_dma_exit);
  1191. MODULE_DESCRIPTION("Atmel AHB DMA Controller driver");
  1192. MODULE_AUTHOR("Nicolas Ferre <nicolas.ferre@atmel.com>");
  1193. MODULE_LICENSE("GPL");
  1194. MODULE_ALIAS("platform:at_hdmac");