dma_v2.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920
  1. /*
  2. * Intel I/OAT DMA Linux driver
  3. * Copyright(c) 2004 - 2009 Intel Corporation.
  4. *
  5. * This program is free software; you can redistribute it and/or modify it
  6. * under the terms and conditions of the GNU General Public License,
  7. * version 2, as published by the Free Software Foundation.
  8. *
  9. * This program is distributed in the hope that it will be useful, but WITHOUT
  10. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  12. * more details.
  13. *
  14. * You should have received a copy of the GNU General Public License along with
  15. * this program; if not, write to the Free Software Foundation, Inc.,
  16. * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17. *
  18. * The full GNU General Public License is included in this distribution in
  19. * the file called "COPYING".
  20. *
  21. */
  22. /*
  23. * This driver supports an Intel I/OAT DMA engine (versions >= 2), which
  24. * does asynchronous data movement and checksumming operations.
  25. */
  26. #include <linux/init.h>
  27. #include <linux/module.h>
  28. #include <linux/slab.h>
  29. #include <linux/pci.h>
  30. #include <linux/interrupt.h>
  31. #include <linux/dmaengine.h>
  32. #include <linux/delay.h>
  33. #include <linux/dma-mapping.h>
  34. #include <linux/workqueue.h>
  35. #include <linux/prefetch.h>
  36. #include <linux/i7300_idle.h>
  37. #include "dma.h"
  38. #include "dma_v2.h"
  39. #include "registers.h"
  40. #include "hw.h"
  41. #include "../dmaengine.h"
  42. int ioat_ring_alloc_order = 8;
  43. module_param(ioat_ring_alloc_order, int, 0644);
  44. MODULE_PARM_DESC(ioat_ring_alloc_order,
  45. "ioat2+: allocate 2^n descriptors per channel"
  46. " (default: 8 max: 16)");
  47. static int ioat_ring_max_alloc_order = IOAT_MAX_ORDER;
  48. module_param(ioat_ring_max_alloc_order, int, 0644);
  49. MODULE_PARM_DESC(ioat_ring_max_alloc_order,
  50. "ioat2+: upper limit for ring size (default: 16)");
  51. void __ioat2_issue_pending(struct ioat2_dma_chan *ioat)
  52. {
  53. struct ioat_chan_common *chan = &ioat->base;
  54. ioat->dmacount += ioat2_ring_pending(ioat);
  55. ioat->issued = ioat->head;
  56. writew(ioat->dmacount, chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
  57. dev_dbg(to_dev(chan),
  58. "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
  59. __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
  60. }
  61. void ioat2_issue_pending(struct dma_chan *c)
  62. {
  63. struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
  64. if (ioat2_ring_pending(ioat)) {
  65. spin_lock_bh(&ioat->prep_lock);
  66. __ioat2_issue_pending(ioat);
  67. spin_unlock_bh(&ioat->prep_lock);
  68. }
  69. }
  70. /**
  71. * ioat2_update_pending - log pending descriptors
  72. * @ioat: ioat2+ channel
  73. *
  74. * Check if the number of unsubmitted descriptors has exceeded the
  75. * watermark. Called with prep_lock held
  76. */
  77. static void ioat2_update_pending(struct ioat2_dma_chan *ioat)
  78. {
  79. if (ioat2_ring_pending(ioat) > ioat_pending_level)
  80. __ioat2_issue_pending(ioat);
  81. }
  82. static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
  83. {
  84. struct ioat_ring_ent *desc;
  85. struct ioat_dma_descriptor *hw;
  86. if (ioat2_ring_space(ioat) < 1) {
  87. dev_err(to_dev(&ioat->base),
  88. "Unable to start null desc - ring full\n");
  89. return;
  90. }
  91. dev_dbg(to_dev(&ioat->base), "%s: head: %#x tail: %#x issued: %#x\n",
  92. __func__, ioat->head, ioat->tail, ioat->issued);
  93. desc = ioat2_get_ring_ent(ioat, ioat->head);
  94. hw = desc->hw;
  95. hw->ctl = 0;
  96. hw->ctl_f.null = 1;
  97. hw->ctl_f.int_en = 1;
  98. hw->ctl_f.compl_write = 1;
  99. /* set size to non-zero value (channel returns error when size is 0) */
  100. hw->size = NULL_DESC_BUFFER_SIZE;
  101. hw->src_addr = 0;
  102. hw->dst_addr = 0;
  103. async_tx_ack(&desc->txd);
  104. ioat2_set_chainaddr(ioat, desc->txd.phys);
  105. dump_desc_dbg(ioat, desc);
  106. wmb();
  107. ioat->head += 1;
  108. __ioat2_issue_pending(ioat);
  109. }
  110. static void ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
  111. {
  112. spin_lock_bh(&ioat->prep_lock);
  113. __ioat2_start_null_desc(ioat);
  114. spin_unlock_bh(&ioat->prep_lock);
  115. }
  116. static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete)
  117. {
  118. struct ioat_chan_common *chan = &ioat->base;
  119. struct dma_async_tx_descriptor *tx;
  120. struct ioat_ring_ent *desc;
  121. bool seen_current = false;
  122. u16 active;
  123. int idx = ioat->tail, i;
  124. dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
  125. __func__, ioat->head, ioat->tail, ioat->issued);
  126. active = ioat2_ring_active(ioat);
  127. for (i = 0; i < active && !seen_current; i++) {
  128. smp_read_barrier_depends();
  129. prefetch(ioat2_get_ring_ent(ioat, idx + i + 1));
  130. desc = ioat2_get_ring_ent(ioat, idx + i);
  131. tx = &desc->txd;
  132. dump_desc_dbg(ioat, desc);
  133. if (tx->cookie) {
  134. dma_descriptor_unmap(tx);
  135. dma_cookie_complete(tx);
  136. if (tx->callback) {
  137. tx->callback(tx->callback_param);
  138. tx->callback = NULL;
  139. }
  140. }
  141. if (tx->phys == phys_complete)
  142. seen_current = true;
  143. }
  144. smp_mb(); /* finish all descriptor reads before incrementing tail */
  145. ioat->tail = idx + i;
  146. BUG_ON(active && !seen_current); /* no active descs have written a completion? */
  147. chan->last_completion = phys_complete;
  148. if (active - i == 0) {
  149. dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
  150. __func__);
  151. clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
  152. mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
  153. }
  154. }
  155. /**
  156. * ioat2_cleanup - clean finished descriptors (advance tail pointer)
  157. * @chan: ioat channel to be cleaned up
  158. */
  159. static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
  160. {
  161. struct ioat_chan_common *chan = &ioat->base;
  162. dma_addr_t phys_complete;
  163. spin_lock_bh(&chan->cleanup_lock);
  164. if (ioat_cleanup_preamble(chan, &phys_complete))
  165. __cleanup(ioat, phys_complete);
  166. spin_unlock_bh(&chan->cleanup_lock);
  167. }
  168. void ioat2_cleanup_event(unsigned long data)
  169. {
  170. struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);
  171. struct ioat_chan_common *chan = &ioat->base;
  172. ioat2_cleanup(ioat);
  173. if (!test_bit(IOAT_RUN, &chan->state))
  174. return;
  175. writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
  176. }
  177. void __ioat2_restart_chan(struct ioat2_dma_chan *ioat)
  178. {
  179. struct ioat_chan_common *chan = &ioat->base;
  180. /* set the tail to be re-issued */
  181. ioat->issued = ioat->tail;
  182. ioat->dmacount = 0;
  183. set_bit(IOAT_COMPLETION_PENDING, &chan->state);
  184. mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
  185. dev_dbg(to_dev(chan),
  186. "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
  187. __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
  188. if (ioat2_ring_pending(ioat)) {
  189. struct ioat_ring_ent *desc;
  190. desc = ioat2_get_ring_ent(ioat, ioat->tail);
  191. ioat2_set_chainaddr(ioat, desc->txd.phys);
  192. __ioat2_issue_pending(ioat);
  193. } else
  194. __ioat2_start_null_desc(ioat);
  195. }
  196. int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo)
  197. {
  198. unsigned long end = jiffies + tmo;
  199. int err = 0;
  200. u32 status;
  201. status = ioat_chansts(chan);
  202. if (is_ioat_active(status) || is_ioat_idle(status))
  203. ioat_suspend(chan);
  204. while (is_ioat_active(status) || is_ioat_idle(status)) {
  205. if (tmo && time_after(jiffies, end)) {
  206. err = -ETIMEDOUT;
  207. break;
  208. }
  209. status = ioat_chansts(chan);
  210. cpu_relax();
  211. }
  212. return err;
  213. }
  214. int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo)
  215. {
  216. unsigned long end = jiffies + tmo;
  217. int err = 0;
  218. ioat_reset(chan);
  219. while (ioat_reset_pending(chan)) {
  220. if (end && time_after(jiffies, end)) {
  221. err = -ETIMEDOUT;
  222. break;
  223. }
  224. cpu_relax();
  225. }
  226. return err;
  227. }
  228. static void ioat2_restart_channel(struct ioat2_dma_chan *ioat)
  229. {
  230. struct ioat_chan_common *chan = &ioat->base;
  231. dma_addr_t phys_complete;
  232. ioat2_quiesce(chan, 0);
  233. if (ioat_cleanup_preamble(chan, &phys_complete))
  234. __cleanup(ioat, phys_complete);
  235. __ioat2_restart_chan(ioat);
  236. }
  237. static void check_active(struct ioat2_dma_chan *ioat)
  238. {
  239. struct ioat_chan_common *chan = &ioat->base;
  240. if (ioat2_ring_active(ioat)) {
  241. mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
  242. return;
  243. }
  244. if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &chan->state))
  245. mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
  246. else if (ioat->alloc_order > ioat_get_alloc_order()) {
  247. /* if the ring is idle, empty, and oversized try to step
  248. * down the size
  249. */
  250. reshape_ring(ioat, ioat->alloc_order - 1);
  251. /* keep shrinking until we get back to our minimum
  252. * default size
  253. */
  254. if (ioat->alloc_order > ioat_get_alloc_order())
  255. mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
  256. }
  257. }
  258. void ioat2_timer_event(unsigned long data)
  259. {
  260. struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);
  261. struct ioat_chan_common *chan = &ioat->base;
  262. dma_addr_t phys_complete;
  263. u64 status;
  264. status = ioat_chansts(chan);
  265. /* when halted due to errors check for channel
  266. * programming errors before advancing the completion state
  267. */
  268. if (is_ioat_halted(status)) {
  269. u32 chanerr;
  270. chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
  271. dev_err(to_dev(chan), "%s: Channel halted (%x)\n",
  272. __func__, chanerr);
  273. if (test_bit(IOAT_RUN, &chan->state))
  274. BUG_ON(is_ioat_bug(chanerr));
  275. else /* we never got off the ground */
  276. return;
  277. }
  278. /* if we haven't made progress and we have already
  279. * acknowledged a pending completion once, then be more
  280. * forceful with a restart
  281. */
  282. spin_lock_bh(&chan->cleanup_lock);
  283. if (ioat_cleanup_preamble(chan, &phys_complete))
  284. __cleanup(ioat, phys_complete);
  285. else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) {
  286. spin_lock_bh(&ioat->prep_lock);
  287. ioat2_restart_channel(ioat);
  288. spin_unlock_bh(&ioat->prep_lock);
  289. spin_unlock_bh(&chan->cleanup_lock);
  290. return;
  291. } else {
  292. set_bit(IOAT_COMPLETION_ACK, &chan->state);
  293. mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
  294. }
  295. if (ioat2_ring_active(ioat))
  296. mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
  297. else {
  298. spin_lock_bh(&ioat->prep_lock);
  299. check_active(ioat);
  300. spin_unlock_bh(&ioat->prep_lock);
  301. }
  302. spin_unlock_bh(&chan->cleanup_lock);
  303. }
  304. static int ioat2_reset_hw(struct ioat_chan_common *chan)
  305. {
  306. /* throw away whatever the channel was doing and get it initialized */
  307. u32 chanerr;
  308. ioat2_quiesce(chan, msecs_to_jiffies(100));
  309. chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
  310. writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
  311. return ioat2_reset_sync(chan, msecs_to_jiffies(200));
  312. }
  313. /**
  314. * ioat2_enumerate_channels - find and initialize the device's channels
  315. * @device: the device to be enumerated
  316. */
  317. int ioat2_enumerate_channels(struct ioatdma_device *device)
  318. {
  319. struct ioat2_dma_chan *ioat;
  320. struct device *dev = &device->pdev->dev;
  321. struct dma_device *dma = &device->common;
  322. u8 xfercap_log;
  323. int i;
  324. INIT_LIST_HEAD(&dma->channels);
  325. dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
  326. dma->chancnt &= 0x1f; /* bits [4:0] valid */
  327. if (dma->chancnt > ARRAY_SIZE(device->idx)) {
  328. dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
  329. dma->chancnt, ARRAY_SIZE(device->idx));
  330. dma->chancnt = ARRAY_SIZE(device->idx);
  331. }
  332. xfercap_log = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
  333. xfercap_log &= 0x1f; /* bits [4:0] valid */
  334. if (xfercap_log == 0)
  335. return 0;
  336. dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log);
  337. /* FIXME which i/oat version is i7300? */
  338. #ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
  339. if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
  340. dma->chancnt--;
  341. #endif
  342. for (i = 0; i < dma->chancnt; i++) {
  343. ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
  344. if (!ioat)
  345. break;
  346. ioat_init_channel(device, &ioat->base, i);
  347. ioat->xfercap_log = xfercap_log;
  348. spin_lock_init(&ioat->prep_lock);
  349. if (device->reset_hw(&ioat->base)) {
  350. i = 0;
  351. break;
  352. }
  353. }
  354. dma->chancnt = i;
  355. return i;
  356. }
  357. static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
  358. {
  359. struct dma_chan *c = tx->chan;
  360. struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
  361. struct ioat_chan_common *chan = &ioat->base;
  362. dma_cookie_t cookie;
  363. cookie = dma_cookie_assign(tx);
  364. dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
  365. if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &chan->state))
  366. mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
  367. /* make descriptor updates visible before advancing ioat->head,
  368. * this is purposefully not smp_wmb() since we are also
  369. * publishing the descriptor updates to a dma device
  370. */
  371. wmb();
  372. ioat->head += ioat->produce;
  373. ioat2_update_pending(ioat);
  374. spin_unlock_bh(&ioat->prep_lock);
  375. return cookie;
  376. }
  377. static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan, gfp_t flags)
  378. {
  379. struct ioat_dma_descriptor *hw;
  380. struct ioat_ring_ent *desc;
  381. struct ioatdma_device *dma;
  382. dma_addr_t phys;
  383. dma = to_ioatdma_device(chan->device);
  384. hw = pci_pool_alloc(dma->dma_pool, flags, &phys);
  385. if (!hw)
  386. return NULL;
  387. memset(hw, 0, sizeof(*hw));
  388. desc = kmem_cache_zalloc(ioat2_cache, flags);
  389. if (!desc) {
  390. pci_pool_free(dma->dma_pool, hw, phys);
  391. return NULL;
  392. }
  393. dma_async_tx_descriptor_init(&desc->txd, chan);
  394. desc->txd.tx_submit = ioat2_tx_submit_unlock;
  395. desc->hw = hw;
  396. desc->txd.phys = phys;
  397. return desc;
  398. }
  399. static void ioat2_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
  400. {
  401. struct ioatdma_device *dma;
  402. dma = to_ioatdma_device(chan->device);
  403. pci_pool_free(dma->dma_pool, desc->hw, desc->txd.phys);
  404. kmem_cache_free(ioat2_cache, desc);
  405. }
  406. static struct ioat_ring_ent **ioat2_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
  407. {
  408. struct ioat_ring_ent **ring;
  409. int descs = 1 << order;
  410. int i;
  411. if (order > ioat_get_max_alloc_order())
  412. return NULL;
  413. /* allocate the array to hold the software ring */
  414. ring = kcalloc(descs, sizeof(*ring), flags);
  415. if (!ring)
  416. return NULL;
  417. for (i = 0; i < descs; i++) {
  418. ring[i] = ioat2_alloc_ring_ent(c, flags);
  419. if (!ring[i]) {
  420. while (i--)
  421. ioat2_free_ring_ent(ring[i], c);
  422. kfree(ring);
  423. return NULL;
  424. }
  425. set_desc_id(ring[i], i);
  426. }
  427. /* link descs */
  428. for (i = 0; i < descs-1; i++) {
  429. struct ioat_ring_ent *next = ring[i+1];
  430. struct ioat_dma_descriptor *hw = ring[i]->hw;
  431. hw->next = next->txd.phys;
  432. }
  433. ring[i]->hw->next = ring[0]->txd.phys;
  434. return ring;
  435. }
  436. void ioat2_free_chan_resources(struct dma_chan *c);
  437. /* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
  438. * @chan: channel to be initialized
  439. */
  440. int ioat2_alloc_chan_resources(struct dma_chan *c)
  441. {
  442. struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
  443. struct ioat_chan_common *chan = &ioat->base;
  444. struct ioat_ring_ent **ring;
  445. u64 status;
  446. int order;
  447. int i = 0;
  448. /* have we already been set up? */
  449. if (ioat->ring)
  450. return 1 << ioat->alloc_order;
  451. /* Setup register to interrupt and write completion status on error */
  452. writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET);
  453. /* allocate a completion writeback area */
  454. /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
  455. chan->completion = pci_pool_alloc(chan->device->completion_pool,
  456. GFP_KERNEL, &chan->completion_dma);
  457. if (!chan->completion)
  458. return -ENOMEM;
  459. memset(chan->completion, 0, sizeof(*chan->completion));
  460. writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
  461. chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
  462. writel(((u64) chan->completion_dma) >> 32,
  463. chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
  464. order = ioat_get_alloc_order();
  465. ring = ioat2_alloc_ring(c, order, GFP_KERNEL);
  466. if (!ring)
  467. return -ENOMEM;
  468. spin_lock_bh(&chan->cleanup_lock);
  469. spin_lock_bh(&ioat->prep_lock);
  470. ioat->ring = ring;
  471. ioat->head = 0;
  472. ioat->issued = 0;
  473. ioat->tail = 0;
  474. ioat->alloc_order = order;
  475. set_bit(IOAT_RUN, &chan->state);
  476. spin_unlock_bh(&ioat->prep_lock);
  477. spin_unlock_bh(&chan->cleanup_lock);
  478. ioat2_start_null_desc(ioat);
  479. /* check that we got off the ground */
  480. do {
  481. udelay(1);
  482. status = ioat_chansts(chan);
  483. } while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status));
  484. if (is_ioat_active(status) || is_ioat_idle(status)) {
  485. return 1 << ioat->alloc_order;
  486. } else {
  487. u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
  488. dev_WARN(to_dev(chan),
  489. "failed to start channel chanerr: %#x\n", chanerr);
  490. ioat2_free_chan_resources(c);
  491. return -EFAULT;
  492. }
  493. }
  494. bool reshape_ring(struct ioat2_dma_chan *ioat, int order)
  495. {
  496. /* reshape differs from normal ring allocation in that we want
  497. * to allocate a new software ring while only
  498. * extending/truncating the hardware ring
  499. */
  500. struct ioat_chan_common *chan = &ioat->base;
  501. struct dma_chan *c = &chan->common;
  502. const u32 curr_size = ioat2_ring_size(ioat);
  503. const u16 active = ioat2_ring_active(ioat);
  504. const u32 new_size = 1 << order;
  505. struct ioat_ring_ent **ring;
  506. u16 i;
  507. if (order > ioat_get_max_alloc_order())
  508. return false;
  509. /* double check that we have at least 1 free descriptor */
  510. if (active == curr_size)
  511. return false;
  512. /* when shrinking, verify that we can hold the current active
  513. * set in the new ring
  514. */
  515. if (active >= new_size)
  516. return false;
  517. /* allocate the array to hold the software ring */
  518. ring = kcalloc(new_size, sizeof(*ring), GFP_NOWAIT);
  519. if (!ring)
  520. return false;
  521. /* allocate/trim descriptors as needed */
  522. if (new_size > curr_size) {
  523. /* copy current descriptors to the new ring */
  524. for (i = 0; i < curr_size; i++) {
  525. u16 curr_idx = (ioat->tail+i) & (curr_size-1);
  526. u16 new_idx = (ioat->tail+i) & (new_size-1);
  527. ring[new_idx] = ioat->ring[curr_idx];
  528. set_desc_id(ring[new_idx], new_idx);
  529. }
  530. /* add new descriptors to the ring */
  531. for (i = curr_size; i < new_size; i++) {
  532. u16 new_idx = (ioat->tail+i) & (new_size-1);
  533. ring[new_idx] = ioat2_alloc_ring_ent(c, GFP_NOWAIT);
  534. if (!ring[new_idx]) {
  535. while (i--) {
  536. u16 new_idx = (ioat->tail+i) & (new_size-1);
  537. ioat2_free_ring_ent(ring[new_idx], c);
  538. }
  539. kfree(ring);
  540. return false;
  541. }
  542. set_desc_id(ring[new_idx], new_idx);
  543. }
  544. /* hw link new descriptors */
  545. for (i = curr_size-1; i < new_size; i++) {
  546. u16 new_idx = (ioat->tail+i) & (new_size-1);
  547. struct ioat_ring_ent *next = ring[(new_idx+1) & (new_size-1)];
  548. struct ioat_dma_descriptor *hw = ring[new_idx]->hw;
  549. hw->next = next->txd.phys;
  550. }
  551. } else {
  552. struct ioat_dma_descriptor *hw;
  553. struct ioat_ring_ent *next;
  554. /* copy current descriptors to the new ring, dropping the
  555. * removed descriptors
  556. */
  557. for (i = 0; i < new_size; i++) {
  558. u16 curr_idx = (ioat->tail+i) & (curr_size-1);
  559. u16 new_idx = (ioat->tail+i) & (new_size-1);
  560. ring[new_idx] = ioat->ring[curr_idx];
  561. set_desc_id(ring[new_idx], new_idx);
  562. }
  563. /* free deleted descriptors */
  564. for (i = new_size; i < curr_size; i++) {
  565. struct ioat_ring_ent *ent;
  566. ent = ioat2_get_ring_ent(ioat, ioat->tail+i);
  567. ioat2_free_ring_ent(ent, c);
  568. }
  569. /* fix up hardware ring */
  570. hw = ring[(ioat->tail+new_size-1) & (new_size-1)]->hw;
  571. next = ring[(ioat->tail+new_size) & (new_size-1)];
  572. hw->next = next->txd.phys;
  573. }
  574. dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n",
  575. __func__, new_size);
  576. kfree(ioat->ring);
  577. ioat->ring = ring;
  578. ioat->alloc_order = order;
  579. return true;
  580. }
  581. /**
  582. * ioat2_check_space_lock - verify space and grab ring producer lock
  583. * @ioat: ioat2,3 channel (ring) to operate on
  584. * @num_descs: allocation length
  585. */
  586. int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs)
  587. {
  588. struct ioat_chan_common *chan = &ioat->base;
  589. bool retry;
  590. retry:
  591. spin_lock_bh(&ioat->prep_lock);
  592. /* never allow the last descriptor to be consumed, we need at
  593. * least one free at all times to allow for on-the-fly ring
  594. * resizing.
  595. */
  596. if (likely(ioat2_ring_space(ioat) > num_descs)) {
  597. dev_dbg(to_dev(chan), "%s: num_descs: %d (%x:%x:%x)\n",
  598. __func__, num_descs, ioat->head, ioat->tail, ioat->issued);
  599. ioat->produce = num_descs;
  600. return 0; /* with ioat->prep_lock held */
  601. }
  602. retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &chan->state);
  603. spin_unlock_bh(&ioat->prep_lock);
  604. /* is another cpu already trying to expand the ring? */
  605. if (retry)
  606. goto retry;
  607. spin_lock_bh(&chan->cleanup_lock);
  608. spin_lock_bh(&ioat->prep_lock);
  609. retry = reshape_ring(ioat, ioat->alloc_order + 1);
  610. clear_bit(IOAT_RESHAPE_PENDING, &chan->state);
  611. spin_unlock_bh(&ioat->prep_lock);
  612. spin_unlock_bh(&chan->cleanup_lock);
  613. /* if we were able to expand the ring retry the allocation */
  614. if (retry)
  615. goto retry;
  616. if (printk_ratelimit())
  617. dev_dbg(to_dev(chan), "%s: ring full! num_descs: %d (%x:%x:%x)\n",
  618. __func__, num_descs, ioat->head, ioat->tail, ioat->issued);
  619. /* progress reclaim in the allocation failure case we may be
  620. * called under bh_disabled so we need to trigger the timer
  621. * event directly
  622. */
  623. if (jiffies > chan->timer.expires && timer_pending(&chan->timer)) {
  624. struct ioatdma_device *device = chan->device;
  625. mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
  626. device->timer_fn((unsigned long) &chan->common);
  627. }
  628. return -ENOMEM;
  629. }
  630. struct dma_async_tx_descriptor *
  631. ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
  632. dma_addr_t dma_src, size_t len, unsigned long flags)
  633. {
  634. struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
  635. struct ioat_dma_descriptor *hw;
  636. struct ioat_ring_ent *desc;
  637. dma_addr_t dst = dma_dest;
  638. dma_addr_t src = dma_src;
  639. size_t total_len = len;
  640. int num_descs, idx, i;
  641. num_descs = ioat2_xferlen_to_descs(ioat, len);
  642. if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs) == 0)
  643. idx = ioat->head;
  644. else
  645. return NULL;
  646. i = 0;
  647. do {
  648. size_t copy = min_t(size_t, len, 1 << ioat->xfercap_log);
  649. desc = ioat2_get_ring_ent(ioat, idx + i);
  650. hw = desc->hw;
  651. hw->size = copy;
  652. hw->ctl = 0;
  653. hw->src_addr = src;
  654. hw->dst_addr = dst;
  655. len -= copy;
  656. dst += copy;
  657. src += copy;
  658. dump_desc_dbg(ioat, desc);
  659. } while (++i < num_descs);
  660. desc->txd.flags = flags;
  661. desc->len = total_len;
  662. hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
  663. hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
  664. hw->ctl_f.compl_write = 1;
  665. dump_desc_dbg(ioat, desc);
  666. /* we leave the channel locked to ensure in order submission */
  667. return &desc->txd;
  668. }
  669. /**
  670. * ioat2_free_chan_resources - release all the descriptors
  671. * @chan: the channel to be cleaned
  672. */
  673. void ioat2_free_chan_resources(struct dma_chan *c)
  674. {
  675. struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
  676. struct ioat_chan_common *chan = &ioat->base;
  677. struct ioatdma_device *device = chan->device;
  678. struct ioat_ring_ent *desc;
  679. const u16 total_descs = 1 << ioat->alloc_order;
  680. int descs;
  681. int i;
  682. /* Before freeing channel resources first check
  683. * if they have been previously allocated for this channel.
  684. */
  685. if (!ioat->ring)
  686. return;
  687. ioat_stop(chan);
  688. device->reset_hw(chan);
  689. spin_lock_bh(&chan->cleanup_lock);
  690. spin_lock_bh(&ioat->prep_lock);
  691. descs = ioat2_ring_space(ioat);
  692. dev_dbg(to_dev(chan), "freeing %d idle descriptors\n", descs);
  693. for (i = 0; i < descs; i++) {
  694. desc = ioat2_get_ring_ent(ioat, ioat->head + i);
  695. ioat2_free_ring_ent(desc, c);
  696. }
  697. if (descs < total_descs)
  698. dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
  699. total_descs - descs);
  700. for (i = 0; i < total_descs - descs; i++) {
  701. desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
  702. dump_desc_dbg(ioat, desc);
  703. ioat2_free_ring_ent(desc, c);
  704. }
  705. kfree(ioat->ring);
  706. ioat->ring = NULL;
  707. ioat->alloc_order = 0;
  708. pci_pool_free(device->completion_pool, chan->completion,
  709. chan->completion_dma);
  710. spin_unlock_bh(&ioat->prep_lock);
  711. spin_unlock_bh(&chan->cleanup_lock);
  712. chan->last_completion = 0;
  713. chan->completion_dma = 0;
  714. ioat->dmacount = 0;
  715. }
  716. static ssize_t ring_size_show(struct dma_chan *c, char *page)
  717. {
  718. struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
  719. return sprintf(page, "%d\n", (1 << ioat->alloc_order) & ~1);
  720. }
  721. static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
  722. static ssize_t ring_active_show(struct dma_chan *c, char *page)
  723. {
  724. struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
  725. /* ...taken outside the lock, no need to be precise */
  726. return sprintf(page, "%d\n", ioat2_ring_active(ioat));
  727. }
  728. static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
  729. static struct attribute *ioat2_attrs[] = {
  730. &ring_size_attr.attr,
  731. &ring_active_attr.attr,
  732. &ioat_cap_attr.attr,
  733. &ioat_version_attr.attr,
  734. NULL,
  735. };
  736. struct kobj_type ioat2_ktype = {
  737. .sysfs_ops = &ioat_sysfs_ops,
  738. .default_attrs = ioat2_attrs,
  739. };
  740. int ioat2_dma_probe(struct ioatdma_device *device, int dca)
  741. {
  742. struct pci_dev *pdev = device->pdev;
  743. struct dma_device *dma;
  744. struct dma_chan *c;
  745. struct ioat_chan_common *chan;
  746. int err;
  747. device->enumerate_channels = ioat2_enumerate_channels;
  748. device->reset_hw = ioat2_reset_hw;
  749. device->cleanup_fn = ioat2_cleanup_event;
  750. device->timer_fn = ioat2_timer_event;
  751. device->self_test = ioat_dma_self_test;
  752. dma = &device->common;
  753. dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
  754. dma->device_issue_pending = ioat2_issue_pending;
  755. dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
  756. dma->device_free_chan_resources = ioat2_free_chan_resources;
  757. dma->device_tx_status = ioat_dma_tx_status;
  758. err = ioat_probe(device);
  759. if (err)
  760. return err;
  761. ioat_set_tcp_copy_break(2048);
  762. list_for_each_entry(c, &dma->channels, device_node) {
  763. chan = to_chan_common(c);
  764. writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
  765. chan->reg_base + IOAT_DCACTRL_OFFSET);
  766. }
  767. err = ioat_register(device);
  768. if (err)
  769. return err;
  770. ioat_kobject_add(device, &ioat2_ktype);
  771. if (dca)
  772. device->dca = ioat2_dca_init(pdev, device->reg_base);
  773. return err;
  774. }