amdgpu_fence.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762
  1. /*
  2. * Copyright 2009 Jerome Glisse.
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining a
  6. * copy of this software and associated documentation files (the
  7. * "Software"), to deal in the Software without restriction, including
  8. * without limitation the rights to use, copy, modify, merge, publish,
  9. * distribute, sub license, and/or sell copies of the Software, and to
  10. * permit persons to whom the Software is furnished to do so, subject to
  11. * the following conditions:
  12. *
  13. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15. * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16. * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17. * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18. * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19. * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20. *
  21. * The above copyright notice and this permission notice (including the
  22. * next paragraph) shall be included in all copies or substantial portions
  23. * of the Software.
  24. *
  25. */
  26. /*
  27. * Authors:
  28. * Jerome Glisse <glisse@freedesktop.org>
  29. * Dave Airlie
  30. */
  31. #include <linux/seq_file.h>
  32. #include <linux/atomic.h>
  33. #include <linux/wait.h>
  34. #include <linux/kref.h>
  35. #include <linux/slab.h>
  36. #include <linux/firmware.h>
  37. #include <drm/drmP.h>
  38. #include "amdgpu.h"
  39. #include "amdgpu_trace.h"
  40. /*
  41. * Fences
  42. * Fences mark an event in the GPUs pipeline and are used
  43. * for GPU/CPU synchronization. When the fence is written,
  44. * it is expected that all buffers associated with that fence
  45. * are no longer in use by the associated ring on the GPU and
  46. * that the the relevant GPU caches have been flushed.
  47. */
  48. static struct kmem_cache *amdgpu_fence_slab;
  49. static atomic_t amdgpu_fence_slab_ref = ATOMIC_INIT(0);
  50. /**
  51. * amdgpu_fence_write - write a fence value
  52. *
  53. * @ring: ring the fence is associated with
  54. * @seq: sequence number to write
  55. *
  56. * Writes a fence value to memory (all asics).
  57. */
  58. static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)
  59. {
  60. struct amdgpu_fence_driver *drv = &ring->fence_drv;
  61. if (drv->cpu_addr)
  62. *drv->cpu_addr = cpu_to_le32(seq);
  63. }
  64. /**
  65. * amdgpu_fence_read - read a fence value
  66. *
  67. * @ring: ring the fence is associated with
  68. *
  69. * Reads a fence value from memory (all asics).
  70. * Returns the value of the fence read from memory.
  71. */
  72. static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
  73. {
  74. struct amdgpu_fence_driver *drv = &ring->fence_drv;
  75. u32 seq = 0;
  76. if (drv->cpu_addr)
  77. seq = le32_to_cpu(*drv->cpu_addr);
  78. else
  79. seq = lower_32_bits(atomic64_read(&drv->last_seq));
  80. return seq;
  81. }
  82. /**
  83. * amdgpu_fence_emit - emit a fence on the requested ring
  84. *
  85. * @ring: ring the fence is associated with
  86. * @owner: creator of the fence
  87. * @fence: amdgpu fence object
  88. *
  89. * Emits a fence command on the requested ring (all asics).
  90. * Returns 0 on success, -ENOMEM on failure.
  91. */
  92. int amdgpu_fence_emit(struct amdgpu_ring *ring, void *owner,
  93. struct amdgpu_fence **fence)
  94. {
  95. struct amdgpu_device *adev = ring->adev;
  96. /* we are protected by the ring emission mutex */
  97. *fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL);
  98. if ((*fence) == NULL) {
  99. return -ENOMEM;
  100. }
  101. (*fence)->seq = ++ring->fence_drv.sync_seq;
  102. (*fence)->ring = ring;
  103. (*fence)->owner = owner;
  104. fence_init(&(*fence)->base, &amdgpu_fence_ops,
  105. &ring->fence_drv.fence_queue.lock,
  106. adev->fence_context + ring->idx,
  107. (*fence)->seq);
  108. amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
  109. (*fence)->seq,
  110. AMDGPU_FENCE_FLAG_INT);
  111. return 0;
  112. }
  113. /**
  114. * amdgpu_fence_schedule_fallback - schedule fallback check
  115. *
  116. * @ring: pointer to struct amdgpu_ring
  117. *
  118. * Start a timer as fallback to our interrupts.
  119. */
  120. static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring)
  121. {
  122. mod_timer(&ring->fence_drv.fallback_timer,
  123. jiffies + AMDGPU_FENCE_JIFFIES_TIMEOUT);
  124. }
  125. /**
  126. * amdgpu_fence_activity - check for fence activity
  127. *
  128. * @ring: pointer to struct amdgpu_ring
  129. *
  130. * Checks the current fence value and calculates the last
  131. * signalled fence value. Returns true if activity occured
  132. * on the ring, and the fence_queue should be waken up.
  133. */
  134. static bool amdgpu_fence_activity(struct amdgpu_ring *ring)
  135. {
  136. uint64_t seq, last_seq, last_emitted;
  137. unsigned count_loop = 0;
  138. bool wake = false;
  139. /* Note there is a scenario here for an infinite loop but it's
  140. * very unlikely to happen. For it to happen, the current polling
  141. * process need to be interrupted by another process and another
  142. * process needs to update the last_seq btw the atomic read and
  143. * xchg of the current process.
  144. *
  145. * More over for this to go in infinite loop there need to be
  146. * continuously new fence signaled ie amdgpu_fence_read needs
  147. * to return a different value each time for both the currently
  148. * polling process and the other process that xchg the last_seq
  149. * btw atomic read and xchg of the current process. And the
  150. * value the other process set as last seq must be higher than
  151. * the seq value we just read. Which means that current process
  152. * need to be interrupted after amdgpu_fence_read and before
  153. * atomic xchg.
  154. *
  155. * To be even more safe we count the number of time we loop and
  156. * we bail after 10 loop just accepting the fact that we might
  157. * have temporarly set the last_seq not to the true real last
  158. * seq but to an older one.
  159. */
  160. last_seq = atomic64_read(&ring->fence_drv.last_seq);
  161. do {
  162. last_emitted = ring->fence_drv.sync_seq;
  163. seq = amdgpu_fence_read(ring);
  164. seq |= last_seq & 0xffffffff00000000LL;
  165. if (seq < last_seq) {
  166. seq &= 0xffffffff;
  167. seq |= last_emitted & 0xffffffff00000000LL;
  168. }
  169. if (seq <= last_seq || seq > last_emitted) {
  170. break;
  171. }
  172. /* If we loop over we don't want to return without
  173. * checking if a fence is signaled as it means that the
  174. * seq we just read is different from the previous on.
  175. */
  176. wake = true;
  177. last_seq = seq;
  178. if ((count_loop++) > 10) {
  179. /* We looped over too many time leave with the
  180. * fact that we might have set an older fence
  181. * seq then the current real last seq as signaled
  182. * by the hw.
  183. */
  184. break;
  185. }
  186. } while (atomic64_xchg(&ring->fence_drv.last_seq, seq) > seq);
  187. if (seq < last_emitted)
  188. amdgpu_fence_schedule_fallback(ring);
  189. return wake;
  190. }
  191. /**
  192. * amdgpu_fence_process - process a fence
  193. *
  194. * @adev: amdgpu_device pointer
  195. * @ring: ring index the fence is associated with
  196. *
  197. * Checks the current fence value and wakes the fence queue
  198. * if the sequence number has increased (all asics).
  199. */
  200. void amdgpu_fence_process(struct amdgpu_ring *ring)
  201. {
  202. if (amdgpu_fence_activity(ring))
  203. wake_up_all(&ring->fence_drv.fence_queue);
  204. }
  205. /**
  206. * amdgpu_fence_fallback - fallback for hardware interrupts
  207. *
  208. * @work: delayed work item
  209. *
  210. * Checks for fence activity.
  211. */
  212. static void amdgpu_fence_fallback(unsigned long arg)
  213. {
  214. struct amdgpu_ring *ring = (void *)arg;
  215. amdgpu_fence_process(ring);
  216. }
  217. /**
  218. * amdgpu_fence_seq_signaled - check if a fence sequence number has signaled
  219. *
  220. * @ring: ring the fence is associated with
  221. * @seq: sequence number
  222. *
  223. * Check if the last signaled fence sequnce number is >= the requested
  224. * sequence number (all asics).
  225. * Returns true if the fence has signaled (current fence value
  226. * is >= requested value) or false if it has not (current fence
  227. * value is < the requested value. Helper function for
  228. * amdgpu_fence_signaled().
  229. */
  230. static bool amdgpu_fence_seq_signaled(struct amdgpu_ring *ring, u64 seq)
  231. {
  232. if (atomic64_read(&ring->fence_drv.last_seq) >= seq)
  233. return true;
  234. /* poll new last sequence at least once */
  235. amdgpu_fence_process(ring);
  236. if (atomic64_read(&ring->fence_drv.last_seq) >= seq)
  237. return true;
  238. return false;
  239. }
  240. /*
  241. * amdgpu_ring_wait_seq_timeout - wait for seq of the specific ring to signal
  242. * @ring: ring to wait on for the seq number
  243. * @seq: seq number wait for
  244. *
  245. * return value:
  246. * 0: seq signaled, and gpu not hang
  247. * -EDEADL: GPU hang detected
  248. * -EINVAL: some paramter is not valid
  249. */
  250. static int amdgpu_fence_ring_wait_seq(struct amdgpu_ring *ring, uint64_t seq)
  251. {
  252. bool signaled = false;
  253. BUG_ON(!ring);
  254. if (seq > ring->fence_drv.sync_seq)
  255. return -EINVAL;
  256. if (atomic64_read(&ring->fence_drv.last_seq) >= seq)
  257. return 0;
  258. amdgpu_fence_schedule_fallback(ring);
  259. wait_event(ring->fence_drv.fence_queue, (
  260. (signaled = amdgpu_fence_seq_signaled(ring, seq))));
  261. if (signaled)
  262. return 0;
  263. else
  264. return -EDEADLK;
  265. }
  266. /**
  267. * amdgpu_fence_wait_next - wait for the next fence to signal
  268. *
  269. * @adev: amdgpu device pointer
  270. * @ring: ring index the fence is associated with
  271. *
  272. * Wait for the next fence on the requested ring to signal (all asics).
  273. * Returns 0 if the next fence has passed, error for all other cases.
  274. * Caller must hold ring lock.
  275. */
  276. int amdgpu_fence_wait_next(struct amdgpu_ring *ring)
  277. {
  278. uint64_t seq = atomic64_read(&ring->fence_drv.last_seq) + 1ULL;
  279. if (seq >= ring->fence_drv.sync_seq)
  280. return -ENOENT;
  281. return amdgpu_fence_ring_wait_seq(ring, seq);
  282. }
  283. /**
  284. * amdgpu_fence_wait_empty - wait for all fences to signal
  285. *
  286. * @adev: amdgpu device pointer
  287. * @ring: ring index the fence is associated with
  288. *
  289. * Wait for all fences on the requested ring to signal (all asics).
  290. * Returns 0 if the fences have passed, error for all other cases.
  291. * Caller must hold ring lock.
  292. */
  293. int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
  294. {
  295. uint64_t seq = ring->fence_drv.sync_seq;
  296. if (!seq)
  297. return 0;
  298. return amdgpu_fence_ring_wait_seq(ring, seq);
  299. }
  300. /**
  301. * amdgpu_fence_count_emitted - get the count of emitted fences
  302. *
  303. * @ring: ring the fence is associated with
  304. *
  305. * Get the number of fences emitted on the requested ring (all asics).
  306. * Returns the number of emitted fences on the ring. Used by the
  307. * dynpm code to ring track activity.
  308. */
  309. unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring)
  310. {
  311. uint64_t emitted;
  312. /* We are not protected by ring lock when reading the last sequence
  313. * but it's ok to report slightly wrong fence count here.
  314. */
  315. amdgpu_fence_process(ring);
  316. emitted = ring->fence_drv.sync_seq
  317. - atomic64_read(&ring->fence_drv.last_seq);
  318. /* to avoid 32bits warp around */
  319. if (emitted > 0x10000000)
  320. emitted = 0x10000000;
  321. return (unsigned)emitted;
  322. }
  323. /**
  324. * amdgpu_fence_driver_start_ring - make the fence driver
  325. * ready for use on the requested ring.
  326. *
  327. * @ring: ring to start the fence driver on
  328. * @irq_src: interrupt source to use for this ring
  329. * @irq_type: interrupt type to use for this ring
  330. *
  331. * Make the fence driver ready for processing (all asics).
  332. * Not all asics have all rings, so each asic will only
  333. * start the fence driver on the rings it has.
  334. * Returns 0 for success, errors for failure.
  335. */
  336. int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
  337. struct amdgpu_irq_src *irq_src,
  338. unsigned irq_type)
  339. {
  340. struct amdgpu_device *adev = ring->adev;
  341. uint64_t index;
  342. if (ring != &adev->uvd.ring) {
  343. ring->fence_drv.cpu_addr = &adev->wb.wb[ring->fence_offs];
  344. ring->fence_drv.gpu_addr = adev->wb.gpu_addr + (ring->fence_offs * 4);
  345. } else {
  346. /* put fence directly behind firmware */
  347. index = ALIGN(adev->uvd.fw->size, 8);
  348. ring->fence_drv.cpu_addr = adev->uvd.cpu_addr + index;
  349. ring->fence_drv.gpu_addr = adev->uvd.gpu_addr + index;
  350. }
  351. amdgpu_fence_write(ring, atomic64_read(&ring->fence_drv.last_seq));
  352. amdgpu_irq_get(adev, irq_src, irq_type);
  353. ring->fence_drv.irq_src = irq_src;
  354. ring->fence_drv.irq_type = irq_type;
  355. ring->fence_drv.initialized = true;
  356. dev_info(adev->dev, "fence driver on ring %d use gpu addr 0x%016llx, "
  357. "cpu addr 0x%p\n", ring->idx,
  358. ring->fence_drv.gpu_addr, ring->fence_drv.cpu_addr);
  359. return 0;
  360. }
  361. /**
  362. * amdgpu_fence_driver_init_ring - init the fence driver
  363. * for the requested ring.
  364. *
  365. * @ring: ring to init the fence driver on
  366. *
  367. * Init the fence driver for the requested ring (all asics).
  368. * Helper function for amdgpu_fence_driver_init().
  369. */
  370. int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring)
  371. {
  372. long timeout;
  373. int r;
  374. ring->fence_drv.cpu_addr = NULL;
  375. ring->fence_drv.gpu_addr = 0;
  376. ring->fence_drv.sync_seq = 0;
  377. atomic64_set(&ring->fence_drv.last_seq, 0);
  378. ring->fence_drv.initialized = false;
  379. setup_timer(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback,
  380. (unsigned long)ring);
  381. init_waitqueue_head(&ring->fence_drv.fence_queue);
  382. timeout = msecs_to_jiffies(amdgpu_lockup_timeout);
  383. if (timeout == 0) {
  384. /*
  385. * FIXME:
  386. * Delayed workqueue cannot use it directly,
  387. * so the scheduler will not use delayed workqueue if
  388. * MAX_SCHEDULE_TIMEOUT is set.
  389. * Currently keep it simple and silly.
  390. */
  391. timeout = MAX_SCHEDULE_TIMEOUT;
  392. }
  393. r = amd_sched_init(&ring->sched, &amdgpu_sched_ops,
  394. amdgpu_sched_hw_submission,
  395. timeout, ring->name);
  396. if (r) {
  397. DRM_ERROR("Failed to create scheduler on ring %s.\n",
  398. ring->name);
  399. return r;
  400. }
  401. return 0;
  402. }
  403. /**
  404. * amdgpu_fence_driver_init - init the fence driver
  405. * for all possible rings.
  406. *
  407. * @adev: amdgpu device pointer
  408. *
  409. * Init the fence driver for all possible rings (all asics).
  410. * Not all asics have all rings, so each asic will only
  411. * start the fence driver on the rings it has using
  412. * amdgpu_fence_driver_start_ring().
  413. * Returns 0 for success.
  414. */
  415. int amdgpu_fence_driver_init(struct amdgpu_device *adev)
  416. {
  417. if (atomic_inc_return(&amdgpu_fence_slab_ref) == 1) {
  418. amdgpu_fence_slab = kmem_cache_create(
  419. "amdgpu_fence", sizeof(struct amdgpu_fence), 0,
  420. SLAB_HWCACHE_ALIGN, NULL);
  421. if (!amdgpu_fence_slab)
  422. return -ENOMEM;
  423. }
  424. if (amdgpu_debugfs_fence_init(adev))
  425. dev_err(adev->dev, "fence debugfs file creation failed\n");
  426. return 0;
  427. }
  428. /**
  429. * amdgpu_fence_driver_fini - tear down the fence driver
  430. * for all possible rings.
  431. *
  432. * @adev: amdgpu device pointer
  433. *
  434. * Tear down the fence driver for all possible rings (all asics).
  435. */
  436. void amdgpu_fence_driver_fini(struct amdgpu_device *adev)
  437. {
  438. int i, r;
  439. if (atomic_dec_and_test(&amdgpu_fence_slab_ref))
  440. kmem_cache_destroy(amdgpu_fence_slab);
  441. for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
  442. struct amdgpu_ring *ring = adev->rings[i];
  443. if (!ring || !ring->fence_drv.initialized)
  444. continue;
  445. r = amdgpu_fence_wait_empty(ring);
  446. if (r) {
  447. /* no need to trigger GPU reset as we are unloading */
  448. amdgpu_fence_driver_force_completion(adev);
  449. }
  450. wake_up_all(&ring->fence_drv.fence_queue);
  451. amdgpu_irq_put(adev, ring->fence_drv.irq_src,
  452. ring->fence_drv.irq_type);
  453. amd_sched_fini(&ring->sched);
  454. del_timer_sync(&ring->fence_drv.fallback_timer);
  455. ring->fence_drv.initialized = false;
  456. }
  457. }
  458. /**
  459. * amdgpu_fence_driver_suspend - suspend the fence driver
  460. * for all possible rings.
  461. *
  462. * @adev: amdgpu device pointer
  463. *
  464. * Suspend the fence driver for all possible rings (all asics).
  465. */
  466. void amdgpu_fence_driver_suspend(struct amdgpu_device *adev)
  467. {
  468. int i, r;
  469. for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
  470. struct amdgpu_ring *ring = adev->rings[i];
  471. if (!ring || !ring->fence_drv.initialized)
  472. continue;
  473. /* wait for gpu to finish processing current batch */
  474. r = amdgpu_fence_wait_empty(ring);
  475. if (r) {
  476. /* delay GPU reset to resume */
  477. amdgpu_fence_driver_force_completion(adev);
  478. }
  479. /* disable the interrupt */
  480. amdgpu_irq_put(adev, ring->fence_drv.irq_src,
  481. ring->fence_drv.irq_type);
  482. }
  483. }
  484. /**
  485. * amdgpu_fence_driver_resume - resume the fence driver
  486. * for all possible rings.
  487. *
  488. * @adev: amdgpu device pointer
  489. *
  490. * Resume the fence driver for all possible rings (all asics).
  491. * Not all asics have all rings, so each asic will only
  492. * start the fence driver on the rings it has using
  493. * amdgpu_fence_driver_start_ring().
  494. * Returns 0 for success.
  495. */
  496. void amdgpu_fence_driver_resume(struct amdgpu_device *adev)
  497. {
  498. int i;
  499. for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
  500. struct amdgpu_ring *ring = adev->rings[i];
  501. if (!ring || !ring->fence_drv.initialized)
  502. continue;
  503. /* enable the interrupt */
  504. amdgpu_irq_get(adev, ring->fence_drv.irq_src,
  505. ring->fence_drv.irq_type);
  506. }
  507. }
  508. /**
  509. * amdgpu_fence_driver_force_completion - force all fence waiter to complete
  510. *
  511. * @adev: amdgpu device pointer
  512. *
  513. * In case of GPU reset failure make sure no process keep waiting on fence
  514. * that will never complete.
  515. */
  516. void amdgpu_fence_driver_force_completion(struct amdgpu_device *adev)
  517. {
  518. int i;
  519. for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
  520. struct amdgpu_ring *ring = adev->rings[i];
  521. if (!ring || !ring->fence_drv.initialized)
  522. continue;
  523. amdgpu_fence_write(ring, ring->fence_drv.sync_seq);
  524. }
  525. }
  526. /*
  527. * Common fence implementation
  528. */
  529. static const char *amdgpu_fence_get_driver_name(struct fence *fence)
  530. {
  531. return "amdgpu";
  532. }
  533. static const char *amdgpu_fence_get_timeline_name(struct fence *f)
  534. {
  535. struct amdgpu_fence *fence = to_amdgpu_fence(f);
  536. return (const char *)fence->ring->name;
  537. }
  538. /**
  539. * amdgpu_fence_is_signaled - test if fence is signaled
  540. *
  541. * @f: fence to test
  542. *
  543. * Test the fence sequence number if it is already signaled. If it isn't
  544. * signaled start fence processing. Returns True if the fence is signaled.
  545. */
  546. static bool amdgpu_fence_is_signaled(struct fence *f)
  547. {
  548. struct amdgpu_fence *fence = to_amdgpu_fence(f);
  549. struct amdgpu_ring *ring = fence->ring;
  550. if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)
  551. return true;
  552. amdgpu_fence_process(ring);
  553. if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)
  554. return true;
  555. return false;
  556. }
  557. /**
  558. * amdgpu_fence_check_signaled - callback from fence_queue
  559. *
  560. * this function is called with fence_queue lock held, which is also used
  561. * for the fence locking itself, so unlocked variants are used for
  562. * fence_signal, and remove_wait_queue.
  563. */
  564. static int amdgpu_fence_check_signaled(wait_queue_t *wait, unsigned mode, int flags, void *key)
  565. {
  566. struct amdgpu_fence *fence;
  567. struct amdgpu_device *adev;
  568. u64 seq;
  569. int ret;
  570. fence = container_of(wait, struct amdgpu_fence, fence_wake);
  571. adev = fence->ring->adev;
  572. /*
  573. * We cannot use amdgpu_fence_process here because we're already
  574. * in the waitqueue, in a call from wake_up_all.
  575. */
  576. seq = atomic64_read(&fence->ring->fence_drv.last_seq);
  577. if (seq >= fence->seq) {
  578. ret = fence_signal_locked(&fence->base);
  579. if (!ret)
  580. FENCE_TRACE(&fence->base, "signaled from irq context\n");
  581. else
  582. FENCE_TRACE(&fence->base, "was already signaled\n");
  583. __remove_wait_queue(&fence->ring->fence_drv.fence_queue, &fence->fence_wake);
  584. fence_put(&fence->base);
  585. } else
  586. FENCE_TRACE(&fence->base, "pending\n");
  587. return 0;
  588. }
  589. /**
  590. * amdgpu_fence_enable_signaling - enable signalling on fence
  591. * @fence: fence
  592. *
  593. * This function is called with fence_queue lock held, and adds a callback
  594. * to fence_queue that checks if this fence is signaled, and if so it
  595. * signals the fence and removes itself.
  596. */
  597. static bool amdgpu_fence_enable_signaling(struct fence *f)
  598. {
  599. struct amdgpu_fence *fence = to_amdgpu_fence(f);
  600. struct amdgpu_ring *ring = fence->ring;
  601. if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)
  602. return false;
  603. fence->fence_wake.flags = 0;
  604. fence->fence_wake.private = NULL;
  605. fence->fence_wake.func = amdgpu_fence_check_signaled;
  606. __add_wait_queue(&ring->fence_drv.fence_queue, &fence->fence_wake);
  607. fence_get(f);
  608. if (!timer_pending(&ring->fence_drv.fallback_timer))
  609. amdgpu_fence_schedule_fallback(ring);
  610. FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx);
  611. return true;
  612. }
  613. static void amdgpu_fence_release(struct fence *f)
  614. {
  615. struct amdgpu_fence *fence = to_amdgpu_fence(f);
  616. kmem_cache_free(amdgpu_fence_slab, fence);
  617. }
  618. const struct fence_ops amdgpu_fence_ops = {
  619. .get_driver_name = amdgpu_fence_get_driver_name,
  620. .get_timeline_name = amdgpu_fence_get_timeline_name,
  621. .enable_signaling = amdgpu_fence_enable_signaling,
  622. .signaled = amdgpu_fence_is_signaled,
  623. .wait = fence_default_wait,
  624. .release = amdgpu_fence_release,
  625. };
  626. /*
  627. * Fence debugfs
  628. */
  629. #if defined(CONFIG_DEBUG_FS)
  630. static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data)
  631. {
  632. struct drm_info_node *node = (struct drm_info_node *)m->private;
  633. struct drm_device *dev = node->minor->dev;
  634. struct amdgpu_device *adev = dev->dev_private;
  635. int i;
  636. for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
  637. struct amdgpu_ring *ring = adev->rings[i];
  638. if (!ring || !ring->fence_drv.initialized)
  639. continue;
  640. amdgpu_fence_process(ring);
  641. seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name);
  642. seq_printf(m, "Last signaled fence 0x%016llx\n",
  643. (unsigned long long)atomic64_read(&ring->fence_drv.last_seq));
  644. seq_printf(m, "Last emitted 0x%016llx\n",
  645. ring->fence_drv.sync_seq);
  646. }
  647. return 0;
  648. }
  649. /**
  650. * amdgpu_debugfs_gpu_reset - manually trigger a gpu reset
  651. *
  652. * Manually trigger a gpu reset at the next fence wait.
  653. */
  654. static int amdgpu_debugfs_gpu_reset(struct seq_file *m, void *data)
  655. {
  656. struct drm_info_node *node = (struct drm_info_node *) m->private;
  657. struct drm_device *dev = node->minor->dev;
  658. struct amdgpu_device *adev = dev->dev_private;
  659. seq_printf(m, "gpu reset\n");
  660. amdgpu_gpu_reset(adev);
  661. return 0;
  662. }
  663. static struct drm_info_list amdgpu_debugfs_fence_list[] = {
  664. {"amdgpu_fence_info", &amdgpu_debugfs_fence_info, 0, NULL},
  665. {"amdgpu_gpu_reset", &amdgpu_debugfs_gpu_reset, 0, NULL}
  666. };
  667. #endif
  668. int amdgpu_debugfs_fence_init(struct amdgpu_device *adev)
  669. {
  670. #if defined(CONFIG_DEBUG_FS)
  671. return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_fence_list, 2);
  672. #else
  673. return 0;
  674. #endif
  675. }