amdgpu_fence.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768
  1. /*
  2. * Copyright 2009 Jerome Glisse.
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining a
  6. * copy of this software and associated documentation files (the
  7. * "Software"), to deal in the Software without restriction, including
  8. * without limitation the rights to use, copy, modify, merge, publish,
  9. * distribute, sub license, and/or sell copies of the Software, and to
  10. * permit persons to whom the Software is furnished to do so, subject to
  11. * the following conditions:
  12. *
  13. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15. * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16. * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17. * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18. * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19. * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20. *
  21. * The above copyright notice and this permission notice (including the
  22. * next paragraph) shall be included in all copies or substantial portions
  23. * of the Software.
  24. *
  25. */
  26. /*
  27. * Authors:
  28. * Jerome Glisse <glisse@freedesktop.org>
  29. * Dave Airlie
  30. */
  31. #include <linux/seq_file.h>
  32. #include <linux/atomic.h>
  33. #include <linux/wait.h>
  34. #include <linux/kref.h>
  35. #include <linux/slab.h>
  36. #include <linux/firmware.h>
  37. #include <drm/drmP.h>
  38. #include "amdgpu.h"
  39. #include "amdgpu_trace.h"
  40. /*
  41. * Fences
  42. * Fences mark an event in the GPUs pipeline and are used
  43. * for GPU/CPU synchronization. When the fence is written,
  44. * it is expected that all buffers associated with that fence
  45. * are no longer in use by the associated ring on the GPU and
  46. * that the the relevant GPU caches have been flushed.
  47. */
  48. static struct kmem_cache *amdgpu_fence_slab;
  49. static atomic_t amdgpu_fence_slab_ref = ATOMIC_INIT(0);
  50. /**
  51. * amdgpu_fence_write - write a fence value
  52. *
  53. * @ring: ring the fence is associated with
  54. * @seq: sequence number to write
  55. *
  56. * Writes a fence value to memory (all asics).
  57. */
  58. static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)
  59. {
  60. struct amdgpu_fence_driver *drv = &ring->fence_drv;
  61. if (drv->cpu_addr)
  62. *drv->cpu_addr = cpu_to_le32(seq);
  63. }
  64. /**
  65. * amdgpu_fence_read - read a fence value
  66. *
  67. * @ring: ring the fence is associated with
  68. *
  69. * Reads a fence value from memory (all asics).
  70. * Returns the value of the fence read from memory.
  71. */
  72. static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
  73. {
  74. struct amdgpu_fence_driver *drv = &ring->fence_drv;
  75. u32 seq = 0;
  76. if (drv->cpu_addr)
  77. seq = le32_to_cpu(*drv->cpu_addr);
  78. else
  79. seq = lower_32_bits(atomic64_read(&drv->last_seq));
  80. return seq;
  81. }
  82. /**
  83. * amdgpu_fence_emit - emit a fence on the requested ring
  84. *
  85. * @ring: ring the fence is associated with
  86. * @owner: creator of the fence
  87. * @fence: amdgpu fence object
  88. *
  89. * Emits a fence command on the requested ring (all asics).
  90. * Returns 0 on success, -ENOMEM on failure.
  91. */
  92. int amdgpu_fence_emit(struct amdgpu_ring *ring, void *owner,
  93. struct amdgpu_fence **fence)
  94. {
  95. struct amdgpu_device *adev = ring->adev;
  96. /* we are protected by the ring emission mutex */
  97. *fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL);
  98. if ((*fence) == NULL) {
  99. return -ENOMEM;
  100. }
  101. (*fence)->seq = ++ring->fence_drv.sync_seq;
  102. (*fence)->ring = ring;
  103. (*fence)->owner = owner;
  104. fence_init(&(*fence)->base, &amdgpu_fence_ops,
  105. &ring->fence_drv.fence_queue.lock,
  106. adev->fence_context + ring->idx,
  107. (*fence)->seq);
  108. amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
  109. (*fence)->seq,
  110. AMDGPU_FENCE_FLAG_INT);
  111. return 0;
  112. }
  113. /**
  114. * amdgpu_fence_schedule_fallback - schedule fallback check
  115. *
  116. * @ring: pointer to struct amdgpu_ring
  117. *
  118. * Start a timer as fallback to our interrupts.
  119. */
  120. static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring)
  121. {
  122. mod_timer(&ring->fence_drv.fallback_timer,
  123. jiffies + AMDGPU_FENCE_JIFFIES_TIMEOUT);
  124. }
  125. /**
  126. * amdgpu_fence_activity - check for fence activity
  127. *
  128. * @ring: pointer to struct amdgpu_ring
  129. *
  130. * Checks the current fence value and calculates the last
  131. * signalled fence value. Returns true if activity occured
  132. * on the ring, and the fence_queue should be waken up.
  133. */
  134. static bool amdgpu_fence_activity(struct amdgpu_ring *ring)
  135. {
  136. uint64_t seq, last_seq, last_emitted;
  137. unsigned count_loop = 0;
  138. bool wake = false;
  139. /* Note there is a scenario here for an infinite loop but it's
  140. * very unlikely to happen. For it to happen, the current polling
  141. * process need to be interrupted by another process and another
  142. * process needs to update the last_seq btw the atomic read and
  143. * xchg of the current process.
  144. *
  145. * More over for this to go in infinite loop there need to be
  146. * continuously new fence signaled ie amdgpu_fence_read needs
  147. * to return a different value each time for both the currently
  148. * polling process and the other process that xchg the last_seq
  149. * btw atomic read and xchg of the current process. And the
  150. * value the other process set as last seq must be higher than
  151. * the seq value we just read. Which means that current process
  152. * need to be interrupted after amdgpu_fence_read and before
  153. * atomic xchg.
  154. *
  155. * To be even more safe we count the number of time we loop and
  156. * we bail after 10 loop just accepting the fact that we might
  157. * have temporarly set the last_seq not to the true real last
  158. * seq but to an older one.
  159. */
  160. last_seq = atomic64_read(&ring->fence_drv.last_seq);
  161. do {
  162. last_emitted = ring->fence_drv.sync_seq;
  163. seq = amdgpu_fence_read(ring);
  164. seq |= last_seq & 0xffffffff00000000LL;
  165. if (seq < last_seq) {
  166. seq &= 0xffffffff;
  167. seq |= last_emitted & 0xffffffff00000000LL;
  168. }
  169. if (seq <= last_seq || seq > last_emitted) {
  170. break;
  171. }
  172. /* If we loop over we don't want to return without
  173. * checking if a fence is signaled as it means that the
  174. * seq we just read is different from the previous on.
  175. */
  176. wake = true;
  177. last_seq = seq;
  178. if ((count_loop++) > 10) {
  179. /* We looped over too many time leave with the
  180. * fact that we might have set an older fence
  181. * seq then the current real last seq as signaled
  182. * by the hw.
  183. */
  184. break;
  185. }
  186. } while (atomic64_xchg(&ring->fence_drv.last_seq, seq) > seq);
  187. if (seq < last_emitted)
  188. amdgpu_fence_schedule_fallback(ring);
  189. return wake;
  190. }
  191. /**
  192. * amdgpu_fence_process - process a fence
  193. *
  194. * @adev: amdgpu_device pointer
  195. * @ring: ring index the fence is associated with
  196. *
  197. * Checks the current fence value and wakes the fence queue
  198. * if the sequence number has increased (all asics).
  199. */
  200. void amdgpu_fence_process(struct amdgpu_ring *ring)
  201. {
  202. if (amdgpu_fence_activity(ring))
  203. wake_up_all(&ring->fence_drv.fence_queue);
  204. }
  205. /**
  206. * amdgpu_fence_fallback - fallback for hardware interrupts
  207. *
  208. * @work: delayed work item
  209. *
  210. * Checks for fence activity.
  211. */
  212. static void amdgpu_fence_fallback(unsigned long arg)
  213. {
  214. struct amdgpu_ring *ring = (void *)arg;
  215. amdgpu_fence_process(ring);
  216. }
  217. /**
  218. * amdgpu_fence_seq_signaled - check if a fence sequence number has signaled
  219. *
  220. * @ring: ring the fence is associated with
  221. * @seq: sequence number
  222. *
  223. * Check if the last signaled fence sequnce number is >= the requested
  224. * sequence number (all asics).
  225. * Returns true if the fence has signaled (current fence value
  226. * is >= requested value) or false if it has not (current fence
  227. * value is < the requested value. Helper function for
  228. * amdgpu_fence_signaled().
  229. */
  230. static bool amdgpu_fence_seq_signaled(struct amdgpu_ring *ring, u64 seq)
  231. {
  232. if (atomic64_read(&ring->fence_drv.last_seq) >= seq)
  233. return true;
  234. /* poll new last sequence at least once */
  235. amdgpu_fence_process(ring);
  236. if (atomic64_read(&ring->fence_drv.last_seq) >= seq)
  237. return true;
  238. return false;
  239. }
  240. /*
  241. * amdgpu_ring_wait_seq_timeout - wait for seq of the specific ring to signal
  242. * @ring: ring to wait on for the seq number
  243. * @seq: seq number wait for
  244. *
  245. * return value:
  246. * 0: seq signaled, and gpu not hang
  247. * -EDEADL: GPU hang detected
  248. * -EINVAL: some paramter is not valid
  249. */
  250. static int amdgpu_fence_ring_wait_seq(struct amdgpu_ring *ring, uint64_t seq)
  251. {
  252. bool signaled = false;
  253. BUG_ON(!ring);
  254. if (seq > ring->fence_drv.sync_seq)
  255. return -EINVAL;
  256. if (atomic64_read(&ring->fence_drv.last_seq) >= seq)
  257. return 0;
  258. amdgpu_fence_schedule_fallback(ring);
  259. wait_event(ring->fence_drv.fence_queue, (
  260. (signaled = amdgpu_fence_seq_signaled(ring, seq))));
  261. if (signaled)
  262. return 0;
  263. else
  264. return -EDEADLK;
  265. }
  266. /**
  267. * amdgpu_fence_wait_next - wait for the next fence to signal
  268. *
  269. * @adev: amdgpu device pointer
  270. * @ring: ring index the fence is associated with
  271. *
  272. * Wait for the next fence on the requested ring to signal (all asics).
  273. * Returns 0 if the next fence has passed, error for all other cases.
  274. * Caller must hold ring lock.
  275. */
  276. int amdgpu_fence_wait_next(struct amdgpu_ring *ring)
  277. {
  278. uint64_t seq = atomic64_read(&ring->fence_drv.last_seq) + 1ULL;
  279. if (seq >= ring->fence_drv.sync_seq)
  280. return -ENOENT;
  281. return amdgpu_fence_ring_wait_seq(ring, seq);
  282. }
  283. /**
  284. * amdgpu_fence_wait_empty - wait for all fences to signal
  285. *
  286. * @adev: amdgpu device pointer
  287. * @ring: ring index the fence is associated with
  288. *
  289. * Wait for all fences on the requested ring to signal (all asics).
  290. * Returns 0 if the fences have passed, error for all other cases.
  291. * Caller must hold ring lock.
  292. */
  293. int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
  294. {
  295. uint64_t seq = ring->fence_drv.sync_seq;
  296. if (!seq)
  297. return 0;
  298. return amdgpu_fence_ring_wait_seq(ring, seq);
  299. }
  300. /**
  301. * amdgpu_fence_count_emitted - get the count of emitted fences
  302. *
  303. * @ring: ring the fence is associated with
  304. *
  305. * Get the number of fences emitted on the requested ring (all asics).
  306. * Returns the number of emitted fences on the ring. Used by the
  307. * dynpm code to ring track activity.
  308. */
  309. unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring)
  310. {
  311. uint64_t emitted;
  312. /* We are not protected by ring lock when reading the last sequence
  313. * but it's ok to report slightly wrong fence count here.
  314. */
  315. amdgpu_fence_process(ring);
  316. emitted = ring->fence_drv.sync_seq
  317. - atomic64_read(&ring->fence_drv.last_seq);
  318. /* to avoid 32bits warp around */
  319. if (emitted > 0x10000000)
  320. emitted = 0x10000000;
  321. return (unsigned)emitted;
  322. }
  323. /**
  324. * amdgpu_fence_driver_start_ring - make the fence driver
  325. * ready for use on the requested ring.
  326. *
  327. * @ring: ring to start the fence driver on
  328. * @irq_src: interrupt source to use for this ring
  329. * @irq_type: interrupt type to use for this ring
  330. *
  331. * Make the fence driver ready for processing (all asics).
  332. * Not all asics have all rings, so each asic will only
  333. * start the fence driver on the rings it has.
  334. * Returns 0 for success, errors for failure.
  335. */
  336. int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
  337. struct amdgpu_irq_src *irq_src,
  338. unsigned irq_type)
  339. {
  340. struct amdgpu_device *adev = ring->adev;
  341. uint64_t index;
  342. if (ring != &adev->uvd.ring) {
  343. ring->fence_drv.cpu_addr = &adev->wb.wb[ring->fence_offs];
  344. ring->fence_drv.gpu_addr = adev->wb.gpu_addr + (ring->fence_offs * 4);
  345. } else {
  346. /* put fence directly behind firmware */
  347. index = ALIGN(adev->uvd.fw->size, 8);
  348. ring->fence_drv.cpu_addr = adev->uvd.cpu_addr + index;
  349. ring->fence_drv.gpu_addr = adev->uvd.gpu_addr + index;
  350. }
  351. amdgpu_fence_write(ring, atomic64_read(&ring->fence_drv.last_seq));
  352. amdgpu_irq_get(adev, irq_src, irq_type);
  353. ring->fence_drv.irq_src = irq_src;
  354. ring->fence_drv.irq_type = irq_type;
  355. ring->fence_drv.initialized = true;
  356. dev_info(adev->dev, "fence driver on ring %d use gpu addr 0x%016llx, "
  357. "cpu addr 0x%p\n", ring->idx,
  358. ring->fence_drv.gpu_addr, ring->fence_drv.cpu_addr);
  359. return 0;
  360. }
  361. /**
  362. * amdgpu_fence_driver_init_ring - init the fence driver
  363. * for the requested ring.
  364. *
  365. * @ring: ring to init the fence driver on
  366. *
  367. * Init the fence driver for the requested ring (all asics).
  368. * Helper function for amdgpu_fence_driver_init().
  369. */
  370. int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring)
  371. {
  372. long timeout;
  373. int r;
  374. ring->fence_drv.cpu_addr = NULL;
  375. ring->fence_drv.gpu_addr = 0;
  376. ring->fence_drv.sync_seq = 0;
  377. atomic64_set(&ring->fence_drv.last_seq, 0);
  378. ring->fence_drv.initialized = false;
  379. setup_timer(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback,
  380. (unsigned long)ring);
  381. init_waitqueue_head(&ring->fence_drv.fence_queue);
  382. timeout = msecs_to_jiffies(amdgpu_lockup_timeout);
  383. if (timeout == 0) {
  384. /*
  385. * FIXME:
  386. * Delayed workqueue cannot use it directly,
  387. * so the scheduler will not use delayed workqueue if
  388. * MAX_SCHEDULE_TIMEOUT is set.
  389. * Currently keep it simple and silly.
  390. */
  391. timeout = MAX_SCHEDULE_TIMEOUT;
  392. }
  393. r = amd_sched_init(&ring->sched, &amdgpu_sched_ops,
  394. amdgpu_sched_hw_submission,
  395. timeout, ring->name);
  396. if (r) {
  397. DRM_ERROR("Failed to create scheduler on ring %s.\n",
  398. ring->name);
  399. return r;
  400. }
  401. return 0;
  402. }
  403. /**
  404. * amdgpu_fence_driver_init - init the fence driver
  405. * for all possible rings.
  406. *
  407. * @adev: amdgpu device pointer
  408. *
  409. * Init the fence driver for all possible rings (all asics).
  410. * Not all asics have all rings, so each asic will only
  411. * start the fence driver on the rings it has using
  412. * amdgpu_fence_driver_start_ring().
  413. * Returns 0 for success.
  414. */
  415. int amdgpu_fence_driver_init(struct amdgpu_device *adev)
  416. {
  417. if (atomic_inc_return(&amdgpu_fence_slab_ref) == 1) {
  418. amdgpu_fence_slab = kmem_cache_create(
  419. "amdgpu_fence", sizeof(struct amdgpu_fence), 0,
  420. SLAB_HWCACHE_ALIGN, NULL);
  421. if (!amdgpu_fence_slab)
  422. return -ENOMEM;
  423. }
  424. if (amdgpu_debugfs_fence_init(adev))
  425. dev_err(adev->dev, "fence debugfs file creation failed\n");
  426. return 0;
  427. }
  428. /**
  429. * amdgpu_fence_driver_fini - tear down the fence driver
  430. * for all possible rings.
  431. *
  432. * @adev: amdgpu device pointer
  433. *
  434. * Tear down the fence driver for all possible rings (all asics).
  435. */
  436. void amdgpu_fence_driver_fini(struct amdgpu_device *adev)
  437. {
  438. int i, r;
  439. if (atomic_dec_and_test(&amdgpu_fence_slab_ref))
  440. kmem_cache_destroy(amdgpu_fence_slab);
  441. mutex_lock(&adev->ring_lock);
  442. for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
  443. struct amdgpu_ring *ring = adev->rings[i];
  444. if (!ring || !ring->fence_drv.initialized)
  445. continue;
  446. r = amdgpu_fence_wait_empty(ring);
  447. if (r) {
  448. /* no need to trigger GPU reset as we are unloading */
  449. amdgpu_fence_driver_force_completion(adev);
  450. }
  451. wake_up_all(&ring->fence_drv.fence_queue);
  452. amdgpu_irq_put(adev, ring->fence_drv.irq_src,
  453. ring->fence_drv.irq_type);
  454. amd_sched_fini(&ring->sched);
  455. del_timer_sync(&ring->fence_drv.fallback_timer);
  456. ring->fence_drv.initialized = false;
  457. }
  458. mutex_unlock(&adev->ring_lock);
  459. }
  460. /**
  461. * amdgpu_fence_driver_suspend - suspend the fence driver
  462. * for all possible rings.
  463. *
  464. * @adev: amdgpu device pointer
  465. *
  466. * Suspend the fence driver for all possible rings (all asics).
  467. */
  468. void amdgpu_fence_driver_suspend(struct amdgpu_device *adev)
  469. {
  470. int i, r;
  471. mutex_lock(&adev->ring_lock);
  472. for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
  473. struct amdgpu_ring *ring = adev->rings[i];
  474. if (!ring || !ring->fence_drv.initialized)
  475. continue;
  476. /* wait for gpu to finish processing current batch */
  477. r = amdgpu_fence_wait_empty(ring);
  478. if (r) {
  479. /* delay GPU reset to resume */
  480. amdgpu_fence_driver_force_completion(adev);
  481. }
  482. /* disable the interrupt */
  483. amdgpu_irq_put(adev, ring->fence_drv.irq_src,
  484. ring->fence_drv.irq_type);
  485. }
  486. mutex_unlock(&adev->ring_lock);
  487. }
  488. /**
  489. * amdgpu_fence_driver_resume - resume the fence driver
  490. * for all possible rings.
  491. *
  492. * @adev: amdgpu device pointer
  493. *
  494. * Resume the fence driver for all possible rings (all asics).
  495. * Not all asics have all rings, so each asic will only
  496. * start the fence driver on the rings it has using
  497. * amdgpu_fence_driver_start_ring().
  498. * Returns 0 for success.
  499. */
  500. void amdgpu_fence_driver_resume(struct amdgpu_device *adev)
  501. {
  502. int i;
  503. mutex_lock(&adev->ring_lock);
  504. for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
  505. struct amdgpu_ring *ring = adev->rings[i];
  506. if (!ring || !ring->fence_drv.initialized)
  507. continue;
  508. /* enable the interrupt */
  509. amdgpu_irq_get(adev, ring->fence_drv.irq_src,
  510. ring->fence_drv.irq_type);
  511. }
  512. mutex_unlock(&adev->ring_lock);
  513. }
  514. /**
  515. * amdgpu_fence_driver_force_completion - force all fence waiter to complete
  516. *
  517. * @adev: amdgpu device pointer
  518. *
  519. * In case of GPU reset failure make sure no process keep waiting on fence
  520. * that will never complete.
  521. */
  522. void amdgpu_fence_driver_force_completion(struct amdgpu_device *adev)
  523. {
  524. int i;
  525. for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
  526. struct amdgpu_ring *ring = adev->rings[i];
  527. if (!ring || !ring->fence_drv.initialized)
  528. continue;
  529. amdgpu_fence_write(ring, ring->fence_drv.sync_seq);
  530. }
  531. }
  532. /*
  533. * Common fence implementation
  534. */
  535. static const char *amdgpu_fence_get_driver_name(struct fence *fence)
  536. {
  537. return "amdgpu";
  538. }
  539. static const char *amdgpu_fence_get_timeline_name(struct fence *f)
  540. {
  541. struct amdgpu_fence *fence = to_amdgpu_fence(f);
  542. return (const char *)fence->ring->name;
  543. }
  544. /**
  545. * amdgpu_fence_is_signaled - test if fence is signaled
  546. *
  547. * @f: fence to test
  548. *
  549. * Test the fence sequence number if it is already signaled. If it isn't
  550. * signaled start fence processing. Returns True if the fence is signaled.
  551. */
  552. static bool amdgpu_fence_is_signaled(struct fence *f)
  553. {
  554. struct amdgpu_fence *fence = to_amdgpu_fence(f);
  555. struct amdgpu_ring *ring = fence->ring;
  556. if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)
  557. return true;
  558. amdgpu_fence_process(ring);
  559. if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)
  560. return true;
  561. return false;
  562. }
  563. /**
  564. * amdgpu_fence_check_signaled - callback from fence_queue
  565. *
  566. * this function is called with fence_queue lock held, which is also used
  567. * for the fence locking itself, so unlocked variants are used for
  568. * fence_signal, and remove_wait_queue.
  569. */
  570. static int amdgpu_fence_check_signaled(wait_queue_t *wait, unsigned mode, int flags, void *key)
  571. {
  572. struct amdgpu_fence *fence;
  573. struct amdgpu_device *adev;
  574. u64 seq;
  575. int ret;
  576. fence = container_of(wait, struct amdgpu_fence, fence_wake);
  577. adev = fence->ring->adev;
  578. /*
  579. * We cannot use amdgpu_fence_process here because we're already
  580. * in the waitqueue, in a call from wake_up_all.
  581. */
  582. seq = atomic64_read(&fence->ring->fence_drv.last_seq);
  583. if (seq >= fence->seq) {
  584. ret = fence_signal_locked(&fence->base);
  585. if (!ret)
  586. FENCE_TRACE(&fence->base, "signaled from irq context\n");
  587. else
  588. FENCE_TRACE(&fence->base, "was already signaled\n");
  589. __remove_wait_queue(&fence->ring->fence_drv.fence_queue, &fence->fence_wake);
  590. fence_put(&fence->base);
  591. } else
  592. FENCE_TRACE(&fence->base, "pending\n");
  593. return 0;
  594. }
  595. /**
  596. * amdgpu_fence_enable_signaling - enable signalling on fence
  597. * @fence: fence
  598. *
  599. * This function is called with fence_queue lock held, and adds a callback
  600. * to fence_queue that checks if this fence is signaled, and if so it
  601. * signals the fence and removes itself.
  602. */
  603. static bool amdgpu_fence_enable_signaling(struct fence *f)
  604. {
  605. struct amdgpu_fence *fence = to_amdgpu_fence(f);
  606. struct amdgpu_ring *ring = fence->ring;
  607. if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)
  608. return false;
  609. fence->fence_wake.flags = 0;
  610. fence->fence_wake.private = NULL;
  611. fence->fence_wake.func = amdgpu_fence_check_signaled;
  612. __add_wait_queue(&ring->fence_drv.fence_queue, &fence->fence_wake);
  613. fence_get(f);
  614. if (!timer_pending(&ring->fence_drv.fallback_timer))
  615. amdgpu_fence_schedule_fallback(ring);
  616. FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx);
  617. return true;
  618. }
  619. static void amdgpu_fence_release(struct fence *f)
  620. {
  621. struct amdgpu_fence *fence = to_amdgpu_fence(f);
  622. kmem_cache_free(amdgpu_fence_slab, fence);
  623. }
  624. const struct fence_ops amdgpu_fence_ops = {
  625. .get_driver_name = amdgpu_fence_get_driver_name,
  626. .get_timeline_name = amdgpu_fence_get_timeline_name,
  627. .enable_signaling = amdgpu_fence_enable_signaling,
  628. .signaled = amdgpu_fence_is_signaled,
  629. .wait = fence_default_wait,
  630. .release = amdgpu_fence_release,
  631. };
  632. /*
  633. * Fence debugfs
  634. */
  635. #if defined(CONFIG_DEBUG_FS)
  636. static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data)
  637. {
  638. struct drm_info_node *node = (struct drm_info_node *)m->private;
  639. struct drm_device *dev = node->minor->dev;
  640. struct amdgpu_device *adev = dev->dev_private;
  641. int i;
  642. for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
  643. struct amdgpu_ring *ring = adev->rings[i];
  644. if (!ring || !ring->fence_drv.initialized)
  645. continue;
  646. amdgpu_fence_process(ring);
  647. seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name);
  648. seq_printf(m, "Last signaled fence 0x%016llx\n",
  649. (unsigned long long)atomic64_read(&ring->fence_drv.last_seq));
  650. seq_printf(m, "Last emitted 0x%016llx\n",
  651. ring->fence_drv.sync_seq);
  652. }
  653. return 0;
  654. }
  655. /**
  656. * amdgpu_debugfs_gpu_reset - manually trigger a gpu reset
  657. *
  658. * Manually trigger a gpu reset at the next fence wait.
  659. */
  660. static int amdgpu_debugfs_gpu_reset(struct seq_file *m, void *data)
  661. {
  662. struct drm_info_node *node = (struct drm_info_node *) m->private;
  663. struct drm_device *dev = node->minor->dev;
  664. struct amdgpu_device *adev = dev->dev_private;
  665. seq_printf(m, "gpu reset\n");
  666. amdgpu_gpu_reset(adev);
  667. return 0;
  668. }
  669. static struct drm_info_list amdgpu_debugfs_fence_list[] = {
  670. {"amdgpu_fence_info", &amdgpu_debugfs_fence_info, 0, NULL},
  671. {"amdgpu_gpu_reset", &amdgpu_debugfs_gpu_reset, 0, NULL}
  672. };
  673. #endif
  674. int amdgpu_debugfs_fence_init(struct amdgpu_device *adev)
  675. {
  676. #if defined(CONFIG_DEBUG_FS)
  677. return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_fence_list, 2);
  678. #else
  679. return 0;
  680. #endif
  681. }