amdgpu_cs.c 28 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157
  1. /*
  2. * Copyright 2008 Jerome Glisse.
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining a
  6. * copy of this software and associated documentation files (the "Software"),
  7. * to deal in the Software without restriction, including without limitation
  8. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  9. * and/or sell copies of the Software, and to permit persons to whom the
  10. * Software is furnished to do so, subject to the following conditions:
  11. *
  12. * The above copyright notice and this permission notice (including the next
  13. * paragraph) shall be included in all copies or substantial portions of the
  14. * Software.
  15. *
  16. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19. * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22. * DEALINGS IN THE SOFTWARE.
  23. *
  24. * Authors:
  25. * Jerome Glisse <glisse@freedesktop.org>
  26. */
  27. #include <linux/pagemap.h>
  28. #include <drm/drmP.h>
  29. #include <drm/amdgpu_drm.h>
  30. #include "amdgpu.h"
  31. #include "amdgpu_trace.h"
  32. int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type,
  33. u32 ip_instance, u32 ring,
  34. struct amdgpu_ring **out_ring)
  35. {
  36. /* Right now all IPs have only one instance - multiple rings. */
  37. if (ip_instance != 0) {
  38. DRM_ERROR("invalid ip instance: %d\n", ip_instance);
  39. return -EINVAL;
  40. }
  41. switch (ip_type) {
  42. default:
  43. DRM_ERROR("unknown ip type: %d\n", ip_type);
  44. return -EINVAL;
  45. case AMDGPU_HW_IP_GFX:
  46. if (ring < adev->gfx.num_gfx_rings) {
  47. *out_ring = &adev->gfx.gfx_ring[ring];
  48. } else {
  49. DRM_ERROR("only %d gfx rings are supported now\n",
  50. adev->gfx.num_gfx_rings);
  51. return -EINVAL;
  52. }
  53. break;
  54. case AMDGPU_HW_IP_COMPUTE:
  55. if (ring < adev->gfx.num_compute_rings) {
  56. *out_ring = &adev->gfx.compute_ring[ring];
  57. } else {
  58. DRM_ERROR("only %d compute rings are supported now\n",
  59. adev->gfx.num_compute_rings);
  60. return -EINVAL;
  61. }
  62. break;
  63. case AMDGPU_HW_IP_DMA:
  64. if (ring < adev->sdma.num_instances) {
  65. *out_ring = &adev->sdma.instance[ring].ring;
  66. } else {
  67. DRM_ERROR("only %d SDMA rings are supported\n",
  68. adev->sdma.num_instances);
  69. return -EINVAL;
  70. }
  71. break;
  72. case AMDGPU_HW_IP_UVD:
  73. *out_ring = &adev->uvd.ring;
  74. break;
  75. case AMDGPU_HW_IP_VCE:
  76. if (ring < 2){
  77. *out_ring = &adev->vce.ring[ring];
  78. } else {
  79. DRM_ERROR("only two VCE rings are supported\n");
  80. return -EINVAL;
  81. }
  82. break;
  83. }
  84. return 0;
  85. }
  86. static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
  87. struct drm_amdgpu_cs_chunk_fence *data,
  88. uint32_t *offset)
  89. {
  90. struct drm_gem_object *gobj;
  91. gobj = drm_gem_object_lookup(p->filp, data->handle);
  92. if (gobj == NULL)
  93. return -EINVAL;
  94. p->uf_entry.robj = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
  95. p->uf_entry.priority = 0;
  96. p->uf_entry.tv.bo = &p->uf_entry.robj->tbo;
  97. p->uf_entry.tv.shared = true;
  98. p->uf_entry.user_pages = NULL;
  99. *offset = data->offset;
  100. drm_gem_object_unreference_unlocked(gobj);
  101. if (amdgpu_ttm_tt_get_usermm(p->uf_entry.robj->tbo.ttm)) {
  102. amdgpu_bo_unref(&p->uf_entry.robj);
  103. return -EINVAL;
  104. }
  105. return 0;
  106. }
  107. int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
  108. {
  109. struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
  110. struct amdgpu_vm *vm = &fpriv->vm;
  111. union drm_amdgpu_cs *cs = data;
  112. uint64_t *chunk_array_user;
  113. uint64_t *chunk_array;
  114. unsigned size, num_ibs = 0;
  115. uint32_t uf_offset = 0;
  116. int i;
  117. int ret;
  118. if (cs->in.num_chunks == 0)
  119. return 0;
  120. chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL);
  121. if (!chunk_array)
  122. return -ENOMEM;
  123. p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id);
  124. if (!p->ctx) {
  125. ret = -EINVAL;
  126. goto free_chunk;
  127. }
  128. /* get chunks */
  129. chunk_array_user = (uint64_t __user *)(unsigned long)(cs->in.chunks);
  130. if (copy_from_user(chunk_array, chunk_array_user,
  131. sizeof(uint64_t)*cs->in.num_chunks)) {
  132. ret = -EFAULT;
  133. goto put_ctx;
  134. }
  135. p->nchunks = cs->in.num_chunks;
  136. p->chunks = kmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk),
  137. GFP_KERNEL);
  138. if (!p->chunks) {
  139. ret = -ENOMEM;
  140. goto put_ctx;
  141. }
  142. for (i = 0; i < p->nchunks; i++) {
  143. struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL;
  144. struct drm_amdgpu_cs_chunk user_chunk;
  145. uint32_t __user *cdata;
  146. chunk_ptr = (void __user *)(unsigned long)chunk_array[i];
  147. if (copy_from_user(&user_chunk, chunk_ptr,
  148. sizeof(struct drm_amdgpu_cs_chunk))) {
  149. ret = -EFAULT;
  150. i--;
  151. goto free_partial_kdata;
  152. }
  153. p->chunks[i].chunk_id = user_chunk.chunk_id;
  154. p->chunks[i].length_dw = user_chunk.length_dw;
  155. size = p->chunks[i].length_dw;
  156. cdata = (void __user *)(unsigned long)user_chunk.chunk_data;
  157. p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t));
  158. if (p->chunks[i].kdata == NULL) {
  159. ret = -ENOMEM;
  160. i--;
  161. goto free_partial_kdata;
  162. }
  163. size *= sizeof(uint32_t);
  164. if (copy_from_user(p->chunks[i].kdata, cdata, size)) {
  165. ret = -EFAULT;
  166. goto free_partial_kdata;
  167. }
  168. switch (p->chunks[i].chunk_id) {
  169. case AMDGPU_CHUNK_ID_IB:
  170. ++num_ibs;
  171. break;
  172. case AMDGPU_CHUNK_ID_FENCE:
  173. size = sizeof(struct drm_amdgpu_cs_chunk_fence);
  174. if (p->chunks[i].length_dw * sizeof(uint32_t) < size) {
  175. ret = -EINVAL;
  176. goto free_partial_kdata;
  177. }
  178. ret = amdgpu_cs_user_fence_chunk(p, p->chunks[i].kdata,
  179. &uf_offset);
  180. if (ret)
  181. goto free_partial_kdata;
  182. break;
  183. case AMDGPU_CHUNK_ID_DEPENDENCIES:
  184. break;
  185. default:
  186. ret = -EINVAL;
  187. goto free_partial_kdata;
  188. }
  189. }
  190. ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job, vm);
  191. if (ret)
  192. goto free_all_kdata;
  193. if (p->uf_entry.robj)
  194. p->job->uf_addr = uf_offset;
  195. kfree(chunk_array);
  196. return 0;
  197. free_all_kdata:
  198. i = p->nchunks - 1;
  199. free_partial_kdata:
  200. for (; i >= 0; i--)
  201. drm_free_large(p->chunks[i].kdata);
  202. kfree(p->chunks);
  203. put_ctx:
  204. amdgpu_ctx_put(p->ctx);
  205. free_chunk:
  206. kfree(chunk_array);
  207. return ret;
  208. }
  209. /* Convert microseconds to bytes. */
  210. static u64 us_to_bytes(struct amdgpu_device *adev, s64 us)
  211. {
  212. if (us <= 0 || !adev->mm_stats.log2_max_MBps)
  213. return 0;
  214. /* Since accum_us is incremented by a million per second, just
  215. * multiply it by the number of MB/s to get the number of bytes.
  216. */
  217. return us << adev->mm_stats.log2_max_MBps;
  218. }
  219. static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes)
  220. {
  221. if (!adev->mm_stats.log2_max_MBps)
  222. return 0;
  223. return bytes >> adev->mm_stats.log2_max_MBps;
  224. }
  225. /* Returns how many bytes TTM can move right now. If no bytes can be moved,
  226. * it returns 0. If it returns non-zero, it's OK to move at least one buffer,
  227. * which means it can go over the threshold once. If that happens, the driver
  228. * will be in debt and no other buffer migrations can be done until that debt
  229. * is repaid.
  230. *
  231. * This approach allows moving a buffer of any size (it's important to allow
  232. * that).
  233. *
  234. * The currency is simply time in microseconds and it increases as the clock
  235. * ticks. The accumulated microseconds (us) are converted to bytes and
  236. * returned.
  237. */
  238. static u64 amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev)
  239. {
  240. s64 time_us, increment_us;
  241. u64 max_bytes;
  242. u64 free_vram, total_vram, used_vram;
  243. /* Allow a maximum of 200 accumulated ms. This is basically per-IB
  244. * throttling.
  245. *
  246. * It means that in order to get full max MBps, at least 5 IBs per
  247. * second must be submitted and not more than 200ms apart from each
  248. * other.
  249. */
  250. const s64 us_upper_bound = 200000;
  251. if (!adev->mm_stats.log2_max_MBps)
  252. return 0;
  253. total_vram = adev->mc.real_vram_size - adev->vram_pin_size;
  254. used_vram = atomic64_read(&adev->vram_usage);
  255. free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram;
  256. spin_lock(&adev->mm_stats.lock);
  257. /* Increase the amount of accumulated us. */
  258. time_us = ktime_to_us(ktime_get());
  259. increment_us = time_us - adev->mm_stats.last_update_us;
  260. adev->mm_stats.last_update_us = time_us;
  261. adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us,
  262. us_upper_bound);
  263. /* This prevents the short period of low performance when the VRAM
  264. * usage is low and the driver is in debt or doesn't have enough
  265. * accumulated us to fill VRAM quickly.
  266. *
  267. * The situation can occur in these cases:
  268. * - a lot of VRAM is freed by userspace
  269. * - the presence of a big buffer causes a lot of evictions
  270. * (solution: split buffers into smaller ones)
  271. *
  272. * If 128 MB or 1/8th of VRAM is free, start filling it now by setting
  273. * accum_us to a positive number.
  274. */
  275. if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) {
  276. s64 min_us;
  277. /* Be more aggresive on dGPUs. Try to fill a portion of free
  278. * VRAM now.
  279. */
  280. if (!(adev->flags & AMD_IS_APU))
  281. min_us = bytes_to_us(adev, free_vram / 4);
  282. else
  283. min_us = 0; /* Reset accum_us on APUs. */
  284. adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us);
  285. }
  286. /* This returns 0 if the driver is in debt to disallow (optional)
  287. * buffer moves.
  288. */
  289. max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us);
  290. spin_unlock(&adev->mm_stats.lock);
  291. return max_bytes;
  292. }
  293. /* Report how many bytes have really been moved for the last command
  294. * submission. This can result in a debt that can stop buffer migrations
  295. * temporarily.
  296. */
  297. static void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev,
  298. u64 num_bytes)
  299. {
  300. spin_lock(&adev->mm_stats.lock);
  301. adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes);
  302. spin_unlock(&adev->mm_stats.lock);
  303. }
  304. static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p,
  305. struct amdgpu_bo *bo)
  306. {
  307. u64 initial_bytes_moved;
  308. uint32_t domain;
  309. int r;
  310. if (bo->pin_count)
  311. return 0;
  312. /* Don't move this buffer if we have depleted our allowance
  313. * to move it. Don't move anything if the threshold is zero.
  314. */
  315. if (p->bytes_moved < p->bytes_moved_threshold)
  316. domain = bo->prefered_domains;
  317. else
  318. domain = bo->allowed_domains;
  319. retry:
  320. amdgpu_ttm_placement_from_domain(bo, domain);
  321. initial_bytes_moved = atomic64_read(&bo->adev->num_bytes_moved);
  322. r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
  323. p->bytes_moved += atomic64_read(&bo->adev->num_bytes_moved) -
  324. initial_bytes_moved;
  325. if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
  326. domain = bo->allowed_domains;
  327. goto retry;
  328. }
  329. return r;
  330. }
  331. /* Last resort, try to evict something from the current working set */
  332. static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
  333. struct amdgpu_bo_list_entry *lobj)
  334. {
  335. uint32_t domain = lobj->robj->allowed_domains;
  336. int r;
  337. if (!p->evictable)
  338. return false;
  339. for (;&p->evictable->tv.head != &p->validated;
  340. p->evictable = list_prev_entry(p->evictable, tv.head)) {
  341. struct amdgpu_bo_list_entry *candidate = p->evictable;
  342. struct amdgpu_bo *bo = candidate->robj;
  343. u64 initial_bytes_moved;
  344. uint32_t other;
  345. /* If we reached our current BO we can forget it */
  346. if (candidate == lobj)
  347. break;
  348. other = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
  349. /* Check if this BO is in one of the domains we need space for */
  350. if (!(other & domain))
  351. continue;
  352. /* Check if we can move this BO somewhere else */
  353. other = bo->allowed_domains & ~domain;
  354. if (!other)
  355. continue;
  356. /* Good we can try to move this BO somewhere else */
  357. amdgpu_ttm_placement_from_domain(bo, other);
  358. initial_bytes_moved = atomic64_read(&bo->adev->num_bytes_moved);
  359. r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
  360. p->bytes_moved += atomic64_read(&bo->adev->num_bytes_moved) -
  361. initial_bytes_moved;
  362. if (unlikely(r))
  363. break;
  364. p->evictable = list_prev_entry(p->evictable, tv.head);
  365. list_move(&candidate->tv.head, &p->validated);
  366. return true;
  367. }
  368. return false;
  369. }
  370. int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
  371. struct list_head *validated)
  372. {
  373. struct amdgpu_bo_list_entry *lobj;
  374. int r;
  375. list_for_each_entry(lobj, validated, tv.head) {
  376. struct amdgpu_bo *bo = lobj->robj;
  377. bool binding_userptr = false;
  378. struct mm_struct *usermm;
  379. usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm);
  380. if (usermm && usermm != current->mm)
  381. return -EPERM;
  382. /* Check if we have user pages and nobody bound the BO already */
  383. if (lobj->user_pages && bo->tbo.ttm->state != tt_bound) {
  384. size_t size = sizeof(struct page *);
  385. size *= bo->tbo.ttm->num_pages;
  386. memcpy(bo->tbo.ttm->pages, lobj->user_pages, size);
  387. binding_userptr = true;
  388. }
  389. if (p->evictable == lobj)
  390. p->evictable = NULL;
  391. do {
  392. r = amdgpu_cs_bo_validate(p, bo);
  393. } while (r == -ENOMEM && amdgpu_cs_try_evict(p, lobj));
  394. if (r)
  395. return r;
  396. if (bo->shadow) {
  397. r = amdgpu_cs_bo_validate(p, bo);
  398. if (r)
  399. return r;
  400. }
  401. if (binding_userptr) {
  402. drm_free_large(lobj->user_pages);
  403. lobj->user_pages = NULL;
  404. }
  405. }
  406. return 0;
  407. }
  408. static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
  409. union drm_amdgpu_cs *cs)
  410. {
  411. struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
  412. struct amdgpu_bo_list_entry *e;
  413. struct list_head duplicates;
  414. bool need_mmap_lock = false;
  415. unsigned i, tries = 10;
  416. int r;
  417. INIT_LIST_HEAD(&p->validated);
  418. p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);
  419. if (p->bo_list) {
  420. need_mmap_lock = p->bo_list->first_userptr !=
  421. p->bo_list->num_entries;
  422. amdgpu_bo_list_get_list(p->bo_list, &p->validated);
  423. }
  424. INIT_LIST_HEAD(&duplicates);
  425. amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd);
  426. if (p->uf_entry.robj)
  427. list_add(&p->uf_entry.tv.head, &p->validated);
  428. if (need_mmap_lock)
  429. down_read(&current->mm->mmap_sem);
  430. while (1) {
  431. struct list_head need_pages;
  432. unsigned i;
  433. r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true,
  434. &duplicates);
  435. if (unlikely(r != 0)) {
  436. DRM_ERROR("ttm_eu_reserve_buffers failed.\n");
  437. goto error_free_pages;
  438. }
  439. /* Without a BO list we don't have userptr BOs */
  440. if (!p->bo_list)
  441. break;
  442. INIT_LIST_HEAD(&need_pages);
  443. for (i = p->bo_list->first_userptr;
  444. i < p->bo_list->num_entries; ++i) {
  445. e = &p->bo_list->array[i];
  446. if (amdgpu_ttm_tt_userptr_invalidated(e->robj->tbo.ttm,
  447. &e->user_invalidated) && e->user_pages) {
  448. /* We acquired a page array, but somebody
  449. * invalidated it. Free it an try again
  450. */
  451. release_pages(e->user_pages,
  452. e->robj->tbo.ttm->num_pages,
  453. false);
  454. drm_free_large(e->user_pages);
  455. e->user_pages = NULL;
  456. }
  457. if (e->robj->tbo.ttm->state != tt_bound &&
  458. !e->user_pages) {
  459. list_del(&e->tv.head);
  460. list_add(&e->tv.head, &need_pages);
  461. amdgpu_bo_unreserve(e->robj);
  462. }
  463. }
  464. if (list_empty(&need_pages))
  465. break;
  466. /* Unreserve everything again. */
  467. ttm_eu_backoff_reservation(&p->ticket, &p->validated);
  468. /* We tried too many times, just abort */
  469. if (!--tries) {
  470. r = -EDEADLK;
  471. DRM_ERROR("deadlock in %s\n", __func__);
  472. goto error_free_pages;
  473. }
  474. /* Fill the page arrays for all useptrs. */
  475. list_for_each_entry(e, &need_pages, tv.head) {
  476. struct ttm_tt *ttm = e->robj->tbo.ttm;
  477. e->user_pages = drm_calloc_large(ttm->num_pages,
  478. sizeof(struct page*));
  479. if (!e->user_pages) {
  480. r = -ENOMEM;
  481. DRM_ERROR("calloc failure in %s\n", __func__);
  482. goto error_free_pages;
  483. }
  484. r = amdgpu_ttm_tt_get_user_pages(ttm, e->user_pages);
  485. if (r) {
  486. DRM_ERROR("amdgpu_ttm_tt_get_user_pages failed.\n");
  487. drm_free_large(e->user_pages);
  488. e->user_pages = NULL;
  489. goto error_free_pages;
  490. }
  491. }
  492. /* And try again. */
  493. list_splice(&need_pages, &p->validated);
  494. }
  495. amdgpu_vm_get_pt_bos(p->adev, &fpriv->vm, &duplicates);
  496. p->bytes_moved_threshold = amdgpu_cs_get_threshold_for_moves(p->adev);
  497. p->bytes_moved = 0;
  498. p->evictable = list_last_entry(&p->validated,
  499. struct amdgpu_bo_list_entry,
  500. tv.head);
  501. r = amdgpu_cs_list_validate(p, &duplicates);
  502. if (r) {
  503. DRM_ERROR("amdgpu_cs_list_validate(duplicates) failed.\n");
  504. goto error_validate;
  505. }
  506. r = amdgpu_cs_list_validate(p, &p->validated);
  507. if (r) {
  508. DRM_ERROR("amdgpu_cs_list_validate(validated) failed.\n");
  509. goto error_validate;
  510. }
  511. amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved);
  512. fpriv->vm.last_eviction_counter =
  513. atomic64_read(&p->adev->num_evictions);
  514. if (p->bo_list) {
  515. struct amdgpu_bo *gds = p->bo_list->gds_obj;
  516. struct amdgpu_bo *gws = p->bo_list->gws_obj;
  517. struct amdgpu_bo *oa = p->bo_list->oa_obj;
  518. struct amdgpu_vm *vm = &fpriv->vm;
  519. unsigned i;
  520. for (i = 0; i < p->bo_list->num_entries; i++) {
  521. struct amdgpu_bo *bo = p->bo_list->array[i].robj;
  522. p->bo_list->array[i].bo_va = amdgpu_vm_bo_find(vm, bo);
  523. }
  524. if (gds) {
  525. p->job->gds_base = amdgpu_bo_gpu_offset(gds);
  526. p->job->gds_size = amdgpu_bo_size(gds);
  527. }
  528. if (gws) {
  529. p->job->gws_base = amdgpu_bo_gpu_offset(gws);
  530. p->job->gws_size = amdgpu_bo_size(gws);
  531. }
  532. if (oa) {
  533. p->job->oa_base = amdgpu_bo_gpu_offset(oa);
  534. p->job->oa_size = amdgpu_bo_size(oa);
  535. }
  536. }
  537. if (p->uf_entry.robj)
  538. p->job->uf_addr += amdgpu_bo_gpu_offset(p->uf_entry.robj);
  539. error_validate:
  540. if (r) {
  541. amdgpu_vm_move_pt_bos_in_lru(p->adev, &fpriv->vm);
  542. ttm_eu_backoff_reservation(&p->ticket, &p->validated);
  543. }
  544. error_free_pages:
  545. if (need_mmap_lock)
  546. up_read(&current->mm->mmap_sem);
  547. if (p->bo_list) {
  548. for (i = p->bo_list->first_userptr;
  549. i < p->bo_list->num_entries; ++i) {
  550. e = &p->bo_list->array[i];
  551. if (!e->user_pages)
  552. continue;
  553. release_pages(e->user_pages,
  554. e->robj->tbo.ttm->num_pages,
  555. false);
  556. drm_free_large(e->user_pages);
  557. }
  558. }
  559. return r;
  560. }
  561. static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
  562. {
  563. struct amdgpu_bo_list_entry *e;
  564. int r;
  565. list_for_each_entry(e, &p->validated, tv.head) {
  566. struct reservation_object *resv = e->robj->tbo.resv;
  567. r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, p->filp);
  568. if (r)
  569. return r;
  570. }
  571. return 0;
  572. }
  573. /**
  574. * cs_parser_fini() - clean parser states
  575. * @parser: parser structure holding parsing context.
  576. * @error: error number
  577. *
  578. * If error is set than unvalidate buffer, otherwise just free memory
  579. * used by parsing context.
  580. **/
  581. static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bool backoff)
  582. {
  583. struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
  584. unsigned i;
  585. if (!error) {
  586. amdgpu_vm_move_pt_bos_in_lru(parser->adev, &fpriv->vm);
  587. ttm_eu_fence_buffer_objects(&parser->ticket,
  588. &parser->validated,
  589. parser->fence);
  590. } else if (backoff) {
  591. ttm_eu_backoff_reservation(&parser->ticket,
  592. &parser->validated);
  593. }
  594. fence_put(parser->fence);
  595. if (parser->ctx)
  596. amdgpu_ctx_put(parser->ctx);
  597. if (parser->bo_list)
  598. amdgpu_bo_list_put(parser->bo_list);
  599. for (i = 0; i < parser->nchunks; i++)
  600. drm_free_large(parser->chunks[i].kdata);
  601. kfree(parser->chunks);
  602. if (parser->job)
  603. amdgpu_job_free(parser->job);
  604. amdgpu_bo_unref(&parser->uf_entry.robj);
  605. }
  606. static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p,
  607. struct amdgpu_vm *vm)
  608. {
  609. struct amdgpu_device *adev = p->adev;
  610. struct amdgpu_bo_va *bo_va;
  611. struct amdgpu_bo *bo;
  612. int i, r;
  613. r = amdgpu_vm_update_page_directory(adev, vm);
  614. if (r)
  615. return r;
  616. r = amdgpu_sync_fence(adev, &p->job->sync, vm->page_directory_fence);
  617. if (r)
  618. return r;
  619. r = amdgpu_vm_clear_freed(adev, vm);
  620. if (r)
  621. return r;
  622. if (p->bo_list) {
  623. for (i = 0; i < p->bo_list->num_entries; i++) {
  624. struct fence *f;
  625. /* ignore duplicates */
  626. bo = p->bo_list->array[i].robj;
  627. if (!bo)
  628. continue;
  629. bo_va = p->bo_list->array[i].bo_va;
  630. if (bo_va == NULL)
  631. continue;
  632. r = amdgpu_vm_bo_update(adev, bo_va, false);
  633. if (r)
  634. return r;
  635. f = bo_va->last_pt_update;
  636. r = amdgpu_sync_fence(adev, &p->job->sync, f);
  637. if (r)
  638. return r;
  639. }
  640. }
  641. r = amdgpu_vm_clear_invalids(adev, vm, &p->job->sync);
  642. if (amdgpu_vm_debug && p->bo_list) {
  643. /* Invalidate all BOs to test for userspace bugs */
  644. for (i = 0; i < p->bo_list->num_entries; i++) {
  645. /* ignore duplicates */
  646. bo = p->bo_list->array[i].robj;
  647. if (!bo)
  648. continue;
  649. amdgpu_vm_bo_invalidate(adev, bo);
  650. }
  651. }
  652. return r;
  653. }
  654. static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
  655. struct amdgpu_cs_parser *p)
  656. {
  657. struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
  658. struct amdgpu_vm *vm = &fpriv->vm;
  659. struct amdgpu_ring *ring = p->job->ring;
  660. int i, r;
  661. /* Only for UVD/VCE VM emulation */
  662. if (ring->funcs->parse_cs) {
  663. p->job->vm = NULL;
  664. for (i = 0; i < p->job->num_ibs; i++) {
  665. r = amdgpu_ring_parse_cs(ring, p, i);
  666. if (r)
  667. return r;
  668. }
  669. } else {
  670. p->job->vm_pd_addr = amdgpu_bo_gpu_offset(vm->page_directory);
  671. r = amdgpu_bo_vm_update_pte(p, vm);
  672. if (r)
  673. return r;
  674. }
  675. return amdgpu_cs_sync_rings(p);
  676. }
  677. static int amdgpu_cs_handle_lockup(struct amdgpu_device *adev, int r)
  678. {
  679. if (r == -EDEADLK) {
  680. r = amdgpu_gpu_reset(adev);
  681. if (!r)
  682. r = -EAGAIN;
  683. }
  684. return r;
  685. }
  686. static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
  687. struct amdgpu_cs_parser *parser)
  688. {
  689. struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
  690. struct amdgpu_vm *vm = &fpriv->vm;
  691. int i, j;
  692. int r;
  693. for (i = 0, j = 0; i < parser->nchunks && j < parser->job->num_ibs; i++) {
  694. struct amdgpu_cs_chunk *chunk;
  695. struct amdgpu_ib *ib;
  696. struct drm_amdgpu_cs_chunk_ib *chunk_ib;
  697. struct amdgpu_ring *ring;
  698. chunk = &parser->chunks[i];
  699. ib = &parser->job->ibs[j];
  700. chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;
  701. if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
  702. continue;
  703. r = amdgpu_cs_get_ring(adev, chunk_ib->ip_type,
  704. chunk_ib->ip_instance, chunk_ib->ring,
  705. &ring);
  706. if (r)
  707. return r;
  708. if (parser->job->ring && parser->job->ring != ring)
  709. return -EINVAL;
  710. parser->job->ring = ring;
  711. if (ring->funcs->parse_cs) {
  712. struct amdgpu_bo_va_mapping *m;
  713. struct amdgpu_bo *aobj = NULL;
  714. uint64_t offset;
  715. uint8_t *kptr;
  716. m = amdgpu_cs_find_mapping(parser, chunk_ib->va_start,
  717. &aobj);
  718. if (!aobj) {
  719. DRM_ERROR("IB va_start is invalid\n");
  720. return -EINVAL;
  721. }
  722. if ((chunk_ib->va_start + chunk_ib->ib_bytes) >
  723. (m->it.last + 1) * AMDGPU_GPU_PAGE_SIZE) {
  724. DRM_ERROR("IB va_start+ib_bytes is invalid\n");
  725. return -EINVAL;
  726. }
  727. /* the IB should be reserved at this point */
  728. r = amdgpu_bo_kmap(aobj, (void **)&kptr);
  729. if (r) {
  730. return r;
  731. }
  732. offset = ((uint64_t)m->it.start) * AMDGPU_GPU_PAGE_SIZE;
  733. kptr += chunk_ib->va_start - offset;
  734. r = amdgpu_ib_get(adev, NULL, chunk_ib->ib_bytes, ib);
  735. if (r) {
  736. DRM_ERROR("Failed to get ib !\n");
  737. return r;
  738. }
  739. memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
  740. amdgpu_bo_kunmap(aobj);
  741. } else {
  742. r = amdgpu_ib_get(adev, vm, 0, ib);
  743. if (r) {
  744. DRM_ERROR("Failed to get ib !\n");
  745. return r;
  746. }
  747. ib->gpu_addr = chunk_ib->va_start;
  748. }
  749. ib->length_dw = chunk_ib->ib_bytes / 4;
  750. ib->flags = chunk_ib->flags;
  751. j++;
  752. }
  753. /* UVD & VCE fw doesn't support user fences */
  754. if (parser->job->uf_addr && (
  755. parser->job->ring->type == AMDGPU_RING_TYPE_UVD ||
  756. parser->job->ring->type == AMDGPU_RING_TYPE_VCE))
  757. return -EINVAL;
  758. return 0;
  759. }
  760. static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
  761. struct amdgpu_cs_parser *p)
  762. {
  763. struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
  764. int i, j, r;
  765. for (i = 0; i < p->nchunks; ++i) {
  766. struct drm_amdgpu_cs_chunk_dep *deps;
  767. struct amdgpu_cs_chunk *chunk;
  768. unsigned num_deps;
  769. chunk = &p->chunks[i];
  770. if (chunk->chunk_id != AMDGPU_CHUNK_ID_DEPENDENCIES)
  771. continue;
  772. deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata;
  773. num_deps = chunk->length_dw * 4 /
  774. sizeof(struct drm_amdgpu_cs_chunk_dep);
  775. for (j = 0; j < num_deps; ++j) {
  776. struct amdgpu_ring *ring;
  777. struct amdgpu_ctx *ctx;
  778. struct fence *fence;
  779. r = amdgpu_cs_get_ring(adev, deps[j].ip_type,
  780. deps[j].ip_instance,
  781. deps[j].ring, &ring);
  782. if (r)
  783. return r;
  784. ctx = amdgpu_ctx_get(fpriv, deps[j].ctx_id);
  785. if (ctx == NULL)
  786. return -EINVAL;
  787. fence = amdgpu_ctx_get_fence(ctx, ring,
  788. deps[j].handle);
  789. if (IS_ERR(fence)) {
  790. r = PTR_ERR(fence);
  791. amdgpu_ctx_put(ctx);
  792. return r;
  793. } else if (fence) {
  794. r = amdgpu_sync_fence(adev, &p->job->sync,
  795. fence);
  796. fence_put(fence);
  797. amdgpu_ctx_put(ctx);
  798. if (r)
  799. return r;
  800. }
  801. }
  802. }
  803. return 0;
  804. }
  805. static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
  806. union drm_amdgpu_cs *cs)
  807. {
  808. struct amdgpu_ring *ring = p->job->ring;
  809. struct amd_sched_entity *entity = &p->ctx->rings[ring->idx].entity;
  810. struct amdgpu_job *job;
  811. int r;
  812. job = p->job;
  813. p->job = NULL;
  814. r = amd_sched_job_init(&job->base, &ring->sched, entity, p->filp);
  815. if (r) {
  816. amdgpu_job_free(job);
  817. return r;
  818. }
  819. job->owner = p->filp;
  820. job->fence_ctx = entity->fence_context;
  821. p->fence = fence_get(&job->base.s_fence->finished);
  822. cs->out.handle = amdgpu_ctx_add_fence(p->ctx, ring, p->fence);
  823. job->uf_sequence = cs->out.handle;
  824. amdgpu_job_free_resources(job);
  825. trace_amdgpu_cs_ioctl(job);
  826. amd_sched_entity_push_job(&job->base);
  827. return 0;
  828. }
  829. int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
  830. {
  831. struct amdgpu_device *adev = dev->dev_private;
  832. union drm_amdgpu_cs *cs = data;
  833. struct amdgpu_cs_parser parser = {};
  834. bool reserved_buffers = false;
  835. int i, r;
  836. if (!adev->accel_working)
  837. return -EBUSY;
  838. parser.adev = adev;
  839. parser.filp = filp;
  840. r = amdgpu_cs_parser_init(&parser, data);
  841. if (r) {
  842. DRM_ERROR("Failed to initialize parser !\n");
  843. amdgpu_cs_parser_fini(&parser, r, false);
  844. r = amdgpu_cs_handle_lockup(adev, r);
  845. return r;
  846. }
  847. r = amdgpu_cs_parser_bos(&parser, data);
  848. if (r == -ENOMEM)
  849. DRM_ERROR("Not enough memory for command submission!\n");
  850. else if (r && r != -ERESTARTSYS)
  851. DRM_ERROR("Failed to process the buffer list %d!\n", r);
  852. else if (!r) {
  853. reserved_buffers = true;
  854. r = amdgpu_cs_ib_fill(adev, &parser);
  855. }
  856. if (!r) {
  857. r = amdgpu_cs_dependencies(adev, &parser);
  858. if (r)
  859. DRM_ERROR("Failed in the dependencies handling %d!\n", r);
  860. }
  861. if (r)
  862. goto out;
  863. for (i = 0; i < parser.job->num_ibs; i++)
  864. trace_amdgpu_cs(&parser, i);
  865. r = amdgpu_cs_ib_vm_chunk(adev, &parser);
  866. if (r)
  867. goto out;
  868. r = amdgpu_cs_submit(&parser, cs);
  869. out:
  870. amdgpu_cs_parser_fini(&parser, r, reserved_buffers);
  871. r = amdgpu_cs_handle_lockup(adev, r);
  872. return r;
  873. }
  874. /**
  875. * amdgpu_cs_wait_ioctl - wait for a command submission to finish
  876. *
  877. * @dev: drm device
  878. * @data: data from userspace
  879. * @filp: file private
  880. *
  881. * Wait for the command submission identified by handle to finish.
  882. */
  883. int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
  884. struct drm_file *filp)
  885. {
  886. union drm_amdgpu_wait_cs *wait = data;
  887. struct amdgpu_device *adev = dev->dev_private;
  888. unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);
  889. struct amdgpu_ring *ring = NULL;
  890. struct amdgpu_ctx *ctx;
  891. struct fence *fence;
  892. long r;
  893. r = amdgpu_cs_get_ring(adev, wait->in.ip_type, wait->in.ip_instance,
  894. wait->in.ring, &ring);
  895. if (r)
  896. return r;
  897. ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);
  898. if (ctx == NULL)
  899. return -EINVAL;
  900. fence = amdgpu_ctx_get_fence(ctx, ring, wait->in.handle);
  901. if (IS_ERR(fence))
  902. r = PTR_ERR(fence);
  903. else if (fence) {
  904. r = fence_wait_timeout(fence, true, timeout);
  905. fence_put(fence);
  906. } else
  907. r = 1;
  908. amdgpu_ctx_put(ctx);
  909. if (r < 0)
  910. return r;
  911. memset(wait, 0, sizeof(*wait));
  912. wait->out.status = (r == 0);
  913. return 0;
  914. }
  915. /**
  916. * amdgpu_cs_find_bo_va - find bo_va for VM address
  917. *
  918. * @parser: command submission parser context
  919. * @addr: VM address
  920. * @bo: resulting BO of the mapping found
  921. *
  922. * Search the buffer objects in the command submission context for a certain
  923. * virtual memory address. Returns allocation structure when found, NULL
  924. * otherwise.
  925. */
  926. struct amdgpu_bo_va_mapping *
  927. amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
  928. uint64_t addr, struct amdgpu_bo **bo)
  929. {
  930. struct amdgpu_bo_va_mapping *mapping;
  931. unsigned i;
  932. if (!parser->bo_list)
  933. return NULL;
  934. addr /= AMDGPU_GPU_PAGE_SIZE;
  935. for (i = 0; i < parser->bo_list->num_entries; i++) {
  936. struct amdgpu_bo_list_entry *lobj;
  937. lobj = &parser->bo_list->array[i];
  938. if (!lobj->bo_va)
  939. continue;
  940. list_for_each_entry(mapping, &lobj->bo_va->valids, list) {
  941. if (mapping->it.start > addr ||
  942. addr > mapping->it.last)
  943. continue;
  944. *bo = lobj->bo_va->bo;
  945. return mapping;
  946. }
  947. list_for_each_entry(mapping, &lobj->bo_va->invalids, list) {
  948. if (mapping->it.start > addr ||
  949. addr > mapping->it.last)
  950. continue;
  951. *bo = lobj->bo_va->bo;
  952. return mapping;
  953. }
  954. }
  955. return NULL;
  956. }