amdgpu_cs.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940
  1. /*
  2. * Copyright 2008 Jerome Glisse.
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining a
  6. * copy of this software and associated documentation files (the "Software"),
  7. * to deal in the Software without restriction, including without limitation
  8. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  9. * and/or sell copies of the Software, and to permit persons to whom the
  10. * Software is furnished to do so, subject to the following conditions:
  11. *
  12. * The above copyright notice and this permission notice (including the next
  13. * paragraph) shall be included in all copies or substantial portions of the
  14. * Software.
  15. *
  16. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19. * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22. * DEALINGS IN THE SOFTWARE.
  23. *
  24. * Authors:
  25. * Jerome Glisse <glisse@freedesktop.org>
  26. */
  27. #include <linux/list_sort.h>
  28. #include <drm/drmP.h>
  29. #include <drm/amdgpu_drm.h>
  30. #include "amdgpu.h"
  31. #include "amdgpu_trace.h"
  32. int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type,
  33. u32 ip_instance, u32 ring,
  34. struct amdgpu_ring **out_ring)
  35. {
  36. /* Right now all IPs have only one instance - multiple rings. */
  37. if (ip_instance != 0) {
  38. DRM_ERROR("invalid ip instance: %d\n", ip_instance);
  39. return -EINVAL;
  40. }
  41. switch (ip_type) {
  42. default:
  43. DRM_ERROR("unknown ip type: %d\n", ip_type);
  44. return -EINVAL;
  45. case AMDGPU_HW_IP_GFX:
  46. if (ring < adev->gfx.num_gfx_rings) {
  47. *out_ring = &adev->gfx.gfx_ring[ring];
  48. } else {
  49. DRM_ERROR("only %d gfx rings are supported now\n",
  50. adev->gfx.num_gfx_rings);
  51. return -EINVAL;
  52. }
  53. break;
  54. case AMDGPU_HW_IP_COMPUTE:
  55. if (ring < adev->gfx.num_compute_rings) {
  56. *out_ring = &adev->gfx.compute_ring[ring];
  57. } else {
  58. DRM_ERROR("only %d compute rings are supported now\n",
  59. adev->gfx.num_compute_rings);
  60. return -EINVAL;
  61. }
  62. break;
  63. case AMDGPU_HW_IP_DMA:
  64. if (ring < adev->sdma.num_instances) {
  65. *out_ring = &adev->sdma.instance[ring].ring;
  66. } else {
  67. DRM_ERROR("only %d SDMA rings are supported\n",
  68. adev->sdma.num_instances);
  69. return -EINVAL;
  70. }
  71. break;
  72. case AMDGPU_HW_IP_UVD:
  73. *out_ring = &adev->uvd.ring;
  74. break;
  75. case AMDGPU_HW_IP_VCE:
  76. if (ring < 2){
  77. *out_ring = &adev->vce.ring[ring];
  78. } else {
  79. DRM_ERROR("only two VCE rings are supported\n");
  80. return -EINVAL;
  81. }
  82. break;
  83. }
  84. return 0;
  85. }
  86. static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
  87. struct amdgpu_user_fence *uf,
  88. struct drm_amdgpu_cs_chunk_fence *fence_data)
  89. {
  90. struct drm_gem_object *gobj;
  91. uint32_t handle;
  92. handle = fence_data->handle;
  93. gobj = drm_gem_object_lookup(p->adev->ddev, p->filp,
  94. fence_data->handle);
  95. if (gobj == NULL)
  96. return -EINVAL;
  97. uf->bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
  98. uf->offset = fence_data->offset;
  99. if (amdgpu_ttm_tt_get_usermm(uf->bo->tbo.ttm)) {
  100. drm_gem_object_unreference_unlocked(gobj);
  101. return -EINVAL;
  102. }
  103. p->uf_entry.robj = amdgpu_bo_ref(uf->bo);
  104. p->uf_entry.priority = 0;
  105. p->uf_entry.tv.bo = &p->uf_entry.robj->tbo;
  106. p->uf_entry.tv.shared = true;
  107. drm_gem_object_unreference_unlocked(gobj);
  108. return 0;
  109. }
  110. int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
  111. {
  112. struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
  113. union drm_amdgpu_cs *cs = data;
  114. uint64_t *chunk_array_user;
  115. uint64_t *chunk_array;
  116. struct amdgpu_user_fence uf = {};
  117. unsigned size, num_ibs = 0;
  118. int i;
  119. int ret;
  120. if (cs->in.num_chunks == 0)
  121. return 0;
  122. chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL);
  123. if (!chunk_array)
  124. return -ENOMEM;
  125. p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id);
  126. if (!p->ctx) {
  127. ret = -EINVAL;
  128. goto free_chunk;
  129. }
  130. /* get chunks */
  131. chunk_array_user = (uint64_t __user *)(unsigned long)(cs->in.chunks);
  132. if (copy_from_user(chunk_array, chunk_array_user,
  133. sizeof(uint64_t)*cs->in.num_chunks)) {
  134. ret = -EFAULT;
  135. goto put_ctx;
  136. }
  137. p->nchunks = cs->in.num_chunks;
  138. p->chunks = kmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk),
  139. GFP_KERNEL);
  140. if (!p->chunks) {
  141. ret = -ENOMEM;
  142. goto put_ctx;
  143. }
  144. for (i = 0; i < p->nchunks; i++) {
  145. struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL;
  146. struct drm_amdgpu_cs_chunk user_chunk;
  147. uint32_t __user *cdata;
  148. chunk_ptr = (void __user *)(unsigned long)chunk_array[i];
  149. if (copy_from_user(&user_chunk, chunk_ptr,
  150. sizeof(struct drm_amdgpu_cs_chunk))) {
  151. ret = -EFAULT;
  152. i--;
  153. goto free_partial_kdata;
  154. }
  155. p->chunks[i].chunk_id = user_chunk.chunk_id;
  156. p->chunks[i].length_dw = user_chunk.length_dw;
  157. size = p->chunks[i].length_dw;
  158. cdata = (void __user *)(unsigned long)user_chunk.chunk_data;
  159. p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t));
  160. if (p->chunks[i].kdata == NULL) {
  161. ret = -ENOMEM;
  162. i--;
  163. goto free_partial_kdata;
  164. }
  165. size *= sizeof(uint32_t);
  166. if (copy_from_user(p->chunks[i].kdata, cdata, size)) {
  167. ret = -EFAULT;
  168. goto free_partial_kdata;
  169. }
  170. switch (p->chunks[i].chunk_id) {
  171. case AMDGPU_CHUNK_ID_IB:
  172. ++num_ibs;
  173. break;
  174. case AMDGPU_CHUNK_ID_FENCE:
  175. size = sizeof(struct drm_amdgpu_cs_chunk_fence);
  176. if (p->chunks[i].length_dw * sizeof(uint32_t) < size) {
  177. ret = -EINVAL;
  178. goto free_partial_kdata;
  179. }
  180. ret = amdgpu_cs_user_fence_chunk(p, &uf, (void *)p->chunks[i].kdata);
  181. if (ret)
  182. goto free_partial_kdata;
  183. break;
  184. case AMDGPU_CHUNK_ID_DEPENDENCIES:
  185. break;
  186. default:
  187. ret = -EINVAL;
  188. goto free_partial_kdata;
  189. }
  190. }
  191. ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job);
  192. if (ret)
  193. goto free_all_kdata;
  194. p->job->uf = uf;
  195. kfree(chunk_array);
  196. return 0;
  197. free_all_kdata:
  198. i = p->nchunks - 1;
  199. free_partial_kdata:
  200. for (; i >= 0; i--)
  201. drm_free_large(p->chunks[i].kdata);
  202. kfree(p->chunks);
  203. put_ctx:
  204. amdgpu_ctx_put(p->ctx);
  205. free_chunk:
  206. kfree(chunk_array);
  207. return ret;
  208. }
  209. /* Returns how many bytes TTM can move per IB.
  210. */
  211. static u64 amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev)
  212. {
  213. u64 real_vram_size = adev->mc.real_vram_size;
  214. u64 vram_usage = atomic64_read(&adev->vram_usage);
  215. /* This function is based on the current VRAM usage.
  216. *
  217. * - If all of VRAM is free, allow relocating the number of bytes that
  218. * is equal to 1/4 of the size of VRAM for this IB.
  219. * - If more than one half of VRAM is occupied, only allow relocating
  220. * 1 MB of data for this IB.
  221. *
  222. * - From 0 to one half of used VRAM, the threshold decreases
  223. * linearly.
  224. * __________________
  225. * 1/4 of -|\ |
  226. * VRAM | \ |
  227. * | \ |
  228. * | \ |
  229. * | \ |
  230. * | \ |
  231. * | \ |
  232. * | \________|1 MB
  233. * |----------------|
  234. * VRAM 0 % 100 %
  235. * used used
  236. *
  237. * Note: It's a threshold, not a limit. The threshold must be crossed
  238. * for buffer relocations to stop, so any buffer of an arbitrary size
  239. * can be moved as long as the threshold isn't crossed before
  240. * the relocation takes place. We don't want to disable buffer
  241. * relocations completely.
  242. *
  243. * The idea is that buffers should be placed in VRAM at creation time
  244. * and TTM should only do a minimum number of relocations during
  245. * command submission. In practice, you need to submit at least
  246. * a dozen IBs to move all buffers to VRAM if they are in GTT.
  247. *
  248. * Also, things can get pretty crazy under memory pressure and actual
  249. * VRAM usage can change a lot, so playing safe even at 50% does
  250. * consistently increase performance.
  251. */
  252. u64 half_vram = real_vram_size >> 1;
  253. u64 half_free_vram = vram_usage >= half_vram ? 0 : half_vram - vram_usage;
  254. u64 bytes_moved_threshold = half_free_vram >> 1;
  255. return max(bytes_moved_threshold, 1024*1024ull);
  256. }
  257. int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
  258. struct list_head *validated)
  259. {
  260. struct amdgpu_bo_list_entry *lobj;
  261. u64 initial_bytes_moved;
  262. int r;
  263. list_for_each_entry(lobj, validated, tv.head) {
  264. struct amdgpu_bo *bo = lobj->robj;
  265. struct mm_struct *usermm;
  266. uint32_t domain;
  267. usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm);
  268. if (usermm && usermm != current->mm)
  269. return -EPERM;
  270. if (bo->pin_count)
  271. continue;
  272. /* Avoid moving this one if we have moved too many buffers
  273. * for this IB already.
  274. *
  275. * Note that this allows moving at least one buffer of
  276. * any size, because it doesn't take the current "bo"
  277. * into account. We don't want to disallow buffer moves
  278. * completely.
  279. */
  280. if (p->bytes_moved <= p->bytes_moved_threshold)
  281. domain = bo->prefered_domains;
  282. else
  283. domain = bo->allowed_domains;
  284. retry:
  285. amdgpu_ttm_placement_from_domain(bo, domain);
  286. initial_bytes_moved = atomic64_read(&bo->adev->num_bytes_moved);
  287. r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
  288. p->bytes_moved += atomic64_read(&bo->adev->num_bytes_moved) -
  289. initial_bytes_moved;
  290. if (unlikely(r)) {
  291. if (r != -ERESTARTSYS && domain != bo->allowed_domains) {
  292. domain = bo->allowed_domains;
  293. goto retry;
  294. }
  295. return r;
  296. }
  297. }
  298. return 0;
  299. }
  300. static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
  301. union drm_amdgpu_cs *cs)
  302. {
  303. struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
  304. struct list_head duplicates;
  305. bool need_mmap_lock = false;
  306. int r;
  307. INIT_LIST_HEAD(&p->validated);
  308. p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);
  309. if (p->bo_list) {
  310. need_mmap_lock = p->bo_list->first_userptr !=
  311. p->bo_list->num_entries;
  312. amdgpu_bo_list_get_list(p->bo_list, &p->validated);
  313. }
  314. INIT_LIST_HEAD(&duplicates);
  315. amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd);
  316. if (p->job->uf.bo)
  317. list_add(&p->uf_entry.tv.head, &p->validated);
  318. if (need_mmap_lock)
  319. down_read(&current->mm->mmap_sem);
  320. r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, &duplicates);
  321. if (unlikely(r != 0))
  322. goto error_reserve;
  323. amdgpu_vm_get_pt_bos(&fpriv->vm, &duplicates);
  324. p->bytes_moved_threshold = amdgpu_cs_get_threshold_for_moves(p->adev);
  325. p->bytes_moved = 0;
  326. r = amdgpu_cs_list_validate(p, &duplicates);
  327. if (r)
  328. goto error_validate;
  329. r = amdgpu_cs_list_validate(p, &p->validated);
  330. if (r)
  331. goto error_validate;
  332. if (p->bo_list) {
  333. struct amdgpu_vm *vm = &fpriv->vm;
  334. unsigned i;
  335. for (i = 0; i < p->bo_list->num_entries; i++) {
  336. struct amdgpu_bo *bo = p->bo_list->array[i].robj;
  337. p->bo_list->array[i].bo_va = amdgpu_vm_bo_find(vm, bo);
  338. }
  339. }
  340. error_validate:
  341. if (r) {
  342. amdgpu_vm_move_pt_bos_in_lru(p->adev, &fpriv->vm);
  343. ttm_eu_backoff_reservation(&p->ticket, &p->validated);
  344. }
  345. error_reserve:
  346. if (need_mmap_lock)
  347. up_read(&current->mm->mmap_sem);
  348. return r;
  349. }
  350. static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
  351. {
  352. struct amdgpu_bo_list_entry *e;
  353. int r;
  354. list_for_each_entry(e, &p->validated, tv.head) {
  355. struct reservation_object *resv = e->robj->tbo.resv;
  356. r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, p->filp);
  357. if (r)
  358. return r;
  359. }
  360. return 0;
  361. }
  362. static int cmp_size_smaller_first(void *priv, struct list_head *a,
  363. struct list_head *b)
  364. {
  365. struct amdgpu_bo_list_entry *la = list_entry(a, struct amdgpu_bo_list_entry, tv.head);
  366. struct amdgpu_bo_list_entry *lb = list_entry(b, struct amdgpu_bo_list_entry, tv.head);
  367. /* Sort A before B if A is smaller. */
  368. return (int)la->robj->tbo.num_pages - (int)lb->robj->tbo.num_pages;
  369. }
  370. /**
  371. * cs_parser_fini() - clean parser states
  372. * @parser: parser structure holding parsing context.
  373. * @error: error number
  374. *
  375. * If error is set than unvalidate buffer, otherwise just free memory
  376. * used by parsing context.
  377. **/
  378. static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bool backoff)
  379. {
  380. struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
  381. unsigned i;
  382. if (!error) {
  383. amdgpu_vm_move_pt_bos_in_lru(parser->adev, &fpriv->vm);
  384. /* Sort the buffer list from the smallest to largest buffer,
  385. * which affects the order of buffers in the LRU list.
  386. * This assures that the smallest buffers are added first
  387. * to the LRU list, so they are likely to be later evicted
  388. * first, instead of large buffers whose eviction is more
  389. * expensive.
  390. *
  391. * This slightly lowers the number of bytes moved by TTM
  392. * per frame under memory pressure.
  393. */
  394. list_sort(NULL, &parser->validated, cmp_size_smaller_first);
  395. ttm_eu_fence_buffer_objects(&parser->ticket,
  396. &parser->validated,
  397. parser->fence);
  398. } else if (backoff) {
  399. ttm_eu_backoff_reservation(&parser->ticket,
  400. &parser->validated);
  401. }
  402. fence_put(parser->fence);
  403. if (parser->ctx)
  404. amdgpu_ctx_put(parser->ctx);
  405. if (parser->bo_list)
  406. amdgpu_bo_list_put(parser->bo_list);
  407. for (i = 0; i < parser->nchunks; i++)
  408. drm_free_large(parser->chunks[i].kdata);
  409. kfree(parser->chunks);
  410. if (parser->job)
  411. amdgpu_job_free(parser->job);
  412. amdgpu_bo_unref(&parser->uf_entry.robj);
  413. }
  414. static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p,
  415. struct amdgpu_vm *vm)
  416. {
  417. struct amdgpu_device *adev = p->adev;
  418. struct amdgpu_bo_va *bo_va;
  419. struct amdgpu_bo *bo;
  420. int i, r;
  421. r = amdgpu_vm_update_page_directory(adev, vm);
  422. if (r)
  423. return r;
  424. r = amdgpu_sync_fence(adev, &p->job->sync, vm->page_directory_fence);
  425. if (r)
  426. return r;
  427. r = amdgpu_vm_clear_freed(adev, vm);
  428. if (r)
  429. return r;
  430. if (p->bo_list) {
  431. for (i = 0; i < p->bo_list->num_entries; i++) {
  432. struct fence *f;
  433. /* ignore duplicates */
  434. bo = p->bo_list->array[i].robj;
  435. if (!bo)
  436. continue;
  437. bo_va = p->bo_list->array[i].bo_va;
  438. if (bo_va == NULL)
  439. continue;
  440. r = amdgpu_vm_bo_update(adev, bo_va, &bo->tbo.mem);
  441. if (r)
  442. return r;
  443. f = bo_va->last_pt_update;
  444. r = amdgpu_sync_fence(adev, &p->job->sync, f);
  445. if (r)
  446. return r;
  447. }
  448. }
  449. r = amdgpu_vm_clear_invalids(adev, vm, &p->job->sync);
  450. if (amdgpu_vm_debug && p->bo_list) {
  451. /* Invalidate all BOs to test for userspace bugs */
  452. for (i = 0; i < p->bo_list->num_entries; i++) {
  453. /* ignore duplicates */
  454. bo = p->bo_list->array[i].robj;
  455. if (!bo)
  456. continue;
  457. amdgpu_vm_bo_invalidate(adev, bo);
  458. }
  459. }
  460. return r;
  461. }
  462. static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
  463. struct amdgpu_cs_parser *p)
  464. {
  465. struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
  466. struct amdgpu_vm *vm = &fpriv->vm;
  467. struct amdgpu_ring *ring = p->job->ring;
  468. int i, r;
  469. /* Only for UVD/VCE VM emulation */
  470. if (ring->funcs->parse_cs) {
  471. for (i = 0; i < p->job->num_ibs; i++) {
  472. r = amdgpu_ring_parse_cs(ring, p, i);
  473. if (r)
  474. return r;
  475. }
  476. }
  477. r = amdgpu_bo_vm_update_pte(p, vm);
  478. if (!r)
  479. amdgpu_cs_sync_rings(p);
  480. return r;
  481. }
  482. static int amdgpu_cs_handle_lockup(struct amdgpu_device *adev, int r)
  483. {
  484. if (r == -EDEADLK) {
  485. r = amdgpu_gpu_reset(adev);
  486. if (!r)
  487. r = -EAGAIN;
  488. }
  489. return r;
  490. }
  491. static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
  492. struct amdgpu_cs_parser *parser)
  493. {
  494. struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
  495. struct amdgpu_vm *vm = &fpriv->vm;
  496. int i, j;
  497. int r;
  498. for (i = 0, j = 0; i < parser->nchunks && j < parser->job->num_ibs; i++) {
  499. struct amdgpu_cs_chunk *chunk;
  500. struct amdgpu_ib *ib;
  501. struct drm_amdgpu_cs_chunk_ib *chunk_ib;
  502. struct amdgpu_ring *ring;
  503. chunk = &parser->chunks[i];
  504. ib = &parser->job->ibs[j];
  505. chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;
  506. if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
  507. continue;
  508. r = amdgpu_cs_get_ring(adev, chunk_ib->ip_type,
  509. chunk_ib->ip_instance, chunk_ib->ring,
  510. &ring);
  511. if (r)
  512. return r;
  513. if (parser->job->ring && parser->job->ring != ring)
  514. return -EINVAL;
  515. parser->job->ring = ring;
  516. if (ring->funcs->parse_cs) {
  517. struct amdgpu_bo_va_mapping *m;
  518. struct amdgpu_bo *aobj = NULL;
  519. uint64_t offset;
  520. uint8_t *kptr;
  521. m = amdgpu_cs_find_mapping(parser, chunk_ib->va_start,
  522. &aobj);
  523. if (!aobj) {
  524. DRM_ERROR("IB va_start is invalid\n");
  525. return -EINVAL;
  526. }
  527. if ((chunk_ib->va_start + chunk_ib->ib_bytes) >
  528. (m->it.last + 1) * AMDGPU_GPU_PAGE_SIZE) {
  529. DRM_ERROR("IB va_start+ib_bytes is invalid\n");
  530. return -EINVAL;
  531. }
  532. /* the IB should be reserved at this point */
  533. r = amdgpu_bo_kmap(aobj, (void **)&kptr);
  534. if (r) {
  535. return r;
  536. }
  537. offset = ((uint64_t)m->it.start) * AMDGPU_GPU_PAGE_SIZE;
  538. kptr += chunk_ib->va_start - offset;
  539. r = amdgpu_ib_get(adev, NULL, chunk_ib->ib_bytes, ib);
  540. if (r) {
  541. DRM_ERROR("Failed to get ib !\n");
  542. return r;
  543. }
  544. memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
  545. amdgpu_bo_kunmap(aobj);
  546. } else {
  547. r = amdgpu_ib_get(adev, vm, 0, ib);
  548. if (r) {
  549. DRM_ERROR("Failed to get ib !\n");
  550. return r;
  551. }
  552. ib->gpu_addr = chunk_ib->va_start;
  553. }
  554. ib->length_dw = chunk_ib->ib_bytes / 4;
  555. ib->flags = chunk_ib->flags;
  556. ib->ctx = parser->ctx;
  557. j++;
  558. }
  559. /* add GDS resources to first IB */
  560. if (parser->bo_list) {
  561. struct amdgpu_bo *gds = parser->bo_list->gds_obj;
  562. struct amdgpu_bo *gws = parser->bo_list->gws_obj;
  563. struct amdgpu_bo *oa = parser->bo_list->oa_obj;
  564. struct amdgpu_ib *ib = &parser->job->ibs[0];
  565. if (gds) {
  566. ib->gds_base = amdgpu_bo_gpu_offset(gds);
  567. ib->gds_size = amdgpu_bo_size(gds);
  568. }
  569. if (gws) {
  570. ib->gws_base = amdgpu_bo_gpu_offset(gws);
  571. ib->gws_size = amdgpu_bo_size(gws);
  572. }
  573. if (oa) {
  574. ib->oa_base = amdgpu_bo_gpu_offset(oa);
  575. ib->oa_size = amdgpu_bo_size(oa);
  576. }
  577. }
  578. /* wrap the last IB with user fence */
  579. if (parser->job->uf.bo) {
  580. struct amdgpu_ib *ib = &parser->job->ibs[parser->job->num_ibs - 1];
  581. /* UVD & VCE fw doesn't support user fences */
  582. if (parser->job->ring->type == AMDGPU_RING_TYPE_UVD ||
  583. parser->job->ring->type == AMDGPU_RING_TYPE_VCE)
  584. return -EINVAL;
  585. ib->user = &parser->job->uf;
  586. }
  587. return 0;
  588. }
  589. static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
  590. struct amdgpu_cs_parser *p)
  591. {
  592. struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
  593. int i, j, r;
  594. for (i = 0; i < p->nchunks; ++i) {
  595. struct drm_amdgpu_cs_chunk_dep *deps;
  596. struct amdgpu_cs_chunk *chunk;
  597. unsigned num_deps;
  598. chunk = &p->chunks[i];
  599. if (chunk->chunk_id != AMDGPU_CHUNK_ID_DEPENDENCIES)
  600. continue;
  601. deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata;
  602. num_deps = chunk->length_dw * 4 /
  603. sizeof(struct drm_amdgpu_cs_chunk_dep);
  604. for (j = 0; j < num_deps; ++j) {
  605. struct amdgpu_ring *ring;
  606. struct amdgpu_ctx *ctx;
  607. struct fence *fence;
  608. r = amdgpu_cs_get_ring(adev, deps[j].ip_type,
  609. deps[j].ip_instance,
  610. deps[j].ring, &ring);
  611. if (r)
  612. return r;
  613. ctx = amdgpu_ctx_get(fpriv, deps[j].ctx_id);
  614. if (ctx == NULL)
  615. return -EINVAL;
  616. fence = amdgpu_ctx_get_fence(ctx, ring,
  617. deps[j].handle);
  618. if (IS_ERR(fence)) {
  619. r = PTR_ERR(fence);
  620. amdgpu_ctx_put(ctx);
  621. return r;
  622. } else if (fence) {
  623. r = amdgpu_sync_fence(adev, &p->job->sync,
  624. fence);
  625. fence_put(fence);
  626. amdgpu_ctx_put(ctx);
  627. if (r)
  628. return r;
  629. }
  630. }
  631. }
  632. return 0;
  633. }
  634. static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
  635. union drm_amdgpu_cs *cs)
  636. {
  637. struct amdgpu_ring *ring = p->job->ring;
  638. struct amd_sched_fence *fence;
  639. struct amdgpu_job *job;
  640. job = p->job;
  641. p->job = NULL;
  642. job->base.sched = &ring->sched;
  643. job->base.s_entity = &p->ctx->rings[ring->idx].entity;
  644. job->owner = p->filp;
  645. fence = amd_sched_fence_create(job->base.s_entity, p->filp);
  646. if (!fence) {
  647. amdgpu_job_free(job);
  648. return -ENOMEM;
  649. }
  650. job->base.s_fence = fence;
  651. p->fence = fence_get(&fence->base);
  652. cs->out.handle = amdgpu_ctx_add_fence(p->ctx, ring,
  653. &fence->base);
  654. job->ibs[job->num_ibs - 1].sequence = cs->out.handle;
  655. trace_amdgpu_cs_ioctl(job);
  656. amd_sched_entity_push_job(&job->base);
  657. return 0;
  658. }
  659. int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
  660. {
  661. struct amdgpu_device *adev = dev->dev_private;
  662. union drm_amdgpu_cs *cs = data;
  663. struct amdgpu_cs_parser parser = {};
  664. bool reserved_buffers = false;
  665. int i, r;
  666. if (!adev->accel_working)
  667. return -EBUSY;
  668. parser.adev = adev;
  669. parser.filp = filp;
  670. r = amdgpu_cs_parser_init(&parser, data);
  671. if (r) {
  672. DRM_ERROR("Failed to initialize parser !\n");
  673. amdgpu_cs_parser_fini(&parser, r, false);
  674. r = amdgpu_cs_handle_lockup(adev, r);
  675. return r;
  676. }
  677. r = amdgpu_cs_parser_bos(&parser, data);
  678. if (r == -ENOMEM)
  679. DRM_ERROR("Not enough memory for command submission!\n");
  680. else if (r && r != -ERESTARTSYS)
  681. DRM_ERROR("Failed to process the buffer list %d!\n", r);
  682. else if (!r) {
  683. reserved_buffers = true;
  684. r = amdgpu_cs_ib_fill(adev, &parser);
  685. }
  686. if (!r) {
  687. r = amdgpu_cs_dependencies(adev, &parser);
  688. if (r)
  689. DRM_ERROR("Failed in the dependencies handling %d!\n", r);
  690. }
  691. if (r)
  692. goto out;
  693. for (i = 0; i < parser.job->num_ibs; i++)
  694. trace_amdgpu_cs(&parser, i);
  695. r = amdgpu_cs_ib_vm_chunk(adev, &parser);
  696. if (r)
  697. goto out;
  698. r = amdgpu_cs_submit(&parser, cs);
  699. out:
  700. amdgpu_cs_parser_fini(&parser, r, reserved_buffers);
  701. r = amdgpu_cs_handle_lockup(adev, r);
  702. return r;
  703. }
  704. /**
  705. * amdgpu_cs_wait_ioctl - wait for a command submission to finish
  706. *
  707. * @dev: drm device
  708. * @data: data from userspace
  709. * @filp: file private
  710. *
  711. * Wait for the command submission identified by handle to finish.
  712. */
  713. int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
  714. struct drm_file *filp)
  715. {
  716. union drm_amdgpu_wait_cs *wait = data;
  717. struct amdgpu_device *adev = dev->dev_private;
  718. unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);
  719. struct amdgpu_ring *ring = NULL;
  720. struct amdgpu_ctx *ctx;
  721. struct fence *fence;
  722. long r;
  723. r = amdgpu_cs_get_ring(adev, wait->in.ip_type, wait->in.ip_instance,
  724. wait->in.ring, &ring);
  725. if (r)
  726. return r;
  727. ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);
  728. if (ctx == NULL)
  729. return -EINVAL;
  730. fence = amdgpu_ctx_get_fence(ctx, ring, wait->in.handle);
  731. if (IS_ERR(fence))
  732. r = PTR_ERR(fence);
  733. else if (fence) {
  734. r = fence_wait_timeout(fence, true, timeout);
  735. fence_put(fence);
  736. } else
  737. r = 1;
  738. amdgpu_ctx_put(ctx);
  739. if (r < 0)
  740. return r;
  741. memset(wait, 0, sizeof(*wait));
  742. wait->out.status = (r == 0);
  743. return 0;
  744. }
  745. /**
  746. * amdgpu_cs_find_bo_va - find bo_va for VM address
  747. *
  748. * @parser: command submission parser context
  749. * @addr: VM address
  750. * @bo: resulting BO of the mapping found
  751. *
  752. * Search the buffer objects in the command submission context for a certain
  753. * virtual memory address. Returns allocation structure when found, NULL
  754. * otherwise.
  755. */
  756. struct amdgpu_bo_va_mapping *
  757. amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
  758. uint64_t addr, struct amdgpu_bo **bo)
  759. {
  760. struct amdgpu_bo_va_mapping *mapping;
  761. unsigned i;
  762. if (!parser->bo_list)
  763. return NULL;
  764. addr /= AMDGPU_GPU_PAGE_SIZE;
  765. for (i = 0; i < parser->bo_list->num_entries; i++) {
  766. struct amdgpu_bo_list_entry *lobj;
  767. lobj = &parser->bo_list->array[i];
  768. if (!lobj->bo_va)
  769. continue;
  770. list_for_each_entry(mapping, &lobj->bo_va->valids, list) {
  771. if (mapping->it.start > addr ||
  772. addr > mapping->it.last)
  773. continue;
  774. *bo = lobj->bo_va->bo;
  775. return mapping;
  776. }
  777. list_for_each_entry(mapping, &lobj->bo_va->invalids, list) {
  778. if (mapping->it.start > addr ||
  779. addr > mapping->it.last)
  780. continue;
  781. *bo = lobj->bo_va->bo;
  782. return mapping;
  783. }
  784. }
  785. return NULL;
  786. }