amdgpu_cs.c 39 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604
  1. /*
  2. * Copyright 2008 Jerome Glisse.
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining a
  6. * copy of this software and associated documentation files (the "Software"),
  7. * to deal in the Software without restriction, including without limitation
  8. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  9. * and/or sell copies of the Software, and to permit persons to whom the
  10. * Software is furnished to do so, subject to the following conditions:
  11. *
  12. * The above copyright notice and this permission notice (including the next
  13. * paragraph) shall be included in all copies or substantial portions of the
  14. * Software.
  15. *
  16. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19. * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22. * DEALINGS IN THE SOFTWARE.
  23. *
  24. * Authors:
  25. * Jerome Glisse <glisse@freedesktop.org>
  26. */
  27. #include <linux/pagemap.h>
  28. #include <linux/sync_file.h>
  29. #include <drm/drmP.h>
  30. #include <drm/amdgpu_drm.h>
  31. #include <drm/drm_syncobj.h>
  32. #include "amdgpu.h"
  33. #include "amdgpu_trace.h"
  34. static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
  35. struct drm_amdgpu_cs_chunk_fence *data,
  36. uint32_t *offset)
  37. {
  38. struct drm_gem_object *gobj;
  39. unsigned long size;
  40. gobj = drm_gem_object_lookup(p->filp, data->handle);
  41. if (gobj == NULL)
  42. return -EINVAL;
  43. p->uf_entry.robj = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
  44. p->uf_entry.priority = 0;
  45. p->uf_entry.tv.bo = &p->uf_entry.robj->tbo;
  46. p->uf_entry.tv.shared = true;
  47. p->uf_entry.user_pages = NULL;
  48. size = amdgpu_bo_size(p->uf_entry.robj);
  49. if (size != PAGE_SIZE || (data->offset + 8) > size)
  50. return -EINVAL;
  51. *offset = data->offset;
  52. drm_gem_object_put_unlocked(gobj);
  53. if (amdgpu_ttm_tt_get_usermm(p->uf_entry.robj->tbo.ttm)) {
  54. amdgpu_bo_unref(&p->uf_entry.robj);
  55. return -EINVAL;
  56. }
  57. return 0;
  58. }
  59. static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
  60. {
  61. struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
  62. struct amdgpu_vm *vm = &fpriv->vm;
  63. union drm_amdgpu_cs *cs = data;
  64. uint64_t *chunk_array_user;
  65. uint64_t *chunk_array;
  66. unsigned size, num_ibs = 0;
  67. uint32_t uf_offset = 0;
  68. int i;
  69. int ret;
  70. if (cs->in.num_chunks == 0)
  71. return 0;
  72. chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL);
  73. if (!chunk_array)
  74. return -ENOMEM;
  75. p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id);
  76. if (!p->ctx) {
  77. ret = -EINVAL;
  78. goto free_chunk;
  79. }
  80. /* skip guilty context job */
  81. if (atomic_read(&p->ctx->guilty) == 1) {
  82. ret = -ECANCELED;
  83. goto free_chunk;
  84. }
  85. mutex_lock(&p->ctx->lock);
  86. /* get chunks */
  87. chunk_array_user = u64_to_user_ptr(cs->in.chunks);
  88. if (copy_from_user(chunk_array, chunk_array_user,
  89. sizeof(uint64_t)*cs->in.num_chunks)) {
  90. ret = -EFAULT;
  91. goto free_chunk;
  92. }
  93. p->nchunks = cs->in.num_chunks;
  94. p->chunks = kmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk),
  95. GFP_KERNEL);
  96. if (!p->chunks) {
  97. ret = -ENOMEM;
  98. goto free_chunk;
  99. }
  100. for (i = 0; i < p->nchunks; i++) {
  101. struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL;
  102. struct drm_amdgpu_cs_chunk user_chunk;
  103. uint32_t __user *cdata;
  104. chunk_ptr = u64_to_user_ptr(chunk_array[i]);
  105. if (copy_from_user(&user_chunk, chunk_ptr,
  106. sizeof(struct drm_amdgpu_cs_chunk))) {
  107. ret = -EFAULT;
  108. i--;
  109. goto free_partial_kdata;
  110. }
  111. p->chunks[i].chunk_id = user_chunk.chunk_id;
  112. p->chunks[i].length_dw = user_chunk.length_dw;
  113. size = p->chunks[i].length_dw;
  114. cdata = u64_to_user_ptr(user_chunk.chunk_data);
  115. p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t), GFP_KERNEL);
  116. if (p->chunks[i].kdata == NULL) {
  117. ret = -ENOMEM;
  118. i--;
  119. goto free_partial_kdata;
  120. }
  121. size *= sizeof(uint32_t);
  122. if (copy_from_user(p->chunks[i].kdata, cdata, size)) {
  123. ret = -EFAULT;
  124. goto free_partial_kdata;
  125. }
  126. switch (p->chunks[i].chunk_id) {
  127. case AMDGPU_CHUNK_ID_IB:
  128. ++num_ibs;
  129. break;
  130. case AMDGPU_CHUNK_ID_FENCE:
  131. size = sizeof(struct drm_amdgpu_cs_chunk_fence);
  132. if (p->chunks[i].length_dw * sizeof(uint32_t) < size) {
  133. ret = -EINVAL;
  134. goto free_partial_kdata;
  135. }
  136. ret = amdgpu_cs_user_fence_chunk(p, p->chunks[i].kdata,
  137. &uf_offset);
  138. if (ret)
  139. goto free_partial_kdata;
  140. break;
  141. case AMDGPU_CHUNK_ID_DEPENDENCIES:
  142. case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
  143. case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
  144. break;
  145. default:
  146. ret = -EINVAL;
  147. goto free_partial_kdata;
  148. }
  149. }
  150. ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job, vm);
  151. if (ret)
  152. goto free_all_kdata;
  153. if (p->ctx->vram_lost_counter != p->job->vram_lost_counter) {
  154. ret = -ECANCELED;
  155. goto free_all_kdata;
  156. }
  157. if (p->uf_entry.robj)
  158. p->job->uf_addr = uf_offset;
  159. kfree(chunk_array);
  160. return 0;
  161. free_all_kdata:
  162. i = p->nchunks - 1;
  163. free_partial_kdata:
  164. for (; i >= 0; i--)
  165. kvfree(p->chunks[i].kdata);
  166. kfree(p->chunks);
  167. p->chunks = NULL;
  168. p->nchunks = 0;
  169. free_chunk:
  170. kfree(chunk_array);
  171. return ret;
  172. }
  173. /* Convert microseconds to bytes. */
  174. static u64 us_to_bytes(struct amdgpu_device *adev, s64 us)
  175. {
  176. if (us <= 0 || !adev->mm_stats.log2_max_MBps)
  177. return 0;
  178. /* Since accum_us is incremented by a million per second, just
  179. * multiply it by the number of MB/s to get the number of bytes.
  180. */
  181. return us << adev->mm_stats.log2_max_MBps;
  182. }
  183. static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes)
  184. {
  185. if (!adev->mm_stats.log2_max_MBps)
  186. return 0;
  187. return bytes >> adev->mm_stats.log2_max_MBps;
  188. }
  189. /* Returns how many bytes TTM can move right now. If no bytes can be moved,
  190. * it returns 0. If it returns non-zero, it's OK to move at least one buffer,
  191. * which means it can go over the threshold once. If that happens, the driver
  192. * will be in debt and no other buffer migrations can be done until that debt
  193. * is repaid.
  194. *
  195. * This approach allows moving a buffer of any size (it's important to allow
  196. * that).
  197. *
  198. * The currency is simply time in microseconds and it increases as the clock
  199. * ticks. The accumulated microseconds (us) are converted to bytes and
  200. * returned.
  201. */
  202. static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev,
  203. u64 *max_bytes,
  204. u64 *max_vis_bytes)
  205. {
  206. s64 time_us, increment_us;
  207. u64 free_vram, total_vram, used_vram;
  208. /* Allow a maximum of 200 accumulated ms. This is basically per-IB
  209. * throttling.
  210. *
  211. * It means that in order to get full max MBps, at least 5 IBs per
  212. * second must be submitted and not more than 200ms apart from each
  213. * other.
  214. */
  215. const s64 us_upper_bound = 200000;
  216. if (!adev->mm_stats.log2_max_MBps) {
  217. *max_bytes = 0;
  218. *max_vis_bytes = 0;
  219. return;
  220. }
  221. total_vram = adev->mc.real_vram_size - adev->vram_pin_size;
  222. used_vram = amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
  223. free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram;
  224. spin_lock(&adev->mm_stats.lock);
  225. /* Increase the amount of accumulated us. */
  226. time_us = ktime_to_us(ktime_get());
  227. increment_us = time_us - adev->mm_stats.last_update_us;
  228. adev->mm_stats.last_update_us = time_us;
  229. adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us,
  230. us_upper_bound);
  231. /* This prevents the short period of low performance when the VRAM
  232. * usage is low and the driver is in debt or doesn't have enough
  233. * accumulated us to fill VRAM quickly.
  234. *
  235. * The situation can occur in these cases:
  236. * - a lot of VRAM is freed by userspace
  237. * - the presence of a big buffer causes a lot of evictions
  238. * (solution: split buffers into smaller ones)
  239. *
  240. * If 128 MB or 1/8th of VRAM is free, start filling it now by setting
  241. * accum_us to a positive number.
  242. */
  243. if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) {
  244. s64 min_us;
  245. /* Be more aggresive on dGPUs. Try to fill a portion of free
  246. * VRAM now.
  247. */
  248. if (!(adev->flags & AMD_IS_APU))
  249. min_us = bytes_to_us(adev, free_vram / 4);
  250. else
  251. min_us = 0; /* Reset accum_us on APUs. */
  252. adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us);
  253. }
  254. /* This is set to 0 if the driver is in debt to disallow (optional)
  255. * buffer moves.
  256. */
  257. *max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us);
  258. /* Do the same for visible VRAM if half of it is free */
  259. if (adev->mc.visible_vram_size < adev->mc.real_vram_size) {
  260. u64 total_vis_vram = adev->mc.visible_vram_size;
  261. u64 used_vis_vram =
  262. amdgpu_vram_mgr_vis_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
  263. if (used_vis_vram < total_vis_vram) {
  264. u64 free_vis_vram = total_vis_vram - used_vis_vram;
  265. adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis +
  266. increment_us, us_upper_bound);
  267. if (free_vis_vram >= total_vis_vram / 2)
  268. adev->mm_stats.accum_us_vis =
  269. max(bytes_to_us(adev, free_vis_vram / 2),
  270. adev->mm_stats.accum_us_vis);
  271. }
  272. *max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis);
  273. } else {
  274. *max_vis_bytes = 0;
  275. }
  276. spin_unlock(&adev->mm_stats.lock);
  277. }
  278. /* Report how many bytes have really been moved for the last command
  279. * submission. This can result in a debt that can stop buffer migrations
  280. * temporarily.
  281. */
  282. void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,
  283. u64 num_vis_bytes)
  284. {
  285. spin_lock(&adev->mm_stats.lock);
  286. adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes);
  287. adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes);
  288. spin_unlock(&adev->mm_stats.lock);
  289. }
  290. static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p,
  291. struct amdgpu_bo *bo)
  292. {
  293. struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
  294. struct ttm_operation_ctx ctx = { true, false };
  295. uint32_t domain;
  296. int r;
  297. if (bo->pin_count)
  298. return 0;
  299. /* Don't move this buffer if we have depleted our allowance
  300. * to move it. Don't move anything if the threshold is zero.
  301. */
  302. if (p->bytes_moved < p->bytes_moved_threshold) {
  303. if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&
  304. (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) {
  305. /* And don't move a CPU_ACCESS_REQUIRED BO to limited
  306. * visible VRAM if we've depleted our allowance to do
  307. * that.
  308. */
  309. if (p->bytes_moved_vis < p->bytes_moved_vis_threshold)
  310. domain = bo->preferred_domains;
  311. else
  312. domain = bo->allowed_domains;
  313. } else {
  314. domain = bo->preferred_domains;
  315. }
  316. } else {
  317. domain = bo->allowed_domains;
  318. }
  319. retry:
  320. amdgpu_ttm_placement_from_domain(bo, domain);
  321. r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
  322. p->bytes_moved += ctx.bytes_moved;
  323. if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&
  324. bo->tbo.mem.mem_type == TTM_PL_VRAM &&
  325. bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT)
  326. p->bytes_moved_vis += ctx.bytes_moved;
  327. if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
  328. domain = bo->allowed_domains;
  329. goto retry;
  330. }
  331. return r;
  332. }
  333. /* Last resort, try to evict something from the current working set */
  334. static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
  335. struct amdgpu_bo *validated)
  336. {
  337. uint32_t domain = validated->allowed_domains;
  338. struct ttm_operation_ctx ctx = { true, false };
  339. int r;
  340. if (!p->evictable)
  341. return false;
  342. for (;&p->evictable->tv.head != &p->validated;
  343. p->evictable = list_prev_entry(p->evictable, tv.head)) {
  344. struct amdgpu_bo_list_entry *candidate = p->evictable;
  345. struct amdgpu_bo *bo = candidate->robj;
  346. struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
  347. u64 initial_bytes_moved, bytes_moved;
  348. bool update_bytes_moved_vis;
  349. uint32_t other;
  350. /* If we reached our current BO we can forget it */
  351. if (candidate->robj == validated)
  352. break;
  353. /* We can't move pinned BOs here */
  354. if (bo->pin_count)
  355. continue;
  356. other = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
  357. /* Check if this BO is in one of the domains we need space for */
  358. if (!(other & domain))
  359. continue;
  360. /* Check if we can move this BO somewhere else */
  361. other = bo->allowed_domains & ~domain;
  362. if (!other)
  363. continue;
  364. /* Good we can try to move this BO somewhere else */
  365. amdgpu_ttm_placement_from_domain(bo, other);
  366. update_bytes_moved_vis =
  367. adev->mc.visible_vram_size < adev->mc.real_vram_size &&
  368. bo->tbo.mem.mem_type == TTM_PL_VRAM &&
  369. bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT;
  370. initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
  371. r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
  372. bytes_moved = atomic64_read(&adev->num_bytes_moved) -
  373. initial_bytes_moved;
  374. p->bytes_moved += bytes_moved;
  375. if (update_bytes_moved_vis)
  376. p->bytes_moved_vis += bytes_moved;
  377. if (unlikely(r))
  378. break;
  379. p->evictable = list_prev_entry(p->evictable, tv.head);
  380. list_move(&candidate->tv.head, &p->validated);
  381. return true;
  382. }
  383. return false;
  384. }
  385. static int amdgpu_cs_validate(void *param, struct amdgpu_bo *bo)
  386. {
  387. struct amdgpu_cs_parser *p = param;
  388. int r;
  389. do {
  390. r = amdgpu_cs_bo_validate(p, bo);
  391. } while (r == -ENOMEM && amdgpu_cs_try_evict(p, bo));
  392. if (r)
  393. return r;
  394. if (bo->shadow)
  395. r = amdgpu_cs_bo_validate(p, bo->shadow);
  396. return r;
  397. }
  398. static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
  399. struct list_head *validated)
  400. {
  401. struct ttm_operation_ctx ctx = { true, false };
  402. struct amdgpu_bo_list_entry *lobj;
  403. int r;
  404. list_for_each_entry(lobj, validated, tv.head) {
  405. struct amdgpu_bo *bo = lobj->robj;
  406. bool binding_userptr = false;
  407. struct mm_struct *usermm;
  408. usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm);
  409. if (usermm && usermm != current->mm)
  410. return -EPERM;
  411. /* Check if we have user pages and nobody bound the BO already */
  412. if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm) &&
  413. lobj->user_pages) {
  414. amdgpu_ttm_placement_from_domain(bo,
  415. AMDGPU_GEM_DOMAIN_CPU);
  416. r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
  417. if (r)
  418. return r;
  419. amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm,
  420. lobj->user_pages);
  421. binding_userptr = true;
  422. }
  423. if (p->evictable == lobj)
  424. p->evictable = NULL;
  425. r = amdgpu_cs_validate(p, bo);
  426. if (r)
  427. return r;
  428. if (binding_userptr) {
  429. kvfree(lobj->user_pages);
  430. lobj->user_pages = NULL;
  431. }
  432. }
  433. return 0;
  434. }
  435. static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
  436. union drm_amdgpu_cs *cs)
  437. {
  438. struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
  439. struct amdgpu_bo_list_entry *e;
  440. struct list_head duplicates;
  441. unsigned i, tries = 10;
  442. int r;
  443. INIT_LIST_HEAD(&p->validated);
  444. p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);
  445. if (p->bo_list) {
  446. amdgpu_bo_list_get_list(p->bo_list, &p->validated);
  447. if (p->bo_list->first_userptr != p->bo_list->num_entries)
  448. p->mn = amdgpu_mn_get(p->adev);
  449. }
  450. INIT_LIST_HEAD(&duplicates);
  451. amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd);
  452. if (p->uf_entry.robj)
  453. list_add(&p->uf_entry.tv.head, &p->validated);
  454. while (1) {
  455. struct list_head need_pages;
  456. unsigned i;
  457. r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true,
  458. &duplicates);
  459. if (unlikely(r != 0)) {
  460. if (r != -ERESTARTSYS)
  461. DRM_ERROR("ttm_eu_reserve_buffers failed.\n");
  462. goto error_free_pages;
  463. }
  464. /* Without a BO list we don't have userptr BOs */
  465. if (!p->bo_list)
  466. break;
  467. INIT_LIST_HEAD(&need_pages);
  468. for (i = p->bo_list->first_userptr;
  469. i < p->bo_list->num_entries; ++i) {
  470. struct amdgpu_bo *bo;
  471. e = &p->bo_list->array[i];
  472. bo = e->robj;
  473. if (amdgpu_ttm_tt_userptr_invalidated(bo->tbo.ttm,
  474. &e->user_invalidated) && e->user_pages) {
  475. /* We acquired a page array, but somebody
  476. * invalidated it. Free it and try again
  477. */
  478. release_pages(e->user_pages,
  479. bo->tbo.ttm->num_pages);
  480. kvfree(e->user_pages);
  481. e->user_pages = NULL;
  482. }
  483. if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm) &&
  484. !e->user_pages) {
  485. list_del(&e->tv.head);
  486. list_add(&e->tv.head, &need_pages);
  487. amdgpu_bo_unreserve(e->robj);
  488. }
  489. }
  490. if (list_empty(&need_pages))
  491. break;
  492. /* Unreserve everything again. */
  493. ttm_eu_backoff_reservation(&p->ticket, &p->validated);
  494. /* We tried too many times, just abort */
  495. if (!--tries) {
  496. r = -EDEADLK;
  497. DRM_ERROR("deadlock in %s\n", __func__);
  498. goto error_free_pages;
  499. }
  500. /* Fill the page arrays for all userptrs. */
  501. list_for_each_entry(e, &need_pages, tv.head) {
  502. struct ttm_tt *ttm = e->robj->tbo.ttm;
  503. e->user_pages = kvmalloc_array(ttm->num_pages,
  504. sizeof(struct page*),
  505. GFP_KERNEL | __GFP_ZERO);
  506. if (!e->user_pages) {
  507. r = -ENOMEM;
  508. DRM_ERROR("calloc failure in %s\n", __func__);
  509. goto error_free_pages;
  510. }
  511. r = amdgpu_ttm_tt_get_user_pages(ttm, e->user_pages);
  512. if (r) {
  513. DRM_ERROR("amdgpu_ttm_tt_get_user_pages failed.\n");
  514. kvfree(e->user_pages);
  515. e->user_pages = NULL;
  516. goto error_free_pages;
  517. }
  518. }
  519. /* And try again. */
  520. list_splice(&need_pages, &p->validated);
  521. }
  522. amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold,
  523. &p->bytes_moved_vis_threshold);
  524. p->bytes_moved = 0;
  525. p->bytes_moved_vis = 0;
  526. p->evictable = list_last_entry(&p->validated,
  527. struct amdgpu_bo_list_entry,
  528. tv.head);
  529. r = amdgpu_vm_validate_pt_bos(p->adev, &fpriv->vm,
  530. amdgpu_cs_validate, p);
  531. if (r) {
  532. DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n");
  533. goto error_validate;
  534. }
  535. r = amdgpu_cs_list_validate(p, &duplicates);
  536. if (r) {
  537. DRM_ERROR("amdgpu_cs_list_validate(duplicates) failed.\n");
  538. goto error_validate;
  539. }
  540. r = amdgpu_cs_list_validate(p, &p->validated);
  541. if (r) {
  542. DRM_ERROR("amdgpu_cs_list_validate(validated) failed.\n");
  543. goto error_validate;
  544. }
  545. amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
  546. p->bytes_moved_vis);
  547. if (p->bo_list) {
  548. struct amdgpu_bo *gds = p->bo_list->gds_obj;
  549. struct amdgpu_bo *gws = p->bo_list->gws_obj;
  550. struct amdgpu_bo *oa = p->bo_list->oa_obj;
  551. struct amdgpu_vm *vm = &fpriv->vm;
  552. unsigned i;
  553. for (i = 0; i < p->bo_list->num_entries; i++) {
  554. struct amdgpu_bo *bo = p->bo_list->array[i].robj;
  555. p->bo_list->array[i].bo_va = amdgpu_vm_bo_find(vm, bo);
  556. }
  557. if (gds) {
  558. p->job->gds_base = amdgpu_bo_gpu_offset(gds);
  559. p->job->gds_size = amdgpu_bo_size(gds);
  560. }
  561. if (gws) {
  562. p->job->gws_base = amdgpu_bo_gpu_offset(gws);
  563. p->job->gws_size = amdgpu_bo_size(gws);
  564. }
  565. if (oa) {
  566. p->job->oa_base = amdgpu_bo_gpu_offset(oa);
  567. p->job->oa_size = amdgpu_bo_size(oa);
  568. }
  569. }
  570. if (!r && p->uf_entry.robj) {
  571. struct amdgpu_bo *uf = p->uf_entry.robj;
  572. r = amdgpu_ttm_alloc_gart(&uf->tbo);
  573. p->job->uf_addr += amdgpu_bo_gpu_offset(uf);
  574. }
  575. error_validate:
  576. if (r)
  577. ttm_eu_backoff_reservation(&p->ticket, &p->validated);
  578. error_free_pages:
  579. if (p->bo_list) {
  580. for (i = p->bo_list->first_userptr;
  581. i < p->bo_list->num_entries; ++i) {
  582. e = &p->bo_list->array[i];
  583. if (!e->user_pages)
  584. continue;
  585. release_pages(e->user_pages,
  586. e->robj->tbo.ttm->num_pages);
  587. kvfree(e->user_pages);
  588. }
  589. }
  590. return r;
  591. }
  592. static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
  593. {
  594. struct amdgpu_bo_list_entry *e;
  595. int r;
  596. list_for_each_entry(e, &p->validated, tv.head) {
  597. struct reservation_object *resv = e->robj->tbo.resv;
  598. r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, p->filp,
  599. amdgpu_bo_explicit_sync(e->robj));
  600. if (r)
  601. return r;
  602. }
  603. return 0;
  604. }
  605. /**
  606. * cs_parser_fini() - clean parser states
  607. * @parser: parser structure holding parsing context.
  608. * @error: error number
  609. *
  610. * If error is set than unvalidate buffer, otherwise just free memory
  611. * used by parsing context.
  612. **/
  613. static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error,
  614. bool backoff)
  615. {
  616. unsigned i;
  617. if (error && backoff)
  618. ttm_eu_backoff_reservation(&parser->ticket,
  619. &parser->validated);
  620. for (i = 0; i < parser->num_post_dep_syncobjs; i++)
  621. drm_syncobj_put(parser->post_dep_syncobjs[i]);
  622. kfree(parser->post_dep_syncobjs);
  623. dma_fence_put(parser->fence);
  624. if (parser->ctx) {
  625. mutex_unlock(&parser->ctx->lock);
  626. amdgpu_ctx_put(parser->ctx);
  627. }
  628. if (parser->bo_list)
  629. amdgpu_bo_list_put(parser->bo_list);
  630. for (i = 0; i < parser->nchunks; i++)
  631. kvfree(parser->chunks[i].kdata);
  632. kfree(parser->chunks);
  633. if (parser->job)
  634. amdgpu_job_free(parser->job);
  635. amdgpu_bo_unref(&parser->uf_entry.robj);
  636. }
  637. static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
  638. {
  639. struct amdgpu_device *adev = p->adev;
  640. struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
  641. struct amdgpu_vm *vm = &fpriv->vm;
  642. struct amdgpu_bo_va *bo_va;
  643. struct amdgpu_bo *bo;
  644. int i, r;
  645. r = amdgpu_vm_update_directories(adev, vm);
  646. if (r)
  647. return r;
  648. r = amdgpu_vm_clear_freed(adev, vm, NULL);
  649. if (r)
  650. return r;
  651. r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false);
  652. if (r)
  653. return r;
  654. r = amdgpu_sync_fence(adev, &p->job->sync,
  655. fpriv->prt_va->last_pt_update, false);
  656. if (r)
  657. return r;
  658. if (amdgpu_sriov_vf(adev)) {
  659. struct dma_fence *f;
  660. bo_va = fpriv->csa_va;
  661. BUG_ON(!bo_va);
  662. r = amdgpu_vm_bo_update(adev, bo_va, false);
  663. if (r)
  664. return r;
  665. f = bo_va->last_pt_update;
  666. r = amdgpu_sync_fence(adev, &p->job->sync, f, false);
  667. if (r)
  668. return r;
  669. }
  670. if (p->bo_list) {
  671. for (i = 0; i < p->bo_list->num_entries; i++) {
  672. struct dma_fence *f;
  673. /* ignore duplicates */
  674. bo = p->bo_list->array[i].robj;
  675. if (!bo)
  676. continue;
  677. bo_va = p->bo_list->array[i].bo_va;
  678. if (bo_va == NULL)
  679. continue;
  680. r = amdgpu_vm_bo_update(adev, bo_va, false);
  681. if (r)
  682. return r;
  683. f = bo_va->last_pt_update;
  684. r = amdgpu_sync_fence(adev, &p->job->sync, f, false);
  685. if (r)
  686. return r;
  687. }
  688. }
  689. r = amdgpu_vm_handle_moved(adev, vm);
  690. if (r)
  691. return r;
  692. r = amdgpu_sync_fence(adev, &p->job->sync, vm->last_update, false);
  693. if (r)
  694. return r;
  695. if (amdgpu_vm_debug && p->bo_list) {
  696. /* Invalidate all BOs to test for userspace bugs */
  697. for (i = 0; i < p->bo_list->num_entries; i++) {
  698. /* ignore duplicates */
  699. bo = p->bo_list->array[i].robj;
  700. if (!bo)
  701. continue;
  702. amdgpu_vm_bo_invalidate(adev, bo, false);
  703. }
  704. }
  705. return r;
  706. }
  707. static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
  708. struct amdgpu_cs_parser *p)
  709. {
  710. struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
  711. struct amdgpu_vm *vm = &fpriv->vm;
  712. struct amdgpu_ring *ring = p->job->ring;
  713. int r;
  714. /* Only for UVD/VCE VM emulation */
  715. if (p->job->ring->funcs->parse_cs) {
  716. unsigned i, j;
  717. for (i = 0, j = 0; i < p->nchunks && j < p->job->num_ibs; i++) {
  718. struct drm_amdgpu_cs_chunk_ib *chunk_ib;
  719. struct amdgpu_bo_va_mapping *m;
  720. struct amdgpu_bo *aobj = NULL;
  721. struct amdgpu_cs_chunk *chunk;
  722. uint64_t offset, va_start;
  723. struct amdgpu_ib *ib;
  724. uint8_t *kptr;
  725. chunk = &p->chunks[i];
  726. ib = &p->job->ibs[j];
  727. chunk_ib = chunk->kdata;
  728. if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
  729. continue;
  730. va_start = chunk_ib->va_start & AMDGPU_VA_HOLE_MASK;
  731. r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m);
  732. if (r) {
  733. DRM_ERROR("IB va_start is invalid\n");
  734. return r;
  735. }
  736. if ((va_start + chunk_ib->ib_bytes) >
  737. (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
  738. DRM_ERROR("IB va_start+ib_bytes is invalid\n");
  739. return -EINVAL;
  740. }
  741. /* the IB should be reserved at this point */
  742. r = amdgpu_bo_kmap(aobj, (void **)&kptr);
  743. if (r) {
  744. return r;
  745. }
  746. offset = m->start * AMDGPU_GPU_PAGE_SIZE;
  747. kptr += va_start - offset;
  748. memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
  749. amdgpu_bo_kunmap(aobj);
  750. r = amdgpu_ring_parse_cs(ring, p, j);
  751. if (r)
  752. return r;
  753. j++;
  754. }
  755. }
  756. if (p->job->vm) {
  757. p->job->vm_pd_addr = amdgpu_bo_gpu_offset(vm->root.base.bo);
  758. r = amdgpu_bo_vm_update_pte(p);
  759. if (r)
  760. return r;
  761. }
  762. return amdgpu_cs_sync_rings(p);
  763. }
  764. static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
  765. struct amdgpu_cs_parser *parser)
  766. {
  767. struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
  768. struct amdgpu_vm *vm = &fpriv->vm;
  769. int i, j;
  770. int r, ce_preempt = 0, de_preempt = 0;
  771. for (i = 0, j = 0; i < parser->nchunks && j < parser->job->num_ibs; i++) {
  772. struct amdgpu_cs_chunk *chunk;
  773. struct amdgpu_ib *ib;
  774. struct drm_amdgpu_cs_chunk_ib *chunk_ib;
  775. struct amdgpu_ring *ring;
  776. chunk = &parser->chunks[i];
  777. ib = &parser->job->ibs[j];
  778. chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;
  779. if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
  780. continue;
  781. if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX && amdgpu_sriov_vf(adev)) {
  782. if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
  783. if (chunk_ib->flags & AMDGPU_IB_FLAG_CE)
  784. ce_preempt++;
  785. else
  786. de_preempt++;
  787. }
  788. /* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */
  789. if (ce_preempt > 1 || de_preempt > 1)
  790. return -EINVAL;
  791. }
  792. r = amdgpu_queue_mgr_map(adev, &parser->ctx->queue_mgr, chunk_ib->ip_type,
  793. chunk_ib->ip_instance, chunk_ib->ring, &ring);
  794. if (r)
  795. return r;
  796. if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE) {
  797. parser->job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT;
  798. if (!parser->ctx->preamble_presented) {
  799. parser->job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST;
  800. parser->ctx->preamble_presented = true;
  801. }
  802. }
  803. if (parser->job->ring && parser->job->ring != ring)
  804. return -EINVAL;
  805. parser->job->ring = ring;
  806. r = amdgpu_ib_get(adev, vm,
  807. ring->funcs->parse_cs ? chunk_ib->ib_bytes : 0,
  808. ib);
  809. if (r) {
  810. DRM_ERROR("Failed to get ib !\n");
  811. return r;
  812. }
  813. ib->gpu_addr = chunk_ib->va_start;
  814. ib->length_dw = chunk_ib->ib_bytes / 4;
  815. ib->flags = chunk_ib->flags;
  816. j++;
  817. }
  818. /* UVD & VCE fw doesn't support user fences */
  819. if (parser->job->uf_addr && (
  820. parser->job->ring->funcs->type == AMDGPU_RING_TYPE_UVD ||
  821. parser->job->ring->funcs->type == AMDGPU_RING_TYPE_VCE))
  822. return -EINVAL;
  823. return amdgpu_ctx_wait_prev_fence(parser->ctx, parser->job->ring->idx);
  824. }
  825. static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p,
  826. struct amdgpu_cs_chunk *chunk)
  827. {
  828. struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
  829. unsigned num_deps;
  830. int i, r;
  831. struct drm_amdgpu_cs_chunk_dep *deps;
  832. deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata;
  833. num_deps = chunk->length_dw * 4 /
  834. sizeof(struct drm_amdgpu_cs_chunk_dep);
  835. for (i = 0; i < num_deps; ++i) {
  836. struct amdgpu_ring *ring;
  837. struct amdgpu_ctx *ctx;
  838. struct dma_fence *fence;
  839. ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id);
  840. if (ctx == NULL)
  841. return -EINVAL;
  842. r = amdgpu_queue_mgr_map(p->adev, &ctx->queue_mgr,
  843. deps[i].ip_type,
  844. deps[i].ip_instance,
  845. deps[i].ring, &ring);
  846. if (r) {
  847. amdgpu_ctx_put(ctx);
  848. return r;
  849. }
  850. fence = amdgpu_ctx_get_fence(ctx, ring,
  851. deps[i].handle);
  852. if (IS_ERR(fence)) {
  853. r = PTR_ERR(fence);
  854. amdgpu_ctx_put(ctx);
  855. return r;
  856. } else if (fence) {
  857. r = amdgpu_sync_fence(p->adev, &p->job->sync, fence,
  858. true);
  859. dma_fence_put(fence);
  860. amdgpu_ctx_put(ctx);
  861. if (r)
  862. return r;
  863. }
  864. }
  865. return 0;
  866. }
  867. static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p,
  868. uint32_t handle)
  869. {
  870. int r;
  871. struct dma_fence *fence;
  872. r = drm_syncobj_find_fence(p->filp, handle, &fence);
  873. if (r)
  874. return r;
  875. r = amdgpu_sync_fence(p->adev, &p->job->sync, fence, true);
  876. dma_fence_put(fence);
  877. return r;
  878. }
  879. static int amdgpu_cs_process_syncobj_in_dep(struct amdgpu_cs_parser *p,
  880. struct amdgpu_cs_chunk *chunk)
  881. {
  882. unsigned num_deps;
  883. int i, r;
  884. struct drm_amdgpu_cs_chunk_sem *deps;
  885. deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata;
  886. num_deps = chunk->length_dw * 4 /
  887. sizeof(struct drm_amdgpu_cs_chunk_sem);
  888. for (i = 0; i < num_deps; ++i) {
  889. r = amdgpu_syncobj_lookup_and_add_to_sync(p, deps[i].handle);
  890. if (r)
  891. return r;
  892. }
  893. return 0;
  894. }
  895. static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser *p,
  896. struct amdgpu_cs_chunk *chunk)
  897. {
  898. unsigned num_deps;
  899. int i;
  900. struct drm_amdgpu_cs_chunk_sem *deps;
  901. deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata;
  902. num_deps = chunk->length_dw * 4 /
  903. sizeof(struct drm_amdgpu_cs_chunk_sem);
  904. p->post_dep_syncobjs = kmalloc_array(num_deps,
  905. sizeof(struct drm_syncobj *),
  906. GFP_KERNEL);
  907. p->num_post_dep_syncobjs = 0;
  908. if (!p->post_dep_syncobjs)
  909. return -ENOMEM;
  910. for (i = 0; i < num_deps; ++i) {
  911. p->post_dep_syncobjs[i] = drm_syncobj_find(p->filp, deps[i].handle);
  912. if (!p->post_dep_syncobjs[i])
  913. return -EINVAL;
  914. p->num_post_dep_syncobjs++;
  915. }
  916. return 0;
  917. }
  918. static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
  919. struct amdgpu_cs_parser *p)
  920. {
  921. int i, r;
  922. for (i = 0; i < p->nchunks; ++i) {
  923. struct amdgpu_cs_chunk *chunk;
  924. chunk = &p->chunks[i];
  925. if (chunk->chunk_id == AMDGPU_CHUNK_ID_DEPENDENCIES) {
  926. r = amdgpu_cs_process_fence_dep(p, chunk);
  927. if (r)
  928. return r;
  929. } else if (chunk->chunk_id == AMDGPU_CHUNK_ID_SYNCOBJ_IN) {
  930. r = amdgpu_cs_process_syncobj_in_dep(p, chunk);
  931. if (r)
  932. return r;
  933. } else if (chunk->chunk_id == AMDGPU_CHUNK_ID_SYNCOBJ_OUT) {
  934. r = amdgpu_cs_process_syncobj_out_dep(p, chunk);
  935. if (r)
  936. return r;
  937. }
  938. }
  939. return 0;
  940. }
  941. static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p)
  942. {
  943. int i;
  944. for (i = 0; i < p->num_post_dep_syncobjs; ++i)
  945. drm_syncobj_replace_fence(p->post_dep_syncobjs[i], p->fence);
  946. }
  947. static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
  948. union drm_amdgpu_cs *cs)
  949. {
  950. struct amdgpu_ring *ring = p->job->ring;
  951. struct drm_sched_entity *entity = &p->ctx->rings[ring->idx].entity;
  952. struct amdgpu_job *job;
  953. unsigned i;
  954. uint64_t seq;
  955. int r;
  956. amdgpu_mn_lock(p->mn);
  957. if (p->bo_list) {
  958. for (i = p->bo_list->first_userptr;
  959. i < p->bo_list->num_entries; ++i) {
  960. struct amdgpu_bo *bo = p->bo_list->array[i].robj;
  961. if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm)) {
  962. amdgpu_mn_unlock(p->mn);
  963. return -ERESTARTSYS;
  964. }
  965. }
  966. }
  967. job = p->job;
  968. p->job = NULL;
  969. r = drm_sched_job_init(&job->base, &ring->sched, entity, p->filp);
  970. if (r) {
  971. amdgpu_job_free(job);
  972. amdgpu_mn_unlock(p->mn);
  973. return r;
  974. }
  975. job->owner = p->filp;
  976. job->fence_ctx = entity->fence_context;
  977. p->fence = dma_fence_get(&job->base.s_fence->finished);
  978. r = amdgpu_ctx_add_fence(p->ctx, ring, p->fence, &seq);
  979. if (r) {
  980. dma_fence_put(p->fence);
  981. dma_fence_put(&job->base.s_fence->finished);
  982. amdgpu_job_free(job);
  983. amdgpu_mn_unlock(p->mn);
  984. return r;
  985. }
  986. amdgpu_cs_post_dependencies(p);
  987. cs->out.handle = seq;
  988. job->uf_sequence = seq;
  989. amdgpu_job_free_resources(job);
  990. amdgpu_ring_priority_get(job->ring, job->base.s_priority);
  991. trace_amdgpu_cs_ioctl(job);
  992. drm_sched_entity_push_job(&job->base, entity);
  993. ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence);
  994. amdgpu_mn_unlock(p->mn);
  995. return 0;
  996. }
  997. int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
  998. {
  999. struct amdgpu_device *adev = dev->dev_private;
  1000. union drm_amdgpu_cs *cs = data;
  1001. struct amdgpu_cs_parser parser = {};
  1002. bool reserved_buffers = false;
  1003. int i, r;
  1004. if (!adev->accel_working)
  1005. return -EBUSY;
  1006. parser.adev = adev;
  1007. parser.filp = filp;
  1008. r = amdgpu_cs_parser_init(&parser, data);
  1009. if (r) {
  1010. DRM_ERROR("Failed to initialize parser !\n");
  1011. goto out;
  1012. }
  1013. r = amdgpu_cs_ib_fill(adev, &parser);
  1014. if (r)
  1015. goto out;
  1016. r = amdgpu_cs_parser_bos(&parser, data);
  1017. if (r) {
  1018. if (r == -ENOMEM)
  1019. DRM_ERROR("Not enough memory for command submission!\n");
  1020. else if (r != -ERESTARTSYS)
  1021. DRM_ERROR("Failed to process the buffer list %d!\n", r);
  1022. goto out;
  1023. }
  1024. reserved_buffers = true;
  1025. r = amdgpu_cs_dependencies(adev, &parser);
  1026. if (r) {
  1027. DRM_ERROR("Failed in the dependencies handling %d!\n", r);
  1028. goto out;
  1029. }
  1030. for (i = 0; i < parser.job->num_ibs; i++)
  1031. trace_amdgpu_cs(&parser, i);
  1032. r = amdgpu_cs_ib_vm_chunk(adev, &parser);
  1033. if (r)
  1034. goto out;
  1035. r = amdgpu_cs_submit(&parser, cs);
  1036. out:
  1037. amdgpu_cs_parser_fini(&parser, r, reserved_buffers);
  1038. return r;
  1039. }
  1040. /**
  1041. * amdgpu_cs_wait_ioctl - wait for a command submission to finish
  1042. *
  1043. * @dev: drm device
  1044. * @data: data from userspace
  1045. * @filp: file private
  1046. *
  1047. * Wait for the command submission identified by handle to finish.
  1048. */
  1049. int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
  1050. struct drm_file *filp)
  1051. {
  1052. union drm_amdgpu_wait_cs *wait = data;
  1053. struct amdgpu_device *adev = dev->dev_private;
  1054. unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);
  1055. struct amdgpu_ring *ring = NULL;
  1056. struct amdgpu_ctx *ctx;
  1057. struct dma_fence *fence;
  1058. long r;
  1059. ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);
  1060. if (ctx == NULL)
  1061. return -EINVAL;
  1062. r = amdgpu_queue_mgr_map(adev, &ctx->queue_mgr,
  1063. wait->in.ip_type, wait->in.ip_instance,
  1064. wait->in.ring, &ring);
  1065. if (r) {
  1066. amdgpu_ctx_put(ctx);
  1067. return r;
  1068. }
  1069. fence = amdgpu_ctx_get_fence(ctx, ring, wait->in.handle);
  1070. if (IS_ERR(fence))
  1071. r = PTR_ERR(fence);
  1072. else if (fence) {
  1073. r = dma_fence_wait_timeout(fence, true, timeout);
  1074. if (r > 0 && fence->error)
  1075. r = fence->error;
  1076. dma_fence_put(fence);
  1077. } else
  1078. r = 1;
  1079. amdgpu_ctx_put(ctx);
  1080. if (r < 0)
  1081. return r;
  1082. memset(wait, 0, sizeof(*wait));
  1083. wait->out.status = (r == 0);
  1084. return 0;
  1085. }
  1086. /**
  1087. * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence
  1088. *
  1089. * @adev: amdgpu device
  1090. * @filp: file private
  1091. * @user: drm_amdgpu_fence copied from user space
  1092. */
  1093. static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev,
  1094. struct drm_file *filp,
  1095. struct drm_amdgpu_fence *user)
  1096. {
  1097. struct amdgpu_ring *ring;
  1098. struct amdgpu_ctx *ctx;
  1099. struct dma_fence *fence;
  1100. int r;
  1101. ctx = amdgpu_ctx_get(filp->driver_priv, user->ctx_id);
  1102. if (ctx == NULL)
  1103. return ERR_PTR(-EINVAL);
  1104. r = amdgpu_queue_mgr_map(adev, &ctx->queue_mgr, user->ip_type,
  1105. user->ip_instance, user->ring, &ring);
  1106. if (r) {
  1107. amdgpu_ctx_put(ctx);
  1108. return ERR_PTR(r);
  1109. }
  1110. fence = amdgpu_ctx_get_fence(ctx, ring, user->seq_no);
  1111. amdgpu_ctx_put(ctx);
  1112. return fence;
  1113. }
  1114. int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,
  1115. struct drm_file *filp)
  1116. {
  1117. struct amdgpu_device *adev = dev->dev_private;
  1118. union drm_amdgpu_fence_to_handle *info = data;
  1119. struct dma_fence *fence;
  1120. struct drm_syncobj *syncobj;
  1121. struct sync_file *sync_file;
  1122. int fd, r;
  1123. fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence);
  1124. if (IS_ERR(fence))
  1125. return PTR_ERR(fence);
  1126. switch (info->in.what) {
  1127. case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ:
  1128. r = drm_syncobj_create(&syncobj, 0, fence);
  1129. dma_fence_put(fence);
  1130. if (r)
  1131. return r;
  1132. r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle);
  1133. drm_syncobj_put(syncobj);
  1134. return r;
  1135. case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD:
  1136. r = drm_syncobj_create(&syncobj, 0, fence);
  1137. dma_fence_put(fence);
  1138. if (r)
  1139. return r;
  1140. r = drm_syncobj_get_fd(syncobj, (int*)&info->out.handle);
  1141. drm_syncobj_put(syncobj);
  1142. return r;
  1143. case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD:
  1144. fd = get_unused_fd_flags(O_CLOEXEC);
  1145. if (fd < 0) {
  1146. dma_fence_put(fence);
  1147. return fd;
  1148. }
  1149. sync_file = sync_file_create(fence);
  1150. dma_fence_put(fence);
  1151. if (!sync_file) {
  1152. put_unused_fd(fd);
  1153. return -ENOMEM;
  1154. }
  1155. fd_install(fd, sync_file->file);
  1156. info->out.handle = fd;
  1157. return 0;
  1158. default:
  1159. return -EINVAL;
  1160. }
  1161. }
  1162. /**
  1163. * amdgpu_cs_wait_all_fence - wait on all fences to signal
  1164. *
  1165. * @adev: amdgpu device
  1166. * @filp: file private
  1167. * @wait: wait parameters
  1168. * @fences: array of drm_amdgpu_fence
  1169. */
  1170. static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev,
  1171. struct drm_file *filp,
  1172. union drm_amdgpu_wait_fences *wait,
  1173. struct drm_amdgpu_fence *fences)
  1174. {
  1175. uint32_t fence_count = wait->in.fence_count;
  1176. unsigned int i;
  1177. long r = 1;
  1178. for (i = 0; i < fence_count; i++) {
  1179. struct dma_fence *fence;
  1180. unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);
  1181. fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);
  1182. if (IS_ERR(fence))
  1183. return PTR_ERR(fence);
  1184. else if (!fence)
  1185. continue;
  1186. r = dma_fence_wait_timeout(fence, true, timeout);
  1187. dma_fence_put(fence);
  1188. if (r < 0)
  1189. return r;
  1190. if (r == 0)
  1191. break;
  1192. if (fence->error)
  1193. return fence->error;
  1194. }
  1195. memset(wait, 0, sizeof(*wait));
  1196. wait->out.status = (r > 0);
  1197. return 0;
  1198. }
  1199. /**
  1200. * amdgpu_cs_wait_any_fence - wait on any fence to signal
  1201. *
  1202. * @adev: amdgpu device
  1203. * @filp: file private
  1204. * @wait: wait parameters
  1205. * @fences: array of drm_amdgpu_fence
  1206. */
  1207. static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev,
  1208. struct drm_file *filp,
  1209. union drm_amdgpu_wait_fences *wait,
  1210. struct drm_amdgpu_fence *fences)
  1211. {
  1212. unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);
  1213. uint32_t fence_count = wait->in.fence_count;
  1214. uint32_t first = ~0;
  1215. struct dma_fence **array;
  1216. unsigned int i;
  1217. long r;
  1218. /* Prepare the fence array */
  1219. array = kcalloc(fence_count, sizeof(struct dma_fence *), GFP_KERNEL);
  1220. if (array == NULL)
  1221. return -ENOMEM;
  1222. for (i = 0; i < fence_count; i++) {
  1223. struct dma_fence *fence;
  1224. fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);
  1225. if (IS_ERR(fence)) {
  1226. r = PTR_ERR(fence);
  1227. goto err_free_fence_array;
  1228. } else if (fence) {
  1229. array[i] = fence;
  1230. } else { /* NULL, the fence has been already signaled */
  1231. r = 1;
  1232. first = i;
  1233. goto out;
  1234. }
  1235. }
  1236. r = dma_fence_wait_any_timeout(array, fence_count, true, timeout,
  1237. &first);
  1238. if (r < 0)
  1239. goto err_free_fence_array;
  1240. out:
  1241. memset(wait, 0, sizeof(*wait));
  1242. wait->out.status = (r > 0);
  1243. wait->out.first_signaled = first;
  1244. if (first < fence_count && array[first])
  1245. r = array[first]->error;
  1246. else
  1247. r = 0;
  1248. err_free_fence_array:
  1249. for (i = 0; i < fence_count; i++)
  1250. dma_fence_put(array[i]);
  1251. kfree(array);
  1252. return r;
  1253. }
  1254. /**
  1255. * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish
  1256. *
  1257. * @dev: drm device
  1258. * @data: data from userspace
  1259. * @filp: file private
  1260. */
  1261. int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
  1262. struct drm_file *filp)
  1263. {
  1264. struct amdgpu_device *adev = dev->dev_private;
  1265. union drm_amdgpu_wait_fences *wait = data;
  1266. uint32_t fence_count = wait->in.fence_count;
  1267. struct drm_amdgpu_fence *fences_user;
  1268. struct drm_amdgpu_fence *fences;
  1269. int r;
  1270. /* Get the fences from userspace */
  1271. fences = kmalloc_array(fence_count, sizeof(struct drm_amdgpu_fence),
  1272. GFP_KERNEL);
  1273. if (fences == NULL)
  1274. return -ENOMEM;
  1275. fences_user = u64_to_user_ptr(wait->in.fences);
  1276. if (copy_from_user(fences, fences_user,
  1277. sizeof(struct drm_amdgpu_fence) * fence_count)) {
  1278. r = -EFAULT;
  1279. goto err_free_fences;
  1280. }
  1281. if (wait->in.wait_all)
  1282. r = amdgpu_cs_wait_all_fences(adev, filp, wait, fences);
  1283. else
  1284. r = amdgpu_cs_wait_any_fence(adev, filp, wait, fences);
  1285. err_free_fences:
  1286. kfree(fences);
  1287. return r;
  1288. }
  1289. /**
  1290. * amdgpu_cs_find_bo_va - find bo_va for VM address
  1291. *
  1292. * @parser: command submission parser context
  1293. * @addr: VM address
  1294. * @bo: resulting BO of the mapping found
  1295. *
  1296. * Search the buffer objects in the command submission context for a certain
  1297. * virtual memory address. Returns allocation structure when found, NULL
  1298. * otherwise.
  1299. */
  1300. int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
  1301. uint64_t addr, struct amdgpu_bo **bo,
  1302. struct amdgpu_bo_va_mapping **map)
  1303. {
  1304. struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
  1305. struct ttm_operation_ctx ctx = { false, false };
  1306. struct amdgpu_vm *vm = &fpriv->vm;
  1307. struct amdgpu_bo_va_mapping *mapping;
  1308. int r;
  1309. addr /= AMDGPU_GPU_PAGE_SIZE;
  1310. mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
  1311. if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo)
  1312. return -EINVAL;
  1313. *bo = mapping->bo_va->base.bo;
  1314. *map = mapping;
  1315. /* Double check that the BO is reserved by this CS */
  1316. if (READ_ONCE((*bo)->tbo.resv->lock.ctx) != &parser->ticket)
  1317. return -EINVAL;
  1318. if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) {
  1319. (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
  1320. amdgpu_ttm_placement_from_domain(*bo, (*bo)->allowed_domains);
  1321. r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, &ctx);
  1322. if (r)
  1323. return r;
  1324. }
  1325. return amdgpu_ttm_alloc_gart(&(*bo)->tbo);
  1326. }