amdgpu_vm.c 67 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660
  1. /*
  2. * Copyright 2008 Advanced Micro Devices, Inc.
  3. * Copyright 2008 Red Hat Inc.
  4. * Copyright 2009 Jerome Glisse.
  5. *
  6. * Permission is hereby granted, free of charge, to any person obtaining a
  7. * copy of this software and associated documentation files (the "Software"),
  8. * to deal in the Software without restriction, including without limitation
  9. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10. * and/or sell copies of the Software, and to permit persons to whom the
  11. * Software is furnished to do so, subject to the following conditions:
  12. *
  13. * The above copyright notice and this permission notice shall be included in
  14. * all copies or substantial portions of the Software.
  15. *
  16. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19. * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22. * OTHER DEALINGS IN THE SOFTWARE.
  23. *
  24. * Authors: Dave Airlie
  25. * Alex Deucher
  26. * Jerome Glisse
  27. */
  28. #include <linux/dma-fence-array.h>
  29. #include <linux/interval_tree_generic.h>
  30. #include <drm/drmP.h>
  31. #include <drm/amdgpu_drm.h>
  32. #include "amdgpu.h"
  33. #include "amdgpu_trace.h"
  34. /*
  35. * GPUVM
  36. * GPUVM is similar to the legacy gart on older asics, however
  37. * rather than there being a single global gart table
  38. * for the entire GPU, there are multiple VM page tables active
  39. * at any given time. The VM page tables can contain a mix
  40. * vram pages and system memory pages and system memory pages
  41. * can be mapped as snooped (cached system pages) or unsnooped
  42. * (uncached system pages).
  43. * Each VM has an ID associated with it and there is a page table
  44. * associated with each VMID. When execting a command buffer,
  45. * the kernel tells the the ring what VMID to use for that command
  46. * buffer. VMIDs are allocated dynamically as commands are submitted.
  47. * The userspace drivers maintain their own address space and the kernel
  48. * sets up their pages tables accordingly when they submit their
  49. * command buffers and a VMID is assigned.
  50. * Cayman/Trinity support up to 8 active VMs at any given time;
  51. * SI supports 16.
  52. */
  53. #define START(node) ((node)->start)
  54. #define LAST(node) ((node)->last)
  55. INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
  56. START, LAST, static, amdgpu_vm_it)
  57. #undef START
  58. #undef LAST
  59. /* Local structure. Encapsulate some VM table update parameters to reduce
  60. * the number of function parameters
  61. */
  62. struct amdgpu_pte_update_params {
  63. /* amdgpu device we do this update for */
  64. struct amdgpu_device *adev;
  65. /* optional amdgpu_vm we do this update for */
  66. struct amdgpu_vm *vm;
  67. /* address where to copy page table entries from */
  68. uint64_t src;
  69. /* indirect buffer to fill with commands */
  70. struct amdgpu_ib *ib;
  71. /* Function which actually does the update */
  72. void (*func)(struct amdgpu_pte_update_params *params, uint64_t pe,
  73. uint64_t addr, unsigned count, uint32_t incr,
  74. uint64_t flags);
  75. /* The next two are used during VM update by CPU
  76. * DMA addresses to use for mapping
  77. * Kernel pointer of PD/PT BO that needs to be updated
  78. */
  79. dma_addr_t *pages_addr;
  80. void *kptr;
  81. };
  82. /* Helper to disable partial resident texture feature from a fence callback */
  83. struct amdgpu_prt_cb {
  84. struct amdgpu_device *adev;
  85. struct dma_fence_cb cb;
  86. };
  87. /**
  88. * amdgpu_vm_num_entries - return the number of entries in a PD/PT
  89. *
  90. * @adev: amdgpu_device pointer
  91. *
  92. * Calculate the number of entries in a page directory or page table.
  93. */
  94. static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
  95. unsigned level)
  96. {
  97. if (level == 0)
  98. /* For the root directory */
  99. return adev->vm_manager.max_pfn >>
  100. (adev->vm_manager.block_size *
  101. adev->vm_manager.num_level);
  102. else if (level == adev->vm_manager.num_level)
  103. /* For the page tables on the leaves */
  104. return AMDGPU_VM_PTE_COUNT(adev);
  105. else
  106. /* Everything in between */
  107. return 1 << adev->vm_manager.block_size;
  108. }
  109. /**
  110. * amdgpu_vm_bo_size - returns the size of the BOs in bytes
  111. *
  112. * @adev: amdgpu_device pointer
  113. *
  114. * Calculate the size of the BO for a page directory or page table in bytes.
  115. */
  116. static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level)
  117. {
  118. return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8);
  119. }
  120. /**
  121. * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
  122. *
  123. * @vm: vm providing the BOs
  124. * @validated: head of validation list
  125. * @entry: entry to add
  126. *
  127. * Add the page directory to the list of BOs to
  128. * validate for command submission.
  129. */
  130. void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
  131. struct list_head *validated,
  132. struct amdgpu_bo_list_entry *entry)
  133. {
  134. entry->robj = vm->root.bo;
  135. entry->priority = 0;
  136. entry->tv.bo = &entry->robj->tbo;
  137. entry->tv.shared = true;
  138. entry->user_pages = NULL;
  139. list_add(&entry->tv.head, validated);
  140. }
  141. /**
  142. * amdgpu_vm_validate_layer - validate a single page table level
  143. *
  144. * @parent: parent page table level
  145. * @validate: callback to do the validation
  146. * @param: parameter for the validation callback
  147. *
  148. * Validate the page table BOs on command submission if neccessary.
  149. */
  150. static int amdgpu_vm_validate_level(struct amdgpu_vm_pt *parent,
  151. int (*validate)(void *, struct amdgpu_bo *),
  152. void *param, bool use_cpu_for_update)
  153. {
  154. unsigned i;
  155. int r;
  156. if (use_cpu_for_update) {
  157. r = amdgpu_bo_kmap(parent->bo, NULL);
  158. if (r)
  159. return r;
  160. }
  161. if (!parent->entries)
  162. return 0;
  163. for (i = 0; i <= parent->last_entry_used; ++i) {
  164. struct amdgpu_vm_pt *entry = &parent->entries[i];
  165. if (!entry->bo)
  166. continue;
  167. r = validate(param, entry->bo);
  168. if (r)
  169. return r;
  170. /*
  171. * Recurse into the sub directory. This is harmless because we
  172. * have only a maximum of 5 layers.
  173. */
  174. r = amdgpu_vm_validate_level(entry, validate, param,
  175. use_cpu_for_update);
  176. if (r)
  177. return r;
  178. }
  179. return r;
  180. }
  181. /**
  182. * amdgpu_vm_validate_pt_bos - validate the page table BOs
  183. *
  184. * @adev: amdgpu device pointer
  185. * @vm: vm providing the BOs
  186. * @validate: callback to do the validation
  187. * @param: parameter for the validation callback
  188. *
  189. * Validate the page table BOs on command submission if neccessary.
  190. */
  191. int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  192. int (*validate)(void *p, struct amdgpu_bo *bo),
  193. void *param)
  194. {
  195. uint64_t num_evictions;
  196. /* We only need to validate the page tables
  197. * if they aren't already valid.
  198. */
  199. num_evictions = atomic64_read(&adev->num_evictions);
  200. if (num_evictions == vm->last_eviction_counter)
  201. return 0;
  202. return amdgpu_vm_validate_level(&vm->root, validate, param,
  203. vm->use_cpu_for_update);
  204. }
  205. /**
  206. * amdgpu_vm_move_level_in_lru - move one level of PT BOs to the LRU tail
  207. *
  208. * @adev: amdgpu device instance
  209. * @vm: vm providing the BOs
  210. *
  211. * Move the PT BOs to the tail of the LRU.
  212. */
  213. static void amdgpu_vm_move_level_in_lru(struct amdgpu_vm_pt *parent)
  214. {
  215. unsigned i;
  216. if (!parent->entries)
  217. return;
  218. for (i = 0; i <= parent->last_entry_used; ++i) {
  219. struct amdgpu_vm_pt *entry = &parent->entries[i];
  220. if (!entry->bo)
  221. continue;
  222. ttm_bo_move_to_lru_tail(&entry->bo->tbo);
  223. amdgpu_vm_move_level_in_lru(entry);
  224. }
  225. }
  226. /**
  227. * amdgpu_vm_move_pt_bos_in_lru - move the PT BOs to the LRU tail
  228. *
  229. * @adev: amdgpu device instance
  230. * @vm: vm providing the BOs
  231. *
  232. * Move the PT BOs to the tail of the LRU.
  233. */
  234. void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
  235. struct amdgpu_vm *vm)
  236. {
  237. struct ttm_bo_global *glob = adev->mman.bdev.glob;
  238. spin_lock(&glob->lru_lock);
  239. amdgpu_vm_move_level_in_lru(&vm->root);
  240. spin_unlock(&glob->lru_lock);
  241. }
  242. /**
  243. * amdgpu_vm_alloc_levels - allocate the PD/PT levels
  244. *
  245. * @adev: amdgpu_device pointer
  246. * @vm: requested vm
  247. * @saddr: start of the address range
  248. * @eaddr: end of the address range
  249. *
  250. * Make sure the page directories and page tables are allocated
  251. */
  252. static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
  253. struct amdgpu_vm *vm,
  254. struct amdgpu_vm_pt *parent,
  255. uint64_t saddr, uint64_t eaddr,
  256. unsigned level)
  257. {
  258. unsigned shift = (adev->vm_manager.num_level - level) *
  259. adev->vm_manager.block_size;
  260. unsigned pt_idx, from, to;
  261. int r;
  262. u64 flags;
  263. if (!parent->entries) {
  264. unsigned num_entries = amdgpu_vm_num_entries(adev, level);
  265. parent->entries = kvmalloc_array(num_entries,
  266. sizeof(struct amdgpu_vm_pt),
  267. GFP_KERNEL | __GFP_ZERO);
  268. if (!parent->entries)
  269. return -ENOMEM;
  270. memset(parent->entries, 0 , sizeof(struct amdgpu_vm_pt));
  271. }
  272. from = saddr >> shift;
  273. to = eaddr >> shift;
  274. if (from >= amdgpu_vm_num_entries(adev, level) ||
  275. to >= amdgpu_vm_num_entries(adev, level))
  276. return -EINVAL;
  277. if (to > parent->last_entry_used)
  278. parent->last_entry_used = to;
  279. ++level;
  280. saddr = saddr & ((1 << shift) - 1);
  281. eaddr = eaddr & ((1 << shift) - 1);
  282. flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
  283. AMDGPU_GEM_CREATE_VRAM_CLEARED;
  284. if (vm->use_cpu_for_update)
  285. flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
  286. else
  287. flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
  288. AMDGPU_GEM_CREATE_SHADOW);
  289. /* walk over the address space and allocate the page tables */
  290. for (pt_idx = from; pt_idx <= to; ++pt_idx) {
  291. struct reservation_object *resv = vm->root.bo->tbo.resv;
  292. struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
  293. struct amdgpu_bo *pt;
  294. if (!entry->bo) {
  295. r = amdgpu_bo_create(adev,
  296. amdgpu_vm_bo_size(adev, level),
  297. AMDGPU_GPU_PAGE_SIZE, true,
  298. AMDGPU_GEM_DOMAIN_VRAM,
  299. flags,
  300. NULL, resv, &pt);
  301. if (r)
  302. return r;
  303. if (vm->use_cpu_for_update) {
  304. r = amdgpu_bo_kmap(pt, NULL);
  305. if (r) {
  306. amdgpu_bo_unref(&pt);
  307. return r;
  308. }
  309. }
  310. /* Keep a reference to the root directory to avoid
  311. * freeing them up in the wrong order.
  312. */
  313. pt->parent = amdgpu_bo_ref(vm->root.bo);
  314. entry->bo = pt;
  315. entry->addr = 0;
  316. }
  317. if (level < adev->vm_manager.num_level) {
  318. uint64_t sub_saddr = (pt_idx == from) ? saddr : 0;
  319. uint64_t sub_eaddr = (pt_idx == to) ? eaddr :
  320. ((1 << shift) - 1);
  321. r = amdgpu_vm_alloc_levels(adev, vm, entry, sub_saddr,
  322. sub_eaddr, level);
  323. if (r)
  324. return r;
  325. }
  326. }
  327. return 0;
  328. }
  329. /**
  330. * amdgpu_vm_alloc_pts - Allocate page tables.
  331. *
  332. * @adev: amdgpu_device pointer
  333. * @vm: VM to allocate page tables for
  334. * @saddr: Start address which needs to be allocated
  335. * @size: Size from start address we need.
  336. *
  337. * Make sure the page tables are allocated.
  338. */
  339. int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
  340. struct amdgpu_vm *vm,
  341. uint64_t saddr, uint64_t size)
  342. {
  343. uint64_t last_pfn;
  344. uint64_t eaddr;
  345. /* validate the parameters */
  346. if (saddr & AMDGPU_GPU_PAGE_MASK || size & AMDGPU_GPU_PAGE_MASK)
  347. return -EINVAL;
  348. eaddr = saddr + size - 1;
  349. last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE;
  350. if (last_pfn >= adev->vm_manager.max_pfn) {
  351. dev_err(adev->dev, "va above limit (0x%08llX >= 0x%08llX)\n",
  352. last_pfn, adev->vm_manager.max_pfn);
  353. return -EINVAL;
  354. }
  355. saddr /= AMDGPU_GPU_PAGE_SIZE;
  356. eaddr /= AMDGPU_GPU_PAGE_SIZE;
  357. return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr, 0);
  358. }
  359. /**
  360. * amdgpu_vm_had_gpu_reset - check if reset occured since last use
  361. *
  362. * @adev: amdgpu_device pointer
  363. * @id: VMID structure
  364. *
  365. * Check if GPU reset occured since last use of the VMID.
  366. */
  367. static bool amdgpu_vm_had_gpu_reset(struct amdgpu_device *adev,
  368. struct amdgpu_vm_id *id)
  369. {
  370. return id->current_gpu_reset_count !=
  371. atomic_read(&adev->gpu_reset_counter);
  372. }
  373. static bool amdgpu_vm_reserved_vmid_ready(struct amdgpu_vm *vm, unsigned vmhub)
  374. {
  375. return !!vm->reserved_vmid[vmhub];
  376. }
  377. /* idr_mgr->lock must be held */
  378. static int amdgpu_vm_grab_reserved_vmid_locked(struct amdgpu_vm *vm,
  379. struct amdgpu_ring *ring,
  380. struct amdgpu_sync *sync,
  381. struct dma_fence *fence,
  382. struct amdgpu_job *job)
  383. {
  384. struct amdgpu_device *adev = ring->adev;
  385. unsigned vmhub = ring->funcs->vmhub;
  386. uint64_t fence_context = adev->fence_context + ring->idx;
  387. struct amdgpu_vm_id *id = vm->reserved_vmid[vmhub];
  388. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  389. struct dma_fence *updates = sync->last_vm_update;
  390. int r = 0;
  391. struct dma_fence *flushed, *tmp;
  392. bool needs_flush = vm->use_cpu_for_update;
  393. flushed = id->flushed_updates;
  394. if ((amdgpu_vm_had_gpu_reset(adev, id)) ||
  395. (atomic64_read(&id->owner) != vm->client_id) ||
  396. (job->vm_pd_addr != id->pd_gpu_addr) ||
  397. (updates && (!flushed || updates->context != flushed->context ||
  398. dma_fence_is_later(updates, flushed))) ||
  399. (!id->last_flush || (id->last_flush->context != fence_context &&
  400. !dma_fence_is_signaled(id->last_flush)))) {
  401. needs_flush = true;
  402. /* to prevent one context starved by another context */
  403. id->pd_gpu_addr = 0;
  404. tmp = amdgpu_sync_peek_fence(&id->active, ring);
  405. if (tmp) {
  406. r = amdgpu_sync_fence(adev, sync, tmp);
  407. return r;
  408. }
  409. }
  410. /* Good we can use this VMID. Remember this submission as
  411. * user of the VMID.
  412. */
  413. r = amdgpu_sync_fence(ring->adev, &id->active, fence);
  414. if (r)
  415. goto out;
  416. if (updates && (!flushed || updates->context != flushed->context ||
  417. dma_fence_is_later(updates, flushed))) {
  418. dma_fence_put(id->flushed_updates);
  419. id->flushed_updates = dma_fence_get(updates);
  420. }
  421. id->pd_gpu_addr = job->vm_pd_addr;
  422. atomic64_set(&id->owner, vm->client_id);
  423. job->vm_needs_flush = needs_flush;
  424. if (needs_flush) {
  425. dma_fence_put(id->last_flush);
  426. id->last_flush = NULL;
  427. }
  428. job->vm_id = id - id_mgr->ids;
  429. trace_amdgpu_vm_grab_id(vm, ring, job);
  430. out:
  431. return r;
  432. }
  433. /**
  434. * amdgpu_vm_grab_id - allocate the next free VMID
  435. *
  436. * @vm: vm to allocate id for
  437. * @ring: ring we want to submit job to
  438. * @sync: sync object where we add dependencies
  439. * @fence: fence protecting ID from reuse
  440. *
  441. * Allocate an id for the vm, adding fences to the sync obj as necessary.
  442. */
  443. int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
  444. struct amdgpu_sync *sync, struct dma_fence *fence,
  445. struct amdgpu_job *job)
  446. {
  447. struct amdgpu_device *adev = ring->adev;
  448. unsigned vmhub = ring->funcs->vmhub;
  449. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  450. uint64_t fence_context = adev->fence_context + ring->idx;
  451. struct dma_fence *updates = sync->last_vm_update;
  452. struct amdgpu_vm_id *id, *idle;
  453. struct dma_fence **fences;
  454. unsigned i;
  455. int r = 0;
  456. mutex_lock(&id_mgr->lock);
  457. if (amdgpu_vm_reserved_vmid_ready(vm, vmhub)) {
  458. r = amdgpu_vm_grab_reserved_vmid_locked(vm, ring, sync, fence, job);
  459. mutex_unlock(&id_mgr->lock);
  460. return r;
  461. }
  462. fences = kmalloc_array(sizeof(void *), id_mgr->num_ids, GFP_KERNEL);
  463. if (!fences) {
  464. mutex_unlock(&id_mgr->lock);
  465. return -ENOMEM;
  466. }
  467. /* Check if we have an idle VMID */
  468. i = 0;
  469. list_for_each_entry(idle, &id_mgr->ids_lru, list) {
  470. fences[i] = amdgpu_sync_peek_fence(&idle->active, ring);
  471. if (!fences[i])
  472. break;
  473. ++i;
  474. }
  475. /* If we can't find a idle VMID to use, wait till one becomes available */
  476. if (&idle->list == &id_mgr->ids_lru) {
  477. u64 fence_context = adev->vm_manager.fence_context + ring->idx;
  478. unsigned seqno = ++adev->vm_manager.seqno[ring->idx];
  479. struct dma_fence_array *array;
  480. unsigned j;
  481. for (j = 0; j < i; ++j)
  482. dma_fence_get(fences[j]);
  483. array = dma_fence_array_create(i, fences, fence_context,
  484. seqno, true);
  485. if (!array) {
  486. for (j = 0; j < i; ++j)
  487. dma_fence_put(fences[j]);
  488. kfree(fences);
  489. r = -ENOMEM;
  490. goto error;
  491. }
  492. r = amdgpu_sync_fence(ring->adev, sync, &array->base);
  493. dma_fence_put(&array->base);
  494. if (r)
  495. goto error;
  496. mutex_unlock(&id_mgr->lock);
  497. return 0;
  498. }
  499. kfree(fences);
  500. job->vm_needs_flush = vm->use_cpu_for_update;
  501. /* Check if we can use a VMID already assigned to this VM */
  502. list_for_each_entry_reverse(id, &id_mgr->ids_lru, list) {
  503. struct dma_fence *flushed;
  504. bool needs_flush = vm->use_cpu_for_update;
  505. /* Check all the prerequisites to using this VMID */
  506. if (amdgpu_vm_had_gpu_reset(adev, id))
  507. continue;
  508. if (atomic64_read(&id->owner) != vm->client_id)
  509. continue;
  510. if (job->vm_pd_addr != id->pd_gpu_addr)
  511. continue;
  512. if (!id->last_flush ||
  513. (id->last_flush->context != fence_context &&
  514. !dma_fence_is_signaled(id->last_flush)))
  515. needs_flush = true;
  516. flushed = id->flushed_updates;
  517. if (updates && (!flushed || dma_fence_is_later(updates, flushed)))
  518. needs_flush = true;
  519. /* Concurrent flushes are only possible starting with Vega10 */
  520. if (adev->asic_type < CHIP_VEGA10 && needs_flush)
  521. continue;
  522. /* Good we can use this VMID. Remember this submission as
  523. * user of the VMID.
  524. */
  525. r = amdgpu_sync_fence(ring->adev, &id->active, fence);
  526. if (r)
  527. goto error;
  528. if (updates && (!flushed || dma_fence_is_later(updates, flushed))) {
  529. dma_fence_put(id->flushed_updates);
  530. id->flushed_updates = dma_fence_get(updates);
  531. }
  532. if (needs_flush)
  533. goto needs_flush;
  534. else
  535. goto no_flush_needed;
  536. };
  537. /* Still no ID to use? Then use the idle one found earlier */
  538. id = idle;
  539. /* Remember this submission as user of the VMID */
  540. r = amdgpu_sync_fence(ring->adev, &id->active, fence);
  541. if (r)
  542. goto error;
  543. id->pd_gpu_addr = job->vm_pd_addr;
  544. dma_fence_put(id->flushed_updates);
  545. id->flushed_updates = dma_fence_get(updates);
  546. atomic64_set(&id->owner, vm->client_id);
  547. needs_flush:
  548. job->vm_needs_flush = true;
  549. dma_fence_put(id->last_flush);
  550. id->last_flush = NULL;
  551. no_flush_needed:
  552. list_move_tail(&id->list, &id_mgr->ids_lru);
  553. job->vm_id = id - id_mgr->ids;
  554. trace_amdgpu_vm_grab_id(vm, ring, job);
  555. error:
  556. mutex_unlock(&id_mgr->lock);
  557. return r;
  558. }
  559. static void amdgpu_vm_free_reserved_vmid(struct amdgpu_device *adev,
  560. struct amdgpu_vm *vm,
  561. unsigned vmhub)
  562. {
  563. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  564. mutex_lock(&id_mgr->lock);
  565. if (vm->reserved_vmid[vmhub]) {
  566. list_add(&vm->reserved_vmid[vmhub]->list,
  567. &id_mgr->ids_lru);
  568. vm->reserved_vmid[vmhub] = NULL;
  569. atomic_dec(&id_mgr->reserved_vmid_num);
  570. }
  571. mutex_unlock(&id_mgr->lock);
  572. }
  573. static int amdgpu_vm_alloc_reserved_vmid(struct amdgpu_device *adev,
  574. struct amdgpu_vm *vm,
  575. unsigned vmhub)
  576. {
  577. struct amdgpu_vm_id_manager *id_mgr;
  578. struct amdgpu_vm_id *idle;
  579. int r = 0;
  580. id_mgr = &adev->vm_manager.id_mgr[vmhub];
  581. mutex_lock(&id_mgr->lock);
  582. if (vm->reserved_vmid[vmhub])
  583. goto unlock;
  584. if (atomic_inc_return(&id_mgr->reserved_vmid_num) >
  585. AMDGPU_VM_MAX_RESERVED_VMID) {
  586. DRM_ERROR("Over limitation of reserved vmid\n");
  587. atomic_dec(&id_mgr->reserved_vmid_num);
  588. r = -EINVAL;
  589. goto unlock;
  590. }
  591. /* Select the first entry VMID */
  592. idle = list_first_entry(&id_mgr->ids_lru, struct amdgpu_vm_id, list);
  593. list_del_init(&idle->list);
  594. vm->reserved_vmid[vmhub] = idle;
  595. mutex_unlock(&id_mgr->lock);
  596. return 0;
  597. unlock:
  598. mutex_unlock(&id_mgr->lock);
  599. return r;
  600. }
  601. /**
  602. * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
  603. *
  604. * @adev: amdgpu_device pointer
  605. */
  606. void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
  607. {
  608. const struct amdgpu_ip_block *ip_block;
  609. bool has_compute_vm_bug;
  610. struct amdgpu_ring *ring;
  611. int i;
  612. has_compute_vm_bug = false;
  613. ip_block = amdgpu_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
  614. if (ip_block) {
  615. /* Compute has a VM bug for GFX version < 7.
  616. Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
  617. if (ip_block->version->major <= 7)
  618. has_compute_vm_bug = true;
  619. else if (ip_block->version->major == 8)
  620. if (adev->gfx.mec_fw_version < 673)
  621. has_compute_vm_bug = true;
  622. }
  623. for (i = 0; i < adev->num_rings; i++) {
  624. ring = adev->rings[i];
  625. if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
  626. /* only compute rings */
  627. ring->has_compute_vm_bug = has_compute_vm_bug;
  628. else
  629. ring->has_compute_vm_bug = false;
  630. }
  631. }
  632. bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
  633. struct amdgpu_job *job)
  634. {
  635. struct amdgpu_device *adev = ring->adev;
  636. unsigned vmhub = ring->funcs->vmhub;
  637. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  638. struct amdgpu_vm_id *id;
  639. bool gds_switch_needed;
  640. bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug;
  641. if (job->vm_id == 0)
  642. return false;
  643. id = &id_mgr->ids[job->vm_id];
  644. gds_switch_needed = ring->funcs->emit_gds_switch && (
  645. id->gds_base != job->gds_base ||
  646. id->gds_size != job->gds_size ||
  647. id->gws_base != job->gws_base ||
  648. id->gws_size != job->gws_size ||
  649. id->oa_base != job->oa_base ||
  650. id->oa_size != job->oa_size);
  651. if (amdgpu_vm_had_gpu_reset(adev, id))
  652. return true;
  653. return vm_flush_needed || gds_switch_needed;
  654. }
  655. static bool amdgpu_vm_is_large_bar(struct amdgpu_device *adev)
  656. {
  657. return (adev->mc.real_vram_size == adev->mc.visible_vram_size);
  658. }
  659. /**
  660. * amdgpu_vm_flush - hardware flush the vm
  661. *
  662. * @ring: ring to use for flush
  663. * @vm_id: vmid number to use
  664. * @pd_addr: address of the page directory
  665. *
  666. * Emit a VM flush when it is necessary.
  667. */
  668. int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync)
  669. {
  670. struct amdgpu_device *adev = ring->adev;
  671. unsigned vmhub = ring->funcs->vmhub;
  672. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  673. struct amdgpu_vm_id *id = &id_mgr->ids[job->vm_id];
  674. bool gds_switch_needed = ring->funcs->emit_gds_switch && (
  675. id->gds_base != job->gds_base ||
  676. id->gds_size != job->gds_size ||
  677. id->gws_base != job->gws_base ||
  678. id->gws_size != job->gws_size ||
  679. id->oa_base != job->oa_base ||
  680. id->oa_size != job->oa_size);
  681. bool vm_flush_needed = job->vm_needs_flush;
  682. unsigned patch_offset = 0;
  683. int r;
  684. if (amdgpu_vm_had_gpu_reset(adev, id)) {
  685. gds_switch_needed = true;
  686. vm_flush_needed = true;
  687. }
  688. if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
  689. return 0;
  690. if (ring->funcs->init_cond_exec)
  691. patch_offset = amdgpu_ring_init_cond_exec(ring);
  692. if (need_pipe_sync)
  693. amdgpu_ring_emit_pipeline_sync(ring);
  694. if (ring->funcs->emit_vm_flush && vm_flush_needed) {
  695. struct dma_fence *fence;
  696. trace_amdgpu_vm_flush(ring, job->vm_id, job->vm_pd_addr);
  697. amdgpu_ring_emit_vm_flush(ring, job->vm_id, job->vm_pd_addr);
  698. r = amdgpu_fence_emit(ring, &fence);
  699. if (r)
  700. return r;
  701. mutex_lock(&id_mgr->lock);
  702. dma_fence_put(id->last_flush);
  703. id->last_flush = fence;
  704. id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
  705. mutex_unlock(&id_mgr->lock);
  706. }
  707. if (ring->funcs->emit_gds_switch && gds_switch_needed) {
  708. id->gds_base = job->gds_base;
  709. id->gds_size = job->gds_size;
  710. id->gws_base = job->gws_base;
  711. id->gws_size = job->gws_size;
  712. id->oa_base = job->oa_base;
  713. id->oa_size = job->oa_size;
  714. amdgpu_ring_emit_gds_switch(ring, job->vm_id, job->gds_base,
  715. job->gds_size, job->gws_base,
  716. job->gws_size, job->oa_base,
  717. job->oa_size);
  718. }
  719. if (ring->funcs->patch_cond_exec)
  720. amdgpu_ring_patch_cond_exec(ring, patch_offset);
  721. /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
  722. if (ring->funcs->emit_switch_buffer) {
  723. amdgpu_ring_emit_switch_buffer(ring);
  724. amdgpu_ring_emit_switch_buffer(ring);
  725. }
  726. return 0;
  727. }
  728. /**
  729. * amdgpu_vm_reset_id - reset VMID to zero
  730. *
  731. * @adev: amdgpu device structure
  732. * @vm_id: vmid number to use
  733. *
  734. * Reset saved GDW, GWS and OA to force switch on next flush.
  735. */
  736. void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vmhub,
  737. unsigned vmid)
  738. {
  739. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  740. struct amdgpu_vm_id *id = &id_mgr->ids[vmid];
  741. atomic64_set(&id->owner, 0);
  742. id->gds_base = 0;
  743. id->gds_size = 0;
  744. id->gws_base = 0;
  745. id->gws_size = 0;
  746. id->oa_base = 0;
  747. id->oa_size = 0;
  748. }
  749. /**
  750. * amdgpu_vm_reset_all_id - reset VMID to zero
  751. *
  752. * @adev: amdgpu device structure
  753. *
  754. * Reset VMID to force flush on next use
  755. */
  756. void amdgpu_vm_reset_all_ids(struct amdgpu_device *adev)
  757. {
  758. unsigned i, j;
  759. for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
  760. struct amdgpu_vm_id_manager *id_mgr =
  761. &adev->vm_manager.id_mgr[i];
  762. for (j = 1; j < id_mgr->num_ids; ++j)
  763. amdgpu_vm_reset_id(adev, i, j);
  764. }
  765. }
  766. /**
  767. * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
  768. *
  769. * @vm: requested vm
  770. * @bo: requested buffer object
  771. *
  772. * Find @bo inside the requested vm.
  773. * Search inside the @bos vm list for the requested vm
  774. * Returns the found bo_va or NULL if none is found
  775. *
  776. * Object has to be reserved!
  777. */
  778. struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
  779. struct amdgpu_bo *bo)
  780. {
  781. struct amdgpu_bo_va *bo_va;
  782. list_for_each_entry(bo_va, &bo->va, bo_list) {
  783. if (bo_va->vm == vm) {
  784. return bo_va;
  785. }
  786. }
  787. return NULL;
  788. }
  789. /**
  790. * amdgpu_vm_do_set_ptes - helper to call the right asic function
  791. *
  792. * @params: see amdgpu_pte_update_params definition
  793. * @pe: addr of the page entry
  794. * @addr: dst addr to write into pe
  795. * @count: number of page entries to update
  796. * @incr: increase next addr by incr bytes
  797. * @flags: hw access flags
  798. *
  799. * Traces the parameters and calls the right asic functions
  800. * to setup the page table using the DMA.
  801. */
  802. static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params *params,
  803. uint64_t pe, uint64_t addr,
  804. unsigned count, uint32_t incr,
  805. uint64_t flags)
  806. {
  807. trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
  808. if (count < 3) {
  809. amdgpu_vm_write_pte(params->adev, params->ib, pe,
  810. addr | flags, count, incr);
  811. } else {
  812. amdgpu_vm_set_pte_pde(params->adev, params->ib, pe, addr,
  813. count, incr, flags);
  814. }
  815. }
  816. /**
  817. * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
  818. *
  819. * @params: see amdgpu_pte_update_params definition
  820. * @pe: addr of the page entry
  821. * @addr: dst addr to write into pe
  822. * @count: number of page entries to update
  823. * @incr: increase next addr by incr bytes
  824. * @flags: hw access flags
  825. *
  826. * Traces the parameters and calls the DMA function to copy the PTEs.
  827. */
  828. static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
  829. uint64_t pe, uint64_t addr,
  830. unsigned count, uint32_t incr,
  831. uint64_t flags)
  832. {
  833. uint64_t src = (params->src + (addr >> 12) * 8);
  834. trace_amdgpu_vm_copy_ptes(pe, src, count);
  835. amdgpu_vm_copy_pte(params->adev, params->ib, pe, src, count);
  836. }
  837. /**
  838. * amdgpu_vm_map_gart - Resolve gart mapping of addr
  839. *
  840. * @pages_addr: optional DMA address to use for lookup
  841. * @addr: the unmapped addr
  842. *
  843. * Look up the physical address of the page that the pte resolves
  844. * to and return the pointer for the page table entry.
  845. */
  846. static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
  847. {
  848. uint64_t result;
  849. /* page table offset */
  850. result = pages_addr[addr >> PAGE_SHIFT];
  851. /* in case cpu page size != gpu page size*/
  852. result |= addr & (~PAGE_MASK);
  853. result &= 0xFFFFFFFFFFFFF000ULL;
  854. return result;
  855. }
  856. /**
  857. * amdgpu_vm_cpu_set_ptes - helper to update page tables via CPU
  858. *
  859. * @params: see amdgpu_pte_update_params definition
  860. * @pe: kmap addr of the page entry
  861. * @addr: dst addr to write into pe
  862. * @count: number of page entries to update
  863. * @incr: increase next addr by incr bytes
  864. * @flags: hw access flags
  865. *
  866. * Write count number of PT/PD entries directly.
  867. */
  868. static void amdgpu_vm_cpu_set_ptes(struct amdgpu_pte_update_params *params,
  869. uint64_t pe, uint64_t addr,
  870. unsigned count, uint32_t incr,
  871. uint64_t flags)
  872. {
  873. unsigned int i;
  874. uint64_t value;
  875. trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
  876. for (i = 0; i < count; i++) {
  877. value = params->pages_addr ?
  878. amdgpu_vm_map_gart(params->pages_addr, addr) :
  879. addr;
  880. amdgpu_gart_set_pte_pde(params->adev, (void *)(uintptr_t)pe,
  881. i, value, flags);
  882. addr += incr;
  883. }
  884. }
  885. static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  886. void *owner)
  887. {
  888. struct amdgpu_sync sync;
  889. int r;
  890. amdgpu_sync_create(&sync);
  891. amdgpu_sync_resv(adev, &sync, vm->root.bo->tbo.resv, owner);
  892. r = amdgpu_sync_wait(&sync, true);
  893. amdgpu_sync_free(&sync);
  894. return r;
  895. }
  896. /*
  897. * amdgpu_vm_update_level - update a single level in the hierarchy
  898. *
  899. * @adev: amdgpu_device pointer
  900. * @vm: requested vm
  901. * @parent: parent directory
  902. *
  903. * Makes sure all entries in @parent are up to date.
  904. * Returns 0 for success, error for failure.
  905. */
  906. static int amdgpu_vm_update_level(struct amdgpu_device *adev,
  907. struct amdgpu_vm *vm,
  908. struct amdgpu_vm_pt *parent,
  909. unsigned level)
  910. {
  911. struct amdgpu_bo *shadow;
  912. struct amdgpu_ring *ring = NULL;
  913. uint64_t pd_addr, shadow_addr = 0;
  914. uint32_t incr = amdgpu_vm_bo_size(adev, level + 1);
  915. uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
  916. unsigned count = 0, pt_idx, ndw = 0;
  917. struct amdgpu_job *job;
  918. struct amdgpu_pte_update_params params;
  919. struct dma_fence *fence = NULL;
  920. int r;
  921. if (!parent->entries)
  922. return 0;
  923. memset(&params, 0, sizeof(params));
  924. params.adev = adev;
  925. shadow = parent->bo->shadow;
  926. if (vm->use_cpu_for_update) {
  927. pd_addr = (unsigned long)parent->bo->kptr;
  928. r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
  929. if (unlikely(r))
  930. return r;
  931. params.func = amdgpu_vm_cpu_set_ptes;
  932. } else {
  933. if (shadow) {
  934. r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
  935. if (r)
  936. return r;
  937. }
  938. ring = container_of(vm->entity.sched, struct amdgpu_ring,
  939. sched);
  940. /* padding, etc. */
  941. ndw = 64;
  942. /* assume the worst case */
  943. ndw += parent->last_entry_used * 6;
  944. pd_addr = amdgpu_bo_gpu_offset(parent->bo);
  945. if (shadow) {
  946. shadow_addr = amdgpu_bo_gpu_offset(shadow);
  947. ndw *= 2;
  948. } else {
  949. shadow_addr = 0;
  950. }
  951. r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
  952. if (r)
  953. return r;
  954. params.ib = &job->ibs[0];
  955. params.func = amdgpu_vm_do_set_ptes;
  956. }
  957. /* walk over the address space and update the directory */
  958. for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
  959. struct amdgpu_bo *bo = parent->entries[pt_idx].bo;
  960. uint64_t pde, pt;
  961. if (bo == NULL)
  962. continue;
  963. if (bo->shadow) {
  964. struct amdgpu_bo *pt_shadow = bo->shadow;
  965. r = amdgpu_ttm_bind(&pt_shadow->tbo,
  966. &pt_shadow->tbo.mem);
  967. if (r)
  968. return r;
  969. }
  970. pt = amdgpu_bo_gpu_offset(bo);
  971. pt = amdgpu_gart_get_vm_pde(adev, pt);
  972. if (parent->entries[pt_idx].addr == pt)
  973. continue;
  974. parent->entries[pt_idx].addr = pt;
  975. pde = pd_addr + pt_idx * 8;
  976. if (((last_pde + 8 * count) != pde) ||
  977. ((last_pt + incr * count) != pt) ||
  978. (count == AMDGPU_VM_MAX_UPDATE_SIZE)) {
  979. if (count) {
  980. if (shadow)
  981. params.func(&params,
  982. last_shadow,
  983. last_pt, count,
  984. incr,
  985. AMDGPU_PTE_VALID);
  986. params.func(&params, last_pde,
  987. last_pt, count, incr,
  988. AMDGPU_PTE_VALID);
  989. }
  990. count = 1;
  991. last_pde = pde;
  992. last_shadow = shadow_addr + pt_idx * 8;
  993. last_pt = pt;
  994. } else {
  995. ++count;
  996. }
  997. }
  998. if (count) {
  999. if (vm->root.bo->shadow)
  1000. params.func(&params, last_shadow, last_pt,
  1001. count, incr, AMDGPU_PTE_VALID);
  1002. params.func(&params, last_pde, last_pt,
  1003. count, incr, AMDGPU_PTE_VALID);
  1004. }
  1005. if (!vm->use_cpu_for_update) {
  1006. if (params.ib->length_dw == 0) {
  1007. amdgpu_job_free(job);
  1008. } else {
  1009. amdgpu_ring_pad_ib(ring, params.ib);
  1010. amdgpu_sync_resv(adev, &job->sync, parent->bo->tbo.resv,
  1011. AMDGPU_FENCE_OWNER_VM);
  1012. if (shadow)
  1013. amdgpu_sync_resv(adev, &job->sync,
  1014. shadow->tbo.resv,
  1015. AMDGPU_FENCE_OWNER_VM);
  1016. WARN_ON(params.ib->length_dw > ndw);
  1017. r = amdgpu_job_submit(job, ring, &vm->entity,
  1018. AMDGPU_FENCE_OWNER_VM, &fence);
  1019. if (r)
  1020. goto error_free;
  1021. amdgpu_bo_fence(parent->bo, fence, true);
  1022. dma_fence_put(vm->last_dir_update);
  1023. vm->last_dir_update = dma_fence_get(fence);
  1024. dma_fence_put(fence);
  1025. }
  1026. }
  1027. /*
  1028. * Recurse into the subdirectories. This recursion is harmless because
  1029. * we only have a maximum of 5 layers.
  1030. */
  1031. for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
  1032. struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
  1033. if (!entry->bo)
  1034. continue;
  1035. r = amdgpu_vm_update_level(adev, vm, entry, level + 1);
  1036. if (r)
  1037. return r;
  1038. }
  1039. return 0;
  1040. error_free:
  1041. amdgpu_job_free(job);
  1042. return r;
  1043. }
  1044. /*
  1045. * amdgpu_vm_invalidate_level - mark all PD levels as invalid
  1046. *
  1047. * @parent: parent PD
  1048. *
  1049. * Mark all PD level as invalid after an error.
  1050. */
  1051. static void amdgpu_vm_invalidate_level(struct amdgpu_vm_pt *parent)
  1052. {
  1053. unsigned pt_idx;
  1054. /*
  1055. * Recurse into the subdirectories. This recursion is harmless because
  1056. * we only have a maximum of 5 layers.
  1057. */
  1058. for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
  1059. struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
  1060. if (!entry->bo)
  1061. continue;
  1062. entry->addr = ~0ULL;
  1063. amdgpu_vm_invalidate_level(entry);
  1064. }
  1065. }
  1066. /*
  1067. * amdgpu_vm_update_directories - make sure that all directories are valid
  1068. *
  1069. * @adev: amdgpu_device pointer
  1070. * @vm: requested vm
  1071. *
  1072. * Makes sure all directories are up to date.
  1073. * Returns 0 for success, error for failure.
  1074. */
  1075. int amdgpu_vm_update_directories(struct amdgpu_device *adev,
  1076. struct amdgpu_vm *vm)
  1077. {
  1078. int r;
  1079. r = amdgpu_vm_update_level(adev, vm, &vm->root, 0);
  1080. if (r)
  1081. amdgpu_vm_invalidate_level(&vm->root);
  1082. if (vm->use_cpu_for_update) {
  1083. /* Flush HDP */
  1084. mb();
  1085. amdgpu_gart_flush_gpu_tlb(adev, 0);
  1086. }
  1087. return r;
  1088. }
  1089. /**
  1090. * amdgpu_vm_find_pt - find the page table for an address
  1091. *
  1092. * @p: see amdgpu_pte_update_params definition
  1093. * @addr: virtual address in question
  1094. *
  1095. * Find the page table BO for a virtual address, return NULL when none found.
  1096. */
  1097. static struct amdgpu_bo *amdgpu_vm_get_pt(struct amdgpu_pte_update_params *p,
  1098. uint64_t addr)
  1099. {
  1100. struct amdgpu_vm_pt *entry = &p->vm->root;
  1101. unsigned idx, level = p->adev->vm_manager.num_level;
  1102. while (entry->entries) {
  1103. idx = addr >> (p->adev->vm_manager.block_size * level--);
  1104. idx %= amdgpu_bo_size(entry->bo) / 8;
  1105. entry = &entry->entries[idx];
  1106. }
  1107. if (level)
  1108. return NULL;
  1109. return entry->bo;
  1110. }
  1111. /**
  1112. * amdgpu_vm_update_ptes - make sure that page tables are valid
  1113. *
  1114. * @params: see amdgpu_pte_update_params definition
  1115. * @vm: requested vm
  1116. * @start: start of GPU address range
  1117. * @end: end of GPU address range
  1118. * @dst: destination address to map to, the next dst inside the function
  1119. * @flags: mapping flags
  1120. *
  1121. * Update the page tables in the range @start - @end.
  1122. * Returns 0 for success, -EINVAL for failure.
  1123. */
  1124. static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
  1125. uint64_t start, uint64_t end,
  1126. uint64_t dst, uint64_t flags)
  1127. {
  1128. struct amdgpu_device *adev = params->adev;
  1129. const uint64_t mask = AMDGPU_VM_PTE_COUNT(adev) - 1;
  1130. uint64_t addr, pe_start;
  1131. struct amdgpu_bo *pt;
  1132. unsigned nptes;
  1133. bool use_cpu_update = (params->func == amdgpu_vm_cpu_set_ptes);
  1134. /* walk over the address space and update the page tables */
  1135. for (addr = start; addr < end; addr += nptes) {
  1136. pt = amdgpu_vm_get_pt(params, addr);
  1137. if (!pt) {
  1138. pr_err("PT not found, aborting update_ptes\n");
  1139. return -EINVAL;
  1140. }
  1141. if ((addr & ~mask) == (end & ~mask))
  1142. nptes = end - addr;
  1143. else
  1144. nptes = AMDGPU_VM_PTE_COUNT(adev) - (addr & mask);
  1145. if (use_cpu_update) {
  1146. pe_start = (unsigned long)pt->kptr;
  1147. } else {
  1148. if (pt->shadow) {
  1149. pe_start = amdgpu_bo_gpu_offset(pt->shadow);
  1150. pe_start += (addr & mask) * 8;
  1151. params->func(params, pe_start, dst, nptes,
  1152. AMDGPU_GPU_PAGE_SIZE, flags);
  1153. }
  1154. pe_start = amdgpu_bo_gpu_offset(pt);
  1155. }
  1156. pe_start += (addr & mask) * 8;
  1157. params->func(params, pe_start, dst, nptes,
  1158. AMDGPU_GPU_PAGE_SIZE, flags);
  1159. dst += nptes * AMDGPU_GPU_PAGE_SIZE;
  1160. }
  1161. return 0;
  1162. }
  1163. /*
  1164. * amdgpu_vm_frag_ptes - add fragment information to PTEs
  1165. *
  1166. * @params: see amdgpu_pte_update_params definition
  1167. * @vm: requested vm
  1168. * @start: first PTE to handle
  1169. * @end: last PTE to handle
  1170. * @dst: addr those PTEs should point to
  1171. * @flags: hw mapping flags
  1172. * Returns 0 for success, -EINVAL for failure.
  1173. */
  1174. static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params *params,
  1175. uint64_t start, uint64_t end,
  1176. uint64_t dst, uint64_t flags)
  1177. {
  1178. int r;
  1179. /**
  1180. * The MC L1 TLB supports variable sized pages, based on a fragment
  1181. * field in the PTE. When this field is set to a non-zero value, page
  1182. * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
  1183. * flags are considered valid for all PTEs within the fragment range
  1184. * and corresponding mappings are assumed to be physically contiguous.
  1185. *
  1186. * The L1 TLB can store a single PTE for the whole fragment,
  1187. * significantly increasing the space available for translation
  1188. * caching. This leads to large improvements in throughput when the
  1189. * TLB is under pressure.
  1190. *
  1191. * The L2 TLB distributes small and large fragments into two
  1192. * asymmetric partitions. The large fragment cache is significantly
  1193. * larger. Thus, we try to use large fragments wherever possible.
  1194. * Userspace can support this by aligning virtual base address and
  1195. * allocation size to the fragment size.
  1196. */
  1197. /* SI and newer are optimized for 64KB */
  1198. unsigned pages_per_frag = AMDGPU_LOG2_PAGES_PER_FRAG(params->adev);
  1199. uint64_t frag_flags = AMDGPU_PTE_FRAG(pages_per_frag);
  1200. uint64_t frag_align = 1 << pages_per_frag;
  1201. uint64_t frag_start = ALIGN(start, frag_align);
  1202. uint64_t frag_end = end & ~(frag_align - 1);
  1203. /* system pages are non continuously */
  1204. if (params->src || !(flags & AMDGPU_PTE_VALID) ||
  1205. (frag_start >= frag_end))
  1206. return amdgpu_vm_update_ptes(params, start, end, dst, flags);
  1207. /* handle the 4K area at the beginning */
  1208. if (start != frag_start) {
  1209. r = amdgpu_vm_update_ptes(params, start, frag_start,
  1210. dst, flags);
  1211. if (r)
  1212. return r;
  1213. dst += (frag_start - start) * AMDGPU_GPU_PAGE_SIZE;
  1214. }
  1215. /* handle the area in the middle */
  1216. r = amdgpu_vm_update_ptes(params, frag_start, frag_end, dst,
  1217. flags | frag_flags);
  1218. if (r)
  1219. return r;
  1220. /* handle the 4K area at the end */
  1221. if (frag_end != end) {
  1222. dst += (frag_end - frag_start) * AMDGPU_GPU_PAGE_SIZE;
  1223. r = amdgpu_vm_update_ptes(params, frag_end, end, dst, flags);
  1224. }
  1225. return r;
  1226. }
  1227. /**
  1228. * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
  1229. *
  1230. * @adev: amdgpu_device pointer
  1231. * @exclusive: fence we need to sync to
  1232. * @src: address where to copy page table entries from
  1233. * @pages_addr: DMA addresses to use for mapping
  1234. * @vm: requested vm
  1235. * @start: start of mapped range
  1236. * @last: last mapped entry
  1237. * @flags: flags for the entries
  1238. * @addr: addr to set the area to
  1239. * @fence: optional resulting fence
  1240. *
  1241. * Fill in the page table entries between @start and @last.
  1242. * Returns 0 for success, -EINVAL for failure.
  1243. */
  1244. static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
  1245. struct dma_fence *exclusive,
  1246. uint64_t src,
  1247. dma_addr_t *pages_addr,
  1248. struct amdgpu_vm *vm,
  1249. uint64_t start, uint64_t last,
  1250. uint64_t flags, uint64_t addr,
  1251. struct dma_fence **fence)
  1252. {
  1253. struct amdgpu_ring *ring;
  1254. void *owner = AMDGPU_FENCE_OWNER_VM;
  1255. unsigned nptes, ncmds, ndw;
  1256. struct amdgpu_job *job;
  1257. struct amdgpu_pte_update_params params;
  1258. struct dma_fence *f = NULL;
  1259. int r;
  1260. memset(&params, 0, sizeof(params));
  1261. params.adev = adev;
  1262. params.vm = vm;
  1263. params.src = src;
  1264. /* sync to everything on unmapping */
  1265. if (!(flags & AMDGPU_PTE_VALID))
  1266. owner = AMDGPU_FENCE_OWNER_UNDEFINED;
  1267. if (vm->use_cpu_for_update) {
  1268. /* params.src is used as flag to indicate system Memory */
  1269. if (pages_addr)
  1270. params.src = ~0;
  1271. /* Wait for PT BOs to be free. PTs share the same resv. object
  1272. * as the root PD BO
  1273. */
  1274. r = amdgpu_vm_wait_pd(adev, vm, owner);
  1275. if (unlikely(r))
  1276. return r;
  1277. params.func = amdgpu_vm_cpu_set_ptes;
  1278. params.pages_addr = pages_addr;
  1279. return amdgpu_vm_frag_ptes(&params, start, last + 1,
  1280. addr, flags);
  1281. }
  1282. ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
  1283. nptes = last - start + 1;
  1284. /*
  1285. * reserve space for one command every (1 << BLOCK_SIZE)
  1286. * entries or 2k dwords (whatever is smaller)
  1287. */
  1288. ncmds = (nptes >> min(adev->vm_manager.block_size, 11u)) + 1;
  1289. /* padding, etc. */
  1290. ndw = 64;
  1291. if (src) {
  1292. /* only copy commands needed */
  1293. ndw += ncmds * 7;
  1294. params.func = amdgpu_vm_do_copy_ptes;
  1295. } else if (pages_addr) {
  1296. /* copy commands needed */
  1297. ndw += ncmds * 7;
  1298. /* and also PTEs */
  1299. ndw += nptes * 2;
  1300. params.func = amdgpu_vm_do_copy_ptes;
  1301. } else {
  1302. /* set page commands needed */
  1303. ndw += ncmds * 10;
  1304. /* two extra commands for begin/end of fragment */
  1305. ndw += 2 * 10;
  1306. params.func = amdgpu_vm_do_set_ptes;
  1307. }
  1308. r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
  1309. if (r)
  1310. return r;
  1311. params.ib = &job->ibs[0];
  1312. if (!src && pages_addr) {
  1313. uint64_t *pte;
  1314. unsigned i;
  1315. /* Put the PTEs at the end of the IB. */
  1316. i = ndw - nptes * 2;
  1317. pte= (uint64_t *)&(job->ibs->ptr[i]);
  1318. params.src = job->ibs->gpu_addr + i * 4;
  1319. for (i = 0; i < nptes; ++i) {
  1320. pte[i] = amdgpu_vm_map_gart(pages_addr, addr + i *
  1321. AMDGPU_GPU_PAGE_SIZE);
  1322. pte[i] |= flags;
  1323. }
  1324. addr = 0;
  1325. }
  1326. r = amdgpu_sync_fence(adev, &job->sync, exclusive);
  1327. if (r)
  1328. goto error_free;
  1329. r = amdgpu_sync_resv(adev, &job->sync, vm->root.bo->tbo.resv,
  1330. owner);
  1331. if (r)
  1332. goto error_free;
  1333. r = reservation_object_reserve_shared(vm->root.bo->tbo.resv);
  1334. if (r)
  1335. goto error_free;
  1336. r = amdgpu_vm_frag_ptes(&params, start, last + 1, addr, flags);
  1337. if (r)
  1338. goto error_free;
  1339. amdgpu_ring_pad_ib(ring, params.ib);
  1340. WARN_ON(params.ib->length_dw > ndw);
  1341. r = amdgpu_job_submit(job, ring, &vm->entity,
  1342. AMDGPU_FENCE_OWNER_VM, &f);
  1343. if (r)
  1344. goto error_free;
  1345. amdgpu_bo_fence(vm->root.bo, f, true);
  1346. dma_fence_put(*fence);
  1347. *fence = f;
  1348. return 0;
  1349. error_free:
  1350. amdgpu_job_free(job);
  1351. return r;
  1352. }
  1353. /**
  1354. * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
  1355. *
  1356. * @adev: amdgpu_device pointer
  1357. * @exclusive: fence we need to sync to
  1358. * @gtt_flags: flags as they are used for GTT
  1359. * @pages_addr: DMA addresses to use for mapping
  1360. * @vm: requested vm
  1361. * @mapping: mapped range and flags to use for the update
  1362. * @flags: HW flags for the mapping
  1363. * @nodes: array of drm_mm_nodes with the MC addresses
  1364. * @fence: optional resulting fence
  1365. *
  1366. * Split the mapping into smaller chunks so that each update fits
  1367. * into a SDMA IB.
  1368. * Returns 0 for success, -EINVAL for failure.
  1369. */
  1370. static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
  1371. struct dma_fence *exclusive,
  1372. uint64_t gtt_flags,
  1373. dma_addr_t *pages_addr,
  1374. struct amdgpu_vm *vm,
  1375. struct amdgpu_bo_va_mapping *mapping,
  1376. uint64_t flags,
  1377. struct drm_mm_node *nodes,
  1378. struct dma_fence **fence)
  1379. {
  1380. uint64_t pfn, src = 0, start = mapping->start;
  1381. int r;
  1382. /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
  1383. * but in case of something, we filter the flags in first place
  1384. */
  1385. if (!(mapping->flags & AMDGPU_PTE_READABLE))
  1386. flags &= ~AMDGPU_PTE_READABLE;
  1387. if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
  1388. flags &= ~AMDGPU_PTE_WRITEABLE;
  1389. flags &= ~AMDGPU_PTE_EXECUTABLE;
  1390. flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
  1391. flags &= ~AMDGPU_PTE_MTYPE_MASK;
  1392. flags |= (mapping->flags & AMDGPU_PTE_MTYPE_MASK);
  1393. if ((mapping->flags & AMDGPU_PTE_PRT) &&
  1394. (adev->asic_type >= CHIP_VEGA10)) {
  1395. flags |= AMDGPU_PTE_PRT;
  1396. flags &= ~AMDGPU_PTE_VALID;
  1397. }
  1398. trace_amdgpu_vm_bo_update(mapping);
  1399. pfn = mapping->offset >> PAGE_SHIFT;
  1400. if (nodes) {
  1401. while (pfn >= nodes->size) {
  1402. pfn -= nodes->size;
  1403. ++nodes;
  1404. }
  1405. }
  1406. do {
  1407. uint64_t max_entries;
  1408. uint64_t addr, last;
  1409. if (nodes) {
  1410. addr = nodes->start << PAGE_SHIFT;
  1411. max_entries = (nodes->size - pfn) *
  1412. (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
  1413. } else {
  1414. addr = 0;
  1415. max_entries = S64_MAX;
  1416. }
  1417. if (pages_addr) {
  1418. if (flags == gtt_flags)
  1419. src = adev->gart.table_addr +
  1420. (addr >> AMDGPU_GPU_PAGE_SHIFT) * 8;
  1421. else
  1422. max_entries = min(max_entries, 16ull * 1024ull);
  1423. addr = 0;
  1424. } else if (flags & AMDGPU_PTE_VALID) {
  1425. addr += adev->vm_manager.vram_base_offset;
  1426. }
  1427. addr += pfn << PAGE_SHIFT;
  1428. last = min((uint64_t)mapping->last, start + max_entries - 1);
  1429. r = amdgpu_vm_bo_update_mapping(adev, exclusive,
  1430. src, pages_addr, vm,
  1431. start, last, flags, addr,
  1432. fence);
  1433. if (r)
  1434. return r;
  1435. pfn += last - start + 1;
  1436. if (nodes && nodes->size == pfn) {
  1437. pfn = 0;
  1438. ++nodes;
  1439. }
  1440. start = last + 1;
  1441. } while (unlikely(start != mapping->last + 1));
  1442. return 0;
  1443. }
  1444. /**
  1445. * amdgpu_vm_bo_update - update all BO mappings in the vm page table
  1446. *
  1447. * @adev: amdgpu_device pointer
  1448. * @bo_va: requested BO and VM object
  1449. * @clear: if true clear the entries
  1450. *
  1451. * Fill in the page table entries for @bo_va.
  1452. * Returns 0 for success, -EINVAL for failure.
  1453. */
  1454. int amdgpu_vm_bo_update(struct amdgpu_device *adev,
  1455. struct amdgpu_bo_va *bo_va,
  1456. bool clear)
  1457. {
  1458. struct amdgpu_vm *vm = bo_va->vm;
  1459. struct amdgpu_bo_va_mapping *mapping;
  1460. dma_addr_t *pages_addr = NULL;
  1461. uint64_t gtt_flags, flags;
  1462. struct ttm_mem_reg *mem;
  1463. struct drm_mm_node *nodes;
  1464. struct dma_fence *exclusive;
  1465. int r;
  1466. if (clear || !bo_va->bo) {
  1467. mem = NULL;
  1468. nodes = NULL;
  1469. exclusive = NULL;
  1470. } else {
  1471. struct ttm_dma_tt *ttm;
  1472. mem = &bo_va->bo->tbo.mem;
  1473. nodes = mem->mm_node;
  1474. if (mem->mem_type == TTM_PL_TT) {
  1475. ttm = container_of(bo_va->bo->tbo.ttm, struct
  1476. ttm_dma_tt, ttm);
  1477. pages_addr = ttm->dma_address;
  1478. }
  1479. exclusive = reservation_object_get_excl(bo_va->bo->tbo.resv);
  1480. }
  1481. if (bo_va->bo) {
  1482. flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem);
  1483. gtt_flags = (amdgpu_ttm_is_bound(bo_va->bo->tbo.ttm) &&
  1484. adev == amdgpu_ttm_adev(bo_va->bo->tbo.bdev)) ?
  1485. flags : 0;
  1486. } else {
  1487. flags = 0x0;
  1488. gtt_flags = ~0x0;
  1489. }
  1490. spin_lock(&vm->status_lock);
  1491. if (!list_empty(&bo_va->vm_status))
  1492. list_splice_init(&bo_va->valids, &bo_va->invalids);
  1493. spin_unlock(&vm->status_lock);
  1494. list_for_each_entry(mapping, &bo_va->invalids, list) {
  1495. r = amdgpu_vm_bo_split_mapping(adev, exclusive,
  1496. gtt_flags, pages_addr, vm,
  1497. mapping, flags, nodes,
  1498. &bo_va->last_pt_update);
  1499. if (r)
  1500. return r;
  1501. }
  1502. if (trace_amdgpu_vm_bo_mapping_enabled()) {
  1503. list_for_each_entry(mapping, &bo_va->valids, list)
  1504. trace_amdgpu_vm_bo_mapping(mapping);
  1505. list_for_each_entry(mapping, &bo_va->invalids, list)
  1506. trace_amdgpu_vm_bo_mapping(mapping);
  1507. }
  1508. spin_lock(&vm->status_lock);
  1509. list_splice_init(&bo_va->invalids, &bo_va->valids);
  1510. list_del_init(&bo_va->vm_status);
  1511. if (clear)
  1512. list_add(&bo_va->vm_status, &vm->cleared);
  1513. spin_unlock(&vm->status_lock);
  1514. if (vm->use_cpu_for_update) {
  1515. /* Flush HDP */
  1516. mb();
  1517. amdgpu_gart_flush_gpu_tlb(adev, 0);
  1518. }
  1519. return 0;
  1520. }
  1521. /**
  1522. * amdgpu_vm_update_prt_state - update the global PRT state
  1523. */
  1524. static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
  1525. {
  1526. unsigned long flags;
  1527. bool enable;
  1528. spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
  1529. enable = !!atomic_read(&adev->vm_manager.num_prt_users);
  1530. adev->gart.gart_funcs->set_prt(adev, enable);
  1531. spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
  1532. }
  1533. /**
  1534. * amdgpu_vm_prt_get - add a PRT user
  1535. */
  1536. static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
  1537. {
  1538. if (!adev->gart.gart_funcs->set_prt)
  1539. return;
  1540. if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
  1541. amdgpu_vm_update_prt_state(adev);
  1542. }
  1543. /**
  1544. * amdgpu_vm_prt_put - drop a PRT user
  1545. */
  1546. static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
  1547. {
  1548. if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
  1549. amdgpu_vm_update_prt_state(adev);
  1550. }
  1551. /**
  1552. * amdgpu_vm_prt_cb - callback for updating the PRT status
  1553. */
  1554. static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
  1555. {
  1556. struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
  1557. amdgpu_vm_prt_put(cb->adev);
  1558. kfree(cb);
  1559. }
  1560. /**
  1561. * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
  1562. */
  1563. static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
  1564. struct dma_fence *fence)
  1565. {
  1566. struct amdgpu_prt_cb *cb;
  1567. if (!adev->gart.gart_funcs->set_prt)
  1568. return;
  1569. cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
  1570. if (!cb) {
  1571. /* Last resort when we are OOM */
  1572. if (fence)
  1573. dma_fence_wait(fence, false);
  1574. amdgpu_vm_prt_put(adev);
  1575. } else {
  1576. cb->adev = adev;
  1577. if (!fence || dma_fence_add_callback(fence, &cb->cb,
  1578. amdgpu_vm_prt_cb))
  1579. amdgpu_vm_prt_cb(fence, &cb->cb);
  1580. }
  1581. }
  1582. /**
  1583. * amdgpu_vm_free_mapping - free a mapping
  1584. *
  1585. * @adev: amdgpu_device pointer
  1586. * @vm: requested vm
  1587. * @mapping: mapping to be freed
  1588. * @fence: fence of the unmap operation
  1589. *
  1590. * Free a mapping and make sure we decrease the PRT usage count if applicable.
  1591. */
  1592. static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
  1593. struct amdgpu_vm *vm,
  1594. struct amdgpu_bo_va_mapping *mapping,
  1595. struct dma_fence *fence)
  1596. {
  1597. if (mapping->flags & AMDGPU_PTE_PRT)
  1598. amdgpu_vm_add_prt_cb(adev, fence);
  1599. kfree(mapping);
  1600. }
  1601. /**
  1602. * amdgpu_vm_prt_fini - finish all prt mappings
  1603. *
  1604. * @adev: amdgpu_device pointer
  1605. * @vm: requested vm
  1606. *
  1607. * Register a cleanup callback to disable PRT support after VM dies.
  1608. */
  1609. static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
  1610. {
  1611. struct reservation_object *resv = vm->root.bo->tbo.resv;
  1612. struct dma_fence *excl, **shared;
  1613. unsigned i, shared_count;
  1614. int r;
  1615. r = reservation_object_get_fences_rcu(resv, &excl,
  1616. &shared_count, &shared);
  1617. if (r) {
  1618. /* Not enough memory to grab the fence list, as last resort
  1619. * block for all the fences to complete.
  1620. */
  1621. reservation_object_wait_timeout_rcu(resv, true, false,
  1622. MAX_SCHEDULE_TIMEOUT);
  1623. return;
  1624. }
  1625. /* Add a callback for each fence in the reservation object */
  1626. amdgpu_vm_prt_get(adev);
  1627. amdgpu_vm_add_prt_cb(adev, excl);
  1628. for (i = 0; i < shared_count; ++i) {
  1629. amdgpu_vm_prt_get(adev);
  1630. amdgpu_vm_add_prt_cb(adev, shared[i]);
  1631. }
  1632. kfree(shared);
  1633. }
  1634. /**
  1635. * amdgpu_vm_clear_freed - clear freed BOs in the PT
  1636. *
  1637. * @adev: amdgpu_device pointer
  1638. * @vm: requested vm
  1639. * @fence: optional resulting fence (unchanged if no work needed to be done
  1640. * or if an error occurred)
  1641. *
  1642. * Make sure all freed BOs are cleared in the PT.
  1643. * Returns 0 for success.
  1644. *
  1645. * PTs have to be reserved and mutex must be locked!
  1646. */
  1647. int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
  1648. struct amdgpu_vm *vm,
  1649. struct dma_fence **fence)
  1650. {
  1651. struct amdgpu_bo_va_mapping *mapping;
  1652. struct dma_fence *f = NULL;
  1653. int r;
  1654. while (!list_empty(&vm->freed)) {
  1655. mapping = list_first_entry(&vm->freed,
  1656. struct amdgpu_bo_va_mapping, list);
  1657. list_del(&mapping->list);
  1658. r = amdgpu_vm_bo_update_mapping(adev, NULL, 0, NULL, vm,
  1659. mapping->start, mapping->last,
  1660. 0, 0, &f);
  1661. amdgpu_vm_free_mapping(adev, vm, mapping, f);
  1662. if (r) {
  1663. dma_fence_put(f);
  1664. return r;
  1665. }
  1666. }
  1667. if (fence && f) {
  1668. dma_fence_put(*fence);
  1669. *fence = f;
  1670. } else {
  1671. dma_fence_put(f);
  1672. }
  1673. return 0;
  1674. }
  1675. /**
  1676. * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT
  1677. *
  1678. * @adev: amdgpu_device pointer
  1679. * @vm: requested vm
  1680. *
  1681. * Make sure all invalidated BOs are cleared in the PT.
  1682. * Returns 0 for success.
  1683. *
  1684. * PTs have to be reserved and mutex must be locked!
  1685. */
  1686. int amdgpu_vm_clear_invalids(struct amdgpu_device *adev,
  1687. struct amdgpu_vm *vm, struct amdgpu_sync *sync)
  1688. {
  1689. struct amdgpu_bo_va *bo_va = NULL;
  1690. int r = 0;
  1691. spin_lock(&vm->status_lock);
  1692. while (!list_empty(&vm->invalidated)) {
  1693. bo_va = list_first_entry(&vm->invalidated,
  1694. struct amdgpu_bo_va, vm_status);
  1695. spin_unlock(&vm->status_lock);
  1696. r = amdgpu_vm_bo_update(adev, bo_va, true);
  1697. if (r)
  1698. return r;
  1699. spin_lock(&vm->status_lock);
  1700. }
  1701. spin_unlock(&vm->status_lock);
  1702. if (bo_va)
  1703. r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
  1704. return r;
  1705. }
  1706. /**
  1707. * amdgpu_vm_bo_add - add a bo to a specific vm
  1708. *
  1709. * @adev: amdgpu_device pointer
  1710. * @vm: requested vm
  1711. * @bo: amdgpu buffer object
  1712. *
  1713. * Add @bo into the requested vm.
  1714. * Add @bo to the list of bos associated with the vm
  1715. * Returns newly added bo_va or NULL for failure
  1716. *
  1717. * Object has to be reserved!
  1718. */
  1719. struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
  1720. struct amdgpu_vm *vm,
  1721. struct amdgpu_bo *bo)
  1722. {
  1723. struct amdgpu_bo_va *bo_va;
  1724. bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
  1725. if (bo_va == NULL) {
  1726. return NULL;
  1727. }
  1728. bo_va->vm = vm;
  1729. bo_va->bo = bo;
  1730. bo_va->ref_count = 1;
  1731. INIT_LIST_HEAD(&bo_va->bo_list);
  1732. INIT_LIST_HEAD(&bo_va->valids);
  1733. INIT_LIST_HEAD(&bo_va->invalids);
  1734. INIT_LIST_HEAD(&bo_va->vm_status);
  1735. if (bo)
  1736. list_add_tail(&bo_va->bo_list, &bo->va);
  1737. return bo_va;
  1738. }
  1739. /**
  1740. * amdgpu_vm_bo_map - map bo inside a vm
  1741. *
  1742. * @adev: amdgpu_device pointer
  1743. * @bo_va: bo_va to store the address
  1744. * @saddr: where to map the BO
  1745. * @offset: requested offset in the BO
  1746. * @flags: attributes of pages (read/write/valid/etc.)
  1747. *
  1748. * Add a mapping of the BO at the specefied addr into the VM.
  1749. * Returns 0 for success, error for failure.
  1750. *
  1751. * Object has to be reserved and unreserved outside!
  1752. */
  1753. int amdgpu_vm_bo_map(struct amdgpu_device *adev,
  1754. struct amdgpu_bo_va *bo_va,
  1755. uint64_t saddr, uint64_t offset,
  1756. uint64_t size, uint64_t flags)
  1757. {
  1758. struct amdgpu_bo_va_mapping *mapping, *tmp;
  1759. struct amdgpu_vm *vm = bo_va->vm;
  1760. uint64_t eaddr;
  1761. /* validate the parameters */
  1762. if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
  1763. size == 0 || size & AMDGPU_GPU_PAGE_MASK)
  1764. return -EINVAL;
  1765. /* make sure object fit at this offset */
  1766. eaddr = saddr + size - 1;
  1767. if (saddr >= eaddr ||
  1768. (bo_va->bo && offset + size > amdgpu_bo_size(bo_va->bo)))
  1769. return -EINVAL;
  1770. saddr /= AMDGPU_GPU_PAGE_SIZE;
  1771. eaddr /= AMDGPU_GPU_PAGE_SIZE;
  1772. tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
  1773. if (tmp) {
  1774. /* bo and tmp overlap, invalid addr */
  1775. dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
  1776. "0x%010Lx-0x%010Lx\n", bo_va->bo, saddr, eaddr,
  1777. tmp->start, tmp->last + 1);
  1778. return -EINVAL;
  1779. }
  1780. mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
  1781. if (!mapping)
  1782. return -ENOMEM;
  1783. INIT_LIST_HEAD(&mapping->list);
  1784. mapping->start = saddr;
  1785. mapping->last = eaddr;
  1786. mapping->offset = offset;
  1787. mapping->flags = flags;
  1788. list_add(&mapping->list, &bo_va->invalids);
  1789. amdgpu_vm_it_insert(mapping, &vm->va);
  1790. if (flags & AMDGPU_PTE_PRT)
  1791. amdgpu_vm_prt_get(adev);
  1792. return 0;
  1793. }
  1794. /**
  1795. * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
  1796. *
  1797. * @adev: amdgpu_device pointer
  1798. * @bo_va: bo_va to store the address
  1799. * @saddr: where to map the BO
  1800. * @offset: requested offset in the BO
  1801. * @flags: attributes of pages (read/write/valid/etc.)
  1802. *
  1803. * Add a mapping of the BO at the specefied addr into the VM. Replace existing
  1804. * mappings as we do so.
  1805. * Returns 0 for success, error for failure.
  1806. *
  1807. * Object has to be reserved and unreserved outside!
  1808. */
  1809. int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
  1810. struct amdgpu_bo_va *bo_va,
  1811. uint64_t saddr, uint64_t offset,
  1812. uint64_t size, uint64_t flags)
  1813. {
  1814. struct amdgpu_bo_va_mapping *mapping;
  1815. struct amdgpu_vm *vm = bo_va->vm;
  1816. uint64_t eaddr;
  1817. int r;
  1818. /* validate the parameters */
  1819. if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
  1820. size == 0 || size & AMDGPU_GPU_PAGE_MASK)
  1821. return -EINVAL;
  1822. /* make sure object fit at this offset */
  1823. eaddr = saddr + size - 1;
  1824. if (saddr >= eaddr ||
  1825. (bo_va->bo && offset + size > amdgpu_bo_size(bo_va->bo)))
  1826. return -EINVAL;
  1827. /* Allocate all the needed memory */
  1828. mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
  1829. if (!mapping)
  1830. return -ENOMEM;
  1831. r = amdgpu_vm_bo_clear_mappings(adev, bo_va->vm, saddr, size);
  1832. if (r) {
  1833. kfree(mapping);
  1834. return r;
  1835. }
  1836. saddr /= AMDGPU_GPU_PAGE_SIZE;
  1837. eaddr /= AMDGPU_GPU_PAGE_SIZE;
  1838. mapping->start = saddr;
  1839. mapping->last = eaddr;
  1840. mapping->offset = offset;
  1841. mapping->flags = flags;
  1842. list_add(&mapping->list, &bo_va->invalids);
  1843. amdgpu_vm_it_insert(mapping, &vm->va);
  1844. if (flags & AMDGPU_PTE_PRT)
  1845. amdgpu_vm_prt_get(adev);
  1846. return 0;
  1847. }
  1848. /**
  1849. * amdgpu_vm_bo_unmap - remove bo mapping from vm
  1850. *
  1851. * @adev: amdgpu_device pointer
  1852. * @bo_va: bo_va to remove the address from
  1853. * @saddr: where to the BO is mapped
  1854. *
  1855. * Remove a mapping of the BO at the specefied addr from the VM.
  1856. * Returns 0 for success, error for failure.
  1857. *
  1858. * Object has to be reserved and unreserved outside!
  1859. */
  1860. int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
  1861. struct amdgpu_bo_va *bo_va,
  1862. uint64_t saddr)
  1863. {
  1864. struct amdgpu_bo_va_mapping *mapping;
  1865. struct amdgpu_vm *vm = bo_va->vm;
  1866. bool valid = true;
  1867. saddr /= AMDGPU_GPU_PAGE_SIZE;
  1868. list_for_each_entry(mapping, &bo_va->valids, list) {
  1869. if (mapping->start == saddr)
  1870. break;
  1871. }
  1872. if (&mapping->list == &bo_va->valids) {
  1873. valid = false;
  1874. list_for_each_entry(mapping, &bo_va->invalids, list) {
  1875. if (mapping->start == saddr)
  1876. break;
  1877. }
  1878. if (&mapping->list == &bo_va->invalids)
  1879. return -ENOENT;
  1880. }
  1881. list_del(&mapping->list);
  1882. amdgpu_vm_it_remove(mapping, &vm->va);
  1883. trace_amdgpu_vm_bo_unmap(bo_va, mapping);
  1884. if (valid)
  1885. list_add(&mapping->list, &vm->freed);
  1886. else
  1887. amdgpu_vm_free_mapping(adev, vm, mapping,
  1888. bo_va->last_pt_update);
  1889. return 0;
  1890. }
  1891. /**
  1892. * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
  1893. *
  1894. * @adev: amdgpu_device pointer
  1895. * @vm: VM structure to use
  1896. * @saddr: start of the range
  1897. * @size: size of the range
  1898. *
  1899. * Remove all mappings in a range, split them as appropriate.
  1900. * Returns 0 for success, error for failure.
  1901. */
  1902. int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
  1903. struct amdgpu_vm *vm,
  1904. uint64_t saddr, uint64_t size)
  1905. {
  1906. struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
  1907. LIST_HEAD(removed);
  1908. uint64_t eaddr;
  1909. eaddr = saddr + size - 1;
  1910. saddr /= AMDGPU_GPU_PAGE_SIZE;
  1911. eaddr /= AMDGPU_GPU_PAGE_SIZE;
  1912. /* Allocate all the needed memory */
  1913. before = kzalloc(sizeof(*before), GFP_KERNEL);
  1914. if (!before)
  1915. return -ENOMEM;
  1916. INIT_LIST_HEAD(&before->list);
  1917. after = kzalloc(sizeof(*after), GFP_KERNEL);
  1918. if (!after) {
  1919. kfree(before);
  1920. return -ENOMEM;
  1921. }
  1922. INIT_LIST_HEAD(&after->list);
  1923. /* Now gather all removed mappings */
  1924. tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
  1925. while (tmp) {
  1926. /* Remember mapping split at the start */
  1927. if (tmp->start < saddr) {
  1928. before->start = tmp->start;
  1929. before->last = saddr - 1;
  1930. before->offset = tmp->offset;
  1931. before->flags = tmp->flags;
  1932. list_add(&before->list, &tmp->list);
  1933. }
  1934. /* Remember mapping split at the end */
  1935. if (tmp->last > eaddr) {
  1936. after->start = eaddr + 1;
  1937. after->last = tmp->last;
  1938. after->offset = tmp->offset;
  1939. after->offset += after->start - tmp->start;
  1940. after->flags = tmp->flags;
  1941. list_add(&after->list, &tmp->list);
  1942. }
  1943. list_del(&tmp->list);
  1944. list_add(&tmp->list, &removed);
  1945. tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
  1946. }
  1947. /* And free them up */
  1948. list_for_each_entry_safe(tmp, next, &removed, list) {
  1949. amdgpu_vm_it_remove(tmp, &vm->va);
  1950. list_del(&tmp->list);
  1951. if (tmp->start < saddr)
  1952. tmp->start = saddr;
  1953. if (tmp->last > eaddr)
  1954. tmp->last = eaddr;
  1955. list_add(&tmp->list, &vm->freed);
  1956. trace_amdgpu_vm_bo_unmap(NULL, tmp);
  1957. }
  1958. /* Insert partial mapping before the range */
  1959. if (!list_empty(&before->list)) {
  1960. amdgpu_vm_it_insert(before, &vm->va);
  1961. if (before->flags & AMDGPU_PTE_PRT)
  1962. amdgpu_vm_prt_get(adev);
  1963. } else {
  1964. kfree(before);
  1965. }
  1966. /* Insert partial mapping after the range */
  1967. if (!list_empty(&after->list)) {
  1968. amdgpu_vm_it_insert(after, &vm->va);
  1969. if (after->flags & AMDGPU_PTE_PRT)
  1970. amdgpu_vm_prt_get(adev);
  1971. } else {
  1972. kfree(after);
  1973. }
  1974. return 0;
  1975. }
  1976. /**
  1977. * amdgpu_vm_bo_rmv - remove a bo to a specific vm
  1978. *
  1979. * @adev: amdgpu_device pointer
  1980. * @bo_va: requested bo_va
  1981. *
  1982. * Remove @bo_va->bo from the requested vm.
  1983. *
  1984. * Object have to be reserved!
  1985. */
  1986. void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
  1987. struct amdgpu_bo_va *bo_va)
  1988. {
  1989. struct amdgpu_bo_va_mapping *mapping, *next;
  1990. struct amdgpu_vm *vm = bo_va->vm;
  1991. list_del(&bo_va->bo_list);
  1992. spin_lock(&vm->status_lock);
  1993. list_del(&bo_va->vm_status);
  1994. spin_unlock(&vm->status_lock);
  1995. list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
  1996. list_del(&mapping->list);
  1997. amdgpu_vm_it_remove(mapping, &vm->va);
  1998. trace_amdgpu_vm_bo_unmap(bo_va, mapping);
  1999. list_add(&mapping->list, &vm->freed);
  2000. }
  2001. list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
  2002. list_del(&mapping->list);
  2003. amdgpu_vm_it_remove(mapping, &vm->va);
  2004. amdgpu_vm_free_mapping(adev, vm, mapping,
  2005. bo_va->last_pt_update);
  2006. }
  2007. dma_fence_put(bo_va->last_pt_update);
  2008. kfree(bo_va);
  2009. }
  2010. /**
  2011. * amdgpu_vm_bo_invalidate - mark the bo as invalid
  2012. *
  2013. * @adev: amdgpu_device pointer
  2014. * @vm: requested vm
  2015. * @bo: amdgpu buffer object
  2016. *
  2017. * Mark @bo as invalid.
  2018. */
  2019. void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
  2020. struct amdgpu_bo *bo)
  2021. {
  2022. struct amdgpu_bo_va *bo_va;
  2023. list_for_each_entry(bo_va, &bo->va, bo_list) {
  2024. spin_lock(&bo_va->vm->status_lock);
  2025. if (list_empty(&bo_va->vm_status))
  2026. list_add(&bo_va->vm_status, &bo_va->vm->invalidated);
  2027. spin_unlock(&bo_va->vm->status_lock);
  2028. }
  2029. }
  2030. static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
  2031. {
  2032. /* Total bits covered by PD + PTs */
  2033. unsigned bits = ilog2(vm_size) + 18;
  2034. /* Make sure the PD is 4K in size up to 8GB address space.
  2035. Above that split equal between PD and PTs */
  2036. if (vm_size <= 8)
  2037. return (bits - 9);
  2038. else
  2039. return ((bits + 3) / 2);
  2040. }
  2041. /**
  2042. * amdgpu_vm_adjust_size - adjust vm size and block size
  2043. *
  2044. * @adev: amdgpu_device pointer
  2045. * @vm_size: the default vm size if it's set auto
  2046. */
  2047. void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size)
  2048. {
  2049. /* adjust vm size firstly */
  2050. if (amdgpu_vm_size == -1)
  2051. adev->vm_manager.vm_size = vm_size;
  2052. else
  2053. adev->vm_manager.vm_size = amdgpu_vm_size;
  2054. /* block size depends on vm size */
  2055. if (amdgpu_vm_block_size == -1)
  2056. adev->vm_manager.block_size =
  2057. amdgpu_vm_get_block_size(adev->vm_manager.vm_size);
  2058. else
  2059. adev->vm_manager.block_size = amdgpu_vm_block_size;
  2060. DRM_INFO("vm size is %llu GB, block size is %u-bit\n",
  2061. adev->vm_manager.vm_size, adev->vm_manager.block_size);
  2062. }
  2063. /**
  2064. * amdgpu_vm_init - initialize a vm instance
  2065. *
  2066. * @adev: amdgpu_device pointer
  2067. * @vm: requested vm
  2068. * @vm_context: Indicates if it GFX or Compute context
  2069. *
  2070. * Init @vm fields.
  2071. */
  2072. int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  2073. int vm_context)
  2074. {
  2075. const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
  2076. AMDGPU_VM_PTE_COUNT(adev) * 8);
  2077. unsigned ring_instance;
  2078. struct amdgpu_ring *ring;
  2079. struct amd_sched_rq *rq;
  2080. int r, i;
  2081. u64 flags;
  2082. vm->va = RB_ROOT;
  2083. vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter);
  2084. for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
  2085. vm->reserved_vmid[i] = NULL;
  2086. spin_lock_init(&vm->status_lock);
  2087. INIT_LIST_HEAD(&vm->invalidated);
  2088. INIT_LIST_HEAD(&vm->cleared);
  2089. INIT_LIST_HEAD(&vm->freed);
  2090. /* create scheduler entity for page table updates */
  2091. ring_instance = atomic_inc_return(&adev->vm_manager.vm_pte_next_ring);
  2092. ring_instance %= adev->vm_manager.vm_pte_num_rings;
  2093. ring = adev->vm_manager.vm_pte_rings[ring_instance];
  2094. rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
  2095. r = amd_sched_entity_init(&ring->sched, &vm->entity,
  2096. rq, amdgpu_sched_jobs);
  2097. if (r)
  2098. return r;
  2099. if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE)
  2100. vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
  2101. AMDGPU_VM_USE_CPU_FOR_COMPUTE);
  2102. else
  2103. vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
  2104. AMDGPU_VM_USE_CPU_FOR_GFX);
  2105. DRM_DEBUG_DRIVER("VM update mode is %s\n",
  2106. vm->use_cpu_for_update ? "CPU" : "SDMA");
  2107. WARN_ONCE((vm->use_cpu_for_update & !amdgpu_vm_is_large_bar(adev)),
  2108. "CPU update of VM recommended only for large BAR system\n");
  2109. vm->last_dir_update = NULL;
  2110. flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
  2111. AMDGPU_GEM_CREATE_VRAM_CLEARED;
  2112. if (vm->use_cpu_for_update)
  2113. flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
  2114. else
  2115. flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
  2116. AMDGPU_GEM_CREATE_SHADOW);
  2117. r = amdgpu_bo_create(adev, amdgpu_vm_bo_size(adev, 0), align, true,
  2118. AMDGPU_GEM_DOMAIN_VRAM,
  2119. flags,
  2120. NULL, NULL, &vm->root.bo);
  2121. if (r)
  2122. goto error_free_sched_entity;
  2123. r = amdgpu_bo_reserve(vm->root.bo, false);
  2124. if (r)
  2125. goto error_free_root;
  2126. vm->last_eviction_counter = atomic64_read(&adev->num_evictions);
  2127. if (vm->use_cpu_for_update) {
  2128. r = amdgpu_bo_kmap(vm->root.bo, NULL);
  2129. if (r)
  2130. goto error_free_root;
  2131. }
  2132. amdgpu_bo_unreserve(vm->root.bo);
  2133. return 0;
  2134. error_free_root:
  2135. amdgpu_bo_unref(&vm->root.bo->shadow);
  2136. amdgpu_bo_unref(&vm->root.bo);
  2137. vm->root.bo = NULL;
  2138. error_free_sched_entity:
  2139. amd_sched_entity_fini(&ring->sched, &vm->entity);
  2140. return r;
  2141. }
  2142. /**
  2143. * amdgpu_vm_free_levels - free PD/PT levels
  2144. *
  2145. * @level: PD/PT starting level to free
  2146. *
  2147. * Free the page directory or page table level and all sub levels.
  2148. */
  2149. static void amdgpu_vm_free_levels(struct amdgpu_vm_pt *level)
  2150. {
  2151. unsigned i;
  2152. if (level->bo) {
  2153. amdgpu_bo_unref(&level->bo->shadow);
  2154. amdgpu_bo_unref(&level->bo);
  2155. }
  2156. if (level->entries)
  2157. for (i = 0; i <= level->last_entry_used; i++)
  2158. amdgpu_vm_free_levels(&level->entries[i]);
  2159. kvfree(level->entries);
  2160. }
  2161. /**
  2162. * amdgpu_vm_fini - tear down a vm instance
  2163. *
  2164. * @adev: amdgpu_device pointer
  2165. * @vm: requested vm
  2166. *
  2167. * Tear down @vm.
  2168. * Unbind the VM and remove all bos from the vm bo list
  2169. */
  2170. void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
  2171. {
  2172. struct amdgpu_bo_va_mapping *mapping, *tmp;
  2173. bool prt_fini_needed = !!adev->gart.gart_funcs->set_prt;
  2174. int i;
  2175. amd_sched_entity_fini(vm->entity.sched, &vm->entity);
  2176. if (!RB_EMPTY_ROOT(&vm->va)) {
  2177. dev_err(adev->dev, "still active bo inside vm\n");
  2178. }
  2179. rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, rb) {
  2180. list_del(&mapping->list);
  2181. amdgpu_vm_it_remove(mapping, &vm->va);
  2182. kfree(mapping);
  2183. }
  2184. list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
  2185. if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
  2186. amdgpu_vm_prt_fini(adev, vm);
  2187. prt_fini_needed = false;
  2188. }
  2189. list_del(&mapping->list);
  2190. amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
  2191. }
  2192. amdgpu_vm_free_levels(&vm->root);
  2193. dma_fence_put(vm->last_dir_update);
  2194. for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
  2195. amdgpu_vm_free_reserved_vmid(adev, vm, i);
  2196. }
  2197. /**
  2198. * amdgpu_vm_manager_init - init the VM manager
  2199. *
  2200. * @adev: amdgpu_device pointer
  2201. *
  2202. * Initialize the VM manager structures
  2203. */
  2204. void amdgpu_vm_manager_init(struct amdgpu_device *adev)
  2205. {
  2206. unsigned i, j;
  2207. for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
  2208. struct amdgpu_vm_id_manager *id_mgr =
  2209. &adev->vm_manager.id_mgr[i];
  2210. mutex_init(&id_mgr->lock);
  2211. INIT_LIST_HEAD(&id_mgr->ids_lru);
  2212. atomic_set(&id_mgr->reserved_vmid_num, 0);
  2213. /* skip over VMID 0, since it is the system VM */
  2214. for (j = 1; j < id_mgr->num_ids; ++j) {
  2215. amdgpu_vm_reset_id(adev, i, j);
  2216. amdgpu_sync_create(&id_mgr->ids[i].active);
  2217. list_add_tail(&id_mgr->ids[j].list, &id_mgr->ids_lru);
  2218. }
  2219. }
  2220. adev->vm_manager.fence_context =
  2221. dma_fence_context_alloc(AMDGPU_MAX_RINGS);
  2222. for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
  2223. adev->vm_manager.seqno[i] = 0;
  2224. atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
  2225. atomic64_set(&adev->vm_manager.client_counter, 0);
  2226. spin_lock_init(&adev->vm_manager.prt_lock);
  2227. atomic_set(&adev->vm_manager.num_prt_users, 0);
  2228. /* If not overridden by the user, by default, only in large BAR systems
  2229. * Compute VM tables will be updated by CPU
  2230. */
  2231. #ifdef CONFIG_X86_64
  2232. if (amdgpu_vm_update_mode == -1) {
  2233. if (amdgpu_vm_is_large_bar(adev))
  2234. adev->vm_manager.vm_update_mode =
  2235. AMDGPU_VM_USE_CPU_FOR_COMPUTE;
  2236. else
  2237. adev->vm_manager.vm_update_mode = 0;
  2238. } else
  2239. adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
  2240. #else
  2241. adev->vm_manager.vm_update_mode = 0;
  2242. #endif
  2243. }
  2244. /**
  2245. * amdgpu_vm_manager_fini - cleanup VM manager
  2246. *
  2247. * @adev: amdgpu_device pointer
  2248. *
  2249. * Cleanup the VM manager and free resources.
  2250. */
  2251. void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
  2252. {
  2253. unsigned i, j;
  2254. for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
  2255. struct amdgpu_vm_id_manager *id_mgr =
  2256. &adev->vm_manager.id_mgr[i];
  2257. mutex_destroy(&id_mgr->lock);
  2258. for (j = 0; j < AMDGPU_NUM_VM; ++j) {
  2259. struct amdgpu_vm_id *id = &id_mgr->ids[j];
  2260. amdgpu_sync_free(&id->active);
  2261. dma_fence_put(id->flushed_updates);
  2262. dma_fence_put(id->last_flush);
  2263. }
  2264. }
  2265. }
  2266. int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
  2267. {
  2268. union drm_amdgpu_vm *args = data;
  2269. struct amdgpu_device *adev = dev->dev_private;
  2270. struct amdgpu_fpriv *fpriv = filp->driver_priv;
  2271. int r;
  2272. switch (args->in.op) {
  2273. case AMDGPU_VM_OP_RESERVE_VMID:
  2274. /* current, we only have requirement to reserve vmid from gfxhub */
  2275. r = amdgpu_vm_alloc_reserved_vmid(adev, &fpriv->vm,
  2276. AMDGPU_GFXHUB);
  2277. if (r)
  2278. return r;
  2279. break;
  2280. case AMDGPU_VM_OP_UNRESERVE_VMID:
  2281. amdgpu_vm_free_reserved_vmid(adev, &fpriv->vm, AMDGPU_GFXHUB);
  2282. break;
  2283. default:
  2284. return -EINVAL;
  2285. }
  2286. return 0;
  2287. }