amdgpu_vm.c 70 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741
  1. /*
  2. * Copyright 2008 Advanced Micro Devices, Inc.
  3. * Copyright 2008 Red Hat Inc.
  4. * Copyright 2009 Jerome Glisse.
  5. *
  6. * Permission is hereby granted, free of charge, to any person obtaining a
  7. * copy of this software and associated documentation files (the "Software"),
  8. * to deal in the Software without restriction, including without limitation
  9. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10. * and/or sell copies of the Software, and to permit persons to whom the
  11. * Software is furnished to do so, subject to the following conditions:
  12. *
  13. * The above copyright notice and this permission notice shall be included in
  14. * all copies or substantial portions of the Software.
  15. *
  16. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19. * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22. * OTHER DEALINGS IN THE SOFTWARE.
  23. *
  24. * Authors: Dave Airlie
  25. * Alex Deucher
  26. * Jerome Glisse
  27. */
  28. #include <linux/dma-fence-array.h>
  29. #include <linux/interval_tree_generic.h>
  30. #include <drm/drmP.h>
  31. #include <drm/amdgpu_drm.h>
  32. #include "amdgpu.h"
  33. #include "amdgpu_trace.h"
  34. /*
  35. * GPUVM
  36. * GPUVM is similar to the legacy gart on older asics, however
  37. * rather than there being a single global gart table
  38. * for the entire GPU, there are multiple VM page tables active
  39. * at any given time. The VM page tables can contain a mix
  40. * vram pages and system memory pages and system memory pages
  41. * can be mapped as snooped (cached system pages) or unsnooped
  42. * (uncached system pages).
  43. * Each VM has an ID associated with it and there is a page table
  44. * associated with each VMID. When execting a command buffer,
  45. * the kernel tells the the ring what VMID to use for that command
  46. * buffer. VMIDs are allocated dynamically as commands are submitted.
  47. * The userspace drivers maintain their own address space and the kernel
  48. * sets up their pages tables accordingly when they submit their
  49. * command buffers and a VMID is assigned.
  50. * Cayman/Trinity support up to 8 active VMs at any given time;
  51. * SI supports 16.
  52. */
  53. #define START(node) ((node)->start)
  54. #define LAST(node) ((node)->last)
  55. INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
  56. START, LAST, static, amdgpu_vm_it)
  57. #undef START
  58. #undef LAST
  59. /* Local structure. Encapsulate some VM table update parameters to reduce
  60. * the number of function parameters
  61. */
  62. struct amdgpu_pte_update_params {
  63. /* amdgpu device we do this update for */
  64. struct amdgpu_device *adev;
  65. /* optional amdgpu_vm we do this update for */
  66. struct amdgpu_vm *vm;
  67. /* address where to copy page table entries from */
  68. uint64_t src;
  69. /* indirect buffer to fill with commands */
  70. struct amdgpu_ib *ib;
  71. /* Function which actually does the update */
  72. void (*func)(struct amdgpu_pte_update_params *params, uint64_t pe,
  73. uint64_t addr, unsigned count, uint32_t incr,
  74. uint64_t flags);
  75. /* The next two are used during VM update by CPU
  76. * DMA addresses to use for mapping
  77. * Kernel pointer of PD/PT BO that needs to be updated
  78. */
  79. dma_addr_t *pages_addr;
  80. void *kptr;
  81. };
  82. /* Helper to disable partial resident texture feature from a fence callback */
  83. struct amdgpu_prt_cb {
  84. struct amdgpu_device *adev;
  85. struct dma_fence_cb cb;
  86. };
  87. /**
  88. * amdgpu_vm_num_entries - return the number of entries in a PD/PT
  89. *
  90. * @adev: amdgpu_device pointer
  91. *
  92. * Calculate the number of entries in a page directory or page table.
  93. */
  94. static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
  95. unsigned level)
  96. {
  97. if (level == 0)
  98. /* For the root directory */
  99. return adev->vm_manager.max_pfn >>
  100. (adev->vm_manager.block_size *
  101. adev->vm_manager.num_level);
  102. else if (level == adev->vm_manager.num_level)
  103. /* For the page tables on the leaves */
  104. return AMDGPU_VM_PTE_COUNT(adev);
  105. else
  106. /* Everything in between */
  107. return 1 << adev->vm_manager.block_size;
  108. }
  109. /**
  110. * amdgpu_vm_bo_size - returns the size of the BOs in bytes
  111. *
  112. * @adev: amdgpu_device pointer
  113. *
  114. * Calculate the size of the BO for a page directory or page table in bytes.
  115. */
  116. static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level)
  117. {
  118. return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8);
  119. }
  120. /**
  121. * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
  122. *
  123. * @vm: vm providing the BOs
  124. * @validated: head of validation list
  125. * @entry: entry to add
  126. *
  127. * Add the page directory to the list of BOs to
  128. * validate for command submission.
  129. */
  130. void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
  131. struct list_head *validated,
  132. struct amdgpu_bo_list_entry *entry)
  133. {
  134. entry->robj = vm->root.bo;
  135. entry->priority = 0;
  136. entry->tv.bo = &entry->robj->tbo;
  137. entry->tv.shared = true;
  138. entry->user_pages = NULL;
  139. list_add(&entry->tv.head, validated);
  140. }
  141. /**
  142. * amdgpu_vm_validate_layer - validate a single page table level
  143. *
  144. * @parent: parent page table level
  145. * @validate: callback to do the validation
  146. * @param: parameter for the validation callback
  147. *
  148. * Validate the page table BOs on command submission if neccessary.
  149. */
  150. static int amdgpu_vm_validate_level(struct amdgpu_vm_pt *parent,
  151. int (*validate)(void *, struct amdgpu_bo *),
  152. void *param, bool use_cpu_for_update,
  153. struct ttm_bo_global *glob)
  154. {
  155. unsigned i;
  156. int r;
  157. if (parent->bo->shadow) {
  158. struct amdgpu_bo *shadow = parent->bo->shadow;
  159. r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
  160. if (r)
  161. return r;
  162. }
  163. if (use_cpu_for_update) {
  164. r = amdgpu_bo_kmap(parent->bo, NULL);
  165. if (r)
  166. return r;
  167. }
  168. if (!parent->entries)
  169. return 0;
  170. for (i = 0; i <= parent->last_entry_used; ++i) {
  171. struct amdgpu_vm_pt *entry = &parent->entries[i];
  172. if (!entry->bo)
  173. continue;
  174. r = validate(param, entry->bo);
  175. if (r)
  176. return r;
  177. spin_lock(&glob->lru_lock);
  178. ttm_bo_move_to_lru_tail(&entry->bo->tbo);
  179. if (entry->bo->shadow)
  180. ttm_bo_move_to_lru_tail(&entry->bo->shadow->tbo);
  181. spin_unlock(&glob->lru_lock);
  182. /*
  183. * Recurse into the sub directory. This is harmless because we
  184. * have only a maximum of 5 layers.
  185. */
  186. r = amdgpu_vm_validate_level(entry, validate, param,
  187. use_cpu_for_update, glob);
  188. if (r)
  189. return r;
  190. }
  191. return r;
  192. }
  193. /**
  194. * amdgpu_vm_validate_pt_bos - validate the page table BOs
  195. *
  196. * @adev: amdgpu device pointer
  197. * @vm: vm providing the BOs
  198. * @validate: callback to do the validation
  199. * @param: parameter for the validation callback
  200. *
  201. * Validate the page table BOs on command submission if neccessary.
  202. */
  203. int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  204. int (*validate)(void *p, struct amdgpu_bo *bo),
  205. void *param)
  206. {
  207. uint64_t num_evictions;
  208. /* We only need to validate the page tables
  209. * if they aren't already valid.
  210. */
  211. num_evictions = atomic64_read(&adev->num_evictions);
  212. if (num_evictions == vm->last_eviction_counter)
  213. return 0;
  214. return amdgpu_vm_validate_level(&vm->root, validate, param,
  215. vm->use_cpu_for_update,
  216. adev->mman.bdev.glob);
  217. }
  218. /**
  219. * amdgpu_vm_alloc_levels - allocate the PD/PT levels
  220. *
  221. * @adev: amdgpu_device pointer
  222. * @vm: requested vm
  223. * @saddr: start of the address range
  224. * @eaddr: end of the address range
  225. *
  226. * Make sure the page directories and page tables are allocated
  227. */
  228. static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
  229. struct amdgpu_vm *vm,
  230. struct amdgpu_vm_pt *parent,
  231. uint64_t saddr, uint64_t eaddr,
  232. unsigned level)
  233. {
  234. unsigned shift = (adev->vm_manager.num_level - level) *
  235. adev->vm_manager.block_size;
  236. unsigned pt_idx, from, to;
  237. int r;
  238. u64 flags;
  239. uint64_t init_value = 0;
  240. if (!parent->entries) {
  241. unsigned num_entries = amdgpu_vm_num_entries(adev, level);
  242. parent->entries = kvmalloc_array(num_entries,
  243. sizeof(struct amdgpu_vm_pt),
  244. GFP_KERNEL | __GFP_ZERO);
  245. if (!parent->entries)
  246. return -ENOMEM;
  247. memset(parent->entries, 0 , sizeof(struct amdgpu_vm_pt));
  248. }
  249. from = saddr >> shift;
  250. to = eaddr >> shift;
  251. if (from >= amdgpu_vm_num_entries(adev, level) ||
  252. to >= amdgpu_vm_num_entries(adev, level))
  253. return -EINVAL;
  254. if (to > parent->last_entry_used)
  255. parent->last_entry_used = to;
  256. ++level;
  257. saddr = saddr & ((1 << shift) - 1);
  258. eaddr = eaddr & ((1 << shift) - 1);
  259. flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
  260. AMDGPU_GEM_CREATE_VRAM_CLEARED;
  261. if (vm->use_cpu_for_update)
  262. flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
  263. else
  264. flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
  265. AMDGPU_GEM_CREATE_SHADOW);
  266. if (vm->pte_support_ats) {
  267. init_value = AMDGPU_PTE_SYSTEM;
  268. if (level != adev->vm_manager.num_level - 1)
  269. init_value |= AMDGPU_PDE_PTE;
  270. }
  271. /* walk over the address space and allocate the page tables */
  272. for (pt_idx = from; pt_idx <= to; ++pt_idx) {
  273. struct reservation_object *resv = vm->root.bo->tbo.resv;
  274. struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
  275. struct amdgpu_bo *pt;
  276. if (!entry->bo) {
  277. r = amdgpu_bo_create(adev,
  278. amdgpu_vm_bo_size(adev, level),
  279. AMDGPU_GPU_PAGE_SIZE, true,
  280. AMDGPU_GEM_DOMAIN_VRAM,
  281. flags,
  282. NULL, resv, init_value, &pt);
  283. if (r)
  284. return r;
  285. if (vm->use_cpu_for_update) {
  286. r = amdgpu_bo_kmap(pt, NULL);
  287. if (r) {
  288. amdgpu_bo_unref(&pt);
  289. return r;
  290. }
  291. }
  292. /* Keep a reference to the root directory to avoid
  293. * freeing them up in the wrong order.
  294. */
  295. pt->parent = amdgpu_bo_ref(vm->root.bo);
  296. entry->bo = pt;
  297. entry->addr = 0;
  298. }
  299. if (level < adev->vm_manager.num_level) {
  300. uint64_t sub_saddr = (pt_idx == from) ? saddr : 0;
  301. uint64_t sub_eaddr = (pt_idx == to) ? eaddr :
  302. ((1 << shift) - 1);
  303. r = amdgpu_vm_alloc_levels(adev, vm, entry, sub_saddr,
  304. sub_eaddr, level);
  305. if (r)
  306. return r;
  307. }
  308. }
  309. return 0;
  310. }
  311. /**
  312. * amdgpu_vm_alloc_pts - Allocate page tables.
  313. *
  314. * @adev: amdgpu_device pointer
  315. * @vm: VM to allocate page tables for
  316. * @saddr: Start address which needs to be allocated
  317. * @size: Size from start address we need.
  318. *
  319. * Make sure the page tables are allocated.
  320. */
  321. int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
  322. struct amdgpu_vm *vm,
  323. uint64_t saddr, uint64_t size)
  324. {
  325. uint64_t last_pfn;
  326. uint64_t eaddr;
  327. /* validate the parameters */
  328. if (saddr & AMDGPU_GPU_PAGE_MASK || size & AMDGPU_GPU_PAGE_MASK)
  329. return -EINVAL;
  330. eaddr = saddr + size - 1;
  331. last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE;
  332. if (last_pfn >= adev->vm_manager.max_pfn) {
  333. dev_err(adev->dev, "va above limit (0x%08llX >= 0x%08llX)\n",
  334. last_pfn, adev->vm_manager.max_pfn);
  335. return -EINVAL;
  336. }
  337. saddr /= AMDGPU_GPU_PAGE_SIZE;
  338. eaddr /= AMDGPU_GPU_PAGE_SIZE;
  339. return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr, 0);
  340. }
  341. /**
  342. * amdgpu_vm_had_gpu_reset - check if reset occured since last use
  343. *
  344. * @adev: amdgpu_device pointer
  345. * @id: VMID structure
  346. *
  347. * Check if GPU reset occured since last use of the VMID.
  348. */
  349. static bool amdgpu_vm_had_gpu_reset(struct amdgpu_device *adev,
  350. struct amdgpu_vm_id *id)
  351. {
  352. return id->current_gpu_reset_count !=
  353. atomic_read(&adev->gpu_reset_counter);
  354. }
  355. static bool amdgpu_vm_reserved_vmid_ready(struct amdgpu_vm *vm, unsigned vmhub)
  356. {
  357. return !!vm->reserved_vmid[vmhub];
  358. }
  359. /* idr_mgr->lock must be held */
  360. static int amdgpu_vm_grab_reserved_vmid_locked(struct amdgpu_vm *vm,
  361. struct amdgpu_ring *ring,
  362. struct amdgpu_sync *sync,
  363. struct dma_fence *fence,
  364. struct amdgpu_job *job)
  365. {
  366. struct amdgpu_device *adev = ring->adev;
  367. unsigned vmhub = ring->funcs->vmhub;
  368. uint64_t fence_context = adev->fence_context + ring->idx;
  369. struct amdgpu_vm_id *id = vm->reserved_vmid[vmhub];
  370. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  371. struct dma_fence *updates = sync->last_vm_update;
  372. int r = 0;
  373. struct dma_fence *flushed, *tmp;
  374. bool needs_flush = vm->use_cpu_for_update;
  375. flushed = id->flushed_updates;
  376. if ((amdgpu_vm_had_gpu_reset(adev, id)) ||
  377. (atomic64_read(&id->owner) != vm->client_id) ||
  378. (job->vm_pd_addr != id->pd_gpu_addr) ||
  379. (updates && (!flushed || updates->context != flushed->context ||
  380. dma_fence_is_later(updates, flushed))) ||
  381. (!id->last_flush || (id->last_flush->context != fence_context &&
  382. !dma_fence_is_signaled(id->last_flush)))) {
  383. needs_flush = true;
  384. /* to prevent one context starved by another context */
  385. id->pd_gpu_addr = 0;
  386. tmp = amdgpu_sync_peek_fence(&id->active, ring);
  387. if (tmp) {
  388. r = amdgpu_sync_fence(adev, sync, tmp);
  389. return r;
  390. }
  391. }
  392. /* Good we can use this VMID. Remember this submission as
  393. * user of the VMID.
  394. */
  395. r = amdgpu_sync_fence(ring->adev, &id->active, fence);
  396. if (r)
  397. goto out;
  398. if (updates && (!flushed || updates->context != flushed->context ||
  399. dma_fence_is_later(updates, flushed))) {
  400. dma_fence_put(id->flushed_updates);
  401. id->flushed_updates = dma_fence_get(updates);
  402. }
  403. id->pd_gpu_addr = job->vm_pd_addr;
  404. atomic64_set(&id->owner, vm->client_id);
  405. job->vm_needs_flush = needs_flush;
  406. if (needs_flush) {
  407. dma_fence_put(id->last_flush);
  408. id->last_flush = NULL;
  409. }
  410. job->vm_id = id - id_mgr->ids;
  411. trace_amdgpu_vm_grab_id(vm, ring, job);
  412. out:
  413. return r;
  414. }
  415. /**
  416. * amdgpu_vm_grab_id - allocate the next free VMID
  417. *
  418. * @vm: vm to allocate id for
  419. * @ring: ring we want to submit job to
  420. * @sync: sync object where we add dependencies
  421. * @fence: fence protecting ID from reuse
  422. *
  423. * Allocate an id for the vm, adding fences to the sync obj as necessary.
  424. */
  425. int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
  426. struct amdgpu_sync *sync, struct dma_fence *fence,
  427. struct amdgpu_job *job)
  428. {
  429. struct amdgpu_device *adev = ring->adev;
  430. unsigned vmhub = ring->funcs->vmhub;
  431. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  432. uint64_t fence_context = adev->fence_context + ring->idx;
  433. struct dma_fence *updates = sync->last_vm_update;
  434. struct amdgpu_vm_id *id, *idle;
  435. struct dma_fence **fences;
  436. unsigned i;
  437. int r = 0;
  438. mutex_lock(&id_mgr->lock);
  439. if (amdgpu_vm_reserved_vmid_ready(vm, vmhub)) {
  440. r = amdgpu_vm_grab_reserved_vmid_locked(vm, ring, sync, fence, job);
  441. mutex_unlock(&id_mgr->lock);
  442. return r;
  443. }
  444. fences = kmalloc_array(sizeof(void *), id_mgr->num_ids, GFP_KERNEL);
  445. if (!fences) {
  446. mutex_unlock(&id_mgr->lock);
  447. return -ENOMEM;
  448. }
  449. /* Check if we have an idle VMID */
  450. i = 0;
  451. list_for_each_entry(idle, &id_mgr->ids_lru, list) {
  452. fences[i] = amdgpu_sync_peek_fence(&idle->active, ring);
  453. if (!fences[i])
  454. break;
  455. ++i;
  456. }
  457. /* If we can't find a idle VMID to use, wait till one becomes available */
  458. if (&idle->list == &id_mgr->ids_lru) {
  459. u64 fence_context = adev->vm_manager.fence_context + ring->idx;
  460. unsigned seqno = ++adev->vm_manager.seqno[ring->idx];
  461. struct dma_fence_array *array;
  462. unsigned j;
  463. for (j = 0; j < i; ++j)
  464. dma_fence_get(fences[j]);
  465. array = dma_fence_array_create(i, fences, fence_context,
  466. seqno, true);
  467. if (!array) {
  468. for (j = 0; j < i; ++j)
  469. dma_fence_put(fences[j]);
  470. kfree(fences);
  471. r = -ENOMEM;
  472. goto error;
  473. }
  474. r = amdgpu_sync_fence(ring->adev, sync, &array->base);
  475. dma_fence_put(&array->base);
  476. if (r)
  477. goto error;
  478. mutex_unlock(&id_mgr->lock);
  479. return 0;
  480. }
  481. kfree(fences);
  482. job->vm_needs_flush = vm->use_cpu_for_update;
  483. /* Check if we can use a VMID already assigned to this VM */
  484. list_for_each_entry_reverse(id, &id_mgr->ids_lru, list) {
  485. struct dma_fence *flushed;
  486. bool needs_flush = vm->use_cpu_for_update;
  487. /* Check all the prerequisites to using this VMID */
  488. if (amdgpu_vm_had_gpu_reset(adev, id))
  489. continue;
  490. if (atomic64_read(&id->owner) != vm->client_id)
  491. continue;
  492. if (job->vm_pd_addr != id->pd_gpu_addr)
  493. continue;
  494. if (!id->last_flush ||
  495. (id->last_flush->context != fence_context &&
  496. !dma_fence_is_signaled(id->last_flush)))
  497. needs_flush = true;
  498. flushed = id->flushed_updates;
  499. if (updates && (!flushed || dma_fence_is_later(updates, flushed)))
  500. needs_flush = true;
  501. /* Concurrent flushes are only possible starting with Vega10 */
  502. if (adev->asic_type < CHIP_VEGA10 && needs_flush)
  503. continue;
  504. /* Good we can use this VMID. Remember this submission as
  505. * user of the VMID.
  506. */
  507. r = amdgpu_sync_fence(ring->adev, &id->active, fence);
  508. if (r)
  509. goto error;
  510. if (updates && (!flushed || dma_fence_is_later(updates, flushed))) {
  511. dma_fence_put(id->flushed_updates);
  512. id->flushed_updates = dma_fence_get(updates);
  513. }
  514. if (needs_flush)
  515. goto needs_flush;
  516. else
  517. goto no_flush_needed;
  518. };
  519. /* Still no ID to use? Then use the idle one found earlier */
  520. id = idle;
  521. /* Remember this submission as user of the VMID */
  522. r = amdgpu_sync_fence(ring->adev, &id->active, fence);
  523. if (r)
  524. goto error;
  525. id->pd_gpu_addr = job->vm_pd_addr;
  526. dma_fence_put(id->flushed_updates);
  527. id->flushed_updates = dma_fence_get(updates);
  528. atomic64_set(&id->owner, vm->client_id);
  529. needs_flush:
  530. job->vm_needs_flush = true;
  531. dma_fence_put(id->last_flush);
  532. id->last_flush = NULL;
  533. no_flush_needed:
  534. list_move_tail(&id->list, &id_mgr->ids_lru);
  535. job->vm_id = id - id_mgr->ids;
  536. trace_amdgpu_vm_grab_id(vm, ring, job);
  537. error:
  538. mutex_unlock(&id_mgr->lock);
  539. return r;
  540. }
  541. static void amdgpu_vm_free_reserved_vmid(struct amdgpu_device *adev,
  542. struct amdgpu_vm *vm,
  543. unsigned vmhub)
  544. {
  545. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  546. mutex_lock(&id_mgr->lock);
  547. if (vm->reserved_vmid[vmhub]) {
  548. list_add(&vm->reserved_vmid[vmhub]->list,
  549. &id_mgr->ids_lru);
  550. vm->reserved_vmid[vmhub] = NULL;
  551. atomic_dec(&id_mgr->reserved_vmid_num);
  552. }
  553. mutex_unlock(&id_mgr->lock);
  554. }
  555. static int amdgpu_vm_alloc_reserved_vmid(struct amdgpu_device *adev,
  556. struct amdgpu_vm *vm,
  557. unsigned vmhub)
  558. {
  559. struct amdgpu_vm_id_manager *id_mgr;
  560. struct amdgpu_vm_id *idle;
  561. int r = 0;
  562. id_mgr = &adev->vm_manager.id_mgr[vmhub];
  563. mutex_lock(&id_mgr->lock);
  564. if (vm->reserved_vmid[vmhub])
  565. goto unlock;
  566. if (atomic_inc_return(&id_mgr->reserved_vmid_num) >
  567. AMDGPU_VM_MAX_RESERVED_VMID) {
  568. DRM_ERROR("Over limitation of reserved vmid\n");
  569. atomic_dec(&id_mgr->reserved_vmid_num);
  570. r = -EINVAL;
  571. goto unlock;
  572. }
  573. /* Select the first entry VMID */
  574. idle = list_first_entry(&id_mgr->ids_lru, struct amdgpu_vm_id, list);
  575. list_del_init(&idle->list);
  576. vm->reserved_vmid[vmhub] = idle;
  577. mutex_unlock(&id_mgr->lock);
  578. return 0;
  579. unlock:
  580. mutex_unlock(&id_mgr->lock);
  581. return r;
  582. }
  583. /**
  584. * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
  585. *
  586. * @adev: amdgpu_device pointer
  587. */
  588. void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
  589. {
  590. const struct amdgpu_ip_block *ip_block;
  591. bool has_compute_vm_bug;
  592. struct amdgpu_ring *ring;
  593. int i;
  594. has_compute_vm_bug = false;
  595. ip_block = amdgpu_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
  596. if (ip_block) {
  597. /* Compute has a VM bug for GFX version < 7.
  598. Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
  599. if (ip_block->version->major <= 7)
  600. has_compute_vm_bug = true;
  601. else if (ip_block->version->major == 8)
  602. if (adev->gfx.mec_fw_version < 673)
  603. has_compute_vm_bug = true;
  604. }
  605. for (i = 0; i < adev->num_rings; i++) {
  606. ring = adev->rings[i];
  607. if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
  608. /* only compute rings */
  609. ring->has_compute_vm_bug = has_compute_vm_bug;
  610. else
  611. ring->has_compute_vm_bug = false;
  612. }
  613. }
  614. bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
  615. struct amdgpu_job *job)
  616. {
  617. struct amdgpu_device *adev = ring->adev;
  618. unsigned vmhub = ring->funcs->vmhub;
  619. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  620. struct amdgpu_vm_id *id;
  621. bool gds_switch_needed;
  622. bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug;
  623. if (job->vm_id == 0)
  624. return false;
  625. id = &id_mgr->ids[job->vm_id];
  626. gds_switch_needed = ring->funcs->emit_gds_switch && (
  627. id->gds_base != job->gds_base ||
  628. id->gds_size != job->gds_size ||
  629. id->gws_base != job->gws_base ||
  630. id->gws_size != job->gws_size ||
  631. id->oa_base != job->oa_base ||
  632. id->oa_size != job->oa_size);
  633. if (amdgpu_vm_had_gpu_reset(adev, id))
  634. return true;
  635. return vm_flush_needed || gds_switch_needed;
  636. }
  637. static bool amdgpu_vm_is_large_bar(struct amdgpu_device *adev)
  638. {
  639. return (adev->mc.real_vram_size == adev->mc.visible_vram_size);
  640. }
  641. /**
  642. * amdgpu_vm_flush - hardware flush the vm
  643. *
  644. * @ring: ring to use for flush
  645. * @vm_id: vmid number to use
  646. * @pd_addr: address of the page directory
  647. *
  648. * Emit a VM flush when it is necessary.
  649. */
  650. int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync)
  651. {
  652. struct amdgpu_device *adev = ring->adev;
  653. unsigned vmhub = ring->funcs->vmhub;
  654. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  655. struct amdgpu_vm_id *id = &id_mgr->ids[job->vm_id];
  656. bool gds_switch_needed = ring->funcs->emit_gds_switch && (
  657. id->gds_base != job->gds_base ||
  658. id->gds_size != job->gds_size ||
  659. id->gws_base != job->gws_base ||
  660. id->gws_size != job->gws_size ||
  661. id->oa_base != job->oa_base ||
  662. id->oa_size != job->oa_size);
  663. bool vm_flush_needed = job->vm_needs_flush;
  664. unsigned patch_offset = 0;
  665. int r;
  666. if (amdgpu_vm_had_gpu_reset(adev, id)) {
  667. gds_switch_needed = true;
  668. vm_flush_needed = true;
  669. }
  670. if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
  671. return 0;
  672. if (ring->funcs->init_cond_exec)
  673. patch_offset = amdgpu_ring_init_cond_exec(ring);
  674. if (need_pipe_sync)
  675. amdgpu_ring_emit_pipeline_sync(ring);
  676. if (ring->funcs->emit_vm_flush && vm_flush_needed) {
  677. struct dma_fence *fence;
  678. trace_amdgpu_vm_flush(ring, job->vm_id, job->vm_pd_addr);
  679. amdgpu_ring_emit_vm_flush(ring, job->vm_id, job->vm_pd_addr);
  680. r = amdgpu_fence_emit(ring, &fence);
  681. if (r)
  682. return r;
  683. mutex_lock(&id_mgr->lock);
  684. dma_fence_put(id->last_flush);
  685. id->last_flush = fence;
  686. id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
  687. mutex_unlock(&id_mgr->lock);
  688. }
  689. if (ring->funcs->emit_gds_switch && gds_switch_needed) {
  690. id->gds_base = job->gds_base;
  691. id->gds_size = job->gds_size;
  692. id->gws_base = job->gws_base;
  693. id->gws_size = job->gws_size;
  694. id->oa_base = job->oa_base;
  695. id->oa_size = job->oa_size;
  696. amdgpu_ring_emit_gds_switch(ring, job->vm_id, job->gds_base,
  697. job->gds_size, job->gws_base,
  698. job->gws_size, job->oa_base,
  699. job->oa_size);
  700. }
  701. if (ring->funcs->patch_cond_exec)
  702. amdgpu_ring_patch_cond_exec(ring, patch_offset);
  703. /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
  704. if (ring->funcs->emit_switch_buffer) {
  705. amdgpu_ring_emit_switch_buffer(ring);
  706. amdgpu_ring_emit_switch_buffer(ring);
  707. }
  708. return 0;
  709. }
  710. /**
  711. * amdgpu_vm_reset_id - reset VMID to zero
  712. *
  713. * @adev: amdgpu device structure
  714. * @vm_id: vmid number to use
  715. *
  716. * Reset saved GDW, GWS and OA to force switch on next flush.
  717. */
  718. void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vmhub,
  719. unsigned vmid)
  720. {
  721. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  722. struct amdgpu_vm_id *id = &id_mgr->ids[vmid];
  723. atomic64_set(&id->owner, 0);
  724. id->gds_base = 0;
  725. id->gds_size = 0;
  726. id->gws_base = 0;
  727. id->gws_size = 0;
  728. id->oa_base = 0;
  729. id->oa_size = 0;
  730. }
  731. /**
  732. * amdgpu_vm_reset_all_id - reset VMID to zero
  733. *
  734. * @adev: amdgpu device structure
  735. *
  736. * Reset VMID to force flush on next use
  737. */
  738. void amdgpu_vm_reset_all_ids(struct amdgpu_device *adev)
  739. {
  740. unsigned i, j;
  741. for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
  742. struct amdgpu_vm_id_manager *id_mgr =
  743. &adev->vm_manager.id_mgr[i];
  744. for (j = 1; j < id_mgr->num_ids; ++j)
  745. amdgpu_vm_reset_id(adev, i, j);
  746. }
  747. }
  748. /**
  749. * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
  750. *
  751. * @vm: requested vm
  752. * @bo: requested buffer object
  753. *
  754. * Find @bo inside the requested vm.
  755. * Search inside the @bos vm list for the requested vm
  756. * Returns the found bo_va or NULL if none is found
  757. *
  758. * Object has to be reserved!
  759. */
  760. struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
  761. struct amdgpu_bo *bo)
  762. {
  763. struct amdgpu_bo_va *bo_va;
  764. list_for_each_entry(bo_va, &bo->va, base.bo_list) {
  765. if (bo_va->base.vm == vm) {
  766. return bo_va;
  767. }
  768. }
  769. return NULL;
  770. }
  771. /**
  772. * amdgpu_vm_do_set_ptes - helper to call the right asic function
  773. *
  774. * @params: see amdgpu_pte_update_params definition
  775. * @pe: addr of the page entry
  776. * @addr: dst addr to write into pe
  777. * @count: number of page entries to update
  778. * @incr: increase next addr by incr bytes
  779. * @flags: hw access flags
  780. *
  781. * Traces the parameters and calls the right asic functions
  782. * to setup the page table using the DMA.
  783. */
  784. static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params *params,
  785. uint64_t pe, uint64_t addr,
  786. unsigned count, uint32_t incr,
  787. uint64_t flags)
  788. {
  789. trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
  790. if (count < 3) {
  791. amdgpu_vm_write_pte(params->adev, params->ib, pe,
  792. addr | flags, count, incr);
  793. } else {
  794. amdgpu_vm_set_pte_pde(params->adev, params->ib, pe, addr,
  795. count, incr, flags);
  796. }
  797. }
  798. /**
  799. * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
  800. *
  801. * @params: see amdgpu_pte_update_params definition
  802. * @pe: addr of the page entry
  803. * @addr: dst addr to write into pe
  804. * @count: number of page entries to update
  805. * @incr: increase next addr by incr bytes
  806. * @flags: hw access flags
  807. *
  808. * Traces the parameters and calls the DMA function to copy the PTEs.
  809. */
  810. static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
  811. uint64_t pe, uint64_t addr,
  812. unsigned count, uint32_t incr,
  813. uint64_t flags)
  814. {
  815. uint64_t src = (params->src + (addr >> 12) * 8);
  816. trace_amdgpu_vm_copy_ptes(pe, src, count);
  817. amdgpu_vm_copy_pte(params->adev, params->ib, pe, src, count);
  818. }
  819. /**
  820. * amdgpu_vm_map_gart - Resolve gart mapping of addr
  821. *
  822. * @pages_addr: optional DMA address to use for lookup
  823. * @addr: the unmapped addr
  824. *
  825. * Look up the physical address of the page that the pte resolves
  826. * to and return the pointer for the page table entry.
  827. */
  828. static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
  829. {
  830. uint64_t result;
  831. /* page table offset */
  832. result = pages_addr[addr >> PAGE_SHIFT];
  833. /* in case cpu page size != gpu page size*/
  834. result |= addr & (~PAGE_MASK);
  835. result &= 0xFFFFFFFFFFFFF000ULL;
  836. return result;
  837. }
  838. /**
  839. * amdgpu_vm_cpu_set_ptes - helper to update page tables via CPU
  840. *
  841. * @params: see amdgpu_pte_update_params definition
  842. * @pe: kmap addr of the page entry
  843. * @addr: dst addr to write into pe
  844. * @count: number of page entries to update
  845. * @incr: increase next addr by incr bytes
  846. * @flags: hw access flags
  847. *
  848. * Write count number of PT/PD entries directly.
  849. */
  850. static void amdgpu_vm_cpu_set_ptes(struct amdgpu_pte_update_params *params,
  851. uint64_t pe, uint64_t addr,
  852. unsigned count, uint32_t incr,
  853. uint64_t flags)
  854. {
  855. unsigned int i;
  856. uint64_t value;
  857. trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
  858. for (i = 0; i < count; i++) {
  859. value = params->pages_addr ?
  860. amdgpu_vm_map_gart(params->pages_addr, addr) :
  861. addr;
  862. amdgpu_gart_set_pte_pde(params->adev, (void *)(uintptr_t)pe,
  863. i, value, flags);
  864. addr += incr;
  865. }
  866. }
  867. static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  868. void *owner)
  869. {
  870. struct amdgpu_sync sync;
  871. int r;
  872. amdgpu_sync_create(&sync);
  873. amdgpu_sync_resv(adev, &sync, vm->root.bo->tbo.resv, owner);
  874. r = amdgpu_sync_wait(&sync, true);
  875. amdgpu_sync_free(&sync);
  876. return r;
  877. }
  878. /*
  879. * amdgpu_vm_update_level - update a single level in the hierarchy
  880. *
  881. * @adev: amdgpu_device pointer
  882. * @vm: requested vm
  883. * @parent: parent directory
  884. *
  885. * Makes sure all entries in @parent are up to date.
  886. * Returns 0 for success, error for failure.
  887. */
  888. static int amdgpu_vm_update_level(struct amdgpu_device *adev,
  889. struct amdgpu_vm *vm,
  890. struct amdgpu_vm_pt *parent,
  891. unsigned level)
  892. {
  893. struct amdgpu_bo *shadow;
  894. struct amdgpu_ring *ring = NULL;
  895. uint64_t pd_addr, shadow_addr = 0;
  896. uint32_t incr = amdgpu_vm_bo_size(adev, level + 1);
  897. uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
  898. unsigned count = 0, pt_idx, ndw = 0;
  899. struct amdgpu_job *job;
  900. struct amdgpu_pte_update_params params;
  901. struct dma_fence *fence = NULL;
  902. int r;
  903. if (!parent->entries)
  904. return 0;
  905. memset(&params, 0, sizeof(params));
  906. params.adev = adev;
  907. shadow = parent->bo->shadow;
  908. if (vm->use_cpu_for_update) {
  909. pd_addr = (unsigned long)amdgpu_bo_kptr(parent->bo);
  910. r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
  911. if (unlikely(r))
  912. return r;
  913. params.func = amdgpu_vm_cpu_set_ptes;
  914. } else {
  915. ring = container_of(vm->entity.sched, struct amdgpu_ring,
  916. sched);
  917. /* padding, etc. */
  918. ndw = 64;
  919. /* assume the worst case */
  920. ndw += parent->last_entry_used * 6;
  921. pd_addr = amdgpu_bo_gpu_offset(parent->bo);
  922. if (shadow) {
  923. shadow_addr = amdgpu_bo_gpu_offset(shadow);
  924. ndw *= 2;
  925. } else {
  926. shadow_addr = 0;
  927. }
  928. r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
  929. if (r)
  930. return r;
  931. params.ib = &job->ibs[0];
  932. params.func = amdgpu_vm_do_set_ptes;
  933. }
  934. /* walk over the address space and update the directory */
  935. for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
  936. struct amdgpu_bo *bo = parent->entries[pt_idx].bo;
  937. uint64_t pde, pt;
  938. if (bo == NULL)
  939. continue;
  940. pt = amdgpu_bo_gpu_offset(bo);
  941. pt = amdgpu_gart_get_vm_pde(adev, pt);
  942. /* Don't update huge pages here */
  943. if ((parent->entries[pt_idx].addr & AMDGPU_PDE_PTE) ||
  944. parent->entries[pt_idx].addr == (pt | AMDGPU_PTE_VALID))
  945. continue;
  946. parent->entries[pt_idx].addr = pt | AMDGPU_PTE_VALID;
  947. pde = pd_addr + pt_idx * 8;
  948. if (((last_pde + 8 * count) != pde) ||
  949. ((last_pt + incr * count) != pt) ||
  950. (count == AMDGPU_VM_MAX_UPDATE_SIZE)) {
  951. if (count) {
  952. if (shadow)
  953. params.func(&params,
  954. last_shadow,
  955. last_pt, count,
  956. incr,
  957. AMDGPU_PTE_VALID);
  958. params.func(&params, last_pde,
  959. last_pt, count, incr,
  960. AMDGPU_PTE_VALID);
  961. }
  962. count = 1;
  963. last_pde = pde;
  964. last_shadow = shadow_addr + pt_idx * 8;
  965. last_pt = pt;
  966. } else {
  967. ++count;
  968. }
  969. }
  970. if (count) {
  971. if (vm->root.bo->shadow)
  972. params.func(&params, last_shadow, last_pt,
  973. count, incr, AMDGPU_PTE_VALID);
  974. params.func(&params, last_pde, last_pt,
  975. count, incr, AMDGPU_PTE_VALID);
  976. }
  977. if (!vm->use_cpu_for_update) {
  978. if (params.ib->length_dw == 0) {
  979. amdgpu_job_free(job);
  980. } else {
  981. amdgpu_ring_pad_ib(ring, params.ib);
  982. amdgpu_sync_resv(adev, &job->sync, parent->bo->tbo.resv,
  983. AMDGPU_FENCE_OWNER_VM);
  984. if (shadow)
  985. amdgpu_sync_resv(adev, &job->sync,
  986. shadow->tbo.resv,
  987. AMDGPU_FENCE_OWNER_VM);
  988. WARN_ON(params.ib->length_dw > ndw);
  989. r = amdgpu_job_submit(job, ring, &vm->entity,
  990. AMDGPU_FENCE_OWNER_VM, &fence);
  991. if (r)
  992. goto error_free;
  993. amdgpu_bo_fence(parent->bo, fence, true);
  994. dma_fence_put(vm->last_dir_update);
  995. vm->last_dir_update = dma_fence_get(fence);
  996. dma_fence_put(fence);
  997. }
  998. }
  999. /*
  1000. * Recurse into the subdirectories. This recursion is harmless because
  1001. * we only have a maximum of 5 layers.
  1002. */
  1003. for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
  1004. struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
  1005. if (!entry->bo)
  1006. continue;
  1007. r = amdgpu_vm_update_level(adev, vm, entry, level + 1);
  1008. if (r)
  1009. return r;
  1010. }
  1011. return 0;
  1012. error_free:
  1013. amdgpu_job_free(job);
  1014. return r;
  1015. }
  1016. /*
  1017. * amdgpu_vm_invalidate_level - mark all PD levels as invalid
  1018. *
  1019. * @parent: parent PD
  1020. *
  1021. * Mark all PD level as invalid after an error.
  1022. */
  1023. static void amdgpu_vm_invalidate_level(struct amdgpu_vm_pt *parent)
  1024. {
  1025. unsigned pt_idx;
  1026. /*
  1027. * Recurse into the subdirectories. This recursion is harmless because
  1028. * we only have a maximum of 5 layers.
  1029. */
  1030. for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
  1031. struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
  1032. if (!entry->bo)
  1033. continue;
  1034. entry->addr = ~0ULL;
  1035. amdgpu_vm_invalidate_level(entry);
  1036. }
  1037. }
  1038. /*
  1039. * amdgpu_vm_update_directories - make sure that all directories are valid
  1040. *
  1041. * @adev: amdgpu_device pointer
  1042. * @vm: requested vm
  1043. *
  1044. * Makes sure all directories are up to date.
  1045. * Returns 0 for success, error for failure.
  1046. */
  1047. int amdgpu_vm_update_directories(struct amdgpu_device *adev,
  1048. struct amdgpu_vm *vm)
  1049. {
  1050. int r;
  1051. r = amdgpu_vm_update_level(adev, vm, &vm->root, 0);
  1052. if (r)
  1053. amdgpu_vm_invalidate_level(&vm->root);
  1054. if (vm->use_cpu_for_update) {
  1055. /* Flush HDP */
  1056. mb();
  1057. amdgpu_gart_flush_gpu_tlb(adev, 0);
  1058. }
  1059. return r;
  1060. }
  1061. /**
  1062. * amdgpu_vm_find_entry - find the entry for an address
  1063. *
  1064. * @p: see amdgpu_pte_update_params definition
  1065. * @addr: virtual address in question
  1066. * @entry: resulting entry or NULL
  1067. * @parent: parent entry
  1068. *
  1069. * Find the vm_pt entry and it's parent for the given address.
  1070. */
  1071. void amdgpu_vm_get_entry(struct amdgpu_pte_update_params *p, uint64_t addr,
  1072. struct amdgpu_vm_pt **entry,
  1073. struct amdgpu_vm_pt **parent)
  1074. {
  1075. unsigned idx, level = p->adev->vm_manager.num_level;
  1076. *parent = NULL;
  1077. *entry = &p->vm->root;
  1078. while ((*entry)->entries) {
  1079. idx = addr >> (p->adev->vm_manager.block_size * level--);
  1080. idx %= amdgpu_bo_size((*entry)->bo) / 8;
  1081. *parent = *entry;
  1082. *entry = &(*entry)->entries[idx];
  1083. }
  1084. if (level)
  1085. *entry = NULL;
  1086. }
  1087. /**
  1088. * amdgpu_vm_handle_huge_pages - handle updating the PD with huge pages
  1089. *
  1090. * @p: see amdgpu_pte_update_params definition
  1091. * @entry: vm_pt entry to check
  1092. * @parent: parent entry
  1093. * @nptes: number of PTEs updated with this operation
  1094. * @dst: destination address where the PTEs should point to
  1095. * @flags: access flags fro the PTEs
  1096. *
  1097. * Check if we can update the PD with a huge page.
  1098. */
  1099. static void amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p,
  1100. struct amdgpu_vm_pt *entry,
  1101. struct amdgpu_vm_pt *parent,
  1102. unsigned nptes, uint64_t dst,
  1103. uint64_t flags)
  1104. {
  1105. bool use_cpu_update = (p->func == amdgpu_vm_cpu_set_ptes);
  1106. uint64_t pd_addr, pde;
  1107. /* In the case of a mixed PT the PDE must point to it*/
  1108. if (p->adev->asic_type < CHIP_VEGA10 ||
  1109. nptes != AMDGPU_VM_PTE_COUNT(p->adev) ||
  1110. p->src ||
  1111. !(flags & AMDGPU_PTE_VALID)) {
  1112. dst = amdgpu_bo_gpu_offset(entry->bo);
  1113. dst = amdgpu_gart_get_vm_pde(p->adev, dst);
  1114. flags = AMDGPU_PTE_VALID;
  1115. } else {
  1116. /* Set the huge page flag to stop scanning at this PDE */
  1117. flags |= AMDGPU_PDE_PTE;
  1118. }
  1119. if (entry->addr == (dst | flags))
  1120. return;
  1121. entry->addr = (dst | flags);
  1122. if (use_cpu_update) {
  1123. /* In case a huge page is replaced with a system
  1124. * memory mapping, p->pages_addr != NULL and
  1125. * amdgpu_vm_cpu_set_ptes would try to translate dst
  1126. * through amdgpu_vm_map_gart. But dst is already a
  1127. * GPU address (of the page table). Disable
  1128. * amdgpu_vm_map_gart temporarily.
  1129. */
  1130. dma_addr_t *tmp;
  1131. tmp = p->pages_addr;
  1132. p->pages_addr = NULL;
  1133. pd_addr = (unsigned long)amdgpu_bo_kptr(parent->bo);
  1134. pde = pd_addr + (entry - parent->entries) * 8;
  1135. amdgpu_vm_cpu_set_ptes(p, pde, dst, 1, 0, flags);
  1136. p->pages_addr = tmp;
  1137. } else {
  1138. if (parent->bo->shadow) {
  1139. pd_addr = amdgpu_bo_gpu_offset(parent->bo->shadow);
  1140. pde = pd_addr + (entry - parent->entries) * 8;
  1141. amdgpu_vm_do_set_ptes(p, pde, dst, 1, 0, flags);
  1142. }
  1143. pd_addr = amdgpu_bo_gpu_offset(parent->bo);
  1144. pde = pd_addr + (entry - parent->entries) * 8;
  1145. amdgpu_vm_do_set_ptes(p, pde, dst, 1, 0, flags);
  1146. }
  1147. }
  1148. /**
  1149. * amdgpu_vm_update_ptes - make sure that page tables are valid
  1150. *
  1151. * @params: see amdgpu_pte_update_params definition
  1152. * @vm: requested vm
  1153. * @start: start of GPU address range
  1154. * @end: end of GPU address range
  1155. * @dst: destination address to map to, the next dst inside the function
  1156. * @flags: mapping flags
  1157. *
  1158. * Update the page tables in the range @start - @end.
  1159. * Returns 0 for success, -EINVAL for failure.
  1160. */
  1161. static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
  1162. uint64_t start, uint64_t end,
  1163. uint64_t dst, uint64_t flags)
  1164. {
  1165. struct amdgpu_device *adev = params->adev;
  1166. const uint64_t mask = AMDGPU_VM_PTE_COUNT(adev) - 1;
  1167. uint64_t addr, pe_start;
  1168. struct amdgpu_bo *pt;
  1169. unsigned nptes;
  1170. bool use_cpu_update = (params->func == amdgpu_vm_cpu_set_ptes);
  1171. /* walk over the address space and update the page tables */
  1172. for (addr = start; addr < end; addr += nptes,
  1173. dst += nptes * AMDGPU_GPU_PAGE_SIZE) {
  1174. struct amdgpu_vm_pt *entry, *parent;
  1175. amdgpu_vm_get_entry(params, addr, &entry, &parent);
  1176. if (!entry)
  1177. return -ENOENT;
  1178. if ((addr & ~mask) == (end & ~mask))
  1179. nptes = end - addr;
  1180. else
  1181. nptes = AMDGPU_VM_PTE_COUNT(adev) - (addr & mask);
  1182. amdgpu_vm_handle_huge_pages(params, entry, parent,
  1183. nptes, dst, flags);
  1184. /* We don't need to update PTEs for huge pages */
  1185. if (entry->addr & AMDGPU_PDE_PTE)
  1186. continue;
  1187. pt = entry->bo;
  1188. if (use_cpu_update) {
  1189. pe_start = (unsigned long)amdgpu_bo_kptr(pt);
  1190. } else {
  1191. if (pt->shadow) {
  1192. pe_start = amdgpu_bo_gpu_offset(pt->shadow);
  1193. pe_start += (addr & mask) * 8;
  1194. params->func(params, pe_start, dst, nptes,
  1195. AMDGPU_GPU_PAGE_SIZE, flags);
  1196. }
  1197. pe_start = amdgpu_bo_gpu_offset(pt);
  1198. }
  1199. pe_start += (addr & mask) * 8;
  1200. params->func(params, pe_start, dst, nptes,
  1201. AMDGPU_GPU_PAGE_SIZE, flags);
  1202. }
  1203. return 0;
  1204. }
  1205. /*
  1206. * amdgpu_vm_frag_ptes - add fragment information to PTEs
  1207. *
  1208. * @params: see amdgpu_pte_update_params definition
  1209. * @vm: requested vm
  1210. * @start: first PTE to handle
  1211. * @end: last PTE to handle
  1212. * @dst: addr those PTEs should point to
  1213. * @flags: hw mapping flags
  1214. * Returns 0 for success, -EINVAL for failure.
  1215. */
  1216. static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params *params,
  1217. uint64_t start, uint64_t end,
  1218. uint64_t dst, uint64_t flags)
  1219. {
  1220. int r;
  1221. /**
  1222. * The MC L1 TLB supports variable sized pages, based on a fragment
  1223. * field in the PTE. When this field is set to a non-zero value, page
  1224. * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
  1225. * flags are considered valid for all PTEs within the fragment range
  1226. * and corresponding mappings are assumed to be physically contiguous.
  1227. *
  1228. * The L1 TLB can store a single PTE for the whole fragment,
  1229. * significantly increasing the space available for translation
  1230. * caching. This leads to large improvements in throughput when the
  1231. * TLB is under pressure.
  1232. *
  1233. * The L2 TLB distributes small and large fragments into two
  1234. * asymmetric partitions. The large fragment cache is significantly
  1235. * larger. Thus, we try to use large fragments wherever possible.
  1236. * Userspace can support this by aligning virtual base address and
  1237. * allocation size to the fragment size.
  1238. */
  1239. unsigned pages_per_frag = params->adev->vm_manager.fragment_size;
  1240. uint64_t frag_flags = AMDGPU_PTE_FRAG(pages_per_frag);
  1241. uint64_t frag_align = 1 << pages_per_frag;
  1242. uint64_t frag_start = ALIGN(start, frag_align);
  1243. uint64_t frag_end = end & ~(frag_align - 1);
  1244. /* system pages are non continuously */
  1245. if (params->src || !(flags & AMDGPU_PTE_VALID) ||
  1246. (frag_start >= frag_end))
  1247. return amdgpu_vm_update_ptes(params, start, end, dst, flags);
  1248. /* handle the 4K area at the beginning */
  1249. if (start != frag_start) {
  1250. r = amdgpu_vm_update_ptes(params, start, frag_start,
  1251. dst, flags);
  1252. if (r)
  1253. return r;
  1254. dst += (frag_start - start) * AMDGPU_GPU_PAGE_SIZE;
  1255. }
  1256. /* handle the area in the middle */
  1257. r = amdgpu_vm_update_ptes(params, frag_start, frag_end, dst,
  1258. flags | frag_flags);
  1259. if (r)
  1260. return r;
  1261. /* handle the 4K area at the end */
  1262. if (frag_end != end) {
  1263. dst += (frag_end - frag_start) * AMDGPU_GPU_PAGE_SIZE;
  1264. r = amdgpu_vm_update_ptes(params, frag_end, end, dst, flags);
  1265. }
  1266. return r;
  1267. }
  1268. /**
  1269. * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
  1270. *
  1271. * @adev: amdgpu_device pointer
  1272. * @exclusive: fence we need to sync to
  1273. * @src: address where to copy page table entries from
  1274. * @pages_addr: DMA addresses to use for mapping
  1275. * @vm: requested vm
  1276. * @start: start of mapped range
  1277. * @last: last mapped entry
  1278. * @flags: flags for the entries
  1279. * @addr: addr to set the area to
  1280. * @fence: optional resulting fence
  1281. *
  1282. * Fill in the page table entries between @start and @last.
  1283. * Returns 0 for success, -EINVAL for failure.
  1284. */
  1285. static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
  1286. struct dma_fence *exclusive,
  1287. uint64_t src,
  1288. dma_addr_t *pages_addr,
  1289. struct amdgpu_vm *vm,
  1290. uint64_t start, uint64_t last,
  1291. uint64_t flags, uint64_t addr,
  1292. struct dma_fence **fence)
  1293. {
  1294. struct amdgpu_ring *ring;
  1295. void *owner = AMDGPU_FENCE_OWNER_VM;
  1296. unsigned nptes, ncmds, ndw;
  1297. struct amdgpu_job *job;
  1298. struct amdgpu_pte_update_params params;
  1299. struct dma_fence *f = NULL;
  1300. int r;
  1301. memset(&params, 0, sizeof(params));
  1302. params.adev = adev;
  1303. params.vm = vm;
  1304. params.src = src;
  1305. /* sync to everything on unmapping */
  1306. if (!(flags & AMDGPU_PTE_VALID))
  1307. owner = AMDGPU_FENCE_OWNER_UNDEFINED;
  1308. if (vm->use_cpu_for_update) {
  1309. /* params.src is used as flag to indicate system Memory */
  1310. if (pages_addr)
  1311. params.src = ~0;
  1312. /* Wait for PT BOs to be free. PTs share the same resv. object
  1313. * as the root PD BO
  1314. */
  1315. r = amdgpu_vm_wait_pd(adev, vm, owner);
  1316. if (unlikely(r))
  1317. return r;
  1318. params.func = amdgpu_vm_cpu_set_ptes;
  1319. params.pages_addr = pages_addr;
  1320. return amdgpu_vm_frag_ptes(&params, start, last + 1,
  1321. addr, flags);
  1322. }
  1323. ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
  1324. nptes = last - start + 1;
  1325. /*
  1326. * reserve space for one command every (1 << BLOCK_SIZE)
  1327. * entries or 2k dwords (whatever is smaller)
  1328. */
  1329. ncmds = (nptes >> min(adev->vm_manager.block_size, 11u)) + 1;
  1330. /* padding, etc. */
  1331. ndw = 64;
  1332. /* one PDE write for each huge page */
  1333. ndw += ((nptes >> adev->vm_manager.block_size) + 1) * 6;
  1334. if (src) {
  1335. /* only copy commands needed */
  1336. ndw += ncmds * 7;
  1337. params.func = amdgpu_vm_do_copy_ptes;
  1338. } else if (pages_addr) {
  1339. /* copy commands needed */
  1340. ndw += ncmds * 7;
  1341. /* and also PTEs */
  1342. ndw += nptes * 2;
  1343. params.func = amdgpu_vm_do_copy_ptes;
  1344. } else {
  1345. /* set page commands needed */
  1346. ndw += ncmds * 10;
  1347. /* two extra commands for begin/end of fragment */
  1348. ndw += 2 * 10;
  1349. params.func = amdgpu_vm_do_set_ptes;
  1350. }
  1351. r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
  1352. if (r)
  1353. return r;
  1354. params.ib = &job->ibs[0];
  1355. if (!src && pages_addr) {
  1356. uint64_t *pte;
  1357. unsigned i;
  1358. /* Put the PTEs at the end of the IB. */
  1359. i = ndw - nptes * 2;
  1360. pte= (uint64_t *)&(job->ibs->ptr[i]);
  1361. params.src = job->ibs->gpu_addr + i * 4;
  1362. for (i = 0; i < nptes; ++i) {
  1363. pte[i] = amdgpu_vm_map_gart(pages_addr, addr + i *
  1364. AMDGPU_GPU_PAGE_SIZE);
  1365. pte[i] |= flags;
  1366. }
  1367. addr = 0;
  1368. }
  1369. r = amdgpu_sync_fence(adev, &job->sync, exclusive);
  1370. if (r)
  1371. goto error_free;
  1372. r = amdgpu_sync_resv(adev, &job->sync, vm->root.bo->tbo.resv,
  1373. owner);
  1374. if (r)
  1375. goto error_free;
  1376. r = reservation_object_reserve_shared(vm->root.bo->tbo.resv);
  1377. if (r)
  1378. goto error_free;
  1379. r = amdgpu_vm_frag_ptes(&params, start, last + 1, addr, flags);
  1380. if (r)
  1381. goto error_free;
  1382. amdgpu_ring_pad_ib(ring, params.ib);
  1383. WARN_ON(params.ib->length_dw > ndw);
  1384. r = amdgpu_job_submit(job, ring, &vm->entity,
  1385. AMDGPU_FENCE_OWNER_VM, &f);
  1386. if (r)
  1387. goto error_free;
  1388. amdgpu_bo_fence(vm->root.bo, f, true);
  1389. dma_fence_put(*fence);
  1390. *fence = f;
  1391. return 0;
  1392. error_free:
  1393. amdgpu_job_free(job);
  1394. amdgpu_vm_invalidate_level(&vm->root);
  1395. return r;
  1396. }
  1397. /**
  1398. * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
  1399. *
  1400. * @adev: amdgpu_device pointer
  1401. * @exclusive: fence we need to sync to
  1402. * @gtt_flags: flags as they are used for GTT
  1403. * @pages_addr: DMA addresses to use for mapping
  1404. * @vm: requested vm
  1405. * @mapping: mapped range and flags to use for the update
  1406. * @flags: HW flags for the mapping
  1407. * @nodes: array of drm_mm_nodes with the MC addresses
  1408. * @fence: optional resulting fence
  1409. *
  1410. * Split the mapping into smaller chunks so that each update fits
  1411. * into a SDMA IB.
  1412. * Returns 0 for success, -EINVAL for failure.
  1413. */
  1414. static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
  1415. struct dma_fence *exclusive,
  1416. uint64_t gtt_flags,
  1417. dma_addr_t *pages_addr,
  1418. struct amdgpu_vm *vm,
  1419. struct amdgpu_bo_va_mapping *mapping,
  1420. uint64_t flags,
  1421. struct drm_mm_node *nodes,
  1422. struct dma_fence **fence)
  1423. {
  1424. uint64_t pfn, src = 0, start = mapping->start;
  1425. int r;
  1426. /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
  1427. * but in case of something, we filter the flags in first place
  1428. */
  1429. if (!(mapping->flags & AMDGPU_PTE_READABLE))
  1430. flags &= ~AMDGPU_PTE_READABLE;
  1431. if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
  1432. flags &= ~AMDGPU_PTE_WRITEABLE;
  1433. flags &= ~AMDGPU_PTE_EXECUTABLE;
  1434. flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
  1435. flags &= ~AMDGPU_PTE_MTYPE_MASK;
  1436. flags |= (mapping->flags & AMDGPU_PTE_MTYPE_MASK);
  1437. if ((mapping->flags & AMDGPU_PTE_PRT) &&
  1438. (adev->asic_type >= CHIP_VEGA10)) {
  1439. flags |= AMDGPU_PTE_PRT;
  1440. flags &= ~AMDGPU_PTE_VALID;
  1441. }
  1442. trace_amdgpu_vm_bo_update(mapping);
  1443. pfn = mapping->offset >> PAGE_SHIFT;
  1444. if (nodes) {
  1445. while (pfn >= nodes->size) {
  1446. pfn -= nodes->size;
  1447. ++nodes;
  1448. }
  1449. }
  1450. do {
  1451. uint64_t max_entries;
  1452. uint64_t addr, last;
  1453. if (nodes) {
  1454. addr = nodes->start << PAGE_SHIFT;
  1455. max_entries = (nodes->size - pfn) *
  1456. (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
  1457. } else {
  1458. addr = 0;
  1459. max_entries = S64_MAX;
  1460. }
  1461. if (pages_addr) {
  1462. if (flags == gtt_flags)
  1463. src = adev->gart.table_addr +
  1464. (addr >> AMDGPU_GPU_PAGE_SHIFT) * 8;
  1465. else
  1466. max_entries = min(max_entries, 16ull * 1024ull);
  1467. addr = 0;
  1468. } else if (flags & AMDGPU_PTE_VALID) {
  1469. addr += adev->vm_manager.vram_base_offset;
  1470. }
  1471. addr += pfn << PAGE_SHIFT;
  1472. last = min((uint64_t)mapping->last, start + max_entries - 1);
  1473. r = amdgpu_vm_bo_update_mapping(adev, exclusive,
  1474. src, pages_addr, vm,
  1475. start, last, flags, addr,
  1476. fence);
  1477. if (r)
  1478. return r;
  1479. pfn += last - start + 1;
  1480. if (nodes && nodes->size == pfn) {
  1481. pfn = 0;
  1482. ++nodes;
  1483. }
  1484. start = last + 1;
  1485. } while (unlikely(start != mapping->last + 1));
  1486. return 0;
  1487. }
  1488. /**
  1489. * amdgpu_vm_bo_update - update all BO mappings in the vm page table
  1490. *
  1491. * @adev: amdgpu_device pointer
  1492. * @bo_va: requested BO and VM object
  1493. * @clear: if true clear the entries
  1494. *
  1495. * Fill in the page table entries for @bo_va.
  1496. * Returns 0 for success, -EINVAL for failure.
  1497. */
  1498. int amdgpu_vm_bo_update(struct amdgpu_device *adev,
  1499. struct amdgpu_bo_va *bo_va,
  1500. bool clear)
  1501. {
  1502. struct amdgpu_bo *bo = bo_va->base.bo;
  1503. struct amdgpu_vm *vm = bo_va->base.vm;
  1504. struct amdgpu_bo_va_mapping *mapping;
  1505. dma_addr_t *pages_addr = NULL;
  1506. uint64_t gtt_flags, flags;
  1507. struct ttm_mem_reg *mem;
  1508. struct drm_mm_node *nodes;
  1509. struct dma_fence *exclusive;
  1510. int r;
  1511. if (clear || !bo_va->base.bo) {
  1512. mem = NULL;
  1513. nodes = NULL;
  1514. exclusive = NULL;
  1515. } else {
  1516. struct ttm_dma_tt *ttm;
  1517. mem = &bo_va->base.bo->tbo.mem;
  1518. nodes = mem->mm_node;
  1519. if (mem->mem_type == TTM_PL_TT) {
  1520. ttm = container_of(bo_va->base.bo->tbo.ttm,
  1521. struct ttm_dma_tt, ttm);
  1522. pages_addr = ttm->dma_address;
  1523. }
  1524. exclusive = reservation_object_get_excl(bo->tbo.resv);
  1525. }
  1526. if (bo) {
  1527. flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
  1528. gtt_flags = (amdgpu_ttm_is_bound(bo->tbo.ttm) &&
  1529. adev == amdgpu_ttm_adev(bo->tbo.bdev)) ?
  1530. flags : 0;
  1531. } else {
  1532. flags = 0x0;
  1533. gtt_flags = ~0x0;
  1534. }
  1535. spin_lock(&vm->status_lock);
  1536. if (!list_empty(&bo_va->base.vm_status))
  1537. list_splice_init(&bo_va->valids, &bo_va->invalids);
  1538. spin_unlock(&vm->status_lock);
  1539. list_for_each_entry(mapping, &bo_va->invalids, list) {
  1540. r = amdgpu_vm_bo_split_mapping(adev, exclusive,
  1541. gtt_flags, pages_addr, vm,
  1542. mapping, flags, nodes,
  1543. &bo_va->last_pt_update);
  1544. if (r)
  1545. return r;
  1546. }
  1547. if (trace_amdgpu_vm_bo_mapping_enabled()) {
  1548. list_for_each_entry(mapping, &bo_va->valids, list)
  1549. trace_amdgpu_vm_bo_mapping(mapping);
  1550. list_for_each_entry(mapping, &bo_va->invalids, list)
  1551. trace_amdgpu_vm_bo_mapping(mapping);
  1552. }
  1553. spin_lock(&vm->status_lock);
  1554. list_splice_init(&bo_va->invalids, &bo_va->valids);
  1555. list_del_init(&bo_va->base.vm_status);
  1556. if (clear)
  1557. list_add(&bo_va->base.vm_status, &vm->cleared);
  1558. spin_unlock(&vm->status_lock);
  1559. if (vm->use_cpu_for_update) {
  1560. /* Flush HDP */
  1561. mb();
  1562. amdgpu_gart_flush_gpu_tlb(adev, 0);
  1563. }
  1564. return 0;
  1565. }
  1566. /**
  1567. * amdgpu_vm_update_prt_state - update the global PRT state
  1568. */
  1569. static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
  1570. {
  1571. unsigned long flags;
  1572. bool enable;
  1573. spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
  1574. enable = !!atomic_read(&adev->vm_manager.num_prt_users);
  1575. adev->gart.gart_funcs->set_prt(adev, enable);
  1576. spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
  1577. }
  1578. /**
  1579. * amdgpu_vm_prt_get - add a PRT user
  1580. */
  1581. static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
  1582. {
  1583. if (!adev->gart.gart_funcs->set_prt)
  1584. return;
  1585. if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
  1586. amdgpu_vm_update_prt_state(adev);
  1587. }
  1588. /**
  1589. * amdgpu_vm_prt_put - drop a PRT user
  1590. */
  1591. static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
  1592. {
  1593. if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
  1594. amdgpu_vm_update_prt_state(adev);
  1595. }
  1596. /**
  1597. * amdgpu_vm_prt_cb - callback for updating the PRT status
  1598. */
  1599. static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
  1600. {
  1601. struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
  1602. amdgpu_vm_prt_put(cb->adev);
  1603. kfree(cb);
  1604. }
  1605. /**
  1606. * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
  1607. */
  1608. static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
  1609. struct dma_fence *fence)
  1610. {
  1611. struct amdgpu_prt_cb *cb;
  1612. if (!adev->gart.gart_funcs->set_prt)
  1613. return;
  1614. cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
  1615. if (!cb) {
  1616. /* Last resort when we are OOM */
  1617. if (fence)
  1618. dma_fence_wait(fence, false);
  1619. amdgpu_vm_prt_put(adev);
  1620. } else {
  1621. cb->adev = adev;
  1622. if (!fence || dma_fence_add_callback(fence, &cb->cb,
  1623. amdgpu_vm_prt_cb))
  1624. amdgpu_vm_prt_cb(fence, &cb->cb);
  1625. }
  1626. }
  1627. /**
  1628. * amdgpu_vm_free_mapping - free a mapping
  1629. *
  1630. * @adev: amdgpu_device pointer
  1631. * @vm: requested vm
  1632. * @mapping: mapping to be freed
  1633. * @fence: fence of the unmap operation
  1634. *
  1635. * Free a mapping and make sure we decrease the PRT usage count if applicable.
  1636. */
  1637. static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
  1638. struct amdgpu_vm *vm,
  1639. struct amdgpu_bo_va_mapping *mapping,
  1640. struct dma_fence *fence)
  1641. {
  1642. if (mapping->flags & AMDGPU_PTE_PRT)
  1643. amdgpu_vm_add_prt_cb(adev, fence);
  1644. kfree(mapping);
  1645. }
  1646. /**
  1647. * amdgpu_vm_prt_fini - finish all prt mappings
  1648. *
  1649. * @adev: amdgpu_device pointer
  1650. * @vm: requested vm
  1651. *
  1652. * Register a cleanup callback to disable PRT support after VM dies.
  1653. */
  1654. static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
  1655. {
  1656. struct reservation_object *resv = vm->root.bo->tbo.resv;
  1657. struct dma_fence *excl, **shared;
  1658. unsigned i, shared_count;
  1659. int r;
  1660. r = reservation_object_get_fences_rcu(resv, &excl,
  1661. &shared_count, &shared);
  1662. if (r) {
  1663. /* Not enough memory to grab the fence list, as last resort
  1664. * block for all the fences to complete.
  1665. */
  1666. reservation_object_wait_timeout_rcu(resv, true, false,
  1667. MAX_SCHEDULE_TIMEOUT);
  1668. return;
  1669. }
  1670. /* Add a callback for each fence in the reservation object */
  1671. amdgpu_vm_prt_get(adev);
  1672. amdgpu_vm_add_prt_cb(adev, excl);
  1673. for (i = 0; i < shared_count; ++i) {
  1674. amdgpu_vm_prt_get(adev);
  1675. amdgpu_vm_add_prt_cb(adev, shared[i]);
  1676. }
  1677. kfree(shared);
  1678. }
  1679. /**
  1680. * amdgpu_vm_clear_freed - clear freed BOs in the PT
  1681. *
  1682. * @adev: amdgpu_device pointer
  1683. * @vm: requested vm
  1684. * @fence: optional resulting fence (unchanged if no work needed to be done
  1685. * or if an error occurred)
  1686. *
  1687. * Make sure all freed BOs are cleared in the PT.
  1688. * Returns 0 for success.
  1689. *
  1690. * PTs have to be reserved and mutex must be locked!
  1691. */
  1692. int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
  1693. struct amdgpu_vm *vm,
  1694. struct dma_fence **fence)
  1695. {
  1696. struct amdgpu_bo_va_mapping *mapping;
  1697. struct dma_fence *f = NULL;
  1698. int r;
  1699. uint64_t init_pte_value = 0;
  1700. while (!list_empty(&vm->freed)) {
  1701. mapping = list_first_entry(&vm->freed,
  1702. struct amdgpu_bo_va_mapping, list);
  1703. list_del(&mapping->list);
  1704. if (vm->pte_support_ats)
  1705. init_pte_value = AMDGPU_PTE_SYSTEM;
  1706. r = amdgpu_vm_bo_update_mapping(adev, NULL, 0, NULL, vm,
  1707. mapping->start, mapping->last,
  1708. init_pte_value, 0, &f);
  1709. amdgpu_vm_free_mapping(adev, vm, mapping, f);
  1710. if (r) {
  1711. dma_fence_put(f);
  1712. return r;
  1713. }
  1714. }
  1715. if (fence && f) {
  1716. dma_fence_put(*fence);
  1717. *fence = f;
  1718. } else {
  1719. dma_fence_put(f);
  1720. }
  1721. return 0;
  1722. }
  1723. /**
  1724. * amdgpu_vm_clear_moved - clear moved BOs in the PT
  1725. *
  1726. * @adev: amdgpu_device pointer
  1727. * @vm: requested vm
  1728. *
  1729. * Make sure all moved BOs are cleared in the PT.
  1730. * Returns 0 for success.
  1731. *
  1732. * PTs have to be reserved and mutex must be locked!
  1733. */
  1734. int amdgpu_vm_clear_moved(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  1735. struct amdgpu_sync *sync)
  1736. {
  1737. struct amdgpu_bo_va *bo_va = NULL;
  1738. int r = 0;
  1739. spin_lock(&vm->status_lock);
  1740. while (!list_empty(&vm->moved)) {
  1741. bo_va = list_first_entry(&vm->moved,
  1742. struct amdgpu_bo_va, base.vm_status);
  1743. spin_unlock(&vm->status_lock);
  1744. r = amdgpu_vm_bo_update(adev, bo_va, true);
  1745. if (r)
  1746. return r;
  1747. spin_lock(&vm->status_lock);
  1748. }
  1749. spin_unlock(&vm->status_lock);
  1750. if (bo_va)
  1751. r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
  1752. return r;
  1753. }
  1754. /**
  1755. * amdgpu_vm_bo_add - add a bo to a specific vm
  1756. *
  1757. * @adev: amdgpu_device pointer
  1758. * @vm: requested vm
  1759. * @bo: amdgpu buffer object
  1760. *
  1761. * Add @bo into the requested vm.
  1762. * Add @bo to the list of bos associated with the vm
  1763. * Returns newly added bo_va or NULL for failure
  1764. *
  1765. * Object has to be reserved!
  1766. */
  1767. struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
  1768. struct amdgpu_vm *vm,
  1769. struct amdgpu_bo *bo)
  1770. {
  1771. struct amdgpu_bo_va *bo_va;
  1772. bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
  1773. if (bo_va == NULL) {
  1774. return NULL;
  1775. }
  1776. bo_va->base.vm = vm;
  1777. bo_va->base.bo = bo;
  1778. INIT_LIST_HEAD(&bo_va->base.bo_list);
  1779. INIT_LIST_HEAD(&bo_va->base.vm_status);
  1780. bo_va->ref_count = 1;
  1781. INIT_LIST_HEAD(&bo_va->valids);
  1782. INIT_LIST_HEAD(&bo_va->invalids);
  1783. if (bo)
  1784. list_add_tail(&bo_va->base.bo_list, &bo->va);
  1785. return bo_va;
  1786. }
  1787. /**
  1788. * amdgpu_vm_bo_map - map bo inside a vm
  1789. *
  1790. * @adev: amdgpu_device pointer
  1791. * @bo_va: bo_va to store the address
  1792. * @saddr: where to map the BO
  1793. * @offset: requested offset in the BO
  1794. * @flags: attributes of pages (read/write/valid/etc.)
  1795. *
  1796. * Add a mapping of the BO at the specefied addr into the VM.
  1797. * Returns 0 for success, error for failure.
  1798. *
  1799. * Object has to be reserved and unreserved outside!
  1800. */
  1801. int amdgpu_vm_bo_map(struct amdgpu_device *adev,
  1802. struct amdgpu_bo_va *bo_va,
  1803. uint64_t saddr, uint64_t offset,
  1804. uint64_t size, uint64_t flags)
  1805. {
  1806. struct amdgpu_bo_va_mapping *mapping, *tmp;
  1807. struct amdgpu_bo *bo = bo_va->base.bo;
  1808. struct amdgpu_vm *vm = bo_va->base.vm;
  1809. uint64_t eaddr;
  1810. /* validate the parameters */
  1811. if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
  1812. size == 0 || size & AMDGPU_GPU_PAGE_MASK)
  1813. return -EINVAL;
  1814. /* make sure object fit at this offset */
  1815. eaddr = saddr + size - 1;
  1816. if (saddr >= eaddr ||
  1817. (bo && offset + size > amdgpu_bo_size(bo)))
  1818. return -EINVAL;
  1819. saddr /= AMDGPU_GPU_PAGE_SIZE;
  1820. eaddr /= AMDGPU_GPU_PAGE_SIZE;
  1821. tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
  1822. if (tmp) {
  1823. /* bo and tmp overlap, invalid addr */
  1824. dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
  1825. "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
  1826. tmp->start, tmp->last + 1);
  1827. return -EINVAL;
  1828. }
  1829. mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
  1830. if (!mapping)
  1831. return -ENOMEM;
  1832. INIT_LIST_HEAD(&mapping->list);
  1833. mapping->start = saddr;
  1834. mapping->last = eaddr;
  1835. mapping->offset = offset;
  1836. mapping->flags = flags;
  1837. list_add(&mapping->list, &bo_va->invalids);
  1838. amdgpu_vm_it_insert(mapping, &vm->va);
  1839. if (flags & AMDGPU_PTE_PRT)
  1840. amdgpu_vm_prt_get(adev);
  1841. return 0;
  1842. }
  1843. /**
  1844. * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
  1845. *
  1846. * @adev: amdgpu_device pointer
  1847. * @bo_va: bo_va to store the address
  1848. * @saddr: where to map the BO
  1849. * @offset: requested offset in the BO
  1850. * @flags: attributes of pages (read/write/valid/etc.)
  1851. *
  1852. * Add a mapping of the BO at the specefied addr into the VM. Replace existing
  1853. * mappings as we do so.
  1854. * Returns 0 for success, error for failure.
  1855. *
  1856. * Object has to be reserved and unreserved outside!
  1857. */
  1858. int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
  1859. struct amdgpu_bo_va *bo_va,
  1860. uint64_t saddr, uint64_t offset,
  1861. uint64_t size, uint64_t flags)
  1862. {
  1863. struct amdgpu_bo_va_mapping *mapping;
  1864. struct amdgpu_bo *bo = bo_va->base.bo;
  1865. struct amdgpu_vm *vm = bo_va->base.vm;
  1866. uint64_t eaddr;
  1867. int r;
  1868. /* validate the parameters */
  1869. if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
  1870. size == 0 || size & AMDGPU_GPU_PAGE_MASK)
  1871. return -EINVAL;
  1872. /* make sure object fit at this offset */
  1873. eaddr = saddr + size - 1;
  1874. if (saddr >= eaddr ||
  1875. (bo && offset + size > amdgpu_bo_size(bo)))
  1876. return -EINVAL;
  1877. /* Allocate all the needed memory */
  1878. mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
  1879. if (!mapping)
  1880. return -ENOMEM;
  1881. r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
  1882. if (r) {
  1883. kfree(mapping);
  1884. return r;
  1885. }
  1886. saddr /= AMDGPU_GPU_PAGE_SIZE;
  1887. eaddr /= AMDGPU_GPU_PAGE_SIZE;
  1888. mapping->start = saddr;
  1889. mapping->last = eaddr;
  1890. mapping->offset = offset;
  1891. mapping->flags = flags;
  1892. list_add(&mapping->list, &bo_va->invalids);
  1893. amdgpu_vm_it_insert(mapping, &vm->va);
  1894. if (flags & AMDGPU_PTE_PRT)
  1895. amdgpu_vm_prt_get(adev);
  1896. return 0;
  1897. }
  1898. /**
  1899. * amdgpu_vm_bo_unmap - remove bo mapping from vm
  1900. *
  1901. * @adev: amdgpu_device pointer
  1902. * @bo_va: bo_va to remove the address from
  1903. * @saddr: where to the BO is mapped
  1904. *
  1905. * Remove a mapping of the BO at the specefied addr from the VM.
  1906. * Returns 0 for success, error for failure.
  1907. *
  1908. * Object has to be reserved and unreserved outside!
  1909. */
  1910. int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
  1911. struct amdgpu_bo_va *bo_va,
  1912. uint64_t saddr)
  1913. {
  1914. struct amdgpu_bo_va_mapping *mapping;
  1915. struct amdgpu_vm *vm = bo_va->base.vm;
  1916. bool valid = true;
  1917. saddr /= AMDGPU_GPU_PAGE_SIZE;
  1918. list_for_each_entry(mapping, &bo_va->valids, list) {
  1919. if (mapping->start == saddr)
  1920. break;
  1921. }
  1922. if (&mapping->list == &bo_va->valids) {
  1923. valid = false;
  1924. list_for_each_entry(mapping, &bo_va->invalids, list) {
  1925. if (mapping->start == saddr)
  1926. break;
  1927. }
  1928. if (&mapping->list == &bo_va->invalids)
  1929. return -ENOENT;
  1930. }
  1931. list_del(&mapping->list);
  1932. amdgpu_vm_it_remove(mapping, &vm->va);
  1933. trace_amdgpu_vm_bo_unmap(bo_va, mapping);
  1934. if (valid)
  1935. list_add(&mapping->list, &vm->freed);
  1936. else
  1937. amdgpu_vm_free_mapping(adev, vm, mapping,
  1938. bo_va->last_pt_update);
  1939. return 0;
  1940. }
  1941. /**
  1942. * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
  1943. *
  1944. * @adev: amdgpu_device pointer
  1945. * @vm: VM structure to use
  1946. * @saddr: start of the range
  1947. * @size: size of the range
  1948. *
  1949. * Remove all mappings in a range, split them as appropriate.
  1950. * Returns 0 for success, error for failure.
  1951. */
  1952. int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
  1953. struct amdgpu_vm *vm,
  1954. uint64_t saddr, uint64_t size)
  1955. {
  1956. struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
  1957. LIST_HEAD(removed);
  1958. uint64_t eaddr;
  1959. eaddr = saddr + size - 1;
  1960. saddr /= AMDGPU_GPU_PAGE_SIZE;
  1961. eaddr /= AMDGPU_GPU_PAGE_SIZE;
  1962. /* Allocate all the needed memory */
  1963. before = kzalloc(sizeof(*before), GFP_KERNEL);
  1964. if (!before)
  1965. return -ENOMEM;
  1966. INIT_LIST_HEAD(&before->list);
  1967. after = kzalloc(sizeof(*after), GFP_KERNEL);
  1968. if (!after) {
  1969. kfree(before);
  1970. return -ENOMEM;
  1971. }
  1972. INIT_LIST_HEAD(&after->list);
  1973. /* Now gather all removed mappings */
  1974. tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
  1975. while (tmp) {
  1976. /* Remember mapping split at the start */
  1977. if (tmp->start < saddr) {
  1978. before->start = tmp->start;
  1979. before->last = saddr - 1;
  1980. before->offset = tmp->offset;
  1981. before->flags = tmp->flags;
  1982. list_add(&before->list, &tmp->list);
  1983. }
  1984. /* Remember mapping split at the end */
  1985. if (tmp->last > eaddr) {
  1986. after->start = eaddr + 1;
  1987. after->last = tmp->last;
  1988. after->offset = tmp->offset;
  1989. after->offset += after->start - tmp->start;
  1990. after->flags = tmp->flags;
  1991. list_add(&after->list, &tmp->list);
  1992. }
  1993. list_del(&tmp->list);
  1994. list_add(&tmp->list, &removed);
  1995. tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
  1996. }
  1997. /* And free them up */
  1998. list_for_each_entry_safe(tmp, next, &removed, list) {
  1999. amdgpu_vm_it_remove(tmp, &vm->va);
  2000. list_del(&tmp->list);
  2001. if (tmp->start < saddr)
  2002. tmp->start = saddr;
  2003. if (tmp->last > eaddr)
  2004. tmp->last = eaddr;
  2005. list_add(&tmp->list, &vm->freed);
  2006. trace_amdgpu_vm_bo_unmap(NULL, tmp);
  2007. }
  2008. /* Insert partial mapping before the range */
  2009. if (!list_empty(&before->list)) {
  2010. amdgpu_vm_it_insert(before, &vm->va);
  2011. if (before->flags & AMDGPU_PTE_PRT)
  2012. amdgpu_vm_prt_get(adev);
  2013. } else {
  2014. kfree(before);
  2015. }
  2016. /* Insert partial mapping after the range */
  2017. if (!list_empty(&after->list)) {
  2018. amdgpu_vm_it_insert(after, &vm->va);
  2019. if (after->flags & AMDGPU_PTE_PRT)
  2020. amdgpu_vm_prt_get(adev);
  2021. } else {
  2022. kfree(after);
  2023. }
  2024. return 0;
  2025. }
  2026. /**
  2027. * amdgpu_vm_bo_rmv - remove a bo to a specific vm
  2028. *
  2029. * @adev: amdgpu_device pointer
  2030. * @bo_va: requested bo_va
  2031. *
  2032. * Remove @bo_va->bo from the requested vm.
  2033. *
  2034. * Object have to be reserved!
  2035. */
  2036. void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
  2037. struct amdgpu_bo_va *bo_va)
  2038. {
  2039. struct amdgpu_bo_va_mapping *mapping, *next;
  2040. struct amdgpu_vm *vm = bo_va->base.vm;
  2041. list_del(&bo_va->base.bo_list);
  2042. spin_lock(&vm->status_lock);
  2043. list_del(&bo_va->base.vm_status);
  2044. spin_unlock(&vm->status_lock);
  2045. list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
  2046. list_del(&mapping->list);
  2047. amdgpu_vm_it_remove(mapping, &vm->va);
  2048. trace_amdgpu_vm_bo_unmap(bo_va, mapping);
  2049. list_add(&mapping->list, &vm->freed);
  2050. }
  2051. list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
  2052. list_del(&mapping->list);
  2053. amdgpu_vm_it_remove(mapping, &vm->va);
  2054. amdgpu_vm_free_mapping(adev, vm, mapping,
  2055. bo_va->last_pt_update);
  2056. }
  2057. dma_fence_put(bo_va->last_pt_update);
  2058. kfree(bo_va);
  2059. }
  2060. /**
  2061. * amdgpu_vm_bo_invalidate - mark the bo as invalid
  2062. *
  2063. * @adev: amdgpu_device pointer
  2064. * @vm: requested vm
  2065. * @bo: amdgpu buffer object
  2066. *
  2067. * Mark @bo as invalid.
  2068. */
  2069. void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
  2070. struct amdgpu_bo *bo)
  2071. {
  2072. struct amdgpu_vm_bo_base *bo_base;
  2073. list_for_each_entry(bo_base, &bo->va, bo_list) {
  2074. spin_lock(&bo_base->vm->status_lock);
  2075. if (list_empty(&bo_base->vm_status))
  2076. list_add(&bo_base->vm_status,
  2077. &bo_base->vm->moved);
  2078. spin_unlock(&bo_base->vm->status_lock);
  2079. }
  2080. }
  2081. static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
  2082. {
  2083. /* Total bits covered by PD + PTs */
  2084. unsigned bits = ilog2(vm_size) + 18;
  2085. /* Make sure the PD is 4K in size up to 8GB address space.
  2086. Above that split equal between PD and PTs */
  2087. if (vm_size <= 8)
  2088. return (bits - 9);
  2089. else
  2090. return ((bits + 3) / 2);
  2091. }
  2092. /**
  2093. * amdgpu_vm_set_fragment_size - adjust fragment size in PTE
  2094. *
  2095. * @adev: amdgpu_device pointer
  2096. * @fragment_size_default: the default fragment size if it's set auto
  2097. */
  2098. void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev, uint32_t fragment_size_default)
  2099. {
  2100. if (amdgpu_vm_fragment_size == -1)
  2101. adev->vm_manager.fragment_size = fragment_size_default;
  2102. else
  2103. adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
  2104. }
  2105. /**
  2106. * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
  2107. *
  2108. * @adev: amdgpu_device pointer
  2109. * @vm_size: the default vm size if it's set auto
  2110. */
  2111. void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size, uint32_t fragment_size_default)
  2112. {
  2113. /* adjust vm size firstly */
  2114. if (amdgpu_vm_size == -1)
  2115. adev->vm_manager.vm_size = vm_size;
  2116. else
  2117. adev->vm_manager.vm_size = amdgpu_vm_size;
  2118. /* block size depends on vm size */
  2119. if (amdgpu_vm_block_size == -1)
  2120. adev->vm_manager.block_size =
  2121. amdgpu_vm_get_block_size(adev->vm_manager.vm_size);
  2122. else
  2123. adev->vm_manager.block_size = amdgpu_vm_block_size;
  2124. amdgpu_vm_set_fragment_size(adev, fragment_size_default);
  2125. DRM_INFO("vm size is %llu GB, block size is %u-bit, fragment size is %u-bit\n",
  2126. adev->vm_manager.vm_size, adev->vm_manager.block_size,
  2127. adev->vm_manager.fragment_size);
  2128. }
  2129. /**
  2130. * amdgpu_vm_init - initialize a vm instance
  2131. *
  2132. * @adev: amdgpu_device pointer
  2133. * @vm: requested vm
  2134. * @vm_context: Indicates if it GFX or Compute context
  2135. *
  2136. * Init @vm fields.
  2137. */
  2138. int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  2139. int vm_context)
  2140. {
  2141. const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
  2142. AMDGPU_VM_PTE_COUNT(adev) * 8);
  2143. unsigned ring_instance;
  2144. struct amdgpu_ring *ring;
  2145. struct amd_sched_rq *rq;
  2146. int r, i;
  2147. u64 flags;
  2148. uint64_t init_pde_value = 0;
  2149. vm->va = RB_ROOT;
  2150. vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter);
  2151. for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
  2152. vm->reserved_vmid[i] = NULL;
  2153. spin_lock_init(&vm->status_lock);
  2154. INIT_LIST_HEAD(&vm->moved);
  2155. INIT_LIST_HEAD(&vm->cleared);
  2156. INIT_LIST_HEAD(&vm->freed);
  2157. /* create scheduler entity for page table updates */
  2158. ring_instance = atomic_inc_return(&adev->vm_manager.vm_pte_next_ring);
  2159. ring_instance %= adev->vm_manager.vm_pte_num_rings;
  2160. ring = adev->vm_manager.vm_pte_rings[ring_instance];
  2161. rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
  2162. r = amd_sched_entity_init(&ring->sched, &vm->entity,
  2163. rq, amdgpu_sched_jobs);
  2164. if (r)
  2165. return r;
  2166. vm->pte_support_ats = false;
  2167. if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) {
  2168. vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
  2169. AMDGPU_VM_USE_CPU_FOR_COMPUTE);
  2170. if (adev->asic_type == CHIP_RAVEN) {
  2171. vm->pte_support_ats = true;
  2172. init_pde_value = AMDGPU_PTE_SYSTEM | AMDGPU_PDE_PTE;
  2173. }
  2174. } else
  2175. vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
  2176. AMDGPU_VM_USE_CPU_FOR_GFX);
  2177. DRM_DEBUG_DRIVER("VM update mode is %s\n",
  2178. vm->use_cpu_for_update ? "CPU" : "SDMA");
  2179. WARN_ONCE((vm->use_cpu_for_update & !amdgpu_vm_is_large_bar(adev)),
  2180. "CPU update of VM recommended only for large BAR system\n");
  2181. vm->last_dir_update = NULL;
  2182. flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
  2183. AMDGPU_GEM_CREATE_VRAM_CLEARED;
  2184. if (vm->use_cpu_for_update)
  2185. flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
  2186. else
  2187. flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
  2188. AMDGPU_GEM_CREATE_SHADOW);
  2189. r = amdgpu_bo_create(adev, amdgpu_vm_bo_size(adev, 0), align, true,
  2190. AMDGPU_GEM_DOMAIN_VRAM,
  2191. flags,
  2192. NULL, NULL, init_pde_value, &vm->root.bo);
  2193. if (r)
  2194. goto error_free_sched_entity;
  2195. r = amdgpu_bo_reserve(vm->root.bo, false);
  2196. if (r)
  2197. goto error_free_root;
  2198. vm->last_eviction_counter = atomic64_read(&adev->num_evictions);
  2199. if (vm->use_cpu_for_update) {
  2200. r = amdgpu_bo_kmap(vm->root.bo, NULL);
  2201. if (r)
  2202. goto error_free_root;
  2203. }
  2204. amdgpu_bo_unreserve(vm->root.bo);
  2205. return 0;
  2206. error_free_root:
  2207. amdgpu_bo_unref(&vm->root.bo->shadow);
  2208. amdgpu_bo_unref(&vm->root.bo);
  2209. vm->root.bo = NULL;
  2210. error_free_sched_entity:
  2211. amd_sched_entity_fini(&ring->sched, &vm->entity);
  2212. return r;
  2213. }
  2214. /**
  2215. * amdgpu_vm_free_levels - free PD/PT levels
  2216. *
  2217. * @level: PD/PT starting level to free
  2218. *
  2219. * Free the page directory or page table level and all sub levels.
  2220. */
  2221. static void amdgpu_vm_free_levels(struct amdgpu_vm_pt *level)
  2222. {
  2223. unsigned i;
  2224. if (level->bo) {
  2225. amdgpu_bo_unref(&level->bo->shadow);
  2226. amdgpu_bo_unref(&level->bo);
  2227. }
  2228. if (level->entries)
  2229. for (i = 0; i <= level->last_entry_used; i++)
  2230. amdgpu_vm_free_levels(&level->entries[i]);
  2231. kvfree(level->entries);
  2232. }
  2233. /**
  2234. * amdgpu_vm_fini - tear down a vm instance
  2235. *
  2236. * @adev: amdgpu_device pointer
  2237. * @vm: requested vm
  2238. *
  2239. * Tear down @vm.
  2240. * Unbind the VM and remove all bos from the vm bo list
  2241. */
  2242. void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
  2243. {
  2244. struct amdgpu_bo_va_mapping *mapping, *tmp;
  2245. bool prt_fini_needed = !!adev->gart.gart_funcs->set_prt;
  2246. int i;
  2247. amd_sched_entity_fini(vm->entity.sched, &vm->entity);
  2248. if (!RB_EMPTY_ROOT(&vm->va)) {
  2249. dev_err(adev->dev, "still active bo inside vm\n");
  2250. }
  2251. rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, rb) {
  2252. list_del(&mapping->list);
  2253. amdgpu_vm_it_remove(mapping, &vm->va);
  2254. kfree(mapping);
  2255. }
  2256. list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
  2257. if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
  2258. amdgpu_vm_prt_fini(adev, vm);
  2259. prt_fini_needed = false;
  2260. }
  2261. list_del(&mapping->list);
  2262. amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
  2263. }
  2264. amdgpu_vm_free_levels(&vm->root);
  2265. dma_fence_put(vm->last_dir_update);
  2266. for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
  2267. amdgpu_vm_free_reserved_vmid(adev, vm, i);
  2268. }
  2269. /**
  2270. * amdgpu_vm_manager_init - init the VM manager
  2271. *
  2272. * @adev: amdgpu_device pointer
  2273. *
  2274. * Initialize the VM manager structures
  2275. */
  2276. void amdgpu_vm_manager_init(struct amdgpu_device *adev)
  2277. {
  2278. unsigned i, j;
  2279. for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
  2280. struct amdgpu_vm_id_manager *id_mgr =
  2281. &adev->vm_manager.id_mgr[i];
  2282. mutex_init(&id_mgr->lock);
  2283. INIT_LIST_HEAD(&id_mgr->ids_lru);
  2284. atomic_set(&id_mgr->reserved_vmid_num, 0);
  2285. /* skip over VMID 0, since it is the system VM */
  2286. for (j = 1; j < id_mgr->num_ids; ++j) {
  2287. amdgpu_vm_reset_id(adev, i, j);
  2288. amdgpu_sync_create(&id_mgr->ids[i].active);
  2289. list_add_tail(&id_mgr->ids[j].list, &id_mgr->ids_lru);
  2290. }
  2291. }
  2292. adev->vm_manager.fence_context =
  2293. dma_fence_context_alloc(AMDGPU_MAX_RINGS);
  2294. for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
  2295. adev->vm_manager.seqno[i] = 0;
  2296. atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
  2297. atomic64_set(&adev->vm_manager.client_counter, 0);
  2298. spin_lock_init(&adev->vm_manager.prt_lock);
  2299. atomic_set(&adev->vm_manager.num_prt_users, 0);
  2300. /* If not overridden by the user, by default, only in large BAR systems
  2301. * Compute VM tables will be updated by CPU
  2302. */
  2303. #ifdef CONFIG_X86_64
  2304. if (amdgpu_vm_update_mode == -1) {
  2305. if (amdgpu_vm_is_large_bar(adev))
  2306. adev->vm_manager.vm_update_mode =
  2307. AMDGPU_VM_USE_CPU_FOR_COMPUTE;
  2308. else
  2309. adev->vm_manager.vm_update_mode = 0;
  2310. } else
  2311. adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
  2312. #else
  2313. adev->vm_manager.vm_update_mode = 0;
  2314. #endif
  2315. }
  2316. /**
  2317. * amdgpu_vm_manager_fini - cleanup VM manager
  2318. *
  2319. * @adev: amdgpu_device pointer
  2320. *
  2321. * Cleanup the VM manager and free resources.
  2322. */
  2323. void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
  2324. {
  2325. unsigned i, j;
  2326. for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
  2327. struct amdgpu_vm_id_manager *id_mgr =
  2328. &adev->vm_manager.id_mgr[i];
  2329. mutex_destroy(&id_mgr->lock);
  2330. for (j = 0; j < AMDGPU_NUM_VM; ++j) {
  2331. struct amdgpu_vm_id *id = &id_mgr->ids[j];
  2332. amdgpu_sync_free(&id->active);
  2333. dma_fence_put(id->flushed_updates);
  2334. dma_fence_put(id->last_flush);
  2335. }
  2336. }
  2337. }
  2338. int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
  2339. {
  2340. union drm_amdgpu_vm *args = data;
  2341. struct amdgpu_device *adev = dev->dev_private;
  2342. struct amdgpu_fpriv *fpriv = filp->driver_priv;
  2343. int r;
  2344. switch (args->in.op) {
  2345. case AMDGPU_VM_OP_RESERVE_VMID:
  2346. /* current, we only have requirement to reserve vmid from gfxhub */
  2347. r = amdgpu_vm_alloc_reserved_vmid(adev, &fpriv->vm,
  2348. AMDGPU_GFXHUB);
  2349. if (r)
  2350. return r;
  2351. break;
  2352. case AMDGPU_VM_OP_UNRESERVE_VMID:
  2353. amdgpu_vm_free_reserved_vmid(adev, &fpriv->vm, AMDGPU_GFXHUB);
  2354. break;
  2355. default:
  2356. return -EINVAL;
  2357. }
  2358. return 0;
  2359. }