amdgpu_vm.c 87 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508
  1. /*
  2. * Copyright 2008 Advanced Micro Devices, Inc.
  3. * Copyright 2008 Red Hat Inc.
  4. * Copyright 2009 Jerome Glisse.
  5. *
  6. * Permission is hereby granted, free of charge, to any person obtaining a
  7. * copy of this software and associated documentation files (the "Software"),
  8. * to deal in the Software without restriction, including without limitation
  9. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10. * and/or sell copies of the Software, and to permit persons to whom the
  11. * Software is furnished to do so, subject to the following conditions:
  12. *
  13. * The above copyright notice and this permission notice shall be included in
  14. * all copies or substantial portions of the Software.
  15. *
  16. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19. * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22. * OTHER DEALINGS IN THE SOFTWARE.
  23. *
  24. * Authors: Dave Airlie
  25. * Alex Deucher
  26. * Jerome Glisse
  27. */
  28. #include <linux/dma-fence-array.h>
  29. #include <linux/interval_tree_generic.h>
  30. #include <linux/idr.h>
  31. #include <drm/drmP.h>
  32. #include <drm/amdgpu_drm.h>
  33. #include "amdgpu.h"
  34. #include "amdgpu_trace.h"
  35. #include "amdgpu_amdkfd.h"
  36. #include "amdgpu_gmc.h"
  37. /**
  38. * DOC: GPUVM
  39. *
  40. * GPUVM is similar to the legacy gart on older asics, however
  41. * rather than there being a single global gart table
  42. * for the entire GPU, there are multiple VM page tables active
  43. * at any given time. The VM page tables can contain a mix
  44. * vram pages and system memory pages and system memory pages
  45. * can be mapped as snooped (cached system pages) or unsnooped
  46. * (uncached system pages).
  47. * Each VM has an ID associated with it and there is a page table
  48. * associated with each VMID. When execting a command buffer,
  49. * the kernel tells the the ring what VMID to use for that command
  50. * buffer. VMIDs are allocated dynamically as commands are submitted.
  51. * The userspace drivers maintain their own address space and the kernel
  52. * sets up their pages tables accordingly when they submit their
  53. * command buffers and a VMID is assigned.
  54. * Cayman/Trinity support up to 8 active VMs at any given time;
  55. * SI supports 16.
  56. */
  57. #define START(node) ((node)->start)
  58. #define LAST(node) ((node)->last)
  59. INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
  60. START, LAST, static, amdgpu_vm_it)
  61. #undef START
  62. #undef LAST
  63. /**
  64. * struct amdgpu_pte_update_params - Local structure
  65. *
  66. * Encapsulate some VM table update parameters to reduce
  67. * the number of function parameters
  68. *
  69. */
  70. struct amdgpu_pte_update_params {
  71. /**
  72. * @adev: amdgpu device we do this update for
  73. */
  74. struct amdgpu_device *adev;
  75. /**
  76. * @vm: optional amdgpu_vm we do this update for
  77. */
  78. struct amdgpu_vm *vm;
  79. /**
  80. * @src: address where to copy page table entries from
  81. */
  82. uint64_t src;
  83. /**
  84. * @ib: indirect buffer to fill with commands
  85. */
  86. struct amdgpu_ib *ib;
  87. /**
  88. * @func: Function which actually does the update
  89. */
  90. void (*func)(struct amdgpu_pte_update_params *params,
  91. struct amdgpu_bo *bo, uint64_t pe,
  92. uint64_t addr, unsigned count, uint32_t incr,
  93. uint64_t flags);
  94. /**
  95. * @pages_addr:
  96. *
  97. * DMA addresses to use for mapping, used during VM update by CPU
  98. */
  99. dma_addr_t *pages_addr;
  100. /**
  101. * @kptr:
  102. *
  103. * Kernel pointer of PD/PT BO that needs to be updated,
  104. * used during VM update by CPU
  105. */
  106. void *kptr;
  107. };
  108. /**
  109. * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback
  110. */
  111. struct amdgpu_prt_cb {
  112. /**
  113. * @adev: amdgpu device
  114. */
  115. struct amdgpu_device *adev;
  116. /**
  117. * @cb: callback
  118. */
  119. struct dma_fence_cb cb;
  120. };
  121. /**
  122. * amdgpu_vm_level_shift - return the addr shift for each level
  123. *
  124. * @adev: amdgpu_device pointer
  125. * @level: VMPT level
  126. *
  127. * Returns:
  128. * The number of bits the pfn needs to be right shifted for a level.
  129. */
  130. static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev,
  131. unsigned level)
  132. {
  133. unsigned shift = 0xff;
  134. switch (level) {
  135. case AMDGPU_VM_PDB2:
  136. case AMDGPU_VM_PDB1:
  137. case AMDGPU_VM_PDB0:
  138. shift = 9 * (AMDGPU_VM_PDB0 - level) +
  139. adev->vm_manager.block_size;
  140. break;
  141. case AMDGPU_VM_PTB:
  142. shift = 0;
  143. break;
  144. default:
  145. dev_err(adev->dev, "the level%d isn't supported.\n", level);
  146. }
  147. return shift;
  148. }
  149. /**
  150. * amdgpu_vm_num_entries - return the number of entries in a PD/PT
  151. *
  152. * @adev: amdgpu_device pointer
  153. * @level: VMPT level
  154. *
  155. * Returns:
  156. * The number of entries in a page directory or page table.
  157. */
  158. static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
  159. unsigned level)
  160. {
  161. unsigned shift = amdgpu_vm_level_shift(adev,
  162. adev->vm_manager.root_level);
  163. if (level == adev->vm_manager.root_level)
  164. /* For the root directory */
  165. return round_up(adev->vm_manager.max_pfn, 1 << shift) >> shift;
  166. else if (level != AMDGPU_VM_PTB)
  167. /* Everything in between */
  168. return 512;
  169. else
  170. /* For the page tables on the leaves */
  171. return AMDGPU_VM_PTE_COUNT(adev);
  172. }
  173. /**
  174. * amdgpu_vm_entries_mask - the mask to get the entry number of a PD/PT
  175. *
  176. * @adev: amdgpu_device pointer
  177. * @level: VMPT level
  178. *
  179. * Returns:
  180. * The mask to extract the entry number of a PD/PT from an address.
  181. */
  182. static uint32_t amdgpu_vm_entries_mask(struct amdgpu_device *adev,
  183. unsigned int level)
  184. {
  185. if (level <= adev->vm_manager.root_level)
  186. return 0xffffffff;
  187. else if (level != AMDGPU_VM_PTB)
  188. return 0x1ff;
  189. else
  190. return AMDGPU_VM_PTE_COUNT(adev) - 1;
  191. }
  192. /**
  193. * amdgpu_vm_bo_size - returns the size of the BOs in bytes
  194. *
  195. * @adev: amdgpu_device pointer
  196. * @level: VMPT level
  197. *
  198. * Returns:
  199. * The size of the BO for a page directory or page table in bytes.
  200. */
  201. static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level)
  202. {
  203. return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8);
  204. }
  205. /**
  206. * amdgpu_vm_bo_evicted - vm_bo is evicted
  207. *
  208. * @vm_bo: vm_bo which is evicted
  209. *
  210. * State for PDs/PTs and per VM BOs which are not at the location they should
  211. * be.
  212. */
  213. static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
  214. {
  215. struct amdgpu_vm *vm = vm_bo->vm;
  216. struct amdgpu_bo *bo = vm_bo->bo;
  217. vm_bo->moved = true;
  218. if (bo->tbo.type == ttm_bo_type_kernel)
  219. list_move(&vm_bo->vm_status, &vm->evicted);
  220. else
  221. list_move_tail(&vm_bo->vm_status, &vm->evicted);
  222. }
  223. /**
  224. * amdgpu_vm_bo_relocated - vm_bo is reloacted
  225. *
  226. * @vm_bo: vm_bo which is relocated
  227. *
  228. * State for PDs/PTs which needs to update their parent PD.
  229. */
  230. static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
  231. {
  232. list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
  233. }
  234. /**
  235. * amdgpu_vm_bo_moved - vm_bo is moved
  236. *
  237. * @vm_bo: vm_bo which is moved
  238. *
  239. * State for per VM BOs which are moved, but that change is not yet reflected
  240. * in the page tables.
  241. */
  242. static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
  243. {
  244. list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
  245. }
  246. /**
  247. * amdgpu_vm_bo_idle - vm_bo is idle
  248. *
  249. * @vm_bo: vm_bo which is now idle
  250. *
  251. * State for PDs/PTs and per VM BOs which have gone through the state machine
  252. * and are now idle.
  253. */
  254. static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
  255. {
  256. list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
  257. vm_bo->moved = false;
  258. }
  259. /**
  260. * amdgpu_vm_bo_invalidated - vm_bo is invalidated
  261. *
  262. * @vm_bo: vm_bo which is now invalidated
  263. *
  264. * State for normal BOs which are invalidated and that change not yet reflected
  265. * in the PTs.
  266. */
  267. static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
  268. {
  269. spin_lock(&vm_bo->vm->invalidated_lock);
  270. list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
  271. spin_unlock(&vm_bo->vm->invalidated_lock);
  272. }
  273. /**
  274. * amdgpu_vm_bo_done - vm_bo is done
  275. *
  276. * @vm_bo: vm_bo which is now done
  277. *
  278. * State for normal BOs which are invalidated and that change has been updated
  279. * in the PTs.
  280. */
  281. static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
  282. {
  283. spin_lock(&vm_bo->vm->invalidated_lock);
  284. list_del_init(&vm_bo->vm_status);
  285. spin_unlock(&vm_bo->vm->invalidated_lock);
  286. }
  287. /**
  288. * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm
  289. *
  290. * @base: base structure for tracking BO usage in a VM
  291. * @vm: vm to which bo is to be added
  292. * @bo: amdgpu buffer object
  293. *
  294. * Initialize a bo_va_base structure and add it to the appropriate lists
  295. *
  296. */
  297. static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
  298. struct amdgpu_vm *vm,
  299. struct amdgpu_bo *bo)
  300. {
  301. base->vm = vm;
  302. base->bo = bo;
  303. base->next = NULL;
  304. INIT_LIST_HEAD(&base->vm_status);
  305. if (!bo)
  306. return;
  307. base->next = bo->vm_bo;
  308. bo->vm_bo = base;
  309. if (bo->tbo.resv != vm->root.base.bo->tbo.resv)
  310. return;
  311. vm->bulk_moveable = false;
  312. if (bo->tbo.type == ttm_bo_type_kernel)
  313. amdgpu_vm_bo_relocated(base);
  314. else
  315. amdgpu_vm_bo_idle(base);
  316. if (bo->preferred_domains &
  317. amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type))
  318. return;
  319. /*
  320. * we checked all the prerequisites, but it looks like this per vm bo
  321. * is currently evicted. add the bo to the evicted list to make sure it
  322. * is validated on next vm use to avoid fault.
  323. * */
  324. amdgpu_vm_bo_evicted(base);
  325. }
  326. /**
  327. * amdgpu_vm_pt_parent - get the parent page directory
  328. *
  329. * @pt: child page table
  330. *
  331. * Helper to get the parent entry for the child page table. NULL if we are at
  332. * the root page directory.
  333. */
  334. static struct amdgpu_vm_pt *amdgpu_vm_pt_parent(struct amdgpu_vm_pt *pt)
  335. {
  336. struct amdgpu_bo *parent = pt->base.bo->parent;
  337. if (!parent)
  338. return NULL;
  339. return container_of(parent->vm_bo, struct amdgpu_vm_pt, base);
  340. }
  341. /**
  342. * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt
  343. */
  344. struct amdgpu_vm_pt_cursor {
  345. uint64_t pfn;
  346. struct amdgpu_vm_pt *parent;
  347. struct amdgpu_vm_pt *entry;
  348. unsigned level;
  349. };
  350. /**
  351. * amdgpu_vm_pt_start - start PD/PT walk
  352. *
  353. * @adev: amdgpu_device pointer
  354. * @vm: amdgpu_vm structure
  355. * @start: start address of the walk
  356. * @cursor: state to initialize
  357. *
  358. * Initialize a amdgpu_vm_pt_cursor to start a walk.
  359. */
  360. static void amdgpu_vm_pt_start(struct amdgpu_device *adev,
  361. struct amdgpu_vm *vm, uint64_t start,
  362. struct amdgpu_vm_pt_cursor *cursor)
  363. {
  364. cursor->pfn = start;
  365. cursor->parent = NULL;
  366. cursor->entry = &vm->root;
  367. cursor->level = adev->vm_manager.root_level;
  368. }
  369. /**
  370. * amdgpu_vm_pt_descendant - go to child node
  371. *
  372. * @adev: amdgpu_device pointer
  373. * @cursor: current state
  374. *
  375. * Walk to the child node of the current node.
  376. * Returns:
  377. * True if the walk was possible, false otherwise.
  378. */
  379. static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev,
  380. struct amdgpu_vm_pt_cursor *cursor)
  381. {
  382. unsigned mask, shift, idx;
  383. if (!cursor->entry->entries)
  384. return false;
  385. BUG_ON(!cursor->entry->base.bo);
  386. mask = amdgpu_vm_entries_mask(adev, cursor->level);
  387. shift = amdgpu_vm_level_shift(adev, cursor->level);
  388. ++cursor->level;
  389. idx = (cursor->pfn >> shift) & mask;
  390. cursor->parent = cursor->entry;
  391. cursor->entry = &cursor->entry->entries[idx];
  392. return true;
  393. }
  394. /**
  395. * amdgpu_vm_pt_sibling - go to sibling node
  396. *
  397. * @adev: amdgpu_device pointer
  398. * @cursor: current state
  399. *
  400. * Walk to the sibling node of the current node.
  401. * Returns:
  402. * True if the walk was possible, false otherwise.
  403. */
  404. static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev,
  405. struct amdgpu_vm_pt_cursor *cursor)
  406. {
  407. unsigned shift, num_entries;
  408. /* Root doesn't have a sibling */
  409. if (!cursor->parent)
  410. return false;
  411. /* Go to our parents and see if we got a sibling */
  412. shift = amdgpu_vm_level_shift(adev, cursor->level - 1);
  413. num_entries = amdgpu_vm_num_entries(adev, cursor->level - 1);
  414. if (cursor->entry == &cursor->parent->entries[num_entries - 1])
  415. return false;
  416. cursor->pfn += 1ULL << shift;
  417. cursor->pfn &= ~((1ULL << shift) - 1);
  418. ++cursor->entry;
  419. return true;
  420. }
  421. /**
  422. * amdgpu_vm_pt_ancestor - go to parent node
  423. *
  424. * @cursor: current state
  425. *
  426. * Walk to the parent node of the current node.
  427. * Returns:
  428. * True if the walk was possible, false otherwise.
  429. */
  430. static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor)
  431. {
  432. if (!cursor->parent)
  433. return false;
  434. --cursor->level;
  435. cursor->entry = cursor->parent;
  436. cursor->parent = amdgpu_vm_pt_parent(cursor->parent);
  437. return true;
  438. }
  439. /**
  440. * amdgpu_vm_pt_next - get next PD/PT in hieratchy
  441. *
  442. * @adev: amdgpu_device pointer
  443. * @cursor: current state
  444. *
  445. * Walk the PD/PT tree to the next node.
  446. */
  447. static void amdgpu_vm_pt_next(struct amdgpu_device *adev,
  448. struct amdgpu_vm_pt_cursor *cursor)
  449. {
  450. /* First try a newborn child */
  451. if (amdgpu_vm_pt_descendant(adev, cursor))
  452. return;
  453. /* If that didn't worked try to find a sibling */
  454. while (!amdgpu_vm_pt_sibling(adev, cursor)) {
  455. /* No sibling, go to our parents and grandparents */
  456. if (!amdgpu_vm_pt_ancestor(cursor)) {
  457. cursor->pfn = ~0ll;
  458. return;
  459. }
  460. }
  461. }
  462. /**
  463. * amdgpu_vm_pt_first_leaf - get first leaf PD/PT
  464. *
  465. * @adev: amdgpu_device pointer
  466. * @vm: amdgpu_vm structure
  467. * @start: start addr of the walk
  468. * @cursor: state to initialize
  469. *
  470. * Start a walk and go directly to the leaf node.
  471. */
  472. static void amdgpu_vm_pt_first_leaf(struct amdgpu_device *adev,
  473. struct amdgpu_vm *vm, uint64_t start,
  474. struct amdgpu_vm_pt_cursor *cursor)
  475. {
  476. amdgpu_vm_pt_start(adev, vm, start, cursor);
  477. while (amdgpu_vm_pt_descendant(adev, cursor));
  478. }
  479. /**
  480. * amdgpu_vm_pt_next_leaf - get next leaf PD/PT
  481. *
  482. * @adev: amdgpu_device pointer
  483. * @cursor: current state
  484. *
  485. * Walk the PD/PT tree to the next leaf node.
  486. */
  487. static void amdgpu_vm_pt_next_leaf(struct amdgpu_device *adev,
  488. struct amdgpu_vm_pt_cursor *cursor)
  489. {
  490. amdgpu_vm_pt_next(adev, cursor);
  491. if (cursor->pfn != ~0ll)
  492. while (amdgpu_vm_pt_descendant(adev, cursor));
  493. }
  494. /**
  495. * for_each_amdgpu_vm_pt_leaf - walk over all leaf PDs/PTs in the hierarchy
  496. */
  497. #define for_each_amdgpu_vm_pt_leaf(adev, vm, start, end, cursor) \
  498. for (amdgpu_vm_pt_first_leaf((adev), (vm), (start), &(cursor)); \
  499. (cursor).pfn <= end; amdgpu_vm_pt_next_leaf((adev), &(cursor)))
  500. /**
  501. * amdgpu_vm_pt_first_dfs - start a deep first search
  502. *
  503. * @adev: amdgpu_device structure
  504. * @vm: amdgpu_vm structure
  505. * @cursor: state to initialize
  506. *
  507. * Starts a deep first traversal of the PD/PT tree.
  508. */
  509. static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev,
  510. struct amdgpu_vm *vm,
  511. struct amdgpu_vm_pt_cursor *cursor)
  512. {
  513. amdgpu_vm_pt_start(adev, vm, 0, cursor);
  514. while (amdgpu_vm_pt_descendant(adev, cursor));
  515. }
  516. /**
  517. * amdgpu_vm_pt_next_dfs - get the next node for a deep first search
  518. *
  519. * @adev: amdgpu_device structure
  520. * @cursor: current state
  521. *
  522. * Move the cursor to the next node in a deep first search.
  523. */
  524. static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev,
  525. struct amdgpu_vm_pt_cursor *cursor)
  526. {
  527. if (!cursor->entry)
  528. return;
  529. if (!cursor->parent)
  530. cursor->entry = NULL;
  531. else if (amdgpu_vm_pt_sibling(adev, cursor))
  532. while (amdgpu_vm_pt_descendant(adev, cursor));
  533. else
  534. amdgpu_vm_pt_ancestor(cursor);
  535. }
  536. /**
  537. * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs
  538. */
  539. #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, cursor, entry) \
  540. for (amdgpu_vm_pt_first_dfs((adev), (vm), &(cursor)), \
  541. (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\
  542. (entry); (entry) = (cursor).entry, \
  543. amdgpu_vm_pt_next_dfs((adev), &(cursor)))
  544. /**
  545. * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
  546. *
  547. * @vm: vm providing the BOs
  548. * @validated: head of validation list
  549. * @entry: entry to add
  550. *
  551. * Add the page directory to the list of BOs to
  552. * validate for command submission.
  553. */
  554. void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
  555. struct list_head *validated,
  556. struct amdgpu_bo_list_entry *entry)
  557. {
  558. entry->priority = 0;
  559. entry->tv.bo = &vm->root.base.bo->tbo;
  560. entry->tv.shared = true;
  561. entry->user_pages = NULL;
  562. list_add(&entry->tv.head, validated);
  563. }
  564. /**
  565. * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU
  566. *
  567. * @adev: amdgpu device pointer
  568. * @vm: vm providing the BOs
  569. *
  570. * Move all BOs to the end of LRU and remember their positions to put them
  571. * together.
  572. */
  573. void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
  574. struct amdgpu_vm *vm)
  575. {
  576. struct ttm_bo_global *glob = adev->mman.bdev.glob;
  577. struct amdgpu_vm_bo_base *bo_base;
  578. if (vm->bulk_moveable) {
  579. spin_lock(&glob->lru_lock);
  580. ttm_bo_bulk_move_lru_tail(&vm->lru_bulk_move);
  581. spin_unlock(&glob->lru_lock);
  582. return;
  583. }
  584. memset(&vm->lru_bulk_move, 0, sizeof(vm->lru_bulk_move));
  585. spin_lock(&glob->lru_lock);
  586. list_for_each_entry(bo_base, &vm->idle, vm_status) {
  587. struct amdgpu_bo *bo = bo_base->bo;
  588. if (!bo->parent)
  589. continue;
  590. ttm_bo_move_to_lru_tail(&bo->tbo, &vm->lru_bulk_move);
  591. if (bo->shadow)
  592. ttm_bo_move_to_lru_tail(&bo->shadow->tbo,
  593. &vm->lru_bulk_move);
  594. }
  595. spin_unlock(&glob->lru_lock);
  596. vm->bulk_moveable = true;
  597. }
  598. /**
  599. * amdgpu_vm_validate_pt_bos - validate the page table BOs
  600. *
  601. * @adev: amdgpu device pointer
  602. * @vm: vm providing the BOs
  603. * @validate: callback to do the validation
  604. * @param: parameter for the validation callback
  605. *
  606. * Validate the page table BOs on command submission if neccessary.
  607. *
  608. * Returns:
  609. * Validation result.
  610. */
  611. int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  612. int (*validate)(void *p, struct amdgpu_bo *bo),
  613. void *param)
  614. {
  615. struct amdgpu_vm_bo_base *bo_base, *tmp;
  616. int r = 0;
  617. vm->bulk_moveable &= list_empty(&vm->evicted);
  618. list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {
  619. struct amdgpu_bo *bo = bo_base->bo;
  620. r = validate(param, bo);
  621. if (r)
  622. break;
  623. if (bo->tbo.type != ttm_bo_type_kernel) {
  624. amdgpu_vm_bo_moved(bo_base);
  625. } else {
  626. if (vm->use_cpu_for_update)
  627. r = amdgpu_bo_kmap(bo, NULL);
  628. else
  629. r = amdgpu_ttm_alloc_gart(&bo->tbo);
  630. if (r)
  631. break;
  632. if (bo->shadow) {
  633. r = amdgpu_ttm_alloc_gart(&bo->shadow->tbo);
  634. if (r)
  635. break;
  636. }
  637. amdgpu_vm_bo_relocated(bo_base);
  638. }
  639. }
  640. return r;
  641. }
  642. /**
  643. * amdgpu_vm_ready - check VM is ready for updates
  644. *
  645. * @vm: VM to check
  646. *
  647. * Check if all VM PDs/PTs are ready for updates
  648. *
  649. * Returns:
  650. * True if eviction list is empty.
  651. */
  652. bool amdgpu_vm_ready(struct amdgpu_vm *vm)
  653. {
  654. return list_empty(&vm->evicted);
  655. }
  656. /**
  657. * amdgpu_vm_clear_bo - initially clear the PDs/PTs
  658. *
  659. * @adev: amdgpu_device pointer
  660. * @vm: VM to clear BO from
  661. * @bo: BO to clear
  662. * @level: level this BO is at
  663. * @pte_support_ats: indicate ATS support from PTE
  664. *
  665. * Root PD needs to be reserved when calling this.
  666. *
  667. * Returns:
  668. * 0 on success, errno otherwise.
  669. */
  670. static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
  671. struct amdgpu_vm *vm, struct amdgpu_bo *bo,
  672. unsigned level, bool pte_support_ats)
  673. {
  674. struct ttm_operation_ctx ctx = { true, false };
  675. struct dma_fence *fence = NULL;
  676. unsigned entries, ats_entries;
  677. struct amdgpu_ring *ring;
  678. struct amdgpu_job *job;
  679. uint64_t addr;
  680. int r;
  681. entries = amdgpu_bo_size(bo) / 8;
  682. if (pte_support_ats) {
  683. if (level == adev->vm_manager.root_level) {
  684. ats_entries = amdgpu_vm_level_shift(adev, level);
  685. ats_entries += AMDGPU_GPU_PAGE_SHIFT;
  686. ats_entries = AMDGPU_GMC_HOLE_START >> ats_entries;
  687. ats_entries = min(ats_entries, entries);
  688. entries -= ats_entries;
  689. } else {
  690. ats_entries = entries;
  691. entries = 0;
  692. }
  693. } else {
  694. ats_entries = 0;
  695. }
  696. ring = container_of(vm->entity.rq->sched, struct amdgpu_ring, sched);
  697. r = reservation_object_reserve_shared(bo->tbo.resv);
  698. if (r)
  699. return r;
  700. r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
  701. if (r)
  702. goto error;
  703. r = amdgpu_ttm_alloc_gart(&bo->tbo);
  704. if (r)
  705. return r;
  706. r = amdgpu_job_alloc_with_ib(adev, 64, &job);
  707. if (r)
  708. goto error;
  709. addr = amdgpu_bo_gpu_offset(bo);
  710. if (ats_entries) {
  711. uint64_t ats_value;
  712. ats_value = AMDGPU_PTE_DEFAULT_ATC;
  713. if (level != AMDGPU_VM_PTB)
  714. ats_value |= AMDGPU_PDE_PTE;
  715. amdgpu_vm_set_pte_pde(adev, &job->ibs[0], addr, 0,
  716. ats_entries, 0, ats_value);
  717. addr += ats_entries * 8;
  718. }
  719. if (entries)
  720. amdgpu_vm_set_pte_pde(adev, &job->ibs[0], addr, 0,
  721. entries, 0, 0);
  722. amdgpu_ring_pad_ib(ring, &job->ibs[0]);
  723. WARN_ON(job->ibs[0].length_dw > 64);
  724. r = amdgpu_sync_resv(adev, &job->sync, bo->tbo.resv,
  725. AMDGPU_FENCE_OWNER_UNDEFINED, false);
  726. if (r)
  727. goto error_free;
  728. r = amdgpu_job_submit(job, &vm->entity, AMDGPU_FENCE_OWNER_UNDEFINED,
  729. &fence);
  730. if (r)
  731. goto error_free;
  732. amdgpu_bo_fence(bo, fence, true);
  733. dma_fence_put(fence);
  734. if (bo->shadow)
  735. return amdgpu_vm_clear_bo(adev, vm, bo->shadow,
  736. level, pte_support_ats);
  737. return 0;
  738. error_free:
  739. amdgpu_job_free(job);
  740. error:
  741. return r;
  742. }
  743. /**
  744. * amdgpu_vm_bo_param - fill in parameters for PD/PT allocation
  745. *
  746. * @adev: amdgpu_device pointer
  747. * @vm: requesting vm
  748. * @bp: resulting BO allocation parameters
  749. */
  750. static void amdgpu_vm_bo_param(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  751. int level, struct amdgpu_bo_param *bp)
  752. {
  753. memset(bp, 0, sizeof(*bp));
  754. bp->size = amdgpu_vm_bo_size(adev, level);
  755. bp->byte_align = AMDGPU_GPU_PAGE_SIZE;
  756. bp->domain = AMDGPU_GEM_DOMAIN_VRAM;
  757. if (bp->size <= PAGE_SIZE && adev->asic_type >= CHIP_VEGA10 &&
  758. adev->flags & AMD_IS_APU)
  759. bp->domain |= AMDGPU_GEM_DOMAIN_GTT;
  760. bp->domain = amdgpu_bo_get_preferred_pin_domain(adev, bp->domain);
  761. bp->flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
  762. AMDGPU_GEM_CREATE_CPU_GTT_USWC;
  763. if (vm->use_cpu_for_update)
  764. bp->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
  765. else if (!vm->root.base.bo || vm->root.base.bo->shadow)
  766. bp->flags |= AMDGPU_GEM_CREATE_SHADOW;
  767. bp->type = ttm_bo_type_kernel;
  768. if (vm->root.base.bo)
  769. bp->resv = vm->root.base.bo->tbo.resv;
  770. }
  771. /**
  772. * amdgpu_vm_alloc_pts - Allocate page tables.
  773. *
  774. * @adev: amdgpu_device pointer
  775. * @vm: VM to allocate page tables for
  776. * @saddr: Start address which needs to be allocated
  777. * @size: Size from start address we need.
  778. *
  779. * Make sure the page directories and page tables are allocated
  780. *
  781. * Returns:
  782. * 0 on success, errno otherwise.
  783. */
  784. int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
  785. struct amdgpu_vm *vm,
  786. uint64_t saddr, uint64_t size)
  787. {
  788. struct amdgpu_vm_pt_cursor cursor;
  789. struct amdgpu_bo *pt;
  790. bool ats = false;
  791. uint64_t eaddr;
  792. int r;
  793. /* validate the parameters */
  794. if (saddr & AMDGPU_GPU_PAGE_MASK || size & AMDGPU_GPU_PAGE_MASK)
  795. return -EINVAL;
  796. eaddr = saddr + size - 1;
  797. if (vm->pte_support_ats)
  798. ats = saddr < AMDGPU_GMC_HOLE_START;
  799. saddr /= AMDGPU_GPU_PAGE_SIZE;
  800. eaddr /= AMDGPU_GPU_PAGE_SIZE;
  801. if (eaddr >= adev->vm_manager.max_pfn) {
  802. dev_err(adev->dev, "va above limit (0x%08llX >= 0x%08llX)\n",
  803. eaddr, adev->vm_manager.max_pfn);
  804. return -EINVAL;
  805. }
  806. for_each_amdgpu_vm_pt_leaf(adev, vm, saddr, eaddr, cursor) {
  807. struct amdgpu_vm_pt *entry = cursor.entry;
  808. struct amdgpu_bo_param bp;
  809. if (cursor.level < AMDGPU_VM_PTB) {
  810. unsigned num_entries;
  811. num_entries = amdgpu_vm_num_entries(adev, cursor.level);
  812. entry->entries = kvmalloc_array(num_entries,
  813. sizeof(*entry->entries),
  814. GFP_KERNEL |
  815. __GFP_ZERO);
  816. if (!entry->entries)
  817. return -ENOMEM;
  818. }
  819. if (entry->base.bo)
  820. continue;
  821. amdgpu_vm_bo_param(adev, vm, cursor.level, &bp);
  822. r = amdgpu_bo_create(adev, &bp, &pt);
  823. if (r)
  824. return r;
  825. r = amdgpu_vm_clear_bo(adev, vm, pt, cursor.level, ats);
  826. if (r)
  827. goto error_free_pt;
  828. if (vm->use_cpu_for_update) {
  829. r = amdgpu_bo_kmap(pt, NULL);
  830. if (r)
  831. goto error_free_pt;
  832. }
  833. /* Keep a reference to the root directory to avoid
  834. * freeing them up in the wrong order.
  835. */
  836. pt->parent = amdgpu_bo_ref(cursor.parent->base.bo);
  837. amdgpu_vm_bo_base_init(&entry->base, vm, pt);
  838. }
  839. return 0;
  840. error_free_pt:
  841. amdgpu_bo_unref(&pt->shadow);
  842. amdgpu_bo_unref(&pt);
  843. return r;
  844. }
  845. /**
  846. * amdgpu_vm_free_pts - free PD/PT levels
  847. *
  848. * @adev: amdgpu device structure
  849. * @vm: amdgpu vm structure
  850. *
  851. * Free the page directory or page table level and all sub levels.
  852. */
  853. static void amdgpu_vm_free_pts(struct amdgpu_device *adev,
  854. struct amdgpu_vm *vm)
  855. {
  856. struct amdgpu_vm_pt_cursor cursor;
  857. struct amdgpu_vm_pt *entry;
  858. for_each_amdgpu_vm_pt_dfs_safe(adev, vm, cursor, entry) {
  859. if (entry->base.bo) {
  860. entry->base.bo->vm_bo = NULL;
  861. list_del(&entry->base.vm_status);
  862. amdgpu_bo_unref(&entry->base.bo->shadow);
  863. amdgpu_bo_unref(&entry->base.bo);
  864. }
  865. kvfree(entry->entries);
  866. }
  867. BUG_ON(vm->root.base.bo);
  868. }
  869. /**
  870. * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
  871. *
  872. * @adev: amdgpu_device pointer
  873. */
  874. void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
  875. {
  876. const struct amdgpu_ip_block *ip_block;
  877. bool has_compute_vm_bug;
  878. struct amdgpu_ring *ring;
  879. int i;
  880. has_compute_vm_bug = false;
  881. ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
  882. if (ip_block) {
  883. /* Compute has a VM bug for GFX version < 7.
  884. Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
  885. if (ip_block->version->major <= 7)
  886. has_compute_vm_bug = true;
  887. else if (ip_block->version->major == 8)
  888. if (adev->gfx.mec_fw_version < 673)
  889. has_compute_vm_bug = true;
  890. }
  891. for (i = 0; i < adev->num_rings; i++) {
  892. ring = adev->rings[i];
  893. if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
  894. /* only compute rings */
  895. ring->has_compute_vm_bug = has_compute_vm_bug;
  896. else
  897. ring->has_compute_vm_bug = false;
  898. }
  899. }
  900. /**
  901. * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job.
  902. *
  903. * @ring: ring on which the job will be submitted
  904. * @job: job to submit
  905. *
  906. * Returns:
  907. * True if sync is needed.
  908. */
  909. bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
  910. struct amdgpu_job *job)
  911. {
  912. struct amdgpu_device *adev = ring->adev;
  913. unsigned vmhub = ring->funcs->vmhub;
  914. struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  915. struct amdgpu_vmid *id;
  916. bool gds_switch_needed;
  917. bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug;
  918. if (job->vmid == 0)
  919. return false;
  920. id = &id_mgr->ids[job->vmid];
  921. gds_switch_needed = ring->funcs->emit_gds_switch && (
  922. id->gds_base != job->gds_base ||
  923. id->gds_size != job->gds_size ||
  924. id->gws_base != job->gws_base ||
  925. id->gws_size != job->gws_size ||
  926. id->oa_base != job->oa_base ||
  927. id->oa_size != job->oa_size);
  928. if (amdgpu_vmid_had_gpu_reset(adev, id))
  929. return true;
  930. return vm_flush_needed || gds_switch_needed;
  931. }
  932. /**
  933. * amdgpu_vm_flush - hardware flush the vm
  934. *
  935. * @ring: ring to use for flush
  936. * @job: related job
  937. * @need_pipe_sync: is pipe sync needed
  938. *
  939. * Emit a VM flush when it is necessary.
  940. *
  941. * Returns:
  942. * 0 on success, errno otherwise.
  943. */
  944. int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync)
  945. {
  946. struct amdgpu_device *adev = ring->adev;
  947. unsigned vmhub = ring->funcs->vmhub;
  948. struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  949. struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
  950. bool gds_switch_needed = ring->funcs->emit_gds_switch && (
  951. id->gds_base != job->gds_base ||
  952. id->gds_size != job->gds_size ||
  953. id->gws_base != job->gws_base ||
  954. id->gws_size != job->gws_size ||
  955. id->oa_base != job->oa_base ||
  956. id->oa_size != job->oa_size);
  957. bool vm_flush_needed = job->vm_needs_flush;
  958. bool pasid_mapping_needed = id->pasid != job->pasid ||
  959. !id->pasid_mapping ||
  960. !dma_fence_is_signaled(id->pasid_mapping);
  961. struct dma_fence *fence = NULL;
  962. unsigned patch_offset = 0;
  963. int r;
  964. if (amdgpu_vmid_had_gpu_reset(adev, id)) {
  965. gds_switch_needed = true;
  966. vm_flush_needed = true;
  967. pasid_mapping_needed = true;
  968. }
  969. gds_switch_needed &= !!ring->funcs->emit_gds_switch;
  970. vm_flush_needed &= !!ring->funcs->emit_vm_flush &&
  971. job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET;
  972. pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping &&
  973. ring->funcs->emit_wreg;
  974. if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
  975. return 0;
  976. if (ring->funcs->init_cond_exec)
  977. patch_offset = amdgpu_ring_init_cond_exec(ring);
  978. if (need_pipe_sync)
  979. amdgpu_ring_emit_pipeline_sync(ring);
  980. if (vm_flush_needed) {
  981. trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
  982. amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);
  983. }
  984. if (pasid_mapping_needed)
  985. amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
  986. if (vm_flush_needed || pasid_mapping_needed) {
  987. r = amdgpu_fence_emit(ring, &fence, 0);
  988. if (r)
  989. return r;
  990. }
  991. if (vm_flush_needed) {
  992. mutex_lock(&id_mgr->lock);
  993. dma_fence_put(id->last_flush);
  994. id->last_flush = dma_fence_get(fence);
  995. id->current_gpu_reset_count =
  996. atomic_read(&adev->gpu_reset_counter);
  997. mutex_unlock(&id_mgr->lock);
  998. }
  999. if (pasid_mapping_needed) {
  1000. id->pasid = job->pasid;
  1001. dma_fence_put(id->pasid_mapping);
  1002. id->pasid_mapping = dma_fence_get(fence);
  1003. }
  1004. dma_fence_put(fence);
  1005. if (ring->funcs->emit_gds_switch && gds_switch_needed) {
  1006. id->gds_base = job->gds_base;
  1007. id->gds_size = job->gds_size;
  1008. id->gws_base = job->gws_base;
  1009. id->gws_size = job->gws_size;
  1010. id->oa_base = job->oa_base;
  1011. id->oa_size = job->oa_size;
  1012. amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
  1013. job->gds_size, job->gws_base,
  1014. job->gws_size, job->oa_base,
  1015. job->oa_size);
  1016. }
  1017. if (ring->funcs->patch_cond_exec)
  1018. amdgpu_ring_patch_cond_exec(ring, patch_offset);
  1019. /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
  1020. if (ring->funcs->emit_switch_buffer) {
  1021. amdgpu_ring_emit_switch_buffer(ring);
  1022. amdgpu_ring_emit_switch_buffer(ring);
  1023. }
  1024. return 0;
  1025. }
  1026. /**
  1027. * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
  1028. *
  1029. * @vm: requested vm
  1030. * @bo: requested buffer object
  1031. *
  1032. * Find @bo inside the requested vm.
  1033. * Search inside the @bos vm list for the requested vm
  1034. * Returns the found bo_va or NULL if none is found
  1035. *
  1036. * Object has to be reserved!
  1037. *
  1038. * Returns:
  1039. * Found bo_va or NULL.
  1040. */
  1041. struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
  1042. struct amdgpu_bo *bo)
  1043. {
  1044. struct amdgpu_vm_bo_base *base;
  1045. for (base = bo->vm_bo; base; base = base->next) {
  1046. if (base->vm != vm)
  1047. continue;
  1048. return container_of(base, struct amdgpu_bo_va, base);
  1049. }
  1050. return NULL;
  1051. }
  1052. /**
  1053. * amdgpu_vm_do_set_ptes - helper to call the right asic function
  1054. *
  1055. * @params: see amdgpu_pte_update_params definition
  1056. * @bo: PD/PT to update
  1057. * @pe: addr of the page entry
  1058. * @addr: dst addr to write into pe
  1059. * @count: number of page entries to update
  1060. * @incr: increase next addr by incr bytes
  1061. * @flags: hw access flags
  1062. *
  1063. * Traces the parameters and calls the right asic functions
  1064. * to setup the page table using the DMA.
  1065. */
  1066. static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params *params,
  1067. struct amdgpu_bo *bo,
  1068. uint64_t pe, uint64_t addr,
  1069. unsigned count, uint32_t incr,
  1070. uint64_t flags)
  1071. {
  1072. pe += amdgpu_bo_gpu_offset(bo);
  1073. trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
  1074. if (count < 3) {
  1075. amdgpu_vm_write_pte(params->adev, params->ib, pe,
  1076. addr | flags, count, incr);
  1077. } else {
  1078. amdgpu_vm_set_pte_pde(params->adev, params->ib, pe, addr,
  1079. count, incr, flags);
  1080. }
  1081. }
  1082. /**
  1083. * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
  1084. *
  1085. * @params: see amdgpu_pte_update_params definition
  1086. * @bo: PD/PT to update
  1087. * @pe: addr of the page entry
  1088. * @addr: dst addr to write into pe
  1089. * @count: number of page entries to update
  1090. * @incr: increase next addr by incr bytes
  1091. * @flags: hw access flags
  1092. *
  1093. * Traces the parameters and calls the DMA function to copy the PTEs.
  1094. */
  1095. static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
  1096. struct amdgpu_bo *bo,
  1097. uint64_t pe, uint64_t addr,
  1098. unsigned count, uint32_t incr,
  1099. uint64_t flags)
  1100. {
  1101. uint64_t src = (params->src + (addr >> 12) * 8);
  1102. pe += amdgpu_bo_gpu_offset(bo);
  1103. trace_amdgpu_vm_copy_ptes(pe, src, count);
  1104. amdgpu_vm_copy_pte(params->adev, params->ib, pe, src, count);
  1105. }
  1106. /**
  1107. * amdgpu_vm_map_gart - Resolve gart mapping of addr
  1108. *
  1109. * @pages_addr: optional DMA address to use for lookup
  1110. * @addr: the unmapped addr
  1111. *
  1112. * Look up the physical address of the page that the pte resolves
  1113. * to.
  1114. *
  1115. * Returns:
  1116. * The pointer for the page table entry.
  1117. */
  1118. static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
  1119. {
  1120. uint64_t result;
  1121. /* page table offset */
  1122. result = pages_addr[addr >> PAGE_SHIFT];
  1123. /* in case cpu page size != gpu page size*/
  1124. result |= addr & (~PAGE_MASK);
  1125. result &= 0xFFFFFFFFFFFFF000ULL;
  1126. return result;
  1127. }
  1128. /**
  1129. * amdgpu_vm_cpu_set_ptes - helper to update page tables via CPU
  1130. *
  1131. * @params: see amdgpu_pte_update_params definition
  1132. * @bo: PD/PT to update
  1133. * @pe: kmap addr of the page entry
  1134. * @addr: dst addr to write into pe
  1135. * @count: number of page entries to update
  1136. * @incr: increase next addr by incr bytes
  1137. * @flags: hw access flags
  1138. *
  1139. * Write count number of PT/PD entries directly.
  1140. */
  1141. static void amdgpu_vm_cpu_set_ptes(struct amdgpu_pte_update_params *params,
  1142. struct amdgpu_bo *bo,
  1143. uint64_t pe, uint64_t addr,
  1144. unsigned count, uint32_t incr,
  1145. uint64_t flags)
  1146. {
  1147. unsigned int i;
  1148. uint64_t value;
  1149. pe += (unsigned long)amdgpu_bo_kptr(bo);
  1150. trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
  1151. for (i = 0; i < count; i++) {
  1152. value = params->pages_addr ?
  1153. amdgpu_vm_map_gart(params->pages_addr, addr) :
  1154. addr;
  1155. amdgpu_gmc_set_pte_pde(params->adev, (void *)(uintptr_t)pe,
  1156. i, value, flags);
  1157. addr += incr;
  1158. }
  1159. }
  1160. /**
  1161. * amdgpu_vm_wait_pd - Wait for PT BOs to be free.
  1162. *
  1163. * @adev: amdgpu_device pointer
  1164. * @vm: related vm
  1165. * @owner: fence owner
  1166. *
  1167. * Returns:
  1168. * 0 on success, errno otherwise.
  1169. */
  1170. static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  1171. void *owner)
  1172. {
  1173. struct amdgpu_sync sync;
  1174. int r;
  1175. amdgpu_sync_create(&sync);
  1176. amdgpu_sync_resv(adev, &sync, vm->root.base.bo->tbo.resv, owner, false);
  1177. r = amdgpu_sync_wait(&sync, true);
  1178. amdgpu_sync_free(&sync);
  1179. return r;
  1180. }
  1181. /**
  1182. * amdgpu_vm_update_func - helper to call update function
  1183. *
  1184. * Calls the update function for both the given BO as well as its shadow.
  1185. */
  1186. static void amdgpu_vm_update_func(struct amdgpu_pte_update_params *params,
  1187. struct amdgpu_bo *bo,
  1188. uint64_t pe, uint64_t addr,
  1189. unsigned count, uint32_t incr,
  1190. uint64_t flags)
  1191. {
  1192. if (bo->shadow)
  1193. params->func(params, bo->shadow, pe, addr, count, incr, flags);
  1194. params->func(params, bo, pe, addr, count, incr, flags);
  1195. }
  1196. /*
  1197. * amdgpu_vm_update_pde - update a single level in the hierarchy
  1198. *
  1199. * @param: parameters for the update
  1200. * @vm: requested vm
  1201. * @parent: parent directory
  1202. * @entry: entry to update
  1203. *
  1204. * Makes sure the requested entry in parent is up to date.
  1205. */
  1206. static void amdgpu_vm_update_pde(struct amdgpu_pte_update_params *params,
  1207. struct amdgpu_vm *vm,
  1208. struct amdgpu_vm_pt *parent,
  1209. struct amdgpu_vm_pt *entry)
  1210. {
  1211. struct amdgpu_bo *bo = parent->base.bo, *pbo;
  1212. uint64_t pde, pt, flags;
  1213. unsigned level;
  1214. /* Don't update huge pages here */
  1215. if (entry->huge)
  1216. return;
  1217. for (level = 0, pbo = bo->parent; pbo; ++level)
  1218. pbo = pbo->parent;
  1219. level += params->adev->vm_manager.root_level;
  1220. amdgpu_gmc_get_pde_for_bo(entry->base.bo, level, &pt, &flags);
  1221. pde = (entry - parent->entries) * 8;
  1222. amdgpu_vm_update_func(params, bo, pde, pt, 1, 0, flags);
  1223. }
  1224. /*
  1225. * amdgpu_vm_invalidate_pds - mark all PDs as invalid
  1226. *
  1227. * @adev: amdgpu_device pointer
  1228. * @vm: related vm
  1229. *
  1230. * Mark all PD level as invalid after an error.
  1231. */
  1232. static void amdgpu_vm_invalidate_pds(struct amdgpu_device *adev,
  1233. struct amdgpu_vm *vm)
  1234. {
  1235. struct amdgpu_vm_pt_cursor cursor;
  1236. struct amdgpu_vm_pt *entry;
  1237. for_each_amdgpu_vm_pt_dfs_safe(adev, vm, cursor, entry)
  1238. if (entry->base.bo && !entry->base.moved)
  1239. amdgpu_vm_bo_relocated(&entry->base);
  1240. }
  1241. /*
  1242. * amdgpu_vm_update_directories - make sure that all directories are valid
  1243. *
  1244. * @adev: amdgpu_device pointer
  1245. * @vm: requested vm
  1246. *
  1247. * Makes sure all directories are up to date.
  1248. *
  1249. * Returns:
  1250. * 0 for success, error for failure.
  1251. */
  1252. int amdgpu_vm_update_directories(struct amdgpu_device *adev,
  1253. struct amdgpu_vm *vm)
  1254. {
  1255. struct amdgpu_pte_update_params params;
  1256. struct amdgpu_job *job;
  1257. unsigned ndw = 0;
  1258. int r = 0;
  1259. if (list_empty(&vm->relocated))
  1260. return 0;
  1261. restart:
  1262. memset(&params, 0, sizeof(params));
  1263. params.adev = adev;
  1264. if (vm->use_cpu_for_update) {
  1265. r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
  1266. if (unlikely(r))
  1267. return r;
  1268. params.func = amdgpu_vm_cpu_set_ptes;
  1269. } else {
  1270. ndw = 512 * 8;
  1271. r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
  1272. if (r)
  1273. return r;
  1274. params.ib = &job->ibs[0];
  1275. params.func = amdgpu_vm_do_set_ptes;
  1276. }
  1277. while (!list_empty(&vm->relocated)) {
  1278. struct amdgpu_vm_pt *pt, *entry;
  1279. entry = list_first_entry(&vm->relocated, struct amdgpu_vm_pt,
  1280. base.vm_status);
  1281. amdgpu_vm_bo_idle(&entry->base);
  1282. pt = amdgpu_vm_pt_parent(entry);
  1283. if (!pt)
  1284. continue;
  1285. amdgpu_vm_update_pde(&params, vm, pt, entry);
  1286. if (!vm->use_cpu_for_update &&
  1287. (ndw - params.ib->length_dw) < 32)
  1288. break;
  1289. }
  1290. if (vm->use_cpu_for_update) {
  1291. /* Flush HDP */
  1292. mb();
  1293. amdgpu_asic_flush_hdp(adev, NULL);
  1294. } else if (params.ib->length_dw == 0) {
  1295. amdgpu_job_free(job);
  1296. } else {
  1297. struct amdgpu_bo *root = vm->root.base.bo;
  1298. struct amdgpu_ring *ring;
  1299. struct dma_fence *fence;
  1300. ring = container_of(vm->entity.rq->sched, struct amdgpu_ring,
  1301. sched);
  1302. amdgpu_ring_pad_ib(ring, params.ib);
  1303. amdgpu_sync_resv(adev, &job->sync, root->tbo.resv,
  1304. AMDGPU_FENCE_OWNER_VM, false);
  1305. WARN_ON(params.ib->length_dw > ndw);
  1306. r = amdgpu_job_submit(job, &vm->entity, AMDGPU_FENCE_OWNER_VM,
  1307. &fence);
  1308. if (r)
  1309. goto error;
  1310. amdgpu_bo_fence(root, fence, true);
  1311. dma_fence_put(vm->last_update);
  1312. vm->last_update = fence;
  1313. }
  1314. if (!list_empty(&vm->relocated))
  1315. goto restart;
  1316. return 0;
  1317. error:
  1318. amdgpu_vm_invalidate_pds(adev, vm);
  1319. amdgpu_job_free(job);
  1320. return r;
  1321. }
  1322. /**
  1323. * amdgpu_vm_update_huge - figure out parameters for PTE updates
  1324. *
  1325. * Make sure to set the right flags for the PTEs at the desired level.
  1326. */
  1327. static void amdgpu_vm_update_huge(struct amdgpu_pte_update_params *params,
  1328. struct amdgpu_bo *bo, unsigned level,
  1329. uint64_t pe, uint64_t addr,
  1330. unsigned count, uint32_t incr,
  1331. uint64_t flags)
  1332. {
  1333. if (level != AMDGPU_VM_PTB) {
  1334. flags |= AMDGPU_PDE_PTE;
  1335. amdgpu_gmc_get_vm_pde(params->adev, level, &addr, &flags);
  1336. }
  1337. amdgpu_vm_update_func(params, bo, pe, addr, count, incr, flags);
  1338. }
  1339. /**
  1340. * amdgpu_vm_fragment - get fragment for PTEs
  1341. *
  1342. * @params: see amdgpu_pte_update_params definition
  1343. * @start: first PTE to handle
  1344. * @end: last PTE to handle
  1345. * @flags: hw mapping flags
  1346. * @frag: resulting fragment size
  1347. * @frag_end: end of this fragment
  1348. *
  1349. * Returns the first possible fragment for the start and end address.
  1350. */
  1351. static void amdgpu_vm_fragment(struct amdgpu_pte_update_params *params,
  1352. uint64_t start, uint64_t end, uint64_t flags,
  1353. unsigned int *frag, uint64_t *frag_end)
  1354. {
  1355. /**
  1356. * The MC L1 TLB supports variable sized pages, based on a fragment
  1357. * field in the PTE. When this field is set to a non-zero value, page
  1358. * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
  1359. * flags are considered valid for all PTEs within the fragment range
  1360. * and corresponding mappings are assumed to be physically contiguous.
  1361. *
  1362. * The L1 TLB can store a single PTE for the whole fragment,
  1363. * significantly increasing the space available for translation
  1364. * caching. This leads to large improvements in throughput when the
  1365. * TLB is under pressure.
  1366. *
  1367. * The L2 TLB distributes small and large fragments into two
  1368. * asymmetric partitions. The large fragment cache is significantly
  1369. * larger. Thus, we try to use large fragments wherever possible.
  1370. * Userspace can support this by aligning virtual base address and
  1371. * allocation size to the fragment size.
  1372. *
  1373. * Starting with Vega10 the fragment size only controls the L1. The L2
  1374. * is now directly feed with small/huge/giant pages from the walker.
  1375. */
  1376. unsigned max_frag;
  1377. if (params->adev->asic_type < CHIP_VEGA10)
  1378. max_frag = params->adev->vm_manager.fragment_size;
  1379. else
  1380. max_frag = 31;
  1381. /* system pages are non continuously */
  1382. if (params->src) {
  1383. *frag = 0;
  1384. *frag_end = end;
  1385. return;
  1386. }
  1387. /* This intentionally wraps around if no bit is set */
  1388. *frag = min((unsigned)ffs(start) - 1, (unsigned)fls64(end - start) - 1);
  1389. if (*frag >= max_frag) {
  1390. *frag = max_frag;
  1391. *frag_end = end & ~((1ULL << max_frag) - 1);
  1392. } else {
  1393. *frag_end = start + (1 << *frag);
  1394. }
  1395. }
  1396. /**
  1397. * amdgpu_vm_update_ptes - make sure that page tables are valid
  1398. *
  1399. * @params: see amdgpu_pte_update_params definition
  1400. * @start: start of GPU address range
  1401. * @end: end of GPU address range
  1402. * @dst: destination address to map to, the next dst inside the function
  1403. * @flags: mapping flags
  1404. *
  1405. * Update the page tables in the range @start - @end.
  1406. *
  1407. * Returns:
  1408. * 0 for success, -EINVAL for failure.
  1409. */
  1410. static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
  1411. uint64_t start, uint64_t end,
  1412. uint64_t dst, uint64_t flags)
  1413. {
  1414. struct amdgpu_device *adev = params->adev;
  1415. struct amdgpu_vm_pt_cursor cursor;
  1416. uint64_t frag_start = start, frag_end;
  1417. unsigned int frag;
  1418. /* figure out the initial fragment */
  1419. amdgpu_vm_fragment(params, frag_start, end, flags, &frag, &frag_end);
  1420. /* walk over the address space and update the PTs */
  1421. amdgpu_vm_pt_start(adev, params->vm, start, &cursor);
  1422. while (cursor.pfn < end) {
  1423. struct amdgpu_bo *pt = cursor.entry->base.bo;
  1424. unsigned shift, parent_shift, mask;
  1425. uint64_t incr, entry_end, pe_start;
  1426. if (!pt)
  1427. return -ENOENT;
  1428. /* The root level can't be a huge page */
  1429. if (cursor.level == adev->vm_manager.root_level) {
  1430. if (!amdgpu_vm_pt_descendant(adev, &cursor))
  1431. return -ENOENT;
  1432. continue;
  1433. }
  1434. /* If it isn't already handled it can't be a huge page */
  1435. if (cursor.entry->huge) {
  1436. /* Add the entry to the relocated list to update it. */
  1437. cursor.entry->huge = false;
  1438. amdgpu_vm_bo_relocated(&cursor.entry->base);
  1439. }
  1440. shift = amdgpu_vm_level_shift(adev, cursor.level);
  1441. parent_shift = amdgpu_vm_level_shift(adev, cursor.level - 1);
  1442. if (adev->asic_type < CHIP_VEGA10) {
  1443. /* No huge page support before GMC v9 */
  1444. if (cursor.level != AMDGPU_VM_PTB) {
  1445. if (!amdgpu_vm_pt_descendant(adev, &cursor))
  1446. return -ENOENT;
  1447. continue;
  1448. }
  1449. } else if (frag < shift) {
  1450. /* We can't use this level when the fragment size is
  1451. * smaller than the address shift. Go to the next
  1452. * child entry and try again.
  1453. */
  1454. if (!amdgpu_vm_pt_descendant(adev, &cursor))
  1455. return -ENOENT;
  1456. continue;
  1457. } else if (frag >= parent_shift) {
  1458. /* If the fragment size is even larger than the parent
  1459. * shift we should go up one level and check it again.
  1460. */
  1461. if (!amdgpu_vm_pt_ancestor(&cursor))
  1462. return -ENOENT;
  1463. continue;
  1464. }
  1465. /* Looks good so far, calculate parameters for the update */
  1466. incr = AMDGPU_GPU_PAGE_SIZE << shift;
  1467. mask = amdgpu_vm_entries_mask(adev, cursor.level);
  1468. pe_start = ((cursor.pfn >> shift) & mask) * 8;
  1469. entry_end = (mask + 1) << shift;
  1470. entry_end += cursor.pfn & ~(entry_end - 1);
  1471. entry_end = min(entry_end, end);
  1472. do {
  1473. uint64_t upd_end = min(entry_end, frag_end);
  1474. unsigned nptes = (upd_end - frag_start) >> shift;
  1475. amdgpu_vm_update_huge(params, pt, cursor.level,
  1476. pe_start, dst, nptes, incr,
  1477. flags | AMDGPU_PTE_FRAG(frag));
  1478. pe_start += nptes * 8;
  1479. dst += nptes * AMDGPU_GPU_PAGE_SIZE << shift;
  1480. frag_start = upd_end;
  1481. if (frag_start >= frag_end) {
  1482. /* figure out the next fragment */
  1483. amdgpu_vm_fragment(params, frag_start, end,
  1484. flags, &frag, &frag_end);
  1485. if (frag < shift)
  1486. break;
  1487. }
  1488. } while (frag_start < entry_end);
  1489. if (amdgpu_vm_pt_descendant(adev, &cursor)) {
  1490. /* Mark all child entries as huge */
  1491. while (cursor.pfn < frag_start) {
  1492. cursor.entry->huge = true;
  1493. amdgpu_vm_pt_next(adev, &cursor);
  1494. }
  1495. } else if (frag >= shift) {
  1496. /* or just move on to the next on the same level. */
  1497. amdgpu_vm_pt_next(adev, &cursor);
  1498. }
  1499. }
  1500. return 0;
  1501. }
  1502. /**
  1503. * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
  1504. *
  1505. * @adev: amdgpu_device pointer
  1506. * @exclusive: fence we need to sync to
  1507. * @pages_addr: DMA addresses to use for mapping
  1508. * @vm: requested vm
  1509. * @start: start of mapped range
  1510. * @last: last mapped entry
  1511. * @flags: flags for the entries
  1512. * @addr: addr to set the area to
  1513. * @fence: optional resulting fence
  1514. *
  1515. * Fill in the page table entries between @start and @last.
  1516. *
  1517. * Returns:
  1518. * 0 for success, -EINVAL for failure.
  1519. */
  1520. static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
  1521. struct dma_fence *exclusive,
  1522. dma_addr_t *pages_addr,
  1523. struct amdgpu_vm *vm,
  1524. uint64_t start, uint64_t last,
  1525. uint64_t flags, uint64_t addr,
  1526. struct dma_fence **fence)
  1527. {
  1528. struct amdgpu_ring *ring;
  1529. void *owner = AMDGPU_FENCE_OWNER_VM;
  1530. unsigned nptes, ncmds, ndw;
  1531. struct amdgpu_job *job;
  1532. struct amdgpu_pte_update_params params;
  1533. struct dma_fence *f = NULL;
  1534. int r;
  1535. memset(&params, 0, sizeof(params));
  1536. params.adev = adev;
  1537. params.vm = vm;
  1538. /* sync to everything on unmapping */
  1539. if (!(flags & AMDGPU_PTE_VALID))
  1540. owner = AMDGPU_FENCE_OWNER_UNDEFINED;
  1541. if (vm->use_cpu_for_update) {
  1542. /* params.src is used as flag to indicate system Memory */
  1543. if (pages_addr)
  1544. params.src = ~0;
  1545. /* Wait for PT BOs to be free. PTs share the same resv. object
  1546. * as the root PD BO
  1547. */
  1548. r = amdgpu_vm_wait_pd(adev, vm, owner);
  1549. if (unlikely(r))
  1550. return r;
  1551. params.func = amdgpu_vm_cpu_set_ptes;
  1552. params.pages_addr = pages_addr;
  1553. return amdgpu_vm_update_ptes(&params, start, last + 1,
  1554. addr, flags);
  1555. }
  1556. ring = container_of(vm->entity.rq->sched, struct amdgpu_ring, sched);
  1557. nptes = last - start + 1;
  1558. /*
  1559. * reserve space for two commands every (1 << BLOCK_SIZE)
  1560. * entries or 2k dwords (whatever is smaller)
  1561. *
  1562. * The second command is for the shadow pagetables.
  1563. */
  1564. if (vm->root.base.bo->shadow)
  1565. ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1) * 2;
  1566. else
  1567. ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1);
  1568. /* padding, etc. */
  1569. ndw = 64;
  1570. if (pages_addr) {
  1571. /* copy commands needed */
  1572. ndw += ncmds * adev->vm_manager.vm_pte_funcs->copy_pte_num_dw;
  1573. /* and also PTEs */
  1574. ndw += nptes * 2;
  1575. params.func = amdgpu_vm_do_copy_ptes;
  1576. } else {
  1577. /* set page commands needed */
  1578. ndw += ncmds * 10;
  1579. /* extra commands for begin/end fragments */
  1580. if (vm->root.base.bo->shadow)
  1581. ndw += 2 * 10 * adev->vm_manager.fragment_size * 2;
  1582. else
  1583. ndw += 2 * 10 * adev->vm_manager.fragment_size;
  1584. params.func = amdgpu_vm_do_set_ptes;
  1585. }
  1586. r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
  1587. if (r)
  1588. return r;
  1589. params.ib = &job->ibs[0];
  1590. if (pages_addr) {
  1591. uint64_t *pte;
  1592. unsigned i;
  1593. /* Put the PTEs at the end of the IB. */
  1594. i = ndw - nptes * 2;
  1595. pte= (uint64_t *)&(job->ibs->ptr[i]);
  1596. params.src = job->ibs->gpu_addr + i * 4;
  1597. for (i = 0; i < nptes; ++i) {
  1598. pte[i] = amdgpu_vm_map_gart(pages_addr, addr + i *
  1599. AMDGPU_GPU_PAGE_SIZE);
  1600. pte[i] |= flags;
  1601. }
  1602. addr = 0;
  1603. }
  1604. r = amdgpu_sync_fence(adev, &job->sync, exclusive, false);
  1605. if (r)
  1606. goto error_free;
  1607. r = amdgpu_sync_resv(adev, &job->sync, vm->root.base.bo->tbo.resv,
  1608. owner, false);
  1609. if (r)
  1610. goto error_free;
  1611. r = reservation_object_reserve_shared(vm->root.base.bo->tbo.resv);
  1612. if (r)
  1613. goto error_free;
  1614. r = amdgpu_vm_update_ptes(&params, start, last + 1, addr, flags);
  1615. if (r)
  1616. goto error_free;
  1617. amdgpu_ring_pad_ib(ring, params.ib);
  1618. WARN_ON(params.ib->length_dw > ndw);
  1619. r = amdgpu_job_submit(job, &vm->entity, AMDGPU_FENCE_OWNER_VM, &f);
  1620. if (r)
  1621. goto error_free;
  1622. amdgpu_bo_fence(vm->root.base.bo, f, true);
  1623. dma_fence_put(*fence);
  1624. *fence = f;
  1625. return 0;
  1626. error_free:
  1627. amdgpu_job_free(job);
  1628. return r;
  1629. }
  1630. /**
  1631. * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
  1632. *
  1633. * @adev: amdgpu_device pointer
  1634. * @exclusive: fence we need to sync to
  1635. * @pages_addr: DMA addresses to use for mapping
  1636. * @vm: requested vm
  1637. * @mapping: mapped range and flags to use for the update
  1638. * @flags: HW flags for the mapping
  1639. * @nodes: array of drm_mm_nodes with the MC addresses
  1640. * @fence: optional resulting fence
  1641. *
  1642. * Split the mapping into smaller chunks so that each update fits
  1643. * into a SDMA IB.
  1644. *
  1645. * Returns:
  1646. * 0 for success, -EINVAL for failure.
  1647. */
  1648. static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
  1649. struct dma_fence *exclusive,
  1650. dma_addr_t *pages_addr,
  1651. struct amdgpu_vm *vm,
  1652. struct amdgpu_bo_va_mapping *mapping,
  1653. uint64_t flags,
  1654. struct drm_mm_node *nodes,
  1655. struct dma_fence **fence)
  1656. {
  1657. unsigned min_linear_pages = 1 << adev->vm_manager.fragment_size;
  1658. uint64_t pfn, start = mapping->start;
  1659. int r;
  1660. /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
  1661. * but in case of something, we filter the flags in first place
  1662. */
  1663. if (!(mapping->flags & AMDGPU_PTE_READABLE))
  1664. flags &= ~AMDGPU_PTE_READABLE;
  1665. if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
  1666. flags &= ~AMDGPU_PTE_WRITEABLE;
  1667. flags &= ~AMDGPU_PTE_EXECUTABLE;
  1668. flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
  1669. flags &= ~AMDGPU_PTE_MTYPE_MASK;
  1670. flags |= (mapping->flags & AMDGPU_PTE_MTYPE_MASK);
  1671. if ((mapping->flags & AMDGPU_PTE_PRT) &&
  1672. (adev->asic_type >= CHIP_VEGA10)) {
  1673. flags |= AMDGPU_PTE_PRT;
  1674. flags &= ~AMDGPU_PTE_VALID;
  1675. }
  1676. trace_amdgpu_vm_bo_update(mapping);
  1677. pfn = mapping->offset >> PAGE_SHIFT;
  1678. if (nodes) {
  1679. while (pfn >= nodes->size) {
  1680. pfn -= nodes->size;
  1681. ++nodes;
  1682. }
  1683. }
  1684. do {
  1685. dma_addr_t *dma_addr = NULL;
  1686. uint64_t max_entries;
  1687. uint64_t addr, last;
  1688. if (nodes) {
  1689. addr = nodes->start << PAGE_SHIFT;
  1690. max_entries = (nodes->size - pfn) *
  1691. AMDGPU_GPU_PAGES_IN_CPU_PAGE;
  1692. } else {
  1693. addr = 0;
  1694. max_entries = S64_MAX;
  1695. }
  1696. if (pages_addr) {
  1697. uint64_t count;
  1698. max_entries = min(max_entries, 16ull * 1024ull);
  1699. for (count = 1;
  1700. count < max_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
  1701. ++count) {
  1702. uint64_t idx = pfn + count;
  1703. if (pages_addr[idx] !=
  1704. (pages_addr[idx - 1] + PAGE_SIZE))
  1705. break;
  1706. }
  1707. if (count < min_linear_pages) {
  1708. addr = pfn << PAGE_SHIFT;
  1709. dma_addr = pages_addr;
  1710. } else {
  1711. addr = pages_addr[pfn];
  1712. max_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
  1713. }
  1714. } else if (flags & AMDGPU_PTE_VALID) {
  1715. addr += adev->vm_manager.vram_base_offset;
  1716. addr += pfn << PAGE_SHIFT;
  1717. }
  1718. last = min((uint64_t)mapping->last, start + max_entries - 1);
  1719. r = amdgpu_vm_bo_update_mapping(adev, exclusive, dma_addr, vm,
  1720. start, last, flags, addr,
  1721. fence);
  1722. if (r)
  1723. return r;
  1724. pfn += (last - start + 1) / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
  1725. if (nodes && nodes->size == pfn) {
  1726. pfn = 0;
  1727. ++nodes;
  1728. }
  1729. start = last + 1;
  1730. } while (unlikely(start != mapping->last + 1));
  1731. return 0;
  1732. }
  1733. /**
  1734. * amdgpu_vm_bo_update - update all BO mappings in the vm page table
  1735. *
  1736. * @adev: amdgpu_device pointer
  1737. * @bo_va: requested BO and VM object
  1738. * @clear: if true clear the entries
  1739. *
  1740. * Fill in the page table entries for @bo_va.
  1741. *
  1742. * Returns:
  1743. * 0 for success, -EINVAL for failure.
  1744. */
  1745. int amdgpu_vm_bo_update(struct amdgpu_device *adev,
  1746. struct amdgpu_bo_va *bo_va,
  1747. bool clear)
  1748. {
  1749. struct amdgpu_bo *bo = bo_va->base.bo;
  1750. struct amdgpu_vm *vm = bo_va->base.vm;
  1751. struct amdgpu_bo_va_mapping *mapping;
  1752. dma_addr_t *pages_addr = NULL;
  1753. struct ttm_mem_reg *mem;
  1754. struct drm_mm_node *nodes;
  1755. struct dma_fence *exclusive, **last_update;
  1756. uint64_t flags;
  1757. int r;
  1758. if (clear || !bo) {
  1759. mem = NULL;
  1760. nodes = NULL;
  1761. exclusive = NULL;
  1762. } else {
  1763. struct ttm_dma_tt *ttm;
  1764. mem = &bo->tbo.mem;
  1765. nodes = mem->mm_node;
  1766. if (mem->mem_type == TTM_PL_TT) {
  1767. ttm = container_of(bo->tbo.ttm, struct ttm_dma_tt, ttm);
  1768. pages_addr = ttm->dma_address;
  1769. }
  1770. exclusive = reservation_object_get_excl(bo->tbo.resv);
  1771. }
  1772. if (bo)
  1773. flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
  1774. else
  1775. flags = 0x0;
  1776. if (clear || (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv))
  1777. last_update = &vm->last_update;
  1778. else
  1779. last_update = &bo_va->last_pt_update;
  1780. if (!clear && bo_va->base.moved) {
  1781. bo_va->base.moved = false;
  1782. list_splice_init(&bo_va->valids, &bo_va->invalids);
  1783. } else if (bo_va->cleared != clear) {
  1784. list_splice_init(&bo_va->valids, &bo_va->invalids);
  1785. }
  1786. list_for_each_entry(mapping, &bo_va->invalids, list) {
  1787. r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm,
  1788. mapping, flags, nodes,
  1789. last_update);
  1790. if (r)
  1791. return r;
  1792. }
  1793. if (vm->use_cpu_for_update) {
  1794. /* Flush HDP */
  1795. mb();
  1796. amdgpu_asic_flush_hdp(adev, NULL);
  1797. }
  1798. /* If the BO is not in its preferred location add it back to
  1799. * the evicted list so that it gets validated again on the
  1800. * next command submission.
  1801. */
  1802. if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
  1803. uint32_t mem_type = bo->tbo.mem.mem_type;
  1804. if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(mem_type)))
  1805. amdgpu_vm_bo_evicted(&bo_va->base);
  1806. else
  1807. amdgpu_vm_bo_idle(&bo_va->base);
  1808. } else {
  1809. amdgpu_vm_bo_done(&bo_va->base);
  1810. }
  1811. list_splice_init(&bo_va->invalids, &bo_va->valids);
  1812. bo_va->cleared = clear;
  1813. if (trace_amdgpu_vm_bo_mapping_enabled()) {
  1814. list_for_each_entry(mapping, &bo_va->valids, list)
  1815. trace_amdgpu_vm_bo_mapping(mapping);
  1816. }
  1817. return 0;
  1818. }
  1819. /**
  1820. * amdgpu_vm_update_prt_state - update the global PRT state
  1821. *
  1822. * @adev: amdgpu_device pointer
  1823. */
  1824. static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
  1825. {
  1826. unsigned long flags;
  1827. bool enable;
  1828. spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
  1829. enable = !!atomic_read(&adev->vm_manager.num_prt_users);
  1830. adev->gmc.gmc_funcs->set_prt(adev, enable);
  1831. spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
  1832. }
  1833. /**
  1834. * amdgpu_vm_prt_get - add a PRT user
  1835. *
  1836. * @adev: amdgpu_device pointer
  1837. */
  1838. static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
  1839. {
  1840. if (!adev->gmc.gmc_funcs->set_prt)
  1841. return;
  1842. if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
  1843. amdgpu_vm_update_prt_state(adev);
  1844. }
  1845. /**
  1846. * amdgpu_vm_prt_put - drop a PRT user
  1847. *
  1848. * @adev: amdgpu_device pointer
  1849. */
  1850. static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
  1851. {
  1852. if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
  1853. amdgpu_vm_update_prt_state(adev);
  1854. }
  1855. /**
  1856. * amdgpu_vm_prt_cb - callback for updating the PRT status
  1857. *
  1858. * @fence: fence for the callback
  1859. * @_cb: the callback function
  1860. */
  1861. static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
  1862. {
  1863. struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
  1864. amdgpu_vm_prt_put(cb->adev);
  1865. kfree(cb);
  1866. }
  1867. /**
  1868. * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
  1869. *
  1870. * @adev: amdgpu_device pointer
  1871. * @fence: fence for the callback
  1872. */
  1873. static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
  1874. struct dma_fence *fence)
  1875. {
  1876. struct amdgpu_prt_cb *cb;
  1877. if (!adev->gmc.gmc_funcs->set_prt)
  1878. return;
  1879. cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
  1880. if (!cb) {
  1881. /* Last resort when we are OOM */
  1882. if (fence)
  1883. dma_fence_wait(fence, false);
  1884. amdgpu_vm_prt_put(adev);
  1885. } else {
  1886. cb->adev = adev;
  1887. if (!fence || dma_fence_add_callback(fence, &cb->cb,
  1888. amdgpu_vm_prt_cb))
  1889. amdgpu_vm_prt_cb(fence, &cb->cb);
  1890. }
  1891. }
  1892. /**
  1893. * amdgpu_vm_free_mapping - free a mapping
  1894. *
  1895. * @adev: amdgpu_device pointer
  1896. * @vm: requested vm
  1897. * @mapping: mapping to be freed
  1898. * @fence: fence of the unmap operation
  1899. *
  1900. * Free a mapping and make sure we decrease the PRT usage count if applicable.
  1901. */
  1902. static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
  1903. struct amdgpu_vm *vm,
  1904. struct amdgpu_bo_va_mapping *mapping,
  1905. struct dma_fence *fence)
  1906. {
  1907. if (mapping->flags & AMDGPU_PTE_PRT)
  1908. amdgpu_vm_add_prt_cb(adev, fence);
  1909. kfree(mapping);
  1910. }
  1911. /**
  1912. * amdgpu_vm_prt_fini - finish all prt mappings
  1913. *
  1914. * @adev: amdgpu_device pointer
  1915. * @vm: requested vm
  1916. *
  1917. * Register a cleanup callback to disable PRT support after VM dies.
  1918. */
  1919. static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
  1920. {
  1921. struct reservation_object *resv = vm->root.base.bo->tbo.resv;
  1922. struct dma_fence *excl, **shared;
  1923. unsigned i, shared_count;
  1924. int r;
  1925. r = reservation_object_get_fences_rcu(resv, &excl,
  1926. &shared_count, &shared);
  1927. if (r) {
  1928. /* Not enough memory to grab the fence list, as last resort
  1929. * block for all the fences to complete.
  1930. */
  1931. reservation_object_wait_timeout_rcu(resv, true, false,
  1932. MAX_SCHEDULE_TIMEOUT);
  1933. return;
  1934. }
  1935. /* Add a callback for each fence in the reservation object */
  1936. amdgpu_vm_prt_get(adev);
  1937. amdgpu_vm_add_prt_cb(adev, excl);
  1938. for (i = 0; i < shared_count; ++i) {
  1939. amdgpu_vm_prt_get(adev);
  1940. amdgpu_vm_add_prt_cb(adev, shared[i]);
  1941. }
  1942. kfree(shared);
  1943. }
  1944. /**
  1945. * amdgpu_vm_clear_freed - clear freed BOs in the PT
  1946. *
  1947. * @adev: amdgpu_device pointer
  1948. * @vm: requested vm
  1949. * @fence: optional resulting fence (unchanged if no work needed to be done
  1950. * or if an error occurred)
  1951. *
  1952. * Make sure all freed BOs are cleared in the PT.
  1953. * PTs have to be reserved and mutex must be locked!
  1954. *
  1955. * Returns:
  1956. * 0 for success.
  1957. *
  1958. */
  1959. int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
  1960. struct amdgpu_vm *vm,
  1961. struct dma_fence **fence)
  1962. {
  1963. struct amdgpu_bo_va_mapping *mapping;
  1964. uint64_t init_pte_value = 0;
  1965. struct dma_fence *f = NULL;
  1966. int r;
  1967. while (!list_empty(&vm->freed)) {
  1968. mapping = list_first_entry(&vm->freed,
  1969. struct amdgpu_bo_va_mapping, list);
  1970. list_del(&mapping->list);
  1971. if (vm->pte_support_ats &&
  1972. mapping->start < AMDGPU_GMC_HOLE_START)
  1973. init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
  1974. r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm,
  1975. mapping->start, mapping->last,
  1976. init_pte_value, 0, &f);
  1977. amdgpu_vm_free_mapping(adev, vm, mapping, f);
  1978. if (r) {
  1979. dma_fence_put(f);
  1980. return r;
  1981. }
  1982. }
  1983. if (fence && f) {
  1984. dma_fence_put(*fence);
  1985. *fence = f;
  1986. } else {
  1987. dma_fence_put(f);
  1988. }
  1989. return 0;
  1990. }
  1991. /**
  1992. * amdgpu_vm_handle_moved - handle moved BOs in the PT
  1993. *
  1994. * @adev: amdgpu_device pointer
  1995. * @vm: requested vm
  1996. *
  1997. * Make sure all BOs which are moved are updated in the PTs.
  1998. *
  1999. * Returns:
  2000. * 0 for success.
  2001. *
  2002. * PTs have to be reserved!
  2003. */
  2004. int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
  2005. struct amdgpu_vm *vm)
  2006. {
  2007. struct amdgpu_bo_va *bo_va, *tmp;
  2008. struct reservation_object *resv;
  2009. bool clear;
  2010. int r;
  2011. list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
  2012. /* Per VM BOs never need to bo cleared in the page tables */
  2013. r = amdgpu_vm_bo_update(adev, bo_va, false);
  2014. if (r)
  2015. return r;
  2016. }
  2017. spin_lock(&vm->invalidated_lock);
  2018. while (!list_empty(&vm->invalidated)) {
  2019. bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
  2020. base.vm_status);
  2021. resv = bo_va->base.bo->tbo.resv;
  2022. spin_unlock(&vm->invalidated_lock);
  2023. /* Try to reserve the BO to avoid clearing its ptes */
  2024. if (!amdgpu_vm_debug && reservation_object_trylock(resv))
  2025. clear = false;
  2026. /* Somebody else is using the BO right now */
  2027. else
  2028. clear = true;
  2029. r = amdgpu_vm_bo_update(adev, bo_va, clear);
  2030. if (r)
  2031. return r;
  2032. if (!clear)
  2033. reservation_object_unlock(resv);
  2034. spin_lock(&vm->invalidated_lock);
  2035. }
  2036. spin_unlock(&vm->invalidated_lock);
  2037. return 0;
  2038. }
  2039. /**
  2040. * amdgpu_vm_bo_add - add a bo to a specific vm
  2041. *
  2042. * @adev: amdgpu_device pointer
  2043. * @vm: requested vm
  2044. * @bo: amdgpu buffer object
  2045. *
  2046. * Add @bo into the requested vm.
  2047. * Add @bo to the list of bos associated with the vm
  2048. *
  2049. * Returns:
  2050. * Newly added bo_va or NULL for failure
  2051. *
  2052. * Object has to be reserved!
  2053. */
  2054. struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
  2055. struct amdgpu_vm *vm,
  2056. struct amdgpu_bo *bo)
  2057. {
  2058. struct amdgpu_bo_va *bo_va;
  2059. bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
  2060. if (bo_va == NULL) {
  2061. return NULL;
  2062. }
  2063. amdgpu_vm_bo_base_init(&bo_va->base, vm, bo);
  2064. bo_va->ref_count = 1;
  2065. INIT_LIST_HEAD(&bo_va->valids);
  2066. INIT_LIST_HEAD(&bo_va->invalids);
  2067. return bo_va;
  2068. }
  2069. /**
  2070. * amdgpu_vm_bo_insert_mapping - insert a new mapping
  2071. *
  2072. * @adev: amdgpu_device pointer
  2073. * @bo_va: bo_va to store the address
  2074. * @mapping: the mapping to insert
  2075. *
  2076. * Insert a new mapping into all structures.
  2077. */
  2078. static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
  2079. struct amdgpu_bo_va *bo_va,
  2080. struct amdgpu_bo_va_mapping *mapping)
  2081. {
  2082. struct amdgpu_vm *vm = bo_va->base.vm;
  2083. struct amdgpu_bo *bo = bo_va->base.bo;
  2084. mapping->bo_va = bo_va;
  2085. list_add(&mapping->list, &bo_va->invalids);
  2086. amdgpu_vm_it_insert(mapping, &vm->va);
  2087. if (mapping->flags & AMDGPU_PTE_PRT)
  2088. amdgpu_vm_prt_get(adev);
  2089. if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv &&
  2090. !bo_va->base.moved) {
  2091. list_move(&bo_va->base.vm_status, &vm->moved);
  2092. }
  2093. trace_amdgpu_vm_bo_map(bo_va, mapping);
  2094. }
  2095. /**
  2096. * amdgpu_vm_bo_map - map bo inside a vm
  2097. *
  2098. * @adev: amdgpu_device pointer
  2099. * @bo_va: bo_va to store the address
  2100. * @saddr: where to map the BO
  2101. * @offset: requested offset in the BO
  2102. * @size: BO size in bytes
  2103. * @flags: attributes of pages (read/write/valid/etc.)
  2104. *
  2105. * Add a mapping of the BO at the specefied addr into the VM.
  2106. *
  2107. * Returns:
  2108. * 0 for success, error for failure.
  2109. *
  2110. * Object has to be reserved and unreserved outside!
  2111. */
  2112. int amdgpu_vm_bo_map(struct amdgpu_device *adev,
  2113. struct amdgpu_bo_va *bo_va,
  2114. uint64_t saddr, uint64_t offset,
  2115. uint64_t size, uint64_t flags)
  2116. {
  2117. struct amdgpu_bo_va_mapping *mapping, *tmp;
  2118. struct amdgpu_bo *bo = bo_va->base.bo;
  2119. struct amdgpu_vm *vm = bo_va->base.vm;
  2120. uint64_t eaddr;
  2121. /* validate the parameters */
  2122. if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
  2123. size == 0 || size & AMDGPU_GPU_PAGE_MASK)
  2124. return -EINVAL;
  2125. /* make sure object fit at this offset */
  2126. eaddr = saddr + size - 1;
  2127. if (saddr >= eaddr ||
  2128. (bo && offset + size > amdgpu_bo_size(bo)))
  2129. return -EINVAL;
  2130. saddr /= AMDGPU_GPU_PAGE_SIZE;
  2131. eaddr /= AMDGPU_GPU_PAGE_SIZE;
  2132. tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
  2133. if (tmp) {
  2134. /* bo and tmp overlap, invalid addr */
  2135. dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
  2136. "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
  2137. tmp->start, tmp->last + 1);
  2138. return -EINVAL;
  2139. }
  2140. mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
  2141. if (!mapping)
  2142. return -ENOMEM;
  2143. mapping->start = saddr;
  2144. mapping->last = eaddr;
  2145. mapping->offset = offset;
  2146. mapping->flags = flags;
  2147. amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
  2148. return 0;
  2149. }
  2150. /**
  2151. * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
  2152. *
  2153. * @adev: amdgpu_device pointer
  2154. * @bo_va: bo_va to store the address
  2155. * @saddr: where to map the BO
  2156. * @offset: requested offset in the BO
  2157. * @size: BO size in bytes
  2158. * @flags: attributes of pages (read/write/valid/etc.)
  2159. *
  2160. * Add a mapping of the BO at the specefied addr into the VM. Replace existing
  2161. * mappings as we do so.
  2162. *
  2163. * Returns:
  2164. * 0 for success, error for failure.
  2165. *
  2166. * Object has to be reserved and unreserved outside!
  2167. */
  2168. int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
  2169. struct amdgpu_bo_va *bo_va,
  2170. uint64_t saddr, uint64_t offset,
  2171. uint64_t size, uint64_t flags)
  2172. {
  2173. struct amdgpu_bo_va_mapping *mapping;
  2174. struct amdgpu_bo *bo = bo_va->base.bo;
  2175. uint64_t eaddr;
  2176. int r;
  2177. /* validate the parameters */
  2178. if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
  2179. size == 0 || size & AMDGPU_GPU_PAGE_MASK)
  2180. return -EINVAL;
  2181. /* make sure object fit at this offset */
  2182. eaddr = saddr + size - 1;
  2183. if (saddr >= eaddr ||
  2184. (bo && offset + size > amdgpu_bo_size(bo)))
  2185. return -EINVAL;
  2186. /* Allocate all the needed memory */
  2187. mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
  2188. if (!mapping)
  2189. return -ENOMEM;
  2190. r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
  2191. if (r) {
  2192. kfree(mapping);
  2193. return r;
  2194. }
  2195. saddr /= AMDGPU_GPU_PAGE_SIZE;
  2196. eaddr /= AMDGPU_GPU_PAGE_SIZE;
  2197. mapping->start = saddr;
  2198. mapping->last = eaddr;
  2199. mapping->offset = offset;
  2200. mapping->flags = flags;
  2201. amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
  2202. return 0;
  2203. }
  2204. /**
  2205. * amdgpu_vm_bo_unmap - remove bo mapping from vm
  2206. *
  2207. * @adev: amdgpu_device pointer
  2208. * @bo_va: bo_va to remove the address from
  2209. * @saddr: where to the BO is mapped
  2210. *
  2211. * Remove a mapping of the BO at the specefied addr from the VM.
  2212. *
  2213. * Returns:
  2214. * 0 for success, error for failure.
  2215. *
  2216. * Object has to be reserved and unreserved outside!
  2217. */
  2218. int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
  2219. struct amdgpu_bo_va *bo_va,
  2220. uint64_t saddr)
  2221. {
  2222. struct amdgpu_bo_va_mapping *mapping;
  2223. struct amdgpu_vm *vm = bo_va->base.vm;
  2224. bool valid = true;
  2225. saddr /= AMDGPU_GPU_PAGE_SIZE;
  2226. list_for_each_entry(mapping, &bo_va->valids, list) {
  2227. if (mapping->start == saddr)
  2228. break;
  2229. }
  2230. if (&mapping->list == &bo_va->valids) {
  2231. valid = false;
  2232. list_for_each_entry(mapping, &bo_va->invalids, list) {
  2233. if (mapping->start == saddr)
  2234. break;
  2235. }
  2236. if (&mapping->list == &bo_va->invalids)
  2237. return -ENOENT;
  2238. }
  2239. list_del(&mapping->list);
  2240. amdgpu_vm_it_remove(mapping, &vm->va);
  2241. mapping->bo_va = NULL;
  2242. trace_amdgpu_vm_bo_unmap(bo_va, mapping);
  2243. if (valid)
  2244. list_add(&mapping->list, &vm->freed);
  2245. else
  2246. amdgpu_vm_free_mapping(adev, vm, mapping,
  2247. bo_va->last_pt_update);
  2248. return 0;
  2249. }
  2250. /**
  2251. * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
  2252. *
  2253. * @adev: amdgpu_device pointer
  2254. * @vm: VM structure to use
  2255. * @saddr: start of the range
  2256. * @size: size of the range
  2257. *
  2258. * Remove all mappings in a range, split them as appropriate.
  2259. *
  2260. * Returns:
  2261. * 0 for success, error for failure.
  2262. */
  2263. int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
  2264. struct amdgpu_vm *vm,
  2265. uint64_t saddr, uint64_t size)
  2266. {
  2267. struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
  2268. LIST_HEAD(removed);
  2269. uint64_t eaddr;
  2270. eaddr = saddr + size - 1;
  2271. saddr /= AMDGPU_GPU_PAGE_SIZE;
  2272. eaddr /= AMDGPU_GPU_PAGE_SIZE;
  2273. /* Allocate all the needed memory */
  2274. before = kzalloc(sizeof(*before), GFP_KERNEL);
  2275. if (!before)
  2276. return -ENOMEM;
  2277. INIT_LIST_HEAD(&before->list);
  2278. after = kzalloc(sizeof(*after), GFP_KERNEL);
  2279. if (!after) {
  2280. kfree(before);
  2281. return -ENOMEM;
  2282. }
  2283. INIT_LIST_HEAD(&after->list);
  2284. /* Now gather all removed mappings */
  2285. tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
  2286. while (tmp) {
  2287. /* Remember mapping split at the start */
  2288. if (tmp->start < saddr) {
  2289. before->start = tmp->start;
  2290. before->last = saddr - 1;
  2291. before->offset = tmp->offset;
  2292. before->flags = tmp->flags;
  2293. before->bo_va = tmp->bo_va;
  2294. list_add(&before->list, &tmp->bo_va->invalids);
  2295. }
  2296. /* Remember mapping split at the end */
  2297. if (tmp->last > eaddr) {
  2298. after->start = eaddr + 1;
  2299. after->last = tmp->last;
  2300. after->offset = tmp->offset;
  2301. after->offset += after->start - tmp->start;
  2302. after->flags = tmp->flags;
  2303. after->bo_va = tmp->bo_va;
  2304. list_add(&after->list, &tmp->bo_va->invalids);
  2305. }
  2306. list_del(&tmp->list);
  2307. list_add(&tmp->list, &removed);
  2308. tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
  2309. }
  2310. /* And free them up */
  2311. list_for_each_entry_safe(tmp, next, &removed, list) {
  2312. amdgpu_vm_it_remove(tmp, &vm->va);
  2313. list_del(&tmp->list);
  2314. if (tmp->start < saddr)
  2315. tmp->start = saddr;
  2316. if (tmp->last > eaddr)
  2317. tmp->last = eaddr;
  2318. tmp->bo_va = NULL;
  2319. list_add(&tmp->list, &vm->freed);
  2320. trace_amdgpu_vm_bo_unmap(NULL, tmp);
  2321. }
  2322. /* Insert partial mapping before the range */
  2323. if (!list_empty(&before->list)) {
  2324. amdgpu_vm_it_insert(before, &vm->va);
  2325. if (before->flags & AMDGPU_PTE_PRT)
  2326. amdgpu_vm_prt_get(adev);
  2327. } else {
  2328. kfree(before);
  2329. }
  2330. /* Insert partial mapping after the range */
  2331. if (!list_empty(&after->list)) {
  2332. amdgpu_vm_it_insert(after, &vm->va);
  2333. if (after->flags & AMDGPU_PTE_PRT)
  2334. amdgpu_vm_prt_get(adev);
  2335. } else {
  2336. kfree(after);
  2337. }
  2338. return 0;
  2339. }
  2340. /**
  2341. * amdgpu_vm_bo_lookup_mapping - find mapping by address
  2342. *
  2343. * @vm: the requested VM
  2344. * @addr: the address
  2345. *
  2346. * Find a mapping by it's address.
  2347. *
  2348. * Returns:
  2349. * The amdgpu_bo_va_mapping matching for addr or NULL
  2350. *
  2351. */
  2352. struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
  2353. uint64_t addr)
  2354. {
  2355. return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
  2356. }
  2357. /**
  2358. * amdgpu_vm_bo_trace_cs - trace all reserved mappings
  2359. *
  2360. * @vm: the requested vm
  2361. * @ticket: CS ticket
  2362. *
  2363. * Trace all mappings of BOs reserved during a command submission.
  2364. */
  2365. void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket)
  2366. {
  2367. struct amdgpu_bo_va_mapping *mapping;
  2368. if (!trace_amdgpu_vm_bo_cs_enabled())
  2369. return;
  2370. for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping;
  2371. mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) {
  2372. if (mapping->bo_va && mapping->bo_va->base.bo) {
  2373. struct amdgpu_bo *bo;
  2374. bo = mapping->bo_va->base.bo;
  2375. if (READ_ONCE(bo->tbo.resv->lock.ctx) != ticket)
  2376. continue;
  2377. }
  2378. trace_amdgpu_vm_bo_cs(mapping);
  2379. }
  2380. }
  2381. /**
  2382. * amdgpu_vm_bo_rmv - remove a bo to a specific vm
  2383. *
  2384. * @adev: amdgpu_device pointer
  2385. * @bo_va: requested bo_va
  2386. *
  2387. * Remove @bo_va->bo from the requested vm.
  2388. *
  2389. * Object have to be reserved!
  2390. */
  2391. void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
  2392. struct amdgpu_bo_va *bo_va)
  2393. {
  2394. struct amdgpu_bo_va_mapping *mapping, *next;
  2395. struct amdgpu_bo *bo = bo_va->base.bo;
  2396. struct amdgpu_vm *vm = bo_va->base.vm;
  2397. struct amdgpu_vm_bo_base **base;
  2398. if (bo) {
  2399. if (bo->tbo.resv == vm->root.base.bo->tbo.resv)
  2400. vm->bulk_moveable = false;
  2401. for (base = &bo_va->base.bo->vm_bo; *base;
  2402. base = &(*base)->next) {
  2403. if (*base != &bo_va->base)
  2404. continue;
  2405. *base = bo_va->base.next;
  2406. break;
  2407. }
  2408. }
  2409. spin_lock(&vm->invalidated_lock);
  2410. list_del(&bo_va->base.vm_status);
  2411. spin_unlock(&vm->invalidated_lock);
  2412. list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
  2413. list_del(&mapping->list);
  2414. amdgpu_vm_it_remove(mapping, &vm->va);
  2415. mapping->bo_va = NULL;
  2416. trace_amdgpu_vm_bo_unmap(bo_va, mapping);
  2417. list_add(&mapping->list, &vm->freed);
  2418. }
  2419. list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
  2420. list_del(&mapping->list);
  2421. amdgpu_vm_it_remove(mapping, &vm->va);
  2422. amdgpu_vm_free_mapping(adev, vm, mapping,
  2423. bo_va->last_pt_update);
  2424. }
  2425. dma_fence_put(bo_va->last_pt_update);
  2426. kfree(bo_va);
  2427. }
  2428. /**
  2429. * amdgpu_vm_bo_invalidate - mark the bo as invalid
  2430. *
  2431. * @adev: amdgpu_device pointer
  2432. * @bo: amdgpu buffer object
  2433. * @evicted: is the BO evicted
  2434. *
  2435. * Mark @bo as invalid.
  2436. */
  2437. void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
  2438. struct amdgpu_bo *bo, bool evicted)
  2439. {
  2440. struct amdgpu_vm_bo_base *bo_base;
  2441. /* shadow bo doesn't have bo base, its validation needs its parent */
  2442. if (bo->parent && bo->parent->shadow == bo)
  2443. bo = bo->parent;
  2444. for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
  2445. struct amdgpu_vm *vm = bo_base->vm;
  2446. if (evicted && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
  2447. amdgpu_vm_bo_evicted(bo_base);
  2448. continue;
  2449. }
  2450. if (bo_base->moved)
  2451. continue;
  2452. bo_base->moved = true;
  2453. if (bo->tbo.type == ttm_bo_type_kernel)
  2454. amdgpu_vm_bo_relocated(bo_base);
  2455. else if (bo->tbo.resv == vm->root.base.bo->tbo.resv)
  2456. amdgpu_vm_bo_moved(bo_base);
  2457. else
  2458. amdgpu_vm_bo_invalidated(bo_base);
  2459. }
  2460. }
  2461. /**
  2462. * amdgpu_vm_get_block_size - calculate VM page table size as power of two
  2463. *
  2464. * @vm_size: VM size
  2465. *
  2466. * Returns:
  2467. * VM page table as power of two
  2468. */
  2469. static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
  2470. {
  2471. /* Total bits covered by PD + PTs */
  2472. unsigned bits = ilog2(vm_size) + 18;
  2473. /* Make sure the PD is 4K in size up to 8GB address space.
  2474. Above that split equal between PD and PTs */
  2475. if (vm_size <= 8)
  2476. return (bits - 9);
  2477. else
  2478. return ((bits + 3) / 2);
  2479. }
  2480. /**
  2481. * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
  2482. *
  2483. * @adev: amdgpu_device pointer
  2484. * @min_vm_size: the minimum vm size in GB if it's set auto
  2485. * @fragment_size_default: Default PTE fragment size
  2486. * @max_level: max VMPT level
  2487. * @max_bits: max address space size in bits
  2488. *
  2489. */
  2490. void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
  2491. uint32_t fragment_size_default, unsigned max_level,
  2492. unsigned max_bits)
  2493. {
  2494. unsigned int max_size = 1 << (max_bits - 30);
  2495. unsigned int vm_size;
  2496. uint64_t tmp;
  2497. /* adjust vm size first */
  2498. if (amdgpu_vm_size != -1) {
  2499. vm_size = amdgpu_vm_size;
  2500. if (vm_size > max_size) {
  2501. dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
  2502. amdgpu_vm_size, max_size);
  2503. vm_size = max_size;
  2504. }
  2505. } else {
  2506. struct sysinfo si;
  2507. unsigned int phys_ram_gb;
  2508. /* Optimal VM size depends on the amount of physical
  2509. * RAM available. Underlying requirements and
  2510. * assumptions:
  2511. *
  2512. * - Need to map system memory and VRAM from all GPUs
  2513. * - VRAM from other GPUs not known here
  2514. * - Assume VRAM <= system memory
  2515. * - On GFX8 and older, VM space can be segmented for
  2516. * different MTYPEs
  2517. * - Need to allow room for fragmentation, guard pages etc.
  2518. *
  2519. * This adds up to a rough guess of system memory x3.
  2520. * Round up to power of two to maximize the available
  2521. * VM size with the given page table size.
  2522. */
  2523. si_meminfo(&si);
  2524. phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit +
  2525. (1 << 30) - 1) >> 30;
  2526. vm_size = roundup_pow_of_two(
  2527. min(max(phys_ram_gb * 3, min_vm_size), max_size));
  2528. }
  2529. adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
  2530. tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
  2531. if (amdgpu_vm_block_size != -1)
  2532. tmp >>= amdgpu_vm_block_size - 9;
  2533. tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
  2534. adev->vm_manager.num_level = min(max_level, (unsigned)tmp);
  2535. switch (adev->vm_manager.num_level) {
  2536. case 3:
  2537. adev->vm_manager.root_level = AMDGPU_VM_PDB2;
  2538. break;
  2539. case 2:
  2540. adev->vm_manager.root_level = AMDGPU_VM_PDB1;
  2541. break;
  2542. case 1:
  2543. adev->vm_manager.root_level = AMDGPU_VM_PDB0;
  2544. break;
  2545. default:
  2546. dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");
  2547. }
  2548. /* block size depends on vm size and hw setup*/
  2549. if (amdgpu_vm_block_size != -1)
  2550. adev->vm_manager.block_size =
  2551. min((unsigned)amdgpu_vm_block_size, max_bits
  2552. - AMDGPU_GPU_PAGE_SHIFT
  2553. - 9 * adev->vm_manager.num_level);
  2554. else if (adev->vm_manager.num_level > 1)
  2555. adev->vm_manager.block_size = 9;
  2556. else
  2557. adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
  2558. if (amdgpu_vm_fragment_size == -1)
  2559. adev->vm_manager.fragment_size = fragment_size_default;
  2560. else
  2561. adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
  2562. DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
  2563. vm_size, adev->vm_manager.num_level + 1,
  2564. adev->vm_manager.block_size,
  2565. adev->vm_manager.fragment_size);
  2566. }
  2567. static struct amdgpu_retryfault_hashtable *init_fault_hash(void)
  2568. {
  2569. struct amdgpu_retryfault_hashtable *fault_hash;
  2570. fault_hash = kmalloc(sizeof(*fault_hash), GFP_KERNEL);
  2571. if (!fault_hash)
  2572. return fault_hash;
  2573. INIT_CHASH_TABLE(fault_hash->hash,
  2574. AMDGPU_PAGEFAULT_HASH_BITS, 8, 0);
  2575. spin_lock_init(&fault_hash->lock);
  2576. fault_hash->count = 0;
  2577. return fault_hash;
  2578. }
  2579. /**
  2580. * amdgpu_vm_init - initialize a vm instance
  2581. *
  2582. * @adev: amdgpu_device pointer
  2583. * @vm: requested vm
  2584. * @vm_context: Indicates if it GFX or Compute context
  2585. * @pasid: Process address space identifier
  2586. *
  2587. * Init @vm fields.
  2588. *
  2589. * Returns:
  2590. * 0 for success, error for failure.
  2591. */
  2592. int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  2593. int vm_context, unsigned int pasid)
  2594. {
  2595. struct amdgpu_bo_param bp;
  2596. struct amdgpu_bo *root;
  2597. int r, i;
  2598. vm->va = RB_ROOT_CACHED;
  2599. for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
  2600. vm->reserved_vmid[i] = NULL;
  2601. INIT_LIST_HEAD(&vm->evicted);
  2602. INIT_LIST_HEAD(&vm->relocated);
  2603. INIT_LIST_HEAD(&vm->moved);
  2604. INIT_LIST_HEAD(&vm->idle);
  2605. INIT_LIST_HEAD(&vm->invalidated);
  2606. spin_lock_init(&vm->invalidated_lock);
  2607. INIT_LIST_HEAD(&vm->freed);
  2608. /* create scheduler entity for page table updates */
  2609. r = drm_sched_entity_init(&vm->entity, adev->vm_manager.vm_pte_rqs,
  2610. adev->vm_manager.vm_pte_num_rqs, NULL);
  2611. if (r)
  2612. return r;
  2613. vm->pte_support_ats = false;
  2614. if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) {
  2615. vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
  2616. AMDGPU_VM_USE_CPU_FOR_COMPUTE);
  2617. if (adev->asic_type == CHIP_RAVEN)
  2618. vm->pte_support_ats = true;
  2619. } else {
  2620. vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
  2621. AMDGPU_VM_USE_CPU_FOR_GFX);
  2622. }
  2623. DRM_DEBUG_DRIVER("VM update mode is %s\n",
  2624. vm->use_cpu_for_update ? "CPU" : "SDMA");
  2625. WARN_ONCE((vm->use_cpu_for_update & !amdgpu_gmc_vram_full_visible(&adev->gmc)),
  2626. "CPU update of VM recommended only for large BAR system\n");
  2627. vm->last_update = NULL;
  2628. amdgpu_vm_bo_param(adev, vm, adev->vm_manager.root_level, &bp);
  2629. if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE)
  2630. bp.flags &= ~AMDGPU_GEM_CREATE_SHADOW;
  2631. r = amdgpu_bo_create(adev, &bp, &root);
  2632. if (r)
  2633. goto error_free_sched_entity;
  2634. r = amdgpu_bo_reserve(root, true);
  2635. if (r)
  2636. goto error_free_root;
  2637. r = amdgpu_vm_clear_bo(adev, vm, root,
  2638. adev->vm_manager.root_level,
  2639. vm->pte_support_ats);
  2640. if (r)
  2641. goto error_unreserve;
  2642. amdgpu_vm_bo_base_init(&vm->root.base, vm, root);
  2643. amdgpu_bo_unreserve(vm->root.base.bo);
  2644. if (pasid) {
  2645. unsigned long flags;
  2646. spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
  2647. r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
  2648. GFP_ATOMIC);
  2649. spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
  2650. if (r < 0)
  2651. goto error_free_root;
  2652. vm->pasid = pasid;
  2653. }
  2654. vm->fault_hash = init_fault_hash();
  2655. if (!vm->fault_hash) {
  2656. r = -ENOMEM;
  2657. goto error_free_root;
  2658. }
  2659. INIT_KFIFO(vm->faults);
  2660. vm->fault_credit = 16;
  2661. return 0;
  2662. error_unreserve:
  2663. amdgpu_bo_unreserve(vm->root.base.bo);
  2664. error_free_root:
  2665. amdgpu_bo_unref(&vm->root.base.bo->shadow);
  2666. amdgpu_bo_unref(&vm->root.base.bo);
  2667. vm->root.base.bo = NULL;
  2668. error_free_sched_entity:
  2669. drm_sched_entity_destroy(&vm->entity);
  2670. return r;
  2671. }
  2672. /**
  2673. * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM
  2674. *
  2675. * @adev: amdgpu_device pointer
  2676. * @vm: requested vm
  2677. *
  2678. * This only works on GFX VMs that don't have any BOs added and no
  2679. * page tables allocated yet.
  2680. *
  2681. * Changes the following VM parameters:
  2682. * - use_cpu_for_update
  2683. * - pte_supports_ats
  2684. * - pasid (old PASID is released, because compute manages its own PASIDs)
  2685. *
  2686. * Reinitializes the page directory to reflect the changed ATS
  2687. * setting.
  2688. *
  2689. * Returns:
  2690. * 0 for success, -errno for errors.
  2691. */
  2692. int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, unsigned int pasid)
  2693. {
  2694. bool pte_support_ats = (adev->asic_type == CHIP_RAVEN);
  2695. int r;
  2696. r = amdgpu_bo_reserve(vm->root.base.bo, true);
  2697. if (r)
  2698. return r;
  2699. /* Sanity checks */
  2700. if (!RB_EMPTY_ROOT(&vm->va.rb_root) || vm->root.entries) {
  2701. r = -EINVAL;
  2702. goto unreserve_bo;
  2703. }
  2704. if (pasid) {
  2705. unsigned long flags;
  2706. spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
  2707. r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
  2708. GFP_ATOMIC);
  2709. spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
  2710. if (r == -ENOSPC)
  2711. goto unreserve_bo;
  2712. r = 0;
  2713. }
  2714. /* Check if PD needs to be reinitialized and do it before
  2715. * changing any other state, in case it fails.
  2716. */
  2717. if (pte_support_ats != vm->pte_support_ats) {
  2718. r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo,
  2719. adev->vm_manager.root_level,
  2720. pte_support_ats);
  2721. if (r)
  2722. goto free_idr;
  2723. }
  2724. /* Update VM state */
  2725. vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
  2726. AMDGPU_VM_USE_CPU_FOR_COMPUTE);
  2727. vm->pte_support_ats = pte_support_ats;
  2728. DRM_DEBUG_DRIVER("VM update mode is %s\n",
  2729. vm->use_cpu_for_update ? "CPU" : "SDMA");
  2730. WARN_ONCE((vm->use_cpu_for_update & !amdgpu_gmc_vram_full_visible(&adev->gmc)),
  2731. "CPU update of VM recommended only for large BAR system\n");
  2732. if (vm->pasid) {
  2733. unsigned long flags;
  2734. spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
  2735. idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
  2736. spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
  2737. /* Free the original amdgpu allocated pasid
  2738. * Will be replaced with kfd allocated pasid
  2739. */
  2740. amdgpu_pasid_free(vm->pasid);
  2741. vm->pasid = 0;
  2742. }
  2743. /* Free the shadow bo for compute VM */
  2744. amdgpu_bo_unref(&vm->root.base.bo->shadow);
  2745. if (pasid)
  2746. vm->pasid = pasid;
  2747. goto unreserve_bo;
  2748. free_idr:
  2749. if (pasid) {
  2750. unsigned long flags;
  2751. spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
  2752. idr_remove(&adev->vm_manager.pasid_idr, pasid);
  2753. spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
  2754. }
  2755. unreserve_bo:
  2756. amdgpu_bo_unreserve(vm->root.base.bo);
  2757. return r;
  2758. }
  2759. /**
  2760. * amdgpu_vm_release_compute - release a compute vm
  2761. * @adev: amdgpu_device pointer
  2762. * @vm: a vm turned into compute vm by calling amdgpu_vm_make_compute
  2763. *
  2764. * This is a correspondant of amdgpu_vm_make_compute. It decouples compute
  2765. * pasid from vm. Compute should stop use of vm after this call.
  2766. */
  2767. void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
  2768. {
  2769. if (vm->pasid) {
  2770. unsigned long flags;
  2771. spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
  2772. idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
  2773. spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
  2774. }
  2775. vm->pasid = 0;
  2776. }
  2777. /**
  2778. * amdgpu_vm_fini - tear down a vm instance
  2779. *
  2780. * @adev: amdgpu_device pointer
  2781. * @vm: requested vm
  2782. *
  2783. * Tear down @vm.
  2784. * Unbind the VM and remove all bos from the vm bo list
  2785. */
  2786. void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
  2787. {
  2788. struct amdgpu_bo_va_mapping *mapping, *tmp;
  2789. bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;
  2790. struct amdgpu_bo *root;
  2791. u64 fault;
  2792. int i, r;
  2793. amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
  2794. /* Clear pending page faults from IH when the VM is destroyed */
  2795. while (kfifo_get(&vm->faults, &fault))
  2796. amdgpu_vm_clear_fault(vm->fault_hash, fault);
  2797. if (vm->pasid) {
  2798. unsigned long flags;
  2799. spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
  2800. idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
  2801. spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
  2802. }
  2803. kfree(vm->fault_hash);
  2804. vm->fault_hash = NULL;
  2805. drm_sched_entity_destroy(&vm->entity);
  2806. if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
  2807. dev_err(adev->dev, "still active bo inside vm\n");
  2808. }
  2809. rbtree_postorder_for_each_entry_safe(mapping, tmp,
  2810. &vm->va.rb_root, rb) {
  2811. /* Don't remove the mapping here, we don't want to trigger a
  2812. * rebalance and the tree is about to be destroyed anyway.
  2813. */
  2814. list_del(&mapping->list);
  2815. kfree(mapping);
  2816. }
  2817. list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
  2818. if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
  2819. amdgpu_vm_prt_fini(adev, vm);
  2820. prt_fini_needed = false;
  2821. }
  2822. list_del(&mapping->list);
  2823. amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
  2824. }
  2825. root = amdgpu_bo_ref(vm->root.base.bo);
  2826. r = amdgpu_bo_reserve(root, true);
  2827. if (r) {
  2828. dev_err(adev->dev, "Leaking page tables because BO reservation failed\n");
  2829. } else {
  2830. amdgpu_vm_free_pts(adev, vm);
  2831. amdgpu_bo_unreserve(root);
  2832. }
  2833. amdgpu_bo_unref(&root);
  2834. dma_fence_put(vm->last_update);
  2835. for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
  2836. amdgpu_vmid_free_reserved(adev, vm, i);
  2837. }
  2838. /**
  2839. * amdgpu_vm_pasid_fault_credit - Check fault credit for given PASID
  2840. *
  2841. * @adev: amdgpu_device pointer
  2842. * @pasid: PASID do identify the VM
  2843. *
  2844. * This function is expected to be called in interrupt context.
  2845. *
  2846. * Returns:
  2847. * True if there was fault credit, false otherwise
  2848. */
  2849. bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev,
  2850. unsigned int pasid)
  2851. {
  2852. struct amdgpu_vm *vm;
  2853. spin_lock(&adev->vm_manager.pasid_lock);
  2854. vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
  2855. if (!vm) {
  2856. /* VM not found, can't track fault credit */
  2857. spin_unlock(&adev->vm_manager.pasid_lock);
  2858. return true;
  2859. }
  2860. /* No lock needed. only accessed by IRQ handler */
  2861. if (!vm->fault_credit) {
  2862. /* Too many faults in this VM */
  2863. spin_unlock(&adev->vm_manager.pasid_lock);
  2864. return false;
  2865. }
  2866. vm->fault_credit--;
  2867. spin_unlock(&adev->vm_manager.pasid_lock);
  2868. return true;
  2869. }
  2870. /**
  2871. * amdgpu_vm_manager_init - init the VM manager
  2872. *
  2873. * @adev: amdgpu_device pointer
  2874. *
  2875. * Initialize the VM manager structures
  2876. */
  2877. void amdgpu_vm_manager_init(struct amdgpu_device *adev)
  2878. {
  2879. unsigned i;
  2880. amdgpu_vmid_mgr_init(adev);
  2881. adev->vm_manager.fence_context =
  2882. dma_fence_context_alloc(AMDGPU_MAX_RINGS);
  2883. for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
  2884. adev->vm_manager.seqno[i] = 0;
  2885. spin_lock_init(&adev->vm_manager.prt_lock);
  2886. atomic_set(&adev->vm_manager.num_prt_users, 0);
  2887. /* If not overridden by the user, by default, only in large BAR systems
  2888. * Compute VM tables will be updated by CPU
  2889. */
  2890. #ifdef CONFIG_X86_64
  2891. if (amdgpu_vm_update_mode == -1) {
  2892. if (amdgpu_gmc_vram_full_visible(&adev->gmc))
  2893. adev->vm_manager.vm_update_mode =
  2894. AMDGPU_VM_USE_CPU_FOR_COMPUTE;
  2895. else
  2896. adev->vm_manager.vm_update_mode = 0;
  2897. } else
  2898. adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
  2899. #else
  2900. adev->vm_manager.vm_update_mode = 0;
  2901. #endif
  2902. idr_init(&adev->vm_manager.pasid_idr);
  2903. spin_lock_init(&adev->vm_manager.pasid_lock);
  2904. }
  2905. /**
  2906. * amdgpu_vm_manager_fini - cleanup VM manager
  2907. *
  2908. * @adev: amdgpu_device pointer
  2909. *
  2910. * Cleanup the VM manager and free resources.
  2911. */
  2912. void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
  2913. {
  2914. WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr));
  2915. idr_destroy(&adev->vm_manager.pasid_idr);
  2916. amdgpu_vmid_mgr_fini(adev);
  2917. }
  2918. /**
  2919. * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs.
  2920. *
  2921. * @dev: drm device pointer
  2922. * @data: drm_amdgpu_vm
  2923. * @filp: drm file pointer
  2924. *
  2925. * Returns:
  2926. * 0 for success, -errno for errors.
  2927. */
  2928. int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
  2929. {
  2930. union drm_amdgpu_vm *args = data;
  2931. struct amdgpu_device *adev = dev->dev_private;
  2932. struct amdgpu_fpriv *fpriv = filp->driver_priv;
  2933. int r;
  2934. switch (args->in.op) {
  2935. case AMDGPU_VM_OP_RESERVE_VMID:
  2936. /* current, we only have requirement to reserve vmid from gfxhub */
  2937. r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);
  2938. if (r)
  2939. return r;
  2940. break;
  2941. case AMDGPU_VM_OP_UNRESERVE_VMID:
  2942. amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);
  2943. break;
  2944. default:
  2945. return -EINVAL;
  2946. }
  2947. return 0;
  2948. }
  2949. /**
  2950. * amdgpu_vm_get_task_info - Extracts task info for a PASID.
  2951. *
  2952. * @adev: drm device pointer
  2953. * @pasid: PASID identifier for VM
  2954. * @task_info: task_info to fill.
  2955. */
  2956. void amdgpu_vm_get_task_info(struct amdgpu_device *adev, unsigned int pasid,
  2957. struct amdgpu_task_info *task_info)
  2958. {
  2959. struct amdgpu_vm *vm;
  2960. spin_lock(&adev->vm_manager.pasid_lock);
  2961. vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
  2962. if (vm)
  2963. *task_info = vm->task_info;
  2964. spin_unlock(&adev->vm_manager.pasid_lock);
  2965. }
  2966. /**
  2967. * amdgpu_vm_set_task_info - Sets VMs task info.
  2968. *
  2969. * @vm: vm for which to set the info
  2970. */
  2971. void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
  2972. {
  2973. if (!vm->task_info.pid) {
  2974. vm->task_info.pid = current->pid;
  2975. get_task_comm(vm->task_info.task_name, current);
  2976. if (current->group_leader->mm == current->mm) {
  2977. vm->task_info.tgid = current->group_leader->pid;
  2978. get_task_comm(vm->task_info.process_name, current->group_leader);
  2979. }
  2980. }
  2981. }
  2982. /**
  2983. * amdgpu_vm_add_fault - Add a page fault record to fault hash table
  2984. *
  2985. * @fault_hash: fault hash table
  2986. * @key: 64-bit encoding of PASID and address
  2987. *
  2988. * This should be called when a retry page fault interrupt is
  2989. * received. If this is a new page fault, it will be added to a hash
  2990. * table. The return value indicates whether this is a new fault, or
  2991. * a fault that was already known and is already being handled.
  2992. *
  2993. * If there are too many pending page faults, this will fail. Retry
  2994. * interrupts should be ignored in this case until there is enough
  2995. * free space.
  2996. *
  2997. * Returns 0 if the fault was added, 1 if the fault was already known,
  2998. * -ENOSPC if there are too many pending faults.
  2999. */
  3000. int amdgpu_vm_add_fault(struct amdgpu_retryfault_hashtable *fault_hash, u64 key)
  3001. {
  3002. unsigned long flags;
  3003. int r = -ENOSPC;
  3004. if (WARN_ON_ONCE(!fault_hash))
  3005. /* Should be allocated in amdgpu_vm_init
  3006. */
  3007. return r;
  3008. spin_lock_irqsave(&fault_hash->lock, flags);
  3009. /* Only let the hash table fill up to 50% for best performance */
  3010. if (fault_hash->count >= (1 << (AMDGPU_PAGEFAULT_HASH_BITS-1)))
  3011. goto unlock_out;
  3012. r = chash_table_copy_in(&fault_hash->hash, key, NULL);
  3013. if (!r)
  3014. fault_hash->count++;
  3015. /* chash_table_copy_in should never fail unless we're losing count */
  3016. WARN_ON_ONCE(r < 0);
  3017. unlock_out:
  3018. spin_unlock_irqrestore(&fault_hash->lock, flags);
  3019. return r;
  3020. }
  3021. /**
  3022. * amdgpu_vm_clear_fault - Remove a page fault record
  3023. *
  3024. * @fault_hash: fault hash table
  3025. * @key: 64-bit encoding of PASID and address
  3026. *
  3027. * This should be called when a page fault has been handled. Any
  3028. * future interrupt with this key will be processed as a new
  3029. * page fault.
  3030. */
  3031. void amdgpu_vm_clear_fault(struct amdgpu_retryfault_hashtable *fault_hash, u64 key)
  3032. {
  3033. unsigned long flags;
  3034. int r;
  3035. if (!fault_hash)
  3036. return;
  3037. spin_lock_irqsave(&fault_hash->lock, flags);
  3038. r = chash_table_remove(&fault_hash->hash, key, NULL);
  3039. if (!WARN_ON_ONCE(r < 0)) {
  3040. fault_hash->count--;
  3041. WARN_ON_ONCE(fault_hash->count < 0);
  3042. }
  3043. spin_unlock_irqrestore(&fault_hash->lock, flags);
  3044. }