amdgpu_device.c 96 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812
  1. /*
  2. * Copyright 2008 Advanced Micro Devices, Inc.
  3. * Copyright 2008 Red Hat Inc.
  4. * Copyright 2009 Jerome Glisse.
  5. *
  6. * Permission is hereby granted, free of charge, to any person obtaining a
  7. * copy of this software and associated documentation files (the "Software"),
  8. * to deal in the Software without restriction, including without limitation
  9. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10. * and/or sell copies of the Software, and to permit persons to whom the
  11. * Software is furnished to do so, subject to the following conditions:
  12. *
  13. * The above copyright notice and this permission notice shall be included in
  14. * all copies or substantial portions of the Software.
  15. *
  16. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19. * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22. * OTHER DEALINGS IN THE SOFTWARE.
  23. *
  24. * Authors: Dave Airlie
  25. * Alex Deucher
  26. * Jerome Glisse
  27. */
  28. #include <linux/kthread.h>
  29. #include <linux/console.h>
  30. #include <linux/slab.h>
  31. #include <linux/debugfs.h>
  32. #include <drm/drmP.h>
  33. #include <drm/drm_crtc_helper.h>
  34. #include <drm/amdgpu_drm.h>
  35. #include <linux/vgaarb.h>
  36. #include <linux/vga_switcheroo.h>
  37. #include <linux/efi.h>
  38. #include "amdgpu.h"
  39. #include "amdgpu_trace.h"
  40. #include "amdgpu_i2c.h"
  41. #include "atom.h"
  42. #include "amdgpu_atombios.h"
  43. #include "amdgpu_atomfirmware.h"
  44. #include "amd_pcie.h"
  45. #ifdef CONFIG_DRM_AMDGPU_SI
  46. #include "si.h"
  47. #endif
  48. #ifdef CONFIG_DRM_AMDGPU_CIK
  49. #include "cik.h"
  50. #endif
  51. #include "vi.h"
  52. #include "soc15.h"
  53. #include "bif/bif_4_1_d.h"
  54. #include <linux/pci.h>
  55. #include <linux/firmware.h>
  56. MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  57. MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  58. #define AMDGPU_RESUME_MS 2000
  59. static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
  60. static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev);
  61. static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev);
  62. static const char *amdgpu_asic_name[] = {
  63. "TAHITI",
  64. "PITCAIRN",
  65. "VERDE",
  66. "OLAND",
  67. "HAINAN",
  68. "BONAIRE",
  69. "KAVERI",
  70. "KABINI",
  71. "HAWAII",
  72. "MULLINS",
  73. "TOPAZ",
  74. "TONGA",
  75. "FIJI",
  76. "CARRIZO",
  77. "STONEY",
  78. "POLARIS10",
  79. "POLARIS11",
  80. "POLARIS12",
  81. "VEGA10",
  82. "RAVEN",
  83. "LAST",
  84. };
  85. bool amdgpu_device_is_px(struct drm_device *dev)
  86. {
  87. struct amdgpu_device *adev = dev->dev_private;
  88. if (adev->flags & AMD_IS_PX)
  89. return true;
  90. return false;
  91. }
  92. /*
  93. * MMIO register access helper functions.
  94. */
  95. uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
  96. uint32_t acc_flags)
  97. {
  98. uint32_t ret;
  99. if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) {
  100. BUG_ON(in_interrupt());
  101. return amdgpu_virt_kiq_rreg(adev, reg);
  102. }
  103. if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
  104. ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
  105. else {
  106. unsigned long flags;
  107. spin_lock_irqsave(&adev->mmio_idx_lock, flags);
  108. writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
  109. ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
  110. spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
  111. }
  112. trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
  113. return ret;
  114. }
  115. void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
  116. uint32_t acc_flags)
  117. {
  118. trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
  119. if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) {
  120. BUG_ON(in_interrupt());
  121. return amdgpu_virt_kiq_wreg(adev, reg, v);
  122. }
  123. if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
  124. writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
  125. else {
  126. unsigned long flags;
  127. spin_lock_irqsave(&adev->mmio_idx_lock, flags);
  128. writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
  129. writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
  130. spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
  131. }
  132. }
  133. u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
  134. {
  135. if ((reg * 4) < adev->rio_mem_size)
  136. return ioread32(adev->rio_mem + (reg * 4));
  137. else {
  138. iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
  139. return ioread32(adev->rio_mem + (mmMM_DATA * 4));
  140. }
  141. }
  142. void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
  143. {
  144. if ((reg * 4) < adev->rio_mem_size)
  145. iowrite32(v, adev->rio_mem + (reg * 4));
  146. else {
  147. iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
  148. iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
  149. }
  150. }
  151. /**
  152. * amdgpu_mm_rdoorbell - read a doorbell dword
  153. *
  154. * @adev: amdgpu_device pointer
  155. * @index: doorbell index
  156. *
  157. * Returns the value in the doorbell aperture at the
  158. * requested doorbell index (CIK).
  159. */
  160. u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
  161. {
  162. if (index < adev->doorbell.num_doorbells) {
  163. return readl(adev->doorbell.ptr + index);
  164. } else {
  165. DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
  166. return 0;
  167. }
  168. }
  169. /**
  170. * amdgpu_mm_wdoorbell - write a doorbell dword
  171. *
  172. * @adev: amdgpu_device pointer
  173. * @index: doorbell index
  174. * @v: value to write
  175. *
  176. * Writes @v to the doorbell aperture at the
  177. * requested doorbell index (CIK).
  178. */
  179. void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
  180. {
  181. if (index < adev->doorbell.num_doorbells) {
  182. writel(v, adev->doorbell.ptr + index);
  183. } else {
  184. DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
  185. }
  186. }
  187. /**
  188. * amdgpu_mm_rdoorbell64 - read a doorbell Qword
  189. *
  190. * @adev: amdgpu_device pointer
  191. * @index: doorbell index
  192. *
  193. * Returns the value in the doorbell aperture at the
  194. * requested doorbell index (VEGA10+).
  195. */
  196. u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
  197. {
  198. if (index < adev->doorbell.num_doorbells) {
  199. return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
  200. } else {
  201. DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
  202. return 0;
  203. }
  204. }
  205. /**
  206. * amdgpu_mm_wdoorbell64 - write a doorbell Qword
  207. *
  208. * @adev: amdgpu_device pointer
  209. * @index: doorbell index
  210. * @v: value to write
  211. *
  212. * Writes @v to the doorbell aperture at the
  213. * requested doorbell index (VEGA10+).
  214. */
  215. void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
  216. {
  217. if (index < adev->doorbell.num_doorbells) {
  218. atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
  219. } else {
  220. DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
  221. }
  222. }
  223. /**
  224. * amdgpu_invalid_rreg - dummy reg read function
  225. *
  226. * @adev: amdgpu device pointer
  227. * @reg: offset of register
  228. *
  229. * Dummy register read function. Used for register blocks
  230. * that certain asics don't have (all asics).
  231. * Returns the value in the register.
  232. */
  233. static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
  234. {
  235. DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
  236. BUG();
  237. return 0;
  238. }
  239. /**
  240. * amdgpu_invalid_wreg - dummy reg write function
  241. *
  242. * @adev: amdgpu device pointer
  243. * @reg: offset of register
  244. * @v: value to write to the register
  245. *
  246. * Dummy register read function. Used for register blocks
  247. * that certain asics don't have (all asics).
  248. */
  249. static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
  250. {
  251. DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
  252. reg, v);
  253. BUG();
  254. }
  255. /**
  256. * amdgpu_block_invalid_rreg - dummy reg read function
  257. *
  258. * @adev: amdgpu device pointer
  259. * @block: offset of instance
  260. * @reg: offset of register
  261. *
  262. * Dummy register read function. Used for register blocks
  263. * that certain asics don't have (all asics).
  264. * Returns the value in the register.
  265. */
  266. static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
  267. uint32_t block, uint32_t reg)
  268. {
  269. DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
  270. reg, block);
  271. BUG();
  272. return 0;
  273. }
  274. /**
  275. * amdgpu_block_invalid_wreg - dummy reg write function
  276. *
  277. * @adev: amdgpu device pointer
  278. * @block: offset of instance
  279. * @reg: offset of register
  280. * @v: value to write to the register
  281. *
  282. * Dummy register read function. Used for register blocks
  283. * that certain asics don't have (all asics).
  284. */
  285. static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
  286. uint32_t block,
  287. uint32_t reg, uint32_t v)
  288. {
  289. DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
  290. reg, block, v);
  291. BUG();
  292. }
  293. static int amdgpu_vram_scratch_init(struct amdgpu_device *adev)
  294. {
  295. int r;
  296. if (adev->vram_scratch.robj == NULL) {
  297. r = amdgpu_bo_create(adev, AMDGPU_GPU_PAGE_SIZE,
  298. PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
  299. AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
  300. AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
  301. NULL, NULL, &adev->vram_scratch.robj);
  302. if (r) {
  303. return r;
  304. }
  305. }
  306. r = amdgpu_bo_reserve(adev->vram_scratch.robj, false);
  307. if (unlikely(r != 0))
  308. return r;
  309. r = amdgpu_bo_pin(adev->vram_scratch.robj,
  310. AMDGPU_GEM_DOMAIN_VRAM, &adev->vram_scratch.gpu_addr);
  311. if (r) {
  312. amdgpu_bo_unreserve(adev->vram_scratch.robj);
  313. return r;
  314. }
  315. r = amdgpu_bo_kmap(adev->vram_scratch.robj,
  316. (void **)&adev->vram_scratch.ptr);
  317. if (r)
  318. amdgpu_bo_unpin(adev->vram_scratch.robj);
  319. amdgpu_bo_unreserve(adev->vram_scratch.robj);
  320. return r;
  321. }
  322. static void amdgpu_vram_scratch_fini(struct amdgpu_device *adev)
  323. {
  324. int r;
  325. if (adev->vram_scratch.robj == NULL) {
  326. return;
  327. }
  328. r = amdgpu_bo_reserve(adev->vram_scratch.robj, true);
  329. if (likely(r == 0)) {
  330. amdgpu_bo_kunmap(adev->vram_scratch.robj);
  331. amdgpu_bo_unpin(adev->vram_scratch.robj);
  332. amdgpu_bo_unreserve(adev->vram_scratch.robj);
  333. }
  334. amdgpu_bo_unref(&adev->vram_scratch.robj);
  335. }
  336. /**
  337. * amdgpu_program_register_sequence - program an array of registers.
  338. *
  339. * @adev: amdgpu_device pointer
  340. * @registers: pointer to the register array
  341. * @array_size: size of the register array
  342. *
  343. * Programs an array or registers with and and or masks.
  344. * This is a helper for setting golden registers.
  345. */
  346. void amdgpu_program_register_sequence(struct amdgpu_device *adev,
  347. const u32 *registers,
  348. const u32 array_size)
  349. {
  350. u32 tmp, reg, and_mask, or_mask;
  351. int i;
  352. if (array_size % 3)
  353. return;
  354. for (i = 0; i < array_size; i +=3) {
  355. reg = registers[i + 0];
  356. and_mask = registers[i + 1];
  357. or_mask = registers[i + 2];
  358. if (and_mask == 0xffffffff) {
  359. tmp = or_mask;
  360. } else {
  361. tmp = RREG32(reg);
  362. tmp &= ~and_mask;
  363. tmp |= or_mask;
  364. }
  365. WREG32(reg, tmp);
  366. }
  367. }
  368. void amdgpu_pci_config_reset(struct amdgpu_device *adev)
  369. {
  370. pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
  371. }
  372. /*
  373. * GPU doorbell aperture helpers function.
  374. */
  375. /**
  376. * amdgpu_doorbell_init - Init doorbell driver information.
  377. *
  378. * @adev: amdgpu_device pointer
  379. *
  380. * Init doorbell driver information (CIK)
  381. * Returns 0 on success, error on failure.
  382. */
  383. static int amdgpu_doorbell_init(struct amdgpu_device *adev)
  384. {
  385. /* doorbell bar mapping */
  386. adev->doorbell.base = pci_resource_start(adev->pdev, 2);
  387. adev->doorbell.size = pci_resource_len(adev->pdev, 2);
  388. adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
  389. AMDGPU_DOORBELL_MAX_ASSIGNMENT+1);
  390. if (adev->doorbell.num_doorbells == 0)
  391. return -EINVAL;
  392. adev->doorbell.ptr = ioremap(adev->doorbell.base,
  393. adev->doorbell.num_doorbells *
  394. sizeof(u32));
  395. if (adev->doorbell.ptr == NULL)
  396. return -ENOMEM;
  397. return 0;
  398. }
  399. /**
  400. * amdgpu_doorbell_fini - Tear down doorbell driver information.
  401. *
  402. * @adev: amdgpu_device pointer
  403. *
  404. * Tear down doorbell driver information (CIK)
  405. */
  406. static void amdgpu_doorbell_fini(struct amdgpu_device *adev)
  407. {
  408. iounmap(adev->doorbell.ptr);
  409. adev->doorbell.ptr = NULL;
  410. }
  411. /**
  412. * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to
  413. * setup amdkfd
  414. *
  415. * @adev: amdgpu_device pointer
  416. * @aperture_base: output returning doorbell aperture base physical address
  417. * @aperture_size: output returning doorbell aperture size in bytes
  418. * @start_offset: output returning # of doorbell bytes reserved for amdgpu.
  419. *
  420. * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up,
  421. * takes doorbells required for its own rings and reports the setup to amdkfd.
  422. * amdgpu reserved doorbells are at the start of the doorbell aperture.
  423. */
  424. void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
  425. phys_addr_t *aperture_base,
  426. size_t *aperture_size,
  427. size_t *start_offset)
  428. {
  429. /*
  430. * The first num_doorbells are used by amdgpu.
  431. * amdkfd takes whatever's left in the aperture.
  432. */
  433. if (adev->doorbell.size > adev->doorbell.num_doorbells * sizeof(u32)) {
  434. *aperture_base = adev->doorbell.base;
  435. *aperture_size = adev->doorbell.size;
  436. *start_offset = adev->doorbell.num_doorbells * sizeof(u32);
  437. } else {
  438. *aperture_base = 0;
  439. *aperture_size = 0;
  440. *start_offset = 0;
  441. }
  442. }
  443. /*
  444. * amdgpu_wb_*()
  445. * Writeback is the method by which the GPU updates special pages in memory
  446. * with the status of certain GPU events (fences, ring pointers,etc.).
  447. */
  448. /**
  449. * amdgpu_wb_fini - Disable Writeback and free memory
  450. *
  451. * @adev: amdgpu_device pointer
  452. *
  453. * Disables Writeback and frees the Writeback memory (all asics).
  454. * Used at driver shutdown.
  455. */
  456. static void amdgpu_wb_fini(struct amdgpu_device *adev)
  457. {
  458. if (adev->wb.wb_obj) {
  459. amdgpu_bo_free_kernel(&adev->wb.wb_obj,
  460. &adev->wb.gpu_addr,
  461. (void **)&adev->wb.wb);
  462. adev->wb.wb_obj = NULL;
  463. }
  464. }
  465. /**
  466. * amdgpu_wb_init- Init Writeback driver info and allocate memory
  467. *
  468. * @adev: amdgpu_device pointer
  469. *
  470. * Initializes writeback and allocates writeback memory (all asics).
  471. * Used at driver startup.
  472. * Returns 0 on success or an -error on failure.
  473. */
  474. static int amdgpu_wb_init(struct amdgpu_device *adev)
  475. {
  476. int r;
  477. if (adev->wb.wb_obj == NULL) {
  478. r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t),
  479. PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
  480. &adev->wb.wb_obj, &adev->wb.gpu_addr,
  481. (void **)&adev->wb.wb);
  482. if (r) {
  483. dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
  484. return r;
  485. }
  486. adev->wb.num_wb = AMDGPU_MAX_WB;
  487. memset(&adev->wb.used, 0, sizeof(adev->wb.used));
  488. /* clear wb memory */
  489. memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t));
  490. }
  491. return 0;
  492. }
  493. /**
  494. * amdgpu_wb_get - Allocate a wb entry
  495. *
  496. * @adev: amdgpu_device pointer
  497. * @wb: wb index
  498. *
  499. * Allocate a wb slot for use by the driver (all asics).
  500. * Returns 0 on success or -EINVAL on failure.
  501. */
  502. int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
  503. {
  504. unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
  505. if (offset < adev->wb.num_wb) {
  506. __set_bit(offset, adev->wb.used);
  507. *wb = offset;
  508. return 0;
  509. } else {
  510. return -EINVAL;
  511. }
  512. }
  513. /**
  514. * amdgpu_wb_get_64bit - Allocate a wb entry
  515. *
  516. * @adev: amdgpu_device pointer
  517. * @wb: wb index
  518. *
  519. * Allocate a wb slot for use by the driver (all asics).
  520. * Returns 0 on success or -EINVAL on failure.
  521. */
  522. int amdgpu_wb_get_64bit(struct amdgpu_device *adev, u32 *wb)
  523. {
  524. unsigned long offset = bitmap_find_next_zero_area_off(adev->wb.used,
  525. adev->wb.num_wb, 0, 2, 7, 0);
  526. if ((offset + 1) < adev->wb.num_wb) {
  527. __set_bit(offset, adev->wb.used);
  528. __set_bit(offset + 1, adev->wb.used);
  529. *wb = offset;
  530. return 0;
  531. } else {
  532. return -EINVAL;
  533. }
  534. }
  535. /**
  536. * amdgpu_wb_free - Free a wb entry
  537. *
  538. * @adev: amdgpu_device pointer
  539. * @wb: wb index
  540. *
  541. * Free a wb slot allocated for use by the driver (all asics)
  542. */
  543. void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb)
  544. {
  545. if (wb < adev->wb.num_wb)
  546. __clear_bit(wb, adev->wb.used);
  547. }
  548. /**
  549. * amdgpu_wb_free_64bit - Free a wb entry
  550. *
  551. * @adev: amdgpu_device pointer
  552. * @wb: wb index
  553. *
  554. * Free a wb slot allocated for use by the driver (all asics)
  555. */
  556. void amdgpu_wb_free_64bit(struct amdgpu_device *adev, u32 wb)
  557. {
  558. if ((wb + 1) < adev->wb.num_wb) {
  559. __clear_bit(wb, adev->wb.used);
  560. __clear_bit(wb + 1, adev->wb.used);
  561. }
  562. }
  563. /**
  564. * amdgpu_vram_location - try to find VRAM location
  565. * @adev: amdgpu device structure holding all necessary informations
  566. * @mc: memory controller structure holding memory informations
  567. * @base: base address at which to put VRAM
  568. *
  569. * Function will try to place VRAM at base address provided
  570. * as parameter (which is so far either PCI aperture address or
  571. * for IGP TOM base address).
  572. *
  573. * If there is not enough space to fit the unvisible VRAM in the 32bits
  574. * address space then we limit the VRAM size to the aperture.
  575. *
  576. * Note: We don't explicitly enforce VRAM start to be aligned on VRAM size,
  577. * this shouldn't be a problem as we are using the PCI aperture as a reference.
  578. * Otherwise this would be needed for rv280, all r3xx, and all r4xx, but
  579. * not IGP.
  580. *
  581. * Note: we use mc_vram_size as on some board we need to program the mc to
  582. * cover the whole aperture even if VRAM size is inferior to aperture size
  583. * Novell bug 204882 + along with lots of ubuntu ones
  584. *
  585. * Note: when limiting vram it's safe to overwritte real_vram_size because
  586. * we are not in case where real_vram_size is inferior to mc_vram_size (ie
  587. * note afected by bogus hw of Novell bug 204882 + along with lots of ubuntu
  588. * ones)
  589. *
  590. * Note: IGP TOM addr should be the same as the aperture addr, we don't
  591. * explicitly check for that though.
  592. *
  593. * FIXME: when reducing VRAM size align new size on power of 2.
  594. */
  595. void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base)
  596. {
  597. uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
  598. mc->vram_start = base;
  599. if (mc->mc_vram_size > (adev->mc.mc_mask - base + 1)) {
  600. dev_warn(adev->dev, "limiting VRAM to PCI aperture size\n");
  601. mc->real_vram_size = mc->aper_size;
  602. mc->mc_vram_size = mc->aper_size;
  603. }
  604. mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
  605. if (limit && limit < mc->real_vram_size)
  606. mc->real_vram_size = limit;
  607. dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
  608. mc->mc_vram_size >> 20, mc->vram_start,
  609. mc->vram_end, mc->real_vram_size >> 20);
  610. }
  611. /**
  612. * amdgpu_gtt_location - try to find GTT location
  613. * @adev: amdgpu device structure holding all necessary informations
  614. * @mc: memory controller structure holding memory informations
  615. *
  616. * Function will place try to place GTT before or after VRAM.
  617. *
  618. * If GTT size is bigger than space left then we ajust GTT size.
  619. * Thus function will never fails.
  620. *
  621. * FIXME: when reducing GTT size align new size on power of 2.
  622. */
  623. void amdgpu_gtt_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
  624. {
  625. u64 size_af, size_bf;
  626. size_af = ((adev->mc.mc_mask - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align;
  627. size_bf = mc->vram_start & ~mc->gtt_base_align;
  628. if (size_bf > size_af) {
  629. if (mc->gtt_size > size_bf) {
  630. dev_warn(adev->dev, "limiting GTT\n");
  631. mc->gtt_size = size_bf;
  632. }
  633. mc->gtt_start = 0;
  634. } else {
  635. if (mc->gtt_size > size_af) {
  636. dev_warn(adev->dev, "limiting GTT\n");
  637. mc->gtt_size = size_af;
  638. }
  639. mc->gtt_start = (mc->vram_end + 1 + mc->gtt_base_align) & ~mc->gtt_base_align;
  640. }
  641. mc->gtt_end = mc->gtt_start + mc->gtt_size - 1;
  642. dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
  643. mc->gtt_size >> 20, mc->gtt_start, mc->gtt_end);
  644. }
  645. /*
  646. * GPU helpers function.
  647. */
  648. /**
  649. * amdgpu_need_post - check if the hw need post or not
  650. *
  651. * @adev: amdgpu_device pointer
  652. *
  653. * Check if the asic has been initialized (all asics) at driver startup
  654. * or post is needed if hw reset is performed.
  655. * Returns true if need or false if not.
  656. */
  657. bool amdgpu_need_post(struct amdgpu_device *adev)
  658. {
  659. uint32_t reg;
  660. if (adev->has_hw_reset) {
  661. adev->has_hw_reset = false;
  662. return true;
  663. }
  664. /* then check MEM_SIZE, in case the crtcs are off */
  665. reg = amdgpu_asic_get_config_memsize(adev);
  666. if ((reg != 0) && (reg != 0xffffffff))
  667. return false;
  668. return true;
  669. }
  670. static bool amdgpu_vpost_needed(struct amdgpu_device *adev)
  671. {
  672. if (amdgpu_sriov_vf(adev))
  673. return false;
  674. if (amdgpu_passthrough(adev)) {
  675. /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
  676. * some old smc fw still need driver do vPost otherwise gpu hang, while
  677. * those smc fw version above 22.15 doesn't have this flaw, so we force
  678. * vpost executed for smc version below 22.15
  679. */
  680. if (adev->asic_type == CHIP_FIJI) {
  681. int err;
  682. uint32_t fw_ver;
  683. err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
  684. /* force vPost if error occured */
  685. if (err)
  686. return true;
  687. fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
  688. if (fw_ver < 0x00160e00)
  689. return true;
  690. }
  691. }
  692. return amdgpu_need_post(adev);
  693. }
  694. /**
  695. * amdgpu_dummy_page_init - init dummy page used by the driver
  696. *
  697. * @adev: amdgpu_device pointer
  698. *
  699. * Allocate the dummy page used by the driver (all asics).
  700. * This dummy page is used by the driver as a filler for gart entries
  701. * when pages are taken out of the GART
  702. * Returns 0 on sucess, -ENOMEM on failure.
  703. */
  704. int amdgpu_dummy_page_init(struct amdgpu_device *adev)
  705. {
  706. if (adev->dummy_page.page)
  707. return 0;
  708. adev->dummy_page.page = alloc_page(GFP_DMA32 | GFP_KERNEL | __GFP_ZERO);
  709. if (adev->dummy_page.page == NULL)
  710. return -ENOMEM;
  711. adev->dummy_page.addr = pci_map_page(adev->pdev, adev->dummy_page.page,
  712. 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
  713. if (pci_dma_mapping_error(adev->pdev, adev->dummy_page.addr)) {
  714. dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n");
  715. __free_page(adev->dummy_page.page);
  716. adev->dummy_page.page = NULL;
  717. return -ENOMEM;
  718. }
  719. return 0;
  720. }
  721. /**
  722. * amdgpu_dummy_page_fini - free dummy page used by the driver
  723. *
  724. * @adev: amdgpu_device pointer
  725. *
  726. * Frees the dummy page used by the driver (all asics).
  727. */
  728. void amdgpu_dummy_page_fini(struct amdgpu_device *adev)
  729. {
  730. if (adev->dummy_page.page == NULL)
  731. return;
  732. pci_unmap_page(adev->pdev, adev->dummy_page.addr,
  733. PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
  734. __free_page(adev->dummy_page.page);
  735. adev->dummy_page.page = NULL;
  736. }
  737. /* ATOM accessor methods */
  738. /*
  739. * ATOM is an interpreted byte code stored in tables in the vbios. The
  740. * driver registers callbacks to access registers and the interpreter
  741. * in the driver parses the tables and executes then to program specific
  742. * actions (set display modes, asic init, etc.). See amdgpu_atombios.c,
  743. * atombios.h, and atom.c
  744. */
  745. /**
  746. * cail_pll_read - read PLL register
  747. *
  748. * @info: atom card_info pointer
  749. * @reg: PLL register offset
  750. *
  751. * Provides a PLL register accessor for the atom interpreter (r4xx+).
  752. * Returns the value of the PLL register.
  753. */
  754. static uint32_t cail_pll_read(struct card_info *info, uint32_t reg)
  755. {
  756. return 0;
  757. }
  758. /**
  759. * cail_pll_write - write PLL register
  760. *
  761. * @info: atom card_info pointer
  762. * @reg: PLL register offset
  763. * @val: value to write to the pll register
  764. *
  765. * Provides a PLL register accessor for the atom interpreter (r4xx+).
  766. */
  767. static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val)
  768. {
  769. }
  770. /**
  771. * cail_mc_read - read MC (Memory Controller) register
  772. *
  773. * @info: atom card_info pointer
  774. * @reg: MC register offset
  775. *
  776. * Provides an MC register accessor for the atom interpreter (r4xx+).
  777. * Returns the value of the MC register.
  778. */
  779. static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)
  780. {
  781. return 0;
  782. }
  783. /**
  784. * cail_mc_write - write MC (Memory Controller) register
  785. *
  786. * @info: atom card_info pointer
  787. * @reg: MC register offset
  788. * @val: value to write to the pll register
  789. *
  790. * Provides a MC register accessor for the atom interpreter (r4xx+).
  791. */
  792. static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)
  793. {
  794. }
  795. /**
  796. * cail_reg_write - write MMIO register
  797. *
  798. * @info: atom card_info pointer
  799. * @reg: MMIO register offset
  800. * @val: value to write to the pll register
  801. *
  802. * Provides a MMIO register accessor for the atom interpreter (r4xx+).
  803. */
  804. static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)
  805. {
  806. struct amdgpu_device *adev = info->dev->dev_private;
  807. WREG32(reg, val);
  808. }
  809. /**
  810. * cail_reg_read - read MMIO register
  811. *
  812. * @info: atom card_info pointer
  813. * @reg: MMIO register offset
  814. *
  815. * Provides an MMIO register accessor for the atom interpreter (r4xx+).
  816. * Returns the value of the MMIO register.
  817. */
  818. static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)
  819. {
  820. struct amdgpu_device *adev = info->dev->dev_private;
  821. uint32_t r;
  822. r = RREG32(reg);
  823. return r;
  824. }
  825. /**
  826. * cail_ioreg_write - write IO register
  827. *
  828. * @info: atom card_info pointer
  829. * @reg: IO register offset
  830. * @val: value to write to the pll register
  831. *
  832. * Provides a IO register accessor for the atom interpreter (r4xx+).
  833. */
  834. static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val)
  835. {
  836. struct amdgpu_device *adev = info->dev->dev_private;
  837. WREG32_IO(reg, val);
  838. }
  839. /**
  840. * cail_ioreg_read - read IO register
  841. *
  842. * @info: atom card_info pointer
  843. * @reg: IO register offset
  844. *
  845. * Provides an IO register accessor for the atom interpreter (r4xx+).
  846. * Returns the value of the IO register.
  847. */
  848. static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)
  849. {
  850. struct amdgpu_device *adev = info->dev->dev_private;
  851. uint32_t r;
  852. r = RREG32_IO(reg);
  853. return r;
  854. }
  855. /**
  856. * amdgpu_atombios_fini - free the driver info and callbacks for atombios
  857. *
  858. * @adev: amdgpu_device pointer
  859. *
  860. * Frees the driver info and register access callbacks for the ATOM
  861. * interpreter (r4xx+).
  862. * Called at driver shutdown.
  863. */
  864. static void amdgpu_atombios_fini(struct amdgpu_device *adev)
  865. {
  866. if (adev->mode_info.atom_context) {
  867. kfree(adev->mode_info.atom_context->scratch);
  868. kfree(adev->mode_info.atom_context->iio);
  869. }
  870. kfree(adev->mode_info.atom_context);
  871. adev->mode_info.atom_context = NULL;
  872. kfree(adev->mode_info.atom_card_info);
  873. adev->mode_info.atom_card_info = NULL;
  874. }
  875. /**
  876. * amdgpu_atombios_init - init the driver info and callbacks for atombios
  877. *
  878. * @adev: amdgpu_device pointer
  879. *
  880. * Initializes the driver info and register access callbacks for the
  881. * ATOM interpreter (r4xx+).
  882. * Returns 0 on sucess, -ENOMEM on failure.
  883. * Called at driver startup.
  884. */
  885. static int amdgpu_atombios_init(struct amdgpu_device *adev)
  886. {
  887. struct card_info *atom_card_info =
  888. kzalloc(sizeof(struct card_info), GFP_KERNEL);
  889. if (!atom_card_info)
  890. return -ENOMEM;
  891. adev->mode_info.atom_card_info = atom_card_info;
  892. atom_card_info->dev = adev->ddev;
  893. atom_card_info->reg_read = cail_reg_read;
  894. atom_card_info->reg_write = cail_reg_write;
  895. /* needed for iio ops */
  896. if (adev->rio_mem) {
  897. atom_card_info->ioreg_read = cail_ioreg_read;
  898. atom_card_info->ioreg_write = cail_ioreg_write;
  899. } else {
  900. DRM_INFO("PCI I/O BAR is not found. Using MMIO to access ATOM BIOS\n");
  901. atom_card_info->ioreg_read = cail_reg_read;
  902. atom_card_info->ioreg_write = cail_reg_write;
  903. }
  904. atom_card_info->mc_read = cail_mc_read;
  905. atom_card_info->mc_write = cail_mc_write;
  906. atom_card_info->pll_read = cail_pll_read;
  907. atom_card_info->pll_write = cail_pll_write;
  908. adev->mode_info.atom_context = amdgpu_atom_parse(atom_card_info, adev->bios);
  909. if (!adev->mode_info.atom_context) {
  910. amdgpu_atombios_fini(adev);
  911. return -ENOMEM;
  912. }
  913. mutex_init(&adev->mode_info.atom_context->mutex);
  914. if (adev->is_atom_fw) {
  915. amdgpu_atomfirmware_scratch_regs_init(adev);
  916. amdgpu_atomfirmware_allocate_fb_scratch(adev);
  917. } else {
  918. amdgpu_atombios_scratch_regs_init(adev);
  919. amdgpu_atombios_allocate_fb_scratch(adev);
  920. }
  921. return 0;
  922. }
  923. /* if we get transitioned to only one device, take VGA back */
  924. /**
  925. * amdgpu_vga_set_decode - enable/disable vga decode
  926. *
  927. * @cookie: amdgpu_device pointer
  928. * @state: enable/disable vga decode
  929. *
  930. * Enable/disable vga decode (all asics).
  931. * Returns VGA resource flags.
  932. */
  933. static unsigned int amdgpu_vga_set_decode(void *cookie, bool state)
  934. {
  935. struct amdgpu_device *adev = cookie;
  936. amdgpu_asic_set_vga_state(adev, state);
  937. if (state)
  938. return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
  939. VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
  940. else
  941. return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
  942. }
  943. /**
  944. * amdgpu_check_pot_argument - check that argument is a power of two
  945. *
  946. * @arg: value to check
  947. *
  948. * Validates that a certain argument is a power of two (all asics).
  949. * Returns true if argument is valid.
  950. */
  951. static bool amdgpu_check_pot_argument(int arg)
  952. {
  953. return (arg & (arg - 1)) == 0;
  954. }
  955. static void amdgpu_check_block_size(struct amdgpu_device *adev)
  956. {
  957. /* defines number of bits in page table versus page directory,
  958. * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
  959. * page table and the remaining bits are in the page directory */
  960. if (amdgpu_vm_block_size == -1)
  961. return;
  962. if (amdgpu_vm_block_size < 9) {
  963. dev_warn(adev->dev, "VM page table size (%d) too small\n",
  964. amdgpu_vm_block_size);
  965. goto def_value;
  966. }
  967. if (amdgpu_vm_block_size > 24 ||
  968. (amdgpu_vm_size * 1024) < (1ull << amdgpu_vm_block_size)) {
  969. dev_warn(adev->dev, "VM page table size (%d) too large\n",
  970. amdgpu_vm_block_size);
  971. goto def_value;
  972. }
  973. return;
  974. def_value:
  975. amdgpu_vm_block_size = -1;
  976. }
  977. static void amdgpu_check_vm_size(struct amdgpu_device *adev)
  978. {
  979. if (!amdgpu_check_pot_argument(amdgpu_vm_size)) {
  980. dev_warn(adev->dev, "VM size (%d) must be a power of 2\n",
  981. amdgpu_vm_size);
  982. goto def_value;
  983. }
  984. if (amdgpu_vm_size < 1) {
  985. dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
  986. amdgpu_vm_size);
  987. goto def_value;
  988. }
  989. /*
  990. * Max GPUVM size for Cayman, SI, CI VI are 40 bits.
  991. */
  992. if (amdgpu_vm_size > 1024) {
  993. dev_warn(adev->dev, "VM size (%d) too large, max is 1TB\n",
  994. amdgpu_vm_size);
  995. goto def_value;
  996. }
  997. return;
  998. def_value:
  999. amdgpu_vm_size = -1;
  1000. }
  1001. /**
  1002. * amdgpu_check_arguments - validate module params
  1003. *
  1004. * @adev: amdgpu_device pointer
  1005. *
  1006. * Validates certain module parameters and updates
  1007. * the associated values used by the driver (all asics).
  1008. */
  1009. static void amdgpu_check_arguments(struct amdgpu_device *adev)
  1010. {
  1011. if (amdgpu_sched_jobs < 4) {
  1012. dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
  1013. amdgpu_sched_jobs);
  1014. amdgpu_sched_jobs = 4;
  1015. } else if (!amdgpu_check_pot_argument(amdgpu_sched_jobs)){
  1016. dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
  1017. amdgpu_sched_jobs);
  1018. amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
  1019. }
  1020. if (amdgpu_gart_size != -1) {
  1021. /* gtt size must be greater or equal to 32M */
  1022. if (amdgpu_gart_size < 32) {
  1023. dev_warn(adev->dev, "gart size (%d) too small\n",
  1024. amdgpu_gart_size);
  1025. amdgpu_gart_size = -1;
  1026. }
  1027. }
  1028. amdgpu_check_vm_size(adev);
  1029. amdgpu_check_block_size(adev);
  1030. if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
  1031. !amdgpu_check_pot_argument(amdgpu_vram_page_split))) {
  1032. dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
  1033. amdgpu_vram_page_split);
  1034. amdgpu_vram_page_split = 1024;
  1035. }
  1036. }
  1037. /**
  1038. * amdgpu_switcheroo_set_state - set switcheroo state
  1039. *
  1040. * @pdev: pci dev pointer
  1041. * @state: vga_switcheroo state
  1042. *
  1043. * Callback for the switcheroo driver. Suspends or resumes the
  1044. * the asics before or after it is powered up using ACPI methods.
  1045. */
  1046. static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
  1047. {
  1048. struct drm_device *dev = pci_get_drvdata(pdev);
  1049. if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
  1050. return;
  1051. if (state == VGA_SWITCHEROO_ON) {
  1052. unsigned d3_delay = dev->pdev->d3_delay;
  1053. pr_info("amdgpu: switched on\n");
  1054. /* don't suspend or resume card normally */
  1055. dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
  1056. amdgpu_device_resume(dev, true, true);
  1057. dev->pdev->d3_delay = d3_delay;
  1058. dev->switch_power_state = DRM_SWITCH_POWER_ON;
  1059. drm_kms_helper_poll_enable(dev);
  1060. } else {
  1061. pr_info("amdgpu: switched off\n");
  1062. drm_kms_helper_poll_disable(dev);
  1063. dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
  1064. amdgpu_device_suspend(dev, true, true);
  1065. dev->switch_power_state = DRM_SWITCH_POWER_OFF;
  1066. }
  1067. }
  1068. /**
  1069. * amdgpu_switcheroo_can_switch - see if switcheroo state can change
  1070. *
  1071. * @pdev: pci dev pointer
  1072. *
  1073. * Callback for the switcheroo driver. Check of the switcheroo
  1074. * state can be changed.
  1075. * Returns true if the state can be changed, false if not.
  1076. */
  1077. static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
  1078. {
  1079. struct drm_device *dev = pci_get_drvdata(pdev);
  1080. /*
  1081. * FIXME: open_count is protected by drm_global_mutex but that would lead to
  1082. * locking inversion with the driver load path. And the access here is
  1083. * completely racy anyway. So don't bother with locking for now.
  1084. */
  1085. return dev->open_count == 0;
  1086. }
  1087. static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
  1088. .set_gpu_state = amdgpu_switcheroo_set_state,
  1089. .reprobe = NULL,
  1090. .can_switch = amdgpu_switcheroo_can_switch,
  1091. };
  1092. int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
  1093. enum amd_ip_block_type block_type,
  1094. enum amd_clockgating_state state)
  1095. {
  1096. int i, r = 0;
  1097. for (i = 0; i < adev->num_ip_blocks; i++) {
  1098. if (!adev->ip_blocks[i].status.valid)
  1099. continue;
  1100. if (adev->ip_blocks[i].version->type != block_type)
  1101. continue;
  1102. if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
  1103. continue;
  1104. r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
  1105. (void *)adev, state);
  1106. if (r)
  1107. DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
  1108. adev->ip_blocks[i].version->funcs->name, r);
  1109. }
  1110. return r;
  1111. }
  1112. int amdgpu_set_powergating_state(struct amdgpu_device *adev,
  1113. enum amd_ip_block_type block_type,
  1114. enum amd_powergating_state state)
  1115. {
  1116. int i, r = 0;
  1117. for (i = 0; i < adev->num_ip_blocks; i++) {
  1118. if (!adev->ip_blocks[i].status.valid)
  1119. continue;
  1120. if (adev->ip_blocks[i].version->type != block_type)
  1121. continue;
  1122. if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
  1123. continue;
  1124. r = adev->ip_blocks[i].version->funcs->set_powergating_state(
  1125. (void *)adev, state);
  1126. if (r)
  1127. DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
  1128. adev->ip_blocks[i].version->funcs->name, r);
  1129. }
  1130. return r;
  1131. }
  1132. void amdgpu_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
  1133. {
  1134. int i;
  1135. for (i = 0; i < adev->num_ip_blocks; i++) {
  1136. if (!adev->ip_blocks[i].status.valid)
  1137. continue;
  1138. if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
  1139. adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
  1140. }
  1141. }
  1142. int amdgpu_wait_for_idle(struct amdgpu_device *adev,
  1143. enum amd_ip_block_type block_type)
  1144. {
  1145. int i, r;
  1146. for (i = 0; i < adev->num_ip_blocks; i++) {
  1147. if (!adev->ip_blocks[i].status.valid)
  1148. continue;
  1149. if (adev->ip_blocks[i].version->type == block_type) {
  1150. r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
  1151. if (r)
  1152. return r;
  1153. break;
  1154. }
  1155. }
  1156. return 0;
  1157. }
  1158. bool amdgpu_is_idle(struct amdgpu_device *adev,
  1159. enum amd_ip_block_type block_type)
  1160. {
  1161. int i;
  1162. for (i = 0; i < adev->num_ip_blocks; i++) {
  1163. if (!adev->ip_blocks[i].status.valid)
  1164. continue;
  1165. if (adev->ip_blocks[i].version->type == block_type)
  1166. return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
  1167. }
  1168. return true;
  1169. }
  1170. struct amdgpu_ip_block * amdgpu_get_ip_block(struct amdgpu_device *adev,
  1171. enum amd_ip_block_type type)
  1172. {
  1173. int i;
  1174. for (i = 0; i < adev->num_ip_blocks; i++)
  1175. if (adev->ip_blocks[i].version->type == type)
  1176. return &adev->ip_blocks[i];
  1177. return NULL;
  1178. }
  1179. /**
  1180. * amdgpu_ip_block_version_cmp
  1181. *
  1182. * @adev: amdgpu_device pointer
  1183. * @type: enum amd_ip_block_type
  1184. * @major: major version
  1185. * @minor: minor version
  1186. *
  1187. * return 0 if equal or greater
  1188. * return 1 if smaller or the ip_block doesn't exist
  1189. */
  1190. int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
  1191. enum amd_ip_block_type type,
  1192. u32 major, u32 minor)
  1193. {
  1194. struct amdgpu_ip_block *ip_block = amdgpu_get_ip_block(adev, type);
  1195. if (ip_block && ((ip_block->version->major > major) ||
  1196. ((ip_block->version->major == major) &&
  1197. (ip_block->version->minor >= minor))))
  1198. return 0;
  1199. return 1;
  1200. }
  1201. /**
  1202. * amdgpu_ip_block_add
  1203. *
  1204. * @adev: amdgpu_device pointer
  1205. * @ip_block_version: pointer to the IP to add
  1206. *
  1207. * Adds the IP block driver information to the collection of IPs
  1208. * on the asic.
  1209. */
  1210. int amdgpu_ip_block_add(struct amdgpu_device *adev,
  1211. const struct amdgpu_ip_block_version *ip_block_version)
  1212. {
  1213. if (!ip_block_version)
  1214. return -EINVAL;
  1215. DRM_DEBUG("add ip block number %d <%s>\n", adev->num_ip_blocks,
  1216. ip_block_version->funcs->name);
  1217. adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
  1218. return 0;
  1219. }
  1220. static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
  1221. {
  1222. adev->enable_virtual_display = false;
  1223. if (amdgpu_virtual_display) {
  1224. struct drm_device *ddev = adev->ddev;
  1225. const char *pci_address_name = pci_name(ddev->pdev);
  1226. char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
  1227. pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
  1228. pciaddstr_tmp = pciaddstr;
  1229. while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
  1230. pciaddname = strsep(&pciaddname_tmp, ",");
  1231. if (!strcmp("all", pciaddname)
  1232. || !strcmp(pci_address_name, pciaddname)) {
  1233. long num_crtc;
  1234. int res = -1;
  1235. adev->enable_virtual_display = true;
  1236. if (pciaddname_tmp)
  1237. res = kstrtol(pciaddname_tmp, 10,
  1238. &num_crtc);
  1239. if (!res) {
  1240. if (num_crtc < 1)
  1241. num_crtc = 1;
  1242. if (num_crtc > 6)
  1243. num_crtc = 6;
  1244. adev->mode_info.num_crtc = num_crtc;
  1245. } else {
  1246. adev->mode_info.num_crtc = 1;
  1247. }
  1248. break;
  1249. }
  1250. }
  1251. DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
  1252. amdgpu_virtual_display, pci_address_name,
  1253. adev->enable_virtual_display, adev->mode_info.num_crtc);
  1254. kfree(pciaddstr);
  1255. }
  1256. }
  1257. static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
  1258. {
  1259. const char *chip_name;
  1260. char fw_name[30];
  1261. int err;
  1262. const struct gpu_info_firmware_header_v1_0 *hdr;
  1263. adev->firmware.gpu_info_fw = NULL;
  1264. switch (adev->asic_type) {
  1265. case CHIP_TOPAZ:
  1266. case CHIP_TONGA:
  1267. case CHIP_FIJI:
  1268. case CHIP_POLARIS11:
  1269. case CHIP_POLARIS10:
  1270. case CHIP_POLARIS12:
  1271. case CHIP_CARRIZO:
  1272. case CHIP_STONEY:
  1273. #ifdef CONFIG_DRM_AMDGPU_SI
  1274. case CHIP_VERDE:
  1275. case CHIP_TAHITI:
  1276. case CHIP_PITCAIRN:
  1277. case CHIP_OLAND:
  1278. case CHIP_HAINAN:
  1279. #endif
  1280. #ifdef CONFIG_DRM_AMDGPU_CIK
  1281. case CHIP_BONAIRE:
  1282. case CHIP_HAWAII:
  1283. case CHIP_KAVERI:
  1284. case CHIP_KABINI:
  1285. case CHIP_MULLINS:
  1286. #endif
  1287. default:
  1288. return 0;
  1289. case CHIP_VEGA10:
  1290. chip_name = "vega10";
  1291. break;
  1292. case CHIP_RAVEN:
  1293. chip_name = "raven";
  1294. break;
  1295. }
  1296. snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
  1297. err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
  1298. if (err) {
  1299. dev_err(adev->dev,
  1300. "Failed to load gpu_info firmware \"%s\"\n",
  1301. fw_name);
  1302. goto out;
  1303. }
  1304. err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
  1305. if (err) {
  1306. dev_err(adev->dev,
  1307. "Failed to validate gpu_info firmware \"%s\"\n",
  1308. fw_name);
  1309. goto out;
  1310. }
  1311. hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
  1312. amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
  1313. switch (hdr->version_major) {
  1314. case 1:
  1315. {
  1316. const struct gpu_info_firmware_v1_0 *gpu_info_fw =
  1317. (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
  1318. le32_to_cpu(hdr->header.ucode_array_offset_bytes));
  1319. adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
  1320. adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
  1321. adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
  1322. adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
  1323. adev->gfx.config.max_texture_channel_caches =
  1324. le32_to_cpu(gpu_info_fw->gc_num_tccs);
  1325. adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
  1326. adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
  1327. adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
  1328. adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
  1329. adev->gfx.config.double_offchip_lds_buf =
  1330. le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
  1331. adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
  1332. adev->gfx.cu_info.max_waves_per_simd =
  1333. le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
  1334. adev->gfx.cu_info.max_scratch_slots_per_cu =
  1335. le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
  1336. adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
  1337. break;
  1338. }
  1339. default:
  1340. dev_err(adev->dev,
  1341. "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
  1342. err = -EINVAL;
  1343. goto out;
  1344. }
  1345. out:
  1346. return err;
  1347. }
  1348. static int amdgpu_early_init(struct amdgpu_device *adev)
  1349. {
  1350. int i, r;
  1351. amdgpu_device_enable_virtual_display(adev);
  1352. switch (adev->asic_type) {
  1353. case CHIP_TOPAZ:
  1354. case CHIP_TONGA:
  1355. case CHIP_FIJI:
  1356. case CHIP_POLARIS11:
  1357. case CHIP_POLARIS10:
  1358. case CHIP_POLARIS12:
  1359. case CHIP_CARRIZO:
  1360. case CHIP_STONEY:
  1361. if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
  1362. adev->family = AMDGPU_FAMILY_CZ;
  1363. else
  1364. adev->family = AMDGPU_FAMILY_VI;
  1365. r = vi_set_ip_blocks(adev);
  1366. if (r)
  1367. return r;
  1368. break;
  1369. #ifdef CONFIG_DRM_AMDGPU_SI
  1370. case CHIP_VERDE:
  1371. case CHIP_TAHITI:
  1372. case CHIP_PITCAIRN:
  1373. case CHIP_OLAND:
  1374. case CHIP_HAINAN:
  1375. adev->family = AMDGPU_FAMILY_SI;
  1376. r = si_set_ip_blocks(adev);
  1377. if (r)
  1378. return r;
  1379. break;
  1380. #endif
  1381. #ifdef CONFIG_DRM_AMDGPU_CIK
  1382. case CHIP_BONAIRE:
  1383. case CHIP_HAWAII:
  1384. case CHIP_KAVERI:
  1385. case CHIP_KABINI:
  1386. case CHIP_MULLINS:
  1387. if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
  1388. adev->family = AMDGPU_FAMILY_CI;
  1389. else
  1390. adev->family = AMDGPU_FAMILY_KV;
  1391. r = cik_set_ip_blocks(adev);
  1392. if (r)
  1393. return r;
  1394. break;
  1395. #endif
  1396. case CHIP_VEGA10:
  1397. case CHIP_RAVEN:
  1398. if (adev->asic_type == CHIP_RAVEN)
  1399. adev->family = AMDGPU_FAMILY_RV;
  1400. else
  1401. adev->family = AMDGPU_FAMILY_AI;
  1402. r = soc15_set_ip_blocks(adev);
  1403. if (r)
  1404. return r;
  1405. break;
  1406. default:
  1407. /* FIXME: not supported yet */
  1408. return -EINVAL;
  1409. }
  1410. r = amdgpu_device_parse_gpu_info_fw(adev);
  1411. if (r)
  1412. return r;
  1413. if (amdgpu_sriov_vf(adev)) {
  1414. r = amdgpu_virt_request_full_gpu(adev, true);
  1415. if (r)
  1416. return r;
  1417. }
  1418. for (i = 0; i < adev->num_ip_blocks; i++) {
  1419. if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
  1420. DRM_ERROR("disabled ip block: %d <%s>\n",
  1421. i, adev->ip_blocks[i].version->funcs->name);
  1422. adev->ip_blocks[i].status.valid = false;
  1423. } else {
  1424. if (adev->ip_blocks[i].version->funcs->early_init) {
  1425. r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
  1426. if (r == -ENOENT) {
  1427. adev->ip_blocks[i].status.valid = false;
  1428. } else if (r) {
  1429. DRM_ERROR("early_init of IP block <%s> failed %d\n",
  1430. adev->ip_blocks[i].version->funcs->name, r);
  1431. return r;
  1432. } else {
  1433. adev->ip_blocks[i].status.valid = true;
  1434. }
  1435. } else {
  1436. adev->ip_blocks[i].status.valid = true;
  1437. }
  1438. }
  1439. }
  1440. adev->cg_flags &= amdgpu_cg_mask;
  1441. adev->pg_flags &= amdgpu_pg_mask;
  1442. return 0;
  1443. }
  1444. static int amdgpu_init(struct amdgpu_device *adev)
  1445. {
  1446. int i, r;
  1447. for (i = 0; i < adev->num_ip_blocks; i++) {
  1448. if (!adev->ip_blocks[i].status.valid)
  1449. continue;
  1450. r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
  1451. if (r) {
  1452. DRM_ERROR("sw_init of IP block <%s> failed %d\n",
  1453. adev->ip_blocks[i].version->funcs->name, r);
  1454. return r;
  1455. }
  1456. adev->ip_blocks[i].status.sw = true;
  1457. /* need to do gmc hw init early so we can allocate gpu mem */
  1458. if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
  1459. r = amdgpu_vram_scratch_init(adev);
  1460. if (r) {
  1461. DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
  1462. return r;
  1463. }
  1464. r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
  1465. if (r) {
  1466. DRM_ERROR("hw_init %d failed %d\n", i, r);
  1467. return r;
  1468. }
  1469. r = amdgpu_wb_init(adev);
  1470. if (r) {
  1471. DRM_ERROR("amdgpu_wb_init failed %d\n", r);
  1472. return r;
  1473. }
  1474. adev->ip_blocks[i].status.hw = true;
  1475. /* right after GMC hw init, we create CSA */
  1476. if (amdgpu_sriov_vf(adev)) {
  1477. r = amdgpu_allocate_static_csa(adev);
  1478. if (r) {
  1479. DRM_ERROR("allocate CSA failed %d\n", r);
  1480. return r;
  1481. }
  1482. }
  1483. }
  1484. }
  1485. for (i = 0; i < adev->num_ip_blocks; i++) {
  1486. if (!adev->ip_blocks[i].status.sw)
  1487. continue;
  1488. /* gmc hw init is done early */
  1489. if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC)
  1490. continue;
  1491. r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
  1492. if (r) {
  1493. DRM_ERROR("hw_init of IP block <%s> failed %d\n",
  1494. adev->ip_blocks[i].version->funcs->name, r);
  1495. return r;
  1496. }
  1497. adev->ip_blocks[i].status.hw = true;
  1498. }
  1499. return 0;
  1500. }
  1501. static void amdgpu_fill_reset_magic(struct amdgpu_device *adev)
  1502. {
  1503. memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
  1504. }
  1505. static bool amdgpu_check_vram_lost(struct amdgpu_device *adev)
  1506. {
  1507. return !!memcmp(adev->gart.ptr, adev->reset_magic,
  1508. AMDGPU_RESET_MAGIC_NUM);
  1509. }
  1510. static int amdgpu_late_set_cg_state(struct amdgpu_device *adev)
  1511. {
  1512. int i = 0, r;
  1513. for (i = 0; i < adev->num_ip_blocks; i++) {
  1514. if (!adev->ip_blocks[i].status.valid)
  1515. continue;
  1516. /* skip CG for VCE/UVD, it's handled specially */
  1517. if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
  1518. adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
  1519. /* enable clockgating to save power */
  1520. r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
  1521. AMD_CG_STATE_GATE);
  1522. if (r) {
  1523. DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
  1524. adev->ip_blocks[i].version->funcs->name, r);
  1525. return r;
  1526. }
  1527. }
  1528. }
  1529. return 0;
  1530. }
  1531. static int amdgpu_late_init(struct amdgpu_device *adev)
  1532. {
  1533. int i = 0, r;
  1534. for (i = 0; i < adev->num_ip_blocks; i++) {
  1535. if (!adev->ip_blocks[i].status.valid)
  1536. continue;
  1537. if (adev->ip_blocks[i].version->funcs->late_init) {
  1538. r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
  1539. if (r) {
  1540. DRM_ERROR("late_init of IP block <%s> failed %d\n",
  1541. adev->ip_blocks[i].version->funcs->name, r);
  1542. return r;
  1543. }
  1544. adev->ip_blocks[i].status.late_initialized = true;
  1545. }
  1546. }
  1547. mod_delayed_work(system_wq, &adev->late_init_work,
  1548. msecs_to_jiffies(AMDGPU_RESUME_MS));
  1549. amdgpu_fill_reset_magic(adev);
  1550. return 0;
  1551. }
  1552. static int amdgpu_fini(struct amdgpu_device *adev)
  1553. {
  1554. int i, r;
  1555. /* need to disable SMC first */
  1556. for (i = 0; i < adev->num_ip_blocks; i++) {
  1557. if (!adev->ip_blocks[i].status.hw)
  1558. continue;
  1559. if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
  1560. /* ungate blocks before hw fini so that we can shutdown the blocks safely */
  1561. r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
  1562. AMD_CG_STATE_UNGATE);
  1563. if (r) {
  1564. DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
  1565. adev->ip_blocks[i].version->funcs->name, r);
  1566. return r;
  1567. }
  1568. r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
  1569. /* XXX handle errors */
  1570. if (r) {
  1571. DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
  1572. adev->ip_blocks[i].version->funcs->name, r);
  1573. }
  1574. adev->ip_blocks[i].status.hw = false;
  1575. break;
  1576. }
  1577. }
  1578. for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
  1579. if (!adev->ip_blocks[i].status.hw)
  1580. continue;
  1581. if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
  1582. amdgpu_wb_fini(adev);
  1583. amdgpu_vram_scratch_fini(adev);
  1584. }
  1585. if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
  1586. adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
  1587. /* ungate blocks before hw fini so that we can shutdown the blocks safely */
  1588. r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
  1589. AMD_CG_STATE_UNGATE);
  1590. if (r) {
  1591. DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
  1592. adev->ip_blocks[i].version->funcs->name, r);
  1593. return r;
  1594. }
  1595. }
  1596. r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
  1597. /* XXX handle errors */
  1598. if (r) {
  1599. DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
  1600. adev->ip_blocks[i].version->funcs->name, r);
  1601. }
  1602. adev->ip_blocks[i].status.hw = false;
  1603. }
  1604. for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
  1605. if (!adev->ip_blocks[i].status.sw)
  1606. continue;
  1607. r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
  1608. /* XXX handle errors */
  1609. if (r) {
  1610. DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
  1611. adev->ip_blocks[i].version->funcs->name, r);
  1612. }
  1613. adev->ip_blocks[i].status.sw = false;
  1614. adev->ip_blocks[i].status.valid = false;
  1615. }
  1616. for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
  1617. if (!adev->ip_blocks[i].status.late_initialized)
  1618. continue;
  1619. if (adev->ip_blocks[i].version->funcs->late_fini)
  1620. adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
  1621. adev->ip_blocks[i].status.late_initialized = false;
  1622. }
  1623. if (amdgpu_sriov_vf(adev)) {
  1624. amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
  1625. amdgpu_virt_release_full_gpu(adev, false);
  1626. }
  1627. return 0;
  1628. }
  1629. static void amdgpu_late_init_func_handler(struct work_struct *work)
  1630. {
  1631. struct amdgpu_device *adev =
  1632. container_of(work, struct amdgpu_device, late_init_work.work);
  1633. amdgpu_late_set_cg_state(adev);
  1634. }
  1635. int amdgpu_suspend(struct amdgpu_device *adev)
  1636. {
  1637. int i, r;
  1638. if (amdgpu_sriov_vf(adev))
  1639. amdgpu_virt_request_full_gpu(adev, false);
  1640. /* ungate SMC block first */
  1641. r = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
  1642. AMD_CG_STATE_UNGATE);
  1643. if (r) {
  1644. DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n",r);
  1645. }
  1646. for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
  1647. if (!adev->ip_blocks[i].status.valid)
  1648. continue;
  1649. /* ungate blocks so that suspend can properly shut them down */
  1650. if (i != AMD_IP_BLOCK_TYPE_SMC) {
  1651. r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
  1652. AMD_CG_STATE_UNGATE);
  1653. if (r) {
  1654. DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
  1655. adev->ip_blocks[i].version->funcs->name, r);
  1656. }
  1657. }
  1658. /* XXX handle errors */
  1659. r = adev->ip_blocks[i].version->funcs->suspend(adev);
  1660. /* XXX handle errors */
  1661. if (r) {
  1662. DRM_ERROR("suspend of IP block <%s> failed %d\n",
  1663. adev->ip_blocks[i].version->funcs->name, r);
  1664. }
  1665. }
  1666. if (amdgpu_sriov_vf(adev))
  1667. amdgpu_virt_release_full_gpu(adev, false);
  1668. return 0;
  1669. }
  1670. static int amdgpu_sriov_reinit_early(struct amdgpu_device *adev)
  1671. {
  1672. int i, r;
  1673. static enum amd_ip_block_type ip_order[] = {
  1674. AMD_IP_BLOCK_TYPE_GMC,
  1675. AMD_IP_BLOCK_TYPE_COMMON,
  1676. AMD_IP_BLOCK_TYPE_IH,
  1677. };
  1678. for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
  1679. int j;
  1680. struct amdgpu_ip_block *block;
  1681. for (j = 0; j < adev->num_ip_blocks; j++) {
  1682. block = &adev->ip_blocks[j];
  1683. if (block->version->type != ip_order[i] ||
  1684. !block->status.valid)
  1685. continue;
  1686. r = block->version->funcs->hw_init(adev);
  1687. DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
  1688. }
  1689. }
  1690. return 0;
  1691. }
  1692. static int amdgpu_sriov_reinit_late(struct amdgpu_device *adev)
  1693. {
  1694. int i, r;
  1695. static enum amd_ip_block_type ip_order[] = {
  1696. AMD_IP_BLOCK_TYPE_SMC,
  1697. AMD_IP_BLOCK_TYPE_DCE,
  1698. AMD_IP_BLOCK_TYPE_GFX,
  1699. AMD_IP_BLOCK_TYPE_SDMA,
  1700. AMD_IP_BLOCK_TYPE_VCE,
  1701. };
  1702. for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
  1703. int j;
  1704. struct amdgpu_ip_block *block;
  1705. for (j = 0; j < adev->num_ip_blocks; j++) {
  1706. block = &adev->ip_blocks[j];
  1707. if (block->version->type != ip_order[i] ||
  1708. !block->status.valid)
  1709. continue;
  1710. r = block->version->funcs->hw_init(adev);
  1711. DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
  1712. }
  1713. }
  1714. return 0;
  1715. }
  1716. static int amdgpu_resume_phase1(struct amdgpu_device *adev)
  1717. {
  1718. int i, r;
  1719. for (i = 0; i < adev->num_ip_blocks; i++) {
  1720. if (!adev->ip_blocks[i].status.valid)
  1721. continue;
  1722. if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
  1723. adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
  1724. adev->ip_blocks[i].version->type ==
  1725. AMD_IP_BLOCK_TYPE_IH) {
  1726. r = adev->ip_blocks[i].version->funcs->resume(adev);
  1727. if (r) {
  1728. DRM_ERROR("resume of IP block <%s> failed %d\n",
  1729. adev->ip_blocks[i].version->funcs->name, r);
  1730. return r;
  1731. }
  1732. }
  1733. }
  1734. return 0;
  1735. }
  1736. static int amdgpu_resume_phase2(struct amdgpu_device *adev)
  1737. {
  1738. int i, r;
  1739. for (i = 0; i < adev->num_ip_blocks; i++) {
  1740. if (!adev->ip_blocks[i].status.valid)
  1741. continue;
  1742. if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
  1743. adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
  1744. adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH )
  1745. continue;
  1746. r = adev->ip_blocks[i].version->funcs->resume(adev);
  1747. if (r) {
  1748. DRM_ERROR("resume of IP block <%s> failed %d\n",
  1749. adev->ip_blocks[i].version->funcs->name, r);
  1750. return r;
  1751. }
  1752. }
  1753. return 0;
  1754. }
  1755. static int amdgpu_resume(struct amdgpu_device *adev)
  1756. {
  1757. int r;
  1758. r = amdgpu_resume_phase1(adev);
  1759. if (r)
  1760. return r;
  1761. r = amdgpu_resume_phase2(adev);
  1762. return r;
  1763. }
  1764. static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
  1765. {
  1766. if (adev->is_atom_fw) {
  1767. if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
  1768. adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
  1769. } else {
  1770. if (amdgpu_atombios_has_gpu_virtualization_table(adev))
  1771. adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
  1772. }
  1773. }
  1774. /**
  1775. * amdgpu_device_init - initialize the driver
  1776. *
  1777. * @adev: amdgpu_device pointer
  1778. * @pdev: drm dev pointer
  1779. * @pdev: pci dev pointer
  1780. * @flags: driver flags
  1781. *
  1782. * Initializes the driver info and hw (all asics).
  1783. * Returns 0 for success or an error on failure.
  1784. * Called at driver startup.
  1785. */
  1786. int amdgpu_device_init(struct amdgpu_device *adev,
  1787. struct drm_device *ddev,
  1788. struct pci_dev *pdev,
  1789. uint32_t flags)
  1790. {
  1791. int r, i;
  1792. bool runtime = false;
  1793. u32 max_MBps;
  1794. adev->shutdown = false;
  1795. adev->dev = &pdev->dev;
  1796. adev->ddev = ddev;
  1797. adev->pdev = pdev;
  1798. adev->flags = flags;
  1799. adev->asic_type = flags & AMD_ASIC_MASK;
  1800. adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
  1801. adev->mc.gtt_size = 512 * 1024 * 1024;
  1802. adev->accel_working = false;
  1803. adev->num_rings = 0;
  1804. adev->mman.buffer_funcs = NULL;
  1805. adev->mman.buffer_funcs_ring = NULL;
  1806. adev->vm_manager.vm_pte_funcs = NULL;
  1807. adev->vm_manager.vm_pte_num_rings = 0;
  1808. adev->gart.gart_funcs = NULL;
  1809. adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
  1810. adev->smc_rreg = &amdgpu_invalid_rreg;
  1811. adev->smc_wreg = &amdgpu_invalid_wreg;
  1812. adev->pcie_rreg = &amdgpu_invalid_rreg;
  1813. adev->pcie_wreg = &amdgpu_invalid_wreg;
  1814. adev->pciep_rreg = &amdgpu_invalid_rreg;
  1815. adev->pciep_wreg = &amdgpu_invalid_wreg;
  1816. adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
  1817. adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
  1818. adev->didt_rreg = &amdgpu_invalid_rreg;
  1819. adev->didt_wreg = &amdgpu_invalid_wreg;
  1820. adev->gc_cac_rreg = &amdgpu_invalid_rreg;
  1821. adev->gc_cac_wreg = &amdgpu_invalid_wreg;
  1822. adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
  1823. adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
  1824. DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
  1825. amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
  1826. pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
  1827. /* mutex initialization are all done here so we
  1828. * can recall function without having locking issues */
  1829. atomic_set(&adev->irq.ih.lock, 0);
  1830. mutex_init(&adev->firmware.mutex);
  1831. mutex_init(&adev->pm.mutex);
  1832. mutex_init(&adev->gfx.gpu_clock_mutex);
  1833. mutex_init(&adev->srbm_mutex);
  1834. mutex_init(&adev->grbm_idx_mutex);
  1835. mutex_init(&adev->mn_lock);
  1836. hash_init(adev->mn_hash);
  1837. amdgpu_check_arguments(adev);
  1838. spin_lock_init(&adev->mmio_idx_lock);
  1839. spin_lock_init(&adev->smc_idx_lock);
  1840. spin_lock_init(&adev->pcie_idx_lock);
  1841. spin_lock_init(&adev->uvd_ctx_idx_lock);
  1842. spin_lock_init(&adev->didt_idx_lock);
  1843. spin_lock_init(&adev->gc_cac_idx_lock);
  1844. spin_lock_init(&adev->audio_endpt_idx_lock);
  1845. spin_lock_init(&adev->mm_stats.lock);
  1846. INIT_LIST_HEAD(&adev->shadow_list);
  1847. mutex_init(&adev->shadow_list_lock);
  1848. INIT_LIST_HEAD(&adev->gtt_list);
  1849. spin_lock_init(&adev->gtt_list_lock);
  1850. INIT_LIST_HEAD(&adev->ring_lru_list);
  1851. spin_lock_init(&adev->ring_lru_list_lock);
  1852. INIT_DELAYED_WORK(&adev->late_init_work, amdgpu_late_init_func_handler);
  1853. /* Registers mapping */
  1854. /* TODO: block userspace mapping of io register */
  1855. if (adev->asic_type >= CHIP_BONAIRE) {
  1856. adev->rmmio_base = pci_resource_start(adev->pdev, 5);
  1857. adev->rmmio_size = pci_resource_len(adev->pdev, 5);
  1858. } else {
  1859. adev->rmmio_base = pci_resource_start(adev->pdev, 2);
  1860. adev->rmmio_size = pci_resource_len(adev->pdev, 2);
  1861. }
  1862. adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
  1863. if (adev->rmmio == NULL) {
  1864. return -ENOMEM;
  1865. }
  1866. DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
  1867. DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
  1868. if (adev->asic_type >= CHIP_BONAIRE)
  1869. /* doorbell bar mapping */
  1870. amdgpu_doorbell_init(adev);
  1871. /* io port mapping */
  1872. for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
  1873. if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
  1874. adev->rio_mem_size = pci_resource_len(adev->pdev, i);
  1875. adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
  1876. break;
  1877. }
  1878. }
  1879. if (adev->rio_mem == NULL)
  1880. DRM_INFO("PCI I/O BAR is not found.\n");
  1881. /* early init functions */
  1882. r = amdgpu_early_init(adev);
  1883. if (r)
  1884. return r;
  1885. /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
  1886. /* this will fail for cards that aren't VGA class devices, just
  1887. * ignore it */
  1888. vga_client_register(adev->pdev, adev, NULL, amdgpu_vga_set_decode);
  1889. if (amdgpu_runtime_pm == 1)
  1890. runtime = true;
  1891. if (amdgpu_device_is_px(ddev))
  1892. runtime = true;
  1893. if (!pci_is_thunderbolt_attached(adev->pdev))
  1894. vga_switcheroo_register_client(adev->pdev,
  1895. &amdgpu_switcheroo_ops, runtime);
  1896. if (runtime)
  1897. vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
  1898. /* Read BIOS */
  1899. if (!amdgpu_get_bios(adev)) {
  1900. r = -EINVAL;
  1901. goto failed;
  1902. }
  1903. r = amdgpu_atombios_init(adev);
  1904. if (r) {
  1905. dev_err(adev->dev, "amdgpu_atombios_init failed\n");
  1906. goto failed;
  1907. }
  1908. /* detect if we are with an SRIOV vbios */
  1909. amdgpu_device_detect_sriov_bios(adev);
  1910. /* Post card if necessary */
  1911. if (amdgpu_vpost_needed(adev)) {
  1912. if (!adev->bios) {
  1913. dev_err(adev->dev, "no vBIOS found\n");
  1914. r = -EINVAL;
  1915. goto failed;
  1916. }
  1917. DRM_INFO("GPU posting now...\n");
  1918. r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
  1919. if (r) {
  1920. dev_err(adev->dev, "gpu post error!\n");
  1921. goto failed;
  1922. }
  1923. } else {
  1924. DRM_INFO("GPU post is not needed\n");
  1925. }
  1926. if (!adev->is_atom_fw) {
  1927. /* Initialize clocks */
  1928. r = amdgpu_atombios_get_clock_info(adev);
  1929. if (r) {
  1930. dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
  1931. return r;
  1932. }
  1933. /* init i2c buses */
  1934. amdgpu_atombios_i2c_init(adev);
  1935. }
  1936. /* Fence driver */
  1937. r = amdgpu_fence_driver_init(adev);
  1938. if (r) {
  1939. dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
  1940. goto failed;
  1941. }
  1942. /* init the mode config */
  1943. drm_mode_config_init(adev->ddev);
  1944. r = amdgpu_init(adev);
  1945. if (r) {
  1946. dev_err(adev->dev, "amdgpu_init failed\n");
  1947. amdgpu_fini(adev);
  1948. goto failed;
  1949. }
  1950. adev->accel_working = true;
  1951. amdgpu_vm_check_compute_bug(adev);
  1952. /* Initialize the buffer migration limit. */
  1953. if (amdgpu_moverate >= 0)
  1954. max_MBps = amdgpu_moverate;
  1955. else
  1956. max_MBps = 8; /* Allow 8 MB/s. */
  1957. /* Get a log2 for easy divisions. */
  1958. adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
  1959. r = amdgpu_ib_pool_init(adev);
  1960. if (r) {
  1961. dev_err(adev->dev, "IB initialization failed (%d).\n", r);
  1962. goto failed;
  1963. }
  1964. r = amdgpu_ib_ring_tests(adev);
  1965. if (r)
  1966. DRM_ERROR("ib ring test failed (%d).\n", r);
  1967. amdgpu_fbdev_init(adev);
  1968. r = amdgpu_gem_debugfs_init(adev);
  1969. if (r)
  1970. DRM_ERROR("registering gem debugfs failed (%d).\n", r);
  1971. r = amdgpu_debugfs_regs_init(adev);
  1972. if (r)
  1973. DRM_ERROR("registering register debugfs failed (%d).\n", r);
  1974. r = amdgpu_debugfs_test_ib_ring_init(adev);
  1975. if (r)
  1976. DRM_ERROR("registering register test ib ring debugfs failed (%d).\n", r);
  1977. r = amdgpu_debugfs_firmware_init(adev);
  1978. if (r)
  1979. DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
  1980. if ((amdgpu_testing & 1)) {
  1981. if (adev->accel_working)
  1982. amdgpu_test_moves(adev);
  1983. else
  1984. DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
  1985. }
  1986. if (amdgpu_benchmarking) {
  1987. if (adev->accel_working)
  1988. amdgpu_benchmark(adev, amdgpu_benchmarking);
  1989. else
  1990. DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
  1991. }
  1992. /* enable clockgating, etc. after ib tests, etc. since some blocks require
  1993. * explicit gating rather than handling it automatically.
  1994. */
  1995. r = amdgpu_late_init(adev);
  1996. if (r) {
  1997. dev_err(adev->dev, "amdgpu_late_init failed\n");
  1998. goto failed;
  1999. }
  2000. return 0;
  2001. failed:
  2002. if (runtime)
  2003. vga_switcheroo_fini_domain_pm_ops(adev->dev);
  2004. return r;
  2005. }
  2006. /**
  2007. * amdgpu_device_fini - tear down the driver
  2008. *
  2009. * @adev: amdgpu_device pointer
  2010. *
  2011. * Tear down the driver info (all asics).
  2012. * Called at driver shutdown.
  2013. */
  2014. void amdgpu_device_fini(struct amdgpu_device *adev)
  2015. {
  2016. int r;
  2017. DRM_INFO("amdgpu: finishing device.\n");
  2018. adev->shutdown = true;
  2019. if (adev->mode_info.mode_config_initialized)
  2020. drm_crtc_force_disable_all(adev->ddev);
  2021. /* evict vram memory */
  2022. amdgpu_bo_evict_vram(adev);
  2023. amdgpu_ib_pool_fini(adev);
  2024. amdgpu_fence_driver_fini(adev);
  2025. amdgpu_fbdev_fini(adev);
  2026. r = amdgpu_fini(adev);
  2027. if (adev->firmware.gpu_info_fw) {
  2028. release_firmware(adev->firmware.gpu_info_fw);
  2029. adev->firmware.gpu_info_fw = NULL;
  2030. }
  2031. adev->accel_working = false;
  2032. cancel_delayed_work_sync(&adev->late_init_work);
  2033. /* free i2c buses */
  2034. amdgpu_i2c_fini(adev);
  2035. amdgpu_atombios_fini(adev);
  2036. kfree(adev->bios);
  2037. adev->bios = NULL;
  2038. if (!pci_is_thunderbolt_attached(adev->pdev))
  2039. vga_switcheroo_unregister_client(adev->pdev);
  2040. if (adev->flags & AMD_IS_PX)
  2041. vga_switcheroo_fini_domain_pm_ops(adev->dev);
  2042. vga_client_register(adev->pdev, NULL, NULL, NULL);
  2043. if (adev->rio_mem)
  2044. pci_iounmap(adev->pdev, adev->rio_mem);
  2045. adev->rio_mem = NULL;
  2046. iounmap(adev->rmmio);
  2047. adev->rmmio = NULL;
  2048. if (adev->asic_type >= CHIP_BONAIRE)
  2049. amdgpu_doorbell_fini(adev);
  2050. amdgpu_debugfs_regs_cleanup(adev);
  2051. }
  2052. /*
  2053. * Suspend & resume.
  2054. */
  2055. /**
  2056. * amdgpu_device_suspend - initiate device suspend
  2057. *
  2058. * @pdev: drm dev pointer
  2059. * @state: suspend state
  2060. *
  2061. * Puts the hw in the suspend state (all asics).
  2062. * Returns 0 for success or an error on failure.
  2063. * Called at driver suspend.
  2064. */
  2065. int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
  2066. {
  2067. struct amdgpu_device *adev;
  2068. struct drm_crtc *crtc;
  2069. struct drm_connector *connector;
  2070. int r;
  2071. if (dev == NULL || dev->dev_private == NULL) {
  2072. return -ENODEV;
  2073. }
  2074. adev = dev->dev_private;
  2075. if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
  2076. return 0;
  2077. drm_kms_helper_poll_disable(dev);
  2078. /* turn off display hw */
  2079. drm_modeset_lock_all(dev);
  2080. list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
  2081. drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
  2082. }
  2083. drm_modeset_unlock_all(dev);
  2084. /* unpin the front buffers and cursors */
  2085. list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
  2086. struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
  2087. struct amdgpu_framebuffer *rfb = to_amdgpu_framebuffer(crtc->primary->fb);
  2088. struct amdgpu_bo *robj;
  2089. if (amdgpu_crtc->cursor_bo) {
  2090. struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
  2091. r = amdgpu_bo_reserve(aobj, true);
  2092. if (r == 0) {
  2093. amdgpu_bo_unpin(aobj);
  2094. amdgpu_bo_unreserve(aobj);
  2095. }
  2096. }
  2097. if (rfb == NULL || rfb->obj == NULL) {
  2098. continue;
  2099. }
  2100. robj = gem_to_amdgpu_bo(rfb->obj);
  2101. /* don't unpin kernel fb objects */
  2102. if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
  2103. r = amdgpu_bo_reserve(robj, true);
  2104. if (r == 0) {
  2105. amdgpu_bo_unpin(robj);
  2106. amdgpu_bo_unreserve(robj);
  2107. }
  2108. }
  2109. }
  2110. /* evict vram memory */
  2111. amdgpu_bo_evict_vram(adev);
  2112. amdgpu_fence_driver_suspend(adev);
  2113. r = amdgpu_suspend(adev);
  2114. /* evict remaining vram memory
  2115. * This second call to evict vram is to evict the gart page table
  2116. * using the CPU.
  2117. */
  2118. amdgpu_bo_evict_vram(adev);
  2119. if (adev->is_atom_fw)
  2120. amdgpu_atomfirmware_scratch_regs_save(adev);
  2121. else
  2122. amdgpu_atombios_scratch_regs_save(adev);
  2123. pci_save_state(dev->pdev);
  2124. if (suspend) {
  2125. /* Shut down the device */
  2126. pci_disable_device(dev->pdev);
  2127. pci_set_power_state(dev->pdev, PCI_D3hot);
  2128. } else {
  2129. r = amdgpu_asic_reset(adev);
  2130. if (r)
  2131. DRM_ERROR("amdgpu asic reset failed\n");
  2132. }
  2133. if (fbcon) {
  2134. console_lock();
  2135. amdgpu_fbdev_set_suspend(adev, 1);
  2136. console_unlock();
  2137. }
  2138. return 0;
  2139. }
  2140. /**
  2141. * amdgpu_device_resume - initiate device resume
  2142. *
  2143. * @pdev: drm dev pointer
  2144. *
  2145. * Bring the hw back to operating state (all asics).
  2146. * Returns 0 for success or an error on failure.
  2147. * Called at driver resume.
  2148. */
  2149. int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
  2150. {
  2151. struct drm_connector *connector;
  2152. struct amdgpu_device *adev = dev->dev_private;
  2153. struct drm_crtc *crtc;
  2154. int r = 0;
  2155. if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
  2156. return 0;
  2157. if (fbcon)
  2158. console_lock();
  2159. if (resume) {
  2160. pci_set_power_state(dev->pdev, PCI_D0);
  2161. pci_restore_state(dev->pdev);
  2162. r = pci_enable_device(dev->pdev);
  2163. if (r)
  2164. goto unlock;
  2165. }
  2166. if (adev->is_atom_fw)
  2167. amdgpu_atomfirmware_scratch_regs_restore(adev);
  2168. else
  2169. amdgpu_atombios_scratch_regs_restore(adev);
  2170. /* post card */
  2171. if (amdgpu_need_post(adev)) {
  2172. r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
  2173. if (r)
  2174. DRM_ERROR("amdgpu asic init failed\n");
  2175. }
  2176. r = amdgpu_resume(adev);
  2177. if (r) {
  2178. DRM_ERROR("amdgpu_resume failed (%d).\n", r);
  2179. goto unlock;
  2180. }
  2181. amdgpu_fence_driver_resume(adev);
  2182. if (resume) {
  2183. r = amdgpu_ib_ring_tests(adev);
  2184. if (r)
  2185. DRM_ERROR("ib ring test failed (%d).\n", r);
  2186. }
  2187. r = amdgpu_late_init(adev);
  2188. if (r)
  2189. goto unlock;
  2190. /* pin cursors */
  2191. list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
  2192. struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
  2193. if (amdgpu_crtc->cursor_bo) {
  2194. struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
  2195. r = amdgpu_bo_reserve(aobj, true);
  2196. if (r == 0) {
  2197. r = amdgpu_bo_pin(aobj,
  2198. AMDGPU_GEM_DOMAIN_VRAM,
  2199. &amdgpu_crtc->cursor_addr);
  2200. if (r != 0)
  2201. DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
  2202. amdgpu_bo_unreserve(aobj);
  2203. }
  2204. }
  2205. }
  2206. /* blat the mode back in */
  2207. if (fbcon) {
  2208. drm_helper_resume_force_mode(dev);
  2209. /* turn on display hw */
  2210. drm_modeset_lock_all(dev);
  2211. list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
  2212. drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
  2213. }
  2214. drm_modeset_unlock_all(dev);
  2215. }
  2216. drm_kms_helper_poll_enable(dev);
  2217. /*
  2218. * Most of the connector probing functions try to acquire runtime pm
  2219. * refs to ensure that the GPU is powered on when connector polling is
  2220. * performed. Since we're calling this from a runtime PM callback,
  2221. * trying to acquire rpm refs will cause us to deadlock.
  2222. *
  2223. * Since we're guaranteed to be holding the rpm lock, it's safe to
  2224. * temporarily disable the rpm helpers so this doesn't deadlock us.
  2225. */
  2226. #ifdef CONFIG_PM
  2227. dev->dev->power.disable_depth++;
  2228. #endif
  2229. drm_helper_hpd_irq_event(dev);
  2230. #ifdef CONFIG_PM
  2231. dev->dev->power.disable_depth--;
  2232. #endif
  2233. if (fbcon)
  2234. amdgpu_fbdev_set_suspend(adev, 0);
  2235. unlock:
  2236. if (fbcon)
  2237. console_unlock();
  2238. return r;
  2239. }
  2240. static bool amdgpu_check_soft_reset(struct amdgpu_device *adev)
  2241. {
  2242. int i;
  2243. bool asic_hang = false;
  2244. for (i = 0; i < adev->num_ip_blocks; i++) {
  2245. if (!adev->ip_blocks[i].status.valid)
  2246. continue;
  2247. if (adev->ip_blocks[i].version->funcs->check_soft_reset)
  2248. adev->ip_blocks[i].status.hang =
  2249. adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
  2250. if (adev->ip_blocks[i].status.hang) {
  2251. DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
  2252. asic_hang = true;
  2253. }
  2254. }
  2255. return asic_hang;
  2256. }
  2257. static int amdgpu_pre_soft_reset(struct amdgpu_device *adev)
  2258. {
  2259. int i, r = 0;
  2260. for (i = 0; i < adev->num_ip_blocks; i++) {
  2261. if (!adev->ip_blocks[i].status.valid)
  2262. continue;
  2263. if (adev->ip_blocks[i].status.hang &&
  2264. adev->ip_blocks[i].version->funcs->pre_soft_reset) {
  2265. r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
  2266. if (r)
  2267. return r;
  2268. }
  2269. }
  2270. return 0;
  2271. }
  2272. static bool amdgpu_need_full_reset(struct amdgpu_device *adev)
  2273. {
  2274. int i;
  2275. for (i = 0; i < adev->num_ip_blocks; i++) {
  2276. if (!adev->ip_blocks[i].status.valid)
  2277. continue;
  2278. if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
  2279. (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
  2280. (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
  2281. (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)) {
  2282. if (adev->ip_blocks[i].status.hang) {
  2283. DRM_INFO("Some block need full reset!\n");
  2284. return true;
  2285. }
  2286. }
  2287. }
  2288. return false;
  2289. }
  2290. static int amdgpu_soft_reset(struct amdgpu_device *adev)
  2291. {
  2292. int i, r = 0;
  2293. for (i = 0; i < adev->num_ip_blocks; i++) {
  2294. if (!adev->ip_blocks[i].status.valid)
  2295. continue;
  2296. if (adev->ip_blocks[i].status.hang &&
  2297. adev->ip_blocks[i].version->funcs->soft_reset) {
  2298. r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
  2299. if (r)
  2300. return r;
  2301. }
  2302. }
  2303. return 0;
  2304. }
  2305. static int amdgpu_post_soft_reset(struct amdgpu_device *adev)
  2306. {
  2307. int i, r = 0;
  2308. for (i = 0; i < adev->num_ip_blocks; i++) {
  2309. if (!adev->ip_blocks[i].status.valid)
  2310. continue;
  2311. if (adev->ip_blocks[i].status.hang &&
  2312. adev->ip_blocks[i].version->funcs->post_soft_reset)
  2313. r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
  2314. if (r)
  2315. return r;
  2316. }
  2317. return 0;
  2318. }
  2319. bool amdgpu_need_backup(struct amdgpu_device *adev)
  2320. {
  2321. if (adev->flags & AMD_IS_APU)
  2322. return false;
  2323. return amdgpu_lockup_timeout > 0 ? true : false;
  2324. }
  2325. static int amdgpu_recover_vram_from_shadow(struct amdgpu_device *adev,
  2326. struct amdgpu_ring *ring,
  2327. struct amdgpu_bo *bo,
  2328. struct dma_fence **fence)
  2329. {
  2330. uint32_t domain;
  2331. int r;
  2332. if (!bo->shadow)
  2333. return 0;
  2334. r = amdgpu_bo_reserve(bo, true);
  2335. if (r)
  2336. return r;
  2337. domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
  2338. /* if bo has been evicted, then no need to recover */
  2339. if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
  2340. r = amdgpu_bo_validate(bo->shadow);
  2341. if (r) {
  2342. DRM_ERROR("bo validate failed!\n");
  2343. goto err;
  2344. }
  2345. r = amdgpu_ttm_bind(&bo->shadow->tbo, &bo->shadow->tbo.mem);
  2346. if (r) {
  2347. DRM_ERROR("%p bind failed\n", bo->shadow);
  2348. goto err;
  2349. }
  2350. r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
  2351. NULL, fence, true);
  2352. if (r) {
  2353. DRM_ERROR("recover page table failed!\n");
  2354. goto err;
  2355. }
  2356. }
  2357. err:
  2358. amdgpu_bo_unreserve(bo);
  2359. return r;
  2360. }
  2361. /**
  2362. * amdgpu_sriov_gpu_reset - reset the asic
  2363. *
  2364. * @adev: amdgpu device pointer
  2365. * @job: which job trigger hang
  2366. *
  2367. * Attempt the reset the GPU if it has hung (all asics).
  2368. * for SRIOV case.
  2369. * Returns 0 for success or an error on failure.
  2370. */
  2371. int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job)
  2372. {
  2373. int i, j, r = 0;
  2374. int resched;
  2375. struct amdgpu_bo *bo, *tmp;
  2376. struct amdgpu_ring *ring;
  2377. struct dma_fence *fence = NULL, *next = NULL;
  2378. mutex_lock(&adev->virt.lock_reset);
  2379. atomic_inc(&adev->gpu_reset_counter);
  2380. adev->gfx.in_reset = true;
  2381. /* block TTM */
  2382. resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
  2383. /* we start from the ring trigger GPU hang */
  2384. j = job ? job->ring->idx : 0;
  2385. /* block scheduler */
  2386. for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) {
  2387. ring = adev->rings[i % AMDGPU_MAX_RINGS];
  2388. if (!ring || !ring->sched.thread)
  2389. continue;
  2390. kthread_park(ring->sched.thread);
  2391. if (job && j != i)
  2392. continue;
  2393. /* here give the last chance to check if job removed from mirror-list
  2394. * since we already pay some time on kthread_park */
  2395. if (job && list_empty(&job->base.node)) {
  2396. kthread_unpark(ring->sched.thread);
  2397. goto give_up_reset;
  2398. }
  2399. if (amd_sched_invalidate_job(&job->base, amdgpu_job_hang_limit))
  2400. amd_sched_job_kickout(&job->base);
  2401. /* only do job_reset on the hang ring if @job not NULL */
  2402. amd_sched_hw_job_reset(&ring->sched);
  2403. /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
  2404. amdgpu_fence_driver_force_completion_ring(ring);
  2405. }
  2406. /* request to take full control of GPU before re-initialization */
  2407. if (job)
  2408. amdgpu_virt_reset_gpu(adev);
  2409. else
  2410. amdgpu_virt_request_full_gpu(adev, true);
  2411. /* Resume IP prior to SMC */
  2412. amdgpu_sriov_reinit_early(adev);
  2413. /* we need recover gart prior to run SMC/CP/SDMA resume */
  2414. amdgpu_ttm_recover_gart(adev);
  2415. /* now we are okay to resume SMC/CP/SDMA */
  2416. amdgpu_sriov_reinit_late(adev);
  2417. amdgpu_irq_gpu_reset_resume_helper(adev);
  2418. if (amdgpu_ib_ring_tests(adev))
  2419. dev_err(adev->dev, "[GPU_RESET] ib ring test failed (%d).\n", r);
  2420. /* release full control of GPU after ib test */
  2421. amdgpu_virt_release_full_gpu(adev, true);
  2422. DRM_INFO("recover vram bo from shadow\n");
  2423. ring = adev->mman.buffer_funcs_ring;
  2424. mutex_lock(&adev->shadow_list_lock);
  2425. list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
  2426. next = NULL;
  2427. amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
  2428. if (fence) {
  2429. r = dma_fence_wait(fence, false);
  2430. if (r) {
  2431. WARN(r, "recovery from shadow isn't completed\n");
  2432. break;
  2433. }
  2434. }
  2435. dma_fence_put(fence);
  2436. fence = next;
  2437. }
  2438. mutex_unlock(&adev->shadow_list_lock);
  2439. if (fence) {
  2440. r = dma_fence_wait(fence, false);
  2441. if (r)
  2442. WARN(r, "recovery from shadow isn't completed\n");
  2443. }
  2444. dma_fence_put(fence);
  2445. for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) {
  2446. ring = adev->rings[i % AMDGPU_MAX_RINGS];
  2447. if (!ring || !ring->sched.thread)
  2448. continue;
  2449. if (job && j != i) {
  2450. kthread_unpark(ring->sched.thread);
  2451. continue;
  2452. }
  2453. amd_sched_job_recovery(&ring->sched);
  2454. kthread_unpark(ring->sched.thread);
  2455. }
  2456. drm_helper_resume_force_mode(adev->ddev);
  2457. give_up_reset:
  2458. ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
  2459. if (r) {
  2460. /* bad news, how to tell it to userspace ? */
  2461. dev_info(adev->dev, "GPU reset failed\n");
  2462. } else {
  2463. dev_info(adev->dev, "GPU reset successed!\n");
  2464. }
  2465. adev->gfx.in_reset = false;
  2466. mutex_unlock(&adev->virt.lock_reset);
  2467. return r;
  2468. }
  2469. /**
  2470. * amdgpu_gpu_reset - reset the asic
  2471. *
  2472. * @adev: amdgpu device pointer
  2473. *
  2474. * Attempt the reset the GPU if it has hung (all asics).
  2475. * Returns 0 for success or an error on failure.
  2476. */
  2477. int amdgpu_gpu_reset(struct amdgpu_device *adev)
  2478. {
  2479. int i, r;
  2480. int resched;
  2481. bool need_full_reset, vram_lost = false;
  2482. if (!amdgpu_check_soft_reset(adev)) {
  2483. DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
  2484. return 0;
  2485. }
  2486. atomic_inc(&adev->gpu_reset_counter);
  2487. /* block TTM */
  2488. resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
  2489. /* block scheduler */
  2490. for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
  2491. struct amdgpu_ring *ring = adev->rings[i];
  2492. if (!ring || !ring->sched.thread)
  2493. continue;
  2494. kthread_park(ring->sched.thread);
  2495. amd_sched_hw_job_reset(&ring->sched);
  2496. }
  2497. /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
  2498. amdgpu_fence_driver_force_completion(adev);
  2499. need_full_reset = amdgpu_need_full_reset(adev);
  2500. if (!need_full_reset) {
  2501. amdgpu_pre_soft_reset(adev);
  2502. r = amdgpu_soft_reset(adev);
  2503. amdgpu_post_soft_reset(adev);
  2504. if (r || amdgpu_check_soft_reset(adev)) {
  2505. DRM_INFO("soft reset failed, will fallback to full reset!\n");
  2506. need_full_reset = true;
  2507. }
  2508. }
  2509. if (need_full_reset) {
  2510. r = amdgpu_suspend(adev);
  2511. retry:
  2512. /* Disable fb access */
  2513. if (adev->mode_info.num_crtc) {
  2514. struct amdgpu_mode_mc_save save;
  2515. amdgpu_display_stop_mc_access(adev, &save);
  2516. amdgpu_wait_for_idle(adev, AMD_IP_BLOCK_TYPE_GMC);
  2517. }
  2518. if (adev->is_atom_fw)
  2519. amdgpu_atomfirmware_scratch_regs_save(adev);
  2520. else
  2521. amdgpu_atombios_scratch_regs_save(adev);
  2522. r = amdgpu_asic_reset(adev);
  2523. if (adev->is_atom_fw)
  2524. amdgpu_atomfirmware_scratch_regs_restore(adev);
  2525. else
  2526. amdgpu_atombios_scratch_regs_restore(adev);
  2527. /* post card */
  2528. amdgpu_atom_asic_init(adev->mode_info.atom_context);
  2529. if (!r) {
  2530. dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
  2531. r = amdgpu_resume_phase1(adev);
  2532. if (r)
  2533. goto out;
  2534. vram_lost = amdgpu_check_vram_lost(adev);
  2535. if (vram_lost) {
  2536. DRM_ERROR("VRAM is lost!\n");
  2537. atomic_inc(&adev->vram_lost_counter);
  2538. }
  2539. r = amdgpu_ttm_recover_gart(adev);
  2540. if (r)
  2541. goto out;
  2542. r = amdgpu_resume_phase2(adev);
  2543. if (r)
  2544. goto out;
  2545. if (vram_lost)
  2546. amdgpu_fill_reset_magic(adev);
  2547. }
  2548. }
  2549. out:
  2550. if (!r) {
  2551. amdgpu_irq_gpu_reset_resume_helper(adev);
  2552. r = amdgpu_ib_ring_tests(adev);
  2553. if (r) {
  2554. dev_err(adev->dev, "ib ring test failed (%d).\n", r);
  2555. r = amdgpu_suspend(adev);
  2556. need_full_reset = true;
  2557. goto retry;
  2558. }
  2559. /**
  2560. * recovery vm page tables, since we cannot depend on VRAM is
  2561. * consistent after gpu full reset.
  2562. */
  2563. if (need_full_reset && amdgpu_need_backup(adev)) {
  2564. struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
  2565. struct amdgpu_bo *bo, *tmp;
  2566. struct dma_fence *fence = NULL, *next = NULL;
  2567. DRM_INFO("recover vram bo from shadow\n");
  2568. mutex_lock(&adev->shadow_list_lock);
  2569. list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
  2570. next = NULL;
  2571. amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
  2572. if (fence) {
  2573. r = dma_fence_wait(fence, false);
  2574. if (r) {
  2575. WARN(r, "recovery from shadow isn't completed\n");
  2576. break;
  2577. }
  2578. }
  2579. dma_fence_put(fence);
  2580. fence = next;
  2581. }
  2582. mutex_unlock(&adev->shadow_list_lock);
  2583. if (fence) {
  2584. r = dma_fence_wait(fence, false);
  2585. if (r)
  2586. WARN(r, "recovery from shadow isn't completed\n");
  2587. }
  2588. dma_fence_put(fence);
  2589. }
  2590. for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
  2591. struct amdgpu_ring *ring = adev->rings[i];
  2592. if (!ring || !ring->sched.thread)
  2593. continue;
  2594. amd_sched_job_recovery(&ring->sched);
  2595. kthread_unpark(ring->sched.thread);
  2596. }
  2597. } else {
  2598. dev_err(adev->dev, "asic resume failed (%d).\n", r);
  2599. for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
  2600. if (adev->rings[i] && adev->rings[i]->sched.thread) {
  2601. kthread_unpark(adev->rings[i]->sched.thread);
  2602. }
  2603. }
  2604. }
  2605. drm_helper_resume_force_mode(adev->ddev);
  2606. ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
  2607. if (r)
  2608. /* bad news, how to tell it to userspace ? */
  2609. dev_info(adev->dev, "GPU reset failed\n");
  2610. else
  2611. dev_info(adev->dev, "GPU reset successed!\n");
  2612. return r;
  2613. }
  2614. void amdgpu_get_pcie_info(struct amdgpu_device *adev)
  2615. {
  2616. u32 mask;
  2617. int ret;
  2618. if (amdgpu_pcie_gen_cap)
  2619. adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
  2620. if (amdgpu_pcie_lane_cap)
  2621. adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
  2622. /* covers APUs as well */
  2623. if (pci_is_root_bus(adev->pdev->bus)) {
  2624. if (adev->pm.pcie_gen_mask == 0)
  2625. adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
  2626. if (adev->pm.pcie_mlw_mask == 0)
  2627. adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
  2628. return;
  2629. }
  2630. if (adev->pm.pcie_gen_mask == 0) {
  2631. ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
  2632. if (!ret) {
  2633. adev->pm.pcie_gen_mask = (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
  2634. CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
  2635. CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
  2636. if (mask & DRM_PCIE_SPEED_25)
  2637. adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
  2638. if (mask & DRM_PCIE_SPEED_50)
  2639. adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2;
  2640. if (mask & DRM_PCIE_SPEED_80)
  2641. adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3;
  2642. } else {
  2643. adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
  2644. }
  2645. }
  2646. if (adev->pm.pcie_mlw_mask == 0) {
  2647. ret = drm_pcie_get_max_link_width(adev->ddev, &mask);
  2648. if (!ret) {
  2649. switch (mask) {
  2650. case 32:
  2651. adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
  2652. CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
  2653. CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
  2654. CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
  2655. CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
  2656. CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
  2657. CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
  2658. break;
  2659. case 16:
  2660. adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
  2661. CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
  2662. CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
  2663. CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
  2664. CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
  2665. CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
  2666. break;
  2667. case 12:
  2668. adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
  2669. CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
  2670. CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
  2671. CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
  2672. CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
  2673. break;
  2674. case 8:
  2675. adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
  2676. CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
  2677. CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
  2678. CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
  2679. break;
  2680. case 4:
  2681. adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
  2682. CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
  2683. CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
  2684. break;
  2685. case 2:
  2686. adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
  2687. CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
  2688. break;
  2689. case 1:
  2690. adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
  2691. break;
  2692. default:
  2693. break;
  2694. }
  2695. } else {
  2696. adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
  2697. }
  2698. }
  2699. }
  2700. /*
  2701. * Debugfs
  2702. */
  2703. int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
  2704. const struct drm_info_list *files,
  2705. unsigned nfiles)
  2706. {
  2707. unsigned i;
  2708. for (i = 0; i < adev->debugfs_count; i++) {
  2709. if (adev->debugfs[i].files == files) {
  2710. /* Already registered */
  2711. return 0;
  2712. }
  2713. }
  2714. i = adev->debugfs_count + 1;
  2715. if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
  2716. DRM_ERROR("Reached maximum number of debugfs components.\n");
  2717. DRM_ERROR("Report so we increase "
  2718. "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
  2719. return -EINVAL;
  2720. }
  2721. adev->debugfs[adev->debugfs_count].files = files;
  2722. adev->debugfs[adev->debugfs_count].num_files = nfiles;
  2723. adev->debugfs_count = i;
  2724. #if defined(CONFIG_DEBUG_FS)
  2725. drm_debugfs_create_files(files, nfiles,
  2726. adev->ddev->primary->debugfs_root,
  2727. adev->ddev->primary);
  2728. #endif
  2729. return 0;
  2730. }
  2731. #if defined(CONFIG_DEBUG_FS)
  2732. static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
  2733. size_t size, loff_t *pos)
  2734. {
  2735. struct amdgpu_device *adev = file_inode(f)->i_private;
  2736. ssize_t result = 0;
  2737. int r;
  2738. bool pm_pg_lock, use_bank;
  2739. unsigned instance_bank, sh_bank, se_bank;
  2740. if (size & 0x3 || *pos & 0x3)
  2741. return -EINVAL;
  2742. /* are we reading registers for which a PG lock is necessary? */
  2743. pm_pg_lock = (*pos >> 23) & 1;
  2744. if (*pos & (1ULL << 62)) {
  2745. se_bank = (*pos >> 24) & 0x3FF;
  2746. sh_bank = (*pos >> 34) & 0x3FF;
  2747. instance_bank = (*pos >> 44) & 0x3FF;
  2748. if (se_bank == 0x3FF)
  2749. se_bank = 0xFFFFFFFF;
  2750. if (sh_bank == 0x3FF)
  2751. sh_bank = 0xFFFFFFFF;
  2752. if (instance_bank == 0x3FF)
  2753. instance_bank = 0xFFFFFFFF;
  2754. use_bank = 1;
  2755. } else {
  2756. use_bank = 0;
  2757. }
  2758. *pos &= (1UL << 22) - 1;
  2759. if (use_bank) {
  2760. if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
  2761. (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
  2762. return -EINVAL;
  2763. mutex_lock(&adev->grbm_idx_mutex);
  2764. amdgpu_gfx_select_se_sh(adev, se_bank,
  2765. sh_bank, instance_bank);
  2766. }
  2767. if (pm_pg_lock)
  2768. mutex_lock(&adev->pm.mutex);
  2769. while (size) {
  2770. uint32_t value;
  2771. if (*pos > adev->rmmio_size)
  2772. goto end;
  2773. value = RREG32(*pos >> 2);
  2774. r = put_user(value, (uint32_t *)buf);
  2775. if (r) {
  2776. result = r;
  2777. goto end;
  2778. }
  2779. result += 4;
  2780. buf += 4;
  2781. *pos += 4;
  2782. size -= 4;
  2783. }
  2784. end:
  2785. if (use_bank) {
  2786. amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
  2787. mutex_unlock(&adev->grbm_idx_mutex);
  2788. }
  2789. if (pm_pg_lock)
  2790. mutex_unlock(&adev->pm.mutex);
  2791. return result;
  2792. }
  2793. static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
  2794. size_t size, loff_t *pos)
  2795. {
  2796. struct amdgpu_device *adev = file_inode(f)->i_private;
  2797. ssize_t result = 0;
  2798. int r;
  2799. bool pm_pg_lock, use_bank;
  2800. unsigned instance_bank, sh_bank, se_bank;
  2801. if (size & 0x3 || *pos & 0x3)
  2802. return -EINVAL;
  2803. /* are we reading registers for which a PG lock is necessary? */
  2804. pm_pg_lock = (*pos >> 23) & 1;
  2805. if (*pos & (1ULL << 62)) {
  2806. se_bank = (*pos >> 24) & 0x3FF;
  2807. sh_bank = (*pos >> 34) & 0x3FF;
  2808. instance_bank = (*pos >> 44) & 0x3FF;
  2809. if (se_bank == 0x3FF)
  2810. se_bank = 0xFFFFFFFF;
  2811. if (sh_bank == 0x3FF)
  2812. sh_bank = 0xFFFFFFFF;
  2813. if (instance_bank == 0x3FF)
  2814. instance_bank = 0xFFFFFFFF;
  2815. use_bank = 1;
  2816. } else {
  2817. use_bank = 0;
  2818. }
  2819. *pos &= (1UL << 22) - 1;
  2820. if (use_bank) {
  2821. if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
  2822. (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
  2823. return -EINVAL;
  2824. mutex_lock(&adev->grbm_idx_mutex);
  2825. amdgpu_gfx_select_se_sh(adev, se_bank,
  2826. sh_bank, instance_bank);
  2827. }
  2828. if (pm_pg_lock)
  2829. mutex_lock(&adev->pm.mutex);
  2830. while (size) {
  2831. uint32_t value;
  2832. if (*pos > adev->rmmio_size)
  2833. return result;
  2834. r = get_user(value, (uint32_t *)buf);
  2835. if (r)
  2836. return r;
  2837. WREG32(*pos >> 2, value);
  2838. result += 4;
  2839. buf += 4;
  2840. *pos += 4;
  2841. size -= 4;
  2842. }
  2843. if (use_bank) {
  2844. amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
  2845. mutex_unlock(&adev->grbm_idx_mutex);
  2846. }
  2847. if (pm_pg_lock)
  2848. mutex_unlock(&adev->pm.mutex);
  2849. return result;
  2850. }
  2851. static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
  2852. size_t size, loff_t *pos)
  2853. {
  2854. struct amdgpu_device *adev = file_inode(f)->i_private;
  2855. ssize_t result = 0;
  2856. int r;
  2857. if (size & 0x3 || *pos & 0x3)
  2858. return -EINVAL;
  2859. while (size) {
  2860. uint32_t value;
  2861. value = RREG32_PCIE(*pos >> 2);
  2862. r = put_user(value, (uint32_t *)buf);
  2863. if (r)
  2864. return r;
  2865. result += 4;
  2866. buf += 4;
  2867. *pos += 4;
  2868. size -= 4;
  2869. }
  2870. return result;
  2871. }
  2872. static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
  2873. size_t size, loff_t *pos)
  2874. {
  2875. struct amdgpu_device *adev = file_inode(f)->i_private;
  2876. ssize_t result = 0;
  2877. int r;
  2878. if (size & 0x3 || *pos & 0x3)
  2879. return -EINVAL;
  2880. while (size) {
  2881. uint32_t value;
  2882. r = get_user(value, (uint32_t *)buf);
  2883. if (r)
  2884. return r;
  2885. WREG32_PCIE(*pos >> 2, value);
  2886. result += 4;
  2887. buf += 4;
  2888. *pos += 4;
  2889. size -= 4;
  2890. }
  2891. return result;
  2892. }
  2893. static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
  2894. size_t size, loff_t *pos)
  2895. {
  2896. struct amdgpu_device *adev = file_inode(f)->i_private;
  2897. ssize_t result = 0;
  2898. int r;
  2899. if (size & 0x3 || *pos & 0x3)
  2900. return -EINVAL;
  2901. while (size) {
  2902. uint32_t value;
  2903. value = RREG32_DIDT(*pos >> 2);
  2904. r = put_user(value, (uint32_t *)buf);
  2905. if (r)
  2906. return r;
  2907. result += 4;
  2908. buf += 4;
  2909. *pos += 4;
  2910. size -= 4;
  2911. }
  2912. return result;
  2913. }
  2914. static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
  2915. size_t size, loff_t *pos)
  2916. {
  2917. struct amdgpu_device *adev = file_inode(f)->i_private;
  2918. ssize_t result = 0;
  2919. int r;
  2920. if (size & 0x3 || *pos & 0x3)
  2921. return -EINVAL;
  2922. while (size) {
  2923. uint32_t value;
  2924. r = get_user(value, (uint32_t *)buf);
  2925. if (r)
  2926. return r;
  2927. WREG32_DIDT(*pos >> 2, value);
  2928. result += 4;
  2929. buf += 4;
  2930. *pos += 4;
  2931. size -= 4;
  2932. }
  2933. return result;
  2934. }
  2935. static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
  2936. size_t size, loff_t *pos)
  2937. {
  2938. struct amdgpu_device *adev = file_inode(f)->i_private;
  2939. ssize_t result = 0;
  2940. int r;
  2941. if (size & 0x3 || *pos & 0x3)
  2942. return -EINVAL;
  2943. while (size) {
  2944. uint32_t value;
  2945. value = RREG32_SMC(*pos);
  2946. r = put_user(value, (uint32_t *)buf);
  2947. if (r)
  2948. return r;
  2949. result += 4;
  2950. buf += 4;
  2951. *pos += 4;
  2952. size -= 4;
  2953. }
  2954. return result;
  2955. }
  2956. static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
  2957. size_t size, loff_t *pos)
  2958. {
  2959. struct amdgpu_device *adev = file_inode(f)->i_private;
  2960. ssize_t result = 0;
  2961. int r;
  2962. if (size & 0x3 || *pos & 0x3)
  2963. return -EINVAL;
  2964. while (size) {
  2965. uint32_t value;
  2966. r = get_user(value, (uint32_t *)buf);
  2967. if (r)
  2968. return r;
  2969. WREG32_SMC(*pos, value);
  2970. result += 4;
  2971. buf += 4;
  2972. *pos += 4;
  2973. size -= 4;
  2974. }
  2975. return result;
  2976. }
  2977. static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
  2978. size_t size, loff_t *pos)
  2979. {
  2980. struct amdgpu_device *adev = file_inode(f)->i_private;
  2981. ssize_t result = 0;
  2982. int r;
  2983. uint32_t *config, no_regs = 0;
  2984. if (size & 0x3 || *pos & 0x3)
  2985. return -EINVAL;
  2986. config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
  2987. if (!config)
  2988. return -ENOMEM;
  2989. /* version, increment each time something is added */
  2990. config[no_regs++] = 3;
  2991. config[no_regs++] = adev->gfx.config.max_shader_engines;
  2992. config[no_regs++] = adev->gfx.config.max_tile_pipes;
  2993. config[no_regs++] = adev->gfx.config.max_cu_per_sh;
  2994. config[no_regs++] = adev->gfx.config.max_sh_per_se;
  2995. config[no_regs++] = adev->gfx.config.max_backends_per_se;
  2996. config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
  2997. config[no_regs++] = adev->gfx.config.max_gprs;
  2998. config[no_regs++] = adev->gfx.config.max_gs_threads;
  2999. config[no_regs++] = adev->gfx.config.max_hw_contexts;
  3000. config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
  3001. config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
  3002. config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
  3003. config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
  3004. config[no_regs++] = adev->gfx.config.num_tile_pipes;
  3005. config[no_regs++] = adev->gfx.config.backend_enable_mask;
  3006. config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
  3007. config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
  3008. config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
  3009. config[no_regs++] = adev->gfx.config.num_gpus;
  3010. config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
  3011. config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
  3012. config[no_regs++] = adev->gfx.config.gb_addr_config;
  3013. config[no_regs++] = adev->gfx.config.num_rbs;
  3014. /* rev==1 */
  3015. config[no_regs++] = adev->rev_id;
  3016. config[no_regs++] = adev->pg_flags;
  3017. config[no_regs++] = adev->cg_flags;
  3018. /* rev==2 */
  3019. config[no_regs++] = adev->family;
  3020. config[no_regs++] = adev->external_rev_id;
  3021. /* rev==3 */
  3022. config[no_regs++] = adev->pdev->device;
  3023. config[no_regs++] = adev->pdev->revision;
  3024. config[no_regs++] = adev->pdev->subsystem_device;
  3025. config[no_regs++] = adev->pdev->subsystem_vendor;
  3026. while (size && (*pos < no_regs * 4)) {
  3027. uint32_t value;
  3028. value = config[*pos >> 2];
  3029. r = put_user(value, (uint32_t *)buf);
  3030. if (r) {
  3031. kfree(config);
  3032. return r;
  3033. }
  3034. result += 4;
  3035. buf += 4;
  3036. *pos += 4;
  3037. size -= 4;
  3038. }
  3039. kfree(config);
  3040. return result;
  3041. }
  3042. static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
  3043. size_t size, loff_t *pos)
  3044. {
  3045. struct amdgpu_device *adev = file_inode(f)->i_private;
  3046. int idx, x, outsize, r, valuesize;
  3047. uint32_t values[16];
  3048. if (size & 3 || *pos & 0x3)
  3049. return -EINVAL;
  3050. if (amdgpu_dpm == 0)
  3051. return -EINVAL;
  3052. /* convert offset to sensor number */
  3053. idx = *pos >> 2;
  3054. valuesize = sizeof(values);
  3055. if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
  3056. r = adev->powerplay.pp_funcs->read_sensor(adev->powerplay.pp_handle, idx, &values[0], &valuesize);
  3057. else if (adev->pm.funcs && adev->pm.funcs->read_sensor)
  3058. r = adev->pm.funcs->read_sensor(adev, idx, &values[0],
  3059. &valuesize);
  3060. else
  3061. return -EINVAL;
  3062. if (size > valuesize)
  3063. return -EINVAL;
  3064. outsize = 0;
  3065. x = 0;
  3066. if (!r) {
  3067. while (size) {
  3068. r = put_user(values[x++], (int32_t *)buf);
  3069. buf += 4;
  3070. size -= 4;
  3071. outsize += 4;
  3072. }
  3073. }
  3074. return !r ? outsize : r;
  3075. }
  3076. static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
  3077. size_t size, loff_t *pos)
  3078. {
  3079. struct amdgpu_device *adev = f->f_inode->i_private;
  3080. int r, x;
  3081. ssize_t result=0;
  3082. uint32_t offset, se, sh, cu, wave, simd, data[32];
  3083. if (size & 3 || *pos & 3)
  3084. return -EINVAL;
  3085. /* decode offset */
  3086. offset = (*pos & 0x7F);
  3087. se = ((*pos >> 7) & 0xFF);
  3088. sh = ((*pos >> 15) & 0xFF);
  3089. cu = ((*pos >> 23) & 0xFF);
  3090. wave = ((*pos >> 31) & 0xFF);
  3091. simd = ((*pos >> 37) & 0xFF);
  3092. /* switch to the specific se/sh/cu */
  3093. mutex_lock(&adev->grbm_idx_mutex);
  3094. amdgpu_gfx_select_se_sh(adev, se, sh, cu);
  3095. x = 0;
  3096. if (adev->gfx.funcs->read_wave_data)
  3097. adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
  3098. amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
  3099. mutex_unlock(&adev->grbm_idx_mutex);
  3100. if (!x)
  3101. return -EINVAL;
  3102. while (size && (offset < x * 4)) {
  3103. uint32_t value;
  3104. value = data[offset >> 2];
  3105. r = put_user(value, (uint32_t *)buf);
  3106. if (r)
  3107. return r;
  3108. result += 4;
  3109. buf += 4;
  3110. offset += 4;
  3111. size -= 4;
  3112. }
  3113. return result;
  3114. }
  3115. static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
  3116. size_t size, loff_t *pos)
  3117. {
  3118. struct amdgpu_device *adev = f->f_inode->i_private;
  3119. int r;
  3120. ssize_t result = 0;
  3121. uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
  3122. if (size & 3 || *pos & 3)
  3123. return -EINVAL;
  3124. /* decode offset */
  3125. offset = (*pos & 0xFFF); /* in dwords */
  3126. se = ((*pos >> 12) & 0xFF);
  3127. sh = ((*pos >> 20) & 0xFF);
  3128. cu = ((*pos >> 28) & 0xFF);
  3129. wave = ((*pos >> 36) & 0xFF);
  3130. simd = ((*pos >> 44) & 0xFF);
  3131. thread = ((*pos >> 52) & 0xFF);
  3132. bank = ((*pos >> 60) & 1);
  3133. data = kmalloc_array(1024, sizeof(*data), GFP_KERNEL);
  3134. if (!data)
  3135. return -ENOMEM;
  3136. /* switch to the specific se/sh/cu */
  3137. mutex_lock(&adev->grbm_idx_mutex);
  3138. amdgpu_gfx_select_se_sh(adev, se, sh, cu);
  3139. if (bank == 0) {
  3140. if (adev->gfx.funcs->read_wave_vgprs)
  3141. adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
  3142. } else {
  3143. if (adev->gfx.funcs->read_wave_sgprs)
  3144. adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
  3145. }
  3146. amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
  3147. mutex_unlock(&adev->grbm_idx_mutex);
  3148. while (size) {
  3149. uint32_t value;
  3150. value = data[offset++];
  3151. r = put_user(value, (uint32_t *)buf);
  3152. if (r) {
  3153. result = r;
  3154. goto err;
  3155. }
  3156. result += 4;
  3157. buf += 4;
  3158. size -= 4;
  3159. }
  3160. err:
  3161. kfree(data);
  3162. return result;
  3163. }
  3164. static const struct file_operations amdgpu_debugfs_regs_fops = {
  3165. .owner = THIS_MODULE,
  3166. .read = amdgpu_debugfs_regs_read,
  3167. .write = amdgpu_debugfs_regs_write,
  3168. .llseek = default_llseek
  3169. };
  3170. static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
  3171. .owner = THIS_MODULE,
  3172. .read = amdgpu_debugfs_regs_didt_read,
  3173. .write = amdgpu_debugfs_regs_didt_write,
  3174. .llseek = default_llseek
  3175. };
  3176. static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
  3177. .owner = THIS_MODULE,
  3178. .read = amdgpu_debugfs_regs_pcie_read,
  3179. .write = amdgpu_debugfs_regs_pcie_write,
  3180. .llseek = default_llseek
  3181. };
  3182. static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
  3183. .owner = THIS_MODULE,
  3184. .read = amdgpu_debugfs_regs_smc_read,
  3185. .write = amdgpu_debugfs_regs_smc_write,
  3186. .llseek = default_llseek
  3187. };
  3188. static const struct file_operations amdgpu_debugfs_gca_config_fops = {
  3189. .owner = THIS_MODULE,
  3190. .read = amdgpu_debugfs_gca_config_read,
  3191. .llseek = default_llseek
  3192. };
  3193. static const struct file_operations amdgpu_debugfs_sensors_fops = {
  3194. .owner = THIS_MODULE,
  3195. .read = amdgpu_debugfs_sensor_read,
  3196. .llseek = default_llseek
  3197. };
  3198. static const struct file_operations amdgpu_debugfs_wave_fops = {
  3199. .owner = THIS_MODULE,
  3200. .read = amdgpu_debugfs_wave_read,
  3201. .llseek = default_llseek
  3202. };
  3203. static const struct file_operations amdgpu_debugfs_gpr_fops = {
  3204. .owner = THIS_MODULE,
  3205. .read = amdgpu_debugfs_gpr_read,
  3206. .llseek = default_llseek
  3207. };
  3208. static const struct file_operations *debugfs_regs[] = {
  3209. &amdgpu_debugfs_regs_fops,
  3210. &amdgpu_debugfs_regs_didt_fops,
  3211. &amdgpu_debugfs_regs_pcie_fops,
  3212. &amdgpu_debugfs_regs_smc_fops,
  3213. &amdgpu_debugfs_gca_config_fops,
  3214. &amdgpu_debugfs_sensors_fops,
  3215. &amdgpu_debugfs_wave_fops,
  3216. &amdgpu_debugfs_gpr_fops,
  3217. };
  3218. static const char *debugfs_regs_names[] = {
  3219. "amdgpu_regs",
  3220. "amdgpu_regs_didt",
  3221. "amdgpu_regs_pcie",
  3222. "amdgpu_regs_smc",
  3223. "amdgpu_gca_config",
  3224. "amdgpu_sensors",
  3225. "amdgpu_wave",
  3226. "amdgpu_gpr",
  3227. };
  3228. static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
  3229. {
  3230. struct drm_minor *minor = adev->ddev->primary;
  3231. struct dentry *ent, *root = minor->debugfs_root;
  3232. unsigned i, j;
  3233. for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
  3234. ent = debugfs_create_file(debugfs_regs_names[i],
  3235. S_IFREG | S_IRUGO, root,
  3236. adev, debugfs_regs[i]);
  3237. if (IS_ERR(ent)) {
  3238. for (j = 0; j < i; j++) {
  3239. debugfs_remove(adev->debugfs_regs[i]);
  3240. adev->debugfs_regs[i] = NULL;
  3241. }
  3242. return PTR_ERR(ent);
  3243. }
  3244. if (!i)
  3245. i_size_write(ent->d_inode, adev->rmmio_size);
  3246. adev->debugfs_regs[i] = ent;
  3247. }
  3248. return 0;
  3249. }
  3250. static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev)
  3251. {
  3252. unsigned i;
  3253. for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
  3254. if (adev->debugfs_regs[i]) {
  3255. debugfs_remove(adev->debugfs_regs[i]);
  3256. adev->debugfs_regs[i] = NULL;
  3257. }
  3258. }
  3259. }
  3260. static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)
  3261. {
  3262. struct drm_info_node *node = (struct drm_info_node *) m->private;
  3263. struct drm_device *dev = node->minor->dev;
  3264. struct amdgpu_device *adev = dev->dev_private;
  3265. int r = 0, i;
  3266. /* hold on the scheduler */
  3267. for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
  3268. struct amdgpu_ring *ring = adev->rings[i];
  3269. if (!ring || !ring->sched.thread)
  3270. continue;
  3271. kthread_park(ring->sched.thread);
  3272. }
  3273. seq_printf(m, "run ib test:\n");
  3274. r = amdgpu_ib_ring_tests(adev);
  3275. if (r)
  3276. seq_printf(m, "ib ring tests failed (%d).\n", r);
  3277. else
  3278. seq_printf(m, "ib ring tests passed.\n");
  3279. /* go on the scheduler */
  3280. for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
  3281. struct amdgpu_ring *ring = adev->rings[i];
  3282. if (!ring || !ring->sched.thread)
  3283. continue;
  3284. kthread_unpark(ring->sched.thread);
  3285. }
  3286. return 0;
  3287. }
  3288. static const struct drm_info_list amdgpu_debugfs_test_ib_ring_list[] = {
  3289. {"amdgpu_test_ib", &amdgpu_debugfs_test_ib}
  3290. };
  3291. static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev)
  3292. {
  3293. return amdgpu_debugfs_add_files(adev,
  3294. amdgpu_debugfs_test_ib_ring_list, 1);
  3295. }
  3296. int amdgpu_debugfs_init(struct drm_minor *minor)
  3297. {
  3298. return 0;
  3299. }
  3300. #else
  3301. static int amdgpu_debugfs_test_ib_init(struct amdgpu_device *adev)
  3302. {
  3303. return 0;
  3304. }
  3305. static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
  3306. {
  3307. return 0;
  3308. }
  3309. static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { }
  3310. #endif