gmc_v8_0.c 47 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670
  1. /*
  2. * Copyright 2014 Advanced Micro Devices, Inc.
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice shall be included in
  12. * all copies or substantial portions of the Software.
  13. *
  14. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  17. * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20. * OTHER DEALINGS IN THE SOFTWARE.
  21. *
  22. */
  23. #include <linux/firmware.h>
  24. #include <drm/drmP.h>
  25. #include "amdgpu.h"
  26. #include "gmc_v8_0.h"
  27. #include "amdgpu_ucode.h"
  28. #include "gmc/gmc_8_1_d.h"
  29. #include "gmc/gmc_8_1_sh_mask.h"
  30. #include "bif/bif_5_0_d.h"
  31. #include "bif/bif_5_0_sh_mask.h"
  32. #include "oss/oss_3_0_d.h"
  33. #include "oss/oss_3_0_sh_mask.h"
  34. #include "dce/dce_10_0_d.h"
  35. #include "dce/dce_10_0_sh_mask.h"
  36. #include "vid.h"
  37. #include "vi.h"
  38. #include "amdgpu_atombios.h"
  39. static void gmc_v8_0_set_gart_funcs(struct amdgpu_device *adev);
  40. static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev);
  41. static int gmc_v8_0_wait_for_idle(void *handle);
  42. MODULE_FIRMWARE("amdgpu/tonga_mc.bin");
  43. MODULE_FIRMWARE("amdgpu/polaris11_mc.bin");
  44. MODULE_FIRMWARE("amdgpu/polaris10_mc.bin");
  45. MODULE_FIRMWARE("amdgpu/polaris12_mc.bin");
  46. static const u32 golden_settings_tonga_a11[] =
  47. {
  48. mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
  49. mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
  50. mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
  51. mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  52. mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  53. mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  54. mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  55. };
  56. static const u32 tonga_mgcg_cgcg_init[] =
  57. {
  58. mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
  59. };
  60. static const u32 golden_settings_fiji_a10[] =
  61. {
  62. mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  63. mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  64. mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  65. mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  66. };
  67. static const u32 fiji_mgcg_cgcg_init[] =
  68. {
  69. mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
  70. };
  71. static const u32 golden_settings_polaris11_a11[] =
  72. {
  73. mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  74. mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  75. mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  76. mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff
  77. };
  78. static const u32 golden_settings_polaris10_a11[] =
  79. {
  80. mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
  81. mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  82. mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  83. mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  84. mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff
  85. };
  86. static const u32 cz_mgcg_cgcg_init[] =
  87. {
  88. mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
  89. };
  90. static const u32 stoney_mgcg_cgcg_init[] =
  91. {
  92. mmATC_MISC_CG, 0xffffffff, 0x000c0200,
  93. mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
  94. };
  95. static const u32 golden_settings_stoney_common[] =
  96. {
  97. mmMC_HUB_RDREQ_UVD, MC_HUB_RDREQ_UVD__PRESCALE_MASK, 0x00000004,
  98. mmMC_RD_GRP_OTH, MC_RD_GRP_OTH__UVD_MASK, 0x00600000
  99. };
  100. static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev)
  101. {
  102. switch (adev->asic_type) {
  103. case CHIP_FIJI:
  104. amdgpu_program_register_sequence(adev,
  105. fiji_mgcg_cgcg_init,
  106. (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init));
  107. amdgpu_program_register_sequence(adev,
  108. golden_settings_fiji_a10,
  109. (const u32)ARRAY_SIZE(golden_settings_fiji_a10));
  110. break;
  111. case CHIP_TONGA:
  112. amdgpu_program_register_sequence(adev,
  113. tonga_mgcg_cgcg_init,
  114. (const u32)ARRAY_SIZE(tonga_mgcg_cgcg_init));
  115. amdgpu_program_register_sequence(adev,
  116. golden_settings_tonga_a11,
  117. (const u32)ARRAY_SIZE(golden_settings_tonga_a11));
  118. break;
  119. case CHIP_POLARIS11:
  120. case CHIP_POLARIS12:
  121. amdgpu_program_register_sequence(adev,
  122. golden_settings_polaris11_a11,
  123. (const u32)ARRAY_SIZE(golden_settings_polaris11_a11));
  124. break;
  125. case CHIP_POLARIS10:
  126. amdgpu_program_register_sequence(adev,
  127. golden_settings_polaris10_a11,
  128. (const u32)ARRAY_SIZE(golden_settings_polaris10_a11));
  129. break;
  130. case CHIP_CARRIZO:
  131. amdgpu_program_register_sequence(adev,
  132. cz_mgcg_cgcg_init,
  133. (const u32)ARRAY_SIZE(cz_mgcg_cgcg_init));
  134. break;
  135. case CHIP_STONEY:
  136. amdgpu_program_register_sequence(adev,
  137. stoney_mgcg_cgcg_init,
  138. (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init));
  139. amdgpu_program_register_sequence(adev,
  140. golden_settings_stoney_common,
  141. (const u32)ARRAY_SIZE(golden_settings_stoney_common));
  142. break;
  143. default:
  144. break;
  145. }
  146. }
  147. static void gmc_v8_0_mc_stop(struct amdgpu_device *adev)
  148. {
  149. u32 blackout;
  150. gmc_v8_0_wait_for_idle(adev);
  151. blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
  152. if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
  153. /* Block CPU access */
  154. WREG32(mmBIF_FB_EN, 0);
  155. /* blackout the MC */
  156. blackout = REG_SET_FIELD(blackout,
  157. MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 1);
  158. WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout);
  159. }
  160. /* wait for the MC to settle */
  161. udelay(100);
  162. }
  163. static void gmc_v8_0_mc_resume(struct amdgpu_device *adev)
  164. {
  165. u32 tmp;
  166. /* unblackout the MC */
  167. tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
  168. tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
  169. WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp);
  170. /* allow CPU access */
  171. tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
  172. tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
  173. WREG32(mmBIF_FB_EN, tmp);
  174. }
  175. /**
  176. * gmc_v8_0_init_microcode - load ucode images from disk
  177. *
  178. * @adev: amdgpu_device pointer
  179. *
  180. * Use the firmware interface to load the ucode images into
  181. * the driver (not loaded into hw).
  182. * Returns 0 on success, error on failure.
  183. */
  184. static int gmc_v8_0_init_microcode(struct amdgpu_device *adev)
  185. {
  186. const char *chip_name;
  187. char fw_name[30];
  188. int err;
  189. DRM_DEBUG("\n");
  190. switch (adev->asic_type) {
  191. case CHIP_TONGA:
  192. chip_name = "tonga";
  193. break;
  194. case CHIP_POLARIS11:
  195. chip_name = "polaris11";
  196. break;
  197. case CHIP_POLARIS10:
  198. chip_name = "polaris10";
  199. break;
  200. case CHIP_POLARIS12:
  201. chip_name = "polaris12";
  202. break;
  203. case CHIP_FIJI:
  204. case CHIP_CARRIZO:
  205. case CHIP_STONEY:
  206. return 0;
  207. default: BUG();
  208. }
  209. snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mc.bin", chip_name);
  210. err = request_firmware(&adev->mc.fw, fw_name, adev->dev);
  211. if (err)
  212. goto out;
  213. err = amdgpu_ucode_validate(adev->mc.fw);
  214. out:
  215. if (err) {
  216. pr_err("mc: Failed to load firmware \"%s\"\n", fw_name);
  217. release_firmware(adev->mc.fw);
  218. adev->mc.fw = NULL;
  219. }
  220. return err;
  221. }
  222. /**
  223. * gmc_v8_0_tonga_mc_load_microcode - load tonga MC ucode into the hw
  224. *
  225. * @adev: amdgpu_device pointer
  226. *
  227. * Load the GDDR MC ucode into the hw (CIK).
  228. * Returns 0 on success, error on failure.
  229. */
  230. static int gmc_v8_0_tonga_mc_load_microcode(struct amdgpu_device *adev)
  231. {
  232. const struct mc_firmware_header_v1_0 *hdr;
  233. const __le32 *fw_data = NULL;
  234. const __le32 *io_mc_regs = NULL;
  235. u32 running;
  236. int i, ucode_size, regs_size;
  237. /* Skip MC ucode loading on SR-IOV capable boards.
  238. * vbios does this for us in asic_init in that case.
  239. * Skip MC ucode loading on VF, because hypervisor will do that
  240. * for this adaptor.
  241. */
  242. if (amdgpu_sriov_bios(adev))
  243. return 0;
  244. if (!adev->mc.fw)
  245. return -EINVAL;
  246. hdr = (const struct mc_firmware_header_v1_0 *)adev->mc.fw->data;
  247. amdgpu_ucode_print_mc_hdr(&hdr->header);
  248. adev->mc.fw_version = le32_to_cpu(hdr->header.ucode_version);
  249. regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
  250. io_mc_regs = (const __le32 *)
  251. (adev->mc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
  252. ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
  253. fw_data = (const __le32 *)
  254. (adev->mc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
  255. running = REG_GET_FIELD(RREG32(mmMC_SEQ_SUP_CNTL), MC_SEQ_SUP_CNTL, RUN);
  256. if (running == 0) {
  257. /* reset the engine and set to writable */
  258. WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
  259. WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
  260. /* load mc io regs */
  261. for (i = 0; i < regs_size; i++) {
  262. WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++));
  263. WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++));
  264. }
  265. /* load the MC ucode */
  266. for (i = 0; i < ucode_size; i++)
  267. WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++));
  268. /* put the engine back into the active state */
  269. WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
  270. WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);
  271. WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);
  272. /* wait for training to complete */
  273. for (i = 0; i < adev->usec_timeout; i++) {
  274. if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
  275. MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D0))
  276. break;
  277. udelay(1);
  278. }
  279. for (i = 0; i < adev->usec_timeout; i++) {
  280. if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
  281. MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D1))
  282. break;
  283. udelay(1);
  284. }
  285. }
  286. return 0;
  287. }
  288. static int gmc_v8_0_polaris_mc_load_microcode(struct amdgpu_device *adev)
  289. {
  290. const struct mc_firmware_header_v1_0 *hdr;
  291. const __le32 *fw_data = NULL;
  292. const __le32 *io_mc_regs = NULL;
  293. u32 data, vbios_version;
  294. int i, ucode_size, regs_size;
  295. /* Skip MC ucode loading on SR-IOV capable boards.
  296. * vbios does this for us in asic_init in that case.
  297. * Skip MC ucode loading on VF, because hypervisor will do that
  298. * for this adaptor.
  299. */
  300. if (amdgpu_sriov_bios(adev))
  301. return 0;
  302. WREG32(mmMC_SEQ_IO_DEBUG_INDEX, 0x9F);
  303. data = RREG32(mmMC_SEQ_IO_DEBUG_DATA);
  304. vbios_version = data & 0xf;
  305. if (vbios_version == 0)
  306. return 0;
  307. if (!adev->mc.fw)
  308. return -EINVAL;
  309. hdr = (const struct mc_firmware_header_v1_0 *)adev->mc.fw->data;
  310. amdgpu_ucode_print_mc_hdr(&hdr->header);
  311. adev->mc.fw_version = le32_to_cpu(hdr->header.ucode_version);
  312. regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
  313. io_mc_regs = (const __le32 *)
  314. (adev->mc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
  315. ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
  316. fw_data = (const __le32 *)
  317. (adev->mc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
  318. data = RREG32(mmMC_SEQ_MISC0);
  319. data &= ~(0x40);
  320. WREG32(mmMC_SEQ_MISC0, data);
  321. /* load mc io regs */
  322. for (i = 0; i < regs_size; i++) {
  323. WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++));
  324. WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++));
  325. }
  326. WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
  327. WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
  328. /* load the MC ucode */
  329. for (i = 0; i < ucode_size; i++)
  330. WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++));
  331. /* put the engine back into the active state */
  332. WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
  333. WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);
  334. WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);
  335. /* wait for training to complete */
  336. for (i = 0; i < adev->usec_timeout; i++) {
  337. data = RREG32(mmMC_SEQ_MISC0);
  338. if (data & 0x80)
  339. break;
  340. udelay(1);
  341. }
  342. return 0;
  343. }
  344. static void gmc_v8_0_vram_gtt_location(struct amdgpu_device *adev,
  345. struct amdgpu_mc *mc)
  346. {
  347. u64 base = 0;
  348. if (!amdgpu_sriov_vf(adev))
  349. base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
  350. base <<= 24;
  351. if (mc->mc_vram_size > 0xFFC0000000ULL) {
  352. /* leave room for at least 1024M GTT */
  353. dev_warn(adev->dev, "limiting VRAM\n");
  354. mc->real_vram_size = 0xFFC0000000ULL;
  355. mc->mc_vram_size = 0xFFC0000000ULL;
  356. }
  357. amdgpu_vram_location(adev, &adev->mc, base);
  358. amdgpu_gart_location(adev, mc);
  359. }
  360. /**
  361. * gmc_v8_0_mc_program - program the GPU memory controller
  362. *
  363. * @adev: amdgpu_device pointer
  364. *
  365. * Set the location of vram, gart, and AGP in the GPU's
  366. * physical address space (CIK).
  367. */
  368. static void gmc_v8_0_mc_program(struct amdgpu_device *adev)
  369. {
  370. u32 tmp;
  371. int i, j;
  372. /* Initialize HDP */
  373. for (i = 0, j = 0; i < 32; i++, j += 0x6) {
  374. WREG32((0xb05 + j), 0x00000000);
  375. WREG32((0xb06 + j), 0x00000000);
  376. WREG32((0xb07 + j), 0x00000000);
  377. WREG32((0xb08 + j), 0x00000000);
  378. WREG32((0xb09 + j), 0x00000000);
  379. }
  380. WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);
  381. if (gmc_v8_0_wait_for_idle((void *)adev)) {
  382. dev_warn(adev->dev, "Wait for MC idle timedout !\n");
  383. }
  384. if (adev->mode_info.num_crtc) {
  385. /* Lockout access through VGA aperture*/
  386. tmp = RREG32(mmVGA_HDP_CONTROL);
  387. tmp = REG_SET_FIELD(tmp, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
  388. WREG32(mmVGA_HDP_CONTROL, tmp);
  389. /* disable VGA render */
  390. tmp = RREG32(mmVGA_RENDER_CONTROL);
  391. tmp = REG_SET_FIELD(tmp, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
  392. WREG32(mmVGA_RENDER_CONTROL, tmp);
  393. }
  394. /* Update configuration */
  395. WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
  396. adev->mc.vram_start >> 12);
  397. WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
  398. adev->mc.vram_end >> 12);
  399. WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
  400. adev->vram_scratch.gpu_addr >> 12);
  401. if (amdgpu_sriov_vf(adev)) {
  402. tmp = ((adev->mc.vram_end >> 24) & 0xFFFF) << 16;
  403. tmp |= ((adev->mc.vram_start >> 24) & 0xFFFF);
  404. WREG32(mmMC_VM_FB_LOCATION, tmp);
  405. /* XXX double check these! */
  406. WREG32(mmHDP_NONSURFACE_BASE, (adev->mc.vram_start >> 8));
  407. WREG32(mmHDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
  408. WREG32(mmHDP_NONSURFACE_SIZE, 0x3FFFFFFF);
  409. }
  410. WREG32(mmMC_VM_AGP_BASE, 0);
  411. WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
  412. WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
  413. if (gmc_v8_0_wait_for_idle((void *)adev)) {
  414. dev_warn(adev->dev, "Wait for MC idle timedout !\n");
  415. }
  416. WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK);
  417. tmp = RREG32(mmHDP_MISC_CNTL);
  418. tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 0);
  419. WREG32(mmHDP_MISC_CNTL, tmp);
  420. tmp = RREG32(mmHDP_HOST_PATH_CNTL);
  421. WREG32(mmHDP_HOST_PATH_CNTL, tmp);
  422. }
  423. /**
  424. * gmc_v8_0_mc_init - initialize the memory controller driver params
  425. *
  426. * @adev: amdgpu_device pointer
  427. *
  428. * Look up the amount of vram, vram width, and decide how to place
  429. * vram and gart within the GPU's physical address space (CIK).
  430. * Returns 0 for success.
  431. */
  432. static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
  433. {
  434. adev->mc.vram_width = amdgpu_atombios_get_vram_width(adev);
  435. if (!adev->mc.vram_width) {
  436. u32 tmp;
  437. int chansize, numchan;
  438. /* Get VRAM informations */
  439. tmp = RREG32(mmMC_ARB_RAMCFG);
  440. if (REG_GET_FIELD(tmp, MC_ARB_RAMCFG, CHANSIZE)) {
  441. chansize = 64;
  442. } else {
  443. chansize = 32;
  444. }
  445. tmp = RREG32(mmMC_SHARED_CHMAP);
  446. switch (REG_GET_FIELD(tmp, MC_SHARED_CHMAP, NOOFCHAN)) {
  447. case 0:
  448. default:
  449. numchan = 1;
  450. break;
  451. case 1:
  452. numchan = 2;
  453. break;
  454. case 2:
  455. numchan = 4;
  456. break;
  457. case 3:
  458. numchan = 8;
  459. break;
  460. case 4:
  461. numchan = 3;
  462. break;
  463. case 5:
  464. numchan = 6;
  465. break;
  466. case 6:
  467. numchan = 10;
  468. break;
  469. case 7:
  470. numchan = 12;
  471. break;
  472. case 8:
  473. numchan = 16;
  474. break;
  475. }
  476. adev->mc.vram_width = numchan * chansize;
  477. }
  478. /* Could aper size report 0 ? */
  479. adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
  480. adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
  481. /* size in MB on si */
  482. adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
  483. adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
  484. #ifdef CONFIG_X86_64
  485. if (adev->flags & AMD_IS_APU) {
  486. adev->mc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;
  487. adev->mc.aper_size = adev->mc.real_vram_size;
  488. }
  489. #endif
  490. /* In case the PCI BAR is larger than the actual amount of vram */
  491. adev->mc.visible_vram_size = adev->mc.aper_size;
  492. if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
  493. adev->mc.visible_vram_size = adev->mc.real_vram_size;
  494. amdgpu_gart_set_defaults(adev);
  495. gmc_v8_0_vram_gtt_location(adev, &adev->mc);
  496. return 0;
  497. }
  498. /*
  499. * GART
  500. * VMID 0 is the physical GPU addresses as used by the kernel.
  501. * VMIDs 1-15 are used for userspace clients and are handled
  502. * by the amdgpu vm/hsa code.
  503. */
  504. /**
  505. * gmc_v8_0_gart_flush_gpu_tlb - gart tlb flush callback
  506. *
  507. * @adev: amdgpu_device pointer
  508. * @vmid: vm instance to flush
  509. *
  510. * Flush the TLB for the requested page table (CIK).
  511. */
  512. static void gmc_v8_0_gart_flush_gpu_tlb(struct amdgpu_device *adev,
  513. uint32_t vmid)
  514. {
  515. /* flush hdp cache */
  516. WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0);
  517. /* bits 0-15 are the VM contexts0-15 */
  518. WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
  519. }
  520. /**
  521. * gmc_v8_0_gart_set_pte_pde - update the page tables using MMIO
  522. *
  523. * @adev: amdgpu_device pointer
  524. * @cpu_pt_addr: cpu address of the page table
  525. * @gpu_page_idx: entry in the page table to update
  526. * @addr: dst addr to write into pte/pde
  527. * @flags: access flags
  528. *
  529. * Update the page tables using the CPU.
  530. */
  531. static int gmc_v8_0_gart_set_pte_pde(struct amdgpu_device *adev,
  532. void *cpu_pt_addr,
  533. uint32_t gpu_page_idx,
  534. uint64_t addr,
  535. uint64_t flags)
  536. {
  537. void __iomem *ptr = (void *)cpu_pt_addr;
  538. uint64_t value;
  539. /*
  540. * PTE format on VI:
  541. * 63:40 reserved
  542. * 39:12 4k physical page base address
  543. * 11:7 fragment
  544. * 6 write
  545. * 5 read
  546. * 4 exe
  547. * 3 reserved
  548. * 2 snooped
  549. * 1 system
  550. * 0 valid
  551. *
  552. * PDE format on VI:
  553. * 63:59 block fragment size
  554. * 58:40 reserved
  555. * 39:1 physical base address of PTE
  556. * bits 5:1 must be 0.
  557. * 0 valid
  558. */
  559. value = addr & 0x000000FFFFFFF000ULL;
  560. value |= flags;
  561. writeq(value, ptr + (gpu_page_idx * 8));
  562. return 0;
  563. }
  564. static uint64_t gmc_v8_0_get_vm_pte_flags(struct amdgpu_device *adev,
  565. uint32_t flags)
  566. {
  567. uint64_t pte_flag = 0;
  568. if (flags & AMDGPU_VM_PAGE_EXECUTABLE)
  569. pte_flag |= AMDGPU_PTE_EXECUTABLE;
  570. if (flags & AMDGPU_VM_PAGE_READABLE)
  571. pte_flag |= AMDGPU_PTE_READABLE;
  572. if (flags & AMDGPU_VM_PAGE_WRITEABLE)
  573. pte_flag |= AMDGPU_PTE_WRITEABLE;
  574. if (flags & AMDGPU_VM_PAGE_PRT)
  575. pte_flag |= AMDGPU_PTE_PRT;
  576. return pte_flag;
  577. }
  578. static uint64_t gmc_v8_0_get_vm_pde(struct amdgpu_device *adev, uint64_t addr)
  579. {
  580. BUG_ON(addr & 0xFFFFFF0000000FFFULL);
  581. return addr;
  582. }
  583. /**
  584. * gmc_v8_0_set_fault_enable_default - update VM fault handling
  585. *
  586. * @adev: amdgpu_device pointer
  587. * @value: true redirects VM faults to the default page
  588. */
  589. static void gmc_v8_0_set_fault_enable_default(struct amdgpu_device *adev,
  590. bool value)
  591. {
  592. u32 tmp;
  593. tmp = RREG32(mmVM_CONTEXT1_CNTL);
  594. tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
  595. RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
  596. tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
  597. DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
  598. tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
  599. PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
  600. tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
  601. VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
  602. tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
  603. READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
  604. tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
  605. WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
  606. tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
  607. EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
  608. WREG32(mmVM_CONTEXT1_CNTL, tmp);
  609. }
  610. /**
  611. * gmc_v8_0_set_prt - set PRT VM fault
  612. *
  613. * @adev: amdgpu_device pointer
  614. * @enable: enable/disable VM fault handling for PRT
  615. */
  616. static void gmc_v8_0_set_prt(struct amdgpu_device *adev, bool enable)
  617. {
  618. u32 tmp;
  619. if (enable && !adev->mc.prt_warning) {
  620. dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n");
  621. adev->mc.prt_warning = true;
  622. }
  623. tmp = RREG32(mmVM_PRT_CNTL);
  624. tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
  625. CB_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable);
  626. tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
  627. CB_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable);
  628. tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
  629. TC_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable);
  630. tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
  631. TC_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable);
  632. tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
  633. L2_CACHE_STORE_INVALID_ENTRIES, enable);
  634. tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
  635. L1_TLB_STORE_INVALID_ENTRIES, enable);
  636. tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
  637. MASK_PDE0_FAULT, enable);
  638. WREG32(mmVM_PRT_CNTL, tmp);
  639. if (enable) {
  640. uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
  641. uint32_t high = adev->vm_manager.max_pfn;
  642. WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
  643. WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);
  644. WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, low);
  645. WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, low);
  646. WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, high);
  647. WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, high);
  648. WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, high);
  649. WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, high);
  650. } else {
  651. WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, 0xfffffff);
  652. WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, 0xfffffff);
  653. WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, 0xfffffff);
  654. WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, 0xfffffff);
  655. WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, 0x0);
  656. WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, 0x0);
  657. WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, 0x0);
  658. WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, 0x0);
  659. }
  660. }
  661. /**
  662. * gmc_v8_0_gart_enable - gart enable
  663. *
  664. * @adev: amdgpu_device pointer
  665. *
  666. * This sets up the TLBs, programs the page tables for VMID0,
  667. * sets up the hw for VMIDs 1-15 which are allocated on
  668. * demand, and sets up the global locations for the LDS, GDS,
  669. * and GPUVM for FSA64 clients (CIK).
  670. * Returns 0 for success, errors for failure.
  671. */
  672. static int gmc_v8_0_gart_enable(struct amdgpu_device *adev)
  673. {
  674. int r, i;
  675. u32 tmp;
  676. if (adev->gart.robj == NULL) {
  677. dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
  678. return -EINVAL;
  679. }
  680. r = amdgpu_gart_table_vram_pin(adev);
  681. if (r)
  682. return r;
  683. /* Setup TLB control */
  684. tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
  685. tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
  686. tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 1);
  687. tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
  688. tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 1);
  689. tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
  690. WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
  691. /* Setup L2 cache */
  692. tmp = RREG32(mmVM_L2_CNTL);
  693. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
  694. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
  695. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE, 1);
  696. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE, 1);
  697. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, EFFECTIVE_L2_QUEUE_SIZE, 7);
  698. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
  699. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
  700. WREG32(mmVM_L2_CNTL, tmp);
  701. tmp = RREG32(mmVM_L2_CNTL2);
  702. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
  703. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
  704. WREG32(mmVM_L2_CNTL2, tmp);
  705. tmp = RREG32(mmVM_L2_CNTL3);
  706. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY, 1);
  707. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 4);
  708. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 4);
  709. WREG32(mmVM_L2_CNTL3, tmp);
  710. /* XXX: set to enable PTE/PDE in system memory */
  711. tmp = RREG32(mmVM_L2_CNTL4);
  712. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_PHYSICAL, 0);
  713. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_SHARED, 0);
  714. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_SNOOP, 0);
  715. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_PHYSICAL, 0);
  716. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_SHARED, 0);
  717. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_SNOOP, 0);
  718. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_PHYSICAL, 0);
  719. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_SHARED, 0);
  720. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_SNOOP, 0);
  721. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_PHYSICAL, 0);
  722. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_SHARED, 0);
  723. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_SNOOP, 0);
  724. WREG32(mmVM_L2_CNTL4, tmp);
  725. /* setup context0 */
  726. WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gart_start >> 12);
  727. WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gart_end >> 12);
  728. WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
  729. WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
  730. (u32)(adev->dummy_page.addr >> 12));
  731. WREG32(mmVM_CONTEXT0_CNTL2, 0);
  732. tmp = RREG32(mmVM_CONTEXT0_CNTL);
  733. tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
  734. tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
  735. tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
  736. WREG32(mmVM_CONTEXT0_CNTL, tmp);
  737. WREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR, 0);
  738. WREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR, 0);
  739. WREG32(mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET, 0);
  740. /* empty context1-15 */
  741. /* FIXME start with 4G, once using 2 level pt switch to full
  742. * vm size space
  743. */
  744. /* set vm size, must be a multiple of 4 */
  745. WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
  746. WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);
  747. for (i = 1; i < 16; i++) {
  748. if (i < 8)
  749. WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
  750. adev->gart.table_addr >> 12);
  751. else
  752. WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
  753. adev->gart.table_addr >> 12);
  754. }
  755. /* enable context1-15 */
  756. WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
  757. (u32)(adev->dummy_page.addr >> 12));
  758. WREG32(mmVM_CONTEXT1_CNTL2, 4);
  759. tmp = RREG32(mmVM_CONTEXT1_CNTL);
  760. tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
  761. tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1);
  762. tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
  763. tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
  764. tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
  765. tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
  766. tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
  767. tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
  768. tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
  769. tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE,
  770. adev->vm_manager.block_size - 9);
  771. WREG32(mmVM_CONTEXT1_CNTL, tmp);
  772. if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
  773. gmc_v8_0_set_fault_enable_default(adev, false);
  774. else
  775. gmc_v8_0_set_fault_enable_default(adev, true);
  776. gmc_v8_0_gart_flush_gpu_tlb(adev, 0);
  777. DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
  778. (unsigned)(adev->mc.gart_size >> 20),
  779. (unsigned long long)adev->gart.table_addr);
  780. adev->gart.ready = true;
  781. return 0;
  782. }
  783. static int gmc_v8_0_gart_init(struct amdgpu_device *adev)
  784. {
  785. int r;
  786. if (adev->gart.robj) {
  787. WARN(1, "R600 PCIE GART already initialized\n");
  788. return 0;
  789. }
  790. /* Initialize common gart structure */
  791. r = amdgpu_gart_init(adev);
  792. if (r)
  793. return r;
  794. adev->gart.table_size = adev->gart.num_gpu_pages * 8;
  795. adev->gart.gart_pte_flags = AMDGPU_PTE_EXECUTABLE;
  796. return amdgpu_gart_table_vram_alloc(adev);
  797. }
  798. /**
  799. * gmc_v8_0_gart_disable - gart disable
  800. *
  801. * @adev: amdgpu_device pointer
  802. *
  803. * This disables all VM page table (CIK).
  804. */
  805. static void gmc_v8_0_gart_disable(struct amdgpu_device *adev)
  806. {
  807. u32 tmp;
  808. /* Disable all tables */
  809. WREG32(mmVM_CONTEXT0_CNTL, 0);
  810. WREG32(mmVM_CONTEXT1_CNTL, 0);
  811. /* Setup TLB control */
  812. tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
  813. tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
  814. tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 0);
  815. tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 0);
  816. WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
  817. /* Setup L2 cache */
  818. tmp = RREG32(mmVM_L2_CNTL);
  819. tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
  820. WREG32(mmVM_L2_CNTL, tmp);
  821. WREG32(mmVM_L2_CNTL2, 0);
  822. amdgpu_gart_table_vram_unpin(adev);
  823. }
  824. /**
  825. * gmc_v8_0_gart_fini - vm fini callback
  826. *
  827. * @adev: amdgpu_device pointer
  828. *
  829. * Tears down the driver GART/VM setup (CIK).
  830. */
  831. static void gmc_v8_0_gart_fini(struct amdgpu_device *adev)
  832. {
  833. amdgpu_gart_table_vram_free(adev);
  834. amdgpu_gart_fini(adev);
  835. }
  836. /**
  837. * gmc_v8_0_vm_decode_fault - print human readable fault info
  838. *
  839. * @adev: amdgpu_device pointer
  840. * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
  841. * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
  842. *
  843. * Print human readable fault information (CIK).
  844. */
  845. static void gmc_v8_0_vm_decode_fault(struct amdgpu_device *adev,
  846. u32 status, u32 addr, u32 mc_client)
  847. {
  848. u32 mc_id;
  849. u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
  850. u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
  851. PROTECTIONS);
  852. char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
  853. (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
  854. mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
  855. MEMORY_CLIENT_ID);
  856. dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
  857. protections, vmid, addr,
  858. REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
  859. MEMORY_CLIENT_RW) ?
  860. "write" : "read", block, mc_client, mc_id);
  861. }
  862. static int gmc_v8_0_convert_vram_type(int mc_seq_vram_type)
  863. {
  864. switch (mc_seq_vram_type) {
  865. case MC_SEQ_MISC0__MT__GDDR1:
  866. return AMDGPU_VRAM_TYPE_GDDR1;
  867. case MC_SEQ_MISC0__MT__DDR2:
  868. return AMDGPU_VRAM_TYPE_DDR2;
  869. case MC_SEQ_MISC0__MT__GDDR3:
  870. return AMDGPU_VRAM_TYPE_GDDR3;
  871. case MC_SEQ_MISC0__MT__GDDR4:
  872. return AMDGPU_VRAM_TYPE_GDDR4;
  873. case MC_SEQ_MISC0__MT__GDDR5:
  874. return AMDGPU_VRAM_TYPE_GDDR5;
  875. case MC_SEQ_MISC0__MT__HBM:
  876. return AMDGPU_VRAM_TYPE_HBM;
  877. case MC_SEQ_MISC0__MT__DDR3:
  878. return AMDGPU_VRAM_TYPE_DDR3;
  879. default:
  880. return AMDGPU_VRAM_TYPE_UNKNOWN;
  881. }
  882. }
  883. static int gmc_v8_0_early_init(void *handle)
  884. {
  885. struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  886. gmc_v8_0_set_gart_funcs(adev);
  887. gmc_v8_0_set_irq_funcs(adev);
  888. adev->mc.shared_aperture_start = 0x2000000000000000ULL;
  889. adev->mc.shared_aperture_end =
  890. adev->mc.shared_aperture_start + (4ULL << 30) - 1;
  891. adev->mc.private_aperture_start =
  892. adev->mc.shared_aperture_end + 1;
  893. adev->mc.private_aperture_end =
  894. adev->mc.private_aperture_start + (4ULL << 30) - 1;
  895. return 0;
  896. }
  897. static int gmc_v8_0_late_init(void *handle)
  898. {
  899. struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  900. if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS)
  901. return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0);
  902. else
  903. return 0;
  904. }
  905. #define mmMC_SEQ_MISC0_FIJI 0xA71
  906. static int gmc_v8_0_sw_init(void *handle)
  907. {
  908. int r;
  909. int dma_bits;
  910. struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  911. if (adev->flags & AMD_IS_APU) {
  912. adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
  913. } else {
  914. u32 tmp;
  915. if (adev->asic_type == CHIP_FIJI)
  916. tmp = RREG32(mmMC_SEQ_MISC0_FIJI);
  917. else
  918. tmp = RREG32(mmMC_SEQ_MISC0);
  919. tmp &= MC_SEQ_MISC0__MT__MASK;
  920. adev->mc.vram_type = gmc_v8_0_convert_vram_type(tmp);
  921. }
  922. r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 146, &adev->mc.vm_fault);
  923. if (r)
  924. return r;
  925. r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 147, &adev->mc.vm_fault);
  926. if (r)
  927. return r;
  928. /* Adjust VM size here.
  929. * Currently set to 4GB ((1 << 20) 4k pages).
  930. * Max GPUVM size for cayman and SI is 40 bits.
  931. */
  932. amdgpu_vm_adjust_size(adev, 64);
  933. adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
  934. /* Set the internal MC address mask
  935. * This is the max address of the GPU's
  936. * internal address space.
  937. */
  938. adev->mc.mc_mask = 0xffffffffffULL; /* 40 bit MC */
  939. adev->mc.stolen_size = 256 * 1024;
  940. /* set DMA mask + need_dma32 flags.
  941. * PCIE - can handle 40-bits.
  942. * IGP - can handle 40-bits
  943. * PCI - dma32 for legacy pci gart, 40 bits on newer asics
  944. */
  945. adev->need_dma32 = false;
  946. dma_bits = adev->need_dma32 ? 32 : 40;
  947. r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
  948. if (r) {
  949. adev->need_dma32 = true;
  950. dma_bits = 32;
  951. pr_warn("amdgpu: No suitable DMA available\n");
  952. }
  953. r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
  954. if (r) {
  955. pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
  956. pr_warn("amdgpu: No coherent DMA available\n");
  957. }
  958. r = gmc_v8_0_init_microcode(adev);
  959. if (r) {
  960. DRM_ERROR("Failed to load mc firmware!\n");
  961. return r;
  962. }
  963. r = gmc_v8_0_mc_init(adev);
  964. if (r)
  965. return r;
  966. /* Memory manager */
  967. r = amdgpu_bo_init(adev);
  968. if (r)
  969. return r;
  970. r = gmc_v8_0_gart_init(adev);
  971. if (r)
  972. return r;
  973. /*
  974. * number of VMs
  975. * VMID 0 is reserved for System
  976. * amdgpu graphics/compute will use VMIDs 1-7
  977. * amdkfd will use VMIDs 8-15
  978. */
  979. adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
  980. adev->vm_manager.num_level = 1;
  981. amdgpu_vm_manager_init(adev);
  982. /* base offset of vram pages */
  983. if (adev->flags & AMD_IS_APU) {
  984. u64 tmp = RREG32(mmMC_VM_FB_OFFSET);
  985. tmp <<= 22;
  986. adev->vm_manager.vram_base_offset = tmp;
  987. } else {
  988. adev->vm_manager.vram_base_offset = 0;
  989. }
  990. return 0;
  991. }
  992. static int gmc_v8_0_sw_fini(void *handle)
  993. {
  994. struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  995. amdgpu_vm_manager_fini(adev);
  996. gmc_v8_0_gart_fini(adev);
  997. amdgpu_gem_force_release(adev);
  998. amdgpu_bo_fini(adev);
  999. return 0;
  1000. }
  1001. static int gmc_v8_0_hw_init(void *handle)
  1002. {
  1003. int r;
  1004. struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  1005. gmc_v8_0_init_golden_registers(adev);
  1006. gmc_v8_0_mc_program(adev);
  1007. if (adev->asic_type == CHIP_TONGA) {
  1008. r = gmc_v8_0_tonga_mc_load_microcode(adev);
  1009. if (r) {
  1010. DRM_ERROR("Failed to load MC firmware!\n");
  1011. return r;
  1012. }
  1013. } else if (adev->asic_type == CHIP_POLARIS11 ||
  1014. adev->asic_type == CHIP_POLARIS10 ||
  1015. adev->asic_type == CHIP_POLARIS12) {
  1016. r = gmc_v8_0_polaris_mc_load_microcode(adev);
  1017. if (r) {
  1018. DRM_ERROR("Failed to load MC firmware!\n");
  1019. return r;
  1020. }
  1021. }
  1022. r = gmc_v8_0_gart_enable(adev);
  1023. if (r)
  1024. return r;
  1025. return r;
  1026. }
  1027. static int gmc_v8_0_hw_fini(void *handle)
  1028. {
  1029. struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  1030. amdgpu_irq_put(adev, &adev->mc.vm_fault, 0);
  1031. gmc_v8_0_gart_disable(adev);
  1032. return 0;
  1033. }
  1034. static int gmc_v8_0_suspend(void *handle)
  1035. {
  1036. struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  1037. gmc_v8_0_hw_fini(adev);
  1038. return 0;
  1039. }
  1040. static int gmc_v8_0_resume(void *handle)
  1041. {
  1042. int r;
  1043. struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  1044. r = gmc_v8_0_hw_init(adev);
  1045. if (r)
  1046. return r;
  1047. amdgpu_vm_reset_all_ids(adev);
  1048. return 0;
  1049. }
  1050. static bool gmc_v8_0_is_idle(void *handle)
  1051. {
  1052. struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  1053. u32 tmp = RREG32(mmSRBM_STATUS);
  1054. if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
  1055. SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK))
  1056. return false;
  1057. return true;
  1058. }
  1059. static int gmc_v8_0_wait_for_idle(void *handle)
  1060. {
  1061. unsigned i;
  1062. u32 tmp;
  1063. struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  1064. for (i = 0; i < adev->usec_timeout; i++) {
  1065. /* read MC_STATUS */
  1066. tmp = RREG32(mmSRBM_STATUS) & (SRBM_STATUS__MCB_BUSY_MASK |
  1067. SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
  1068. SRBM_STATUS__MCC_BUSY_MASK |
  1069. SRBM_STATUS__MCD_BUSY_MASK |
  1070. SRBM_STATUS__VMC_BUSY_MASK |
  1071. SRBM_STATUS__VMC1_BUSY_MASK);
  1072. if (!tmp)
  1073. return 0;
  1074. udelay(1);
  1075. }
  1076. return -ETIMEDOUT;
  1077. }
  1078. static bool gmc_v8_0_check_soft_reset(void *handle)
  1079. {
  1080. u32 srbm_soft_reset = 0;
  1081. struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  1082. u32 tmp = RREG32(mmSRBM_STATUS);
  1083. if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
  1084. srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
  1085. SRBM_SOFT_RESET, SOFT_RESET_VMC, 1);
  1086. if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
  1087. SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
  1088. if (!(adev->flags & AMD_IS_APU))
  1089. srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
  1090. SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
  1091. }
  1092. if (srbm_soft_reset) {
  1093. adev->mc.srbm_soft_reset = srbm_soft_reset;
  1094. return true;
  1095. } else {
  1096. adev->mc.srbm_soft_reset = 0;
  1097. return false;
  1098. }
  1099. }
  1100. static int gmc_v8_0_pre_soft_reset(void *handle)
  1101. {
  1102. struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  1103. if (!adev->mc.srbm_soft_reset)
  1104. return 0;
  1105. gmc_v8_0_mc_stop(adev);
  1106. if (gmc_v8_0_wait_for_idle(adev)) {
  1107. dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
  1108. }
  1109. return 0;
  1110. }
  1111. static int gmc_v8_0_soft_reset(void *handle)
  1112. {
  1113. struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  1114. u32 srbm_soft_reset;
  1115. if (!adev->mc.srbm_soft_reset)
  1116. return 0;
  1117. srbm_soft_reset = adev->mc.srbm_soft_reset;
  1118. if (srbm_soft_reset) {
  1119. u32 tmp;
  1120. tmp = RREG32(mmSRBM_SOFT_RESET);
  1121. tmp |= srbm_soft_reset;
  1122. dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
  1123. WREG32(mmSRBM_SOFT_RESET, tmp);
  1124. tmp = RREG32(mmSRBM_SOFT_RESET);
  1125. udelay(50);
  1126. tmp &= ~srbm_soft_reset;
  1127. WREG32(mmSRBM_SOFT_RESET, tmp);
  1128. tmp = RREG32(mmSRBM_SOFT_RESET);
  1129. /* Wait a little for things to settle down */
  1130. udelay(50);
  1131. }
  1132. return 0;
  1133. }
  1134. static int gmc_v8_0_post_soft_reset(void *handle)
  1135. {
  1136. struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  1137. if (!adev->mc.srbm_soft_reset)
  1138. return 0;
  1139. gmc_v8_0_mc_resume(adev);
  1140. return 0;
  1141. }
  1142. static int gmc_v8_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
  1143. struct amdgpu_irq_src *src,
  1144. unsigned type,
  1145. enum amdgpu_interrupt_state state)
  1146. {
  1147. u32 tmp;
  1148. u32 bits = (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
  1149. VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
  1150. VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
  1151. VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
  1152. VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
  1153. VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
  1154. VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK);
  1155. switch (state) {
  1156. case AMDGPU_IRQ_STATE_DISABLE:
  1157. /* system context */
  1158. tmp = RREG32(mmVM_CONTEXT0_CNTL);
  1159. tmp &= ~bits;
  1160. WREG32(mmVM_CONTEXT0_CNTL, tmp);
  1161. /* VMs */
  1162. tmp = RREG32(mmVM_CONTEXT1_CNTL);
  1163. tmp &= ~bits;
  1164. WREG32(mmVM_CONTEXT1_CNTL, tmp);
  1165. break;
  1166. case AMDGPU_IRQ_STATE_ENABLE:
  1167. /* system context */
  1168. tmp = RREG32(mmVM_CONTEXT0_CNTL);
  1169. tmp |= bits;
  1170. WREG32(mmVM_CONTEXT0_CNTL, tmp);
  1171. /* VMs */
  1172. tmp = RREG32(mmVM_CONTEXT1_CNTL);
  1173. tmp |= bits;
  1174. WREG32(mmVM_CONTEXT1_CNTL, tmp);
  1175. break;
  1176. default:
  1177. break;
  1178. }
  1179. return 0;
  1180. }
  1181. static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev,
  1182. struct amdgpu_irq_src *source,
  1183. struct amdgpu_iv_entry *entry)
  1184. {
  1185. u32 addr, status, mc_client;
  1186. if (amdgpu_sriov_vf(adev)) {
  1187. dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
  1188. entry->src_id, entry->src_data[0]);
  1189. dev_err(adev->dev, " Can't decode VM fault info here on SRIOV VF\n");
  1190. return 0;
  1191. }
  1192. addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
  1193. status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
  1194. mc_client = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
  1195. /* reset addr and status */
  1196. WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
  1197. if (!addr && !status)
  1198. return 0;
  1199. if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
  1200. gmc_v8_0_set_fault_enable_default(adev, false);
  1201. if (printk_ratelimit()) {
  1202. dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
  1203. entry->src_id, entry->src_data[0]);
  1204. dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
  1205. addr);
  1206. dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
  1207. status);
  1208. gmc_v8_0_vm_decode_fault(adev, status, addr, mc_client);
  1209. }
  1210. return 0;
  1211. }
  1212. static void fiji_update_mc_medium_grain_clock_gating(struct amdgpu_device *adev,
  1213. bool enable)
  1214. {
  1215. uint32_t data;
  1216. if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG)) {
  1217. data = RREG32(mmMC_HUB_MISC_HUB_CG);
  1218. data |= MC_HUB_MISC_HUB_CG__ENABLE_MASK;
  1219. WREG32(mmMC_HUB_MISC_HUB_CG, data);
  1220. data = RREG32(mmMC_HUB_MISC_SIP_CG);
  1221. data |= MC_HUB_MISC_SIP_CG__ENABLE_MASK;
  1222. WREG32(mmMC_HUB_MISC_SIP_CG, data);
  1223. data = RREG32(mmMC_HUB_MISC_VM_CG);
  1224. data |= MC_HUB_MISC_VM_CG__ENABLE_MASK;
  1225. WREG32(mmMC_HUB_MISC_VM_CG, data);
  1226. data = RREG32(mmMC_XPB_CLK_GAT);
  1227. data |= MC_XPB_CLK_GAT__ENABLE_MASK;
  1228. WREG32(mmMC_XPB_CLK_GAT, data);
  1229. data = RREG32(mmATC_MISC_CG);
  1230. data |= ATC_MISC_CG__ENABLE_MASK;
  1231. WREG32(mmATC_MISC_CG, data);
  1232. data = RREG32(mmMC_CITF_MISC_WR_CG);
  1233. data |= MC_CITF_MISC_WR_CG__ENABLE_MASK;
  1234. WREG32(mmMC_CITF_MISC_WR_CG, data);
  1235. data = RREG32(mmMC_CITF_MISC_RD_CG);
  1236. data |= MC_CITF_MISC_RD_CG__ENABLE_MASK;
  1237. WREG32(mmMC_CITF_MISC_RD_CG, data);
  1238. data = RREG32(mmMC_CITF_MISC_VM_CG);
  1239. data |= MC_CITF_MISC_VM_CG__ENABLE_MASK;
  1240. WREG32(mmMC_CITF_MISC_VM_CG, data);
  1241. data = RREG32(mmVM_L2_CG);
  1242. data |= VM_L2_CG__ENABLE_MASK;
  1243. WREG32(mmVM_L2_CG, data);
  1244. } else {
  1245. data = RREG32(mmMC_HUB_MISC_HUB_CG);
  1246. data &= ~MC_HUB_MISC_HUB_CG__ENABLE_MASK;
  1247. WREG32(mmMC_HUB_MISC_HUB_CG, data);
  1248. data = RREG32(mmMC_HUB_MISC_SIP_CG);
  1249. data &= ~MC_HUB_MISC_SIP_CG__ENABLE_MASK;
  1250. WREG32(mmMC_HUB_MISC_SIP_CG, data);
  1251. data = RREG32(mmMC_HUB_MISC_VM_CG);
  1252. data &= ~MC_HUB_MISC_VM_CG__ENABLE_MASK;
  1253. WREG32(mmMC_HUB_MISC_VM_CG, data);
  1254. data = RREG32(mmMC_XPB_CLK_GAT);
  1255. data &= ~MC_XPB_CLK_GAT__ENABLE_MASK;
  1256. WREG32(mmMC_XPB_CLK_GAT, data);
  1257. data = RREG32(mmATC_MISC_CG);
  1258. data &= ~ATC_MISC_CG__ENABLE_MASK;
  1259. WREG32(mmATC_MISC_CG, data);
  1260. data = RREG32(mmMC_CITF_MISC_WR_CG);
  1261. data &= ~MC_CITF_MISC_WR_CG__ENABLE_MASK;
  1262. WREG32(mmMC_CITF_MISC_WR_CG, data);
  1263. data = RREG32(mmMC_CITF_MISC_RD_CG);
  1264. data &= ~MC_CITF_MISC_RD_CG__ENABLE_MASK;
  1265. WREG32(mmMC_CITF_MISC_RD_CG, data);
  1266. data = RREG32(mmMC_CITF_MISC_VM_CG);
  1267. data &= ~MC_CITF_MISC_VM_CG__ENABLE_MASK;
  1268. WREG32(mmMC_CITF_MISC_VM_CG, data);
  1269. data = RREG32(mmVM_L2_CG);
  1270. data &= ~VM_L2_CG__ENABLE_MASK;
  1271. WREG32(mmVM_L2_CG, data);
  1272. }
  1273. }
  1274. static void fiji_update_mc_light_sleep(struct amdgpu_device *adev,
  1275. bool enable)
  1276. {
  1277. uint32_t data;
  1278. if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS)) {
  1279. data = RREG32(mmMC_HUB_MISC_HUB_CG);
  1280. data |= MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK;
  1281. WREG32(mmMC_HUB_MISC_HUB_CG, data);
  1282. data = RREG32(mmMC_HUB_MISC_SIP_CG);
  1283. data |= MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK;
  1284. WREG32(mmMC_HUB_MISC_SIP_CG, data);
  1285. data = RREG32(mmMC_HUB_MISC_VM_CG);
  1286. data |= MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK;
  1287. WREG32(mmMC_HUB_MISC_VM_CG, data);
  1288. data = RREG32(mmMC_XPB_CLK_GAT);
  1289. data |= MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK;
  1290. WREG32(mmMC_XPB_CLK_GAT, data);
  1291. data = RREG32(mmATC_MISC_CG);
  1292. data |= ATC_MISC_CG__MEM_LS_ENABLE_MASK;
  1293. WREG32(mmATC_MISC_CG, data);
  1294. data = RREG32(mmMC_CITF_MISC_WR_CG);
  1295. data |= MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK;
  1296. WREG32(mmMC_CITF_MISC_WR_CG, data);
  1297. data = RREG32(mmMC_CITF_MISC_RD_CG);
  1298. data |= MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK;
  1299. WREG32(mmMC_CITF_MISC_RD_CG, data);
  1300. data = RREG32(mmMC_CITF_MISC_VM_CG);
  1301. data |= MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK;
  1302. WREG32(mmMC_CITF_MISC_VM_CG, data);
  1303. data = RREG32(mmVM_L2_CG);
  1304. data |= VM_L2_CG__MEM_LS_ENABLE_MASK;
  1305. WREG32(mmVM_L2_CG, data);
  1306. } else {
  1307. data = RREG32(mmMC_HUB_MISC_HUB_CG);
  1308. data &= ~MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK;
  1309. WREG32(mmMC_HUB_MISC_HUB_CG, data);
  1310. data = RREG32(mmMC_HUB_MISC_SIP_CG);
  1311. data &= ~MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK;
  1312. WREG32(mmMC_HUB_MISC_SIP_CG, data);
  1313. data = RREG32(mmMC_HUB_MISC_VM_CG);
  1314. data &= ~MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK;
  1315. WREG32(mmMC_HUB_MISC_VM_CG, data);
  1316. data = RREG32(mmMC_XPB_CLK_GAT);
  1317. data &= ~MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK;
  1318. WREG32(mmMC_XPB_CLK_GAT, data);
  1319. data = RREG32(mmATC_MISC_CG);
  1320. data &= ~ATC_MISC_CG__MEM_LS_ENABLE_MASK;
  1321. WREG32(mmATC_MISC_CG, data);
  1322. data = RREG32(mmMC_CITF_MISC_WR_CG);
  1323. data &= ~MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK;
  1324. WREG32(mmMC_CITF_MISC_WR_CG, data);
  1325. data = RREG32(mmMC_CITF_MISC_RD_CG);
  1326. data &= ~MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK;
  1327. WREG32(mmMC_CITF_MISC_RD_CG, data);
  1328. data = RREG32(mmMC_CITF_MISC_VM_CG);
  1329. data &= ~MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK;
  1330. WREG32(mmMC_CITF_MISC_VM_CG, data);
  1331. data = RREG32(mmVM_L2_CG);
  1332. data &= ~VM_L2_CG__MEM_LS_ENABLE_MASK;
  1333. WREG32(mmVM_L2_CG, data);
  1334. }
  1335. }
  1336. static int gmc_v8_0_set_clockgating_state(void *handle,
  1337. enum amd_clockgating_state state)
  1338. {
  1339. struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  1340. if (amdgpu_sriov_vf(adev))
  1341. return 0;
  1342. switch (adev->asic_type) {
  1343. case CHIP_FIJI:
  1344. fiji_update_mc_medium_grain_clock_gating(adev,
  1345. state == AMD_CG_STATE_GATE);
  1346. fiji_update_mc_light_sleep(adev,
  1347. state == AMD_CG_STATE_GATE);
  1348. break;
  1349. default:
  1350. break;
  1351. }
  1352. return 0;
  1353. }
  1354. static int gmc_v8_0_set_powergating_state(void *handle,
  1355. enum amd_powergating_state state)
  1356. {
  1357. return 0;
  1358. }
  1359. static void gmc_v8_0_get_clockgating_state(void *handle, u32 *flags)
  1360. {
  1361. struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  1362. int data;
  1363. if (amdgpu_sriov_vf(adev))
  1364. *flags = 0;
  1365. /* AMD_CG_SUPPORT_MC_MGCG */
  1366. data = RREG32(mmMC_HUB_MISC_HUB_CG);
  1367. if (data & MC_HUB_MISC_HUB_CG__ENABLE_MASK)
  1368. *flags |= AMD_CG_SUPPORT_MC_MGCG;
  1369. /* AMD_CG_SUPPORT_MC_LS */
  1370. if (data & MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK)
  1371. *flags |= AMD_CG_SUPPORT_MC_LS;
  1372. }
  1373. static const struct amd_ip_funcs gmc_v8_0_ip_funcs = {
  1374. .name = "gmc_v8_0",
  1375. .early_init = gmc_v8_0_early_init,
  1376. .late_init = gmc_v8_0_late_init,
  1377. .sw_init = gmc_v8_0_sw_init,
  1378. .sw_fini = gmc_v8_0_sw_fini,
  1379. .hw_init = gmc_v8_0_hw_init,
  1380. .hw_fini = gmc_v8_0_hw_fini,
  1381. .suspend = gmc_v8_0_suspend,
  1382. .resume = gmc_v8_0_resume,
  1383. .is_idle = gmc_v8_0_is_idle,
  1384. .wait_for_idle = gmc_v8_0_wait_for_idle,
  1385. .check_soft_reset = gmc_v8_0_check_soft_reset,
  1386. .pre_soft_reset = gmc_v8_0_pre_soft_reset,
  1387. .soft_reset = gmc_v8_0_soft_reset,
  1388. .post_soft_reset = gmc_v8_0_post_soft_reset,
  1389. .set_clockgating_state = gmc_v8_0_set_clockgating_state,
  1390. .set_powergating_state = gmc_v8_0_set_powergating_state,
  1391. .get_clockgating_state = gmc_v8_0_get_clockgating_state,
  1392. };
  1393. static const struct amdgpu_gart_funcs gmc_v8_0_gart_funcs = {
  1394. .flush_gpu_tlb = gmc_v8_0_gart_flush_gpu_tlb,
  1395. .set_pte_pde = gmc_v8_0_gart_set_pte_pde,
  1396. .set_prt = gmc_v8_0_set_prt,
  1397. .get_vm_pte_flags = gmc_v8_0_get_vm_pte_flags,
  1398. .get_vm_pde = gmc_v8_0_get_vm_pde
  1399. };
  1400. static const struct amdgpu_irq_src_funcs gmc_v8_0_irq_funcs = {
  1401. .set = gmc_v8_0_vm_fault_interrupt_state,
  1402. .process = gmc_v8_0_process_interrupt,
  1403. };
  1404. static void gmc_v8_0_set_gart_funcs(struct amdgpu_device *adev)
  1405. {
  1406. if (adev->gart.gart_funcs == NULL)
  1407. adev->gart.gart_funcs = &gmc_v8_0_gart_funcs;
  1408. }
  1409. static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev)
  1410. {
  1411. adev->mc.vm_fault.num_types = 1;
  1412. adev->mc.vm_fault.funcs = &gmc_v8_0_irq_funcs;
  1413. }
  1414. const struct amdgpu_ip_block_version gmc_v8_0_ip_block =
  1415. {
  1416. .type = AMD_IP_BLOCK_TYPE_GMC,
  1417. .major = 8,
  1418. .minor = 0,
  1419. .rev = 0,
  1420. .funcs = &gmc_v8_0_ip_funcs,
  1421. };
  1422. const struct amdgpu_ip_block_version gmc_v8_1_ip_block =
  1423. {
  1424. .type = AMD_IP_BLOCK_TYPE_GMC,
  1425. .major = 8,
  1426. .minor = 1,
  1427. .rev = 0,
  1428. .funcs = &gmc_v8_0_ip_funcs,
  1429. };
  1430. const struct amdgpu_ip_block_version gmc_v8_5_ip_block =
  1431. {
  1432. .type = AMD_IP_BLOCK_TYPE_GMC,
  1433. .major = 8,
  1434. .minor = 5,
  1435. .rev = 0,
  1436. .funcs = &gmc_v8_0_ip_funcs,
  1437. };