amdgpu_ttm.c 40 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605
  1. /*
  2. * Copyright 2009 Jerome Glisse.
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining a
  6. * copy of this software and associated documentation files (the
  7. * "Software"), to deal in the Software without restriction, including
  8. * without limitation the rights to use, copy, modify, merge, publish,
  9. * distribute, sub license, and/or sell copies of the Software, and to
  10. * permit persons to whom the Software is furnished to do so, subject to
  11. * the following conditions:
  12. *
  13. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15. * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16. * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17. * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18. * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19. * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20. *
  21. * The above copyright notice and this permission notice (including the
  22. * next paragraph) shall be included in all copies or substantial portions
  23. * of the Software.
  24. *
  25. */
  26. /*
  27. * Authors:
  28. * Jerome Glisse <glisse@freedesktop.org>
  29. * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
  30. * Dave Airlie
  31. */
  32. #include <ttm/ttm_bo_api.h>
  33. #include <ttm/ttm_bo_driver.h>
  34. #include <ttm/ttm_placement.h>
  35. #include <ttm/ttm_module.h>
  36. #include <ttm/ttm_page_alloc.h>
  37. #include <ttm/ttm_memory.h>
  38. #include <drm/drmP.h>
  39. #include <drm/amdgpu_drm.h>
  40. #include <linux/seq_file.h>
  41. #include <linux/slab.h>
  42. #include <linux/swiotlb.h>
  43. #include <linux/swap.h>
  44. #include <linux/pagemap.h>
  45. #include <linux/debugfs.h>
  46. #include "amdgpu.h"
  47. #include "bif/bif_4_1_d.h"
  48. #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
  49. static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);
  50. static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev);
  51. /*
  52. * Global memory.
  53. */
  54. static int amdgpu_ttm_mem_global_init(struct drm_global_reference *ref)
  55. {
  56. return ttm_mem_global_init(ref->object);
  57. }
  58. static void amdgpu_ttm_mem_global_release(struct drm_global_reference *ref)
  59. {
  60. ttm_mem_global_release(ref->object);
  61. }
  62. int amdgpu_ttm_global_init(struct amdgpu_device *adev)
  63. {
  64. struct drm_global_reference *global_ref;
  65. struct amdgpu_ring *ring;
  66. struct amd_sched_rq *rq;
  67. int r;
  68. adev->mman.mem_global_referenced = false;
  69. global_ref = &adev->mman.mem_global_ref;
  70. global_ref->global_type = DRM_GLOBAL_TTM_MEM;
  71. global_ref->size = sizeof(struct ttm_mem_global);
  72. global_ref->init = &amdgpu_ttm_mem_global_init;
  73. global_ref->release = &amdgpu_ttm_mem_global_release;
  74. r = drm_global_item_ref(global_ref);
  75. if (r) {
  76. DRM_ERROR("Failed setting up TTM memory accounting "
  77. "subsystem.\n");
  78. goto error_mem;
  79. }
  80. adev->mman.bo_global_ref.mem_glob =
  81. adev->mman.mem_global_ref.object;
  82. global_ref = &adev->mman.bo_global_ref.ref;
  83. global_ref->global_type = DRM_GLOBAL_TTM_BO;
  84. global_ref->size = sizeof(struct ttm_bo_global);
  85. global_ref->init = &ttm_bo_global_init;
  86. global_ref->release = &ttm_bo_global_release;
  87. r = drm_global_item_ref(global_ref);
  88. if (r) {
  89. DRM_ERROR("Failed setting up TTM BO subsystem.\n");
  90. goto error_bo;
  91. }
  92. ring = adev->mman.buffer_funcs_ring;
  93. rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
  94. r = amd_sched_entity_init(&ring->sched, &adev->mman.entity,
  95. rq, amdgpu_sched_jobs);
  96. if (r) {
  97. DRM_ERROR("Failed setting up TTM BO move run queue.\n");
  98. goto error_entity;
  99. }
  100. adev->mman.mem_global_referenced = true;
  101. return 0;
  102. error_entity:
  103. drm_global_item_unref(&adev->mman.bo_global_ref.ref);
  104. error_bo:
  105. drm_global_item_unref(&adev->mman.mem_global_ref);
  106. error_mem:
  107. return r;
  108. }
  109. static void amdgpu_ttm_global_fini(struct amdgpu_device *adev)
  110. {
  111. if (adev->mman.mem_global_referenced) {
  112. amd_sched_entity_fini(adev->mman.entity.sched,
  113. &adev->mman.entity);
  114. drm_global_item_unref(&adev->mman.bo_global_ref.ref);
  115. drm_global_item_unref(&adev->mman.mem_global_ref);
  116. adev->mman.mem_global_referenced = false;
  117. }
  118. }
  119. static int amdgpu_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
  120. {
  121. return 0;
  122. }
  123. static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
  124. struct ttm_mem_type_manager *man)
  125. {
  126. struct amdgpu_device *adev;
  127. adev = amdgpu_ttm_adev(bdev);
  128. switch (type) {
  129. case TTM_PL_SYSTEM:
  130. /* System memory */
  131. man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
  132. man->available_caching = TTM_PL_MASK_CACHING;
  133. man->default_caching = TTM_PL_FLAG_CACHED;
  134. break;
  135. case TTM_PL_TT:
  136. man->func = &amdgpu_gtt_mgr_func;
  137. man->gpu_offset = adev->mc.gtt_start;
  138. man->available_caching = TTM_PL_MASK_CACHING;
  139. man->default_caching = TTM_PL_FLAG_CACHED;
  140. man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
  141. break;
  142. case TTM_PL_VRAM:
  143. /* "On-card" video ram */
  144. man->func = &amdgpu_vram_mgr_func;
  145. man->gpu_offset = adev->mc.vram_start;
  146. man->flags = TTM_MEMTYPE_FLAG_FIXED |
  147. TTM_MEMTYPE_FLAG_MAPPABLE;
  148. man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;
  149. man->default_caching = TTM_PL_FLAG_WC;
  150. break;
  151. case AMDGPU_PL_GDS:
  152. case AMDGPU_PL_GWS:
  153. case AMDGPU_PL_OA:
  154. /* On-chip GDS memory*/
  155. man->func = &ttm_bo_manager_func;
  156. man->gpu_offset = 0;
  157. man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_CMA;
  158. man->available_caching = TTM_PL_FLAG_UNCACHED;
  159. man->default_caching = TTM_PL_FLAG_UNCACHED;
  160. break;
  161. default:
  162. DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
  163. return -EINVAL;
  164. }
  165. return 0;
  166. }
  167. static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
  168. struct ttm_placement *placement)
  169. {
  170. struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
  171. struct amdgpu_bo *abo;
  172. static struct ttm_place placements = {
  173. .fpfn = 0,
  174. .lpfn = 0,
  175. .flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM
  176. };
  177. unsigned i;
  178. if (!amdgpu_ttm_bo_is_amdgpu_bo(bo)) {
  179. placement->placement = &placements;
  180. placement->busy_placement = &placements;
  181. placement->num_placement = 1;
  182. placement->num_busy_placement = 1;
  183. return;
  184. }
  185. abo = container_of(bo, struct amdgpu_bo, tbo);
  186. switch (bo->mem.mem_type) {
  187. case TTM_PL_VRAM:
  188. if (adev->mman.buffer_funcs_ring->ready == false) {
  189. amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
  190. } else {
  191. amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
  192. for (i = 0; i < abo->placement.num_placement; ++i) {
  193. if (!(abo->placements[i].flags &
  194. TTM_PL_FLAG_TT))
  195. continue;
  196. if (abo->placements[i].lpfn)
  197. continue;
  198. /* set an upper limit to force directly
  199. * allocating address space for the BO.
  200. */
  201. abo->placements[i].lpfn =
  202. adev->mc.gtt_size >> PAGE_SHIFT;
  203. }
  204. }
  205. break;
  206. case TTM_PL_TT:
  207. default:
  208. amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
  209. }
  210. *placement = abo->placement;
  211. }
  212. static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp)
  213. {
  214. struct amdgpu_bo *abo = container_of(bo, struct amdgpu_bo, tbo);
  215. if (amdgpu_ttm_tt_get_usermm(bo->ttm))
  216. return -EPERM;
  217. return drm_vma_node_verify_access(&abo->gem_base.vma_node,
  218. filp->private_data);
  219. }
  220. static void amdgpu_move_null(struct ttm_buffer_object *bo,
  221. struct ttm_mem_reg *new_mem)
  222. {
  223. struct ttm_mem_reg *old_mem = &bo->mem;
  224. BUG_ON(old_mem->mm_node != NULL);
  225. *old_mem = *new_mem;
  226. new_mem->mm_node = NULL;
  227. }
  228. static int amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
  229. struct drm_mm_node *mm_node,
  230. struct ttm_mem_reg *mem,
  231. uint64_t *addr)
  232. {
  233. int r;
  234. switch (mem->mem_type) {
  235. case TTM_PL_TT:
  236. r = amdgpu_ttm_bind(bo, mem);
  237. if (r)
  238. return r;
  239. case TTM_PL_VRAM:
  240. *addr = mm_node->start << PAGE_SHIFT;
  241. *addr += bo->bdev->man[mem->mem_type].gpu_offset;
  242. break;
  243. default:
  244. DRM_ERROR("Unknown placement %d\n", mem->mem_type);
  245. return -EINVAL;
  246. }
  247. return 0;
  248. }
  249. static int amdgpu_move_blit(struct ttm_buffer_object *bo,
  250. bool evict, bool no_wait_gpu,
  251. struct ttm_mem_reg *new_mem,
  252. struct ttm_mem_reg *old_mem)
  253. {
  254. struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
  255. struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
  256. struct drm_mm_node *old_mm, *new_mm;
  257. uint64_t old_start, old_size, new_start, new_size;
  258. unsigned long num_pages;
  259. struct fence *fence = NULL;
  260. int r;
  261. BUILD_BUG_ON((PAGE_SIZE % AMDGPU_GPU_PAGE_SIZE) != 0);
  262. if (!ring->ready) {
  263. DRM_ERROR("Trying to move memory with ring turned off.\n");
  264. return -EINVAL;
  265. }
  266. old_mm = old_mem->mm_node;
  267. r = amdgpu_mm_node_addr(bo, old_mm, old_mem, &old_start);
  268. if (r)
  269. return r;
  270. old_size = old_mm->size;
  271. new_mm = new_mem->mm_node;
  272. r = amdgpu_mm_node_addr(bo, new_mm, new_mem, &new_start);
  273. if (r)
  274. return r;
  275. new_size = new_mm->size;
  276. num_pages = new_mem->num_pages;
  277. while (num_pages) {
  278. unsigned long cur_pages = min(old_size, new_size);
  279. struct fence *next;
  280. r = amdgpu_copy_buffer(ring, old_start, new_start,
  281. cur_pages * PAGE_SIZE,
  282. bo->resv, &next, false);
  283. if (r)
  284. goto error;
  285. fence_put(fence);
  286. fence = next;
  287. num_pages -= cur_pages;
  288. if (!num_pages)
  289. break;
  290. old_size -= cur_pages;
  291. if (!old_size) {
  292. r = amdgpu_mm_node_addr(bo, ++old_mm, old_mem,
  293. &old_start);
  294. if (r)
  295. goto error;
  296. old_size = old_mm->size;
  297. } else {
  298. old_start += cur_pages * PAGE_SIZE;
  299. }
  300. new_size -= cur_pages;
  301. if (!new_size) {
  302. r = amdgpu_mm_node_addr(bo, ++new_mm, new_mem,
  303. &new_start);
  304. if (r)
  305. goto error;
  306. new_size = new_mm->size;
  307. } else {
  308. new_start += cur_pages * PAGE_SIZE;
  309. }
  310. }
  311. r = ttm_bo_pipeline_move(bo, fence, evict, new_mem);
  312. fence_put(fence);
  313. return r;
  314. error:
  315. if (fence)
  316. fence_wait(fence, false);
  317. fence_put(fence);
  318. return r;
  319. }
  320. static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo,
  321. bool evict, bool interruptible,
  322. bool no_wait_gpu,
  323. struct ttm_mem_reg *new_mem)
  324. {
  325. struct amdgpu_device *adev;
  326. struct ttm_mem_reg *old_mem = &bo->mem;
  327. struct ttm_mem_reg tmp_mem;
  328. struct ttm_place placements;
  329. struct ttm_placement placement;
  330. int r;
  331. adev = amdgpu_ttm_adev(bo->bdev);
  332. tmp_mem = *new_mem;
  333. tmp_mem.mm_node = NULL;
  334. placement.num_placement = 1;
  335. placement.placement = &placements;
  336. placement.num_busy_placement = 1;
  337. placement.busy_placement = &placements;
  338. placements.fpfn = 0;
  339. placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT;
  340. placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
  341. r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
  342. interruptible, no_wait_gpu);
  343. if (unlikely(r)) {
  344. return r;
  345. }
  346. r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);
  347. if (unlikely(r)) {
  348. goto out_cleanup;
  349. }
  350. r = ttm_tt_bind(bo->ttm, &tmp_mem);
  351. if (unlikely(r)) {
  352. goto out_cleanup;
  353. }
  354. r = amdgpu_move_blit(bo, true, no_wait_gpu, &tmp_mem, old_mem);
  355. if (unlikely(r)) {
  356. goto out_cleanup;
  357. }
  358. r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, new_mem);
  359. out_cleanup:
  360. ttm_bo_mem_put(bo, &tmp_mem);
  361. return r;
  362. }
  363. static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo,
  364. bool evict, bool interruptible,
  365. bool no_wait_gpu,
  366. struct ttm_mem_reg *new_mem)
  367. {
  368. struct amdgpu_device *adev;
  369. struct ttm_mem_reg *old_mem = &bo->mem;
  370. struct ttm_mem_reg tmp_mem;
  371. struct ttm_placement placement;
  372. struct ttm_place placements;
  373. int r;
  374. adev = amdgpu_ttm_adev(bo->bdev);
  375. tmp_mem = *new_mem;
  376. tmp_mem.mm_node = NULL;
  377. placement.num_placement = 1;
  378. placement.placement = &placements;
  379. placement.num_busy_placement = 1;
  380. placement.busy_placement = &placements;
  381. placements.fpfn = 0;
  382. placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT;
  383. placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
  384. r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
  385. interruptible, no_wait_gpu);
  386. if (unlikely(r)) {
  387. return r;
  388. }
  389. r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, &tmp_mem);
  390. if (unlikely(r)) {
  391. goto out_cleanup;
  392. }
  393. r = amdgpu_move_blit(bo, true, no_wait_gpu, new_mem, old_mem);
  394. if (unlikely(r)) {
  395. goto out_cleanup;
  396. }
  397. out_cleanup:
  398. ttm_bo_mem_put(bo, &tmp_mem);
  399. return r;
  400. }
  401. static int amdgpu_bo_move(struct ttm_buffer_object *bo,
  402. bool evict, bool interruptible,
  403. bool no_wait_gpu,
  404. struct ttm_mem_reg *new_mem)
  405. {
  406. struct amdgpu_device *adev;
  407. struct amdgpu_bo *abo;
  408. struct ttm_mem_reg *old_mem = &bo->mem;
  409. int r;
  410. /* Can't move a pinned BO */
  411. abo = container_of(bo, struct amdgpu_bo, tbo);
  412. if (WARN_ON_ONCE(abo->pin_count > 0))
  413. return -EINVAL;
  414. adev = amdgpu_ttm_adev(bo->bdev);
  415. /* remember the eviction */
  416. if (evict)
  417. atomic64_inc(&adev->num_evictions);
  418. if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
  419. amdgpu_move_null(bo, new_mem);
  420. return 0;
  421. }
  422. if ((old_mem->mem_type == TTM_PL_TT &&
  423. new_mem->mem_type == TTM_PL_SYSTEM) ||
  424. (old_mem->mem_type == TTM_PL_SYSTEM &&
  425. new_mem->mem_type == TTM_PL_TT)) {
  426. /* bind is enough */
  427. amdgpu_move_null(bo, new_mem);
  428. return 0;
  429. }
  430. if (adev->mman.buffer_funcs == NULL ||
  431. adev->mman.buffer_funcs_ring == NULL ||
  432. !adev->mman.buffer_funcs_ring->ready) {
  433. /* use memcpy */
  434. goto memcpy;
  435. }
  436. if (old_mem->mem_type == TTM_PL_VRAM &&
  437. new_mem->mem_type == TTM_PL_SYSTEM) {
  438. r = amdgpu_move_vram_ram(bo, evict, interruptible,
  439. no_wait_gpu, new_mem);
  440. } else if (old_mem->mem_type == TTM_PL_SYSTEM &&
  441. new_mem->mem_type == TTM_PL_VRAM) {
  442. r = amdgpu_move_ram_vram(bo, evict, interruptible,
  443. no_wait_gpu, new_mem);
  444. } else {
  445. r = amdgpu_move_blit(bo, evict, no_wait_gpu, new_mem, old_mem);
  446. }
  447. if (r) {
  448. memcpy:
  449. r = ttm_bo_move_memcpy(bo, interruptible, no_wait_gpu, new_mem);
  450. if (r) {
  451. return r;
  452. }
  453. }
  454. /* update statistics */
  455. atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved);
  456. return 0;
  457. }
  458. static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
  459. {
  460. struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
  461. struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
  462. mem->bus.addr = NULL;
  463. mem->bus.offset = 0;
  464. mem->bus.size = mem->num_pages << PAGE_SHIFT;
  465. mem->bus.base = 0;
  466. mem->bus.is_iomem = false;
  467. if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
  468. return -EINVAL;
  469. switch (mem->mem_type) {
  470. case TTM_PL_SYSTEM:
  471. /* system memory */
  472. return 0;
  473. case TTM_PL_TT:
  474. break;
  475. case TTM_PL_VRAM:
  476. mem->bus.offset = mem->start << PAGE_SHIFT;
  477. /* check if it's visible */
  478. if ((mem->bus.offset + mem->bus.size) > adev->mc.visible_vram_size)
  479. return -EINVAL;
  480. mem->bus.base = adev->mc.aper_base;
  481. mem->bus.is_iomem = true;
  482. #ifdef __alpha__
  483. /*
  484. * Alpha: use bus.addr to hold the ioremap() return,
  485. * so we can modify bus.base below.
  486. */
  487. if (mem->placement & TTM_PL_FLAG_WC)
  488. mem->bus.addr =
  489. ioremap_wc(mem->bus.base + mem->bus.offset,
  490. mem->bus.size);
  491. else
  492. mem->bus.addr =
  493. ioremap_nocache(mem->bus.base + mem->bus.offset,
  494. mem->bus.size);
  495. /*
  496. * Alpha: Use just the bus offset plus
  497. * the hose/domain memory base for bus.base.
  498. * It then can be used to build PTEs for VRAM
  499. * access, as done in ttm_bo_vm_fault().
  500. */
  501. mem->bus.base = (mem->bus.base & 0x0ffffffffUL) +
  502. adev->ddev->hose->dense_mem_base;
  503. #endif
  504. break;
  505. default:
  506. return -EINVAL;
  507. }
  508. return 0;
  509. }
  510. static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
  511. {
  512. }
  513. /*
  514. * TTM backend functions.
  515. */
  516. struct amdgpu_ttm_gup_task_list {
  517. struct list_head list;
  518. struct task_struct *task;
  519. };
  520. struct amdgpu_ttm_tt {
  521. struct ttm_dma_tt ttm;
  522. struct amdgpu_device *adev;
  523. u64 offset;
  524. uint64_t userptr;
  525. struct mm_struct *usermm;
  526. uint32_t userflags;
  527. spinlock_t guptasklock;
  528. struct list_head guptasks;
  529. atomic_t mmu_invalidations;
  530. struct list_head list;
  531. };
  532. int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
  533. {
  534. struct amdgpu_ttm_tt *gtt = (void *)ttm;
  535. unsigned int flags = 0;
  536. unsigned pinned = 0;
  537. int r;
  538. if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
  539. flags |= FOLL_WRITE;
  540. if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) {
  541. /* check that we only use anonymous memory
  542. to prevent problems with writeback */
  543. unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;
  544. struct vm_area_struct *vma;
  545. vma = find_vma(gtt->usermm, gtt->userptr);
  546. if (!vma || vma->vm_file || vma->vm_end < end)
  547. return -EPERM;
  548. }
  549. do {
  550. unsigned num_pages = ttm->num_pages - pinned;
  551. uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE;
  552. struct page **p = pages + pinned;
  553. struct amdgpu_ttm_gup_task_list guptask;
  554. guptask.task = current;
  555. spin_lock(&gtt->guptasklock);
  556. list_add(&guptask.list, &gtt->guptasks);
  557. spin_unlock(&gtt->guptasklock);
  558. r = get_user_pages(userptr, num_pages, flags, p, NULL);
  559. spin_lock(&gtt->guptasklock);
  560. list_del(&guptask.list);
  561. spin_unlock(&gtt->guptasklock);
  562. if (r < 0)
  563. goto release_pages;
  564. pinned += r;
  565. } while (pinned < ttm->num_pages);
  566. return 0;
  567. release_pages:
  568. release_pages(pages, pinned, 0);
  569. return r;
  570. }
  571. /* prepare the sg table with the user pages */
  572. static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
  573. {
  574. struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
  575. struct amdgpu_ttm_tt *gtt = (void *)ttm;
  576. unsigned nents;
  577. int r;
  578. int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
  579. enum dma_data_direction direction = write ?
  580. DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
  581. r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,
  582. ttm->num_pages << PAGE_SHIFT,
  583. GFP_KERNEL);
  584. if (r)
  585. goto release_sg;
  586. r = -ENOMEM;
  587. nents = dma_map_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
  588. if (nents != ttm->sg->nents)
  589. goto release_sg;
  590. drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
  591. gtt->ttm.dma_address, ttm->num_pages);
  592. return 0;
  593. release_sg:
  594. kfree(ttm->sg);
  595. return r;
  596. }
  597. static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
  598. {
  599. struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
  600. struct amdgpu_ttm_tt *gtt = (void *)ttm;
  601. struct sg_page_iter sg_iter;
  602. int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
  603. enum dma_data_direction direction = write ?
  604. DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
  605. /* double check that we don't free the table twice */
  606. if (!ttm->sg->sgl)
  607. return;
  608. /* free the sg table and pages again */
  609. dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
  610. for_each_sg_page(ttm->sg->sgl, &sg_iter, ttm->sg->nents, 0) {
  611. struct page *page = sg_page_iter_page(&sg_iter);
  612. if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
  613. set_page_dirty(page);
  614. mark_page_accessed(page);
  615. put_page(page);
  616. }
  617. sg_free_table(ttm->sg);
  618. }
  619. static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
  620. struct ttm_mem_reg *bo_mem)
  621. {
  622. struct amdgpu_ttm_tt *gtt = (void*)ttm;
  623. int r;
  624. if (gtt->userptr) {
  625. r = amdgpu_ttm_tt_pin_userptr(ttm);
  626. if (r) {
  627. DRM_ERROR("failed to pin userptr\n");
  628. return r;
  629. }
  630. }
  631. if (!ttm->num_pages) {
  632. WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n",
  633. ttm->num_pages, bo_mem, ttm);
  634. }
  635. if (bo_mem->mem_type == AMDGPU_PL_GDS ||
  636. bo_mem->mem_type == AMDGPU_PL_GWS ||
  637. bo_mem->mem_type == AMDGPU_PL_OA)
  638. return -EINVAL;
  639. return 0;
  640. }
  641. bool amdgpu_ttm_is_bound(struct ttm_tt *ttm)
  642. {
  643. struct amdgpu_ttm_tt *gtt = (void *)ttm;
  644. return gtt && !list_empty(&gtt->list);
  645. }
  646. int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem)
  647. {
  648. struct ttm_tt *ttm = bo->ttm;
  649. struct amdgpu_ttm_tt *gtt = (void *)bo->ttm;
  650. uint32_t flags;
  651. int r;
  652. if (!ttm || amdgpu_ttm_is_bound(ttm))
  653. return 0;
  654. r = amdgpu_gtt_mgr_alloc(&bo->bdev->man[TTM_PL_TT], bo,
  655. NULL, bo_mem);
  656. if (r) {
  657. DRM_ERROR("Failed to allocate GTT address space (%d)\n", r);
  658. return r;
  659. }
  660. flags = amdgpu_ttm_tt_pte_flags(gtt->adev, ttm, bo_mem);
  661. gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
  662. r = amdgpu_gart_bind(gtt->adev, gtt->offset, ttm->num_pages,
  663. ttm->pages, gtt->ttm.dma_address, flags);
  664. if (r) {
  665. DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
  666. ttm->num_pages, gtt->offset);
  667. return r;
  668. }
  669. spin_lock(&gtt->adev->gtt_list_lock);
  670. list_add_tail(&gtt->list, &gtt->adev->gtt_list);
  671. spin_unlock(&gtt->adev->gtt_list_lock);
  672. return 0;
  673. }
  674. int amdgpu_ttm_recover_gart(struct amdgpu_device *adev)
  675. {
  676. struct amdgpu_ttm_tt *gtt, *tmp;
  677. struct ttm_mem_reg bo_mem;
  678. uint32_t flags;
  679. int r;
  680. bo_mem.mem_type = TTM_PL_TT;
  681. spin_lock(&adev->gtt_list_lock);
  682. list_for_each_entry_safe(gtt, tmp, &adev->gtt_list, list) {
  683. flags = amdgpu_ttm_tt_pte_flags(gtt->adev, &gtt->ttm.ttm, &bo_mem);
  684. r = amdgpu_gart_bind(adev, gtt->offset, gtt->ttm.ttm.num_pages,
  685. gtt->ttm.ttm.pages, gtt->ttm.dma_address,
  686. flags);
  687. if (r) {
  688. spin_unlock(&adev->gtt_list_lock);
  689. DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
  690. gtt->ttm.ttm.num_pages, gtt->offset);
  691. return r;
  692. }
  693. }
  694. spin_unlock(&adev->gtt_list_lock);
  695. return 0;
  696. }
  697. static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
  698. {
  699. struct amdgpu_ttm_tt *gtt = (void *)ttm;
  700. if (gtt->userptr)
  701. amdgpu_ttm_tt_unpin_userptr(ttm);
  702. if (!amdgpu_ttm_is_bound(ttm))
  703. return 0;
  704. /* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */
  705. if (gtt->adev->gart.ready)
  706. amdgpu_gart_unbind(gtt->adev, gtt->offset, ttm->num_pages);
  707. spin_lock(&gtt->adev->gtt_list_lock);
  708. list_del_init(&gtt->list);
  709. spin_unlock(&gtt->adev->gtt_list_lock);
  710. return 0;
  711. }
  712. static void amdgpu_ttm_backend_destroy(struct ttm_tt *ttm)
  713. {
  714. struct amdgpu_ttm_tt *gtt = (void *)ttm;
  715. ttm_dma_tt_fini(&gtt->ttm);
  716. kfree(gtt);
  717. }
  718. static struct ttm_backend_func amdgpu_backend_func = {
  719. .bind = &amdgpu_ttm_backend_bind,
  720. .unbind = &amdgpu_ttm_backend_unbind,
  721. .destroy = &amdgpu_ttm_backend_destroy,
  722. };
  723. static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_bo_device *bdev,
  724. unsigned long size, uint32_t page_flags,
  725. struct page *dummy_read_page)
  726. {
  727. struct amdgpu_device *adev;
  728. struct amdgpu_ttm_tt *gtt;
  729. adev = amdgpu_ttm_adev(bdev);
  730. gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL);
  731. if (gtt == NULL) {
  732. return NULL;
  733. }
  734. gtt->ttm.ttm.func = &amdgpu_backend_func;
  735. gtt->adev = adev;
  736. if (ttm_dma_tt_init(&gtt->ttm, bdev, size, page_flags, dummy_read_page)) {
  737. kfree(gtt);
  738. return NULL;
  739. }
  740. INIT_LIST_HEAD(&gtt->list);
  741. return &gtt->ttm.ttm;
  742. }
  743. static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm)
  744. {
  745. struct amdgpu_device *adev;
  746. struct amdgpu_ttm_tt *gtt = (void *)ttm;
  747. unsigned i;
  748. int r;
  749. bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
  750. if (ttm->state != tt_unpopulated)
  751. return 0;
  752. if (gtt && gtt->userptr) {
  753. ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
  754. if (!ttm->sg)
  755. return -ENOMEM;
  756. ttm->page_flags |= TTM_PAGE_FLAG_SG;
  757. ttm->state = tt_unbound;
  758. return 0;
  759. }
  760. if (slave && ttm->sg) {
  761. drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
  762. gtt->ttm.dma_address, ttm->num_pages);
  763. ttm->state = tt_unbound;
  764. return 0;
  765. }
  766. adev = amdgpu_ttm_adev(ttm->bdev);
  767. #ifdef CONFIG_SWIOTLB
  768. if (swiotlb_nr_tbl()) {
  769. return ttm_dma_populate(&gtt->ttm, adev->dev);
  770. }
  771. #endif
  772. r = ttm_pool_populate(ttm);
  773. if (r) {
  774. return r;
  775. }
  776. for (i = 0; i < ttm->num_pages; i++) {
  777. gtt->ttm.dma_address[i] = pci_map_page(adev->pdev, ttm->pages[i],
  778. 0, PAGE_SIZE,
  779. PCI_DMA_BIDIRECTIONAL);
  780. if (pci_dma_mapping_error(adev->pdev, gtt->ttm.dma_address[i])) {
  781. while (i--) {
  782. pci_unmap_page(adev->pdev, gtt->ttm.dma_address[i],
  783. PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
  784. gtt->ttm.dma_address[i] = 0;
  785. }
  786. ttm_pool_unpopulate(ttm);
  787. return -EFAULT;
  788. }
  789. }
  790. return 0;
  791. }
  792. static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
  793. {
  794. struct amdgpu_device *adev;
  795. struct amdgpu_ttm_tt *gtt = (void *)ttm;
  796. unsigned i;
  797. bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
  798. if (gtt && gtt->userptr) {
  799. kfree(ttm->sg);
  800. ttm->page_flags &= ~TTM_PAGE_FLAG_SG;
  801. return;
  802. }
  803. if (slave)
  804. return;
  805. adev = amdgpu_ttm_adev(ttm->bdev);
  806. #ifdef CONFIG_SWIOTLB
  807. if (swiotlb_nr_tbl()) {
  808. ttm_dma_unpopulate(&gtt->ttm, adev->dev);
  809. return;
  810. }
  811. #endif
  812. for (i = 0; i < ttm->num_pages; i++) {
  813. if (gtt->ttm.dma_address[i]) {
  814. pci_unmap_page(adev->pdev, gtt->ttm.dma_address[i],
  815. PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
  816. }
  817. }
  818. ttm_pool_unpopulate(ttm);
  819. }
  820. int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
  821. uint32_t flags)
  822. {
  823. struct amdgpu_ttm_tt *gtt = (void *)ttm;
  824. if (gtt == NULL)
  825. return -EINVAL;
  826. gtt->userptr = addr;
  827. gtt->usermm = current->mm;
  828. gtt->userflags = flags;
  829. spin_lock_init(&gtt->guptasklock);
  830. INIT_LIST_HEAD(&gtt->guptasks);
  831. atomic_set(&gtt->mmu_invalidations, 0);
  832. return 0;
  833. }
  834. struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)
  835. {
  836. struct amdgpu_ttm_tt *gtt = (void *)ttm;
  837. if (gtt == NULL)
  838. return NULL;
  839. return gtt->usermm;
  840. }
  841. bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
  842. unsigned long end)
  843. {
  844. struct amdgpu_ttm_tt *gtt = (void *)ttm;
  845. struct amdgpu_ttm_gup_task_list *entry;
  846. unsigned long size;
  847. if (gtt == NULL || !gtt->userptr)
  848. return false;
  849. size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE;
  850. if (gtt->userptr > end || gtt->userptr + size <= start)
  851. return false;
  852. spin_lock(&gtt->guptasklock);
  853. list_for_each_entry(entry, &gtt->guptasks, list) {
  854. if (entry->task == current) {
  855. spin_unlock(&gtt->guptasklock);
  856. return false;
  857. }
  858. }
  859. spin_unlock(&gtt->guptasklock);
  860. atomic_inc(&gtt->mmu_invalidations);
  861. return true;
  862. }
  863. bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
  864. int *last_invalidated)
  865. {
  866. struct amdgpu_ttm_tt *gtt = (void *)ttm;
  867. int prev_invalidated = *last_invalidated;
  868. *last_invalidated = atomic_read(&gtt->mmu_invalidations);
  869. return prev_invalidated != *last_invalidated;
  870. }
  871. bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
  872. {
  873. struct amdgpu_ttm_tt *gtt = (void *)ttm;
  874. if (gtt == NULL)
  875. return false;
  876. return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
  877. }
  878. uint32_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
  879. struct ttm_mem_reg *mem)
  880. {
  881. uint32_t flags = 0;
  882. if (mem && mem->mem_type != TTM_PL_SYSTEM)
  883. flags |= AMDGPU_PTE_VALID;
  884. if (mem && mem->mem_type == TTM_PL_TT) {
  885. flags |= AMDGPU_PTE_SYSTEM;
  886. if (ttm->caching_state == tt_cached)
  887. flags |= AMDGPU_PTE_SNOOPED;
  888. }
  889. if (adev->asic_type >= CHIP_TONGA)
  890. flags |= AMDGPU_PTE_EXECUTABLE;
  891. flags |= AMDGPU_PTE_READABLE;
  892. if (!amdgpu_ttm_tt_is_readonly(ttm))
  893. flags |= AMDGPU_PTE_WRITEABLE;
  894. return flags;
  895. }
  896. static void amdgpu_ttm_lru_removal(struct ttm_buffer_object *tbo)
  897. {
  898. struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
  899. unsigned i, j;
  900. for (i = 0; i < AMDGPU_TTM_LRU_SIZE; ++i) {
  901. struct amdgpu_mman_lru *lru = &adev->mman.log2_size[i];
  902. for (j = 0; j < TTM_NUM_MEM_TYPES; ++j)
  903. if (&tbo->lru == lru->lru[j])
  904. lru->lru[j] = tbo->lru.prev;
  905. if (&tbo->swap == lru->swap_lru)
  906. lru->swap_lru = tbo->swap.prev;
  907. }
  908. }
  909. static struct amdgpu_mman_lru *amdgpu_ttm_lru(struct ttm_buffer_object *tbo)
  910. {
  911. struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
  912. unsigned log2_size = min(ilog2(tbo->num_pages),
  913. AMDGPU_TTM_LRU_SIZE - 1);
  914. return &adev->mman.log2_size[log2_size];
  915. }
  916. static struct list_head *amdgpu_ttm_lru_tail(struct ttm_buffer_object *tbo)
  917. {
  918. struct amdgpu_mman_lru *lru = amdgpu_ttm_lru(tbo);
  919. struct list_head *res = lru->lru[tbo->mem.mem_type];
  920. lru->lru[tbo->mem.mem_type] = &tbo->lru;
  921. while ((++lru)->lru[tbo->mem.mem_type] == res)
  922. lru->lru[tbo->mem.mem_type] = &tbo->lru;
  923. return res;
  924. }
  925. static struct list_head *amdgpu_ttm_swap_lru_tail(struct ttm_buffer_object *tbo)
  926. {
  927. struct amdgpu_mman_lru *lru = amdgpu_ttm_lru(tbo);
  928. struct list_head *res = lru->swap_lru;
  929. lru->swap_lru = &tbo->swap;
  930. while ((++lru)->swap_lru == res)
  931. lru->swap_lru = &tbo->swap;
  932. return res;
  933. }
  934. static struct ttm_bo_driver amdgpu_bo_driver = {
  935. .ttm_tt_create = &amdgpu_ttm_tt_create,
  936. .ttm_tt_populate = &amdgpu_ttm_tt_populate,
  937. .ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate,
  938. .invalidate_caches = &amdgpu_invalidate_caches,
  939. .init_mem_type = &amdgpu_init_mem_type,
  940. .evict_flags = &amdgpu_evict_flags,
  941. .move = &amdgpu_bo_move,
  942. .verify_access = &amdgpu_verify_access,
  943. .move_notify = &amdgpu_bo_move_notify,
  944. .fault_reserve_notify = &amdgpu_bo_fault_reserve_notify,
  945. .io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
  946. .io_mem_free = &amdgpu_ttm_io_mem_free,
  947. .lru_removal = &amdgpu_ttm_lru_removal,
  948. .lru_tail = &amdgpu_ttm_lru_tail,
  949. .swap_lru_tail = &amdgpu_ttm_swap_lru_tail,
  950. };
  951. int amdgpu_ttm_init(struct amdgpu_device *adev)
  952. {
  953. unsigned i, j;
  954. int r;
  955. /* No others user of address space so set it to 0 */
  956. r = ttm_bo_device_init(&adev->mman.bdev,
  957. adev->mman.bo_global_ref.ref.object,
  958. &amdgpu_bo_driver,
  959. adev->ddev->anon_inode->i_mapping,
  960. DRM_FILE_PAGE_OFFSET,
  961. adev->need_dma32);
  962. if (r) {
  963. DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
  964. return r;
  965. }
  966. for (i = 0; i < AMDGPU_TTM_LRU_SIZE; ++i) {
  967. struct amdgpu_mman_lru *lru = &adev->mman.log2_size[i];
  968. for (j = 0; j < TTM_NUM_MEM_TYPES; ++j)
  969. lru->lru[j] = &adev->mman.bdev.man[j].lru;
  970. lru->swap_lru = &adev->mman.bdev.glob->swap_lru;
  971. }
  972. for (j = 0; j < TTM_NUM_MEM_TYPES; ++j)
  973. adev->mman.guard.lru[j] = NULL;
  974. adev->mman.guard.swap_lru = NULL;
  975. adev->mman.initialized = true;
  976. r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_VRAM,
  977. adev->mc.real_vram_size >> PAGE_SHIFT);
  978. if (r) {
  979. DRM_ERROR("Failed initializing VRAM heap.\n");
  980. return r;
  981. }
  982. /* Change the size here instead of the init above so only lpfn is affected */
  983. amdgpu_ttm_set_active_vram_size(adev, adev->mc.visible_vram_size);
  984. r = amdgpu_bo_create(adev, 256 * 1024, PAGE_SIZE, true,
  985. AMDGPU_GEM_DOMAIN_VRAM,
  986. AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
  987. AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
  988. NULL, NULL, &adev->stollen_vga_memory);
  989. if (r) {
  990. return r;
  991. }
  992. r = amdgpu_bo_reserve(adev->stollen_vga_memory, false);
  993. if (r)
  994. return r;
  995. r = amdgpu_bo_pin(adev->stollen_vga_memory, AMDGPU_GEM_DOMAIN_VRAM, NULL);
  996. amdgpu_bo_unreserve(adev->stollen_vga_memory);
  997. if (r) {
  998. amdgpu_bo_unref(&adev->stollen_vga_memory);
  999. return r;
  1000. }
  1001. DRM_INFO("amdgpu: %uM of VRAM memory ready\n",
  1002. (unsigned) (adev->mc.real_vram_size / (1024 * 1024)));
  1003. r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT,
  1004. adev->mc.gtt_size >> PAGE_SHIFT);
  1005. if (r) {
  1006. DRM_ERROR("Failed initializing GTT heap.\n");
  1007. return r;
  1008. }
  1009. DRM_INFO("amdgpu: %uM of GTT memory ready.\n",
  1010. (unsigned)(adev->mc.gtt_size / (1024 * 1024)));
  1011. adev->gds.mem.total_size = adev->gds.mem.total_size << AMDGPU_GDS_SHIFT;
  1012. adev->gds.mem.gfx_partition_size = adev->gds.mem.gfx_partition_size << AMDGPU_GDS_SHIFT;
  1013. adev->gds.mem.cs_partition_size = adev->gds.mem.cs_partition_size << AMDGPU_GDS_SHIFT;
  1014. adev->gds.gws.total_size = adev->gds.gws.total_size << AMDGPU_GWS_SHIFT;
  1015. adev->gds.gws.gfx_partition_size = adev->gds.gws.gfx_partition_size << AMDGPU_GWS_SHIFT;
  1016. adev->gds.gws.cs_partition_size = adev->gds.gws.cs_partition_size << AMDGPU_GWS_SHIFT;
  1017. adev->gds.oa.total_size = adev->gds.oa.total_size << AMDGPU_OA_SHIFT;
  1018. adev->gds.oa.gfx_partition_size = adev->gds.oa.gfx_partition_size << AMDGPU_OA_SHIFT;
  1019. adev->gds.oa.cs_partition_size = adev->gds.oa.cs_partition_size << AMDGPU_OA_SHIFT;
  1020. /* GDS Memory */
  1021. r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GDS,
  1022. adev->gds.mem.total_size >> PAGE_SHIFT);
  1023. if (r) {
  1024. DRM_ERROR("Failed initializing GDS heap.\n");
  1025. return r;
  1026. }
  1027. /* GWS */
  1028. r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GWS,
  1029. adev->gds.gws.total_size >> PAGE_SHIFT);
  1030. if (r) {
  1031. DRM_ERROR("Failed initializing gws heap.\n");
  1032. return r;
  1033. }
  1034. /* OA */
  1035. r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_OA,
  1036. adev->gds.oa.total_size >> PAGE_SHIFT);
  1037. if (r) {
  1038. DRM_ERROR("Failed initializing oa heap.\n");
  1039. return r;
  1040. }
  1041. r = amdgpu_ttm_debugfs_init(adev);
  1042. if (r) {
  1043. DRM_ERROR("Failed to init debugfs\n");
  1044. return r;
  1045. }
  1046. return 0;
  1047. }
  1048. void amdgpu_ttm_fini(struct amdgpu_device *adev)
  1049. {
  1050. int r;
  1051. if (!adev->mman.initialized)
  1052. return;
  1053. amdgpu_ttm_debugfs_fini(adev);
  1054. if (adev->stollen_vga_memory) {
  1055. r = amdgpu_bo_reserve(adev->stollen_vga_memory, false);
  1056. if (r == 0) {
  1057. amdgpu_bo_unpin(adev->stollen_vga_memory);
  1058. amdgpu_bo_unreserve(adev->stollen_vga_memory);
  1059. }
  1060. amdgpu_bo_unref(&adev->stollen_vga_memory);
  1061. }
  1062. ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_VRAM);
  1063. ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_TT);
  1064. ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GDS);
  1065. ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GWS);
  1066. ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_OA);
  1067. ttm_bo_device_release(&adev->mman.bdev);
  1068. amdgpu_gart_fini(adev);
  1069. amdgpu_ttm_global_fini(adev);
  1070. adev->mman.initialized = false;
  1071. DRM_INFO("amdgpu: ttm finalized\n");
  1072. }
  1073. /* this should only be called at bootup or when userspace
  1074. * isn't running */
  1075. void amdgpu_ttm_set_active_vram_size(struct amdgpu_device *adev, u64 size)
  1076. {
  1077. struct ttm_mem_type_manager *man;
  1078. if (!adev->mman.initialized)
  1079. return;
  1080. man = &adev->mman.bdev.man[TTM_PL_VRAM];
  1081. /* this just adjusts TTM size idea, which sets lpfn to the correct value */
  1082. man->size = size >> PAGE_SHIFT;
  1083. }
  1084. int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma)
  1085. {
  1086. struct drm_file *file_priv;
  1087. struct amdgpu_device *adev;
  1088. if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
  1089. return -EINVAL;
  1090. file_priv = filp->private_data;
  1091. adev = file_priv->minor->dev->dev_private;
  1092. if (adev == NULL)
  1093. return -EINVAL;
  1094. return ttm_bo_mmap(filp, vma, &adev->mman.bdev);
  1095. }
  1096. int amdgpu_copy_buffer(struct amdgpu_ring *ring,
  1097. uint64_t src_offset,
  1098. uint64_t dst_offset,
  1099. uint32_t byte_count,
  1100. struct reservation_object *resv,
  1101. struct fence **fence, bool direct_submit)
  1102. {
  1103. struct amdgpu_device *adev = ring->adev;
  1104. struct amdgpu_job *job;
  1105. uint32_t max_bytes;
  1106. unsigned num_loops, num_dw;
  1107. unsigned i;
  1108. int r;
  1109. max_bytes = adev->mman.buffer_funcs->copy_max_bytes;
  1110. num_loops = DIV_ROUND_UP(byte_count, max_bytes);
  1111. num_dw = num_loops * adev->mman.buffer_funcs->copy_num_dw;
  1112. /* for IB padding */
  1113. while (num_dw & 0x7)
  1114. num_dw++;
  1115. r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
  1116. if (r)
  1117. return r;
  1118. if (resv) {
  1119. r = amdgpu_sync_resv(adev, &job->sync, resv,
  1120. AMDGPU_FENCE_OWNER_UNDEFINED);
  1121. if (r) {
  1122. DRM_ERROR("sync failed (%d).\n", r);
  1123. goto error_free;
  1124. }
  1125. }
  1126. for (i = 0; i < num_loops; i++) {
  1127. uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
  1128. amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset,
  1129. dst_offset, cur_size_in_bytes);
  1130. src_offset += cur_size_in_bytes;
  1131. dst_offset += cur_size_in_bytes;
  1132. byte_count -= cur_size_in_bytes;
  1133. }
  1134. amdgpu_ring_pad_ib(ring, &job->ibs[0]);
  1135. WARN_ON(job->ibs[0].length_dw > num_dw);
  1136. if (direct_submit) {
  1137. r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs,
  1138. NULL, NULL, fence);
  1139. job->fence = fence_get(*fence);
  1140. if (r)
  1141. DRM_ERROR("Error scheduling IBs (%d)\n", r);
  1142. amdgpu_job_free(job);
  1143. } else {
  1144. r = amdgpu_job_submit(job, ring, &adev->mman.entity,
  1145. AMDGPU_FENCE_OWNER_UNDEFINED, fence);
  1146. if (r)
  1147. goto error_free;
  1148. }
  1149. return r;
  1150. error_free:
  1151. amdgpu_job_free(job);
  1152. return r;
  1153. }
  1154. int amdgpu_fill_buffer(struct amdgpu_bo *bo,
  1155. uint32_t src_data,
  1156. struct reservation_object *resv,
  1157. struct fence **fence)
  1158. {
  1159. struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
  1160. struct amdgpu_job *job;
  1161. struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
  1162. uint32_t max_bytes, byte_count;
  1163. uint64_t dst_offset;
  1164. unsigned int num_loops, num_dw;
  1165. unsigned int i;
  1166. int r;
  1167. byte_count = bo->tbo.num_pages << PAGE_SHIFT;
  1168. max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
  1169. num_loops = DIV_ROUND_UP(byte_count, max_bytes);
  1170. num_dw = num_loops * adev->mman.buffer_funcs->fill_num_dw;
  1171. /* for IB padding */
  1172. while (num_dw & 0x7)
  1173. num_dw++;
  1174. r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
  1175. if (r)
  1176. return r;
  1177. if (resv) {
  1178. r = amdgpu_sync_resv(adev, &job->sync, resv,
  1179. AMDGPU_FENCE_OWNER_UNDEFINED);
  1180. if (r) {
  1181. DRM_ERROR("sync failed (%d).\n", r);
  1182. goto error_free;
  1183. }
  1184. }
  1185. dst_offset = bo->tbo.mem.start << PAGE_SHIFT;
  1186. for (i = 0; i < num_loops; i++) {
  1187. uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
  1188. amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data,
  1189. dst_offset, cur_size_in_bytes);
  1190. dst_offset += cur_size_in_bytes;
  1191. byte_count -= cur_size_in_bytes;
  1192. }
  1193. amdgpu_ring_pad_ib(ring, &job->ibs[0]);
  1194. WARN_ON(job->ibs[0].length_dw > num_dw);
  1195. r = amdgpu_job_submit(job, ring, &adev->mman.entity,
  1196. AMDGPU_FENCE_OWNER_UNDEFINED, fence);
  1197. if (r)
  1198. goto error_free;
  1199. return 0;
  1200. error_free:
  1201. amdgpu_job_free(job);
  1202. return r;
  1203. }
  1204. #if defined(CONFIG_DEBUG_FS)
  1205. static int amdgpu_mm_dump_table(struct seq_file *m, void *data)
  1206. {
  1207. struct drm_info_node *node = (struct drm_info_node *)m->private;
  1208. unsigned ttm_pl = *(int *)node->info_ent->data;
  1209. struct drm_device *dev = node->minor->dev;
  1210. struct amdgpu_device *adev = dev->dev_private;
  1211. struct drm_mm *mm = (struct drm_mm *)adev->mman.bdev.man[ttm_pl].priv;
  1212. int ret;
  1213. struct ttm_bo_global *glob = adev->mman.bdev.glob;
  1214. spin_lock(&glob->lru_lock);
  1215. ret = drm_mm_dump_table(m, mm);
  1216. spin_unlock(&glob->lru_lock);
  1217. if (ttm_pl == TTM_PL_VRAM)
  1218. seq_printf(m, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n",
  1219. adev->mman.bdev.man[ttm_pl].size,
  1220. (u64)atomic64_read(&adev->vram_usage) >> 20,
  1221. (u64)atomic64_read(&adev->vram_vis_usage) >> 20);
  1222. return ret;
  1223. }
  1224. static int ttm_pl_vram = TTM_PL_VRAM;
  1225. static int ttm_pl_tt = TTM_PL_TT;
  1226. static const struct drm_info_list amdgpu_ttm_debugfs_list[] = {
  1227. {"amdgpu_vram_mm", amdgpu_mm_dump_table, 0, &ttm_pl_vram},
  1228. {"amdgpu_gtt_mm", amdgpu_mm_dump_table, 0, &ttm_pl_tt},
  1229. {"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL},
  1230. #ifdef CONFIG_SWIOTLB
  1231. {"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL}
  1232. #endif
  1233. };
  1234. static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
  1235. size_t size, loff_t *pos)
  1236. {
  1237. struct amdgpu_device *adev = f->f_inode->i_private;
  1238. ssize_t result = 0;
  1239. int r;
  1240. if (size & 0x3 || *pos & 0x3)
  1241. return -EINVAL;
  1242. while (size) {
  1243. unsigned long flags;
  1244. uint32_t value;
  1245. if (*pos >= adev->mc.mc_vram_size)
  1246. return result;
  1247. spin_lock_irqsave(&adev->mmio_idx_lock, flags);
  1248. WREG32(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
  1249. WREG32(mmMM_INDEX_HI, *pos >> 31);
  1250. value = RREG32(mmMM_DATA);
  1251. spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
  1252. r = put_user(value, (uint32_t *)buf);
  1253. if (r)
  1254. return r;
  1255. result += 4;
  1256. buf += 4;
  1257. *pos += 4;
  1258. size -= 4;
  1259. }
  1260. return result;
  1261. }
  1262. static const struct file_operations amdgpu_ttm_vram_fops = {
  1263. .owner = THIS_MODULE,
  1264. .read = amdgpu_ttm_vram_read,
  1265. .llseek = default_llseek
  1266. };
  1267. #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
  1268. static ssize_t amdgpu_ttm_gtt_read(struct file *f, char __user *buf,
  1269. size_t size, loff_t *pos)
  1270. {
  1271. struct amdgpu_device *adev = f->f_inode->i_private;
  1272. ssize_t result = 0;
  1273. int r;
  1274. while (size) {
  1275. loff_t p = *pos / PAGE_SIZE;
  1276. unsigned off = *pos & ~PAGE_MASK;
  1277. size_t cur_size = min_t(size_t, size, PAGE_SIZE - off);
  1278. struct page *page;
  1279. void *ptr;
  1280. if (p >= adev->gart.num_cpu_pages)
  1281. return result;
  1282. page = adev->gart.pages[p];
  1283. if (page) {
  1284. ptr = kmap(page);
  1285. ptr += off;
  1286. r = copy_to_user(buf, ptr, cur_size);
  1287. kunmap(adev->gart.pages[p]);
  1288. } else
  1289. r = clear_user(buf, cur_size);
  1290. if (r)
  1291. return -EFAULT;
  1292. result += cur_size;
  1293. buf += cur_size;
  1294. *pos += cur_size;
  1295. size -= cur_size;
  1296. }
  1297. return result;
  1298. }
  1299. static const struct file_operations amdgpu_ttm_gtt_fops = {
  1300. .owner = THIS_MODULE,
  1301. .read = amdgpu_ttm_gtt_read,
  1302. .llseek = default_llseek
  1303. };
  1304. #endif
  1305. #endif
  1306. static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
  1307. {
  1308. #if defined(CONFIG_DEBUG_FS)
  1309. unsigned count;
  1310. struct drm_minor *minor = adev->ddev->primary;
  1311. struct dentry *ent, *root = minor->debugfs_root;
  1312. ent = debugfs_create_file("amdgpu_vram", S_IFREG | S_IRUGO, root,
  1313. adev, &amdgpu_ttm_vram_fops);
  1314. if (IS_ERR(ent))
  1315. return PTR_ERR(ent);
  1316. i_size_write(ent->d_inode, adev->mc.mc_vram_size);
  1317. adev->mman.vram = ent;
  1318. #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
  1319. ent = debugfs_create_file("amdgpu_gtt", S_IFREG | S_IRUGO, root,
  1320. adev, &amdgpu_ttm_gtt_fops);
  1321. if (IS_ERR(ent))
  1322. return PTR_ERR(ent);
  1323. i_size_write(ent->d_inode, adev->mc.gtt_size);
  1324. adev->mman.gtt = ent;
  1325. #endif
  1326. count = ARRAY_SIZE(amdgpu_ttm_debugfs_list);
  1327. #ifdef CONFIG_SWIOTLB
  1328. if (!swiotlb_nr_tbl())
  1329. --count;
  1330. #endif
  1331. return amdgpu_debugfs_add_files(adev, amdgpu_ttm_debugfs_list, count);
  1332. #else
  1333. return 0;
  1334. #endif
  1335. }
  1336. static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev)
  1337. {
  1338. #if defined(CONFIG_DEBUG_FS)
  1339. debugfs_remove(adev->mman.vram);
  1340. adev->mman.vram = NULL;
  1341. #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
  1342. debugfs_remove(adev->mman.gtt);
  1343. adev->mman.gtt = NULL;
  1344. #endif
  1345. #endif
  1346. }
  1347. u64 amdgpu_ttm_get_gtt_mem_size(struct amdgpu_device *adev)
  1348. {
  1349. return ttm_get_kernel_zone_memory_size(adev->mman.mem_global_ref.object);
  1350. }