amdgpu_object.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832
  1. /*
  2. * Copyright 2009 Jerome Glisse.
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining a
  6. * copy of this software and associated documentation files (the
  7. * "Software"), to deal in the Software without restriction, including
  8. * without limitation the rights to use, copy, modify, merge, publish,
  9. * distribute, sub license, and/or sell copies of the Software, and to
  10. * permit persons to whom the Software is furnished to do so, subject to
  11. * the following conditions:
  12. *
  13. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15. * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16. * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17. * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18. * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19. * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20. *
  21. * The above copyright notice and this permission notice (including the
  22. * next paragraph) shall be included in all copies or substantial portions
  23. * of the Software.
  24. *
  25. */
  26. /*
  27. * Authors:
  28. * Jerome Glisse <glisse@freedesktop.org>
  29. * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
  30. * Dave Airlie
  31. */
  32. #include <linux/list.h>
  33. #include <linux/slab.h>
  34. #include <drm/drmP.h>
  35. #include <drm/amdgpu_drm.h>
  36. #include <drm/drm_cache.h>
  37. #include "amdgpu.h"
  38. #include "amdgpu_trace.h"
  39. int amdgpu_ttm_init(struct amdgpu_device *adev);
  40. void amdgpu_ttm_fini(struct amdgpu_device *adev);
  41. static u64 amdgpu_get_vis_part_size(struct amdgpu_device *adev,
  42. struct ttm_mem_reg *mem)
  43. {
  44. u64 ret = 0;
  45. if (mem->start << PAGE_SHIFT < adev->mc.visible_vram_size) {
  46. ret = (u64)((mem->start << PAGE_SHIFT) + mem->size) >
  47. adev->mc.visible_vram_size ?
  48. adev->mc.visible_vram_size - (mem->start << PAGE_SHIFT) :
  49. mem->size;
  50. }
  51. return ret;
  52. }
  53. static void amdgpu_update_memory_usage(struct amdgpu_device *adev,
  54. struct ttm_mem_reg *old_mem,
  55. struct ttm_mem_reg *new_mem)
  56. {
  57. u64 vis_size;
  58. if (!adev)
  59. return;
  60. if (new_mem) {
  61. switch (new_mem->mem_type) {
  62. case TTM_PL_TT:
  63. atomic64_add(new_mem->size, &adev->gtt_usage);
  64. break;
  65. case TTM_PL_VRAM:
  66. atomic64_add(new_mem->size, &adev->vram_usage);
  67. vis_size = amdgpu_get_vis_part_size(adev, new_mem);
  68. atomic64_add(vis_size, &adev->vram_vis_usage);
  69. break;
  70. }
  71. }
  72. if (old_mem) {
  73. switch (old_mem->mem_type) {
  74. case TTM_PL_TT:
  75. atomic64_sub(old_mem->size, &adev->gtt_usage);
  76. break;
  77. case TTM_PL_VRAM:
  78. atomic64_sub(old_mem->size, &adev->vram_usage);
  79. vis_size = amdgpu_get_vis_part_size(adev, old_mem);
  80. atomic64_sub(vis_size, &adev->vram_vis_usage);
  81. break;
  82. }
  83. }
  84. }
  85. static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
  86. {
  87. struct amdgpu_bo *bo;
  88. bo = container_of(tbo, struct amdgpu_bo, tbo);
  89. amdgpu_update_memory_usage(bo->adev, &bo->tbo.mem, NULL);
  90. drm_gem_object_release(&bo->gem_base);
  91. amdgpu_bo_unref(&bo->parent);
  92. kfree(bo->metadata);
  93. kfree(bo);
  94. }
  95. bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
  96. {
  97. if (bo->destroy == &amdgpu_ttm_bo_destroy)
  98. return true;
  99. return false;
  100. }
  101. static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
  102. struct ttm_placement *placement,
  103. struct ttm_place *placements,
  104. u32 domain, u64 flags)
  105. {
  106. u32 c = 0, i;
  107. placement->placement = placements;
  108. placement->busy_placement = placements;
  109. if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
  110. if (flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS &&
  111. adev->mc.visible_vram_size < adev->mc.real_vram_size) {
  112. placements[c].fpfn =
  113. adev->mc.visible_vram_size >> PAGE_SHIFT;
  114. placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
  115. TTM_PL_FLAG_VRAM | TTM_PL_FLAG_TOPDOWN;
  116. }
  117. placements[c].fpfn = 0;
  118. placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
  119. TTM_PL_FLAG_VRAM;
  120. if (!(flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED))
  121. placements[c - 1].flags |= TTM_PL_FLAG_TOPDOWN;
  122. }
  123. if (domain & AMDGPU_GEM_DOMAIN_GTT) {
  124. if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) {
  125. placements[c].fpfn = 0;
  126. placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_TT |
  127. TTM_PL_FLAG_UNCACHED;
  128. } else {
  129. placements[c].fpfn = 0;
  130. placements[c++].flags = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_TT;
  131. }
  132. }
  133. if (domain & AMDGPU_GEM_DOMAIN_CPU) {
  134. if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) {
  135. placements[c].fpfn = 0;
  136. placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_SYSTEM |
  137. TTM_PL_FLAG_UNCACHED;
  138. } else {
  139. placements[c].fpfn = 0;
  140. placements[c++].flags = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM;
  141. }
  142. }
  143. if (domain & AMDGPU_GEM_DOMAIN_GDS) {
  144. placements[c].fpfn = 0;
  145. placements[c++].flags = TTM_PL_FLAG_UNCACHED |
  146. AMDGPU_PL_FLAG_GDS;
  147. }
  148. if (domain & AMDGPU_GEM_DOMAIN_GWS) {
  149. placements[c].fpfn = 0;
  150. placements[c++].flags = TTM_PL_FLAG_UNCACHED |
  151. AMDGPU_PL_FLAG_GWS;
  152. }
  153. if (domain & AMDGPU_GEM_DOMAIN_OA) {
  154. placements[c].fpfn = 0;
  155. placements[c++].flags = TTM_PL_FLAG_UNCACHED |
  156. AMDGPU_PL_FLAG_OA;
  157. }
  158. if (!c) {
  159. placements[c].fpfn = 0;
  160. placements[c++].flags = TTM_PL_MASK_CACHING |
  161. TTM_PL_FLAG_SYSTEM;
  162. }
  163. placement->num_placement = c;
  164. placement->num_busy_placement = c;
  165. for (i = 0; i < c; i++) {
  166. if ((flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
  167. (placements[i].flags & TTM_PL_FLAG_VRAM) &&
  168. !placements[i].fpfn)
  169. placements[i].lpfn =
  170. adev->mc.visible_vram_size >> PAGE_SHIFT;
  171. else
  172. placements[i].lpfn = 0;
  173. }
  174. }
  175. void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *rbo, u32 domain)
  176. {
  177. amdgpu_ttm_placement_init(rbo->adev, &rbo->placement,
  178. rbo->placements, domain, rbo->flags);
  179. }
  180. static void amdgpu_fill_placement_to_bo(struct amdgpu_bo *bo,
  181. struct ttm_placement *placement)
  182. {
  183. BUG_ON(placement->num_placement > (AMDGPU_GEM_DOMAIN_MAX + 1));
  184. memcpy(bo->placements, placement->placement,
  185. placement->num_placement * sizeof(struct ttm_place));
  186. bo->placement.num_placement = placement->num_placement;
  187. bo->placement.num_busy_placement = placement->num_busy_placement;
  188. bo->placement.placement = bo->placements;
  189. bo->placement.busy_placement = bo->placements;
  190. }
  191. /**
  192. * amdgpu_bo_create_kernel - create BO for kernel use
  193. *
  194. * @adev: amdgpu device object
  195. * @size: size for the new BO
  196. * @align: alignment for the new BO
  197. * @domain: where to place it
  198. * @bo_ptr: resulting BO
  199. * @gpu_addr: GPU addr of the pinned BO
  200. * @cpu_addr: optional CPU address mapping
  201. *
  202. * Allocates and pins a BO for kernel internal use.
  203. *
  204. * Returns 0 on success, negative error code otherwise.
  205. */
  206. int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
  207. unsigned long size, int align,
  208. u32 domain, struct amdgpu_bo **bo_ptr,
  209. u64 *gpu_addr, void **cpu_addr)
  210. {
  211. int r;
  212. r = amdgpu_bo_create(adev, size, align, true, domain,
  213. AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
  214. NULL, NULL, bo_ptr);
  215. if (r) {
  216. dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", r);
  217. return r;
  218. }
  219. r = amdgpu_bo_reserve(*bo_ptr, false);
  220. if (r) {
  221. dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
  222. goto error_free;
  223. }
  224. r = amdgpu_bo_pin(*bo_ptr, domain, gpu_addr);
  225. if (r) {
  226. dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
  227. goto error_unreserve;
  228. }
  229. if (cpu_addr) {
  230. r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
  231. if (r) {
  232. dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
  233. goto error_unreserve;
  234. }
  235. }
  236. amdgpu_bo_unreserve(*bo_ptr);
  237. return 0;
  238. error_unreserve:
  239. amdgpu_bo_unreserve(*bo_ptr);
  240. error_free:
  241. amdgpu_bo_unref(bo_ptr);
  242. return r;
  243. }
  244. int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
  245. unsigned long size, int byte_align,
  246. bool kernel, u32 domain, u64 flags,
  247. struct sg_table *sg,
  248. struct ttm_placement *placement,
  249. struct reservation_object *resv,
  250. struct amdgpu_bo **bo_ptr)
  251. {
  252. struct amdgpu_bo *bo;
  253. enum ttm_bo_type type;
  254. unsigned long page_align;
  255. size_t acc_size;
  256. int r;
  257. page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
  258. size = ALIGN(size, PAGE_SIZE);
  259. if (kernel) {
  260. type = ttm_bo_type_kernel;
  261. } else if (sg) {
  262. type = ttm_bo_type_sg;
  263. } else {
  264. type = ttm_bo_type_device;
  265. }
  266. *bo_ptr = NULL;
  267. acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
  268. sizeof(struct amdgpu_bo));
  269. bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL);
  270. if (bo == NULL)
  271. return -ENOMEM;
  272. r = drm_gem_object_init(adev->ddev, &bo->gem_base, size);
  273. if (unlikely(r)) {
  274. kfree(bo);
  275. return r;
  276. }
  277. bo->adev = adev;
  278. INIT_LIST_HEAD(&bo->list);
  279. INIT_LIST_HEAD(&bo->va);
  280. bo->prefered_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM |
  281. AMDGPU_GEM_DOMAIN_GTT |
  282. AMDGPU_GEM_DOMAIN_CPU |
  283. AMDGPU_GEM_DOMAIN_GDS |
  284. AMDGPU_GEM_DOMAIN_GWS |
  285. AMDGPU_GEM_DOMAIN_OA);
  286. bo->allowed_domains = bo->prefered_domains;
  287. if (!kernel && bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
  288. bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
  289. bo->flags = flags;
  290. /* For architectures that don't support WC memory,
  291. * mask out the WC flag from the BO
  292. */
  293. if (!drm_arch_can_wc_memory())
  294. bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
  295. amdgpu_fill_placement_to_bo(bo, placement);
  296. /* Kernel allocation are uninterruptible */
  297. r = ttm_bo_init(&adev->mman.bdev, &bo->tbo, size, type,
  298. &bo->placement, page_align, !kernel, NULL,
  299. acc_size, sg, resv, &amdgpu_ttm_bo_destroy);
  300. if (unlikely(r != 0)) {
  301. return r;
  302. }
  303. if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
  304. bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {
  305. struct fence *fence;
  306. if (adev->mman.buffer_funcs_ring == NULL ||
  307. !adev->mman.buffer_funcs_ring->ready) {
  308. r = -EBUSY;
  309. goto fail_free;
  310. }
  311. r = amdgpu_bo_reserve(bo, false);
  312. if (unlikely(r != 0))
  313. goto fail_free;
  314. amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM);
  315. r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
  316. if (unlikely(r != 0))
  317. goto fail_unreserve;
  318. amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence);
  319. amdgpu_bo_fence(bo, fence, false);
  320. amdgpu_bo_unreserve(bo);
  321. fence_put(bo->tbo.moving);
  322. bo->tbo.moving = fence_get(fence);
  323. fence_put(fence);
  324. }
  325. *bo_ptr = bo;
  326. trace_amdgpu_bo_create(bo);
  327. return 0;
  328. fail_unreserve:
  329. amdgpu_bo_unreserve(bo);
  330. fail_free:
  331. amdgpu_bo_unref(&bo);
  332. return r;
  333. }
  334. static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
  335. unsigned long size, int byte_align,
  336. struct amdgpu_bo *bo)
  337. {
  338. struct ttm_placement placement = {0};
  339. struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
  340. int r;
  341. if (bo->shadow)
  342. return 0;
  343. bo->flags |= AMDGPU_GEM_CREATE_SHADOW;
  344. memset(&placements, 0,
  345. (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place));
  346. amdgpu_ttm_placement_init(adev, &placement,
  347. placements, AMDGPU_GEM_DOMAIN_GTT,
  348. AMDGPU_GEM_CREATE_CPU_GTT_USWC);
  349. r = amdgpu_bo_create_restricted(adev, size, byte_align, true,
  350. AMDGPU_GEM_DOMAIN_GTT,
  351. AMDGPU_GEM_CREATE_CPU_GTT_USWC,
  352. NULL, &placement,
  353. bo->tbo.resv,
  354. &bo->shadow);
  355. if (!r)
  356. bo->shadow->parent = amdgpu_bo_ref(bo);
  357. return r;
  358. }
  359. int amdgpu_bo_create(struct amdgpu_device *adev,
  360. unsigned long size, int byte_align,
  361. bool kernel, u32 domain, u64 flags,
  362. struct sg_table *sg,
  363. struct reservation_object *resv,
  364. struct amdgpu_bo **bo_ptr)
  365. {
  366. struct ttm_placement placement = {0};
  367. struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
  368. int r;
  369. memset(&placements, 0,
  370. (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place));
  371. amdgpu_ttm_placement_init(adev, &placement,
  372. placements, domain, flags);
  373. r = amdgpu_bo_create_restricted(adev, size, byte_align, kernel,
  374. domain, flags, sg, &placement,
  375. resv, bo_ptr);
  376. if (r)
  377. return r;
  378. if (flags & AMDGPU_GEM_CREATE_SHADOW) {
  379. r = amdgpu_bo_create_shadow(adev, size, byte_align, (*bo_ptr));
  380. if (r)
  381. amdgpu_bo_unref(bo_ptr);
  382. }
  383. return r;
  384. }
  385. int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
  386. {
  387. bool is_iomem;
  388. long r;
  389. if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
  390. return -EPERM;
  391. if (bo->kptr) {
  392. if (ptr) {
  393. *ptr = bo->kptr;
  394. }
  395. return 0;
  396. }
  397. r = reservation_object_wait_timeout_rcu(bo->tbo.resv, false, false,
  398. MAX_SCHEDULE_TIMEOUT);
  399. if (r < 0)
  400. return r;
  401. r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap);
  402. if (r)
  403. return r;
  404. bo->kptr = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
  405. if (ptr)
  406. *ptr = bo->kptr;
  407. return 0;
  408. }
  409. void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
  410. {
  411. if (bo->kptr == NULL)
  412. return;
  413. bo->kptr = NULL;
  414. ttm_bo_kunmap(&bo->kmap);
  415. }
  416. struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
  417. {
  418. if (bo == NULL)
  419. return NULL;
  420. ttm_bo_reference(&bo->tbo);
  421. return bo;
  422. }
  423. void amdgpu_bo_unref(struct amdgpu_bo **bo)
  424. {
  425. struct ttm_buffer_object *tbo;
  426. if ((*bo) == NULL)
  427. return;
  428. tbo = &((*bo)->tbo);
  429. ttm_bo_unref(&tbo);
  430. if (tbo == NULL)
  431. *bo = NULL;
  432. }
  433. int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
  434. u64 min_offset, u64 max_offset,
  435. u64 *gpu_addr)
  436. {
  437. int r, i;
  438. unsigned fpfn, lpfn;
  439. if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
  440. return -EPERM;
  441. if (WARN_ON_ONCE(min_offset > max_offset))
  442. return -EINVAL;
  443. if (bo->pin_count) {
  444. bo->pin_count++;
  445. if (gpu_addr)
  446. *gpu_addr = amdgpu_bo_gpu_offset(bo);
  447. if (max_offset != 0) {
  448. u64 domain_start;
  449. if (domain == AMDGPU_GEM_DOMAIN_VRAM)
  450. domain_start = bo->adev->mc.vram_start;
  451. else
  452. domain_start = bo->adev->mc.gtt_start;
  453. WARN_ON_ONCE(max_offset <
  454. (amdgpu_bo_gpu_offset(bo) - domain_start));
  455. }
  456. return 0;
  457. }
  458. amdgpu_ttm_placement_from_domain(bo, domain);
  459. for (i = 0; i < bo->placement.num_placement; i++) {
  460. /* force to pin into visible video ram */
  461. if ((bo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
  462. !(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) &&
  463. (!max_offset || max_offset > bo->adev->mc.visible_vram_size)) {
  464. if (WARN_ON_ONCE(min_offset >
  465. bo->adev->mc.visible_vram_size))
  466. return -EINVAL;
  467. fpfn = min_offset >> PAGE_SHIFT;
  468. lpfn = bo->adev->mc.visible_vram_size >> PAGE_SHIFT;
  469. } else {
  470. fpfn = min_offset >> PAGE_SHIFT;
  471. lpfn = max_offset >> PAGE_SHIFT;
  472. }
  473. if (fpfn > bo->placements[i].fpfn)
  474. bo->placements[i].fpfn = fpfn;
  475. if (!bo->placements[i].lpfn ||
  476. (lpfn && lpfn < bo->placements[i].lpfn))
  477. bo->placements[i].lpfn = lpfn;
  478. bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
  479. }
  480. r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
  481. if (likely(r == 0)) {
  482. bo->pin_count = 1;
  483. if (gpu_addr != NULL)
  484. *gpu_addr = amdgpu_bo_gpu_offset(bo);
  485. if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
  486. bo->adev->vram_pin_size += amdgpu_bo_size(bo);
  487. if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
  488. bo->adev->invisible_pin_size += amdgpu_bo_size(bo);
  489. } else
  490. bo->adev->gart_pin_size += amdgpu_bo_size(bo);
  491. } else {
  492. dev_err(bo->adev->dev, "%p pin failed\n", bo);
  493. }
  494. return r;
  495. }
  496. int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain, u64 *gpu_addr)
  497. {
  498. return amdgpu_bo_pin_restricted(bo, domain, 0, 0, gpu_addr);
  499. }
  500. int amdgpu_bo_unpin(struct amdgpu_bo *bo)
  501. {
  502. int r, i;
  503. if (!bo->pin_count) {
  504. dev_warn(bo->adev->dev, "%p unpin not necessary\n", bo);
  505. return 0;
  506. }
  507. bo->pin_count--;
  508. if (bo->pin_count)
  509. return 0;
  510. for (i = 0; i < bo->placement.num_placement; i++) {
  511. bo->placements[i].lpfn = 0;
  512. bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
  513. }
  514. r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
  515. if (likely(r == 0)) {
  516. if (bo->tbo.mem.mem_type == TTM_PL_VRAM) {
  517. bo->adev->vram_pin_size -= amdgpu_bo_size(bo);
  518. if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
  519. bo->adev->invisible_pin_size -= amdgpu_bo_size(bo);
  520. } else
  521. bo->adev->gart_pin_size -= amdgpu_bo_size(bo);
  522. } else {
  523. dev_err(bo->adev->dev, "%p validate failed for unpin\n", bo);
  524. }
  525. return r;
  526. }
  527. int amdgpu_bo_evict_vram(struct amdgpu_device *adev)
  528. {
  529. /* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */
  530. if (0 && (adev->flags & AMD_IS_APU)) {
  531. /* Useless to evict on IGP chips */
  532. return 0;
  533. }
  534. return ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_VRAM);
  535. }
  536. static const char *amdgpu_vram_names[] = {
  537. "UNKNOWN",
  538. "GDDR1",
  539. "DDR2",
  540. "GDDR3",
  541. "GDDR4",
  542. "GDDR5",
  543. "HBM",
  544. "DDR3"
  545. };
  546. int amdgpu_bo_init(struct amdgpu_device *adev)
  547. {
  548. /* Add an MTRR for the VRAM */
  549. adev->mc.vram_mtrr = arch_phys_wc_add(adev->mc.aper_base,
  550. adev->mc.aper_size);
  551. DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
  552. adev->mc.mc_vram_size >> 20,
  553. (unsigned long long)adev->mc.aper_size >> 20);
  554. DRM_INFO("RAM width %dbits %s\n",
  555. adev->mc.vram_width, amdgpu_vram_names[adev->mc.vram_type]);
  556. return amdgpu_ttm_init(adev);
  557. }
  558. void amdgpu_bo_fini(struct amdgpu_device *adev)
  559. {
  560. amdgpu_ttm_fini(adev);
  561. arch_phys_wc_del(adev->mc.vram_mtrr);
  562. }
  563. int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo,
  564. struct vm_area_struct *vma)
  565. {
  566. return ttm_fbdev_mmap(vma, &bo->tbo);
  567. }
  568. int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
  569. {
  570. if (AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
  571. return -EINVAL;
  572. bo->tiling_flags = tiling_flags;
  573. return 0;
  574. }
  575. void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
  576. {
  577. lockdep_assert_held(&bo->tbo.resv->lock.base);
  578. if (tiling_flags)
  579. *tiling_flags = bo->tiling_flags;
  580. }
  581. int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata,
  582. uint32_t metadata_size, uint64_t flags)
  583. {
  584. void *buffer;
  585. if (!metadata_size) {
  586. if (bo->metadata_size) {
  587. kfree(bo->metadata);
  588. bo->metadata = NULL;
  589. bo->metadata_size = 0;
  590. }
  591. return 0;
  592. }
  593. if (metadata == NULL)
  594. return -EINVAL;
  595. buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
  596. if (buffer == NULL)
  597. return -ENOMEM;
  598. kfree(bo->metadata);
  599. bo->metadata_flags = flags;
  600. bo->metadata = buffer;
  601. bo->metadata_size = metadata_size;
  602. return 0;
  603. }
  604. int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
  605. size_t buffer_size, uint32_t *metadata_size,
  606. uint64_t *flags)
  607. {
  608. if (!buffer && !metadata_size)
  609. return -EINVAL;
  610. if (buffer) {
  611. if (buffer_size < bo->metadata_size)
  612. return -EINVAL;
  613. if (bo->metadata_size)
  614. memcpy(buffer, bo->metadata, bo->metadata_size);
  615. }
  616. if (metadata_size)
  617. *metadata_size = bo->metadata_size;
  618. if (flags)
  619. *flags = bo->metadata_flags;
  620. return 0;
  621. }
  622. void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
  623. struct ttm_mem_reg *new_mem)
  624. {
  625. struct amdgpu_bo *rbo;
  626. struct ttm_mem_reg *old_mem = &bo->mem;
  627. if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
  628. return;
  629. rbo = container_of(bo, struct amdgpu_bo, tbo);
  630. amdgpu_vm_bo_invalidate(rbo->adev, rbo);
  631. /* update statistics */
  632. if (!new_mem)
  633. return;
  634. /* move_notify is called before move happens */
  635. amdgpu_update_memory_usage(rbo->adev, &bo->mem, new_mem);
  636. trace_amdgpu_ttm_bo_move(rbo, new_mem->mem_type, old_mem->mem_type);
  637. }
  638. int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
  639. {
  640. struct amdgpu_device *adev;
  641. struct amdgpu_bo *abo;
  642. unsigned long offset, size, lpfn;
  643. int i, r;
  644. if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
  645. return 0;
  646. abo = container_of(bo, struct amdgpu_bo, tbo);
  647. adev = abo->adev;
  648. if (bo->mem.mem_type != TTM_PL_VRAM)
  649. return 0;
  650. size = bo->mem.num_pages << PAGE_SHIFT;
  651. offset = bo->mem.start << PAGE_SHIFT;
  652. if ((offset + size) <= adev->mc.visible_vram_size)
  653. return 0;
  654. /* Can't move a pinned BO to visible VRAM */
  655. if (abo->pin_count > 0)
  656. return -EINVAL;
  657. /* hurrah the memory is not visible ! */
  658. amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM);
  659. lpfn = adev->mc.visible_vram_size >> PAGE_SHIFT;
  660. for (i = 0; i < abo->placement.num_placement; i++) {
  661. /* Force into visible VRAM */
  662. if ((abo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
  663. (!abo->placements[i].lpfn || abo->placements[i].lpfn > lpfn))
  664. abo->placements[i].lpfn = lpfn;
  665. }
  666. r = ttm_bo_validate(bo, &abo->placement, false, false);
  667. if (unlikely(r == -ENOMEM)) {
  668. amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
  669. return ttm_bo_validate(bo, &abo->placement, false, false);
  670. } else if (unlikely(r != 0)) {
  671. return r;
  672. }
  673. offset = bo->mem.start << PAGE_SHIFT;
  674. /* this should never happen */
  675. if ((offset + size) > adev->mc.visible_vram_size)
  676. return -EINVAL;
  677. return 0;
  678. }
  679. /**
  680. * amdgpu_bo_fence - add fence to buffer object
  681. *
  682. * @bo: buffer object in question
  683. * @fence: fence to add
  684. * @shared: true if fence should be added shared
  685. *
  686. */
  687. void amdgpu_bo_fence(struct amdgpu_bo *bo, struct fence *fence,
  688. bool shared)
  689. {
  690. struct reservation_object *resv = bo->tbo.resv;
  691. if (shared)
  692. reservation_object_add_shared_fence(resv, fence);
  693. else
  694. reservation_object_add_excl_fence(resv, fence);
  695. }
  696. /**
  697. * amdgpu_bo_gpu_offset - return GPU offset of bo
  698. * @bo: amdgpu object for which we query the offset
  699. *
  700. * Returns current GPU offset of the object.
  701. *
  702. * Note: object should either be pinned or reserved when calling this
  703. * function, it might be useful to add check for this for debugging.
  704. */
  705. u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
  706. {
  707. WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM);
  708. WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) &&
  709. !bo->pin_count);
  710. return bo->tbo.offset;
  711. }