i915_gem_shrinker.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509
  1. /*
  2. * Copyright © 2008-2015 Intel Corporation
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice (including the next
  12. * paragraph) shall be included in all copies or substantial portions of the
  13. * Software.
  14. *
  15. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18. * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21. * IN THE SOFTWARE.
  22. *
  23. */
  24. #include <linux/oom.h>
  25. #include <linux/shmem_fs.h>
  26. #include <linux/slab.h>
  27. #include <linux/swap.h>
  28. #include <linux/pci.h>
  29. #include <linux/dma-buf.h>
  30. #include <linux/vmalloc.h>
  31. #include <drm/drmP.h>
  32. #include <drm/i915_drm.h>
  33. #include "i915_drv.h"
  34. #include "i915_trace.h"
  35. static bool shrinker_lock(struct drm_i915_private *dev_priv, bool *unlock)
  36. {
  37. switch (mutex_trylock_recursive(&dev_priv->drm.struct_mutex)) {
  38. case MUTEX_TRYLOCK_RECURSIVE:
  39. *unlock = false;
  40. return true;
  41. case MUTEX_TRYLOCK_FAILED:
  42. *unlock = false;
  43. preempt_disable();
  44. do {
  45. cpu_relax();
  46. if (mutex_trylock(&dev_priv->drm.struct_mutex)) {
  47. *unlock = true;
  48. break;
  49. }
  50. } while (!need_resched());
  51. preempt_enable();
  52. return *unlock;
  53. case MUTEX_TRYLOCK_SUCCESS:
  54. *unlock = true;
  55. return true;
  56. }
  57. BUG();
  58. }
  59. static void shrinker_unlock(struct drm_i915_private *dev_priv, bool unlock)
  60. {
  61. if (!unlock)
  62. return;
  63. mutex_unlock(&dev_priv->drm.struct_mutex);
  64. }
  65. static bool any_vma_pinned(struct drm_i915_gem_object *obj)
  66. {
  67. struct i915_vma *vma;
  68. list_for_each_entry(vma, &obj->vma_list, obj_link) {
  69. /* Only GGTT vma may be permanently pinned, and are always
  70. * at the start of the list. We can stop hunting as soon
  71. * as we see a ppGTT vma.
  72. */
  73. if (!i915_vma_is_ggtt(vma))
  74. break;
  75. if (i915_vma_is_pinned(vma))
  76. return true;
  77. }
  78. return false;
  79. }
  80. static bool swap_available(void)
  81. {
  82. return get_nr_swap_pages() > 0;
  83. }
  84. static bool can_release_pages(struct drm_i915_gem_object *obj)
  85. {
  86. if (!obj->mm.pages)
  87. return false;
  88. /* Consider only shrinkable ojects. */
  89. if (!i915_gem_object_is_shrinkable(obj))
  90. return false;
  91. /* Only report true if by unbinding the object and putting its pages
  92. * we can actually make forward progress towards freeing physical
  93. * pages.
  94. *
  95. * If the pages are pinned for any other reason than being bound
  96. * to the GPU, simply unbinding from the GPU is not going to succeed
  97. * in releasing our pin count on the pages themselves.
  98. */
  99. if (atomic_read(&obj->mm.pages_pin_count) > obj->bind_count)
  100. return false;
  101. if (any_vma_pinned(obj))
  102. return false;
  103. /* We can only return physical pages to the system if we can either
  104. * discard the contents (because the user has marked them as being
  105. * purgeable) or if we can move their contents out to swap.
  106. */
  107. return swap_available() || obj->mm.madv == I915_MADV_DONTNEED;
  108. }
  109. static bool unsafe_drop_pages(struct drm_i915_gem_object *obj)
  110. {
  111. if (i915_gem_object_unbind(obj) == 0)
  112. __i915_gem_object_put_pages(obj, I915_MM_SHRINKER);
  113. return !READ_ONCE(obj->mm.pages);
  114. }
  115. /**
  116. * i915_gem_shrink - Shrink buffer object caches
  117. * @dev_priv: i915 device
  118. * @target: amount of memory to make available, in pages
  119. * @flags: control flags for selecting cache types
  120. *
  121. * This function is the main interface to the shrinker. It will try to release
  122. * up to @target pages of main memory backing storage from buffer objects.
  123. * Selection of the specific caches can be done with @flags. This is e.g. useful
  124. * when purgeable objects should be removed from caches preferentially.
  125. *
  126. * Note that it's not guaranteed that released amount is actually available as
  127. * free system memory - the pages might still be in-used to due to other reasons
  128. * (like cpu mmaps) or the mm core has reused them before we could grab them.
  129. * Therefore code that needs to explicitly shrink buffer objects caches (e.g. to
  130. * avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all().
  131. *
  132. * Also note that any kind of pinning (both per-vma address space pins and
  133. * backing storage pins at the buffer object level) result in the shrinker code
  134. * having to skip the object.
  135. *
  136. * Returns:
  137. * The number of pages of backing storage actually released.
  138. */
  139. unsigned long
  140. i915_gem_shrink(struct drm_i915_private *dev_priv,
  141. unsigned long target, unsigned flags)
  142. {
  143. const struct {
  144. struct list_head *list;
  145. unsigned int bit;
  146. } phases[] = {
  147. { &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND },
  148. { &dev_priv->mm.bound_list, I915_SHRINK_BOUND },
  149. { NULL, 0 },
  150. }, *phase;
  151. unsigned long count = 0;
  152. bool unlock;
  153. if (!shrinker_lock(dev_priv, &unlock))
  154. return 0;
  155. trace_i915_gem_shrink(dev_priv, target, flags);
  156. i915_gem_retire_requests(dev_priv);
  157. /*
  158. * Unbinding of objects will require HW access; Let us not wake the
  159. * device just to recover a little memory. If absolutely necessary,
  160. * we will force the wake during oom-notifier.
  161. */
  162. if ((flags & I915_SHRINK_BOUND) &&
  163. !intel_runtime_pm_get_if_in_use(dev_priv))
  164. flags &= ~I915_SHRINK_BOUND;
  165. /*
  166. * As we may completely rewrite the (un)bound list whilst unbinding
  167. * (due to retiring requests) we have to strictly process only
  168. * one element of the list at the time, and recheck the list
  169. * on every iteration.
  170. *
  171. * In particular, we must hold a reference whilst removing the
  172. * object as we may end up waiting for and/or retiring the objects.
  173. * This might release the final reference (held by the active list)
  174. * and result in the object being freed from under us. This is
  175. * similar to the precautions the eviction code must take whilst
  176. * removing objects.
  177. *
  178. * Also note that although these lists do not hold a reference to
  179. * the object we can safely grab one here: The final object
  180. * unreferencing and the bound_list are both protected by the
  181. * dev->struct_mutex and so we won't ever be able to observe an
  182. * object on the bound_list with a reference count equals 0.
  183. */
  184. for (phase = phases; phase->list; phase++) {
  185. struct list_head still_in_list;
  186. struct drm_i915_gem_object *obj;
  187. if ((flags & phase->bit) == 0)
  188. continue;
  189. INIT_LIST_HEAD(&still_in_list);
  190. while (count < target &&
  191. (obj = list_first_entry_or_null(phase->list,
  192. typeof(*obj),
  193. global_link))) {
  194. list_move_tail(&obj->global_link, &still_in_list);
  195. if (!obj->mm.pages) {
  196. list_del_init(&obj->global_link);
  197. continue;
  198. }
  199. if (flags & I915_SHRINK_PURGEABLE &&
  200. obj->mm.madv != I915_MADV_DONTNEED)
  201. continue;
  202. if (flags & I915_SHRINK_VMAPS &&
  203. !is_vmalloc_addr(obj->mm.mapping))
  204. continue;
  205. if (!(flags & I915_SHRINK_ACTIVE) &&
  206. (i915_gem_object_is_active(obj) ||
  207. i915_gem_object_is_framebuffer(obj)))
  208. continue;
  209. if (!can_release_pages(obj))
  210. continue;
  211. if (unsafe_drop_pages(obj)) {
  212. /* May arrive from get_pages on another bo */
  213. mutex_lock_nested(&obj->mm.lock,
  214. I915_MM_SHRINKER);
  215. if (!obj->mm.pages) {
  216. __i915_gem_object_invalidate(obj);
  217. list_del_init(&obj->global_link);
  218. count += obj->base.size >> PAGE_SHIFT;
  219. }
  220. mutex_unlock(&obj->mm.lock);
  221. }
  222. }
  223. list_splice_tail(&still_in_list, phase->list);
  224. }
  225. if (flags & I915_SHRINK_BOUND)
  226. intel_runtime_pm_put(dev_priv);
  227. i915_gem_retire_requests(dev_priv);
  228. shrinker_unlock(dev_priv, unlock);
  229. return count;
  230. }
  231. /**
  232. * i915_gem_shrink_all - Shrink buffer object caches completely
  233. * @dev_priv: i915 device
  234. *
  235. * This is a simple wraper around i915_gem_shrink() to aggressively shrink all
  236. * caches completely. It also first waits for and retires all outstanding
  237. * requests to also be able to release backing storage for active objects.
  238. *
  239. * This should only be used in code to intentionally quiescent the gpu or as a
  240. * last-ditch effort when memory seems to have run out.
  241. *
  242. * Returns:
  243. * The number of pages of backing storage actually released.
  244. */
  245. unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv)
  246. {
  247. unsigned long freed;
  248. intel_runtime_pm_get(dev_priv);
  249. freed = i915_gem_shrink(dev_priv, -1UL,
  250. I915_SHRINK_BOUND |
  251. I915_SHRINK_UNBOUND |
  252. I915_SHRINK_ACTIVE);
  253. intel_runtime_pm_put(dev_priv);
  254. return freed;
  255. }
  256. static unsigned long
  257. i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
  258. {
  259. struct drm_i915_private *dev_priv =
  260. container_of(shrinker, struct drm_i915_private, mm.shrinker);
  261. struct drm_i915_gem_object *obj;
  262. unsigned long count;
  263. bool unlock;
  264. if (!shrinker_lock(dev_priv, &unlock))
  265. return 0;
  266. i915_gem_retire_requests(dev_priv);
  267. count = 0;
  268. list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link)
  269. if (can_release_pages(obj))
  270. count += obj->base.size >> PAGE_SHIFT;
  271. list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) {
  272. if (!i915_gem_object_is_active(obj) && can_release_pages(obj))
  273. count += obj->base.size >> PAGE_SHIFT;
  274. }
  275. shrinker_unlock(dev_priv, unlock);
  276. return count;
  277. }
  278. static unsigned long
  279. i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
  280. {
  281. struct drm_i915_private *dev_priv =
  282. container_of(shrinker, struct drm_i915_private, mm.shrinker);
  283. unsigned long freed;
  284. bool unlock;
  285. if (!shrinker_lock(dev_priv, &unlock))
  286. return SHRINK_STOP;
  287. freed = i915_gem_shrink(dev_priv,
  288. sc->nr_to_scan,
  289. I915_SHRINK_BOUND |
  290. I915_SHRINK_UNBOUND |
  291. I915_SHRINK_PURGEABLE);
  292. if (freed < sc->nr_to_scan)
  293. freed += i915_gem_shrink(dev_priv,
  294. sc->nr_to_scan - freed,
  295. I915_SHRINK_BOUND |
  296. I915_SHRINK_UNBOUND);
  297. if (freed < sc->nr_to_scan && current_is_kswapd()) {
  298. intel_runtime_pm_get(dev_priv);
  299. freed += i915_gem_shrink(dev_priv,
  300. sc->nr_to_scan - freed,
  301. I915_SHRINK_ACTIVE |
  302. I915_SHRINK_BOUND |
  303. I915_SHRINK_UNBOUND);
  304. intel_runtime_pm_put(dev_priv);
  305. }
  306. shrinker_unlock(dev_priv, unlock);
  307. return freed;
  308. }
  309. static bool
  310. shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv, bool *unlock,
  311. int timeout_ms)
  312. {
  313. unsigned long timeout = jiffies + msecs_to_jiffies_timeout(timeout_ms);
  314. do {
  315. if (i915_gem_wait_for_idle(dev_priv, 0) == 0 &&
  316. shrinker_lock(dev_priv, unlock))
  317. break;
  318. schedule_timeout_killable(1);
  319. if (fatal_signal_pending(current))
  320. return false;
  321. if (time_after(jiffies, timeout)) {
  322. pr_err("Unable to lock GPU to purge memory.\n");
  323. return false;
  324. }
  325. } while (1);
  326. return true;
  327. }
  328. static int
  329. i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
  330. {
  331. struct drm_i915_private *dev_priv =
  332. container_of(nb, struct drm_i915_private, mm.oom_notifier);
  333. struct drm_i915_gem_object *obj;
  334. unsigned long unevictable, bound, unbound, freed_pages;
  335. bool unlock;
  336. if (!shrinker_lock_uninterruptible(dev_priv, &unlock, 5000))
  337. return NOTIFY_DONE;
  338. freed_pages = i915_gem_shrink_all(dev_priv);
  339. /* Because we may be allocating inside our own driver, we cannot
  340. * assert that there are no objects with pinned pages that are not
  341. * being pointed to by hardware.
  342. */
  343. unbound = bound = unevictable = 0;
  344. list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) {
  345. if (!obj->mm.pages)
  346. continue;
  347. if (!can_release_pages(obj))
  348. unevictable += obj->base.size >> PAGE_SHIFT;
  349. else
  350. unbound += obj->base.size >> PAGE_SHIFT;
  351. }
  352. list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) {
  353. if (!obj->mm.pages)
  354. continue;
  355. if (!can_release_pages(obj))
  356. unevictable += obj->base.size >> PAGE_SHIFT;
  357. else
  358. bound += obj->base.size >> PAGE_SHIFT;
  359. }
  360. shrinker_unlock(dev_priv, unlock);
  361. if (freed_pages || unbound || bound)
  362. pr_info("Purging GPU memory, %lu pages freed, "
  363. "%lu pages still pinned.\n",
  364. freed_pages, unevictable);
  365. if (unbound || bound)
  366. pr_err("%lu and %lu pages still available in the "
  367. "bound and unbound GPU page lists.\n",
  368. bound, unbound);
  369. *(unsigned long *)ptr += freed_pages;
  370. return NOTIFY_DONE;
  371. }
  372. static int
  373. i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
  374. {
  375. struct drm_i915_private *dev_priv =
  376. container_of(nb, struct drm_i915_private, mm.vmap_notifier);
  377. struct i915_vma *vma, *next;
  378. unsigned long freed_pages = 0;
  379. bool unlock;
  380. int ret;
  381. if (!shrinker_lock_uninterruptible(dev_priv, &unlock, 5000))
  382. return NOTIFY_DONE;
  383. /* Force everything onto the inactive lists */
  384. ret = i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED);
  385. if (ret)
  386. goto out;
  387. intel_runtime_pm_get(dev_priv);
  388. freed_pages += i915_gem_shrink(dev_priv, -1UL,
  389. I915_SHRINK_BOUND |
  390. I915_SHRINK_UNBOUND |
  391. I915_SHRINK_ACTIVE |
  392. I915_SHRINK_VMAPS);
  393. intel_runtime_pm_put(dev_priv);
  394. /* We also want to clear any cached iomaps as they wrap vmap */
  395. list_for_each_entry_safe(vma, next,
  396. &dev_priv->ggtt.base.inactive_list, vm_link) {
  397. unsigned long count = vma->node.size >> PAGE_SHIFT;
  398. if (vma->iomap && i915_vma_unbind(vma) == 0)
  399. freed_pages += count;
  400. }
  401. out:
  402. shrinker_unlock(dev_priv, unlock);
  403. *(unsigned long *)ptr += freed_pages;
  404. return NOTIFY_DONE;
  405. }
  406. /**
  407. * i915_gem_shrinker_init - Initialize i915 shrinker
  408. * @dev_priv: i915 device
  409. *
  410. * This function registers and sets up the i915 shrinker and OOM handler.
  411. */
  412. void i915_gem_shrinker_init(struct drm_i915_private *dev_priv)
  413. {
  414. dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan;
  415. dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count;
  416. dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS;
  417. WARN_ON(register_shrinker(&dev_priv->mm.shrinker));
  418. dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
  419. WARN_ON(register_oom_notifier(&dev_priv->mm.oom_notifier));
  420. dev_priv->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap;
  421. WARN_ON(register_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
  422. }
  423. /**
  424. * i915_gem_shrinker_cleanup - Clean up i915 shrinker
  425. * @dev_priv: i915 device
  426. *
  427. * This function unregisters the i915 shrinker and OOM handler.
  428. */
  429. void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv)
  430. {
  431. WARN_ON(unregister_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
  432. WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier));
  433. unregister_shrinker(&dev_priv->mm.shrinker);
  434. }