balloon.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837
  1. /******************************************************************************
  2. * Xen balloon driver - enables returning/claiming memory to/from Xen.
  3. *
  4. * Copyright (c) 2003, B Dragovic
  5. * Copyright (c) 2003-2004, M Williamson, K Fraser
  6. * Copyright (c) 2005 Dan M. Smith, IBM Corporation
  7. * Copyright (c) 2010 Daniel Kiper
  8. *
  9. * Memory hotplug support was written by Daniel Kiper. Work on
  10. * it was sponsored by Google under Google Summer of Code 2010
  11. * program. Jeremy Fitzhardinge from Citrix was the mentor for
  12. * this project.
  13. *
  14. * This program is free software; you can redistribute it and/or
  15. * modify it under the terms of the GNU General Public License version 2
  16. * as published by the Free Software Foundation; or, when distributed
  17. * separately from the Linux kernel or incorporated into other
  18. * software packages, subject to the following license:
  19. *
  20. * Permission is hereby granted, free of charge, to any person obtaining a copy
  21. * of this source file (the "Software"), to deal in the Software without
  22. * restriction, including without limitation the rights to use, copy, modify,
  23. * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  24. * and to permit persons to whom the Software is furnished to do so, subject to
  25. * the following conditions:
  26. *
  27. * The above copyright notice and this permission notice shall be included in
  28. * all copies or substantial portions of the Software.
  29. *
  30. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  31. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  32. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  33. * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  34. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  35. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  36. * IN THE SOFTWARE.
  37. */
  38. #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
  39. #include <linux/cpu.h>
  40. #include <linux/kernel.h>
  41. #include <linux/sched.h>
  42. #include <linux/cred.h>
  43. #include <linux/errno.h>
  44. #include <linux/mm.h>
  45. #include <linux/bootmem.h>
  46. #include <linux/pagemap.h>
  47. #include <linux/highmem.h>
  48. #include <linux/mutex.h>
  49. #include <linux/list.h>
  50. #include <linux/gfp.h>
  51. #include <linux/notifier.h>
  52. #include <linux/memory.h>
  53. #include <linux/memory_hotplug.h>
  54. #include <linux/percpu-defs.h>
  55. #include <linux/slab.h>
  56. #include <linux/sysctl.h>
  57. #include <asm/page.h>
  58. #include <asm/pgalloc.h>
  59. #include <asm/pgtable.h>
  60. #include <asm/tlb.h>
  61. #include <asm/xen/hypervisor.h>
  62. #include <asm/xen/hypercall.h>
  63. #include <xen/xen.h>
  64. #include <xen/interface/xen.h>
  65. #include <xen/interface/memory.h>
  66. #include <xen/balloon.h>
  67. #include <xen/features.h>
  68. #include <xen/page.h>
  69. static int xen_hotplug_unpopulated;
  70. #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
  71. static int zero;
  72. static int one = 1;
  73. static struct ctl_table balloon_table[] = {
  74. {
  75. .procname = "hotplug_unpopulated",
  76. .data = &xen_hotplug_unpopulated,
  77. .maxlen = sizeof(int),
  78. .mode = 0644,
  79. .proc_handler = proc_dointvec_minmax,
  80. .extra1 = &zero,
  81. .extra2 = &one,
  82. },
  83. { }
  84. };
  85. static struct ctl_table balloon_root[] = {
  86. {
  87. .procname = "balloon",
  88. .mode = 0555,
  89. .child = balloon_table,
  90. },
  91. { }
  92. };
  93. static struct ctl_table xen_root[] = {
  94. {
  95. .procname = "xen",
  96. .mode = 0555,
  97. .child = balloon_root,
  98. },
  99. { }
  100. };
  101. #endif
  102. /*
  103. * Use one extent per PAGE_SIZE to avoid to break down the page into
  104. * multiple frame.
  105. */
  106. #define EXTENT_ORDER (fls(XEN_PFN_PER_PAGE) - 1)
  107. /*
  108. * balloon_process() state:
  109. *
  110. * BP_DONE: done or nothing to do,
  111. * BP_WAIT: wait to be rescheduled,
  112. * BP_EAGAIN: error, go to sleep,
  113. * BP_ECANCELED: error, balloon operation canceled.
  114. */
  115. enum bp_state {
  116. BP_DONE,
  117. BP_WAIT,
  118. BP_EAGAIN,
  119. BP_ECANCELED
  120. };
  121. static DEFINE_MUTEX(balloon_mutex);
  122. struct balloon_stats balloon_stats;
  123. EXPORT_SYMBOL_GPL(balloon_stats);
  124. /* We increase/decrease in batches which fit in a page */
  125. static xen_pfn_t frame_list[PAGE_SIZE / sizeof(xen_pfn_t)];
  126. /* List of ballooned pages, threaded through the mem_map array. */
  127. static LIST_HEAD(ballooned_pages);
  128. static DECLARE_WAIT_QUEUE_HEAD(balloon_wq);
  129. /* Main work function, always executed in process context. */
  130. static void balloon_process(struct work_struct *work);
  131. static DECLARE_DELAYED_WORK(balloon_worker, balloon_process);
  132. /* When ballooning out (allocating memory to return to Xen) we don't really
  133. want the kernel to try too hard since that can trigger the oom killer. */
  134. #define GFP_BALLOON \
  135. (GFP_HIGHUSER | __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC)
  136. static void scrub_page(struct page *page)
  137. {
  138. #ifdef CONFIG_XEN_SCRUB_PAGES
  139. clear_highpage(page);
  140. #endif
  141. }
  142. /* balloon_append: add the given page to the balloon. */
  143. static void __balloon_append(struct page *page)
  144. {
  145. /* Lowmem is re-populated first, so highmem pages go at list tail. */
  146. if (PageHighMem(page)) {
  147. list_add_tail(&page->lru, &ballooned_pages);
  148. balloon_stats.balloon_high++;
  149. } else {
  150. list_add(&page->lru, &ballooned_pages);
  151. balloon_stats.balloon_low++;
  152. }
  153. wake_up(&balloon_wq);
  154. }
  155. static void balloon_append(struct page *page)
  156. {
  157. __balloon_append(page);
  158. }
  159. /* balloon_retrieve: rescue a page from the balloon, if it is not empty. */
  160. static struct page *balloon_retrieve(bool require_lowmem)
  161. {
  162. struct page *page;
  163. if (list_empty(&ballooned_pages))
  164. return NULL;
  165. page = list_entry(ballooned_pages.next, struct page, lru);
  166. if (require_lowmem && PageHighMem(page))
  167. return NULL;
  168. list_del(&page->lru);
  169. if (PageHighMem(page))
  170. balloon_stats.balloon_high--;
  171. else
  172. balloon_stats.balloon_low--;
  173. return page;
  174. }
  175. static struct page *balloon_next_page(struct page *page)
  176. {
  177. struct list_head *next = page->lru.next;
  178. if (next == &ballooned_pages)
  179. return NULL;
  180. return list_entry(next, struct page, lru);
  181. }
  182. static enum bp_state update_schedule(enum bp_state state)
  183. {
  184. if (state == BP_WAIT)
  185. return BP_WAIT;
  186. if (state == BP_ECANCELED)
  187. return BP_ECANCELED;
  188. if (state == BP_DONE) {
  189. balloon_stats.schedule_delay = 1;
  190. balloon_stats.retry_count = 1;
  191. return BP_DONE;
  192. }
  193. ++balloon_stats.retry_count;
  194. if (balloon_stats.max_retry_count != RETRY_UNLIMITED &&
  195. balloon_stats.retry_count > balloon_stats.max_retry_count) {
  196. balloon_stats.schedule_delay = 1;
  197. balloon_stats.retry_count = 1;
  198. return BP_ECANCELED;
  199. }
  200. balloon_stats.schedule_delay <<= 1;
  201. if (balloon_stats.schedule_delay > balloon_stats.max_schedule_delay)
  202. balloon_stats.schedule_delay = balloon_stats.max_schedule_delay;
  203. return BP_EAGAIN;
  204. }
  205. #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
  206. static void release_memory_resource(struct resource *resource)
  207. {
  208. if (!resource)
  209. return;
  210. /*
  211. * No need to reset region to identity mapped since we now
  212. * know that no I/O can be in this region
  213. */
  214. release_resource(resource);
  215. kfree(resource);
  216. }
  217. /*
  218. * Host memory not allocated to dom0. We can use this range for hotplug-based
  219. * ballooning.
  220. *
  221. * It's a type-less resource. Setting IORESOURCE_MEM will make resource
  222. * management algorithms (arch_remove_reservations()) look into guest e820,
  223. * which we don't want.
  224. */
  225. static struct resource hostmem_resource = {
  226. .name = "Host RAM",
  227. };
  228. void __attribute__((weak)) __init arch_xen_balloon_init(struct resource *res)
  229. {}
  230. static struct resource *additional_memory_resource(phys_addr_t size)
  231. {
  232. struct resource *res, *res_hostmem;
  233. int ret = -ENOMEM;
  234. res = kzalloc(sizeof(*res), GFP_KERNEL);
  235. if (!res)
  236. return NULL;
  237. res->name = "System RAM";
  238. res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
  239. res_hostmem = kzalloc(sizeof(*res), GFP_KERNEL);
  240. if (res_hostmem) {
  241. /* Try to grab a range from hostmem */
  242. res_hostmem->name = "Host memory";
  243. ret = allocate_resource(&hostmem_resource, res_hostmem,
  244. size, 0, -1,
  245. PAGES_PER_SECTION * PAGE_SIZE, NULL, NULL);
  246. }
  247. if (!ret) {
  248. /*
  249. * Insert this resource into iomem. Because hostmem_resource
  250. * tracks portion of guest e820 marked as UNUSABLE noone else
  251. * should try to use it.
  252. */
  253. res->start = res_hostmem->start;
  254. res->end = res_hostmem->end;
  255. ret = insert_resource(&iomem_resource, res);
  256. if (ret < 0) {
  257. pr_err("Can't insert iomem_resource [%llx - %llx]\n",
  258. res->start, res->end);
  259. release_memory_resource(res_hostmem);
  260. res_hostmem = NULL;
  261. res->start = res->end = 0;
  262. }
  263. }
  264. if (ret) {
  265. ret = allocate_resource(&iomem_resource, res,
  266. size, 0, -1,
  267. PAGES_PER_SECTION * PAGE_SIZE, NULL, NULL);
  268. if (ret < 0) {
  269. pr_err("Cannot allocate new System RAM resource\n");
  270. kfree(res);
  271. return NULL;
  272. }
  273. }
  274. #ifdef CONFIG_SPARSEMEM
  275. {
  276. unsigned long limit = 1UL << (MAX_PHYSMEM_BITS - PAGE_SHIFT);
  277. unsigned long pfn = res->start >> PAGE_SHIFT;
  278. if (pfn > limit) {
  279. pr_err("New System RAM resource outside addressable RAM (%lu > %lu)\n",
  280. pfn, limit);
  281. release_memory_resource(res);
  282. release_memory_resource(res_hostmem);
  283. return NULL;
  284. }
  285. }
  286. #endif
  287. return res;
  288. }
  289. static enum bp_state reserve_additional_memory(void)
  290. {
  291. long credit;
  292. struct resource *resource;
  293. int nid, rc;
  294. unsigned long balloon_hotplug;
  295. credit = balloon_stats.target_pages + balloon_stats.target_unpopulated
  296. - balloon_stats.total_pages;
  297. /*
  298. * Already hotplugged enough pages? Wait for them to be
  299. * onlined.
  300. */
  301. if (credit <= 0)
  302. return BP_WAIT;
  303. balloon_hotplug = round_up(credit, PAGES_PER_SECTION);
  304. resource = additional_memory_resource(balloon_hotplug * PAGE_SIZE);
  305. if (!resource)
  306. goto err;
  307. nid = memory_add_physaddr_to_nid(resource->start);
  308. #ifdef CONFIG_XEN_HAVE_PVMMU
  309. /*
  310. * We don't support PV MMU when Linux and Xen is using
  311. * different page granularity.
  312. */
  313. BUILD_BUG_ON(XEN_PAGE_SIZE != PAGE_SIZE);
  314. /*
  315. * add_memory() will build page tables for the new memory so
  316. * the p2m must contain invalid entries so the correct
  317. * non-present PTEs will be written.
  318. *
  319. * If a failure occurs, the original (identity) p2m entries
  320. * are not restored since this region is now known not to
  321. * conflict with any devices.
  322. */
  323. if (!xen_feature(XENFEAT_auto_translated_physmap)) {
  324. unsigned long pfn, i;
  325. pfn = PFN_DOWN(resource->start);
  326. for (i = 0; i < balloon_hotplug; i++) {
  327. if (!set_phys_to_machine(pfn + i, INVALID_P2M_ENTRY)) {
  328. pr_warn("set_phys_to_machine() failed, no memory added\n");
  329. goto err;
  330. }
  331. }
  332. }
  333. #endif
  334. /*
  335. * add_memory_resource() will call online_pages() which in its turn
  336. * will call xen_online_page() callback causing deadlock if we don't
  337. * release balloon_mutex here. Unlocking here is safe because the
  338. * callers drop the mutex before trying again.
  339. */
  340. mutex_unlock(&balloon_mutex);
  341. rc = add_memory_resource(nid, resource, memhp_auto_online);
  342. mutex_lock(&balloon_mutex);
  343. if (rc) {
  344. pr_warn("Cannot add additional memory (%i)\n", rc);
  345. goto err;
  346. }
  347. balloon_stats.total_pages += balloon_hotplug;
  348. return BP_WAIT;
  349. err:
  350. release_memory_resource(resource);
  351. return BP_ECANCELED;
  352. }
  353. static void xen_online_page(struct page *page)
  354. {
  355. __online_page_set_limits(page);
  356. mutex_lock(&balloon_mutex);
  357. __balloon_append(page);
  358. mutex_unlock(&balloon_mutex);
  359. }
  360. static int xen_memory_notifier(struct notifier_block *nb, unsigned long val, void *v)
  361. {
  362. if (val == MEM_ONLINE)
  363. schedule_delayed_work(&balloon_worker, 0);
  364. return NOTIFY_OK;
  365. }
  366. static struct notifier_block xen_memory_nb = {
  367. .notifier_call = xen_memory_notifier,
  368. .priority = 0
  369. };
  370. #else
  371. static enum bp_state reserve_additional_memory(void)
  372. {
  373. balloon_stats.target_pages = balloon_stats.current_pages;
  374. return BP_ECANCELED;
  375. }
  376. #endif /* CONFIG_XEN_BALLOON_MEMORY_HOTPLUG */
  377. static long current_credit(void)
  378. {
  379. return balloon_stats.target_pages - balloon_stats.current_pages;
  380. }
  381. static bool balloon_is_inflated(void)
  382. {
  383. return balloon_stats.balloon_low || balloon_stats.balloon_high;
  384. }
  385. static enum bp_state increase_reservation(unsigned long nr_pages)
  386. {
  387. int rc;
  388. unsigned long i;
  389. struct page *page;
  390. struct xen_memory_reservation reservation = {
  391. .address_bits = 0,
  392. .extent_order = EXTENT_ORDER,
  393. .domid = DOMID_SELF
  394. };
  395. if (nr_pages > ARRAY_SIZE(frame_list))
  396. nr_pages = ARRAY_SIZE(frame_list);
  397. page = list_first_entry_or_null(&ballooned_pages, struct page, lru);
  398. for (i = 0; i < nr_pages; i++) {
  399. if (!page) {
  400. nr_pages = i;
  401. break;
  402. }
  403. /* XENMEM_populate_physmap requires a PFN based on Xen
  404. * granularity.
  405. */
  406. frame_list[i] = page_to_xen_pfn(page);
  407. page = balloon_next_page(page);
  408. }
  409. set_xen_guest_handle(reservation.extent_start, frame_list);
  410. reservation.nr_extents = nr_pages;
  411. rc = HYPERVISOR_memory_op(XENMEM_populate_physmap, &reservation);
  412. if (rc <= 0)
  413. return BP_EAGAIN;
  414. for (i = 0; i < rc; i++) {
  415. page = balloon_retrieve(false);
  416. BUG_ON(page == NULL);
  417. #ifdef CONFIG_XEN_HAVE_PVMMU
  418. /*
  419. * We don't support PV MMU when Linux and Xen is using
  420. * different page granularity.
  421. */
  422. BUILD_BUG_ON(XEN_PAGE_SIZE != PAGE_SIZE);
  423. if (!xen_feature(XENFEAT_auto_translated_physmap)) {
  424. unsigned long pfn = page_to_pfn(page);
  425. set_phys_to_machine(pfn, frame_list[i]);
  426. /* Link back into the page tables if not highmem. */
  427. if (!PageHighMem(page)) {
  428. int ret;
  429. ret = HYPERVISOR_update_va_mapping(
  430. (unsigned long)__va(pfn << PAGE_SHIFT),
  431. mfn_pte(frame_list[i], PAGE_KERNEL),
  432. 0);
  433. BUG_ON(ret);
  434. }
  435. }
  436. #endif
  437. /* Relinquish the page back to the allocator. */
  438. free_reserved_page(page);
  439. }
  440. balloon_stats.current_pages += rc;
  441. return BP_DONE;
  442. }
  443. static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
  444. {
  445. enum bp_state state = BP_DONE;
  446. unsigned long i;
  447. struct page *page, *tmp;
  448. int ret;
  449. struct xen_memory_reservation reservation = {
  450. .address_bits = 0,
  451. .extent_order = EXTENT_ORDER,
  452. .domid = DOMID_SELF
  453. };
  454. LIST_HEAD(pages);
  455. if (nr_pages > ARRAY_SIZE(frame_list))
  456. nr_pages = ARRAY_SIZE(frame_list);
  457. for (i = 0; i < nr_pages; i++) {
  458. page = alloc_page(gfp);
  459. if (page == NULL) {
  460. nr_pages = i;
  461. state = BP_EAGAIN;
  462. break;
  463. }
  464. adjust_managed_page_count(page, -1);
  465. scrub_page(page);
  466. list_add(&page->lru, &pages);
  467. }
  468. /*
  469. * Ensure that ballooned highmem pages don't have kmaps.
  470. *
  471. * Do this before changing the p2m as kmap_flush_unused()
  472. * reads PTEs to obtain pages (and hence needs the original
  473. * p2m entry).
  474. */
  475. kmap_flush_unused();
  476. /*
  477. * Setup the frame, update direct mapping, invalidate P2M,
  478. * and add to balloon.
  479. */
  480. i = 0;
  481. list_for_each_entry_safe(page, tmp, &pages, lru) {
  482. /* XENMEM_decrease_reservation requires a GFN */
  483. frame_list[i++] = xen_page_to_gfn(page);
  484. #ifdef CONFIG_XEN_HAVE_PVMMU
  485. /*
  486. * We don't support PV MMU when Linux and Xen is using
  487. * different page granularity.
  488. */
  489. BUILD_BUG_ON(XEN_PAGE_SIZE != PAGE_SIZE);
  490. if (!xen_feature(XENFEAT_auto_translated_physmap)) {
  491. unsigned long pfn = page_to_pfn(page);
  492. if (!PageHighMem(page)) {
  493. ret = HYPERVISOR_update_va_mapping(
  494. (unsigned long)__va(pfn << PAGE_SHIFT),
  495. __pte_ma(0), 0);
  496. BUG_ON(ret);
  497. }
  498. __set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
  499. }
  500. #endif
  501. list_del(&page->lru);
  502. balloon_append(page);
  503. }
  504. flush_tlb_all();
  505. set_xen_guest_handle(reservation.extent_start, frame_list);
  506. reservation.nr_extents = nr_pages;
  507. ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation);
  508. BUG_ON(ret != nr_pages);
  509. balloon_stats.current_pages -= nr_pages;
  510. return state;
  511. }
  512. /*
  513. * As this is a work item it is guaranteed to run as a single instance only.
  514. * We may of course race updates of the target counts (which are protected
  515. * by the balloon lock), or with changes to the Xen hard limit, but we will
  516. * recover from these in time.
  517. */
  518. static void balloon_process(struct work_struct *work)
  519. {
  520. enum bp_state state = BP_DONE;
  521. long credit;
  522. do {
  523. mutex_lock(&balloon_mutex);
  524. credit = current_credit();
  525. if (credit > 0) {
  526. if (balloon_is_inflated())
  527. state = increase_reservation(credit);
  528. else
  529. state = reserve_additional_memory();
  530. }
  531. if (credit < 0)
  532. state = decrease_reservation(-credit, GFP_BALLOON);
  533. state = update_schedule(state);
  534. mutex_unlock(&balloon_mutex);
  535. cond_resched();
  536. } while (credit && state == BP_DONE);
  537. /* Schedule more work if there is some still to be done. */
  538. if (state == BP_EAGAIN)
  539. schedule_delayed_work(&balloon_worker, balloon_stats.schedule_delay * HZ);
  540. }
  541. /* Resets the Xen limit, sets new target, and kicks off processing. */
  542. void balloon_set_new_target(unsigned long target)
  543. {
  544. /* No need for lock. Not read-modify-write updates. */
  545. balloon_stats.target_pages = target;
  546. schedule_delayed_work(&balloon_worker, 0);
  547. }
  548. EXPORT_SYMBOL_GPL(balloon_set_new_target);
  549. static int add_ballooned_pages(int nr_pages)
  550. {
  551. enum bp_state st;
  552. if (xen_hotplug_unpopulated) {
  553. st = reserve_additional_memory();
  554. if (st != BP_ECANCELED) {
  555. mutex_unlock(&balloon_mutex);
  556. wait_event(balloon_wq,
  557. !list_empty(&ballooned_pages));
  558. mutex_lock(&balloon_mutex);
  559. return 0;
  560. }
  561. }
  562. st = decrease_reservation(nr_pages, GFP_USER);
  563. if (st != BP_DONE)
  564. return -ENOMEM;
  565. return 0;
  566. }
  567. /**
  568. * alloc_xenballooned_pages - get pages that have been ballooned out
  569. * @nr_pages: Number of pages to get
  570. * @pages: pages returned
  571. * @return 0 on success, error otherwise
  572. */
  573. int alloc_xenballooned_pages(int nr_pages, struct page **pages)
  574. {
  575. int pgno = 0;
  576. struct page *page;
  577. int ret;
  578. mutex_lock(&balloon_mutex);
  579. balloon_stats.target_unpopulated += nr_pages;
  580. while (pgno < nr_pages) {
  581. page = balloon_retrieve(true);
  582. if (page) {
  583. pages[pgno++] = page;
  584. #ifdef CONFIG_XEN_HAVE_PVMMU
  585. /*
  586. * We don't support PV MMU when Linux and Xen is using
  587. * different page granularity.
  588. */
  589. BUILD_BUG_ON(XEN_PAGE_SIZE != PAGE_SIZE);
  590. if (!xen_feature(XENFEAT_auto_translated_physmap)) {
  591. ret = xen_alloc_p2m_entry(page_to_pfn(page));
  592. if (ret < 0)
  593. goto out_undo;
  594. }
  595. #endif
  596. } else {
  597. ret = add_ballooned_pages(nr_pages - pgno);
  598. if (ret < 0)
  599. goto out_undo;
  600. }
  601. }
  602. mutex_unlock(&balloon_mutex);
  603. return 0;
  604. out_undo:
  605. mutex_unlock(&balloon_mutex);
  606. free_xenballooned_pages(pgno, pages);
  607. return ret;
  608. }
  609. EXPORT_SYMBOL(alloc_xenballooned_pages);
  610. /**
  611. * free_xenballooned_pages - return pages retrieved with get_ballooned_pages
  612. * @nr_pages: Number of pages
  613. * @pages: pages to return
  614. */
  615. void free_xenballooned_pages(int nr_pages, struct page **pages)
  616. {
  617. int i;
  618. mutex_lock(&balloon_mutex);
  619. for (i = 0; i < nr_pages; i++) {
  620. if (pages[i])
  621. balloon_append(pages[i]);
  622. }
  623. balloon_stats.target_unpopulated -= nr_pages;
  624. /* The balloon may be too large now. Shrink it if needed. */
  625. if (current_credit())
  626. schedule_delayed_work(&balloon_worker, 0);
  627. mutex_unlock(&balloon_mutex);
  628. }
  629. EXPORT_SYMBOL(free_xenballooned_pages);
  630. #ifdef CONFIG_XEN_PV
  631. static void __init balloon_add_region(unsigned long start_pfn,
  632. unsigned long pages)
  633. {
  634. unsigned long pfn, extra_pfn_end;
  635. struct page *page;
  636. /*
  637. * If the amount of usable memory has been limited (e.g., with
  638. * the 'mem' command line parameter), don't add pages beyond
  639. * this limit.
  640. */
  641. extra_pfn_end = min(max_pfn, start_pfn + pages);
  642. for (pfn = start_pfn; pfn < extra_pfn_end; pfn++) {
  643. page = pfn_to_page(pfn);
  644. /* totalram_pages and totalhigh_pages do not
  645. include the boot-time balloon extension, so
  646. don't subtract from it. */
  647. __balloon_append(page);
  648. }
  649. balloon_stats.total_pages += extra_pfn_end - start_pfn;
  650. }
  651. #endif
  652. static int __init balloon_init(void)
  653. {
  654. if (!xen_domain())
  655. return -ENODEV;
  656. pr_info("Initialising balloon driver\n");
  657. #ifdef CONFIG_XEN_PV
  658. balloon_stats.current_pages = xen_pv_domain()
  659. ? min(xen_start_info->nr_pages - xen_released_pages, max_pfn)
  660. : get_num_physpages();
  661. #else
  662. balloon_stats.current_pages = get_num_physpages();
  663. #endif
  664. balloon_stats.target_pages = balloon_stats.current_pages;
  665. balloon_stats.balloon_low = 0;
  666. balloon_stats.balloon_high = 0;
  667. balloon_stats.total_pages = balloon_stats.current_pages;
  668. balloon_stats.schedule_delay = 1;
  669. balloon_stats.max_schedule_delay = 32;
  670. balloon_stats.retry_count = 1;
  671. balloon_stats.max_retry_count = RETRY_UNLIMITED;
  672. #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
  673. set_online_page_callback(&xen_online_page);
  674. register_memory_notifier(&xen_memory_nb);
  675. register_sysctl_table(xen_root);
  676. arch_xen_balloon_init(&hostmem_resource);
  677. #endif
  678. #ifdef CONFIG_XEN_PV
  679. {
  680. int i;
  681. /*
  682. * Initialize the balloon with pages from the extra memory
  683. * regions (see arch/x86/xen/setup.c).
  684. */
  685. for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++)
  686. if (xen_extra_mem[i].n_pfns)
  687. balloon_add_region(xen_extra_mem[i].start_pfn,
  688. xen_extra_mem[i].n_pfns);
  689. }
  690. #endif
  691. /* Init the xen-balloon driver. */
  692. xen_balloon_init();
  693. return 0;
  694. }
  695. subsys_initcall(balloon_init);