virtio_balloon.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710
  1. /*
  2. * Virtio balloon implementation, inspired by Dor Laor and Marcelo
  3. * Tosatti's implementations.
  4. *
  5. * Copyright 2008 Rusty Russell IBM Corporation
  6. *
  7. * This program is free software; you can redistribute it and/or modify
  8. * it under the terms of the GNU General Public License as published by
  9. * the Free Software Foundation; either version 2 of the License, or
  10. * (at your option) any later version.
  11. *
  12. * This program is distributed in the hope that it will be useful,
  13. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. * GNU General Public License for more details.
  16. *
  17. * You should have received a copy of the GNU General Public License
  18. * along with this program; if not, write to the Free Software
  19. * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  20. */
  21. #include <linux/virtio.h>
  22. #include <linux/virtio_balloon.h>
  23. #include <linux/swap.h>
  24. #include <linux/workqueue.h>
  25. #include <linux/delay.h>
  26. #include <linux/slab.h>
  27. #include <linux/module.h>
  28. #include <linux/balloon_compaction.h>
  29. #include <linux/oom.h>
  30. #include <linux/wait.h>
  31. #include <linux/mm.h>
  32. #include <linux/mount.h>
  33. #include <linux/magic.h>
  34. /*
  35. * Balloon device works in 4K page units. So each page is pointed to by
  36. * multiple balloon pages. All memory counters in this driver are in balloon
  37. * page units.
  38. */
  39. #define VIRTIO_BALLOON_PAGES_PER_PAGE (unsigned)(PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT)
  40. #define VIRTIO_BALLOON_ARRAY_PFNS_MAX 256
  41. #define OOM_VBALLOON_DEFAULT_PAGES 256
  42. #define VIRTBALLOON_OOM_NOTIFY_PRIORITY 80
  43. static int oom_pages = OOM_VBALLOON_DEFAULT_PAGES;
  44. module_param(oom_pages, int, S_IRUSR | S_IWUSR);
  45. MODULE_PARM_DESC(oom_pages, "pages to free on OOM");
  46. #ifdef CONFIG_BALLOON_COMPACTION
  47. static struct vfsmount *balloon_mnt;
  48. #endif
  49. struct virtio_balloon {
  50. struct virtio_device *vdev;
  51. struct virtqueue *inflate_vq, *deflate_vq, *stats_vq;
  52. /* The balloon servicing is delegated to a freezable workqueue. */
  53. struct work_struct update_balloon_stats_work;
  54. struct work_struct update_balloon_size_work;
  55. /* Prevent updating balloon when it is being canceled. */
  56. spinlock_t stop_update_lock;
  57. bool stop_update;
  58. /* Waiting for host to ack the pages we released. */
  59. wait_queue_head_t acked;
  60. /* Number of balloon pages we've told the Host we're not using. */
  61. unsigned int num_pages;
  62. /*
  63. * The pages we've told the Host we're not using are enqueued
  64. * at vb_dev_info->pages list.
  65. * Each page on this list adds VIRTIO_BALLOON_PAGES_PER_PAGE
  66. * to num_pages above.
  67. */
  68. struct balloon_dev_info vb_dev_info;
  69. /* Synchronize access/update to this struct virtio_balloon elements */
  70. struct mutex balloon_lock;
  71. /* The array of pfns we tell the Host about. */
  72. unsigned int num_pfns;
  73. __virtio32 pfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX];
  74. /* Memory statistics */
  75. struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR];
  76. /* To register callback in oom notifier call chain */
  77. struct notifier_block nb;
  78. };
  79. static struct virtio_device_id id_table[] = {
  80. { VIRTIO_ID_BALLOON, VIRTIO_DEV_ANY_ID },
  81. { 0 },
  82. };
  83. static u32 page_to_balloon_pfn(struct page *page)
  84. {
  85. unsigned long pfn = page_to_pfn(page);
  86. BUILD_BUG_ON(PAGE_SHIFT < VIRTIO_BALLOON_PFN_SHIFT);
  87. /* Convert pfn from Linux page size to balloon page size. */
  88. return pfn * VIRTIO_BALLOON_PAGES_PER_PAGE;
  89. }
  90. static void balloon_ack(struct virtqueue *vq)
  91. {
  92. struct virtio_balloon *vb = vq->vdev->priv;
  93. wake_up(&vb->acked);
  94. }
  95. static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq)
  96. {
  97. struct scatterlist sg;
  98. unsigned int len;
  99. sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns);
  100. /* We should always be able to add one buffer to an empty queue. */
  101. virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);
  102. virtqueue_kick(vq);
  103. /* When host has read buffer, this completes via balloon_ack */
  104. wait_event(vb->acked, virtqueue_get_buf(vq, &len));
  105. }
  106. static void set_page_pfns(struct virtio_balloon *vb,
  107. __virtio32 pfns[], struct page *page)
  108. {
  109. unsigned int i;
  110. /*
  111. * Set balloon pfns pointing at this page.
  112. * Note that the first pfn points at start of the page.
  113. */
  114. for (i = 0; i < VIRTIO_BALLOON_PAGES_PER_PAGE; i++)
  115. pfns[i] = cpu_to_virtio32(vb->vdev,
  116. page_to_balloon_pfn(page) + i);
  117. }
  118. static unsigned fill_balloon(struct virtio_balloon *vb, size_t num)
  119. {
  120. unsigned num_allocated_pages;
  121. unsigned num_pfns;
  122. struct page *page;
  123. LIST_HEAD(pages);
  124. /* We can only do one array worth at a time. */
  125. num = min(num, ARRAY_SIZE(vb->pfns));
  126. for (num_pfns = 0; num_pfns < num;
  127. num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
  128. struct page *page = balloon_page_alloc();
  129. if (!page) {
  130. dev_info_ratelimited(&vb->vdev->dev,
  131. "Out of puff! Can't get %u pages\n",
  132. VIRTIO_BALLOON_PAGES_PER_PAGE);
  133. /* Sleep for at least 1/5 of a second before retry. */
  134. msleep(200);
  135. break;
  136. }
  137. balloon_page_push(&pages, page);
  138. }
  139. mutex_lock(&vb->balloon_lock);
  140. vb->num_pfns = 0;
  141. while ((page = balloon_page_pop(&pages))) {
  142. balloon_page_enqueue(&vb->vb_dev_info, page);
  143. set_page_pfns(vb, vb->pfns + vb->num_pfns, page);
  144. vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;
  145. if (!virtio_has_feature(vb->vdev,
  146. VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
  147. adjust_managed_page_count(page, -1);
  148. vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE;
  149. }
  150. num_allocated_pages = vb->num_pfns;
  151. /* Did we get any? */
  152. if (vb->num_pfns != 0)
  153. tell_host(vb, vb->inflate_vq);
  154. mutex_unlock(&vb->balloon_lock);
  155. return num_allocated_pages;
  156. }
  157. static void release_pages_balloon(struct virtio_balloon *vb,
  158. struct list_head *pages)
  159. {
  160. struct page *page, *next;
  161. list_for_each_entry_safe(page, next, pages, lru) {
  162. if (!virtio_has_feature(vb->vdev,
  163. VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
  164. adjust_managed_page_count(page, 1);
  165. list_del(&page->lru);
  166. put_page(page); /* balloon reference */
  167. }
  168. }
  169. static unsigned leak_balloon(struct virtio_balloon *vb, size_t num)
  170. {
  171. unsigned num_freed_pages;
  172. struct page *page;
  173. struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info;
  174. LIST_HEAD(pages);
  175. /* We can only do one array worth at a time. */
  176. num = min(num, ARRAY_SIZE(vb->pfns));
  177. mutex_lock(&vb->balloon_lock);
  178. /* We can't release more pages than taken */
  179. num = min(num, (size_t)vb->num_pages);
  180. for (vb->num_pfns = 0; vb->num_pfns < num;
  181. vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
  182. page = balloon_page_dequeue(vb_dev_info);
  183. if (!page)
  184. break;
  185. set_page_pfns(vb, vb->pfns + vb->num_pfns, page);
  186. list_add(&page->lru, &pages);
  187. vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE;
  188. }
  189. num_freed_pages = vb->num_pfns;
  190. /*
  191. * Note that if
  192. * virtio_has_feature(vdev, VIRTIO_BALLOON_F_MUST_TELL_HOST);
  193. * is true, we *have* to do it in this order
  194. */
  195. if (vb->num_pfns != 0)
  196. tell_host(vb, vb->deflate_vq);
  197. release_pages_balloon(vb, &pages);
  198. mutex_unlock(&vb->balloon_lock);
  199. return num_freed_pages;
  200. }
  201. static inline void update_stat(struct virtio_balloon *vb, int idx,
  202. u16 tag, u64 val)
  203. {
  204. BUG_ON(idx >= VIRTIO_BALLOON_S_NR);
  205. vb->stats[idx].tag = cpu_to_virtio16(vb->vdev, tag);
  206. vb->stats[idx].val = cpu_to_virtio64(vb->vdev, val);
  207. }
  208. #define pages_to_bytes(x) ((u64)(x) << PAGE_SHIFT)
  209. static unsigned int update_balloon_stats(struct virtio_balloon *vb)
  210. {
  211. unsigned long events[NR_VM_EVENT_ITEMS];
  212. struct sysinfo i;
  213. unsigned int idx = 0;
  214. long available;
  215. unsigned long caches;
  216. all_vm_events(events);
  217. si_meminfo(&i);
  218. available = si_mem_available();
  219. caches = global_node_page_state(NR_FILE_PAGES);
  220. #ifdef CONFIG_VM_EVENT_COUNTERS
  221. update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN,
  222. pages_to_bytes(events[PSWPIN]));
  223. update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT,
  224. pages_to_bytes(events[PSWPOUT]));
  225. update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]);
  226. update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]);
  227. #endif
  228. update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE,
  229. pages_to_bytes(i.freeram));
  230. update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT,
  231. pages_to_bytes(i.totalram));
  232. update_stat(vb, idx++, VIRTIO_BALLOON_S_AVAIL,
  233. pages_to_bytes(available));
  234. update_stat(vb, idx++, VIRTIO_BALLOON_S_CACHES,
  235. pages_to_bytes(caches));
  236. return idx;
  237. }
  238. /*
  239. * While most virtqueues communicate guest-initiated requests to the hypervisor,
  240. * the stats queue operates in reverse. The driver initializes the virtqueue
  241. * with a single buffer. From that point forward, all conversations consist of
  242. * a hypervisor request (a call to this function) which directs us to refill
  243. * the virtqueue with a fresh stats buffer. Since stats collection can sleep,
  244. * we delegate the job to a freezable workqueue that will do the actual work via
  245. * stats_handle_request().
  246. */
  247. static void stats_request(struct virtqueue *vq)
  248. {
  249. struct virtio_balloon *vb = vq->vdev->priv;
  250. spin_lock(&vb->stop_update_lock);
  251. if (!vb->stop_update)
  252. queue_work(system_freezable_wq, &vb->update_balloon_stats_work);
  253. spin_unlock(&vb->stop_update_lock);
  254. }
  255. static void stats_handle_request(struct virtio_balloon *vb)
  256. {
  257. struct virtqueue *vq;
  258. struct scatterlist sg;
  259. unsigned int len, num_stats;
  260. num_stats = update_balloon_stats(vb);
  261. vq = vb->stats_vq;
  262. if (!virtqueue_get_buf(vq, &len))
  263. return;
  264. sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats);
  265. virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);
  266. virtqueue_kick(vq);
  267. }
  268. static void virtballoon_changed(struct virtio_device *vdev)
  269. {
  270. struct virtio_balloon *vb = vdev->priv;
  271. unsigned long flags;
  272. spin_lock_irqsave(&vb->stop_update_lock, flags);
  273. if (!vb->stop_update)
  274. queue_work(system_freezable_wq, &vb->update_balloon_size_work);
  275. spin_unlock_irqrestore(&vb->stop_update_lock, flags);
  276. }
  277. static inline s64 towards_target(struct virtio_balloon *vb)
  278. {
  279. s64 target;
  280. u32 num_pages;
  281. virtio_cread(vb->vdev, struct virtio_balloon_config, num_pages,
  282. &num_pages);
  283. /* Legacy balloon config space is LE, unlike all other devices. */
  284. if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1))
  285. num_pages = le32_to_cpu((__force __le32)num_pages);
  286. target = num_pages;
  287. return target - vb->num_pages;
  288. }
  289. static void update_balloon_size(struct virtio_balloon *vb)
  290. {
  291. u32 actual = vb->num_pages;
  292. /* Legacy balloon config space is LE, unlike all other devices. */
  293. if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1))
  294. actual = (__force u32)cpu_to_le32(actual);
  295. virtio_cwrite(vb->vdev, struct virtio_balloon_config, actual,
  296. &actual);
  297. }
  298. /*
  299. * virtballoon_oom_notify - release pages when system is under severe
  300. * memory pressure (called from out_of_memory())
  301. * @self : notifier block struct
  302. * @dummy: not used
  303. * @parm : returned - number of freed pages
  304. *
  305. * The balancing of memory by use of the virtio balloon should not cause
  306. * the termination of processes while there are pages in the balloon.
  307. * If virtio balloon manages to release some memory, it will make the
  308. * system return and retry the allocation that forced the OOM killer
  309. * to run.
  310. */
  311. static int virtballoon_oom_notify(struct notifier_block *self,
  312. unsigned long dummy, void *parm)
  313. {
  314. struct virtio_balloon *vb;
  315. unsigned long *freed;
  316. unsigned num_freed_pages;
  317. vb = container_of(self, struct virtio_balloon, nb);
  318. if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
  319. return NOTIFY_OK;
  320. freed = parm;
  321. num_freed_pages = leak_balloon(vb, oom_pages);
  322. update_balloon_size(vb);
  323. *freed += num_freed_pages;
  324. return NOTIFY_OK;
  325. }
  326. static void update_balloon_stats_func(struct work_struct *work)
  327. {
  328. struct virtio_balloon *vb;
  329. vb = container_of(work, struct virtio_balloon,
  330. update_balloon_stats_work);
  331. stats_handle_request(vb);
  332. }
  333. static void update_balloon_size_func(struct work_struct *work)
  334. {
  335. struct virtio_balloon *vb;
  336. s64 diff;
  337. vb = container_of(work, struct virtio_balloon,
  338. update_balloon_size_work);
  339. diff = towards_target(vb);
  340. if (diff > 0)
  341. diff -= fill_balloon(vb, diff);
  342. else if (diff < 0)
  343. diff += leak_balloon(vb, -diff);
  344. update_balloon_size(vb);
  345. if (diff)
  346. queue_work(system_freezable_wq, work);
  347. }
  348. static int init_vqs(struct virtio_balloon *vb)
  349. {
  350. struct virtqueue *vqs[3];
  351. vq_callback_t *callbacks[] = { balloon_ack, balloon_ack, stats_request };
  352. static const char * const names[] = { "inflate", "deflate", "stats" };
  353. int err, nvqs;
  354. /*
  355. * We expect two virtqueues: inflate and deflate, and
  356. * optionally stat.
  357. */
  358. nvqs = virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ) ? 3 : 2;
  359. err = virtio_find_vqs(vb->vdev, nvqs, vqs, callbacks, names, NULL);
  360. if (err)
  361. return err;
  362. vb->inflate_vq = vqs[0];
  363. vb->deflate_vq = vqs[1];
  364. if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
  365. struct scatterlist sg;
  366. unsigned int num_stats;
  367. vb->stats_vq = vqs[2];
  368. /*
  369. * Prime this virtqueue with one buffer so the hypervisor can
  370. * use it to signal us later (it can't be broken yet!).
  371. */
  372. num_stats = update_balloon_stats(vb);
  373. sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats);
  374. if (virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb, GFP_KERNEL)
  375. < 0)
  376. BUG();
  377. virtqueue_kick(vb->stats_vq);
  378. }
  379. return 0;
  380. }
  381. #ifdef CONFIG_BALLOON_COMPACTION
  382. /*
  383. * virtballoon_migratepage - perform the balloon page migration on behalf of
  384. * a compation thread. (called under page lock)
  385. * @vb_dev_info: the balloon device
  386. * @newpage: page that will replace the isolated page after migration finishes.
  387. * @page : the isolated (old) page that is about to be migrated to newpage.
  388. * @mode : compaction mode -- not used for balloon page migration.
  389. *
  390. * After a ballooned page gets isolated by compaction procedures, this is the
  391. * function that performs the page migration on behalf of a compaction thread
  392. * The page migration for virtio balloon is done in a simple swap fashion which
  393. * follows these two macro steps:
  394. * 1) insert newpage into vb->pages list and update the host about it;
  395. * 2) update the host about the old page removed from vb->pages list;
  396. *
  397. * This function preforms the balloon page migration task.
  398. * Called through balloon_mapping->a_ops->migratepage
  399. */
  400. static int virtballoon_migratepage(struct balloon_dev_info *vb_dev_info,
  401. struct page *newpage, struct page *page, enum migrate_mode mode)
  402. {
  403. struct virtio_balloon *vb = container_of(vb_dev_info,
  404. struct virtio_balloon, vb_dev_info);
  405. unsigned long flags;
  406. /*
  407. * In order to avoid lock contention while migrating pages concurrently
  408. * to leak_balloon() or fill_balloon() we just give up the balloon_lock
  409. * this turn, as it is easier to retry the page migration later.
  410. * This also prevents fill_balloon() getting stuck into a mutex
  411. * recursion in the case it ends up triggering memory compaction
  412. * while it is attempting to inflate the ballon.
  413. */
  414. if (!mutex_trylock(&vb->balloon_lock))
  415. return -EAGAIN;
  416. get_page(newpage); /* balloon reference */
  417. /* balloon's page migration 1st step -- inflate "newpage" */
  418. spin_lock_irqsave(&vb_dev_info->pages_lock, flags);
  419. balloon_page_insert(vb_dev_info, newpage);
  420. vb_dev_info->isolated_pages--;
  421. __count_vm_event(BALLOON_MIGRATE);
  422. spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags);
  423. vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
  424. set_page_pfns(vb, vb->pfns, newpage);
  425. tell_host(vb, vb->inflate_vq);
  426. /* balloon's page migration 2nd step -- deflate "page" */
  427. balloon_page_delete(page);
  428. vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
  429. set_page_pfns(vb, vb->pfns, page);
  430. tell_host(vb, vb->deflate_vq);
  431. mutex_unlock(&vb->balloon_lock);
  432. put_page(page); /* balloon reference */
  433. return MIGRATEPAGE_SUCCESS;
  434. }
  435. static struct dentry *balloon_mount(struct file_system_type *fs_type,
  436. int flags, const char *dev_name, void *data)
  437. {
  438. static const struct dentry_operations ops = {
  439. .d_dname = simple_dname,
  440. };
  441. return mount_pseudo(fs_type, "balloon-kvm:", NULL, &ops,
  442. BALLOON_KVM_MAGIC);
  443. }
  444. static struct file_system_type balloon_fs = {
  445. .name = "balloon-kvm",
  446. .mount = balloon_mount,
  447. .kill_sb = kill_anon_super,
  448. };
  449. #endif /* CONFIG_BALLOON_COMPACTION */
  450. static int virtballoon_probe(struct virtio_device *vdev)
  451. {
  452. struct virtio_balloon *vb;
  453. int err;
  454. if (!vdev->config->get) {
  455. dev_err(&vdev->dev, "%s failure: config access disabled\n",
  456. __func__);
  457. return -EINVAL;
  458. }
  459. vdev->priv = vb = kmalloc(sizeof(*vb), GFP_KERNEL);
  460. if (!vb) {
  461. err = -ENOMEM;
  462. goto out;
  463. }
  464. INIT_WORK(&vb->update_balloon_stats_work, update_balloon_stats_func);
  465. INIT_WORK(&vb->update_balloon_size_work, update_balloon_size_func);
  466. spin_lock_init(&vb->stop_update_lock);
  467. vb->stop_update = false;
  468. vb->num_pages = 0;
  469. mutex_init(&vb->balloon_lock);
  470. init_waitqueue_head(&vb->acked);
  471. vb->vdev = vdev;
  472. balloon_devinfo_init(&vb->vb_dev_info);
  473. err = init_vqs(vb);
  474. if (err)
  475. goto out_free_vb;
  476. vb->nb.notifier_call = virtballoon_oom_notify;
  477. vb->nb.priority = VIRTBALLOON_OOM_NOTIFY_PRIORITY;
  478. err = register_oom_notifier(&vb->nb);
  479. if (err < 0)
  480. goto out_del_vqs;
  481. #ifdef CONFIG_BALLOON_COMPACTION
  482. balloon_mnt = kern_mount(&balloon_fs);
  483. if (IS_ERR(balloon_mnt)) {
  484. err = PTR_ERR(balloon_mnt);
  485. unregister_oom_notifier(&vb->nb);
  486. goto out_del_vqs;
  487. }
  488. vb->vb_dev_info.migratepage = virtballoon_migratepage;
  489. vb->vb_dev_info.inode = alloc_anon_inode(balloon_mnt->mnt_sb);
  490. if (IS_ERR(vb->vb_dev_info.inode)) {
  491. err = PTR_ERR(vb->vb_dev_info.inode);
  492. kern_unmount(balloon_mnt);
  493. unregister_oom_notifier(&vb->nb);
  494. vb->vb_dev_info.inode = NULL;
  495. goto out_del_vqs;
  496. }
  497. vb->vb_dev_info.inode->i_mapping->a_ops = &balloon_aops;
  498. #endif
  499. virtio_device_ready(vdev);
  500. if (towards_target(vb))
  501. virtballoon_changed(vdev);
  502. return 0;
  503. out_del_vqs:
  504. vdev->config->del_vqs(vdev);
  505. out_free_vb:
  506. kfree(vb);
  507. out:
  508. return err;
  509. }
  510. static void remove_common(struct virtio_balloon *vb)
  511. {
  512. /* There might be pages left in the balloon: free them. */
  513. while (vb->num_pages)
  514. leak_balloon(vb, vb->num_pages);
  515. update_balloon_size(vb);
  516. /* Now we reset the device so we can clean up the queues. */
  517. vb->vdev->config->reset(vb->vdev);
  518. vb->vdev->config->del_vqs(vb->vdev);
  519. }
  520. static void virtballoon_remove(struct virtio_device *vdev)
  521. {
  522. struct virtio_balloon *vb = vdev->priv;
  523. unregister_oom_notifier(&vb->nb);
  524. spin_lock_irq(&vb->stop_update_lock);
  525. vb->stop_update = true;
  526. spin_unlock_irq(&vb->stop_update_lock);
  527. cancel_work_sync(&vb->update_balloon_size_work);
  528. cancel_work_sync(&vb->update_balloon_stats_work);
  529. remove_common(vb);
  530. #ifdef CONFIG_BALLOON_COMPACTION
  531. if (vb->vb_dev_info.inode)
  532. iput(vb->vb_dev_info.inode);
  533. kern_unmount(balloon_mnt);
  534. #endif
  535. kfree(vb);
  536. }
  537. #ifdef CONFIG_PM_SLEEP
  538. static int virtballoon_freeze(struct virtio_device *vdev)
  539. {
  540. struct virtio_balloon *vb = vdev->priv;
  541. /*
  542. * The workqueue is already frozen by the PM core before this
  543. * function is called.
  544. */
  545. remove_common(vb);
  546. return 0;
  547. }
  548. static int virtballoon_restore(struct virtio_device *vdev)
  549. {
  550. struct virtio_balloon *vb = vdev->priv;
  551. int ret;
  552. ret = init_vqs(vdev->priv);
  553. if (ret)
  554. return ret;
  555. virtio_device_ready(vdev);
  556. if (towards_target(vb))
  557. virtballoon_changed(vdev);
  558. update_balloon_size(vb);
  559. return 0;
  560. }
  561. #endif
  562. static int virtballoon_validate(struct virtio_device *vdev)
  563. {
  564. __virtio_clear_bit(vdev, VIRTIO_F_IOMMU_PLATFORM);
  565. return 0;
  566. }
  567. static unsigned int features[] = {
  568. VIRTIO_BALLOON_F_MUST_TELL_HOST,
  569. VIRTIO_BALLOON_F_STATS_VQ,
  570. VIRTIO_BALLOON_F_DEFLATE_ON_OOM,
  571. };
  572. static struct virtio_driver virtio_balloon_driver = {
  573. .feature_table = features,
  574. .feature_table_size = ARRAY_SIZE(features),
  575. .driver.name = KBUILD_MODNAME,
  576. .driver.owner = THIS_MODULE,
  577. .id_table = id_table,
  578. .validate = virtballoon_validate,
  579. .probe = virtballoon_probe,
  580. .remove = virtballoon_remove,
  581. .config_changed = virtballoon_changed,
  582. #ifdef CONFIG_PM_SLEEP
  583. .freeze = virtballoon_freeze,
  584. .restore = virtballoon_restore,
  585. #endif
  586. };
  587. module_virtio_driver(virtio_balloon_driver);
  588. MODULE_DEVICE_TABLE(virtio, id_table);
  589. MODULE_DESCRIPTION("Virtio balloon driver");
  590. MODULE_LICENSE("GPL");