eventfd.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928
  1. /*
  2. * kvm eventfd support - use eventfd objects to signal various KVM events
  3. *
  4. * Copyright 2009 Novell. All Rights Reserved.
  5. * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  6. *
  7. * Author:
  8. * Gregory Haskins <ghaskins@novell.com>
  9. *
  10. * This file is free software; you can redistribute it and/or modify
  11. * it under the terms of version 2 of the GNU General Public License
  12. * as published by the Free Software Foundation.
  13. *
  14. * This program is distributed in the hope that it will be useful,
  15. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17. * GNU General Public License for more details.
  18. *
  19. * You should have received a copy of the GNU General Public License
  20. * along with this program; if not, write to the Free Software Foundation,
  21. * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
  22. */
  23. #include <linux/kvm_host.h>
  24. #include <linux/kvm.h>
  25. #include <linux/workqueue.h>
  26. #include <linux/syscalls.h>
  27. #include <linux/wait.h>
  28. #include <linux/poll.h>
  29. #include <linux/file.h>
  30. #include <linux/list.h>
  31. #include <linux/eventfd.h>
  32. #include <linux/kernel.h>
  33. #include <linux/srcu.h>
  34. #include <linux/slab.h>
  35. #include <linux/seqlock.h>
  36. #include <trace/events/kvm.h>
  37. #include <kvm/iodev.h>
  38. #ifdef CONFIG_HAVE_KVM_IRQFD
  39. /*
  40. * --------------------------------------------------------------------
  41. * irqfd: Allows an fd to be used to inject an interrupt to the guest
  42. *
  43. * Credit goes to Avi Kivity for the original idea.
  44. * --------------------------------------------------------------------
  45. */
  46. /*
  47. * Resampling irqfds are a special variety of irqfds used to emulate
  48. * level triggered interrupts. The interrupt is asserted on eventfd
  49. * trigger. On acknowledgement through the irq ack notifier, the
  50. * interrupt is de-asserted and userspace is notified through the
  51. * resamplefd. All resamplers on the same gsi are de-asserted
  52. * together, so we don't need to track the state of each individual
  53. * user. We can also therefore share the same irq source ID.
  54. */
  55. struct _irqfd_resampler {
  56. struct kvm *kvm;
  57. /*
  58. * List of resampling struct _irqfd objects sharing this gsi.
  59. * RCU list modified under kvm->irqfds.resampler_lock
  60. */
  61. struct list_head list;
  62. struct kvm_irq_ack_notifier notifier;
  63. /*
  64. * Entry in list of kvm->irqfd.resampler_list. Use for sharing
  65. * resamplers among irqfds on the same gsi.
  66. * Accessed and modified under kvm->irqfds.resampler_lock
  67. */
  68. struct list_head link;
  69. };
  70. struct _irqfd {
  71. /* Used for MSI fast-path */
  72. struct kvm *kvm;
  73. wait_queue_t wait;
  74. /* Update side is protected by irqfds.lock */
  75. struct kvm_kernel_irq_routing_entry irq_entry;
  76. seqcount_t irq_entry_sc;
  77. /* Used for level IRQ fast-path */
  78. int gsi;
  79. struct work_struct inject;
  80. /* The resampler used by this irqfd (resampler-only) */
  81. struct _irqfd_resampler *resampler;
  82. /* Eventfd notified on resample (resampler-only) */
  83. struct eventfd_ctx *resamplefd;
  84. /* Entry in list of irqfds for a resampler (resampler-only) */
  85. struct list_head resampler_link;
  86. /* Used for setup/shutdown */
  87. struct eventfd_ctx *eventfd;
  88. struct list_head list;
  89. poll_table pt;
  90. struct work_struct shutdown;
  91. };
  92. static struct workqueue_struct *irqfd_cleanup_wq;
  93. static void
  94. irqfd_inject(struct work_struct *work)
  95. {
  96. struct _irqfd *irqfd = container_of(work, struct _irqfd, inject);
  97. struct kvm *kvm = irqfd->kvm;
  98. if (!irqfd->resampler) {
  99. kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 1,
  100. false);
  101. kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 0,
  102. false);
  103. } else
  104. kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
  105. irqfd->gsi, 1, false);
  106. }
  107. /*
  108. * Since resampler irqfds share an IRQ source ID, we de-assert once
  109. * then notify all of the resampler irqfds using this GSI. We can't
  110. * do multiple de-asserts or we risk racing with incoming re-asserts.
  111. */
  112. static void
  113. irqfd_resampler_ack(struct kvm_irq_ack_notifier *kian)
  114. {
  115. struct _irqfd_resampler *resampler;
  116. struct kvm *kvm;
  117. struct _irqfd *irqfd;
  118. int idx;
  119. resampler = container_of(kian, struct _irqfd_resampler, notifier);
  120. kvm = resampler->kvm;
  121. kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
  122. resampler->notifier.gsi, 0, false);
  123. idx = srcu_read_lock(&kvm->irq_srcu);
  124. list_for_each_entry_rcu(irqfd, &resampler->list, resampler_link)
  125. eventfd_signal(irqfd->resamplefd, 1);
  126. srcu_read_unlock(&kvm->irq_srcu, idx);
  127. }
  128. static void
  129. irqfd_resampler_shutdown(struct _irqfd *irqfd)
  130. {
  131. struct _irqfd_resampler *resampler = irqfd->resampler;
  132. struct kvm *kvm = resampler->kvm;
  133. mutex_lock(&kvm->irqfds.resampler_lock);
  134. list_del_rcu(&irqfd->resampler_link);
  135. synchronize_srcu(&kvm->irq_srcu);
  136. if (list_empty(&resampler->list)) {
  137. list_del(&resampler->link);
  138. kvm_unregister_irq_ack_notifier(kvm, &resampler->notifier);
  139. kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
  140. resampler->notifier.gsi, 0, false);
  141. kfree(resampler);
  142. }
  143. mutex_unlock(&kvm->irqfds.resampler_lock);
  144. }
  145. /*
  146. * Race-free decouple logic (ordering is critical)
  147. */
  148. static void
  149. irqfd_shutdown(struct work_struct *work)
  150. {
  151. struct _irqfd *irqfd = container_of(work, struct _irqfd, shutdown);
  152. u64 cnt;
  153. /*
  154. * Synchronize with the wait-queue and unhook ourselves to prevent
  155. * further events.
  156. */
  157. eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt);
  158. /*
  159. * We know no new events will be scheduled at this point, so block
  160. * until all previously outstanding events have completed
  161. */
  162. flush_work(&irqfd->inject);
  163. if (irqfd->resampler) {
  164. irqfd_resampler_shutdown(irqfd);
  165. eventfd_ctx_put(irqfd->resamplefd);
  166. }
  167. /*
  168. * It is now safe to release the object's resources
  169. */
  170. eventfd_ctx_put(irqfd->eventfd);
  171. kfree(irqfd);
  172. }
  173. /* assumes kvm->irqfds.lock is held */
  174. static bool
  175. irqfd_is_active(struct _irqfd *irqfd)
  176. {
  177. return list_empty(&irqfd->list) ? false : true;
  178. }
  179. /*
  180. * Mark the irqfd as inactive and schedule it for removal
  181. *
  182. * assumes kvm->irqfds.lock is held
  183. */
  184. static void
  185. irqfd_deactivate(struct _irqfd *irqfd)
  186. {
  187. BUG_ON(!irqfd_is_active(irqfd));
  188. list_del_init(&irqfd->list);
  189. queue_work(irqfd_cleanup_wq, &irqfd->shutdown);
  190. }
  191. /*
  192. * Called with wqh->lock held and interrupts disabled
  193. */
  194. static int
  195. irqfd_wakeup(wait_queue_t *wait, unsigned mode, int sync, void *key)
  196. {
  197. struct _irqfd *irqfd = container_of(wait, struct _irqfd, wait);
  198. unsigned long flags = (unsigned long)key;
  199. struct kvm_kernel_irq_routing_entry irq;
  200. struct kvm *kvm = irqfd->kvm;
  201. unsigned seq;
  202. int idx;
  203. if (flags & POLLIN) {
  204. idx = srcu_read_lock(&kvm->irq_srcu);
  205. do {
  206. seq = read_seqcount_begin(&irqfd->irq_entry_sc);
  207. irq = irqfd->irq_entry;
  208. } while (read_seqcount_retry(&irqfd->irq_entry_sc, seq));
  209. /* An event has been signaled, inject an interrupt */
  210. if (irq.type == KVM_IRQ_ROUTING_MSI)
  211. kvm_set_msi(&irq, kvm, KVM_USERSPACE_IRQ_SOURCE_ID, 1,
  212. false);
  213. else
  214. schedule_work(&irqfd->inject);
  215. srcu_read_unlock(&kvm->irq_srcu, idx);
  216. }
  217. if (flags & POLLHUP) {
  218. /* The eventfd is closing, detach from KVM */
  219. unsigned long flags;
  220. spin_lock_irqsave(&kvm->irqfds.lock, flags);
  221. /*
  222. * We must check if someone deactivated the irqfd before
  223. * we could acquire the irqfds.lock since the item is
  224. * deactivated from the KVM side before it is unhooked from
  225. * the wait-queue. If it is already deactivated, we can
  226. * simply return knowing the other side will cleanup for us.
  227. * We cannot race against the irqfd going away since the
  228. * other side is required to acquire wqh->lock, which we hold
  229. */
  230. if (irqfd_is_active(irqfd))
  231. irqfd_deactivate(irqfd);
  232. spin_unlock_irqrestore(&kvm->irqfds.lock, flags);
  233. }
  234. return 0;
  235. }
  236. static void
  237. irqfd_ptable_queue_proc(struct file *file, wait_queue_head_t *wqh,
  238. poll_table *pt)
  239. {
  240. struct _irqfd *irqfd = container_of(pt, struct _irqfd, pt);
  241. add_wait_queue(wqh, &irqfd->wait);
  242. }
  243. /* Must be called under irqfds.lock */
  244. static void irqfd_update(struct kvm *kvm, struct _irqfd *irqfd)
  245. {
  246. struct kvm_kernel_irq_routing_entry *e;
  247. struct kvm_kernel_irq_routing_entry entries[KVM_NR_IRQCHIPS];
  248. int i, n_entries;
  249. n_entries = kvm_irq_map_gsi(kvm, entries, irqfd->gsi);
  250. write_seqcount_begin(&irqfd->irq_entry_sc);
  251. irqfd->irq_entry.type = 0;
  252. e = entries;
  253. for (i = 0; i < n_entries; ++i, ++e) {
  254. /* Only fast-path MSI. */
  255. if (e->type == KVM_IRQ_ROUTING_MSI)
  256. irqfd->irq_entry = *e;
  257. }
  258. write_seqcount_end(&irqfd->irq_entry_sc);
  259. }
  260. static int
  261. kvm_irqfd_assign(struct kvm *kvm, struct kvm_irqfd *args)
  262. {
  263. struct _irqfd *irqfd, *tmp;
  264. struct fd f;
  265. struct eventfd_ctx *eventfd = NULL, *resamplefd = NULL;
  266. int ret;
  267. unsigned int events;
  268. int idx;
  269. if (!kvm_arch_intc_initialized(kvm))
  270. return -EAGAIN;
  271. irqfd = kzalloc(sizeof(*irqfd), GFP_KERNEL);
  272. if (!irqfd)
  273. return -ENOMEM;
  274. irqfd->kvm = kvm;
  275. irqfd->gsi = args->gsi;
  276. INIT_LIST_HEAD(&irqfd->list);
  277. INIT_WORK(&irqfd->inject, irqfd_inject);
  278. INIT_WORK(&irqfd->shutdown, irqfd_shutdown);
  279. seqcount_init(&irqfd->irq_entry_sc);
  280. f = fdget(args->fd);
  281. if (!f.file) {
  282. ret = -EBADF;
  283. goto out;
  284. }
  285. eventfd = eventfd_ctx_fileget(f.file);
  286. if (IS_ERR(eventfd)) {
  287. ret = PTR_ERR(eventfd);
  288. goto fail;
  289. }
  290. irqfd->eventfd = eventfd;
  291. if (args->flags & KVM_IRQFD_FLAG_RESAMPLE) {
  292. struct _irqfd_resampler *resampler;
  293. resamplefd = eventfd_ctx_fdget(args->resamplefd);
  294. if (IS_ERR(resamplefd)) {
  295. ret = PTR_ERR(resamplefd);
  296. goto fail;
  297. }
  298. irqfd->resamplefd = resamplefd;
  299. INIT_LIST_HEAD(&irqfd->resampler_link);
  300. mutex_lock(&kvm->irqfds.resampler_lock);
  301. list_for_each_entry(resampler,
  302. &kvm->irqfds.resampler_list, link) {
  303. if (resampler->notifier.gsi == irqfd->gsi) {
  304. irqfd->resampler = resampler;
  305. break;
  306. }
  307. }
  308. if (!irqfd->resampler) {
  309. resampler = kzalloc(sizeof(*resampler), GFP_KERNEL);
  310. if (!resampler) {
  311. ret = -ENOMEM;
  312. mutex_unlock(&kvm->irqfds.resampler_lock);
  313. goto fail;
  314. }
  315. resampler->kvm = kvm;
  316. INIT_LIST_HEAD(&resampler->list);
  317. resampler->notifier.gsi = irqfd->gsi;
  318. resampler->notifier.irq_acked = irqfd_resampler_ack;
  319. INIT_LIST_HEAD(&resampler->link);
  320. list_add(&resampler->link, &kvm->irqfds.resampler_list);
  321. kvm_register_irq_ack_notifier(kvm,
  322. &resampler->notifier);
  323. irqfd->resampler = resampler;
  324. }
  325. list_add_rcu(&irqfd->resampler_link, &irqfd->resampler->list);
  326. synchronize_srcu(&kvm->irq_srcu);
  327. mutex_unlock(&kvm->irqfds.resampler_lock);
  328. }
  329. /*
  330. * Install our own custom wake-up handling so we are notified via
  331. * a callback whenever someone signals the underlying eventfd
  332. */
  333. init_waitqueue_func_entry(&irqfd->wait, irqfd_wakeup);
  334. init_poll_funcptr(&irqfd->pt, irqfd_ptable_queue_proc);
  335. spin_lock_irq(&kvm->irqfds.lock);
  336. ret = 0;
  337. list_for_each_entry(tmp, &kvm->irqfds.items, list) {
  338. if (irqfd->eventfd != tmp->eventfd)
  339. continue;
  340. /* This fd is used for another irq already. */
  341. ret = -EBUSY;
  342. spin_unlock_irq(&kvm->irqfds.lock);
  343. goto fail;
  344. }
  345. idx = srcu_read_lock(&kvm->irq_srcu);
  346. irqfd_update(kvm, irqfd);
  347. srcu_read_unlock(&kvm->irq_srcu, idx);
  348. list_add_tail(&irqfd->list, &kvm->irqfds.items);
  349. spin_unlock_irq(&kvm->irqfds.lock);
  350. /*
  351. * Check if there was an event already pending on the eventfd
  352. * before we registered, and trigger it as if we didn't miss it.
  353. */
  354. events = f.file->f_op->poll(f.file, &irqfd->pt);
  355. if (events & POLLIN)
  356. schedule_work(&irqfd->inject);
  357. /*
  358. * do not drop the file until the irqfd is fully initialized, otherwise
  359. * we might race against the POLLHUP
  360. */
  361. fdput(f);
  362. return 0;
  363. fail:
  364. if (irqfd->resampler)
  365. irqfd_resampler_shutdown(irqfd);
  366. if (resamplefd && !IS_ERR(resamplefd))
  367. eventfd_ctx_put(resamplefd);
  368. if (eventfd && !IS_ERR(eventfd))
  369. eventfd_ctx_put(eventfd);
  370. fdput(f);
  371. out:
  372. kfree(irqfd);
  373. return ret;
  374. }
  375. bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin)
  376. {
  377. struct kvm_irq_ack_notifier *kian;
  378. int gsi, idx;
  379. idx = srcu_read_lock(&kvm->irq_srcu);
  380. gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);
  381. if (gsi != -1)
  382. hlist_for_each_entry_rcu(kian, &kvm->irq_ack_notifier_list,
  383. link)
  384. if (kian->gsi == gsi) {
  385. srcu_read_unlock(&kvm->irq_srcu, idx);
  386. return true;
  387. }
  388. srcu_read_unlock(&kvm->irq_srcu, idx);
  389. return false;
  390. }
  391. EXPORT_SYMBOL_GPL(kvm_irq_has_notifier);
  392. void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin)
  393. {
  394. struct kvm_irq_ack_notifier *kian;
  395. int gsi, idx;
  396. trace_kvm_ack_irq(irqchip, pin);
  397. idx = srcu_read_lock(&kvm->irq_srcu);
  398. gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);
  399. if (gsi != -1)
  400. hlist_for_each_entry_rcu(kian, &kvm->irq_ack_notifier_list,
  401. link)
  402. if (kian->gsi == gsi)
  403. kian->irq_acked(kian);
  404. srcu_read_unlock(&kvm->irq_srcu, idx);
  405. }
  406. void kvm_register_irq_ack_notifier(struct kvm *kvm,
  407. struct kvm_irq_ack_notifier *kian)
  408. {
  409. mutex_lock(&kvm->irq_lock);
  410. hlist_add_head_rcu(&kian->link, &kvm->irq_ack_notifier_list);
  411. mutex_unlock(&kvm->irq_lock);
  412. kvm_vcpu_request_scan_ioapic(kvm);
  413. }
  414. void kvm_unregister_irq_ack_notifier(struct kvm *kvm,
  415. struct kvm_irq_ack_notifier *kian)
  416. {
  417. mutex_lock(&kvm->irq_lock);
  418. hlist_del_init_rcu(&kian->link);
  419. mutex_unlock(&kvm->irq_lock);
  420. synchronize_srcu(&kvm->irq_srcu);
  421. kvm_vcpu_request_scan_ioapic(kvm);
  422. }
  423. #endif
  424. void
  425. kvm_eventfd_init(struct kvm *kvm)
  426. {
  427. #ifdef CONFIG_HAVE_KVM_IRQFD
  428. spin_lock_init(&kvm->irqfds.lock);
  429. INIT_LIST_HEAD(&kvm->irqfds.items);
  430. INIT_LIST_HEAD(&kvm->irqfds.resampler_list);
  431. mutex_init(&kvm->irqfds.resampler_lock);
  432. #endif
  433. INIT_LIST_HEAD(&kvm->ioeventfds);
  434. }
  435. #ifdef CONFIG_HAVE_KVM_IRQFD
  436. /*
  437. * shutdown any irqfd's that match fd+gsi
  438. */
  439. static int
  440. kvm_irqfd_deassign(struct kvm *kvm, struct kvm_irqfd *args)
  441. {
  442. struct _irqfd *irqfd, *tmp;
  443. struct eventfd_ctx *eventfd;
  444. eventfd = eventfd_ctx_fdget(args->fd);
  445. if (IS_ERR(eventfd))
  446. return PTR_ERR(eventfd);
  447. spin_lock_irq(&kvm->irqfds.lock);
  448. list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) {
  449. if (irqfd->eventfd == eventfd && irqfd->gsi == args->gsi) {
  450. /*
  451. * This clearing of irq_entry.type is needed for when
  452. * another thread calls kvm_irq_routing_update before
  453. * we flush workqueue below (we synchronize with
  454. * kvm_irq_routing_update using irqfds.lock).
  455. */
  456. write_seqcount_begin(&irqfd->irq_entry_sc);
  457. irqfd->irq_entry.type = 0;
  458. write_seqcount_end(&irqfd->irq_entry_sc);
  459. irqfd_deactivate(irqfd);
  460. }
  461. }
  462. spin_unlock_irq(&kvm->irqfds.lock);
  463. eventfd_ctx_put(eventfd);
  464. /*
  465. * Block until we know all outstanding shutdown jobs have completed
  466. * so that we guarantee there will not be any more interrupts on this
  467. * gsi once this deassign function returns.
  468. */
  469. flush_workqueue(irqfd_cleanup_wq);
  470. return 0;
  471. }
  472. int
  473. kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)
  474. {
  475. if (args->flags & ~(KVM_IRQFD_FLAG_DEASSIGN | KVM_IRQFD_FLAG_RESAMPLE))
  476. return -EINVAL;
  477. if (args->flags & KVM_IRQFD_FLAG_DEASSIGN)
  478. return kvm_irqfd_deassign(kvm, args);
  479. return kvm_irqfd_assign(kvm, args);
  480. }
  481. /*
  482. * This function is called as the kvm VM fd is being released. Shutdown all
  483. * irqfds that still remain open
  484. */
  485. void
  486. kvm_irqfd_release(struct kvm *kvm)
  487. {
  488. struct _irqfd *irqfd, *tmp;
  489. spin_lock_irq(&kvm->irqfds.lock);
  490. list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list)
  491. irqfd_deactivate(irqfd);
  492. spin_unlock_irq(&kvm->irqfds.lock);
  493. /*
  494. * Block until we know all outstanding shutdown jobs have completed
  495. * since we do not take a kvm* reference.
  496. */
  497. flush_workqueue(irqfd_cleanup_wq);
  498. }
  499. /*
  500. * Take note of a change in irq routing.
  501. * Caller must invoke synchronize_srcu(&kvm->irq_srcu) afterwards.
  502. */
  503. void kvm_irq_routing_update(struct kvm *kvm)
  504. {
  505. struct _irqfd *irqfd;
  506. spin_lock_irq(&kvm->irqfds.lock);
  507. list_for_each_entry(irqfd, &kvm->irqfds.items, list)
  508. irqfd_update(kvm, irqfd);
  509. spin_unlock_irq(&kvm->irqfds.lock);
  510. }
  511. /*
  512. * create a host-wide workqueue for issuing deferred shutdown requests
  513. * aggregated from all vm* instances. We need our own isolated single-thread
  514. * queue to prevent deadlock against flushing the normal work-queue.
  515. */
  516. int kvm_irqfd_init(void)
  517. {
  518. irqfd_cleanup_wq = create_singlethread_workqueue("kvm-irqfd-cleanup");
  519. if (!irqfd_cleanup_wq)
  520. return -ENOMEM;
  521. return 0;
  522. }
  523. void kvm_irqfd_exit(void)
  524. {
  525. destroy_workqueue(irqfd_cleanup_wq);
  526. }
  527. #endif
  528. /*
  529. * --------------------------------------------------------------------
  530. * ioeventfd: translate a PIO/MMIO memory write to an eventfd signal.
  531. *
  532. * userspace can register a PIO/MMIO address with an eventfd for receiving
  533. * notification when the memory has been touched.
  534. * --------------------------------------------------------------------
  535. */
  536. struct _ioeventfd {
  537. struct list_head list;
  538. u64 addr;
  539. int length;
  540. struct eventfd_ctx *eventfd;
  541. u64 datamatch;
  542. struct kvm_io_device dev;
  543. u8 bus_idx;
  544. bool wildcard;
  545. };
  546. static inline struct _ioeventfd *
  547. to_ioeventfd(struct kvm_io_device *dev)
  548. {
  549. return container_of(dev, struct _ioeventfd, dev);
  550. }
  551. static void
  552. ioeventfd_release(struct _ioeventfd *p)
  553. {
  554. eventfd_ctx_put(p->eventfd);
  555. list_del(&p->list);
  556. kfree(p);
  557. }
  558. static bool
  559. ioeventfd_in_range(struct _ioeventfd *p, gpa_t addr, int len, const void *val)
  560. {
  561. u64 _val;
  562. if (addr != p->addr)
  563. /* address must be precise for a hit */
  564. return false;
  565. if (!p->length)
  566. /* length = 0 means only look at the address, so always a hit */
  567. return true;
  568. if (len != p->length)
  569. /* address-range must be precise for a hit */
  570. return false;
  571. if (p->wildcard)
  572. /* all else equal, wildcard is always a hit */
  573. return true;
  574. /* otherwise, we have to actually compare the data */
  575. BUG_ON(!IS_ALIGNED((unsigned long)val, len));
  576. switch (len) {
  577. case 1:
  578. _val = *(u8 *)val;
  579. break;
  580. case 2:
  581. _val = *(u16 *)val;
  582. break;
  583. case 4:
  584. _val = *(u32 *)val;
  585. break;
  586. case 8:
  587. _val = *(u64 *)val;
  588. break;
  589. default:
  590. return false;
  591. }
  592. return _val == p->datamatch ? true : false;
  593. }
  594. /* MMIO/PIO writes trigger an event if the addr/val match */
  595. static int
  596. ioeventfd_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this, gpa_t addr,
  597. int len, const void *val)
  598. {
  599. struct _ioeventfd *p = to_ioeventfd(this);
  600. if (!ioeventfd_in_range(p, addr, len, val))
  601. return -EOPNOTSUPP;
  602. eventfd_signal(p->eventfd, 1);
  603. return 0;
  604. }
  605. /*
  606. * This function is called as KVM is completely shutting down. We do not
  607. * need to worry about locking just nuke anything we have as quickly as possible
  608. */
  609. static void
  610. ioeventfd_destructor(struct kvm_io_device *this)
  611. {
  612. struct _ioeventfd *p = to_ioeventfd(this);
  613. ioeventfd_release(p);
  614. }
  615. static const struct kvm_io_device_ops ioeventfd_ops = {
  616. .write = ioeventfd_write,
  617. .destructor = ioeventfd_destructor,
  618. };
  619. /* assumes kvm->slots_lock held */
  620. static bool
  621. ioeventfd_check_collision(struct kvm *kvm, struct _ioeventfd *p)
  622. {
  623. struct _ioeventfd *_p;
  624. list_for_each_entry(_p, &kvm->ioeventfds, list)
  625. if (_p->bus_idx == p->bus_idx &&
  626. _p->addr == p->addr &&
  627. (!_p->length || !p->length ||
  628. (_p->length == p->length &&
  629. (_p->wildcard || p->wildcard ||
  630. _p->datamatch == p->datamatch))))
  631. return true;
  632. return false;
  633. }
  634. static enum kvm_bus ioeventfd_bus_from_flags(__u32 flags)
  635. {
  636. if (flags & KVM_IOEVENTFD_FLAG_PIO)
  637. return KVM_PIO_BUS;
  638. if (flags & KVM_IOEVENTFD_FLAG_VIRTIO_CCW_NOTIFY)
  639. return KVM_VIRTIO_CCW_NOTIFY_BUS;
  640. return KVM_MMIO_BUS;
  641. }
  642. static int
  643. kvm_assign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
  644. {
  645. enum kvm_bus bus_idx;
  646. struct _ioeventfd *p;
  647. struct eventfd_ctx *eventfd;
  648. int ret;
  649. bus_idx = ioeventfd_bus_from_flags(args->flags);
  650. /* must be natural-word sized, or 0 to ignore length */
  651. switch (args->len) {
  652. case 0:
  653. case 1:
  654. case 2:
  655. case 4:
  656. case 8:
  657. break;
  658. default:
  659. return -EINVAL;
  660. }
  661. /* check for range overflow */
  662. if (args->addr + args->len < args->addr)
  663. return -EINVAL;
  664. /* check for extra flags that we don't understand */
  665. if (args->flags & ~KVM_IOEVENTFD_VALID_FLAG_MASK)
  666. return -EINVAL;
  667. /* ioeventfd with no length can't be combined with DATAMATCH */
  668. if (!args->len &&
  669. args->flags & (KVM_IOEVENTFD_FLAG_PIO |
  670. KVM_IOEVENTFD_FLAG_DATAMATCH))
  671. return -EINVAL;
  672. eventfd = eventfd_ctx_fdget(args->fd);
  673. if (IS_ERR(eventfd))
  674. return PTR_ERR(eventfd);
  675. p = kzalloc(sizeof(*p), GFP_KERNEL);
  676. if (!p) {
  677. ret = -ENOMEM;
  678. goto fail;
  679. }
  680. INIT_LIST_HEAD(&p->list);
  681. p->addr = args->addr;
  682. p->bus_idx = bus_idx;
  683. p->length = args->len;
  684. p->eventfd = eventfd;
  685. /* The datamatch feature is optional, otherwise this is a wildcard */
  686. if (args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH)
  687. p->datamatch = args->datamatch;
  688. else
  689. p->wildcard = true;
  690. mutex_lock(&kvm->slots_lock);
  691. /* Verify that there isn't a match already */
  692. if (ioeventfd_check_collision(kvm, p)) {
  693. ret = -EEXIST;
  694. goto unlock_fail;
  695. }
  696. kvm_iodevice_init(&p->dev, &ioeventfd_ops);
  697. ret = kvm_io_bus_register_dev(kvm, bus_idx, p->addr, p->length,
  698. &p->dev);
  699. if (ret < 0)
  700. goto unlock_fail;
  701. /* When length is ignored, MMIO is also put on a separate bus, for
  702. * faster lookups.
  703. */
  704. if (!args->len && !(args->flags & KVM_IOEVENTFD_FLAG_PIO)) {
  705. ret = kvm_io_bus_register_dev(kvm, KVM_FAST_MMIO_BUS,
  706. p->addr, 0, &p->dev);
  707. if (ret < 0)
  708. goto register_fail;
  709. }
  710. kvm->buses[bus_idx]->ioeventfd_count++;
  711. list_add_tail(&p->list, &kvm->ioeventfds);
  712. mutex_unlock(&kvm->slots_lock);
  713. return 0;
  714. register_fail:
  715. kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev);
  716. unlock_fail:
  717. mutex_unlock(&kvm->slots_lock);
  718. fail:
  719. kfree(p);
  720. eventfd_ctx_put(eventfd);
  721. return ret;
  722. }
  723. static int
  724. kvm_deassign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
  725. {
  726. enum kvm_bus bus_idx;
  727. struct _ioeventfd *p, *tmp;
  728. struct eventfd_ctx *eventfd;
  729. int ret = -ENOENT;
  730. bus_idx = ioeventfd_bus_from_flags(args->flags);
  731. eventfd = eventfd_ctx_fdget(args->fd);
  732. if (IS_ERR(eventfd))
  733. return PTR_ERR(eventfd);
  734. mutex_lock(&kvm->slots_lock);
  735. list_for_each_entry_safe(p, tmp, &kvm->ioeventfds, list) {
  736. bool wildcard = !(args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH);
  737. if (p->bus_idx != bus_idx ||
  738. p->eventfd != eventfd ||
  739. p->addr != args->addr ||
  740. p->length != args->len ||
  741. p->wildcard != wildcard)
  742. continue;
  743. if (!p->wildcard && p->datamatch != args->datamatch)
  744. continue;
  745. kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev);
  746. if (!p->length) {
  747. kvm_io_bus_unregister_dev(kvm, KVM_FAST_MMIO_BUS,
  748. &p->dev);
  749. }
  750. kvm->buses[bus_idx]->ioeventfd_count--;
  751. ioeventfd_release(p);
  752. ret = 0;
  753. break;
  754. }
  755. mutex_unlock(&kvm->slots_lock);
  756. eventfd_ctx_put(eventfd);
  757. return ret;
  758. }
  759. int
  760. kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
  761. {
  762. if (args->flags & KVM_IOEVENTFD_FLAG_DEASSIGN)
  763. return kvm_deassign_ioeventfd(kvm, args);
  764. return kvm_assign_ioeventfd(kvm, args);
  765. }