vhost.c 39 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555
  1. /* Copyright (C) 2009 Red Hat, Inc.
  2. * Copyright (C) 2006 Rusty Russell IBM Corporation
  3. *
  4. * Author: Michael S. Tsirkin <mst@redhat.com>
  5. *
  6. * Inspiration, some code, and most witty comments come from
  7. * Documentation/virtual/lguest/lguest.c, by Rusty Russell
  8. *
  9. * This work is licensed under the terms of the GNU GPL, version 2.
  10. *
  11. * Generic code for virtio server in host kernel.
  12. */
  13. #include <linux/eventfd.h>
  14. #include <linux/vhost.h>
  15. #include <linux/uio.h>
  16. #include <linux/mm.h>
  17. #include <linux/mmu_context.h>
  18. #include <linux/miscdevice.h>
  19. #include <linux/mutex.h>
  20. #include <linux/rcupdate.h>
  21. #include <linux/poll.h>
  22. #include <linux/file.h>
  23. #include <linux/highmem.h>
  24. #include <linux/slab.h>
  25. #include <linux/kthread.h>
  26. #include <linux/cgroup.h>
  27. #include <linux/module.h>
  28. #include "vhost.h"
  29. enum {
  30. VHOST_MEMORY_MAX_NREGIONS = 64,
  31. VHOST_MEMORY_F_LOG = 0x1,
  32. };
  33. #define vhost_used_event(vq) ((u16 __user *)&vq->avail->ring[vq->num])
  34. #define vhost_avail_event(vq) ((u16 __user *)&vq->used->ring[vq->num])
  35. static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh,
  36. poll_table *pt)
  37. {
  38. struct vhost_poll *poll;
  39. poll = container_of(pt, struct vhost_poll, table);
  40. poll->wqh = wqh;
  41. add_wait_queue(wqh, &poll->wait);
  42. }
  43. static int vhost_poll_wakeup(wait_queue_t *wait, unsigned mode, int sync,
  44. void *key)
  45. {
  46. struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait);
  47. if (!((unsigned long)key & poll->mask))
  48. return 0;
  49. vhost_poll_queue(poll);
  50. return 0;
  51. }
  52. void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn)
  53. {
  54. INIT_LIST_HEAD(&work->node);
  55. work->fn = fn;
  56. init_waitqueue_head(&work->done);
  57. work->flushing = 0;
  58. work->queue_seq = work->done_seq = 0;
  59. }
  60. EXPORT_SYMBOL_GPL(vhost_work_init);
  61. /* Init poll structure */
  62. void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn,
  63. unsigned long mask, struct vhost_dev *dev)
  64. {
  65. init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup);
  66. init_poll_funcptr(&poll->table, vhost_poll_func);
  67. poll->mask = mask;
  68. poll->dev = dev;
  69. poll->wqh = NULL;
  70. vhost_work_init(&poll->work, fn);
  71. }
  72. EXPORT_SYMBOL_GPL(vhost_poll_init);
  73. /* Start polling a file. We add ourselves to file's wait queue. The caller must
  74. * keep a reference to a file until after vhost_poll_stop is called. */
  75. int vhost_poll_start(struct vhost_poll *poll, struct file *file)
  76. {
  77. unsigned long mask;
  78. int ret = 0;
  79. if (poll->wqh)
  80. return 0;
  81. mask = file->f_op->poll(file, &poll->table);
  82. if (mask)
  83. vhost_poll_wakeup(&poll->wait, 0, 0, (void *)mask);
  84. if (mask & POLLERR) {
  85. if (poll->wqh)
  86. remove_wait_queue(poll->wqh, &poll->wait);
  87. ret = -EINVAL;
  88. }
  89. return ret;
  90. }
  91. EXPORT_SYMBOL_GPL(vhost_poll_start);
  92. /* Stop polling a file. After this function returns, it becomes safe to drop the
  93. * file reference. You must also flush afterwards. */
  94. void vhost_poll_stop(struct vhost_poll *poll)
  95. {
  96. if (poll->wqh) {
  97. remove_wait_queue(poll->wqh, &poll->wait);
  98. poll->wqh = NULL;
  99. }
  100. }
  101. EXPORT_SYMBOL_GPL(vhost_poll_stop);
  102. static bool vhost_work_seq_done(struct vhost_dev *dev, struct vhost_work *work,
  103. unsigned seq)
  104. {
  105. int left;
  106. spin_lock_irq(&dev->work_lock);
  107. left = seq - work->done_seq;
  108. spin_unlock_irq(&dev->work_lock);
  109. return left <= 0;
  110. }
  111. void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work)
  112. {
  113. unsigned seq;
  114. int flushing;
  115. spin_lock_irq(&dev->work_lock);
  116. seq = work->queue_seq;
  117. work->flushing++;
  118. spin_unlock_irq(&dev->work_lock);
  119. wait_event(work->done, vhost_work_seq_done(dev, work, seq));
  120. spin_lock_irq(&dev->work_lock);
  121. flushing = --work->flushing;
  122. spin_unlock_irq(&dev->work_lock);
  123. BUG_ON(flushing < 0);
  124. }
  125. EXPORT_SYMBOL_GPL(vhost_work_flush);
  126. /* Flush any work that has been scheduled. When calling this, don't hold any
  127. * locks that are also used by the callback. */
  128. void vhost_poll_flush(struct vhost_poll *poll)
  129. {
  130. vhost_work_flush(poll->dev, &poll->work);
  131. }
  132. EXPORT_SYMBOL_GPL(vhost_poll_flush);
  133. void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work)
  134. {
  135. unsigned long flags;
  136. spin_lock_irqsave(&dev->work_lock, flags);
  137. if (list_empty(&work->node)) {
  138. list_add_tail(&work->node, &dev->work_list);
  139. work->queue_seq++;
  140. spin_unlock_irqrestore(&dev->work_lock, flags);
  141. wake_up_process(dev->worker);
  142. } else {
  143. spin_unlock_irqrestore(&dev->work_lock, flags);
  144. }
  145. }
  146. EXPORT_SYMBOL_GPL(vhost_work_queue);
  147. void vhost_poll_queue(struct vhost_poll *poll)
  148. {
  149. vhost_work_queue(poll->dev, &poll->work);
  150. }
  151. EXPORT_SYMBOL_GPL(vhost_poll_queue);
  152. static void vhost_vq_reset(struct vhost_dev *dev,
  153. struct vhost_virtqueue *vq)
  154. {
  155. vq->num = 1;
  156. vq->desc = NULL;
  157. vq->avail = NULL;
  158. vq->used = NULL;
  159. vq->last_avail_idx = 0;
  160. vq->avail_idx = 0;
  161. vq->last_used_idx = 0;
  162. vq->signalled_used = 0;
  163. vq->signalled_used_valid = false;
  164. vq->used_flags = 0;
  165. vq->log_used = false;
  166. vq->log_addr = -1ull;
  167. vq->private_data = NULL;
  168. vq->log_base = NULL;
  169. vq->error_ctx = NULL;
  170. vq->error = NULL;
  171. vq->kick = NULL;
  172. vq->call_ctx = NULL;
  173. vq->call = NULL;
  174. vq->log_ctx = NULL;
  175. }
  176. static int vhost_worker(void *data)
  177. {
  178. struct vhost_dev *dev = data;
  179. struct vhost_work *work = NULL;
  180. unsigned uninitialized_var(seq);
  181. mm_segment_t oldfs = get_fs();
  182. set_fs(USER_DS);
  183. use_mm(dev->mm);
  184. for (;;) {
  185. /* mb paired w/ kthread_stop */
  186. set_current_state(TASK_INTERRUPTIBLE);
  187. spin_lock_irq(&dev->work_lock);
  188. if (work) {
  189. work->done_seq = seq;
  190. if (work->flushing)
  191. wake_up_all(&work->done);
  192. }
  193. if (kthread_should_stop()) {
  194. spin_unlock_irq(&dev->work_lock);
  195. __set_current_state(TASK_RUNNING);
  196. break;
  197. }
  198. if (!list_empty(&dev->work_list)) {
  199. work = list_first_entry(&dev->work_list,
  200. struct vhost_work, node);
  201. list_del_init(&work->node);
  202. seq = work->queue_seq;
  203. } else
  204. work = NULL;
  205. spin_unlock_irq(&dev->work_lock);
  206. if (work) {
  207. __set_current_state(TASK_RUNNING);
  208. work->fn(work);
  209. if (need_resched())
  210. schedule();
  211. } else
  212. schedule();
  213. }
  214. unuse_mm(dev->mm);
  215. set_fs(oldfs);
  216. return 0;
  217. }
  218. static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq)
  219. {
  220. kfree(vq->indirect);
  221. vq->indirect = NULL;
  222. kfree(vq->log);
  223. vq->log = NULL;
  224. kfree(vq->heads);
  225. vq->heads = NULL;
  226. }
  227. /* Helper to allocate iovec buffers for all vqs. */
  228. static long vhost_dev_alloc_iovecs(struct vhost_dev *dev)
  229. {
  230. struct vhost_virtqueue *vq;
  231. int i;
  232. for (i = 0; i < dev->nvqs; ++i) {
  233. vq = dev->vqs[i];
  234. vq->indirect = kmalloc(sizeof *vq->indirect * UIO_MAXIOV,
  235. GFP_KERNEL);
  236. vq->log = kmalloc(sizeof *vq->log * UIO_MAXIOV, GFP_KERNEL);
  237. vq->heads = kmalloc(sizeof *vq->heads * UIO_MAXIOV, GFP_KERNEL);
  238. if (!vq->indirect || !vq->log || !vq->heads)
  239. goto err_nomem;
  240. }
  241. return 0;
  242. err_nomem:
  243. for (; i >= 0; --i)
  244. vhost_vq_free_iovecs(dev->vqs[i]);
  245. return -ENOMEM;
  246. }
  247. static void vhost_dev_free_iovecs(struct vhost_dev *dev)
  248. {
  249. int i;
  250. for (i = 0; i < dev->nvqs; ++i)
  251. vhost_vq_free_iovecs(dev->vqs[i]);
  252. }
  253. void vhost_dev_init(struct vhost_dev *dev,
  254. struct vhost_virtqueue **vqs, int nvqs)
  255. {
  256. struct vhost_virtqueue *vq;
  257. int i;
  258. dev->vqs = vqs;
  259. dev->nvqs = nvqs;
  260. mutex_init(&dev->mutex);
  261. dev->log_ctx = NULL;
  262. dev->log_file = NULL;
  263. dev->memory = NULL;
  264. dev->mm = NULL;
  265. spin_lock_init(&dev->work_lock);
  266. INIT_LIST_HEAD(&dev->work_list);
  267. dev->worker = NULL;
  268. for (i = 0; i < dev->nvqs; ++i) {
  269. vq = dev->vqs[i];
  270. vq->log = NULL;
  271. vq->indirect = NULL;
  272. vq->heads = NULL;
  273. vq->dev = dev;
  274. mutex_init(&vq->mutex);
  275. vhost_vq_reset(dev, vq);
  276. if (vq->handle_kick)
  277. vhost_poll_init(&vq->poll, vq->handle_kick,
  278. POLLIN, dev);
  279. }
  280. }
  281. EXPORT_SYMBOL_GPL(vhost_dev_init);
  282. /* Caller should have device mutex */
  283. long vhost_dev_check_owner(struct vhost_dev *dev)
  284. {
  285. /* Are you the owner? If not, I don't think you mean to do that */
  286. return dev->mm == current->mm ? 0 : -EPERM;
  287. }
  288. EXPORT_SYMBOL_GPL(vhost_dev_check_owner);
  289. struct vhost_attach_cgroups_struct {
  290. struct vhost_work work;
  291. struct task_struct *owner;
  292. int ret;
  293. };
  294. static void vhost_attach_cgroups_work(struct vhost_work *work)
  295. {
  296. struct vhost_attach_cgroups_struct *s;
  297. s = container_of(work, struct vhost_attach_cgroups_struct, work);
  298. s->ret = cgroup_attach_task_all(s->owner, current);
  299. }
  300. static int vhost_attach_cgroups(struct vhost_dev *dev)
  301. {
  302. struct vhost_attach_cgroups_struct attach;
  303. attach.owner = current;
  304. vhost_work_init(&attach.work, vhost_attach_cgroups_work);
  305. vhost_work_queue(dev, &attach.work);
  306. vhost_work_flush(dev, &attach.work);
  307. return attach.ret;
  308. }
  309. /* Caller should have device mutex */
  310. bool vhost_dev_has_owner(struct vhost_dev *dev)
  311. {
  312. return dev->mm;
  313. }
  314. EXPORT_SYMBOL_GPL(vhost_dev_has_owner);
  315. /* Caller should have device mutex */
  316. long vhost_dev_set_owner(struct vhost_dev *dev)
  317. {
  318. struct task_struct *worker;
  319. int err;
  320. /* Is there an owner already? */
  321. if (vhost_dev_has_owner(dev)) {
  322. err = -EBUSY;
  323. goto err_mm;
  324. }
  325. /* No owner, become one */
  326. dev->mm = get_task_mm(current);
  327. worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid);
  328. if (IS_ERR(worker)) {
  329. err = PTR_ERR(worker);
  330. goto err_worker;
  331. }
  332. dev->worker = worker;
  333. wake_up_process(worker); /* avoid contributing to loadavg */
  334. err = vhost_attach_cgroups(dev);
  335. if (err)
  336. goto err_cgroup;
  337. err = vhost_dev_alloc_iovecs(dev);
  338. if (err)
  339. goto err_cgroup;
  340. return 0;
  341. err_cgroup:
  342. kthread_stop(worker);
  343. dev->worker = NULL;
  344. err_worker:
  345. if (dev->mm)
  346. mmput(dev->mm);
  347. dev->mm = NULL;
  348. err_mm:
  349. return err;
  350. }
  351. EXPORT_SYMBOL_GPL(vhost_dev_set_owner);
  352. struct vhost_memory *vhost_dev_reset_owner_prepare(void)
  353. {
  354. return kmalloc(offsetof(struct vhost_memory, regions), GFP_KERNEL);
  355. }
  356. EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare);
  357. /* Caller should have device mutex */
  358. void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_memory *memory)
  359. {
  360. vhost_dev_cleanup(dev, true);
  361. /* Restore memory to default empty mapping. */
  362. memory->nregions = 0;
  363. RCU_INIT_POINTER(dev->memory, memory);
  364. }
  365. EXPORT_SYMBOL_GPL(vhost_dev_reset_owner);
  366. void vhost_dev_stop(struct vhost_dev *dev)
  367. {
  368. int i;
  369. for (i = 0; i < dev->nvqs; ++i) {
  370. if (dev->vqs[i]->kick && dev->vqs[i]->handle_kick) {
  371. vhost_poll_stop(&dev->vqs[i]->poll);
  372. vhost_poll_flush(&dev->vqs[i]->poll);
  373. }
  374. }
  375. }
  376. EXPORT_SYMBOL_GPL(vhost_dev_stop);
  377. /* Caller should have device mutex if and only if locked is set */
  378. void vhost_dev_cleanup(struct vhost_dev *dev, bool locked)
  379. {
  380. int i;
  381. for (i = 0; i < dev->nvqs; ++i) {
  382. if (dev->vqs[i]->error_ctx)
  383. eventfd_ctx_put(dev->vqs[i]->error_ctx);
  384. if (dev->vqs[i]->error)
  385. fput(dev->vqs[i]->error);
  386. if (dev->vqs[i]->kick)
  387. fput(dev->vqs[i]->kick);
  388. if (dev->vqs[i]->call_ctx)
  389. eventfd_ctx_put(dev->vqs[i]->call_ctx);
  390. if (dev->vqs[i]->call)
  391. fput(dev->vqs[i]->call);
  392. vhost_vq_reset(dev, dev->vqs[i]);
  393. }
  394. vhost_dev_free_iovecs(dev);
  395. if (dev->log_ctx)
  396. eventfd_ctx_put(dev->log_ctx);
  397. dev->log_ctx = NULL;
  398. if (dev->log_file)
  399. fput(dev->log_file);
  400. dev->log_file = NULL;
  401. /* No one will access memory at this point */
  402. kfree(rcu_dereference_protected(dev->memory,
  403. locked ==
  404. lockdep_is_held(&dev->mutex)));
  405. RCU_INIT_POINTER(dev->memory, NULL);
  406. WARN_ON(!list_empty(&dev->work_list));
  407. if (dev->worker) {
  408. kthread_stop(dev->worker);
  409. dev->worker = NULL;
  410. }
  411. if (dev->mm)
  412. mmput(dev->mm);
  413. dev->mm = NULL;
  414. }
  415. EXPORT_SYMBOL_GPL(vhost_dev_cleanup);
  416. static int log_access_ok(void __user *log_base, u64 addr, unsigned long sz)
  417. {
  418. u64 a = addr / VHOST_PAGE_SIZE / 8;
  419. /* Make sure 64 bit math will not overflow. */
  420. if (a > ULONG_MAX - (unsigned long)log_base ||
  421. a + (unsigned long)log_base > ULONG_MAX)
  422. return 0;
  423. return access_ok(VERIFY_WRITE, log_base + a,
  424. (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8);
  425. }
  426. /* Caller should have vq mutex and device mutex. */
  427. static int vq_memory_access_ok(void __user *log_base, struct vhost_memory *mem,
  428. int log_all)
  429. {
  430. int i;
  431. if (!mem)
  432. return 0;
  433. for (i = 0; i < mem->nregions; ++i) {
  434. struct vhost_memory_region *m = mem->regions + i;
  435. unsigned long a = m->userspace_addr;
  436. if (m->memory_size > ULONG_MAX)
  437. return 0;
  438. else if (!access_ok(VERIFY_WRITE, (void __user *)a,
  439. m->memory_size))
  440. return 0;
  441. else if (log_all && !log_access_ok(log_base,
  442. m->guest_phys_addr,
  443. m->memory_size))
  444. return 0;
  445. }
  446. return 1;
  447. }
  448. /* Can we switch to this memory table? */
  449. /* Caller should have device mutex but not vq mutex */
  450. static int memory_access_ok(struct vhost_dev *d, struct vhost_memory *mem,
  451. int log_all)
  452. {
  453. int i;
  454. for (i = 0; i < d->nvqs; ++i) {
  455. int ok;
  456. mutex_lock(&d->vqs[i]->mutex);
  457. /* If ring is inactive, will check when it's enabled. */
  458. if (d->vqs[i]->private_data)
  459. ok = vq_memory_access_ok(d->vqs[i]->log_base, mem,
  460. log_all);
  461. else
  462. ok = 1;
  463. mutex_unlock(&d->vqs[i]->mutex);
  464. if (!ok)
  465. return 0;
  466. }
  467. return 1;
  468. }
  469. static int vq_access_ok(struct vhost_dev *d, unsigned int num,
  470. struct vring_desc __user *desc,
  471. struct vring_avail __user *avail,
  472. struct vring_used __user *used)
  473. {
  474. size_t s = vhost_has_feature(d, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
  475. return access_ok(VERIFY_READ, desc, num * sizeof *desc) &&
  476. access_ok(VERIFY_READ, avail,
  477. sizeof *avail + num * sizeof *avail->ring + s) &&
  478. access_ok(VERIFY_WRITE, used,
  479. sizeof *used + num * sizeof *used->ring + s);
  480. }
  481. /* Can we log writes? */
  482. /* Caller should have device mutex but not vq mutex */
  483. int vhost_log_access_ok(struct vhost_dev *dev)
  484. {
  485. struct vhost_memory *mp;
  486. mp = rcu_dereference_protected(dev->memory,
  487. lockdep_is_held(&dev->mutex));
  488. return memory_access_ok(dev, mp, 1);
  489. }
  490. EXPORT_SYMBOL_GPL(vhost_log_access_ok);
  491. /* Verify access for write logging. */
  492. /* Caller should have vq mutex and device mutex */
  493. static int vq_log_access_ok(struct vhost_dev *d, struct vhost_virtqueue *vq,
  494. void __user *log_base)
  495. {
  496. struct vhost_memory *mp;
  497. size_t s = vhost_has_feature(d, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
  498. mp = rcu_dereference_protected(vq->dev->memory,
  499. lockdep_is_held(&vq->mutex));
  500. return vq_memory_access_ok(log_base, mp,
  501. vhost_has_feature(vq->dev, VHOST_F_LOG_ALL)) &&
  502. (!vq->log_used || log_access_ok(log_base, vq->log_addr,
  503. sizeof *vq->used +
  504. vq->num * sizeof *vq->used->ring + s));
  505. }
  506. /* Can we start vq? */
  507. /* Caller should have vq mutex and device mutex */
  508. int vhost_vq_access_ok(struct vhost_virtqueue *vq)
  509. {
  510. return vq_access_ok(vq->dev, vq->num, vq->desc, vq->avail, vq->used) &&
  511. vq_log_access_ok(vq->dev, vq, vq->log_base);
  512. }
  513. EXPORT_SYMBOL_GPL(vhost_vq_access_ok);
  514. static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
  515. {
  516. struct vhost_memory mem, *newmem, *oldmem;
  517. unsigned long size = offsetof(struct vhost_memory, regions);
  518. if (copy_from_user(&mem, m, size))
  519. return -EFAULT;
  520. if (mem.padding)
  521. return -EOPNOTSUPP;
  522. if (mem.nregions > VHOST_MEMORY_MAX_NREGIONS)
  523. return -E2BIG;
  524. newmem = kmalloc(size + mem.nregions * sizeof *m->regions, GFP_KERNEL);
  525. if (!newmem)
  526. return -ENOMEM;
  527. memcpy(newmem, &mem, size);
  528. if (copy_from_user(newmem->regions, m->regions,
  529. mem.nregions * sizeof *m->regions)) {
  530. kfree(newmem);
  531. return -EFAULT;
  532. }
  533. if (!memory_access_ok(d, newmem,
  534. vhost_has_feature(d, VHOST_F_LOG_ALL))) {
  535. kfree(newmem);
  536. return -EFAULT;
  537. }
  538. oldmem = rcu_dereference_protected(d->memory,
  539. lockdep_is_held(&d->mutex));
  540. rcu_assign_pointer(d->memory, newmem);
  541. synchronize_rcu();
  542. kfree(oldmem);
  543. return 0;
  544. }
  545. long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp)
  546. {
  547. struct file *eventfp, *filep = NULL;
  548. bool pollstart = false, pollstop = false;
  549. struct eventfd_ctx *ctx = NULL;
  550. u32 __user *idxp = argp;
  551. struct vhost_virtqueue *vq;
  552. struct vhost_vring_state s;
  553. struct vhost_vring_file f;
  554. struct vhost_vring_addr a;
  555. u32 idx;
  556. long r;
  557. r = get_user(idx, idxp);
  558. if (r < 0)
  559. return r;
  560. if (idx >= d->nvqs)
  561. return -ENOBUFS;
  562. vq = d->vqs[idx];
  563. mutex_lock(&vq->mutex);
  564. switch (ioctl) {
  565. case VHOST_SET_VRING_NUM:
  566. /* Resizing ring with an active backend?
  567. * You don't want to do that. */
  568. if (vq->private_data) {
  569. r = -EBUSY;
  570. break;
  571. }
  572. if (copy_from_user(&s, argp, sizeof s)) {
  573. r = -EFAULT;
  574. break;
  575. }
  576. if (!s.num || s.num > 0xffff || (s.num & (s.num - 1))) {
  577. r = -EINVAL;
  578. break;
  579. }
  580. vq->num = s.num;
  581. break;
  582. case VHOST_SET_VRING_BASE:
  583. /* Moving base with an active backend?
  584. * You don't want to do that. */
  585. if (vq->private_data) {
  586. r = -EBUSY;
  587. break;
  588. }
  589. if (copy_from_user(&s, argp, sizeof s)) {
  590. r = -EFAULT;
  591. break;
  592. }
  593. if (s.num > 0xffff) {
  594. r = -EINVAL;
  595. break;
  596. }
  597. vq->last_avail_idx = s.num;
  598. /* Forget the cached index value. */
  599. vq->avail_idx = vq->last_avail_idx;
  600. break;
  601. case VHOST_GET_VRING_BASE:
  602. s.index = idx;
  603. s.num = vq->last_avail_idx;
  604. if (copy_to_user(argp, &s, sizeof s))
  605. r = -EFAULT;
  606. break;
  607. case VHOST_SET_VRING_ADDR:
  608. if (copy_from_user(&a, argp, sizeof a)) {
  609. r = -EFAULT;
  610. break;
  611. }
  612. if (a.flags & ~(0x1 << VHOST_VRING_F_LOG)) {
  613. r = -EOPNOTSUPP;
  614. break;
  615. }
  616. /* For 32bit, verify that the top 32bits of the user
  617. data are set to zero. */
  618. if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr ||
  619. (u64)(unsigned long)a.used_user_addr != a.used_user_addr ||
  620. (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr) {
  621. r = -EFAULT;
  622. break;
  623. }
  624. if ((a.avail_user_addr & (sizeof *vq->avail->ring - 1)) ||
  625. (a.used_user_addr & (sizeof *vq->used->ring - 1)) ||
  626. (a.log_guest_addr & (sizeof *vq->used->ring - 1))) {
  627. r = -EINVAL;
  628. break;
  629. }
  630. /* We only verify access here if backend is configured.
  631. * If it is not, we don't as size might not have been setup.
  632. * We will verify when backend is configured. */
  633. if (vq->private_data) {
  634. if (!vq_access_ok(d, vq->num,
  635. (void __user *)(unsigned long)a.desc_user_addr,
  636. (void __user *)(unsigned long)a.avail_user_addr,
  637. (void __user *)(unsigned long)a.used_user_addr)) {
  638. r = -EINVAL;
  639. break;
  640. }
  641. /* Also validate log access for used ring if enabled. */
  642. if ((a.flags & (0x1 << VHOST_VRING_F_LOG)) &&
  643. !log_access_ok(vq->log_base, a.log_guest_addr,
  644. sizeof *vq->used +
  645. vq->num * sizeof *vq->used->ring)) {
  646. r = -EINVAL;
  647. break;
  648. }
  649. }
  650. vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG));
  651. vq->desc = (void __user *)(unsigned long)a.desc_user_addr;
  652. vq->avail = (void __user *)(unsigned long)a.avail_user_addr;
  653. vq->log_addr = a.log_guest_addr;
  654. vq->used = (void __user *)(unsigned long)a.used_user_addr;
  655. break;
  656. case VHOST_SET_VRING_KICK:
  657. if (copy_from_user(&f, argp, sizeof f)) {
  658. r = -EFAULT;
  659. break;
  660. }
  661. eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
  662. if (IS_ERR(eventfp)) {
  663. r = PTR_ERR(eventfp);
  664. break;
  665. }
  666. if (eventfp != vq->kick) {
  667. pollstop = (filep = vq->kick) != NULL;
  668. pollstart = (vq->kick = eventfp) != NULL;
  669. } else
  670. filep = eventfp;
  671. break;
  672. case VHOST_SET_VRING_CALL:
  673. if (copy_from_user(&f, argp, sizeof f)) {
  674. r = -EFAULT;
  675. break;
  676. }
  677. eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
  678. if (IS_ERR(eventfp)) {
  679. r = PTR_ERR(eventfp);
  680. break;
  681. }
  682. if (eventfp != vq->call) {
  683. filep = vq->call;
  684. ctx = vq->call_ctx;
  685. vq->call = eventfp;
  686. vq->call_ctx = eventfp ?
  687. eventfd_ctx_fileget(eventfp) : NULL;
  688. } else
  689. filep = eventfp;
  690. break;
  691. case VHOST_SET_VRING_ERR:
  692. if (copy_from_user(&f, argp, sizeof f)) {
  693. r = -EFAULT;
  694. break;
  695. }
  696. eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
  697. if (IS_ERR(eventfp)) {
  698. r = PTR_ERR(eventfp);
  699. break;
  700. }
  701. if (eventfp != vq->error) {
  702. filep = vq->error;
  703. vq->error = eventfp;
  704. ctx = vq->error_ctx;
  705. vq->error_ctx = eventfp ?
  706. eventfd_ctx_fileget(eventfp) : NULL;
  707. } else
  708. filep = eventfp;
  709. break;
  710. default:
  711. r = -ENOIOCTLCMD;
  712. }
  713. if (pollstop && vq->handle_kick)
  714. vhost_poll_stop(&vq->poll);
  715. if (ctx)
  716. eventfd_ctx_put(ctx);
  717. if (filep)
  718. fput(filep);
  719. if (pollstart && vq->handle_kick)
  720. r = vhost_poll_start(&vq->poll, vq->kick);
  721. mutex_unlock(&vq->mutex);
  722. if (pollstop && vq->handle_kick)
  723. vhost_poll_flush(&vq->poll);
  724. return r;
  725. }
  726. EXPORT_SYMBOL_GPL(vhost_vring_ioctl);
  727. /* Caller must have device mutex */
  728. long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
  729. {
  730. struct file *eventfp, *filep = NULL;
  731. struct eventfd_ctx *ctx = NULL;
  732. u64 p;
  733. long r;
  734. int i, fd;
  735. /* If you are not the owner, you can become one */
  736. if (ioctl == VHOST_SET_OWNER) {
  737. r = vhost_dev_set_owner(d);
  738. goto done;
  739. }
  740. /* You must be the owner to do anything else */
  741. r = vhost_dev_check_owner(d);
  742. if (r)
  743. goto done;
  744. switch (ioctl) {
  745. case VHOST_SET_MEM_TABLE:
  746. r = vhost_set_memory(d, argp);
  747. break;
  748. case VHOST_SET_LOG_BASE:
  749. if (copy_from_user(&p, argp, sizeof p)) {
  750. r = -EFAULT;
  751. break;
  752. }
  753. if ((u64)(unsigned long)p != p) {
  754. r = -EFAULT;
  755. break;
  756. }
  757. for (i = 0; i < d->nvqs; ++i) {
  758. struct vhost_virtqueue *vq;
  759. void __user *base = (void __user *)(unsigned long)p;
  760. vq = d->vqs[i];
  761. mutex_lock(&vq->mutex);
  762. /* If ring is inactive, will check when it's enabled. */
  763. if (vq->private_data && !vq_log_access_ok(d, vq, base))
  764. r = -EFAULT;
  765. else
  766. vq->log_base = base;
  767. mutex_unlock(&vq->mutex);
  768. }
  769. break;
  770. case VHOST_SET_LOG_FD:
  771. r = get_user(fd, (int __user *)argp);
  772. if (r < 0)
  773. break;
  774. eventfp = fd == -1 ? NULL : eventfd_fget(fd);
  775. if (IS_ERR(eventfp)) {
  776. r = PTR_ERR(eventfp);
  777. break;
  778. }
  779. if (eventfp != d->log_file) {
  780. filep = d->log_file;
  781. ctx = d->log_ctx;
  782. d->log_ctx = eventfp ?
  783. eventfd_ctx_fileget(eventfp) : NULL;
  784. } else
  785. filep = eventfp;
  786. for (i = 0; i < d->nvqs; ++i) {
  787. mutex_lock(&d->vqs[i]->mutex);
  788. d->vqs[i]->log_ctx = d->log_ctx;
  789. mutex_unlock(&d->vqs[i]->mutex);
  790. }
  791. if (ctx)
  792. eventfd_ctx_put(ctx);
  793. if (filep)
  794. fput(filep);
  795. break;
  796. default:
  797. r = -ENOIOCTLCMD;
  798. break;
  799. }
  800. done:
  801. return r;
  802. }
  803. EXPORT_SYMBOL_GPL(vhost_dev_ioctl);
  804. static const struct vhost_memory_region *find_region(struct vhost_memory *mem,
  805. __u64 addr, __u32 len)
  806. {
  807. struct vhost_memory_region *reg;
  808. int i;
  809. /* linear search is not brilliant, but we really have on the order of 6
  810. * regions in practice */
  811. for (i = 0; i < mem->nregions; ++i) {
  812. reg = mem->regions + i;
  813. if (reg->guest_phys_addr <= addr &&
  814. reg->guest_phys_addr + reg->memory_size - 1 >= addr)
  815. return reg;
  816. }
  817. return NULL;
  818. }
  819. /* TODO: This is really inefficient. We need something like get_user()
  820. * (instruction directly accesses the data, with an exception table entry
  821. * returning -EFAULT). See Documentation/x86/exception-tables.txt.
  822. */
  823. static int set_bit_to_user(int nr, void __user *addr)
  824. {
  825. unsigned long log = (unsigned long)addr;
  826. struct page *page;
  827. void *base;
  828. int bit = nr + (log % PAGE_SIZE) * 8;
  829. int r;
  830. r = get_user_pages_fast(log, 1, 1, &page);
  831. if (r < 0)
  832. return r;
  833. BUG_ON(r != 1);
  834. base = kmap_atomic(page);
  835. set_bit(bit, base);
  836. kunmap_atomic(base);
  837. set_page_dirty_lock(page);
  838. put_page(page);
  839. return 0;
  840. }
  841. static int log_write(void __user *log_base,
  842. u64 write_address, u64 write_length)
  843. {
  844. u64 write_page = write_address / VHOST_PAGE_SIZE;
  845. int r;
  846. if (!write_length)
  847. return 0;
  848. write_length += write_address % VHOST_PAGE_SIZE;
  849. for (;;) {
  850. u64 base = (u64)(unsigned long)log_base;
  851. u64 log = base + write_page / 8;
  852. int bit = write_page % 8;
  853. if ((u64)(unsigned long)log != log)
  854. return -EFAULT;
  855. r = set_bit_to_user(bit, (void __user *)(unsigned long)log);
  856. if (r < 0)
  857. return r;
  858. if (write_length <= VHOST_PAGE_SIZE)
  859. break;
  860. write_length -= VHOST_PAGE_SIZE;
  861. write_page += 1;
  862. }
  863. return r;
  864. }
  865. int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,
  866. unsigned int log_num, u64 len)
  867. {
  868. int i, r;
  869. /* Make sure data written is seen before log. */
  870. smp_wmb();
  871. for (i = 0; i < log_num; ++i) {
  872. u64 l = min(log[i].len, len);
  873. r = log_write(vq->log_base, log[i].addr, l);
  874. if (r < 0)
  875. return r;
  876. len -= l;
  877. if (!len) {
  878. if (vq->log_ctx)
  879. eventfd_signal(vq->log_ctx, 1);
  880. return 0;
  881. }
  882. }
  883. /* Length written exceeds what we have stored. This is a bug. */
  884. BUG();
  885. return 0;
  886. }
  887. EXPORT_SYMBOL_GPL(vhost_log_write);
  888. static int vhost_update_used_flags(struct vhost_virtqueue *vq)
  889. {
  890. void __user *used;
  891. if (__put_user(vq->used_flags, &vq->used->flags) < 0)
  892. return -EFAULT;
  893. if (unlikely(vq->log_used)) {
  894. /* Make sure the flag is seen before log. */
  895. smp_wmb();
  896. /* Log used flag write. */
  897. used = &vq->used->flags;
  898. log_write(vq->log_base, vq->log_addr +
  899. (used - (void __user *)vq->used),
  900. sizeof vq->used->flags);
  901. if (vq->log_ctx)
  902. eventfd_signal(vq->log_ctx, 1);
  903. }
  904. return 0;
  905. }
  906. static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event)
  907. {
  908. if (__put_user(vq->avail_idx, vhost_avail_event(vq)))
  909. return -EFAULT;
  910. if (unlikely(vq->log_used)) {
  911. void __user *used;
  912. /* Make sure the event is seen before log. */
  913. smp_wmb();
  914. /* Log avail event write */
  915. used = vhost_avail_event(vq);
  916. log_write(vq->log_base, vq->log_addr +
  917. (used - (void __user *)vq->used),
  918. sizeof *vhost_avail_event(vq));
  919. if (vq->log_ctx)
  920. eventfd_signal(vq->log_ctx, 1);
  921. }
  922. return 0;
  923. }
  924. int vhost_init_used(struct vhost_virtqueue *vq)
  925. {
  926. int r;
  927. if (!vq->private_data)
  928. return 0;
  929. r = vhost_update_used_flags(vq);
  930. if (r)
  931. return r;
  932. vq->signalled_used_valid = false;
  933. return get_user(vq->last_used_idx, &vq->used->idx);
  934. }
  935. EXPORT_SYMBOL_GPL(vhost_init_used);
  936. static int translate_desc(struct vhost_dev *dev, u64 addr, u32 len,
  937. struct iovec iov[], int iov_size)
  938. {
  939. const struct vhost_memory_region *reg;
  940. struct vhost_memory *mem;
  941. struct iovec *_iov;
  942. u64 s = 0;
  943. int ret = 0;
  944. rcu_read_lock();
  945. mem = rcu_dereference(dev->memory);
  946. while ((u64)len > s) {
  947. u64 size;
  948. if (unlikely(ret >= iov_size)) {
  949. ret = -ENOBUFS;
  950. break;
  951. }
  952. reg = find_region(mem, addr, len);
  953. if (unlikely(!reg)) {
  954. ret = -EFAULT;
  955. break;
  956. }
  957. _iov = iov + ret;
  958. size = reg->memory_size - addr + reg->guest_phys_addr;
  959. _iov->iov_len = min((u64)len - s, size);
  960. _iov->iov_base = (void __user *)(unsigned long)
  961. (reg->userspace_addr + addr - reg->guest_phys_addr);
  962. s += size;
  963. addr += size;
  964. ++ret;
  965. }
  966. rcu_read_unlock();
  967. return ret;
  968. }
  969. /* Each buffer in the virtqueues is actually a chain of descriptors. This
  970. * function returns the next descriptor in the chain,
  971. * or -1U if we're at the end. */
  972. static unsigned next_desc(struct vring_desc *desc)
  973. {
  974. unsigned int next;
  975. /* If this descriptor says it doesn't chain, we're done. */
  976. if (!(desc->flags & VRING_DESC_F_NEXT))
  977. return -1U;
  978. /* Check they're not leading us off end of descriptors. */
  979. next = desc->next;
  980. /* Make sure compiler knows to grab that: we don't want it changing! */
  981. /* We will use the result as an index in an array, so most
  982. * architectures only need a compiler barrier here. */
  983. read_barrier_depends();
  984. return next;
  985. }
  986. static int get_indirect(struct vhost_dev *dev, struct vhost_virtqueue *vq,
  987. struct iovec iov[], unsigned int iov_size,
  988. unsigned int *out_num, unsigned int *in_num,
  989. struct vhost_log *log, unsigned int *log_num,
  990. struct vring_desc *indirect)
  991. {
  992. struct vring_desc desc;
  993. unsigned int i = 0, count, found = 0;
  994. int ret;
  995. /* Sanity check */
  996. if (unlikely(indirect->len % sizeof desc)) {
  997. vq_err(vq, "Invalid length in indirect descriptor: "
  998. "len 0x%llx not multiple of 0x%zx\n",
  999. (unsigned long long)indirect->len,
  1000. sizeof desc);
  1001. return -EINVAL;
  1002. }
  1003. ret = translate_desc(dev, indirect->addr, indirect->len, vq->indirect,
  1004. UIO_MAXIOV);
  1005. if (unlikely(ret < 0)) {
  1006. vq_err(vq, "Translation failure %d in indirect.\n", ret);
  1007. return ret;
  1008. }
  1009. /* We will use the result as an address to read from, so most
  1010. * architectures only need a compiler barrier here. */
  1011. read_barrier_depends();
  1012. count = indirect->len / sizeof desc;
  1013. /* Buffers are chained via a 16 bit next field, so
  1014. * we can have at most 2^16 of these. */
  1015. if (unlikely(count > USHRT_MAX + 1)) {
  1016. vq_err(vq, "Indirect buffer length too big: %d\n",
  1017. indirect->len);
  1018. return -E2BIG;
  1019. }
  1020. do {
  1021. unsigned iov_count = *in_num + *out_num;
  1022. if (unlikely(++found > count)) {
  1023. vq_err(vq, "Loop detected: last one at %u "
  1024. "indirect size %u\n",
  1025. i, count);
  1026. return -EINVAL;
  1027. }
  1028. if (unlikely(memcpy_fromiovec((unsigned char *)&desc,
  1029. vq->indirect, sizeof desc))) {
  1030. vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n",
  1031. i, (size_t)indirect->addr + i * sizeof desc);
  1032. return -EINVAL;
  1033. }
  1034. if (unlikely(desc.flags & VRING_DESC_F_INDIRECT)) {
  1035. vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n",
  1036. i, (size_t)indirect->addr + i * sizeof desc);
  1037. return -EINVAL;
  1038. }
  1039. ret = translate_desc(dev, desc.addr, desc.len, iov + iov_count,
  1040. iov_size - iov_count);
  1041. if (unlikely(ret < 0)) {
  1042. vq_err(vq, "Translation failure %d indirect idx %d\n",
  1043. ret, i);
  1044. return ret;
  1045. }
  1046. /* If this is an input descriptor, increment that count. */
  1047. if (desc.flags & VRING_DESC_F_WRITE) {
  1048. *in_num += ret;
  1049. if (unlikely(log)) {
  1050. log[*log_num].addr = desc.addr;
  1051. log[*log_num].len = desc.len;
  1052. ++*log_num;
  1053. }
  1054. } else {
  1055. /* If it's an output descriptor, they're all supposed
  1056. * to come before any input descriptors. */
  1057. if (unlikely(*in_num)) {
  1058. vq_err(vq, "Indirect descriptor "
  1059. "has out after in: idx %d\n", i);
  1060. return -EINVAL;
  1061. }
  1062. *out_num += ret;
  1063. }
  1064. } while ((i = next_desc(&desc)) != -1);
  1065. return 0;
  1066. }
  1067. /* This looks in the virtqueue and for the first available buffer, and converts
  1068. * it to an iovec for convenient access. Since descriptors consist of some
  1069. * number of output then some number of input descriptors, it's actually two
  1070. * iovecs, but we pack them into one and note how many of each there were.
  1071. *
  1072. * This function returns the descriptor number found, or vq->num (which is
  1073. * never a valid descriptor number) if none was found. A negative code is
  1074. * returned on error. */
  1075. int vhost_get_vq_desc(struct vhost_dev *dev, struct vhost_virtqueue *vq,
  1076. struct iovec iov[], unsigned int iov_size,
  1077. unsigned int *out_num, unsigned int *in_num,
  1078. struct vhost_log *log, unsigned int *log_num)
  1079. {
  1080. struct vring_desc desc;
  1081. unsigned int i, head, found = 0;
  1082. u16 last_avail_idx;
  1083. int ret;
  1084. /* Check it isn't doing very strange things with descriptor numbers. */
  1085. last_avail_idx = vq->last_avail_idx;
  1086. if (unlikely(__get_user(vq->avail_idx, &vq->avail->idx))) {
  1087. vq_err(vq, "Failed to access avail idx at %p\n",
  1088. &vq->avail->idx);
  1089. return -EFAULT;
  1090. }
  1091. if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) {
  1092. vq_err(vq, "Guest moved used index from %u to %u",
  1093. last_avail_idx, vq->avail_idx);
  1094. return -EFAULT;
  1095. }
  1096. /* If there's nothing new since last we looked, return invalid. */
  1097. if (vq->avail_idx == last_avail_idx)
  1098. return vq->num;
  1099. /* Only get avail ring entries after they have been exposed by guest. */
  1100. smp_rmb();
  1101. /* Grab the next descriptor number they're advertising, and increment
  1102. * the index we've seen. */
  1103. if (unlikely(__get_user(head,
  1104. &vq->avail->ring[last_avail_idx % vq->num]))) {
  1105. vq_err(vq, "Failed to read head: idx %d address %p\n",
  1106. last_avail_idx,
  1107. &vq->avail->ring[last_avail_idx % vq->num]);
  1108. return -EFAULT;
  1109. }
  1110. /* If their number is silly, that's an error. */
  1111. if (unlikely(head >= vq->num)) {
  1112. vq_err(vq, "Guest says index %u > %u is available",
  1113. head, vq->num);
  1114. return -EINVAL;
  1115. }
  1116. /* When we start there are none of either input nor output. */
  1117. *out_num = *in_num = 0;
  1118. if (unlikely(log))
  1119. *log_num = 0;
  1120. i = head;
  1121. do {
  1122. unsigned iov_count = *in_num + *out_num;
  1123. if (unlikely(i >= vq->num)) {
  1124. vq_err(vq, "Desc index is %u > %u, head = %u",
  1125. i, vq->num, head);
  1126. return -EINVAL;
  1127. }
  1128. if (unlikely(++found > vq->num)) {
  1129. vq_err(vq, "Loop detected: last one at %u "
  1130. "vq size %u head %u\n",
  1131. i, vq->num, head);
  1132. return -EINVAL;
  1133. }
  1134. ret = __copy_from_user(&desc, vq->desc + i, sizeof desc);
  1135. if (unlikely(ret)) {
  1136. vq_err(vq, "Failed to get descriptor: idx %d addr %p\n",
  1137. i, vq->desc + i);
  1138. return -EFAULT;
  1139. }
  1140. if (desc.flags & VRING_DESC_F_INDIRECT) {
  1141. ret = get_indirect(dev, vq, iov, iov_size,
  1142. out_num, in_num,
  1143. log, log_num, &desc);
  1144. if (unlikely(ret < 0)) {
  1145. vq_err(vq, "Failure detected "
  1146. "in indirect descriptor at idx %d\n", i);
  1147. return ret;
  1148. }
  1149. continue;
  1150. }
  1151. ret = translate_desc(dev, desc.addr, desc.len, iov + iov_count,
  1152. iov_size - iov_count);
  1153. if (unlikely(ret < 0)) {
  1154. vq_err(vq, "Translation failure %d descriptor idx %d\n",
  1155. ret, i);
  1156. return ret;
  1157. }
  1158. if (desc.flags & VRING_DESC_F_WRITE) {
  1159. /* If this is an input descriptor,
  1160. * increment that count. */
  1161. *in_num += ret;
  1162. if (unlikely(log)) {
  1163. log[*log_num].addr = desc.addr;
  1164. log[*log_num].len = desc.len;
  1165. ++*log_num;
  1166. }
  1167. } else {
  1168. /* If it's an output descriptor, they're all supposed
  1169. * to come before any input descriptors. */
  1170. if (unlikely(*in_num)) {
  1171. vq_err(vq, "Descriptor has out after in: "
  1172. "idx %d\n", i);
  1173. return -EINVAL;
  1174. }
  1175. *out_num += ret;
  1176. }
  1177. } while ((i = next_desc(&desc)) != -1);
  1178. /* On success, increment avail index. */
  1179. vq->last_avail_idx++;
  1180. /* Assume notifications from guest are disabled at this point,
  1181. * if they aren't we would need to update avail_event index. */
  1182. BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY));
  1183. return head;
  1184. }
  1185. EXPORT_SYMBOL_GPL(vhost_get_vq_desc);
  1186. /* Reverse the effect of vhost_get_vq_desc. Useful for error handling. */
  1187. void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n)
  1188. {
  1189. vq->last_avail_idx -= n;
  1190. }
  1191. EXPORT_SYMBOL_GPL(vhost_discard_vq_desc);
  1192. /* After we've used one of their buffers, we tell them about it. We'll then
  1193. * want to notify the guest, using eventfd. */
  1194. int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len)
  1195. {
  1196. struct vring_used_elem heads = { head, len };
  1197. return vhost_add_used_n(vq, &heads, 1);
  1198. }
  1199. EXPORT_SYMBOL_GPL(vhost_add_used);
  1200. static int __vhost_add_used_n(struct vhost_virtqueue *vq,
  1201. struct vring_used_elem *heads,
  1202. unsigned count)
  1203. {
  1204. struct vring_used_elem __user *used;
  1205. u16 old, new;
  1206. int start;
  1207. start = vq->last_used_idx % vq->num;
  1208. used = vq->used->ring + start;
  1209. if (count == 1) {
  1210. if (__put_user(heads[0].id, &used->id)) {
  1211. vq_err(vq, "Failed to write used id");
  1212. return -EFAULT;
  1213. }
  1214. if (__put_user(heads[0].len, &used->len)) {
  1215. vq_err(vq, "Failed to write used len");
  1216. return -EFAULT;
  1217. }
  1218. } else if (__copy_to_user(used, heads, count * sizeof *used)) {
  1219. vq_err(vq, "Failed to write used");
  1220. return -EFAULT;
  1221. }
  1222. if (unlikely(vq->log_used)) {
  1223. /* Make sure data is seen before log. */
  1224. smp_wmb();
  1225. /* Log used ring entry write. */
  1226. log_write(vq->log_base,
  1227. vq->log_addr +
  1228. ((void __user *)used - (void __user *)vq->used),
  1229. count * sizeof *used);
  1230. }
  1231. old = vq->last_used_idx;
  1232. new = (vq->last_used_idx += count);
  1233. /* If the driver never bothers to signal in a very long while,
  1234. * used index might wrap around. If that happens, invalidate
  1235. * signalled_used index we stored. TODO: make sure driver
  1236. * signals at least once in 2^16 and remove this. */
  1237. if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old)))
  1238. vq->signalled_used_valid = false;
  1239. return 0;
  1240. }
  1241. /* After we've used one of their buffers, we tell them about it. We'll then
  1242. * want to notify the guest, using eventfd. */
  1243. int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads,
  1244. unsigned count)
  1245. {
  1246. int start, n, r;
  1247. start = vq->last_used_idx % vq->num;
  1248. n = vq->num - start;
  1249. if (n < count) {
  1250. r = __vhost_add_used_n(vq, heads, n);
  1251. if (r < 0)
  1252. return r;
  1253. heads += n;
  1254. count -= n;
  1255. }
  1256. r = __vhost_add_used_n(vq, heads, count);
  1257. /* Make sure buffer is written before we update index. */
  1258. smp_wmb();
  1259. if (put_user(vq->last_used_idx, &vq->used->idx)) {
  1260. vq_err(vq, "Failed to increment used idx");
  1261. return -EFAULT;
  1262. }
  1263. if (unlikely(vq->log_used)) {
  1264. /* Log used index update. */
  1265. log_write(vq->log_base,
  1266. vq->log_addr + offsetof(struct vring_used, idx),
  1267. sizeof vq->used->idx);
  1268. if (vq->log_ctx)
  1269. eventfd_signal(vq->log_ctx, 1);
  1270. }
  1271. return r;
  1272. }
  1273. EXPORT_SYMBOL_GPL(vhost_add_used_n);
  1274. static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
  1275. {
  1276. __u16 old, new, event;
  1277. bool v;
  1278. /* Flush out used index updates. This is paired
  1279. * with the barrier that the Guest executes when enabling
  1280. * interrupts. */
  1281. smp_mb();
  1282. if (vhost_has_feature(dev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
  1283. unlikely(vq->avail_idx == vq->last_avail_idx))
  1284. return true;
  1285. if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
  1286. __u16 flags;
  1287. if (__get_user(flags, &vq->avail->flags)) {
  1288. vq_err(vq, "Failed to get flags");
  1289. return true;
  1290. }
  1291. return !(flags & VRING_AVAIL_F_NO_INTERRUPT);
  1292. }
  1293. old = vq->signalled_used;
  1294. v = vq->signalled_used_valid;
  1295. new = vq->signalled_used = vq->last_used_idx;
  1296. vq->signalled_used_valid = true;
  1297. if (unlikely(!v))
  1298. return true;
  1299. if (get_user(event, vhost_used_event(vq))) {
  1300. vq_err(vq, "Failed to get used event idx");
  1301. return true;
  1302. }
  1303. return vring_need_event(event, new, old);
  1304. }
  1305. /* This actually signals the guest, using eventfd. */
  1306. void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)
  1307. {
  1308. /* Signal the Guest tell them we used something up. */
  1309. if (vq->call_ctx && vhost_notify(dev, vq))
  1310. eventfd_signal(vq->call_ctx, 1);
  1311. }
  1312. EXPORT_SYMBOL_GPL(vhost_signal);
  1313. /* And here's the combo meal deal. Supersize me! */
  1314. void vhost_add_used_and_signal(struct vhost_dev *dev,
  1315. struct vhost_virtqueue *vq,
  1316. unsigned int head, int len)
  1317. {
  1318. vhost_add_used(vq, head, len);
  1319. vhost_signal(dev, vq);
  1320. }
  1321. EXPORT_SYMBOL_GPL(vhost_add_used_and_signal);
  1322. /* multi-buffer version of vhost_add_used_and_signal */
  1323. void vhost_add_used_and_signal_n(struct vhost_dev *dev,
  1324. struct vhost_virtqueue *vq,
  1325. struct vring_used_elem *heads, unsigned count)
  1326. {
  1327. vhost_add_used_n(vq, heads, count);
  1328. vhost_signal(dev, vq);
  1329. }
  1330. EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n);
  1331. /* OK, now we need to know about added descriptors. */
  1332. bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
  1333. {
  1334. u16 avail_idx;
  1335. int r;
  1336. if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY))
  1337. return false;
  1338. vq->used_flags &= ~VRING_USED_F_NO_NOTIFY;
  1339. if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
  1340. r = vhost_update_used_flags(vq);
  1341. if (r) {
  1342. vq_err(vq, "Failed to enable notification at %p: %d\n",
  1343. &vq->used->flags, r);
  1344. return false;
  1345. }
  1346. } else {
  1347. r = vhost_update_avail_event(vq, vq->avail_idx);
  1348. if (r) {
  1349. vq_err(vq, "Failed to update avail event index at %p: %d\n",
  1350. vhost_avail_event(vq), r);
  1351. return false;
  1352. }
  1353. }
  1354. /* They could have slipped one in as we were doing that: make
  1355. * sure it's written, then check again. */
  1356. smp_mb();
  1357. r = __get_user(avail_idx, &vq->avail->idx);
  1358. if (r) {
  1359. vq_err(vq, "Failed to check avail idx at %p: %d\n",
  1360. &vq->avail->idx, r);
  1361. return false;
  1362. }
  1363. return avail_idx != vq->avail_idx;
  1364. }
  1365. EXPORT_SYMBOL_GPL(vhost_enable_notify);
  1366. /* We don't need to be notified again. */
  1367. void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
  1368. {
  1369. int r;
  1370. if (vq->used_flags & VRING_USED_F_NO_NOTIFY)
  1371. return;
  1372. vq->used_flags |= VRING_USED_F_NO_NOTIFY;
  1373. if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
  1374. r = vhost_update_used_flags(vq);
  1375. if (r)
  1376. vq_err(vq, "Failed to enable notification at %p: %d\n",
  1377. &vq->used->flags, r);
  1378. }
  1379. }
  1380. EXPORT_SYMBOL_GPL(vhost_disable_notify);
  1381. static int __init vhost_init(void)
  1382. {
  1383. return 0;
  1384. }
  1385. static void __exit vhost_exit(void)
  1386. {
  1387. }
  1388. module_init(vhost_init);
  1389. module_exit(vhost_exit);
  1390. MODULE_VERSION("0.0.1");
  1391. MODULE_LICENSE("GPL v2");
  1392. MODULE_AUTHOR("Michael S. Tsirkin");
  1393. MODULE_DESCRIPTION("Host kernel accelerator for virtio");