vmci_queue_pair.c 93 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268
  1. /*
  2. * VMware VMCI Driver
  3. *
  4. * Copyright (C) 2012 VMware, Inc. All rights reserved.
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms of the GNU General Public License as published by the
  8. * Free Software Foundation version 2 and no later version.
  9. *
  10. * This program is distributed in the hope that it will be useful, but
  11. * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  12. * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
  13. * for more details.
  14. */
  15. #include <linux/vmw_vmci_defs.h>
  16. #include <linux/vmw_vmci_api.h>
  17. #include <linux/highmem.h>
  18. #include <linux/kernel.h>
  19. #include <linux/mm.h>
  20. #include <linux/module.h>
  21. #include <linux/mutex.h>
  22. #include <linux/pagemap.h>
  23. #include <linux/pci.h>
  24. #include <linux/sched.h>
  25. #include <linux/slab.h>
  26. #include <linux/uio.h>
  27. #include <linux/wait.h>
  28. #include <linux/vmalloc.h>
  29. #include <linux/skbuff.h>
  30. #include "vmci_handle_array.h"
  31. #include "vmci_queue_pair.h"
  32. #include "vmci_datagram.h"
  33. #include "vmci_resource.h"
  34. #include "vmci_context.h"
  35. #include "vmci_driver.h"
  36. #include "vmci_event.h"
  37. #include "vmci_route.h"
  38. /*
  39. * In the following, we will distinguish between two kinds of VMX processes -
  40. * the ones with versions lower than VMCI_VERSION_NOVMVM that use specialized
  41. * VMCI page files in the VMX and supporting VM to VM communication and the
  42. * newer ones that use the guest memory directly. We will in the following
  43. * refer to the older VMX versions as old-style VMX'en, and the newer ones as
  44. * new-style VMX'en.
  45. *
  46. * The state transition datagram is as follows (the VMCIQPB_ prefix has been
  47. * removed for readability) - see below for more details on the transtions:
  48. *
  49. * -------------- NEW -------------
  50. * | |
  51. * \_/ \_/
  52. * CREATED_NO_MEM <-----------------> CREATED_MEM
  53. * | | |
  54. * | o-----------------------o |
  55. * | | |
  56. * \_/ \_/ \_/
  57. * ATTACHED_NO_MEM <----------------> ATTACHED_MEM
  58. * | | |
  59. * | o----------------------o |
  60. * | | |
  61. * \_/ \_/ \_/
  62. * SHUTDOWN_NO_MEM <----------------> SHUTDOWN_MEM
  63. * | |
  64. * | |
  65. * -------------> gone <-------------
  66. *
  67. * In more detail. When a VMCI queue pair is first created, it will be in the
  68. * VMCIQPB_NEW state. It will then move into one of the following states:
  69. *
  70. * - VMCIQPB_CREATED_NO_MEM: this state indicates that either:
  71. *
  72. * - the created was performed by a host endpoint, in which case there is
  73. * no backing memory yet.
  74. *
  75. * - the create was initiated by an old-style VMX, that uses
  76. * vmci_qp_broker_set_page_store to specify the UVAs of the queue pair at
  77. * a later point in time. This state can be distinguished from the one
  78. * above by the context ID of the creator. A host side is not allowed to
  79. * attach until the page store has been set.
  80. *
  81. * - VMCIQPB_CREATED_MEM: this state is the result when the queue pair
  82. * is created by a VMX using the queue pair device backend that
  83. * sets the UVAs of the queue pair immediately and stores the
  84. * information for later attachers. At this point, it is ready for
  85. * the host side to attach to it.
  86. *
  87. * Once the queue pair is in one of the created states (with the exception of
  88. * the case mentioned for older VMX'en above), it is possible to attach to the
  89. * queue pair. Again we have two new states possible:
  90. *
  91. * - VMCIQPB_ATTACHED_MEM: this state can be reached through the following
  92. * paths:
  93. *
  94. * - from VMCIQPB_CREATED_NO_MEM when a new-style VMX allocates a queue
  95. * pair, and attaches to a queue pair previously created by the host side.
  96. *
  97. * - from VMCIQPB_CREATED_MEM when the host side attaches to a queue pair
  98. * already created by a guest.
  99. *
  100. * - from VMCIQPB_ATTACHED_NO_MEM, when an old-style VMX calls
  101. * vmci_qp_broker_set_page_store (see below).
  102. *
  103. * - VMCIQPB_ATTACHED_NO_MEM: If the queue pair already was in the
  104. * VMCIQPB_CREATED_NO_MEM due to a host side create, an old-style VMX will
  105. * bring the queue pair into this state. Once vmci_qp_broker_set_page_store
  106. * is called to register the user memory, the VMCIQPB_ATTACH_MEM state
  107. * will be entered.
  108. *
  109. * From the attached queue pair, the queue pair can enter the shutdown states
  110. * when either side of the queue pair detaches. If the guest side detaches
  111. * first, the queue pair will enter the VMCIQPB_SHUTDOWN_NO_MEM state, where
  112. * the content of the queue pair will no longer be available. If the host
  113. * side detaches first, the queue pair will either enter the
  114. * VMCIQPB_SHUTDOWN_MEM, if the guest memory is currently mapped, or
  115. * VMCIQPB_SHUTDOWN_NO_MEM, if the guest memory is not mapped
  116. * (e.g., the host detaches while a guest is stunned).
  117. *
  118. * New-style VMX'en will also unmap guest memory, if the guest is
  119. * quiesced, e.g., during a snapshot operation. In that case, the guest
  120. * memory will no longer be available, and the queue pair will transition from
  121. * *_MEM state to a *_NO_MEM state. The VMX may later map the memory once more,
  122. * in which case the queue pair will transition from the *_NO_MEM state at that
  123. * point back to the *_MEM state. Note that the *_NO_MEM state may have changed,
  124. * since the peer may have either attached or detached in the meantime. The
  125. * values are laid out such that ++ on a state will move from a *_NO_MEM to a
  126. * *_MEM state, and vice versa.
  127. */
  128. /* The Kernel specific component of the struct vmci_queue structure. */
  129. struct vmci_queue_kern_if {
  130. struct mutex __mutex; /* Protects the queue. */
  131. struct mutex *mutex; /* Shared by producer and consumer queues. */
  132. size_t num_pages; /* Number of pages incl. header. */
  133. bool host; /* Host or guest? */
  134. union {
  135. struct {
  136. dma_addr_t *pas;
  137. void **vas;
  138. } g; /* Used by the guest. */
  139. struct {
  140. struct page **page;
  141. struct page **header_page;
  142. } h; /* Used by the host. */
  143. } u;
  144. };
  145. /*
  146. * This structure is opaque to the clients.
  147. */
  148. struct vmci_qp {
  149. struct vmci_handle handle;
  150. struct vmci_queue *produce_q;
  151. struct vmci_queue *consume_q;
  152. u64 produce_q_size;
  153. u64 consume_q_size;
  154. u32 peer;
  155. u32 flags;
  156. u32 priv_flags;
  157. bool guest_endpoint;
  158. unsigned int blocked;
  159. unsigned int generation;
  160. wait_queue_head_t event;
  161. };
  162. enum qp_broker_state {
  163. VMCIQPB_NEW,
  164. VMCIQPB_CREATED_NO_MEM,
  165. VMCIQPB_CREATED_MEM,
  166. VMCIQPB_ATTACHED_NO_MEM,
  167. VMCIQPB_ATTACHED_MEM,
  168. VMCIQPB_SHUTDOWN_NO_MEM,
  169. VMCIQPB_SHUTDOWN_MEM,
  170. VMCIQPB_GONE
  171. };
  172. #define QPBROKERSTATE_HAS_MEM(_qpb) (_qpb->state == VMCIQPB_CREATED_MEM || \
  173. _qpb->state == VMCIQPB_ATTACHED_MEM || \
  174. _qpb->state == VMCIQPB_SHUTDOWN_MEM)
  175. /*
  176. * In the queue pair broker, we always use the guest point of view for
  177. * the produce and consume queue values and references, e.g., the
  178. * produce queue size stored is the guests produce queue size. The
  179. * host endpoint will need to swap these around. The only exception is
  180. * the local queue pairs on the host, in which case the host endpoint
  181. * that creates the queue pair will have the right orientation, and
  182. * the attaching host endpoint will need to swap.
  183. */
  184. struct qp_entry {
  185. struct list_head list_item;
  186. struct vmci_handle handle;
  187. u32 peer;
  188. u32 flags;
  189. u64 produce_size;
  190. u64 consume_size;
  191. u32 ref_count;
  192. };
  193. struct qp_broker_entry {
  194. struct vmci_resource resource;
  195. struct qp_entry qp;
  196. u32 create_id;
  197. u32 attach_id;
  198. enum qp_broker_state state;
  199. bool require_trusted_attach;
  200. bool created_by_trusted;
  201. bool vmci_page_files; /* Created by VMX using VMCI page files */
  202. struct vmci_queue *produce_q;
  203. struct vmci_queue *consume_q;
  204. struct vmci_queue_header saved_produce_q;
  205. struct vmci_queue_header saved_consume_q;
  206. vmci_event_release_cb wakeup_cb;
  207. void *client_data;
  208. void *local_mem; /* Kernel memory for local queue pair */
  209. };
  210. struct qp_guest_endpoint {
  211. struct vmci_resource resource;
  212. struct qp_entry qp;
  213. u64 num_ppns;
  214. void *produce_q;
  215. void *consume_q;
  216. struct ppn_set ppn_set;
  217. };
  218. struct qp_list {
  219. struct list_head head;
  220. struct mutex mutex; /* Protect queue list. */
  221. };
  222. static struct qp_list qp_broker_list = {
  223. .head = LIST_HEAD_INIT(qp_broker_list.head),
  224. .mutex = __MUTEX_INITIALIZER(qp_broker_list.mutex),
  225. };
  226. static struct qp_list qp_guest_endpoints = {
  227. .head = LIST_HEAD_INIT(qp_guest_endpoints.head),
  228. .mutex = __MUTEX_INITIALIZER(qp_guest_endpoints.mutex),
  229. };
  230. #define INVALID_VMCI_GUEST_MEM_ID 0
  231. #define QPE_NUM_PAGES(_QPE) ((u32) \
  232. (DIV_ROUND_UP(_QPE.produce_size, PAGE_SIZE) + \
  233. DIV_ROUND_UP(_QPE.consume_size, PAGE_SIZE) + 2))
  234. /*
  235. * Frees kernel VA space for a given queue and its queue header, and
  236. * frees physical data pages.
  237. */
  238. static void qp_free_queue(void *q, u64 size)
  239. {
  240. struct vmci_queue *queue = q;
  241. if (queue) {
  242. u64 i;
  243. /* Given size does not include header, so add in a page here. */
  244. for (i = 0; i < DIV_ROUND_UP(size, PAGE_SIZE) + 1; i++) {
  245. dma_free_coherent(&vmci_pdev->dev, PAGE_SIZE,
  246. queue->kernel_if->u.g.vas[i],
  247. queue->kernel_if->u.g.pas[i]);
  248. }
  249. vfree(queue);
  250. }
  251. }
  252. /*
  253. * Allocates kernel queue pages of specified size with IOMMU mappings,
  254. * plus space for the queue structure/kernel interface and the queue
  255. * header.
  256. */
  257. static void *qp_alloc_queue(u64 size, u32 flags)
  258. {
  259. u64 i;
  260. struct vmci_queue *queue;
  261. size_t pas_size;
  262. size_t vas_size;
  263. size_t queue_size = sizeof(*queue) + sizeof(*queue->kernel_if);
  264. u64 num_pages;
  265. if (size > SIZE_MAX - PAGE_SIZE)
  266. return NULL;
  267. num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
  268. if (num_pages >
  269. (SIZE_MAX - queue_size) /
  270. (sizeof(*queue->kernel_if->u.g.pas) +
  271. sizeof(*queue->kernel_if->u.g.vas)))
  272. return NULL;
  273. pas_size = num_pages * sizeof(*queue->kernel_if->u.g.pas);
  274. vas_size = num_pages * sizeof(*queue->kernel_if->u.g.vas);
  275. queue_size += pas_size + vas_size;
  276. queue = vmalloc(queue_size);
  277. if (!queue)
  278. return NULL;
  279. queue->q_header = NULL;
  280. queue->saved_header = NULL;
  281. queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
  282. queue->kernel_if->mutex = NULL;
  283. queue->kernel_if->num_pages = num_pages;
  284. queue->kernel_if->u.g.pas = (dma_addr_t *)(queue->kernel_if + 1);
  285. queue->kernel_if->u.g.vas =
  286. (void **)((u8 *)queue->kernel_if->u.g.pas + pas_size);
  287. queue->kernel_if->host = false;
  288. for (i = 0; i < num_pages; i++) {
  289. queue->kernel_if->u.g.vas[i] =
  290. dma_alloc_coherent(&vmci_pdev->dev, PAGE_SIZE,
  291. &queue->kernel_if->u.g.pas[i],
  292. GFP_KERNEL);
  293. if (!queue->kernel_if->u.g.vas[i]) {
  294. /* Size excl. the header. */
  295. qp_free_queue(queue, i * PAGE_SIZE);
  296. return NULL;
  297. }
  298. }
  299. /* Queue header is the first page. */
  300. queue->q_header = queue->kernel_if->u.g.vas[0];
  301. return queue;
  302. }
  303. /*
  304. * Copies from a given buffer or iovector to a VMCI Queue. Uses
  305. * kmap()/kunmap() to dynamically map/unmap required portions of the queue
  306. * by traversing the offset -> page translation structure for the queue.
  307. * Assumes that offset + size does not wrap around in the queue.
  308. */
  309. static int qp_memcpy_to_queue_iter(struct vmci_queue *queue,
  310. u64 queue_offset,
  311. struct iov_iter *from,
  312. size_t size)
  313. {
  314. struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
  315. size_t bytes_copied = 0;
  316. while (bytes_copied < size) {
  317. const u64 page_index =
  318. (queue_offset + bytes_copied) / PAGE_SIZE;
  319. const size_t page_offset =
  320. (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
  321. void *va;
  322. size_t to_copy;
  323. if (kernel_if->host)
  324. va = kmap(kernel_if->u.h.page[page_index]);
  325. else
  326. va = kernel_if->u.g.vas[page_index + 1];
  327. /* Skip header. */
  328. if (size - bytes_copied > PAGE_SIZE - page_offset)
  329. /* Enough payload to fill up from this page. */
  330. to_copy = PAGE_SIZE - page_offset;
  331. else
  332. to_copy = size - bytes_copied;
  333. if (!copy_from_iter_full((u8 *)va + page_offset, to_copy,
  334. from)) {
  335. if (kernel_if->host)
  336. kunmap(kernel_if->u.h.page[page_index]);
  337. return VMCI_ERROR_INVALID_ARGS;
  338. }
  339. bytes_copied += to_copy;
  340. if (kernel_if->host)
  341. kunmap(kernel_if->u.h.page[page_index]);
  342. }
  343. return VMCI_SUCCESS;
  344. }
  345. /*
  346. * Copies to a given buffer or iovector from a VMCI Queue. Uses
  347. * kmap()/kunmap() to dynamically map/unmap required portions of the queue
  348. * by traversing the offset -> page translation structure for the queue.
  349. * Assumes that offset + size does not wrap around in the queue.
  350. */
  351. static int qp_memcpy_from_queue_iter(struct iov_iter *to,
  352. const struct vmci_queue *queue,
  353. u64 queue_offset, size_t size)
  354. {
  355. struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
  356. size_t bytes_copied = 0;
  357. while (bytes_copied < size) {
  358. const u64 page_index =
  359. (queue_offset + bytes_copied) / PAGE_SIZE;
  360. const size_t page_offset =
  361. (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
  362. void *va;
  363. size_t to_copy;
  364. int err;
  365. if (kernel_if->host)
  366. va = kmap(kernel_if->u.h.page[page_index]);
  367. else
  368. va = kernel_if->u.g.vas[page_index + 1];
  369. /* Skip header. */
  370. if (size - bytes_copied > PAGE_SIZE - page_offset)
  371. /* Enough payload to fill up this page. */
  372. to_copy = PAGE_SIZE - page_offset;
  373. else
  374. to_copy = size - bytes_copied;
  375. err = copy_to_iter((u8 *)va + page_offset, to_copy, to);
  376. if (err != to_copy) {
  377. if (kernel_if->host)
  378. kunmap(kernel_if->u.h.page[page_index]);
  379. return VMCI_ERROR_INVALID_ARGS;
  380. }
  381. bytes_copied += to_copy;
  382. if (kernel_if->host)
  383. kunmap(kernel_if->u.h.page[page_index]);
  384. }
  385. return VMCI_SUCCESS;
  386. }
  387. /*
  388. * Allocates two list of PPNs --- one for the pages in the produce queue,
  389. * and the other for the pages in the consume queue. Intializes the list
  390. * of PPNs with the page frame numbers of the KVA for the two queues (and
  391. * the queue headers).
  392. */
  393. static int qp_alloc_ppn_set(void *prod_q,
  394. u64 num_produce_pages,
  395. void *cons_q,
  396. u64 num_consume_pages, struct ppn_set *ppn_set)
  397. {
  398. u32 *produce_ppns;
  399. u32 *consume_ppns;
  400. struct vmci_queue *produce_q = prod_q;
  401. struct vmci_queue *consume_q = cons_q;
  402. u64 i;
  403. if (!produce_q || !num_produce_pages || !consume_q ||
  404. !num_consume_pages || !ppn_set)
  405. return VMCI_ERROR_INVALID_ARGS;
  406. if (ppn_set->initialized)
  407. return VMCI_ERROR_ALREADY_EXISTS;
  408. produce_ppns =
  409. kmalloc(num_produce_pages * sizeof(*produce_ppns), GFP_KERNEL);
  410. if (!produce_ppns)
  411. return VMCI_ERROR_NO_MEM;
  412. consume_ppns =
  413. kmalloc(num_consume_pages * sizeof(*consume_ppns), GFP_KERNEL);
  414. if (!consume_ppns) {
  415. kfree(produce_ppns);
  416. return VMCI_ERROR_NO_MEM;
  417. }
  418. for (i = 0; i < num_produce_pages; i++) {
  419. unsigned long pfn;
  420. produce_ppns[i] =
  421. produce_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
  422. pfn = produce_ppns[i];
  423. /* Fail allocation if PFN isn't supported by hypervisor. */
  424. if (sizeof(pfn) > sizeof(*produce_ppns)
  425. && pfn != produce_ppns[i])
  426. goto ppn_error;
  427. }
  428. for (i = 0; i < num_consume_pages; i++) {
  429. unsigned long pfn;
  430. consume_ppns[i] =
  431. consume_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
  432. pfn = consume_ppns[i];
  433. /* Fail allocation if PFN isn't supported by hypervisor. */
  434. if (sizeof(pfn) > sizeof(*consume_ppns)
  435. && pfn != consume_ppns[i])
  436. goto ppn_error;
  437. }
  438. ppn_set->num_produce_pages = num_produce_pages;
  439. ppn_set->num_consume_pages = num_consume_pages;
  440. ppn_set->produce_ppns = produce_ppns;
  441. ppn_set->consume_ppns = consume_ppns;
  442. ppn_set->initialized = true;
  443. return VMCI_SUCCESS;
  444. ppn_error:
  445. kfree(produce_ppns);
  446. kfree(consume_ppns);
  447. return VMCI_ERROR_INVALID_ARGS;
  448. }
  449. /*
  450. * Frees the two list of PPNs for a queue pair.
  451. */
  452. static void qp_free_ppn_set(struct ppn_set *ppn_set)
  453. {
  454. if (ppn_set->initialized) {
  455. /* Do not call these functions on NULL inputs. */
  456. kfree(ppn_set->produce_ppns);
  457. kfree(ppn_set->consume_ppns);
  458. }
  459. memset(ppn_set, 0, sizeof(*ppn_set));
  460. }
  461. /*
  462. * Populates the list of PPNs in the hypercall structure with the PPNS
  463. * of the produce queue and the consume queue.
  464. */
  465. static int qp_populate_ppn_set(u8 *call_buf, const struct ppn_set *ppn_set)
  466. {
  467. memcpy(call_buf, ppn_set->produce_ppns,
  468. ppn_set->num_produce_pages * sizeof(*ppn_set->produce_ppns));
  469. memcpy(call_buf +
  470. ppn_set->num_produce_pages * sizeof(*ppn_set->produce_ppns),
  471. ppn_set->consume_ppns,
  472. ppn_set->num_consume_pages * sizeof(*ppn_set->consume_ppns));
  473. return VMCI_SUCCESS;
  474. }
  475. /*
  476. * Allocates kernel VA space of specified size plus space for the queue
  477. * and kernel interface. This is different from the guest queue allocator,
  478. * because we do not allocate our own queue header/data pages here but
  479. * share those of the guest.
  480. */
  481. static struct vmci_queue *qp_host_alloc_queue(u64 size)
  482. {
  483. struct vmci_queue *queue;
  484. size_t queue_page_size;
  485. u64 num_pages;
  486. const size_t queue_size = sizeof(*queue) + sizeof(*(queue->kernel_if));
  487. if (size > SIZE_MAX - PAGE_SIZE)
  488. return NULL;
  489. num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
  490. if (num_pages > (SIZE_MAX - queue_size) /
  491. sizeof(*queue->kernel_if->u.h.page))
  492. return NULL;
  493. queue_page_size = num_pages * sizeof(*queue->kernel_if->u.h.page);
  494. queue = kzalloc(queue_size + queue_page_size, GFP_KERNEL);
  495. if (queue) {
  496. queue->q_header = NULL;
  497. queue->saved_header = NULL;
  498. queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
  499. queue->kernel_if->host = true;
  500. queue->kernel_if->mutex = NULL;
  501. queue->kernel_if->num_pages = num_pages;
  502. queue->kernel_if->u.h.header_page =
  503. (struct page **)((u8 *)queue + queue_size);
  504. queue->kernel_if->u.h.page =
  505. &queue->kernel_if->u.h.header_page[1];
  506. }
  507. return queue;
  508. }
  509. /*
  510. * Frees kernel memory for a given queue (header plus translation
  511. * structure).
  512. */
  513. static void qp_host_free_queue(struct vmci_queue *queue, u64 queue_size)
  514. {
  515. kfree(queue);
  516. }
  517. /*
  518. * Initialize the mutex for the pair of queues. This mutex is used to
  519. * protect the q_header and the buffer from changing out from under any
  520. * users of either queue. Of course, it's only any good if the mutexes
  521. * are actually acquired. Queue structure must lie on non-paged memory
  522. * or we cannot guarantee access to the mutex.
  523. */
  524. static void qp_init_queue_mutex(struct vmci_queue *produce_q,
  525. struct vmci_queue *consume_q)
  526. {
  527. /*
  528. * Only the host queue has shared state - the guest queues do not
  529. * need to synchronize access using a queue mutex.
  530. */
  531. if (produce_q->kernel_if->host) {
  532. produce_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
  533. consume_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
  534. mutex_init(produce_q->kernel_if->mutex);
  535. }
  536. }
  537. /*
  538. * Cleans up the mutex for the pair of queues.
  539. */
  540. static void qp_cleanup_queue_mutex(struct vmci_queue *produce_q,
  541. struct vmci_queue *consume_q)
  542. {
  543. if (produce_q->kernel_if->host) {
  544. produce_q->kernel_if->mutex = NULL;
  545. consume_q->kernel_if->mutex = NULL;
  546. }
  547. }
  548. /*
  549. * Acquire the mutex for the queue. Note that the produce_q and
  550. * the consume_q share a mutex. So, only one of the two need to
  551. * be passed in to this routine. Either will work just fine.
  552. */
  553. static void qp_acquire_queue_mutex(struct vmci_queue *queue)
  554. {
  555. if (queue->kernel_if->host)
  556. mutex_lock(queue->kernel_if->mutex);
  557. }
  558. /*
  559. * Release the mutex for the queue. Note that the produce_q and
  560. * the consume_q share a mutex. So, only one of the two need to
  561. * be passed in to this routine. Either will work just fine.
  562. */
  563. static void qp_release_queue_mutex(struct vmci_queue *queue)
  564. {
  565. if (queue->kernel_if->host)
  566. mutex_unlock(queue->kernel_if->mutex);
  567. }
  568. /*
  569. * Helper function to release pages in the PageStoreAttachInfo
  570. * previously obtained using get_user_pages.
  571. */
  572. static void qp_release_pages(struct page **pages,
  573. u64 num_pages, bool dirty)
  574. {
  575. int i;
  576. for (i = 0; i < num_pages; i++) {
  577. if (dirty)
  578. set_page_dirty(pages[i]);
  579. put_page(pages[i]);
  580. pages[i] = NULL;
  581. }
  582. }
  583. /*
  584. * Lock the user pages referenced by the {produce,consume}Buffer
  585. * struct into memory and populate the {produce,consume}Pages
  586. * arrays in the attach structure with them.
  587. */
  588. static int qp_host_get_user_memory(u64 produce_uva,
  589. u64 consume_uva,
  590. struct vmci_queue *produce_q,
  591. struct vmci_queue *consume_q)
  592. {
  593. int retval;
  594. int err = VMCI_SUCCESS;
  595. retval = get_user_pages_fast((uintptr_t) produce_uva,
  596. produce_q->kernel_if->num_pages, 1,
  597. produce_q->kernel_if->u.h.header_page);
  598. if (retval < produce_q->kernel_if->num_pages) {
  599. pr_debug("get_user_pages_fast(produce) failed (retval=%d)",
  600. retval);
  601. qp_release_pages(produce_q->kernel_if->u.h.header_page,
  602. retval, false);
  603. err = VMCI_ERROR_NO_MEM;
  604. goto out;
  605. }
  606. retval = get_user_pages_fast((uintptr_t) consume_uva,
  607. consume_q->kernel_if->num_pages, 1,
  608. consume_q->kernel_if->u.h.header_page);
  609. if (retval < consume_q->kernel_if->num_pages) {
  610. pr_debug("get_user_pages_fast(consume) failed (retval=%d)",
  611. retval);
  612. qp_release_pages(consume_q->kernel_if->u.h.header_page,
  613. retval, false);
  614. qp_release_pages(produce_q->kernel_if->u.h.header_page,
  615. produce_q->kernel_if->num_pages, false);
  616. err = VMCI_ERROR_NO_MEM;
  617. }
  618. out:
  619. return err;
  620. }
  621. /*
  622. * Registers the specification of the user pages used for backing a queue
  623. * pair. Enough information to map in pages is stored in the OS specific
  624. * part of the struct vmci_queue structure.
  625. */
  626. static int qp_host_register_user_memory(struct vmci_qp_page_store *page_store,
  627. struct vmci_queue *produce_q,
  628. struct vmci_queue *consume_q)
  629. {
  630. u64 produce_uva;
  631. u64 consume_uva;
  632. /*
  633. * The new style and the old style mapping only differs in
  634. * that we either get a single or two UVAs, so we split the
  635. * single UVA range at the appropriate spot.
  636. */
  637. produce_uva = page_store->pages;
  638. consume_uva = page_store->pages +
  639. produce_q->kernel_if->num_pages * PAGE_SIZE;
  640. return qp_host_get_user_memory(produce_uva, consume_uva, produce_q,
  641. consume_q);
  642. }
  643. /*
  644. * Releases and removes the references to user pages stored in the attach
  645. * struct. Pages are released from the page cache and may become
  646. * swappable again.
  647. */
  648. static void qp_host_unregister_user_memory(struct vmci_queue *produce_q,
  649. struct vmci_queue *consume_q)
  650. {
  651. qp_release_pages(produce_q->kernel_if->u.h.header_page,
  652. produce_q->kernel_if->num_pages, true);
  653. memset(produce_q->kernel_if->u.h.header_page, 0,
  654. sizeof(*produce_q->kernel_if->u.h.header_page) *
  655. produce_q->kernel_if->num_pages);
  656. qp_release_pages(consume_q->kernel_if->u.h.header_page,
  657. consume_q->kernel_if->num_pages, true);
  658. memset(consume_q->kernel_if->u.h.header_page, 0,
  659. sizeof(*consume_q->kernel_if->u.h.header_page) *
  660. consume_q->kernel_if->num_pages);
  661. }
  662. /*
  663. * Once qp_host_register_user_memory has been performed on a
  664. * queue, the queue pair headers can be mapped into the
  665. * kernel. Once mapped, they must be unmapped with
  666. * qp_host_unmap_queues prior to calling
  667. * qp_host_unregister_user_memory.
  668. * Pages are pinned.
  669. */
  670. static int qp_host_map_queues(struct vmci_queue *produce_q,
  671. struct vmci_queue *consume_q)
  672. {
  673. int result;
  674. if (!produce_q->q_header || !consume_q->q_header) {
  675. struct page *headers[2];
  676. if (produce_q->q_header != consume_q->q_header)
  677. return VMCI_ERROR_QUEUEPAIR_MISMATCH;
  678. if (produce_q->kernel_if->u.h.header_page == NULL ||
  679. *produce_q->kernel_if->u.h.header_page == NULL)
  680. return VMCI_ERROR_UNAVAILABLE;
  681. headers[0] = *produce_q->kernel_if->u.h.header_page;
  682. headers[1] = *consume_q->kernel_if->u.h.header_page;
  683. produce_q->q_header = vmap(headers, 2, VM_MAP, PAGE_KERNEL);
  684. if (produce_q->q_header != NULL) {
  685. consume_q->q_header =
  686. (struct vmci_queue_header *)((u8 *)
  687. produce_q->q_header +
  688. PAGE_SIZE);
  689. result = VMCI_SUCCESS;
  690. } else {
  691. pr_warn("vmap failed\n");
  692. result = VMCI_ERROR_NO_MEM;
  693. }
  694. } else {
  695. result = VMCI_SUCCESS;
  696. }
  697. return result;
  698. }
  699. /*
  700. * Unmaps previously mapped queue pair headers from the kernel.
  701. * Pages are unpinned.
  702. */
  703. static int qp_host_unmap_queues(u32 gid,
  704. struct vmci_queue *produce_q,
  705. struct vmci_queue *consume_q)
  706. {
  707. if (produce_q->q_header) {
  708. if (produce_q->q_header < consume_q->q_header)
  709. vunmap(produce_q->q_header);
  710. else
  711. vunmap(consume_q->q_header);
  712. produce_q->q_header = NULL;
  713. consume_q->q_header = NULL;
  714. }
  715. return VMCI_SUCCESS;
  716. }
  717. /*
  718. * Finds the entry in the list corresponding to a given handle. Assumes
  719. * that the list is locked.
  720. */
  721. static struct qp_entry *qp_list_find(struct qp_list *qp_list,
  722. struct vmci_handle handle)
  723. {
  724. struct qp_entry *entry;
  725. if (vmci_handle_is_invalid(handle))
  726. return NULL;
  727. list_for_each_entry(entry, &qp_list->head, list_item) {
  728. if (vmci_handle_is_equal(entry->handle, handle))
  729. return entry;
  730. }
  731. return NULL;
  732. }
  733. /*
  734. * Finds the entry in the list corresponding to a given handle.
  735. */
  736. static struct qp_guest_endpoint *
  737. qp_guest_handle_to_entry(struct vmci_handle handle)
  738. {
  739. struct qp_guest_endpoint *entry;
  740. struct qp_entry *qp = qp_list_find(&qp_guest_endpoints, handle);
  741. entry = qp ? container_of(
  742. qp, struct qp_guest_endpoint, qp) : NULL;
  743. return entry;
  744. }
  745. /*
  746. * Finds the entry in the list corresponding to a given handle.
  747. */
  748. static struct qp_broker_entry *
  749. qp_broker_handle_to_entry(struct vmci_handle handle)
  750. {
  751. struct qp_broker_entry *entry;
  752. struct qp_entry *qp = qp_list_find(&qp_broker_list, handle);
  753. entry = qp ? container_of(
  754. qp, struct qp_broker_entry, qp) : NULL;
  755. return entry;
  756. }
  757. /*
  758. * Dispatches a queue pair event message directly into the local event
  759. * queue.
  760. */
  761. static int qp_notify_peer_local(bool attach, struct vmci_handle handle)
  762. {
  763. u32 context_id = vmci_get_context_id();
  764. struct vmci_event_qp ev;
  765. ev.msg.hdr.dst = vmci_make_handle(context_id, VMCI_EVENT_HANDLER);
  766. ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
  767. VMCI_CONTEXT_RESOURCE_ID);
  768. ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
  769. ev.msg.event_data.event =
  770. attach ? VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
  771. ev.payload.peer_id = context_id;
  772. ev.payload.handle = handle;
  773. return vmci_event_dispatch(&ev.msg.hdr);
  774. }
  775. /*
  776. * Allocates and initializes a qp_guest_endpoint structure.
  777. * Allocates a queue_pair rid (and handle) iff the given entry has
  778. * an invalid handle. 0 through VMCI_RESERVED_RESOURCE_ID_MAX
  779. * are reserved handles. Assumes that the QP list mutex is held
  780. * by the caller.
  781. */
  782. static struct qp_guest_endpoint *
  783. qp_guest_endpoint_create(struct vmci_handle handle,
  784. u32 peer,
  785. u32 flags,
  786. u64 produce_size,
  787. u64 consume_size,
  788. void *produce_q,
  789. void *consume_q)
  790. {
  791. int result;
  792. struct qp_guest_endpoint *entry;
  793. /* One page each for the queue headers. */
  794. const u64 num_ppns = DIV_ROUND_UP(produce_size, PAGE_SIZE) +
  795. DIV_ROUND_UP(consume_size, PAGE_SIZE) + 2;
  796. if (vmci_handle_is_invalid(handle)) {
  797. u32 context_id = vmci_get_context_id();
  798. handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
  799. }
  800. entry = kzalloc(sizeof(*entry), GFP_KERNEL);
  801. if (entry) {
  802. entry->qp.peer = peer;
  803. entry->qp.flags = flags;
  804. entry->qp.produce_size = produce_size;
  805. entry->qp.consume_size = consume_size;
  806. entry->qp.ref_count = 0;
  807. entry->num_ppns = num_ppns;
  808. entry->produce_q = produce_q;
  809. entry->consume_q = consume_q;
  810. INIT_LIST_HEAD(&entry->qp.list_item);
  811. /* Add resource obj */
  812. result = vmci_resource_add(&entry->resource,
  813. VMCI_RESOURCE_TYPE_QPAIR_GUEST,
  814. handle);
  815. entry->qp.handle = vmci_resource_handle(&entry->resource);
  816. if ((result != VMCI_SUCCESS) ||
  817. qp_list_find(&qp_guest_endpoints, entry->qp.handle)) {
  818. pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
  819. handle.context, handle.resource, result);
  820. kfree(entry);
  821. entry = NULL;
  822. }
  823. }
  824. return entry;
  825. }
  826. /*
  827. * Frees a qp_guest_endpoint structure.
  828. */
  829. static void qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry)
  830. {
  831. qp_free_ppn_set(&entry->ppn_set);
  832. qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
  833. qp_free_queue(entry->produce_q, entry->qp.produce_size);
  834. qp_free_queue(entry->consume_q, entry->qp.consume_size);
  835. /* Unlink from resource hash table and free callback */
  836. vmci_resource_remove(&entry->resource);
  837. kfree(entry);
  838. }
  839. /*
  840. * Helper to make a queue_pairAlloc hypercall when the driver is
  841. * supporting a guest device.
  842. */
  843. static int qp_alloc_hypercall(const struct qp_guest_endpoint *entry)
  844. {
  845. struct vmci_qp_alloc_msg *alloc_msg;
  846. size_t msg_size;
  847. int result;
  848. if (!entry || entry->num_ppns <= 2)
  849. return VMCI_ERROR_INVALID_ARGS;
  850. msg_size = sizeof(*alloc_msg) +
  851. (size_t) entry->num_ppns * sizeof(u32);
  852. alloc_msg = kmalloc(msg_size, GFP_KERNEL);
  853. if (!alloc_msg)
  854. return VMCI_ERROR_NO_MEM;
  855. alloc_msg->hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
  856. VMCI_QUEUEPAIR_ALLOC);
  857. alloc_msg->hdr.src = VMCI_ANON_SRC_HANDLE;
  858. alloc_msg->hdr.payload_size = msg_size - VMCI_DG_HEADERSIZE;
  859. alloc_msg->handle = entry->qp.handle;
  860. alloc_msg->peer = entry->qp.peer;
  861. alloc_msg->flags = entry->qp.flags;
  862. alloc_msg->produce_size = entry->qp.produce_size;
  863. alloc_msg->consume_size = entry->qp.consume_size;
  864. alloc_msg->num_ppns = entry->num_ppns;
  865. result = qp_populate_ppn_set((u8 *)alloc_msg + sizeof(*alloc_msg),
  866. &entry->ppn_set);
  867. if (result == VMCI_SUCCESS)
  868. result = vmci_send_datagram(&alloc_msg->hdr);
  869. kfree(alloc_msg);
  870. return result;
  871. }
  872. /*
  873. * Helper to make a queue_pairDetach hypercall when the driver is
  874. * supporting a guest device.
  875. */
  876. static int qp_detatch_hypercall(struct vmci_handle handle)
  877. {
  878. struct vmci_qp_detach_msg detach_msg;
  879. detach_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
  880. VMCI_QUEUEPAIR_DETACH);
  881. detach_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
  882. detach_msg.hdr.payload_size = sizeof(handle);
  883. detach_msg.handle = handle;
  884. return vmci_send_datagram(&detach_msg.hdr);
  885. }
  886. /*
  887. * Adds the given entry to the list. Assumes that the list is locked.
  888. */
  889. static void qp_list_add_entry(struct qp_list *qp_list, struct qp_entry *entry)
  890. {
  891. if (entry)
  892. list_add(&entry->list_item, &qp_list->head);
  893. }
  894. /*
  895. * Removes the given entry from the list. Assumes that the list is locked.
  896. */
  897. static void qp_list_remove_entry(struct qp_list *qp_list,
  898. struct qp_entry *entry)
  899. {
  900. if (entry)
  901. list_del(&entry->list_item);
  902. }
  903. /*
  904. * Helper for VMCI queue_pair detach interface. Frees the physical
  905. * pages for the queue pair.
  906. */
  907. static int qp_detatch_guest_work(struct vmci_handle handle)
  908. {
  909. int result;
  910. struct qp_guest_endpoint *entry;
  911. u32 ref_count = ~0; /* To avoid compiler warning below */
  912. mutex_lock(&qp_guest_endpoints.mutex);
  913. entry = qp_guest_handle_to_entry(handle);
  914. if (!entry) {
  915. mutex_unlock(&qp_guest_endpoints.mutex);
  916. return VMCI_ERROR_NOT_FOUND;
  917. }
  918. if (entry->qp.flags & VMCI_QPFLAG_LOCAL) {
  919. result = VMCI_SUCCESS;
  920. if (entry->qp.ref_count > 1) {
  921. result = qp_notify_peer_local(false, handle);
  922. /*
  923. * We can fail to notify a local queuepair
  924. * because we can't allocate. We still want
  925. * to release the entry if that happens, so
  926. * don't bail out yet.
  927. */
  928. }
  929. } else {
  930. result = qp_detatch_hypercall(handle);
  931. if (result < VMCI_SUCCESS) {
  932. /*
  933. * We failed to notify a non-local queuepair.
  934. * That other queuepair might still be
  935. * accessing the shared memory, so don't
  936. * release the entry yet. It will get cleaned
  937. * up by VMCIqueue_pair_Exit() if necessary
  938. * (assuming we are going away, otherwise why
  939. * did this fail?).
  940. */
  941. mutex_unlock(&qp_guest_endpoints.mutex);
  942. return result;
  943. }
  944. }
  945. /*
  946. * If we get here then we either failed to notify a local queuepair, or
  947. * we succeeded in all cases. Release the entry if required.
  948. */
  949. entry->qp.ref_count--;
  950. if (entry->qp.ref_count == 0)
  951. qp_list_remove_entry(&qp_guest_endpoints, &entry->qp);
  952. /* If we didn't remove the entry, this could change once we unlock. */
  953. if (entry)
  954. ref_count = entry->qp.ref_count;
  955. mutex_unlock(&qp_guest_endpoints.mutex);
  956. if (ref_count == 0)
  957. qp_guest_endpoint_destroy(entry);
  958. return result;
  959. }
  960. /*
  961. * This functions handles the actual allocation of a VMCI queue
  962. * pair guest endpoint. Allocates physical pages for the queue
  963. * pair. It makes OS dependent calls through generic wrappers.
  964. */
  965. static int qp_alloc_guest_work(struct vmci_handle *handle,
  966. struct vmci_queue **produce_q,
  967. u64 produce_size,
  968. struct vmci_queue **consume_q,
  969. u64 consume_size,
  970. u32 peer,
  971. u32 flags,
  972. u32 priv_flags)
  973. {
  974. const u64 num_produce_pages =
  975. DIV_ROUND_UP(produce_size, PAGE_SIZE) + 1;
  976. const u64 num_consume_pages =
  977. DIV_ROUND_UP(consume_size, PAGE_SIZE) + 1;
  978. void *my_produce_q = NULL;
  979. void *my_consume_q = NULL;
  980. int result;
  981. struct qp_guest_endpoint *queue_pair_entry = NULL;
  982. if (priv_flags != VMCI_NO_PRIVILEGE_FLAGS)
  983. return VMCI_ERROR_NO_ACCESS;
  984. mutex_lock(&qp_guest_endpoints.mutex);
  985. queue_pair_entry = qp_guest_handle_to_entry(*handle);
  986. if (queue_pair_entry) {
  987. if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
  988. /* Local attach case. */
  989. if (queue_pair_entry->qp.ref_count > 1) {
  990. pr_devel("Error attempting to attach more than once\n");
  991. result = VMCI_ERROR_UNAVAILABLE;
  992. goto error_keep_entry;
  993. }
  994. if (queue_pair_entry->qp.produce_size != consume_size ||
  995. queue_pair_entry->qp.consume_size !=
  996. produce_size ||
  997. queue_pair_entry->qp.flags !=
  998. (flags & ~VMCI_QPFLAG_ATTACH_ONLY)) {
  999. pr_devel("Error mismatched queue pair in local attach\n");
  1000. result = VMCI_ERROR_QUEUEPAIR_MISMATCH;
  1001. goto error_keep_entry;
  1002. }
  1003. /*
  1004. * Do a local attach. We swap the consume and
  1005. * produce queues for the attacher and deliver
  1006. * an attach event.
  1007. */
  1008. result = qp_notify_peer_local(true, *handle);
  1009. if (result < VMCI_SUCCESS)
  1010. goto error_keep_entry;
  1011. my_produce_q = queue_pair_entry->consume_q;
  1012. my_consume_q = queue_pair_entry->produce_q;
  1013. goto out;
  1014. }
  1015. result = VMCI_ERROR_ALREADY_EXISTS;
  1016. goto error_keep_entry;
  1017. }
  1018. my_produce_q = qp_alloc_queue(produce_size, flags);
  1019. if (!my_produce_q) {
  1020. pr_warn("Error allocating pages for produce queue\n");
  1021. result = VMCI_ERROR_NO_MEM;
  1022. goto error;
  1023. }
  1024. my_consume_q = qp_alloc_queue(consume_size, flags);
  1025. if (!my_consume_q) {
  1026. pr_warn("Error allocating pages for consume queue\n");
  1027. result = VMCI_ERROR_NO_MEM;
  1028. goto error;
  1029. }
  1030. queue_pair_entry = qp_guest_endpoint_create(*handle, peer, flags,
  1031. produce_size, consume_size,
  1032. my_produce_q, my_consume_q);
  1033. if (!queue_pair_entry) {
  1034. pr_warn("Error allocating memory in %s\n", __func__);
  1035. result = VMCI_ERROR_NO_MEM;
  1036. goto error;
  1037. }
  1038. result = qp_alloc_ppn_set(my_produce_q, num_produce_pages, my_consume_q,
  1039. num_consume_pages,
  1040. &queue_pair_entry->ppn_set);
  1041. if (result < VMCI_SUCCESS) {
  1042. pr_warn("qp_alloc_ppn_set failed\n");
  1043. goto error;
  1044. }
  1045. /*
  1046. * It's only necessary to notify the host if this queue pair will be
  1047. * attached to from another context.
  1048. */
  1049. if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
  1050. /* Local create case. */
  1051. u32 context_id = vmci_get_context_id();
  1052. /*
  1053. * Enforce similar checks on local queue pairs as we
  1054. * do for regular ones. The handle's context must
  1055. * match the creator or attacher context id (here they
  1056. * are both the current context id) and the
  1057. * attach-only flag cannot exist during create. We
  1058. * also ensure specified peer is this context or an
  1059. * invalid one.
  1060. */
  1061. if (queue_pair_entry->qp.handle.context != context_id ||
  1062. (queue_pair_entry->qp.peer != VMCI_INVALID_ID &&
  1063. queue_pair_entry->qp.peer != context_id)) {
  1064. result = VMCI_ERROR_NO_ACCESS;
  1065. goto error;
  1066. }
  1067. if (queue_pair_entry->qp.flags & VMCI_QPFLAG_ATTACH_ONLY) {
  1068. result = VMCI_ERROR_NOT_FOUND;
  1069. goto error;
  1070. }
  1071. } else {
  1072. result = qp_alloc_hypercall(queue_pair_entry);
  1073. if (result < VMCI_SUCCESS) {
  1074. pr_warn("qp_alloc_hypercall result = %d\n", result);
  1075. goto error;
  1076. }
  1077. }
  1078. qp_init_queue_mutex((struct vmci_queue *)my_produce_q,
  1079. (struct vmci_queue *)my_consume_q);
  1080. qp_list_add_entry(&qp_guest_endpoints, &queue_pair_entry->qp);
  1081. out:
  1082. queue_pair_entry->qp.ref_count++;
  1083. *handle = queue_pair_entry->qp.handle;
  1084. *produce_q = (struct vmci_queue *)my_produce_q;
  1085. *consume_q = (struct vmci_queue *)my_consume_q;
  1086. /*
  1087. * We should initialize the queue pair header pages on a local
  1088. * queue pair create. For non-local queue pairs, the
  1089. * hypervisor initializes the header pages in the create step.
  1090. */
  1091. if ((queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) &&
  1092. queue_pair_entry->qp.ref_count == 1) {
  1093. vmci_q_header_init((*produce_q)->q_header, *handle);
  1094. vmci_q_header_init((*consume_q)->q_header, *handle);
  1095. }
  1096. mutex_unlock(&qp_guest_endpoints.mutex);
  1097. return VMCI_SUCCESS;
  1098. error:
  1099. mutex_unlock(&qp_guest_endpoints.mutex);
  1100. if (queue_pair_entry) {
  1101. /* The queues will be freed inside the destroy routine. */
  1102. qp_guest_endpoint_destroy(queue_pair_entry);
  1103. } else {
  1104. qp_free_queue(my_produce_q, produce_size);
  1105. qp_free_queue(my_consume_q, consume_size);
  1106. }
  1107. return result;
  1108. error_keep_entry:
  1109. /* This path should only be used when an existing entry was found. */
  1110. mutex_unlock(&qp_guest_endpoints.mutex);
  1111. return result;
  1112. }
  1113. /*
  1114. * The first endpoint issuing a queue pair allocation will create the state
  1115. * of the queue pair in the queue pair broker.
  1116. *
  1117. * If the creator is a guest, it will associate a VMX virtual address range
  1118. * with the queue pair as specified by the page_store. For compatibility with
  1119. * older VMX'en, that would use a separate step to set the VMX virtual
  1120. * address range, the virtual address range can be registered later using
  1121. * vmci_qp_broker_set_page_store. In that case, a page_store of NULL should be
  1122. * used.
  1123. *
  1124. * If the creator is the host, a page_store of NULL should be used as well,
  1125. * since the host is not able to supply a page store for the queue pair.
  1126. *
  1127. * For older VMX and host callers, the queue pair will be created in the
  1128. * VMCIQPB_CREATED_NO_MEM state, and for current VMX callers, it will be
  1129. * created in VMCOQPB_CREATED_MEM state.
  1130. */
  1131. static int qp_broker_create(struct vmci_handle handle,
  1132. u32 peer,
  1133. u32 flags,
  1134. u32 priv_flags,
  1135. u64 produce_size,
  1136. u64 consume_size,
  1137. struct vmci_qp_page_store *page_store,
  1138. struct vmci_ctx *context,
  1139. vmci_event_release_cb wakeup_cb,
  1140. void *client_data, struct qp_broker_entry **ent)
  1141. {
  1142. struct qp_broker_entry *entry = NULL;
  1143. const u32 context_id = vmci_ctx_get_id(context);
  1144. bool is_local = flags & VMCI_QPFLAG_LOCAL;
  1145. int result;
  1146. u64 guest_produce_size;
  1147. u64 guest_consume_size;
  1148. /* Do not create if the caller asked not to. */
  1149. if (flags & VMCI_QPFLAG_ATTACH_ONLY)
  1150. return VMCI_ERROR_NOT_FOUND;
  1151. /*
  1152. * Creator's context ID should match handle's context ID or the creator
  1153. * must allow the context in handle's context ID as the "peer".
  1154. */
  1155. if (handle.context != context_id && handle.context != peer)
  1156. return VMCI_ERROR_NO_ACCESS;
  1157. if (VMCI_CONTEXT_IS_VM(context_id) && VMCI_CONTEXT_IS_VM(peer))
  1158. return VMCI_ERROR_DST_UNREACHABLE;
  1159. /*
  1160. * Creator's context ID for local queue pairs should match the
  1161. * peer, if a peer is specified.
  1162. */
  1163. if (is_local && peer != VMCI_INVALID_ID && context_id != peer)
  1164. return VMCI_ERROR_NO_ACCESS;
  1165. entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
  1166. if (!entry)
  1167. return VMCI_ERROR_NO_MEM;
  1168. if (vmci_ctx_get_id(context) == VMCI_HOST_CONTEXT_ID && !is_local) {
  1169. /*
  1170. * The queue pair broker entry stores values from the guest
  1171. * point of view, so a creating host side endpoint should swap
  1172. * produce and consume values -- unless it is a local queue
  1173. * pair, in which case no swapping is necessary, since the local
  1174. * attacher will swap queues.
  1175. */
  1176. guest_produce_size = consume_size;
  1177. guest_consume_size = produce_size;
  1178. } else {
  1179. guest_produce_size = produce_size;
  1180. guest_consume_size = consume_size;
  1181. }
  1182. entry->qp.handle = handle;
  1183. entry->qp.peer = peer;
  1184. entry->qp.flags = flags;
  1185. entry->qp.produce_size = guest_produce_size;
  1186. entry->qp.consume_size = guest_consume_size;
  1187. entry->qp.ref_count = 1;
  1188. entry->create_id = context_id;
  1189. entry->attach_id = VMCI_INVALID_ID;
  1190. entry->state = VMCIQPB_NEW;
  1191. entry->require_trusted_attach =
  1192. !!(context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED);
  1193. entry->created_by_trusted =
  1194. !!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED);
  1195. entry->vmci_page_files = false;
  1196. entry->wakeup_cb = wakeup_cb;
  1197. entry->client_data = client_data;
  1198. entry->produce_q = qp_host_alloc_queue(guest_produce_size);
  1199. if (entry->produce_q == NULL) {
  1200. result = VMCI_ERROR_NO_MEM;
  1201. goto error;
  1202. }
  1203. entry->consume_q = qp_host_alloc_queue(guest_consume_size);
  1204. if (entry->consume_q == NULL) {
  1205. result = VMCI_ERROR_NO_MEM;
  1206. goto error;
  1207. }
  1208. qp_init_queue_mutex(entry->produce_q, entry->consume_q);
  1209. INIT_LIST_HEAD(&entry->qp.list_item);
  1210. if (is_local) {
  1211. u8 *tmp;
  1212. entry->local_mem = kcalloc(QPE_NUM_PAGES(entry->qp),
  1213. PAGE_SIZE, GFP_KERNEL);
  1214. if (entry->local_mem == NULL) {
  1215. result = VMCI_ERROR_NO_MEM;
  1216. goto error;
  1217. }
  1218. entry->state = VMCIQPB_CREATED_MEM;
  1219. entry->produce_q->q_header = entry->local_mem;
  1220. tmp = (u8 *)entry->local_mem + PAGE_SIZE *
  1221. (DIV_ROUND_UP(entry->qp.produce_size, PAGE_SIZE) + 1);
  1222. entry->consume_q->q_header = (struct vmci_queue_header *)tmp;
  1223. } else if (page_store) {
  1224. /*
  1225. * The VMX already initialized the queue pair headers, so no
  1226. * need for the kernel side to do that.
  1227. */
  1228. result = qp_host_register_user_memory(page_store,
  1229. entry->produce_q,
  1230. entry->consume_q);
  1231. if (result < VMCI_SUCCESS)
  1232. goto error;
  1233. entry->state = VMCIQPB_CREATED_MEM;
  1234. } else {
  1235. /*
  1236. * A create without a page_store may be either a host
  1237. * side create (in which case we are waiting for the
  1238. * guest side to supply the memory) or an old style
  1239. * queue pair create (in which case we will expect a
  1240. * set page store call as the next step).
  1241. */
  1242. entry->state = VMCIQPB_CREATED_NO_MEM;
  1243. }
  1244. qp_list_add_entry(&qp_broker_list, &entry->qp);
  1245. if (ent != NULL)
  1246. *ent = entry;
  1247. /* Add to resource obj */
  1248. result = vmci_resource_add(&entry->resource,
  1249. VMCI_RESOURCE_TYPE_QPAIR_HOST,
  1250. handle);
  1251. if (result != VMCI_SUCCESS) {
  1252. pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
  1253. handle.context, handle.resource, result);
  1254. goto error;
  1255. }
  1256. entry->qp.handle = vmci_resource_handle(&entry->resource);
  1257. if (is_local) {
  1258. vmci_q_header_init(entry->produce_q->q_header,
  1259. entry->qp.handle);
  1260. vmci_q_header_init(entry->consume_q->q_header,
  1261. entry->qp.handle);
  1262. }
  1263. vmci_ctx_qp_create(context, entry->qp.handle);
  1264. return VMCI_SUCCESS;
  1265. error:
  1266. if (entry != NULL) {
  1267. qp_host_free_queue(entry->produce_q, guest_produce_size);
  1268. qp_host_free_queue(entry->consume_q, guest_consume_size);
  1269. kfree(entry);
  1270. }
  1271. return result;
  1272. }
  1273. /*
  1274. * Enqueues an event datagram to notify the peer VM attached to
  1275. * the given queue pair handle about attach/detach event by the
  1276. * given VM. Returns Payload size of datagram enqueued on
  1277. * success, error code otherwise.
  1278. */
  1279. static int qp_notify_peer(bool attach,
  1280. struct vmci_handle handle,
  1281. u32 my_id,
  1282. u32 peer_id)
  1283. {
  1284. int rv;
  1285. struct vmci_event_qp ev;
  1286. if (vmci_handle_is_invalid(handle) || my_id == VMCI_INVALID_ID ||
  1287. peer_id == VMCI_INVALID_ID)
  1288. return VMCI_ERROR_INVALID_ARGS;
  1289. /*
  1290. * In vmci_ctx_enqueue_datagram() we enforce the upper limit on
  1291. * number of pending events from the hypervisor to a given VM
  1292. * otherwise a rogue VM could do an arbitrary number of attach
  1293. * and detach operations causing memory pressure in the host
  1294. * kernel.
  1295. */
  1296. ev.msg.hdr.dst = vmci_make_handle(peer_id, VMCI_EVENT_HANDLER);
  1297. ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
  1298. VMCI_CONTEXT_RESOURCE_ID);
  1299. ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
  1300. ev.msg.event_data.event = attach ?
  1301. VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
  1302. ev.payload.handle = handle;
  1303. ev.payload.peer_id = my_id;
  1304. rv = vmci_datagram_dispatch(VMCI_HYPERVISOR_CONTEXT_ID,
  1305. &ev.msg.hdr, false);
  1306. if (rv < VMCI_SUCCESS)
  1307. pr_warn("Failed to enqueue queue_pair %s event datagram for context (ID=0x%x)\n",
  1308. attach ? "ATTACH" : "DETACH", peer_id);
  1309. return rv;
  1310. }
  1311. /*
  1312. * The second endpoint issuing a queue pair allocation will attach to
  1313. * the queue pair registered with the queue pair broker.
  1314. *
  1315. * If the attacher is a guest, it will associate a VMX virtual address
  1316. * range with the queue pair as specified by the page_store. At this
  1317. * point, the already attach host endpoint may start using the queue
  1318. * pair, and an attach event is sent to it. For compatibility with
  1319. * older VMX'en, that used a separate step to set the VMX virtual
  1320. * address range, the virtual address range can be registered later
  1321. * using vmci_qp_broker_set_page_store. In that case, a page_store of
  1322. * NULL should be used, and the attach event will be generated once
  1323. * the actual page store has been set.
  1324. *
  1325. * If the attacher is the host, a page_store of NULL should be used as
  1326. * well, since the page store information is already set by the guest.
  1327. *
  1328. * For new VMX and host callers, the queue pair will be moved to the
  1329. * VMCIQPB_ATTACHED_MEM state, and for older VMX callers, it will be
  1330. * moved to the VMCOQPB_ATTACHED_NO_MEM state.
  1331. */
  1332. static int qp_broker_attach(struct qp_broker_entry *entry,
  1333. u32 peer,
  1334. u32 flags,
  1335. u32 priv_flags,
  1336. u64 produce_size,
  1337. u64 consume_size,
  1338. struct vmci_qp_page_store *page_store,
  1339. struct vmci_ctx *context,
  1340. vmci_event_release_cb wakeup_cb,
  1341. void *client_data,
  1342. struct qp_broker_entry **ent)
  1343. {
  1344. const u32 context_id = vmci_ctx_get_id(context);
  1345. bool is_local = flags & VMCI_QPFLAG_LOCAL;
  1346. int result;
  1347. if (entry->state != VMCIQPB_CREATED_NO_MEM &&
  1348. entry->state != VMCIQPB_CREATED_MEM)
  1349. return VMCI_ERROR_UNAVAILABLE;
  1350. if (is_local) {
  1351. if (!(entry->qp.flags & VMCI_QPFLAG_LOCAL) ||
  1352. context_id != entry->create_id) {
  1353. return VMCI_ERROR_INVALID_ARGS;
  1354. }
  1355. } else if (context_id == entry->create_id ||
  1356. context_id == entry->attach_id) {
  1357. return VMCI_ERROR_ALREADY_EXISTS;
  1358. }
  1359. if (VMCI_CONTEXT_IS_VM(context_id) &&
  1360. VMCI_CONTEXT_IS_VM(entry->create_id))
  1361. return VMCI_ERROR_DST_UNREACHABLE;
  1362. /*
  1363. * If we are attaching from a restricted context then the queuepair
  1364. * must have been created by a trusted endpoint.
  1365. */
  1366. if ((context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED) &&
  1367. !entry->created_by_trusted)
  1368. return VMCI_ERROR_NO_ACCESS;
  1369. /*
  1370. * If we are attaching to a queuepair that was created by a restricted
  1371. * context then we must be trusted.
  1372. */
  1373. if (entry->require_trusted_attach &&
  1374. (!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED)))
  1375. return VMCI_ERROR_NO_ACCESS;
  1376. /*
  1377. * If the creator specifies VMCI_INVALID_ID in "peer" field, access
  1378. * control check is not performed.
  1379. */
  1380. if (entry->qp.peer != VMCI_INVALID_ID && entry->qp.peer != context_id)
  1381. return VMCI_ERROR_NO_ACCESS;
  1382. if (entry->create_id == VMCI_HOST_CONTEXT_ID) {
  1383. /*
  1384. * Do not attach if the caller doesn't support Host Queue Pairs
  1385. * and a host created this queue pair.
  1386. */
  1387. if (!vmci_ctx_supports_host_qp(context))
  1388. return VMCI_ERROR_INVALID_RESOURCE;
  1389. } else if (context_id == VMCI_HOST_CONTEXT_ID) {
  1390. struct vmci_ctx *create_context;
  1391. bool supports_host_qp;
  1392. /*
  1393. * Do not attach a host to a user created queue pair if that
  1394. * user doesn't support host queue pair end points.
  1395. */
  1396. create_context = vmci_ctx_get(entry->create_id);
  1397. supports_host_qp = vmci_ctx_supports_host_qp(create_context);
  1398. vmci_ctx_put(create_context);
  1399. if (!supports_host_qp)
  1400. return VMCI_ERROR_INVALID_RESOURCE;
  1401. }
  1402. if ((entry->qp.flags & ~VMCI_QP_ASYMM) != (flags & ~VMCI_QP_ASYMM_PEER))
  1403. return VMCI_ERROR_QUEUEPAIR_MISMATCH;
  1404. if (context_id != VMCI_HOST_CONTEXT_ID) {
  1405. /*
  1406. * The queue pair broker entry stores values from the guest
  1407. * point of view, so an attaching guest should match the values
  1408. * stored in the entry.
  1409. */
  1410. if (entry->qp.produce_size != produce_size ||
  1411. entry->qp.consume_size != consume_size) {
  1412. return VMCI_ERROR_QUEUEPAIR_MISMATCH;
  1413. }
  1414. } else if (entry->qp.produce_size != consume_size ||
  1415. entry->qp.consume_size != produce_size) {
  1416. return VMCI_ERROR_QUEUEPAIR_MISMATCH;
  1417. }
  1418. if (context_id != VMCI_HOST_CONTEXT_ID) {
  1419. /*
  1420. * If a guest attached to a queue pair, it will supply
  1421. * the backing memory. If this is a pre NOVMVM vmx,
  1422. * the backing memory will be supplied by calling
  1423. * vmci_qp_broker_set_page_store() following the
  1424. * return of the vmci_qp_broker_alloc() call. If it is
  1425. * a vmx of version NOVMVM or later, the page store
  1426. * must be supplied as part of the
  1427. * vmci_qp_broker_alloc call. Under all circumstances
  1428. * must the initially created queue pair not have any
  1429. * memory associated with it already.
  1430. */
  1431. if (entry->state != VMCIQPB_CREATED_NO_MEM)
  1432. return VMCI_ERROR_INVALID_ARGS;
  1433. if (page_store != NULL) {
  1434. /*
  1435. * Patch up host state to point to guest
  1436. * supplied memory. The VMX already
  1437. * initialized the queue pair headers, so no
  1438. * need for the kernel side to do that.
  1439. */
  1440. result = qp_host_register_user_memory(page_store,
  1441. entry->produce_q,
  1442. entry->consume_q);
  1443. if (result < VMCI_SUCCESS)
  1444. return result;
  1445. entry->state = VMCIQPB_ATTACHED_MEM;
  1446. } else {
  1447. entry->state = VMCIQPB_ATTACHED_NO_MEM;
  1448. }
  1449. } else if (entry->state == VMCIQPB_CREATED_NO_MEM) {
  1450. /*
  1451. * The host side is attempting to attach to a queue
  1452. * pair that doesn't have any memory associated with
  1453. * it. This must be a pre NOVMVM vmx that hasn't set
  1454. * the page store information yet, or a quiesced VM.
  1455. */
  1456. return VMCI_ERROR_UNAVAILABLE;
  1457. } else {
  1458. /* The host side has successfully attached to a queue pair. */
  1459. entry->state = VMCIQPB_ATTACHED_MEM;
  1460. }
  1461. if (entry->state == VMCIQPB_ATTACHED_MEM) {
  1462. result =
  1463. qp_notify_peer(true, entry->qp.handle, context_id,
  1464. entry->create_id);
  1465. if (result < VMCI_SUCCESS)
  1466. pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
  1467. entry->create_id, entry->qp.handle.context,
  1468. entry->qp.handle.resource);
  1469. }
  1470. entry->attach_id = context_id;
  1471. entry->qp.ref_count++;
  1472. if (wakeup_cb) {
  1473. entry->wakeup_cb = wakeup_cb;
  1474. entry->client_data = client_data;
  1475. }
  1476. /*
  1477. * When attaching to local queue pairs, the context already has
  1478. * an entry tracking the queue pair, so don't add another one.
  1479. */
  1480. if (!is_local)
  1481. vmci_ctx_qp_create(context, entry->qp.handle);
  1482. if (ent != NULL)
  1483. *ent = entry;
  1484. return VMCI_SUCCESS;
  1485. }
  1486. /*
  1487. * queue_pair_Alloc for use when setting up queue pair endpoints
  1488. * on the host.
  1489. */
  1490. static int qp_broker_alloc(struct vmci_handle handle,
  1491. u32 peer,
  1492. u32 flags,
  1493. u32 priv_flags,
  1494. u64 produce_size,
  1495. u64 consume_size,
  1496. struct vmci_qp_page_store *page_store,
  1497. struct vmci_ctx *context,
  1498. vmci_event_release_cb wakeup_cb,
  1499. void *client_data,
  1500. struct qp_broker_entry **ent,
  1501. bool *swap)
  1502. {
  1503. const u32 context_id = vmci_ctx_get_id(context);
  1504. bool create;
  1505. struct qp_broker_entry *entry = NULL;
  1506. bool is_local = flags & VMCI_QPFLAG_LOCAL;
  1507. int result;
  1508. if (vmci_handle_is_invalid(handle) ||
  1509. (flags & ~VMCI_QP_ALL_FLAGS) || is_local ||
  1510. !(produce_size || consume_size) ||
  1511. !context || context_id == VMCI_INVALID_ID ||
  1512. handle.context == VMCI_INVALID_ID) {
  1513. return VMCI_ERROR_INVALID_ARGS;
  1514. }
  1515. if (page_store && !VMCI_QP_PAGESTORE_IS_WELLFORMED(page_store))
  1516. return VMCI_ERROR_INVALID_ARGS;
  1517. /*
  1518. * In the initial argument check, we ensure that non-vmkernel hosts
  1519. * are not allowed to create local queue pairs.
  1520. */
  1521. mutex_lock(&qp_broker_list.mutex);
  1522. if (!is_local && vmci_ctx_qp_exists(context, handle)) {
  1523. pr_devel("Context (ID=0x%x) already attached to queue pair (handle=0x%x:0x%x)\n",
  1524. context_id, handle.context, handle.resource);
  1525. mutex_unlock(&qp_broker_list.mutex);
  1526. return VMCI_ERROR_ALREADY_EXISTS;
  1527. }
  1528. if (handle.resource != VMCI_INVALID_ID)
  1529. entry = qp_broker_handle_to_entry(handle);
  1530. if (!entry) {
  1531. create = true;
  1532. result =
  1533. qp_broker_create(handle, peer, flags, priv_flags,
  1534. produce_size, consume_size, page_store,
  1535. context, wakeup_cb, client_data, ent);
  1536. } else {
  1537. create = false;
  1538. result =
  1539. qp_broker_attach(entry, peer, flags, priv_flags,
  1540. produce_size, consume_size, page_store,
  1541. context, wakeup_cb, client_data, ent);
  1542. }
  1543. mutex_unlock(&qp_broker_list.mutex);
  1544. if (swap)
  1545. *swap = (context_id == VMCI_HOST_CONTEXT_ID) &&
  1546. !(create && is_local);
  1547. return result;
  1548. }
  1549. /*
  1550. * This function implements the kernel API for allocating a queue
  1551. * pair.
  1552. */
  1553. static int qp_alloc_host_work(struct vmci_handle *handle,
  1554. struct vmci_queue **produce_q,
  1555. u64 produce_size,
  1556. struct vmci_queue **consume_q,
  1557. u64 consume_size,
  1558. u32 peer,
  1559. u32 flags,
  1560. u32 priv_flags,
  1561. vmci_event_release_cb wakeup_cb,
  1562. void *client_data)
  1563. {
  1564. struct vmci_handle new_handle;
  1565. struct vmci_ctx *context;
  1566. struct qp_broker_entry *entry;
  1567. int result;
  1568. bool swap;
  1569. if (vmci_handle_is_invalid(*handle)) {
  1570. new_handle = vmci_make_handle(
  1571. VMCI_HOST_CONTEXT_ID, VMCI_INVALID_ID);
  1572. } else
  1573. new_handle = *handle;
  1574. context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
  1575. entry = NULL;
  1576. result =
  1577. qp_broker_alloc(new_handle, peer, flags, priv_flags,
  1578. produce_size, consume_size, NULL, context,
  1579. wakeup_cb, client_data, &entry, &swap);
  1580. if (result == VMCI_SUCCESS) {
  1581. if (swap) {
  1582. /*
  1583. * If this is a local queue pair, the attacher
  1584. * will swap around produce and consume
  1585. * queues.
  1586. */
  1587. *produce_q = entry->consume_q;
  1588. *consume_q = entry->produce_q;
  1589. } else {
  1590. *produce_q = entry->produce_q;
  1591. *consume_q = entry->consume_q;
  1592. }
  1593. *handle = vmci_resource_handle(&entry->resource);
  1594. } else {
  1595. *handle = VMCI_INVALID_HANDLE;
  1596. pr_devel("queue pair broker failed to alloc (result=%d)\n",
  1597. result);
  1598. }
  1599. vmci_ctx_put(context);
  1600. return result;
  1601. }
  1602. /*
  1603. * Allocates a VMCI queue_pair. Only checks validity of input
  1604. * arguments. The real work is done in the host or guest
  1605. * specific function.
  1606. */
  1607. int vmci_qp_alloc(struct vmci_handle *handle,
  1608. struct vmci_queue **produce_q,
  1609. u64 produce_size,
  1610. struct vmci_queue **consume_q,
  1611. u64 consume_size,
  1612. u32 peer,
  1613. u32 flags,
  1614. u32 priv_flags,
  1615. bool guest_endpoint,
  1616. vmci_event_release_cb wakeup_cb,
  1617. void *client_data)
  1618. {
  1619. if (!handle || !produce_q || !consume_q ||
  1620. (!produce_size && !consume_size) || (flags & ~VMCI_QP_ALL_FLAGS))
  1621. return VMCI_ERROR_INVALID_ARGS;
  1622. if (guest_endpoint) {
  1623. return qp_alloc_guest_work(handle, produce_q,
  1624. produce_size, consume_q,
  1625. consume_size, peer,
  1626. flags, priv_flags);
  1627. } else {
  1628. return qp_alloc_host_work(handle, produce_q,
  1629. produce_size, consume_q,
  1630. consume_size, peer, flags,
  1631. priv_flags, wakeup_cb, client_data);
  1632. }
  1633. }
  1634. /*
  1635. * This function implements the host kernel API for detaching from
  1636. * a queue pair.
  1637. */
  1638. static int qp_detatch_host_work(struct vmci_handle handle)
  1639. {
  1640. int result;
  1641. struct vmci_ctx *context;
  1642. context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
  1643. result = vmci_qp_broker_detach(handle, context);
  1644. vmci_ctx_put(context);
  1645. return result;
  1646. }
  1647. /*
  1648. * Detaches from a VMCI queue_pair. Only checks validity of input argument.
  1649. * Real work is done in the host or guest specific function.
  1650. */
  1651. static int qp_detatch(struct vmci_handle handle, bool guest_endpoint)
  1652. {
  1653. if (vmci_handle_is_invalid(handle))
  1654. return VMCI_ERROR_INVALID_ARGS;
  1655. if (guest_endpoint)
  1656. return qp_detatch_guest_work(handle);
  1657. else
  1658. return qp_detatch_host_work(handle);
  1659. }
  1660. /*
  1661. * Returns the entry from the head of the list. Assumes that the list is
  1662. * locked.
  1663. */
  1664. static struct qp_entry *qp_list_get_head(struct qp_list *qp_list)
  1665. {
  1666. if (!list_empty(&qp_list->head)) {
  1667. struct qp_entry *entry =
  1668. list_first_entry(&qp_list->head, struct qp_entry,
  1669. list_item);
  1670. return entry;
  1671. }
  1672. return NULL;
  1673. }
  1674. void vmci_qp_broker_exit(void)
  1675. {
  1676. struct qp_entry *entry;
  1677. struct qp_broker_entry *be;
  1678. mutex_lock(&qp_broker_list.mutex);
  1679. while ((entry = qp_list_get_head(&qp_broker_list))) {
  1680. be = (struct qp_broker_entry *)entry;
  1681. qp_list_remove_entry(&qp_broker_list, entry);
  1682. kfree(be);
  1683. }
  1684. mutex_unlock(&qp_broker_list.mutex);
  1685. }
  1686. /*
  1687. * Requests that a queue pair be allocated with the VMCI queue
  1688. * pair broker. Allocates a queue pair entry if one does not
  1689. * exist. Attaches to one if it exists, and retrieves the page
  1690. * files backing that queue_pair. Assumes that the queue pair
  1691. * broker lock is held.
  1692. */
  1693. int vmci_qp_broker_alloc(struct vmci_handle handle,
  1694. u32 peer,
  1695. u32 flags,
  1696. u32 priv_flags,
  1697. u64 produce_size,
  1698. u64 consume_size,
  1699. struct vmci_qp_page_store *page_store,
  1700. struct vmci_ctx *context)
  1701. {
  1702. return qp_broker_alloc(handle, peer, flags, priv_flags,
  1703. produce_size, consume_size,
  1704. page_store, context, NULL, NULL, NULL, NULL);
  1705. }
  1706. /*
  1707. * VMX'en with versions lower than VMCI_VERSION_NOVMVM use a separate
  1708. * step to add the UVAs of the VMX mapping of the queue pair. This function
  1709. * provides backwards compatibility with such VMX'en, and takes care of
  1710. * registering the page store for a queue pair previously allocated by the
  1711. * VMX during create or attach. This function will move the queue pair state
  1712. * to either from VMCIQBP_CREATED_NO_MEM to VMCIQBP_CREATED_MEM or
  1713. * VMCIQBP_ATTACHED_NO_MEM to VMCIQBP_ATTACHED_MEM. If moving to the
  1714. * attached state with memory, the queue pair is ready to be used by the
  1715. * host peer, and an attached event will be generated.
  1716. *
  1717. * Assumes that the queue pair broker lock is held.
  1718. *
  1719. * This function is only used by the hosted platform, since there is no
  1720. * issue with backwards compatibility for vmkernel.
  1721. */
  1722. int vmci_qp_broker_set_page_store(struct vmci_handle handle,
  1723. u64 produce_uva,
  1724. u64 consume_uva,
  1725. struct vmci_ctx *context)
  1726. {
  1727. struct qp_broker_entry *entry;
  1728. int result;
  1729. const u32 context_id = vmci_ctx_get_id(context);
  1730. if (vmci_handle_is_invalid(handle) || !context ||
  1731. context_id == VMCI_INVALID_ID)
  1732. return VMCI_ERROR_INVALID_ARGS;
  1733. /*
  1734. * We only support guest to host queue pairs, so the VMX must
  1735. * supply UVAs for the mapped page files.
  1736. */
  1737. if (produce_uva == 0 || consume_uva == 0)
  1738. return VMCI_ERROR_INVALID_ARGS;
  1739. mutex_lock(&qp_broker_list.mutex);
  1740. if (!vmci_ctx_qp_exists(context, handle)) {
  1741. pr_warn("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
  1742. context_id, handle.context, handle.resource);
  1743. result = VMCI_ERROR_NOT_FOUND;
  1744. goto out;
  1745. }
  1746. entry = qp_broker_handle_to_entry(handle);
  1747. if (!entry) {
  1748. result = VMCI_ERROR_NOT_FOUND;
  1749. goto out;
  1750. }
  1751. /*
  1752. * If I'm the owner then I can set the page store.
  1753. *
  1754. * Or, if a host created the queue_pair and I'm the attached peer
  1755. * then I can set the page store.
  1756. */
  1757. if (entry->create_id != context_id &&
  1758. (entry->create_id != VMCI_HOST_CONTEXT_ID ||
  1759. entry->attach_id != context_id)) {
  1760. result = VMCI_ERROR_QUEUEPAIR_NOTOWNER;
  1761. goto out;
  1762. }
  1763. if (entry->state != VMCIQPB_CREATED_NO_MEM &&
  1764. entry->state != VMCIQPB_ATTACHED_NO_MEM) {
  1765. result = VMCI_ERROR_UNAVAILABLE;
  1766. goto out;
  1767. }
  1768. result = qp_host_get_user_memory(produce_uva, consume_uva,
  1769. entry->produce_q, entry->consume_q);
  1770. if (result < VMCI_SUCCESS)
  1771. goto out;
  1772. result = qp_host_map_queues(entry->produce_q, entry->consume_q);
  1773. if (result < VMCI_SUCCESS) {
  1774. qp_host_unregister_user_memory(entry->produce_q,
  1775. entry->consume_q);
  1776. goto out;
  1777. }
  1778. if (entry->state == VMCIQPB_CREATED_NO_MEM)
  1779. entry->state = VMCIQPB_CREATED_MEM;
  1780. else
  1781. entry->state = VMCIQPB_ATTACHED_MEM;
  1782. entry->vmci_page_files = true;
  1783. if (entry->state == VMCIQPB_ATTACHED_MEM) {
  1784. result =
  1785. qp_notify_peer(true, handle, context_id, entry->create_id);
  1786. if (result < VMCI_SUCCESS) {
  1787. pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
  1788. entry->create_id, entry->qp.handle.context,
  1789. entry->qp.handle.resource);
  1790. }
  1791. }
  1792. result = VMCI_SUCCESS;
  1793. out:
  1794. mutex_unlock(&qp_broker_list.mutex);
  1795. return result;
  1796. }
  1797. /*
  1798. * Resets saved queue headers for the given QP broker
  1799. * entry. Should be used when guest memory becomes available
  1800. * again, or the guest detaches.
  1801. */
  1802. static void qp_reset_saved_headers(struct qp_broker_entry *entry)
  1803. {
  1804. entry->produce_q->saved_header = NULL;
  1805. entry->consume_q->saved_header = NULL;
  1806. }
  1807. /*
  1808. * The main entry point for detaching from a queue pair registered with the
  1809. * queue pair broker. If more than one endpoint is attached to the queue
  1810. * pair, the first endpoint will mainly decrement a reference count and
  1811. * generate a notification to its peer. The last endpoint will clean up
  1812. * the queue pair state registered with the broker.
  1813. *
  1814. * When a guest endpoint detaches, it will unmap and unregister the guest
  1815. * memory backing the queue pair. If the host is still attached, it will
  1816. * no longer be able to access the queue pair content.
  1817. *
  1818. * If the queue pair is already in a state where there is no memory
  1819. * registered for the queue pair (any *_NO_MEM state), it will transition to
  1820. * the VMCIQPB_SHUTDOWN_NO_MEM state. This will also happen, if a guest
  1821. * endpoint is the first of two endpoints to detach. If the host endpoint is
  1822. * the first out of two to detach, the queue pair will move to the
  1823. * VMCIQPB_SHUTDOWN_MEM state.
  1824. */
  1825. int vmci_qp_broker_detach(struct vmci_handle handle, struct vmci_ctx *context)
  1826. {
  1827. struct qp_broker_entry *entry;
  1828. const u32 context_id = vmci_ctx_get_id(context);
  1829. u32 peer_id;
  1830. bool is_local = false;
  1831. int result;
  1832. if (vmci_handle_is_invalid(handle) || !context ||
  1833. context_id == VMCI_INVALID_ID) {
  1834. return VMCI_ERROR_INVALID_ARGS;
  1835. }
  1836. mutex_lock(&qp_broker_list.mutex);
  1837. if (!vmci_ctx_qp_exists(context, handle)) {
  1838. pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
  1839. context_id, handle.context, handle.resource);
  1840. result = VMCI_ERROR_NOT_FOUND;
  1841. goto out;
  1842. }
  1843. entry = qp_broker_handle_to_entry(handle);
  1844. if (!entry) {
  1845. pr_devel("Context (ID=0x%x) reports being attached to queue pair(handle=0x%x:0x%x) that isn't present in broker\n",
  1846. context_id, handle.context, handle.resource);
  1847. result = VMCI_ERROR_NOT_FOUND;
  1848. goto out;
  1849. }
  1850. if (context_id != entry->create_id && context_id != entry->attach_id) {
  1851. result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
  1852. goto out;
  1853. }
  1854. if (context_id == entry->create_id) {
  1855. peer_id = entry->attach_id;
  1856. entry->create_id = VMCI_INVALID_ID;
  1857. } else {
  1858. peer_id = entry->create_id;
  1859. entry->attach_id = VMCI_INVALID_ID;
  1860. }
  1861. entry->qp.ref_count--;
  1862. is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;
  1863. if (context_id != VMCI_HOST_CONTEXT_ID) {
  1864. bool headers_mapped;
  1865. /*
  1866. * Pre NOVMVM vmx'en may detach from a queue pair
  1867. * before setting the page store, and in that case
  1868. * there is no user memory to detach from. Also, more
  1869. * recent VMX'en may detach from a queue pair in the
  1870. * quiesced state.
  1871. */
  1872. qp_acquire_queue_mutex(entry->produce_q);
  1873. headers_mapped = entry->produce_q->q_header ||
  1874. entry->consume_q->q_header;
  1875. if (QPBROKERSTATE_HAS_MEM(entry)) {
  1876. result =
  1877. qp_host_unmap_queues(INVALID_VMCI_GUEST_MEM_ID,
  1878. entry->produce_q,
  1879. entry->consume_q);
  1880. if (result < VMCI_SUCCESS)
  1881. pr_warn("Failed to unmap queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
  1882. handle.context, handle.resource,
  1883. result);
  1884. qp_host_unregister_user_memory(entry->produce_q,
  1885. entry->consume_q);
  1886. }
  1887. if (!headers_mapped)
  1888. qp_reset_saved_headers(entry);
  1889. qp_release_queue_mutex(entry->produce_q);
  1890. if (!headers_mapped && entry->wakeup_cb)
  1891. entry->wakeup_cb(entry->client_data);
  1892. } else {
  1893. if (entry->wakeup_cb) {
  1894. entry->wakeup_cb = NULL;
  1895. entry->client_data = NULL;
  1896. }
  1897. }
  1898. if (entry->qp.ref_count == 0) {
  1899. qp_list_remove_entry(&qp_broker_list, &entry->qp);
  1900. if (is_local)
  1901. kfree(entry->local_mem);
  1902. qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
  1903. qp_host_free_queue(entry->produce_q, entry->qp.produce_size);
  1904. qp_host_free_queue(entry->consume_q, entry->qp.consume_size);
  1905. /* Unlink from resource hash table and free callback */
  1906. vmci_resource_remove(&entry->resource);
  1907. kfree(entry);
  1908. vmci_ctx_qp_destroy(context, handle);
  1909. } else {
  1910. qp_notify_peer(false, handle, context_id, peer_id);
  1911. if (context_id == VMCI_HOST_CONTEXT_ID &&
  1912. QPBROKERSTATE_HAS_MEM(entry)) {
  1913. entry->state = VMCIQPB_SHUTDOWN_MEM;
  1914. } else {
  1915. entry->state = VMCIQPB_SHUTDOWN_NO_MEM;
  1916. }
  1917. if (!is_local)
  1918. vmci_ctx_qp_destroy(context, handle);
  1919. }
  1920. result = VMCI_SUCCESS;
  1921. out:
  1922. mutex_unlock(&qp_broker_list.mutex);
  1923. return result;
  1924. }
  1925. /*
  1926. * Establishes the necessary mappings for a queue pair given a
  1927. * reference to the queue pair guest memory. This is usually
  1928. * called when a guest is unquiesced and the VMX is allowed to
  1929. * map guest memory once again.
  1930. */
  1931. int vmci_qp_broker_map(struct vmci_handle handle,
  1932. struct vmci_ctx *context,
  1933. u64 guest_mem)
  1934. {
  1935. struct qp_broker_entry *entry;
  1936. const u32 context_id = vmci_ctx_get_id(context);
  1937. bool is_local = false;
  1938. int result;
  1939. if (vmci_handle_is_invalid(handle) || !context ||
  1940. context_id == VMCI_INVALID_ID)
  1941. return VMCI_ERROR_INVALID_ARGS;
  1942. mutex_lock(&qp_broker_list.mutex);
  1943. if (!vmci_ctx_qp_exists(context, handle)) {
  1944. pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
  1945. context_id, handle.context, handle.resource);
  1946. result = VMCI_ERROR_NOT_FOUND;
  1947. goto out;
  1948. }
  1949. entry = qp_broker_handle_to_entry(handle);
  1950. if (!entry) {
  1951. pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
  1952. context_id, handle.context, handle.resource);
  1953. result = VMCI_ERROR_NOT_FOUND;
  1954. goto out;
  1955. }
  1956. if (context_id != entry->create_id && context_id != entry->attach_id) {
  1957. result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
  1958. goto out;
  1959. }
  1960. is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;
  1961. result = VMCI_SUCCESS;
  1962. if (context_id != VMCI_HOST_CONTEXT_ID) {
  1963. struct vmci_qp_page_store page_store;
  1964. page_store.pages = guest_mem;
  1965. page_store.len = QPE_NUM_PAGES(entry->qp);
  1966. qp_acquire_queue_mutex(entry->produce_q);
  1967. qp_reset_saved_headers(entry);
  1968. result =
  1969. qp_host_register_user_memory(&page_store,
  1970. entry->produce_q,
  1971. entry->consume_q);
  1972. qp_release_queue_mutex(entry->produce_q);
  1973. if (result == VMCI_SUCCESS) {
  1974. /* Move state from *_NO_MEM to *_MEM */
  1975. entry->state++;
  1976. if (entry->wakeup_cb)
  1977. entry->wakeup_cb(entry->client_data);
  1978. }
  1979. }
  1980. out:
  1981. mutex_unlock(&qp_broker_list.mutex);
  1982. return result;
  1983. }
  1984. /*
  1985. * Saves a snapshot of the queue headers for the given QP broker
  1986. * entry. Should be used when guest memory is unmapped.
  1987. * Results:
  1988. * VMCI_SUCCESS on success, appropriate error code if guest memory
  1989. * can't be accessed..
  1990. */
  1991. static int qp_save_headers(struct qp_broker_entry *entry)
  1992. {
  1993. int result;
  1994. if (entry->produce_q->saved_header != NULL &&
  1995. entry->consume_q->saved_header != NULL) {
  1996. /*
  1997. * If the headers have already been saved, we don't need to do
  1998. * it again, and we don't want to map in the headers
  1999. * unnecessarily.
  2000. */
  2001. return VMCI_SUCCESS;
  2002. }
  2003. if (NULL == entry->produce_q->q_header ||
  2004. NULL == entry->consume_q->q_header) {
  2005. result = qp_host_map_queues(entry->produce_q, entry->consume_q);
  2006. if (result < VMCI_SUCCESS)
  2007. return result;
  2008. }
  2009. memcpy(&entry->saved_produce_q, entry->produce_q->q_header,
  2010. sizeof(entry->saved_produce_q));
  2011. entry->produce_q->saved_header = &entry->saved_produce_q;
  2012. memcpy(&entry->saved_consume_q, entry->consume_q->q_header,
  2013. sizeof(entry->saved_consume_q));
  2014. entry->consume_q->saved_header = &entry->saved_consume_q;
  2015. return VMCI_SUCCESS;
  2016. }
  2017. /*
  2018. * Removes all references to the guest memory of a given queue pair, and
  2019. * will move the queue pair from state *_MEM to *_NO_MEM. It is usually
  2020. * called when a VM is being quiesced where access to guest memory should
  2021. * avoided.
  2022. */
  2023. int vmci_qp_broker_unmap(struct vmci_handle handle,
  2024. struct vmci_ctx *context,
  2025. u32 gid)
  2026. {
  2027. struct qp_broker_entry *entry;
  2028. const u32 context_id = vmci_ctx_get_id(context);
  2029. bool is_local = false;
  2030. int result;
  2031. if (vmci_handle_is_invalid(handle) || !context ||
  2032. context_id == VMCI_INVALID_ID)
  2033. return VMCI_ERROR_INVALID_ARGS;
  2034. mutex_lock(&qp_broker_list.mutex);
  2035. if (!vmci_ctx_qp_exists(context, handle)) {
  2036. pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
  2037. context_id, handle.context, handle.resource);
  2038. result = VMCI_ERROR_NOT_FOUND;
  2039. goto out;
  2040. }
  2041. entry = qp_broker_handle_to_entry(handle);
  2042. if (!entry) {
  2043. pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
  2044. context_id, handle.context, handle.resource);
  2045. result = VMCI_ERROR_NOT_FOUND;
  2046. goto out;
  2047. }
  2048. if (context_id != entry->create_id && context_id != entry->attach_id) {
  2049. result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
  2050. goto out;
  2051. }
  2052. is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;
  2053. if (context_id != VMCI_HOST_CONTEXT_ID) {
  2054. qp_acquire_queue_mutex(entry->produce_q);
  2055. result = qp_save_headers(entry);
  2056. if (result < VMCI_SUCCESS)
  2057. pr_warn("Failed to save queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
  2058. handle.context, handle.resource, result);
  2059. qp_host_unmap_queues(gid, entry->produce_q, entry->consume_q);
  2060. /*
  2061. * On hosted, when we unmap queue pairs, the VMX will also
  2062. * unmap the guest memory, so we invalidate the previously
  2063. * registered memory. If the queue pair is mapped again at a
  2064. * later point in time, we will need to reregister the user
  2065. * memory with a possibly new user VA.
  2066. */
  2067. qp_host_unregister_user_memory(entry->produce_q,
  2068. entry->consume_q);
  2069. /*
  2070. * Move state from *_MEM to *_NO_MEM.
  2071. */
  2072. entry->state--;
  2073. qp_release_queue_mutex(entry->produce_q);
  2074. }
  2075. result = VMCI_SUCCESS;
  2076. out:
  2077. mutex_unlock(&qp_broker_list.mutex);
  2078. return result;
  2079. }
  2080. /*
  2081. * Destroys all guest queue pair endpoints. If active guest queue
  2082. * pairs still exist, hypercalls to attempt detach from these
  2083. * queue pairs will be made. Any failure to detach is silently
  2084. * ignored.
  2085. */
  2086. void vmci_qp_guest_endpoints_exit(void)
  2087. {
  2088. struct qp_entry *entry;
  2089. struct qp_guest_endpoint *ep;
  2090. mutex_lock(&qp_guest_endpoints.mutex);
  2091. while ((entry = qp_list_get_head(&qp_guest_endpoints))) {
  2092. ep = (struct qp_guest_endpoint *)entry;
  2093. /* Don't make a hypercall for local queue_pairs. */
  2094. if (!(entry->flags & VMCI_QPFLAG_LOCAL))
  2095. qp_detatch_hypercall(entry->handle);
  2096. /* We cannot fail the exit, so let's reset ref_count. */
  2097. entry->ref_count = 0;
  2098. qp_list_remove_entry(&qp_guest_endpoints, entry);
  2099. qp_guest_endpoint_destroy(ep);
  2100. }
  2101. mutex_unlock(&qp_guest_endpoints.mutex);
  2102. }
  2103. /*
  2104. * Helper routine that will lock the queue pair before subsequent
  2105. * operations.
  2106. * Note: Non-blocking on the host side is currently only implemented in ESX.
  2107. * Since non-blocking isn't yet implemented on the host personality we
  2108. * have no reason to acquire a spin lock. So to avoid the use of an
  2109. * unnecessary lock only acquire the mutex if we can block.
  2110. */
  2111. static void qp_lock(const struct vmci_qp *qpair)
  2112. {
  2113. qp_acquire_queue_mutex(qpair->produce_q);
  2114. }
  2115. /*
  2116. * Helper routine that unlocks the queue pair after calling
  2117. * qp_lock.
  2118. */
  2119. static void qp_unlock(const struct vmci_qp *qpair)
  2120. {
  2121. qp_release_queue_mutex(qpair->produce_q);
  2122. }
  2123. /*
  2124. * The queue headers may not be mapped at all times. If a queue is
  2125. * currently not mapped, it will be attempted to do so.
  2126. */
  2127. static int qp_map_queue_headers(struct vmci_queue *produce_q,
  2128. struct vmci_queue *consume_q)
  2129. {
  2130. int result;
  2131. if (NULL == produce_q->q_header || NULL == consume_q->q_header) {
  2132. result = qp_host_map_queues(produce_q, consume_q);
  2133. if (result < VMCI_SUCCESS)
  2134. return (produce_q->saved_header &&
  2135. consume_q->saved_header) ?
  2136. VMCI_ERROR_QUEUEPAIR_NOT_READY :
  2137. VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
  2138. }
  2139. return VMCI_SUCCESS;
  2140. }
  2141. /*
  2142. * Helper routine that will retrieve the produce and consume
  2143. * headers of a given queue pair. If the guest memory of the
  2144. * queue pair is currently not available, the saved queue headers
  2145. * will be returned, if these are available.
  2146. */
  2147. static int qp_get_queue_headers(const struct vmci_qp *qpair,
  2148. struct vmci_queue_header **produce_q_header,
  2149. struct vmci_queue_header **consume_q_header)
  2150. {
  2151. int result;
  2152. result = qp_map_queue_headers(qpair->produce_q, qpair->consume_q);
  2153. if (result == VMCI_SUCCESS) {
  2154. *produce_q_header = qpair->produce_q->q_header;
  2155. *consume_q_header = qpair->consume_q->q_header;
  2156. } else if (qpair->produce_q->saved_header &&
  2157. qpair->consume_q->saved_header) {
  2158. *produce_q_header = qpair->produce_q->saved_header;
  2159. *consume_q_header = qpair->consume_q->saved_header;
  2160. result = VMCI_SUCCESS;
  2161. }
  2162. return result;
  2163. }
  2164. /*
  2165. * Callback from VMCI queue pair broker indicating that a queue
  2166. * pair that was previously not ready, now either is ready or
  2167. * gone forever.
  2168. */
  2169. static int qp_wakeup_cb(void *client_data)
  2170. {
  2171. struct vmci_qp *qpair = (struct vmci_qp *)client_data;
  2172. qp_lock(qpair);
  2173. while (qpair->blocked > 0) {
  2174. qpair->blocked--;
  2175. qpair->generation++;
  2176. wake_up(&qpair->event);
  2177. }
  2178. qp_unlock(qpair);
  2179. return VMCI_SUCCESS;
  2180. }
  2181. /*
  2182. * Makes the calling thread wait for the queue pair to become
  2183. * ready for host side access. Returns true when thread is
  2184. * woken up after queue pair state change, false otherwise.
  2185. */
  2186. static bool qp_wait_for_ready_queue(struct vmci_qp *qpair)
  2187. {
  2188. unsigned int generation;
  2189. qpair->blocked++;
  2190. generation = qpair->generation;
  2191. qp_unlock(qpair);
  2192. wait_event(qpair->event, generation != qpair->generation);
  2193. qp_lock(qpair);
  2194. return true;
  2195. }
  2196. /*
  2197. * Enqueues a given buffer to the produce queue using the provided
  2198. * function. As many bytes as possible (space available in the queue)
  2199. * are enqueued. Assumes the queue->mutex has been acquired. Returns
  2200. * VMCI_ERROR_QUEUEPAIR_NOSPACE if no space was available to enqueue
  2201. * data, VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the
  2202. * queue (as defined by the queue size), VMCI_ERROR_INVALID_ARGS, if
  2203. * an error occured when accessing the buffer,
  2204. * VMCI_ERROR_QUEUEPAIR_NOTATTACHED, if the queue pair pages aren't
  2205. * available. Otherwise, the number of bytes written to the queue is
  2206. * returned. Updates the tail pointer of the produce queue.
  2207. */
  2208. static ssize_t qp_enqueue_locked(struct vmci_queue *produce_q,
  2209. struct vmci_queue *consume_q,
  2210. const u64 produce_q_size,
  2211. struct iov_iter *from)
  2212. {
  2213. s64 free_space;
  2214. u64 tail;
  2215. size_t buf_size = iov_iter_count(from);
  2216. size_t written;
  2217. ssize_t result;
  2218. result = qp_map_queue_headers(produce_q, consume_q);
  2219. if (unlikely(result != VMCI_SUCCESS))
  2220. return result;
  2221. free_space = vmci_q_header_free_space(produce_q->q_header,
  2222. consume_q->q_header,
  2223. produce_q_size);
  2224. if (free_space == 0)
  2225. return VMCI_ERROR_QUEUEPAIR_NOSPACE;
  2226. if (free_space < VMCI_SUCCESS)
  2227. return (ssize_t) free_space;
  2228. written = (size_t) (free_space > buf_size ? buf_size : free_space);
  2229. tail = vmci_q_header_producer_tail(produce_q->q_header);
  2230. if (likely(tail + written < produce_q_size)) {
  2231. result = qp_memcpy_to_queue_iter(produce_q, tail, from, written);
  2232. } else {
  2233. /* Tail pointer wraps around. */
  2234. const size_t tmp = (size_t) (produce_q_size - tail);
  2235. result = qp_memcpy_to_queue_iter(produce_q, tail, from, tmp);
  2236. if (result >= VMCI_SUCCESS)
  2237. result = qp_memcpy_to_queue_iter(produce_q, 0, from,
  2238. written - tmp);
  2239. }
  2240. if (result < VMCI_SUCCESS)
  2241. return result;
  2242. vmci_q_header_add_producer_tail(produce_q->q_header, written,
  2243. produce_q_size);
  2244. return written;
  2245. }
  2246. /*
  2247. * Dequeues data (if available) from the given consume queue. Writes data
  2248. * to the user provided buffer using the provided function.
  2249. * Assumes the queue->mutex has been acquired.
  2250. * Results:
  2251. * VMCI_ERROR_QUEUEPAIR_NODATA if no data was available to dequeue.
  2252. * VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the queue
  2253. * (as defined by the queue size).
  2254. * VMCI_ERROR_INVALID_ARGS, if an error occured when accessing the buffer.
  2255. * Otherwise the number of bytes dequeued is returned.
  2256. * Side effects:
  2257. * Updates the head pointer of the consume queue.
  2258. */
  2259. static ssize_t qp_dequeue_locked(struct vmci_queue *produce_q,
  2260. struct vmci_queue *consume_q,
  2261. const u64 consume_q_size,
  2262. struct iov_iter *to,
  2263. bool update_consumer)
  2264. {
  2265. size_t buf_size = iov_iter_count(to);
  2266. s64 buf_ready;
  2267. u64 head;
  2268. size_t read;
  2269. ssize_t result;
  2270. result = qp_map_queue_headers(produce_q, consume_q);
  2271. if (unlikely(result != VMCI_SUCCESS))
  2272. return result;
  2273. buf_ready = vmci_q_header_buf_ready(consume_q->q_header,
  2274. produce_q->q_header,
  2275. consume_q_size);
  2276. if (buf_ready == 0)
  2277. return VMCI_ERROR_QUEUEPAIR_NODATA;
  2278. if (buf_ready < VMCI_SUCCESS)
  2279. return (ssize_t) buf_ready;
  2280. read = (size_t) (buf_ready > buf_size ? buf_size : buf_ready);
  2281. head = vmci_q_header_consumer_head(produce_q->q_header);
  2282. if (likely(head + read < consume_q_size)) {
  2283. result = qp_memcpy_from_queue_iter(to, consume_q, head, read);
  2284. } else {
  2285. /* Head pointer wraps around. */
  2286. const size_t tmp = (size_t) (consume_q_size - head);
  2287. result = qp_memcpy_from_queue_iter(to, consume_q, head, tmp);
  2288. if (result >= VMCI_SUCCESS)
  2289. result = qp_memcpy_from_queue_iter(to, consume_q, 0,
  2290. read - tmp);
  2291. }
  2292. if (result < VMCI_SUCCESS)
  2293. return result;
  2294. if (update_consumer)
  2295. vmci_q_header_add_consumer_head(produce_q->q_header,
  2296. read, consume_q_size);
  2297. return read;
  2298. }
  2299. /*
  2300. * vmci_qpair_alloc() - Allocates a queue pair.
  2301. * @qpair: Pointer for the new vmci_qp struct.
  2302. * @handle: Handle to track the resource.
  2303. * @produce_qsize: Desired size of the producer queue.
  2304. * @consume_qsize: Desired size of the consumer queue.
  2305. * @peer: ContextID of the peer.
  2306. * @flags: VMCI flags.
  2307. * @priv_flags: VMCI priviledge flags.
  2308. *
  2309. * This is the client interface for allocating the memory for a
  2310. * vmci_qp structure and then attaching to the underlying
  2311. * queue. If an error occurs allocating the memory for the
  2312. * vmci_qp structure no attempt is made to attach. If an
  2313. * error occurs attaching, then the structure is freed.
  2314. */
  2315. int vmci_qpair_alloc(struct vmci_qp **qpair,
  2316. struct vmci_handle *handle,
  2317. u64 produce_qsize,
  2318. u64 consume_qsize,
  2319. u32 peer,
  2320. u32 flags,
  2321. u32 priv_flags)
  2322. {
  2323. struct vmci_qp *my_qpair;
  2324. int retval;
  2325. struct vmci_handle src = VMCI_INVALID_HANDLE;
  2326. struct vmci_handle dst = vmci_make_handle(peer, VMCI_INVALID_ID);
  2327. enum vmci_route route;
  2328. vmci_event_release_cb wakeup_cb;
  2329. void *client_data;
  2330. /*
  2331. * Restrict the size of a queuepair. The device already
  2332. * enforces a limit on the total amount of memory that can be
  2333. * allocated to queuepairs for a guest. However, we try to
  2334. * allocate this memory before we make the queuepair
  2335. * allocation hypercall. On Linux, we allocate each page
  2336. * separately, which means rather than fail, the guest will
  2337. * thrash while it tries to allocate, and will become
  2338. * increasingly unresponsive to the point where it appears to
  2339. * be hung. So we place a limit on the size of an individual
  2340. * queuepair here, and leave the device to enforce the
  2341. * restriction on total queuepair memory. (Note that this
  2342. * doesn't prevent all cases; a user with only this much
  2343. * physical memory could still get into trouble.) The error
  2344. * used by the device is NO_RESOURCES, so use that here too.
  2345. */
  2346. if (produce_qsize + consume_qsize < max(produce_qsize, consume_qsize) ||
  2347. produce_qsize + consume_qsize > VMCI_MAX_GUEST_QP_MEMORY)
  2348. return VMCI_ERROR_NO_RESOURCES;
  2349. retval = vmci_route(&src, &dst, false, &route);
  2350. if (retval < VMCI_SUCCESS)
  2351. route = vmci_guest_code_active() ?
  2352. VMCI_ROUTE_AS_GUEST : VMCI_ROUTE_AS_HOST;
  2353. if (flags & (VMCI_QPFLAG_NONBLOCK | VMCI_QPFLAG_PINNED)) {
  2354. pr_devel("NONBLOCK OR PINNED set");
  2355. return VMCI_ERROR_INVALID_ARGS;
  2356. }
  2357. my_qpair = kzalloc(sizeof(*my_qpair), GFP_KERNEL);
  2358. if (!my_qpair)
  2359. return VMCI_ERROR_NO_MEM;
  2360. my_qpair->produce_q_size = produce_qsize;
  2361. my_qpair->consume_q_size = consume_qsize;
  2362. my_qpair->peer = peer;
  2363. my_qpair->flags = flags;
  2364. my_qpair->priv_flags = priv_flags;
  2365. wakeup_cb = NULL;
  2366. client_data = NULL;
  2367. if (VMCI_ROUTE_AS_HOST == route) {
  2368. my_qpair->guest_endpoint = false;
  2369. if (!(flags & VMCI_QPFLAG_LOCAL)) {
  2370. my_qpair->blocked = 0;
  2371. my_qpair->generation = 0;
  2372. init_waitqueue_head(&my_qpair->event);
  2373. wakeup_cb = qp_wakeup_cb;
  2374. client_data = (void *)my_qpair;
  2375. }
  2376. } else {
  2377. my_qpair->guest_endpoint = true;
  2378. }
  2379. retval = vmci_qp_alloc(handle,
  2380. &my_qpair->produce_q,
  2381. my_qpair->produce_q_size,
  2382. &my_qpair->consume_q,
  2383. my_qpair->consume_q_size,
  2384. my_qpair->peer,
  2385. my_qpair->flags,
  2386. my_qpair->priv_flags,
  2387. my_qpair->guest_endpoint,
  2388. wakeup_cb, client_data);
  2389. if (retval < VMCI_SUCCESS) {
  2390. kfree(my_qpair);
  2391. return retval;
  2392. }
  2393. *qpair = my_qpair;
  2394. my_qpair->handle = *handle;
  2395. return retval;
  2396. }
  2397. EXPORT_SYMBOL_GPL(vmci_qpair_alloc);
  2398. /*
  2399. * vmci_qpair_detach() - Detatches the client from a queue pair.
  2400. * @qpair: Reference of a pointer to the qpair struct.
  2401. *
  2402. * This is the client interface for detaching from a VMCIQPair.
  2403. * Note that this routine will free the memory allocated for the
  2404. * vmci_qp structure too.
  2405. */
  2406. int vmci_qpair_detach(struct vmci_qp **qpair)
  2407. {
  2408. int result;
  2409. struct vmci_qp *old_qpair;
  2410. if (!qpair || !(*qpair))
  2411. return VMCI_ERROR_INVALID_ARGS;
  2412. old_qpair = *qpair;
  2413. result = qp_detatch(old_qpair->handle, old_qpair->guest_endpoint);
  2414. /*
  2415. * The guest can fail to detach for a number of reasons, and
  2416. * if it does so, it will cleanup the entry (if there is one).
  2417. * The host can fail too, but it won't cleanup the entry
  2418. * immediately, it will do that later when the context is
  2419. * freed. Either way, we need to release the qpair struct
  2420. * here; there isn't much the caller can do, and we don't want
  2421. * to leak.
  2422. */
  2423. memset(old_qpair, 0, sizeof(*old_qpair));
  2424. old_qpair->handle = VMCI_INVALID_HANDLE;
  2425. old_qpair->peer = VMCI_INVALID_ID;
  2426. kfree(old_qpair);
  2427. *qpair = NULL;
  2428. return result;
  2429. }
  2430. EXPORT_SYMBOL_GPL(vmci_qpair_detach);
  2431. /*
  2432. * vmci_qpair_get_produce_indexes() - Retrieves the indexes of the producer.
  2433. * @qpair: Pointer to the queue pair struct.
  2434. * @producer_tail: Reference used for storing producer tail index.
  2435. * @consumer_head: Reference used for storing the consumer head index.
  2436. *
  2437. * This is the client interface for getting the current indexes of the
  2438. * QPair from the point of the view of the caller as the producer.
  2439. */
  2440. int vmci_qpair_get_produce_indexes(const struct vmci_qp *qpair,
  2441. u64 *producer_tail,
  2442. u64 *consumer_head)
  2443. {
  2444. struct vmci_queue_header *produce_q_header;
  2445. struct vmci_queue_header *consume_q_header;
  2446. int result;
  2447. if (!qpair)
  2448. return VMCI_ERROR_INVALID_ARGS;
  2449. qp_lock(qpair);
  2450. result =
  2451. qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
  2452. if (result == VMCI_SUCCESS)
  2453. vmci_q_header_get_pointers(produce_q_header, consume_q_header,
  2454. producer_tail, consumer_head);
  2455. qp_unlock(qpair);
  2456. if (result == VMCI_SUCCESS &&
  2457. ((producer_tail && *producer_tail >= qpair->produce_q_size) ||
  2458. (consumer_head && *consumer_head >= qpair->produce_q_size)))
  2459. return VMCI_ERROR_INVALID_SIZE;
  2460. return result;
  2461. }
  2462. EXPORT_SYMBOL_GPL(vmci_qpair_get_produce_indexes);
  2463. /*
  2464. * vmci_qpair_get_consume_indexes() - Retrieves the indexes of the consumer.
  2465. * @qpair: Pointer to the queue pair struct.
  2466. * @consumer_tail: Reference used for storing consumer tail index.
  2467. * @producer_head: Reference used for storing the producer head index.
  2468. *
  2469. * This is the client interface for getting the current indexes of the
  2470. * QPair from the point of the view of the caller as the consumer.
  2471. */
  2472. int vmci_qpair_get_consume_indexes(const struct vmci_qp *qpair,
  2473. u64 *consumer_tail,
  2474. u64 *producer_head)
  2475. {
  2476. struct vmci_queue_header *produce_q_header;
  2477. struct vmci_queue_header *consume_q_header;
  2478. int result;
  2479. if (!qpair)
  2480. return VMCI_ERROR_INVALID_ARGS;
  2481. qp_lock(qpair);
  2482. result =
  2483. qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
  2484. if (result == VMCI_SUCCESS)
  2485. vmci_q_header_get_pointers(consume_q_header, produce_q_header,
  2486. consumer_tail, producer_head);
  2487. qp_unlock(qpair);
  2488. if (result == VMCI_SUCCESS &&
  2489. ((consumer_tail && *consumer_tail >= qpair->consume_q_size) ||
  2490. (producer_head && *producer_head >= qpair->consume_q_size)))
  2491. return VMCI_ERROR_INVALID_SIZE;
  2492. return result;
  2493. }
  2494. EXPORT_SYMBOL_GPL(vmci_qpair_get_consume_indexes);
  2495. /*
  2496. * vmci_qpair_produce_free_space() - Retrieves free space in producer queue.
  2497. * @qpair: Pointer to the queue pair struct.
  2498. *
  2499. * This is the client interface for getting the amount of free
  2500. * space in the QPair from the point of the view of the caller as
  2501. * the producer which is the common case. Returns < 0 if err, else
  2502. * available bytes into which data can be enqueued if > 0.
  2503. */
  2504. s64 vmci_qpair_produce_free_space(const struct vmci_qp *qpair)
  2505. {
  2506. struct vmci_queue_header *produce_q_header;
  2507. struct vmci_queue_header *consume_q_header;
  2508. s64 result;
  2509. if (!qpair)
  2510. return VMCI_ERROR_INVALID_ARGS;
  2511. qp_lock(qpair);
  2512. result =
  2513. qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
  2514. if (result == VMCI_SUCCESS)
  2515. result = vmci_q_header_free_space(produce_q_header,
  2516. consume_q_header,
  2517. qpair->produce_q_size);
  2518. else
  2519. result = 0;
  2520. qp_unlock(qpair);
  2521. return result;
  2522. }
  2523. EXPORT_SYMBOL_GPL(vmci_qpair_produce_free_space);
  2524. /*
  2525. * vmci_qpair_consume_free_space() - Retrieves free space in consumer queue.
  2526. * @qpair: Pointer to the queue pair struct.
  2527. *
  2528. * This is the client interface for getting the amount of free
  2529. * space in the QPair from the point of the view of the caller as
  2530. * the consumer which is not the common case. Returns < 0 if err, else
  2531. * available bytes into which data can be enqueued if > 0.
  2532. */
  2533. s64 vmci_qpair_consume_free_space(const struct vmci_qp *qpair)
  2534. {
  2535. struct vmci_queue_header *produce_q_header;
  2536. struct vmci_queue_header *consume_q_header;
  2537. s64 result;
  2538. if (!qpair)
  2539. return VMCI_ERROR_INVALID_ARGS;
  2540. qp_lock(qpair);
  2541. result =
  2542. qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
  2543. if (result == VMCI_SUCCESS)
  2544. result = vmci_q_header_free_space(consume_q_header,
  2545. produce_q_header,
  2546. qpair->consume_q_size);
  2547. else
  2548. result = 0;
  2549. qp_unlock(qpair);
  2550. return result;
  2551. }
  2552. EXPORT_SYMBOL_GPL(vmci_qpair_consume_free_space);
  2553. /*
  2554. * vmci_qpair_produce_buf_ready() - Gets bytes ready to read from
  2555. * producer queue.
  2556. * @qpair: Pointer to the queue pair struct.
  2557. *
  2558. * This is the client interface for getting the amount of
  2559. * enqueued data in the QPair from the point of the view of the
  2560. * caller as the producer which is not the common case. Returns < 0 if err,
  2561. * else available bytes that may be read.
  2562. */
  2563. s64 vmci_qpair_produce_buf_ready(const struct vmci_qp *qpair)
  2564. {
  2565. struct vmci_queue_header *produce_q_header;
  2566. struct vmci_queue_header *consume_q_header;
  2567. s64 result;
  2568. if (!qpair)
  2569. return VMCI_ERROR_INVALID_ARGS;
  2570. qp_lock(qpair);
  2571. result =
  2572. qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
  2573. if (result == VMCI_SUCCESS)
  2574. result = vmci_q_header_buf_ready(produce_q_header,
  2575. consume_q_header,
  2576. qpair->produce_q_size);
  2577. else
  2578. result = 0;
  2579. qp_unlock(qpair);
  2580. return result;
  2581. }
  2582. EXPORT_SYMBOL_GPL(vmci_qpair_produce_buf_ready);
  2583. /*
  2584. * vmci_qpair_consume_buf_ready() - Gets bytes ready to read from
  2585. * consumer queue.
  2586. * @qpair: Pointer to the queue pair struct.
  2587. *
  2588. * This is the client interface for getting the amount of
  2589. * enqueued data in the QPair from the point of the view of the
  2590. * caller as the consumer which is the normal case. Returns < 0 if err,
  2591. * else available bytes that may be read.
  2592. */
  2593. s64 vmci_qpair_consume_buf_ready(const struct vmci_qp *qpair)
  2594. {
  2595. struct vmci_queue_header *produce_q_header;
  2596. struct vmci_queue_header *consume_q_header;
  2597. s64 result;
  2598. if (!qpair)
  2599. return VMCI_ERROR_INVALID_ARGS;
  2600. qp_lock(qpair);
  2601. result =
  2602. qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
  2603. if (result == VMCI_SUCCESS)
  2604. result = vmci_q_header_buf_ready(consume_q_header,
  2605. produce_q_header,
  2606. qpair->consume_q_size);
  2607. else
  2608. result = 0;
  2609. qp_unlock(qpair);
  2610. return result;
  2611. }
  2612. EXPORT_SYMBOL_GPL(vmci_qpair_consume_buf_ready);
  2613. /*
  2614. * vmci_qpair_enqueue() - Throw data on the queue.
  2615. * @qpair: Pointer to the queue pair struct.
  2616. * @buf: Pointer to buffer containing data
  2617. * @buf_size: Length of buffer.
  2618. * @buf_type: Buffer type (Unused).
  2619. *
  2620. * This is the client interface for enqueueing data into the queue.
  2621. * Returns number of bytes enqueued or < 0 on error.
  2622. */
  2623. ssize_t vmci_qpair_enqueue(struct vmci_qp *qpair,
  2624. const void *buf,
  2625. size_t buf_size,
  2626. int buf_type)
  2627. {
  2628. ssize_t result;
  2629. struct iov_iter from;
  2630. struct kvec v = {.iov_base = (void *)buf, .iov_len = buf_size};
  2631. if (!qpair || !buf)
  2632. return VMCI_ERROR_INVALID_ARGS;
  2633. iov_iter_kvec(&from, WRITE | ITER_KVEC, &v, 1, buf_size);
  2634. qp_lock(qpair);
  2635. do {
  2636. result = qp_enqueue_locked(qpair->produce_q,
  2637. qpair->consume_q,
  2638. qpair->produce_q_size,
  2639. &from);
  2640. if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
  2641. !qp_wait_for_ready_queue(qpair))
  2642. result = VMCI_ERROR_WOULD_BLOCK;
  2643. } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
  2644. qp_unlock(qpair);
  2645. return result;
  2646. }
  2647. EXPORT_SYMBOL_GPL(vmci_qpair_enqueue);
  2648. /*
  2649. * vmci_qpair_dequeue() - Get data from the queue.
  2650. * @qpair: Pointer to the queue pair struct.
  2651. * @buf: Pointer to buffer for the data
  2652. * @buf_size: Length of buffer.
  2653. * @buf_type: Buffer type (Unused).
  2654. *
  2655. * This is the client interface for dequeueing data from the queue.
  2656. * Returns number of bytes dequeued or < 0 on error.
  2657. */
  2658. ssize_t vmci_qpair_dequeue(struct vmci_qp *qpair,
  2659. void *buf,
  2660. size_t buf_size,
  2661. int buf_type)
  2662. {
  2663. ssize_t result;
  2664. struct iov_iter to;
  2665. struct kvec v = {.iov_base = buf, .iov_len = buf_size};
  2666. if (!qpair || !buf)
  2667. return VMCI_ERROR_INVALID_ARGS;
  2668. iov_iter_kvec(&to, READ | ITER_KVEC, &v, 1, buf_size);
  2669. qp_lock(qpair);
  2670. do {
  2671. result = qp_dequeue_locked(qpair->produce_q,
  2672. qpair->consume_q,
  2673. qpair->consume_q_size,
  2674. &to, true);
  2675. if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
  2676. !qp_wait_for_ready_queue(qpair))
  2677. result = VMCI_ERROR_WOULD_BLOCK;
  2678. } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
  2679. qp_unlock(qpair);
  2680. return result;
  2681. }
  2682. EXPORT_SYMBOL_GPL(vmci_qpair_dequeue);
  2683. /*
  2684. * vmci_qpair_peek() - Peek at the data in the queue.
  2685. * @qpair: Pointer to the queue pair struct.
  2686. * @buf: Pointer to buffer for the data
  2687. * @buf_size: Length of buffer.
  2688. * @buf_type: Buffer type (Unused on Linux).
  2689. *
  2690. * This is the client interface for peeking into a queue. (I.e.,
  2691. * copy data from the queue without updating the head pointer.)
  2692. * Returns number of bytes dequeued or < 0 on error.
  2693. */
  2694. ssize_t vmci_qpair_peek(struct vmci_qp *qpair,
  2695. void *buf,
  2696. size_t buf_size,
  2697. int buf_type)
  2698. {
  2699. struct iov_iter to;
  2700. struct kvec v = {.iov_base = buf, .iov_len = buf_size};
  2701. ssize_t result;
  2702. if (!qpair || !buf)
  2703. return VMCI_ERROR_INVALID_ARGS;
  2704. iov_iter_kvec(&to, READ | ITER_KVEC, &v, 1, buf_size);
  2705. qp_lock(qpair);
  2706. do {
  2707. result = qp_dequeue_locked(qpair->produce_q,
  2708. qpair->consume_q,
  2709. qpair->consume_q_size,
  2710. &to, false);
  2711. if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
  2712. !qp_wait_for_ready_queue(qpair))
  2713. result = VMCI_ERROR_WOULD_BLOCK;
  2714. } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
  2715. qp_unlock(qpair);
  2716. return result;
  2717. }
  2718. EXPORT_SYMBOL_GPL(vmci_qpair_peek);
  2719. /*
  2720. * vmci_qpair_enquev() - Throw data on the queue using iov.
  2721. * @qpair: Pointer to the queue pair struct.
  2722. * @iov: Pointer to buffer containing data
  2723. * @iov_size: Length of buffer.
  2724. * @buf_type: Buffer type (Unused).
  2725. *
  2726. * This is the client interface for enqueueing data into the queue.
  2727. * This function uses IO vectors to handle the work. Returns number
  2728. * of bytes enqueued or < 0 on error.
  2729. */
  2730. ssize_t vmci_qpair_enquev(struct vmci_qp *qpair,
  2731. struct msghdr *msg,
  2732. size_t iov_size,
  2733. int buf_type)
  2734. {
  2735. ssize_t result;
  2736. if (!qpair)
  2737. return VMCI_ERROR_INVALID_ARGS;
  2738. qp_lock(qpair);
  2739. do {
  2740. result = qp_enqueue_locked(qpair->produce_q,
  2741. qpair->consume_q,
  2742. qpair->produce_q_size,
  2743. &msg->msg_iter);
  2744. if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
  2745. !qp_wait_for_ready_queue(qpair))
  2746. result = VMCI_ERROR_WOULD_BLOCK;
  2747. } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
  2748. qp_unlock(qpair);
  2749. return result;
  2750. }
  2751. EXPORT_SYMBOL_GPL(vmci_qpair_enquev);
  2752. /*
  2753. * vmci_qpair_dequev() - Get data from the queue using iov.
  2754. * @qpair: Pointer to the queue pair struct.
  2755. * @iov: Pointer to buffer for the data
  2756. * @iov_size: Length of buffer.
  2757. * @buf_type: Buffer type (Unused).
  2758. *
  2759. * This is the client interface for dequeueing data from the queue.
  2760. * This function uses IO vectors to handle the work. Returns number
  2761. * of bytes dequeued or < 0 on error.
  2762. */
  2763. ssize_t vmci_qpair_dequev(struct vmci_qp *qpair,
  2764. struct msghdr *msg,
  2765. size_t iov_size,
  2766. int buf_type)
  2767. {
  2768. ssize_t result;
  2769. if (!qpair)
  2770. return VMCI_ERROR_INVALID_ARGS;
  2771. qp_lock(qpair);
  2772. do {
  2773. result = qp_dequeue_locked(qpair->produce_q,
  2774. qpair->consume_q,
  2775. qpair->consume_q_size,
  2776. &msg->msg_iter, true);
  2777. if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
  2778. !qp_wait_for_ready_queue(qpair))
  2779. result = VMCI_ERROR_WOULD_BLOCK;
  2780. } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
  2781. qp_unlock(qpair);
  2782. return result;
  2783. }
  2784. EXPORT_SYMBOL_GPL(vmci_qpair_dequev);
  2785. /*
  2786. * vmci_qpair_peekv() - Peek at the data in the queue using iov.
  2787. * @qpair: Pointer to the queue pair struct.
  2788. * @iov: Pointer to buffer for the data
  2789. * @iov_size: Length of buffer.
  2790. * @buf_type: Buffer type (Unused on Linux).
  2791. *
  2792. * This is the client interface for peeking into a queue. (I.e.,
  2793. * copy data from the queue without updating the head pointer.)
  2794. * This function uses IO vectors to handle the work. Returns number
  2795. * of bytes peeked or < 0 on error.
  2796. */
  2797. ssize_t vmci_qpair_peekv(struct vmci_qp *qpair,
  2798. struct msghdr *msg,
  2799. size_t iov_size,
  2800. int buf_type)
  2801. {
  2802. ssize_t result;
  2803. if (!qpair)
  2804. return VMCI_ERROR_INVALID_ARGS;
  2805. qp_lock(qpair);
  2806. do {
  2807. result = qp_dequeue_locked(qpair->produce_q,
  2808. qpair->consume_q,
  2809. qpair->consume_q_size,
  2810. &msg->msg_iter, false);
  2811. if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
  2812. !qp_wait_for_ready_queue(qpair))
  2813. result = VMCI_ERROR_WOULD_BLOCK;
  2814. } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
  2815. qp_unlock(qpair);
  2816. return result;
  2817. }
  2818. EXPORT_SYMBOL_GPL(vmci_qpair_peekv);