vmx.c 330 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385738673877388738973907391739273937394739573967397739873997400740174027403740474057406740774087409741074117412741374147415741674177418741974207421742274237424742574267427742874297430743174327433743474357436743774387439744074417442744374447445744674477448744974507451745274537454745574567457745874597460746174627463746474657466746774687469747074717472747374747475747674777478747974807481748274837484748574867487748874897490749174927493749474957496749774987499750075017502750375047505750675077508750975107511751275137514751575167517751875197520752175227523752475257526752775287529753075317532753375347535753675377538753975407541754275437544754575467547754875497550755175527553755475557556755775587559756075617562756375647565756675677568756975707571757275737574757575767577757875797580758175827583758475857586758775887589759075917592759375947595759675977598759976007601760276037604760576067607760876097610761176127613761476157616761776187619762076217622762376247625762676277628762976307631763276337634763576367637763876397640764176427643764476457646764776487649765076517652765376547655765676577658765976607661766276637664766576667667766876697670767176727673767476757676767776787679768076817682768376847685768676877688768976907691769276937694769576967697769876997700770177027703770477057706770777087709771077117712771377147715771677177718771977207721772277237724772577267727772877297730773177327733773477357736773777387739774077417742774377447745774677477748774977507751775277537754775577567757775877597760776177627763776477657766776777687769777077717772777377747775777677777778777977807781778277837784778577867787778877897790779177927793779477957796779777987799780078017802780378047805780678077808780978107811781278137814781578167817781878197820782178227823782478257826782778287829783078317832783378347835783678377838783978407841784278437844784578467847784878497850785178527853785478557856785778587859786078617862786378647865786678677868786978707871787278737874787578767877787878797880788178827883788478857886788778887889789078917892789378947895789678977898789979007901790279037904790579067907790879097910791179127913791479157916791779187919792079217922792379247925792679277928792979307931793279337934793579367937793879397940794179427943794479457946794779487949795079517952795379547955795679577958795979607961796279637964796579667967796879697970797179727973797479757976797779787979798079817982798379847985798679877988798979907991799279937994799579967997799879998000800180028003800480058006800780088009801080118012801380148015801680178018801980208021802280238024802580268027802880298030803180328033803480358036803780388039804080418042804380448045804680478048804980508051805280538054805580568057805880598060806180628063806480658066806780688069807080718072807380748075807680778078807980808081808280838084808580868087808880898090809180928093809480958096809780988099810081018102810381048105810681078108810981108111811281138114811581168117811881198120812181228123812481258126812781288129813081318132813381348135813681378138813981408141814281438144814581468147814881498150815181528153815481558156815781588159816081618162816381648165816681678168816981708171817281738174817581768177817881798180818181828183818481858186818781888189819081918192819381948195819681978198819982008201820282038204820582068207820882098210821182128213821482158216821782188219822082218222822382248225822682278228822982308231823282338234823582368237823882398240824182428243824482458246824782488249825082518252825382548255825682578258825982608261826282638264826582668267826882698270827182728273827482758276827782788279828082818282828382848285828682878288828982908291829282938294829582968297829882998300830183028303830483058306830783088309831083118312831383148315831683178318831983208321832283238324832583268327832883298330833183328333833483358336833783388339834083418342834383448345834683478348834983508351835283538354835583568357835883598360836183628363836483658366836783688369837083718372837383748375837683778378837983808381838283838384838583868387838883898390839183928393839483958396839783988399840084018402840384048405840684078408840984108411841284138414841584168417841884198420842184228423842484258426842784288429843084318432843384348435843684378438843984408441844284438444844584468447844884498450845184528453845484558456845784588459846084618462846384648465846684678468846984708471847284738474847584768477847884798480848184828483848484858486848784888489849084918492849384948495849684978498849985008501850285038504850585068507850885098510851185128513851485158516851785188519852085218522852385248525852685278528852985308531853285338534853585368537853885398540854185428543854485458546854785488549855085518552855385548555855685578558855985608561856285638564856585668567856885698570857185728573857485758576857785788579858085818582858385848585858685878588858985908591859285938594859585968597859885998600860186028603860486058606860786088609861086118612861386148615861686178618861986208621862286238624862586268627862886298630863186328633863486358636863786388639864086418642864386448645864686478648864986508651865286538654865586568657865886598660866186628663866486658666866786688669867086718672867386748675867686778678867986808681868286838684868586868687868886898690869186928693869486958696869786988699870087018702870387048705870687078708870987108711871287138714871587168717871887198720872187228723872487258726872787288729873087318732873387348735873687378738873987408741874287438744874587468747874887498750875187528753875487558756875787588759876087618762876387648765876687678768876987708771877287738774877587768777877887798780878187828783878487858786878787888789879087918792879387948795879687978798879988008801880288038804880588068807880888098810881188128813881488158816881788188819882088218822882388248825882688278828882988308831883288338834883588368837883888398840884188428843884488458846884788488849885088518852885388548855885688578858885988608861886288638864886588668867886888698870887188728873887488758876887788788879888088818882888388848885888688878888888988908891889288938894889588968897889888998900890189028903890489058906890789088909891089118912891389148915891689178918891989208921892289238924892589268927892889298930893189328933893489358936893789388939894089418942894389448945894689478948894989508951895289538954895589568957895889598960896189628963896489658966896789688969897089718972897389748975897689778978897989808981898289838984898589868987898889898990899189928993899489958996899789988999900090019002900390049005900690079008900990109011901290139014901590169017901890199020902190229023902490259026902790289029903090319032903390349035903690379038903990409041904290439044904590469047904890499050905190529053905490559056905790589059906090619062906390649065906690679068906990709071907290739074907590769077907890799080908190829083908490859086908790889089909090919092909390949095909690979098909991009101910291039104910591069107910891099110911191129113911491159116911791189119912091219122912391249125912691279128912991309131913291339134913591369137913891399140914191429143914491459146914791489149915091519152915391549155915691579158915991609161916291639164916591669167916891699170917191729173917491759176917791789179918091819182918391849185918691879188918991909191919291939194919591969197919891999200920192029203920492059206920792089209921092119212921392149215921692179218921992209221922292239224922592269227922892299230923192329233923492359236923792389239924092419242924392449245924692479248924992509251925292539254925592569257925892599260926192629263926492659266926792689269927092719272927392749275927692779278927992809281928292839284928592869287928892899290929192929293929492959296929792989299930093019302930393049305930693079308930993109311931293139314931593169317931893199320932193229323932493259326932793289329933093319332933393349335933693379338933993409341934293439344934593469347934893499350935193529353935493559356935793589359936093619362936393649365936693679368936993709371937293739374937593769377937893799380938193829383938493859386938793889389939093919392939393949395939693979398939994009401940294039404940594069407940894099410941194129413941494159416941794189419942094219422942394249425942694279428942994309431943294339434943594369437943894399440944194429443944494459446944794489449945094519452945394549455945694579458945994609461946294639464946594669467946894699470947194729473947494759476947794789479948094819482948394849485948694879488948994909491949294939494949594969497949894999500950195029503950495059506950795089509951095119512951395149515951695179518951995209521952295239524952595269527952895299530953195329533953495359536953795389539954095419542954395449545954695479548954995509551955295539554955595569557955895599560956195629563956495659566956795689569957095719572957395749575957695779578957995809581958295839584958595869587958895899590959195929593959495959596959795989599960096019602960396049605960696079608960996109611961296139614961596169617961896199620962196229623962496259626962796289629963096319632963396349635963696379638963996409641964296439644964596469647964896499650965196529653965496559656965796589659966096619662966396649665966696679668966996709671967296739674967596769677967896799680968196829683968496859686968796889689969096919692969396949695969696979698969997009701970297039704970597069707970897099710971197129713971497159716971797189719972097219722972397249725972697279728972997309731973297339734973597369737973897399740974197429743974497459746974797489749975097519752975397549755975697579758975997609761976297639764976597669767976897699770977197729773977497759776977797789779978097819782978397849785978697879788978997909791979297939794979597969797979897999800980198029803980498059806980798089809981098119812981398149815981698179818981998209821982298239824982598269827982898299830983198329833983498359836983798389839984098419842984398449845984698479848984998509851985298539854985598569857985898599860986198629863986498659866986798689869987098719872987398749875987698779878987998809881988298839884988598869887988898899890989198929893989498959896989798989899990099019902990399049905990699079908990999109911991299139914991599169917991899199920992199229923992499259926992799289929993099319932993399349935993699379938993999409941994299439944994599469947994899499950995199529953995499559956995799589959996099619962996399649965996699679968996999709971997299739974997599769977997899799980998199829983998499859986998799889989999099919992999399949995999699979998999910000100011000210003100041000510006100071000810009100101001110012100131001410015100161001710018100191002010021100221002310024100251002610027100281002910030100311003210033100341003510036100371003810039100401004110042100431004410045100461004710048100491005010051100521005310054100551005610057100581005910060100611006210063100641006510066100671006810069100701007110072100731007410075100761007710078100791008010081100821008310084100851008610087100881008910090100911009210093100941009510096100971009810099101001010110102101031010410105101061010710108101091011010111101121011310114101151011610117101181011910120101211012210123101241012510126101271012810129101301013110132101331013410135101361013710138101391014010141101421014310144101451014610147101481014910150101511015210153101541015510156101571015810159101601016110162101631016410165101661016710168101691017010171101721017310174101751017610177101781017910180101811018210183101841018510186101871018810189101901019110192101931019410195101961019710198101991020010201102021020310204102051020610207102081020910210102111021210213102141021510216102171021810219102201022110222102231022410225102261022710228102291023010231102321023310234102351023610237102381023910240102411024210243102441024510246102471024810249102501025110252102531025410255102561025710258102591026010261102621026310264102651026610267102681026910270102711027210273102741027510276102771027810279102801028110282102831028410285102861028710288102891029010291102921029310294102951029610297102981029910300103011030210303103041030510306103071030810309103101031110312103131031410315103161031710318103191032010321103221032310324103251032610327103281032910330103311033210333103341033510336103371033810339103401034110342103431034410345103461034710348103491035010351103521035310354103551035610357103581035910360103611036210363103641036510366103671036810369103701037110372103731037410375103761037710378103791038010381103821038310384103851038610387103881038910390103911039210393103941039510396103971039810399104001040110402104031040410405104061040710408104091041010411104121041310414104151041610417104181041910420104211042210423104241042510426104271042810429104301043110432104331043410435104361043710438104391044010441104421044310444104451044610447104481044910450104511045210453104541045510456104571045810459104601046110462104631046410465104661046710468104691047010471104721047310474104751047610477104781047910480104811048210483104841048510486104871048810489104901049110492104931049410495104961049710498104991050010501105021050310504105051050610507105081050910510105111051210513105141051510516105171051810519105201052110522105231052410525105261052710528105291053010531105321053310534105351053610537105381053910540105411054210543105441054510546105471054810549105501055110552105531055410555105561055710558105591056010561105621056310564105651056610567105681056910570105711057210573105741057510576105771057810579105801058110582105831058410585105861058710588105891059010591105921059310594105951059610597105981059910600106011060210603106041060510606106071060810609106101061110612106131061410615106161061710618106191062010621106221062310624106251062610627106281062910630106311063210633106341063510636106371063810639106401064110642106431064410645106461064710648106491065010651106521065310654106551065610657106581065910660106611066210663106641066510666106671066810669106701067110672106731067410675106761067710678106791068010681106821068310684106851068610687106881068910690106911069210693106941069510696106971069810699107001070110702107031070410705107061070710708107091071010711107121071310714107151071610717107181071910720107211072210723107241072510726107271072810729107301073110732107331073410735107361073710738107391074010741107421074310744107451074610747107481074910750107511075210753107541075510756107571075810759107601076110762107631076410765107661076710768107691077010771107721077310774107751077610777107781077910780107811078210783107841078510786107871078810789107901079110792107931079410795107961079710798107991080010801108021080310804108051080610807108081080910810108111081210813108141081510816108171081810819108201082110822108231082410825108261082710828108291083010831108321083310834108351083610837108381083910840108411084210843108441084510846108471084810849108501085110852108531085410855108561085710858108591086010861108621086310864108651086610867108681086910870108711087210873108741087510876108771087810879108801088110882108831088410885108861088710888108891089010891108921089310894108951089610897108981089910900109011090210903109041090510906109071090810909109101091110912109131091410915109161091710918109191092010921109221092310924109251092610927109281092910930109311093210933109341093510936109371093810939109401094110942109431094410945109461094710948109491095010951109521095310954109551095610957109581095910960109611096210963109641096510966109671096810969109701097110972109731097410975109761097710978109791098010981109821098310984109851098610987109881098910990109911099210993109941099510996109971099810999110001100111002110031100411005110061100711008110091101011011110121101311014110151101611017110181101911020110211102211023110241102511026110271102811029110301103111032110331103411035110361103711038110391104011041110421104311044110451104611047110481104911050110511105211053110541105511056110571105811059110601106111062110631106411065110661106711068110691107011071110721107311074110751107611077110781107911080110811108211083110841108511086110871108811089110901109111092110931109411095110961109711098110991110011101111021110311104111051110611107111081110911110111111111211113111141111511116111171111811119111201112111122111231112411125111261112711128111291113011131111321113311134111351113611137111381113911140111411114211143111441114511146111471114811149111501115111152111531115411155111561115711158111591116011161111621116311164111651116611167111681116911170111711117211173111741117511176111771117811179111801118111182111831118411185111861118711188111891119011191111921119311194111951119611197111981119911200112011120211203112041120511206112071120811209112101121111212112131121411215112161121711218112191122011221112221122311224112251122611227112281122911230112311123211233112341123511236112371123811239112401124111242112431124411245112461124711248112491125011251112521125311254112551125611257112581125911260112611126211263112641126511266112671126811269112701127111272112731127411275112761127711278112791128011281112821128311284112851128611287112881128911290112911129211293112941129511296112971129811299113001130111302113031130411305113061130711308113091131011311113121131311314113151131611317113181131911320113211132211323113241132511326113271132811329113301133111332113331133411335113361133711338113391134011341113421134311344113451134611347113481134911350113511135211353113541135511356113571135811359113601136111362113631136411365113661136711368113691137011371113721137311374113751137611377113781137911380113811138211383113841138511386113871138811389113901139111392113931139411395113961139711398113991140011401114021140311404114051140611407114081140911410114111141211413114141141511416114171141811419114201142111422114231142411425114261142711428114291143011431114321143311434114351143611437114381143911440114411144211443114441144511446114471144811449114501145111452114531145411455114561145711458114591146011461114621146311464114651146611467114681146911470114711147211473114741147511476114771147811479114801148111482114831148411485114861148711488114891149011491114921149311494114951149611497114981149911500115011150211503115041150511506115071150811509115101151111512115131151411515115161151711518115191152011521115221152311524115251152611527115281152911530115311153211533115341153511536115371153811539115401154111542115431154411545115461154711548115491155011551115521155311554115551155611557115581155911560115611156211563115641156511566115671156811569115701157111572115731157411575115761157711578115791158011581115821158311584115851158611587115881158911590115911159211593115941159511596115971159811599116001160111602116031160411605116061160711608116091161011611116121161311614116151161611617116181161911620116211162211623116241162511626116271162811629116301163111632116331163411635116361163711638116391164011641116421164311644116451164611647116481164911650116511165211653116541165511656
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * This module enables machines with Intel VT-x extensions to run virtual
  5. * machines without emulation or binary translation.
  6. *
  7. * Copyright (C) 2006 Qumranet, Inc.
  8. * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  9. *
  10. * Authors:
  11. * Avi Kivity <avi@qumranet.com>
  12. * Yaniv Kamay <yaniv@qumranet.com>
  13. *
  14. * This work is licensed under the terms of the GNU GPL, version 2. See
  15. * the COPYING file in the top-level directory.
  16. *
  17. */
  18. #include "irq.h"
  19. #include "mmu.h"
  20. #include "cpuid.h"
  21. #include "lapic.h"
  22. #include <linux/kvm_host.h>
  23. #include <linux/module.h>
  24. #include <linux/kernel.h>
  25. #include <linux/mm.h>
  26. #include <linux/highmem.h>
  27. #include <linux/sched.h>
  28. #include <linux/moduleparam.h>
  29. #include <linux/mod_devicetable.h>
  30. #include <linux/trace_events.h>
  31. #include <linux/slab.h>
  32. #include <linux/tboot.h>
  33. #include <linux/hrtimer.h>
  34. #include "kvm_cache_regs.h"
  35. #include "x86.h"
  36. #include <asm/cpu.h>
  37. #include <asm/io.h>
  38. #include <asm/desc.h>
  39. #include <asm/vmx.h>
  40. #include <asm/virtext.h>
  41. #include <asm/mce.h>
  42. #include <asm/fpu/internal.h>
  43. #include <asm/perf_event.h>
  44. #include <asm/debugreg.h>
  45. #include <asm/kexec.h>
  46. #include <asm/apic.h>
  47. #include <asm/irq_remapping.h>
  48. #include "trace.h"
  49. #include "pmu.h"
  50. #define __ex(x) __kvm_handle_fault_on_reboot(x)
  51. #define __ex_clear(x, reg) \
  52. ____kvm_handle_fault_on_reboot(x, "xor " reg " , " reg)
  53. MODULE_AUTHOR("Qumranet");
  54. MODULE_LICENSE("GPL");
  55. static const struct x86_cpu_id vmx_cpu_id[] = {
  56. X86_FEATURE_MATCH(X86_FEATURE_VMX),
  57. {}
  58. };
  59. MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id);
  60. static bool __read_mostly enable_vpid = 1;
  61. module_param_named(vpid, enable_vpid, bool, 0444);
  62. static bool __read_mostly flexpriority_enabled = 1;
  63. module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
  64. static bool __read_mostly enable_ept = 1;
  65. module_param_named(ept, enable_ept, bool, S_IRUGO);
  66. static bool __read_mostly enable_unrestricted_guest = 1;
  67. module_param_named(unrestricted_guest,
  68. enable_unrestricted_guest, bool, S_IRUGO);
  69. static bool __read_mostly enable_ept_ad_bits = 1;
  70. module_param_named(eptad, enable_ept_ad_bits, bool, S_IRUGO);
  71. static bool __read_mostly emulate_invalid_guest_state = true;
  72. module_param(emulate_invalid_guest_state, bool, S_IRUGO);
  73. static bool __read_mostly vmm_exclusive = 1;
  74. module_param(vmm_exclusive, bool, S_IRUGO);
  75. static bool __read_mostly fasteoi = 1;
  76. module_param(fasteoi, bool, S_IRUGO);
  77. static bool __read_mostly enable_apicv = 1;
  78. module_param(enable_apicv, bool, S_IRUGO);
  79. static bool __read_mostly enable_shadow_vmcs = 1;
  80. module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);
  81. /*
  82. * If nested=1, nested virtualization is supported, i.e., guests may use
  83. * VMX and be a hypervisor for its own guests. If nested=0, guests may not
  84. * use VMX instructions.
  85. */
  86. static bool __read_mostly nested = 0;
  87. module_param(nested, bool, S_IRUGO);
  88. static u64 __read_mostly host_xss;
  89. static bool __read_mostly enable_pml = 1;
  90. module_param_named(pml, enable_pml, bool, S_IRUGO);
  91. #define KVM_VMX_TSC_MULTIPLIER_MAX 0xffffffffffffffffULL
  92. /* Guest_tsc -> host_tsc conversion requires 64-bit division. */
  93. static int __read_mostly cpu_preemption_timer_multi;
  94. static bool __read_mostly enable_preemption_timer = 1;
  95. #ifdef CONFIG_X86_64
  96. module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO);
  97. #endif
  98. #define KVM_GUEST_CR0_MASK (X86_CR0_NW | X86_CR0_CD)
  99. #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST (X86_CR0_WP | X86_CR0_NE)
  100. #define KVM_VM_CR0_ALWAYS_ON \
  101. (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
  102. #define KVM_CR4_GUEST_OWNED_BITS \
  103. (X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR \
  104. | X86_CR4_OSXMMEXCPT | X86_CR4_TSD)
  105. #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
  106. #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)
  107. #define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM))
  108. #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5
  109. /*
  110. * Hyper-V requires all of these, so mark them as supported even though
  111. * they are just treated the same as all-context.
  112. */
  113. #define VMX_VPID_EXTENT_SUPPORTED_MASK \
  114. (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \
  115. VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \
  116. VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \
  117. VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT)
  118. /*
  119. * These 2 parameters are used to config the controls for Pause-Loop Exiting:
  120. * ple_gap: upper bound on the amount of time between two successive
  121. * executions of PAUSE in a loop. Also indicate if ple enabled.
  122. * According to test, this time is usually smaller than 128 cycles.
  123. * ple_window: upper bound on the amount of time a guest is allowed to execute
  124. * in a PAUSE loop. Tests indicate that most spinlocks are held for
  125. * less than 2^12 cycles
  126. * Time is measured based on a counter that runs at the same rate as the TSC,
  127. * refer SDM volume 3b section 21.6.13 & 22.1.3.
  128. */
  129. #define KVM_VMX_DEFAULT_PLE_GAP 128
  130. #define KVM_VMX_DEFAULT_PLE_WINDOW 4096
  131. #define KVM_VMX_DEFAULT_PLE_WINDOW_GROW 2
  132. #define KVM_VMX_DEFAULT_PLE_WINDOW_SHRINK 0
  133. #define KVM_VMX_DEFAULT_PLE_WINDOW_MAX \
  134. INT_MAX / KVM_VMX_DEFAULT_PLE_WINDOW_GROW
  135. static int ple_gap = KVM_VMX_DEFAULT_PLE_GAP;
  136. module_param(ple_gap, int, S_IRUGO);
  137. static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
  138. module_param(ple_window, int, S_IRUGO);
  139. /* Default doubles per-vcpu window every exit. */
  140. static int ple_window_grow = KVM_VMX_DEFAULT_PLE_WINDOW_GROW;
  141. module_param(ple_window_grow, int, S_IRUGO);
  142. /* Default resets per-vcpu window every exit to ple_window. */
  143. static int ple_window_shrink = KVM_VMX_DEFAULT_PLE_WINDOW_SHRINK;
  144. module_param(ple_window_shrink, int, S_IRUGO);
  145. /* Default is to compute the maximum so we can never overflow. */
  146. static int ple_window_actual_max = KVM_VMX_DEFAULT_PLE_WINDOW_MAX;
  147. static int ple_window_max = KVM_VMX_DEFAULT_PLE_WINDOW_MAX;
  148. module_param(ple_window_max, int, S_IRUGO);
  149. extern const ulong vmx_return;
  150. #define NR_AUTOLOAD_MSRS 8
  151. #define VMCS02_POOL_SIZE 1
  152. struct vmcs {
  153. u32 revision_id;
  154. u32 abort;
  155. char data[0];
  156. };
  157. /*
  158. * Track a VMCS that may be loaded on a certain CPU. If it is (cpu!=-1), also
  159. * remember whether it was VMLAUNCHed, and maintain a linked list of all VMCSs
  160. * loaded on this CPU (so we can clear them if the CPU goes down).
  161. */
  162. struct loaded_vmcs {
  163. struct vmcs *vmcs;
  164. struct vmcs *shadow_vmcs;
  165. int cpu;
  166. int launched;
  167. struct list_head loaded_vmcss_on_cpu_link;
  168. };
  169. struct shared_msr_entry {
  170. unsigned index;
  171. u64 data;
  172. u64 mask;
  173. };
  174. /*
  175. * struct vmcs12 describes the state that our guest hypervisor (L1) keeps for a
  176. * single nested guest (L2), hence the name vmcs12. Any VMX implementation has
  177. * a VMCS structure, and vmcs12 is our emulated VMX's VMCS. This structure is
  178. * stored in guest memory specified by VMPTRLD, but is opaque to the guest,
  179. * which must access it using VMREAD/VMWRITE/VMCLEAR instructions.
  180. * More than one of these structures may exist, if L1 runs multiple L2 guests.
  181. * nested_vmx_run() will use the data here to build a vmcs02: a VMCS for the
  182. * underlying hardware which will be used to run L2.
  183. * This structure is packed to ensure that its layout is identical across
  184. * machines (necessary for live migration).
  185. * If there are changes in this struct, VMCS12_REVISION must be changed.
  186. */
  187. typedef u64 natural_width;
  188. struct __packed vmcs12 {
  189. /* According to the Intel spec, a VMCS region must start with the
  190. * following two fields. Then follow implementation-specific data.
  191. */
  192. u32 revision_id;
  193. u32 abort;
  194. u32 launch_state; /* set to 0 by VMCLEAR, to 1 by VMLAUNCH */
  195. u32 padding[7]; /* room for future expansion */
  196. u64 io_bitmap_a;
  197. u64 io_bitmap_b;
  198. u64 msr_bitmap;
  199. u64 vm_exit_msr_store_addr;
  200. u64 vm_exit_msr_load_addr;
  201. u64 vm_entry_msr_load_addr;
  202. u64 tsc_offset;
  203. u64 virtual_apic_page_addr;
  204. u64 apic_access_addr;
  205. u64 posted_intr_desc_addr;
  206. u64 ept_pointer;
  207. u64 eoi_exit_bitmap0;
  208. u64 eoi_exit_bitmap1;
  209. u64 eoi_exit_bitmap2;
  210. u64 eoi_exit_bitmap3;
  211. u64 xss_exit_bitmap;
  212. u64 guest_physical_address;
  213. u64 vmcs_link_pointer;
  214. u64 guest_ia32_debugctl;
  215. u64 guest_ia32_pat;
  216. u64 guest_ia32_efer;
  217. u64 guest_ia32_perf_global_ctrl;
  218. u64 guest_pdptr0;
  219. u64 guest_pdptr1;
  220. u64 guest_pdptr2;
  221. u64 guest_pdptr3;
  222. u64 guest_bndcfgs;
  223. u64 host_ia32_pat;
  224. u64 host_ia32_efer;
  225. u64 host_ia32_perf_global_ctrl;
  226. u64 padding64[8]; /* room for future expansion */
  227. /*
  228. * To allow migration of L1 (complete with its L2 guests) between
  229. * machines of different natural widths (32 or 64 bit), we cannot have
  230. * unsigned long fields with no explict size. We use u64 (aliased
  231. * natural_width) instead. Luckily, x86 is little-endian.
  232. */
  233. natural_width cr0_guest_host_mask;
  234. natural_width cr4_guest_host_mask;
  235. natural_width cr0_read_shadow;
  236. natural_width cr4_read_shadow;
  237. natural_width cr3_target_value0;
  238. natural_width cr3_target_value1;
  239. natural_width cr3_target_value2;
  240. natural_width cr3_target_value3;
  241. natural_width exit_qualification;
  242. natural_width guest_linear_address;
  243. natural_width guest_cr0;
  244. natural_width guest_cr3;
  245. natural_width guest_cr4;
  246. natural_width guest_es_base;
  247. natural_width guest_cs_base;
  248. natural_width guest_ss_base;
  249. natural_width guest_ds_base;
  250. natural_width guest_fs_base;
  251. natural_width guest_gs_base;
  252. natural_width guest_ldtr_base;
  253. natural_width guest_tr_base;
  254. natural_width guest_gdtr_base;
  255. natural_width guest_idtr_base;
  256. natural_width guest_dr7;
  257. natural_width guest_rsp;
  258. natural_width guest_rip;
  259. natural_width guest_rflags;
  260. natural_width guest_pending_dbg_exceptions;
  261. natural_width guest_sysenter_esp;
  262. natural_width guest_sysenter_eip;
  263. natural_width host_cr0;
  264. natural_width host_cr3;
  265. natural_width host_cr4;
  266. natural_width host_fs_base;
  267. natural_width host_gs_base;
  268. natural_width host_tr_base;
  269. natural_width host_gdtr_base;
  270. natural_width host_idtr_base;
  271. natural_width host_ia32_sysenter_esp;
  272. natural_width host_ia32_sysenter_eip;
  273. natural_width host_rsp;
  274. natural_width host_rip;
  275. natural_width paddingl[8]; /* room for future expansion */
  276. u32 pin_based_vm_exec_control;
  277. u32 cpu_based_vm_exec_control;
  278. u32 exception_bitmap;
  279. u32 page_fault_error_code_mask;
  280. u32 page_fault_error_code_match;
  281. u32 cr3_target_count;
  282. u32 vm_exit_controls;
  283. u32 vm_exit_msr_store_count;
  284. u32 vm_exit_msr_load_count;
  285. u32 vm_entry_controls;
  286. u32 vm_entry_msr_load_count;
  287. u32 vm_entry_intr_info_field;
  288. u32 vm_entry_exception_error_code;
  289. u32 vm_entry_instruction_len;
  290. u32 tpr_threshold;
  291. u32 secondary_vm_exec_control;
  292. u32 vm_instruction_error;
  293. u32 vm_exit_reason;
  294. u32 vm_exit_intr_info;
  295. u32 vm_exit_intr_error_code;
  296. u32 idt_vectoring_info_field;
  297. u32 idt_vectoring_error_code;
  298. u32 vm_exit_instruction_len;
  299. u32 vmx_instruction_info;
  300. u32 guest_es_limit;
  301. u32 guest_cs_limit;
  302. u32 guest_ss_limit;
  303. u32 guest_ds_limit;
  304. u32 guest_fs_limit;
  305. u32 guest_gs_limit;
  306. u32 guest_ldtr_limit;
  307. u32 guest_tr_limit;
  308. u32 guest_gdtr_limit;
  309. u32 guest_idtr_limit;
  310. u32 guest_es_ar_bytes;
  311. u32 guest_cs_ar_bytes;
  312. u32 guest_ss_ar_bytes;
  313. u32 guest_ds_ar_bytes;
  314. u32 guest_fs_ar_bytes;
  315. u32 guest_gs_ar_bytes;
  316. u32 guest_ldtr_ar_bytes;
  317. u32 guest_tr_ar_bytes;
  318. u32 guest_interruptibility_info;
  319. u32 guest_activity_state;
  320. u32 guest_sysenter_cs;
  321. u32 host_ia32_sysenter_cs;
  322. u32 vmx_preemption_timer_value;
  323. u32 padding32[7]; /* room for future expansion */
  324. u16 virtual_processor_id;
  325. u16 posted_intr_nv;
  326. u16 guest_es_selector;
  327. u16 guest_cs_selector;
  328. u16 guest_ss_selector;
  329. u16 guest_ds_selector;
  330. u16 guest_fs_selector;
  331. u16 guest_gs_selector;
  332. u16 guest_ldtr_selector;
  333. u16 guest_tr_selector;
  334. u16 guest_intr_status;
  335. u16 host_es_selector;
  336. u16 host_cs_selector;
  337. u16 host_ss_selector;
  338. u16 host_ds_selector;
  339. u16 host_fs_selector;
  340. u16 host_gs_selector;
  341. u16 host_tr_selector;
  342. };
  343. /*
  344. * VMCS12_REVISION is an arbitrary id that should be changed if the content or
  345. * layout of struct vmcs12 is changed. MSR_IA32_VMX_BASIC returns this id, and
  346. * VMPTRLD verifies that the VMCS region that L1 is loading contains this id.
  347. */
  348. #define VMCS12_REVISION 0x11e57ed0
  349. /*
  350. * VMCS12_SIZE is the number of bytes L1 should allocate for the VMXON region
  351. * and any VMCS region. Although only sizeof(struct vmcs12) are used by the
  352. * current implementation, 4K are reserved to avoid future complications.
  353. */
  354. #define VMCS12_SIZE 0x1000
  355. /* Used to remember the last vmcs02 used for some recently used vmcs12s */
  356. struct vmcs02_list {
  357. struct list_head list;
  358. gpa_t vmptr;
  359. struct loaded_vmcs vmcs02;
  360. };
  361. /*
  362. * The nested_vmx structure is part of vcpu_vmx, and holds information we need
  363. * for correct emulation of VMX (i.e., nested VMX) on this vcpu.
  364. */
  365. struct nested_vmx {
  366. /* Has the level1 guest done vmxon? */
  367. bool vmxon;
  368. gpa_t vmxon_ptr;
  369. /* The guest-physical address of the current VMCS L1 keeps for L2 */
  370. gpa_t current_vmptr;
  371. /* The host-usable pointer to the above */
  372. struct page *current_vmcs12_page;
  373. struct vmcs12 *current_vmcs12;
  374. /*
  375. * Cache of the guest's VMCS, existing outside of guest memory.
  376. * Loaded from guest memory during VMPTRLD. Flushed to guest
  377. * memory during VMXOFF, VMCLEAR, VMPTRLD.
  378. */
  379. struct vmcs12 *cached_vmcs12;
  380. /*
  381. * Indicates if the shadow vmcs must be updated with the
  382. * data hold by vmcs12
  383. */
  384. bool sync_shadow_vmcs;
  385. /* vmcs02_list cache of VMCSs recently used to run L2 guests */
  386. struct list_head vmcs02_pool;
  387. int vmcs02_num;
  388. bool change_vmcs01_virtual_x2apic_mode;
  389. /* L2 must run next, and mustn't decide to exit to L1. */
  390. bool nested_run_pending;
  391. /*
  392. * Guest pages referred to in vmcs02 with host-physical pointers, so
  393. * we must keep them pinned while L2 runs.
  394. */
  395. struct page *apic_access_page;
  396. struct page *virtual_apic_page;
  397. struct page *pi_desc_page;
  398. struct pi_desc *pi_desc;
  399. bool pi_pending;
  400. u16 posted_intr_nv;
  401. unsigned long *msr_bitmap;
  402. struct hrtimer preemption_timer;
  403. bool preemption_timer_expired;
  404. /* to migrate it to L2 if VM_ENTRY_LOAD_DEBUG_CONTROLS is off */
  405. u64 vmcs01_debugctl;
  406. u16 vpid02;
  407. u16 last_vpid;
  408. /*
  409. * We only store the "true" versions of the VMX capability MSRs. We
  410. * generate the "non-true" versions by setting the must-be-1 bits
  411. * according to the SDM.
  412. */
  413. u32 nested_vmx_procbased_ctls_low;
  414. u32 nested_vmx_procbased_ctls_high;
  415. u32 nested_vmx_secondary_ctls_low;
  416. u32 nested_vmx_secondary_ctls_high;
  417. u32 nested_vmx_pinbased_ctls_low;
  418. u32 nested_vmx_pinbased_ctls_high;
  419. u32 nested_vmx_exit_ctls_low;
  420. u32 nested_vmx_exit_ctls_high;
  421. u32 nested_vmx_entry_ctls_low;
  422. u32 nested_vmx_entry_ctls_high;
  423. u32 nested_vmx_misc_low;
  424. u32 nested_vmx_misc_high;
  425. u32 nested_vmx_ept_caps;
  426. u32 nested_vmx_vpid_caps;
  427. u64 nested_vmx_basic;
  428. u64 nested_vmx_cr0_fixed0;
  429. u64 nested_vmx_cr0_fixed1;
  430. u64 nested_vmx_cr4_fixed0;
  431. u64 nested_vmx_cr4_fixed1;
  432. u64 nested_vmx_vmcs_enum;
  433. };
  434. #define POSTED_INTR_ON 0
  435. #define POSTED_INTR_SN 1
  436. /* Posted-Interrupt Descriptor */
  437. struct pi_desc {
  438. u32 pir[8]; /* Posted interrupt requested */
  439. union {
  440. struct {
  441. /* bit 256 - Outstanding Notification */
  442. u16 on : 1,
  443. /* bit 257 - Suppress Notification */
  444. sn : 1,
  445. /* bit 271:258 - Reserved */
  446. rsvd_1 : 14;
  447. /* bit 279:272 - Notification Vector */
  448. u8 nv;
  449. /* bit 287:280 - Reserved */
  450. u8 rsvd_2;
  451. /* bit 319:288 - Notification Destination */
  452. u32 ndst;
  453. };
  454. u64 control;
  455. };
  456. u32 rsvd[6];
  457. } __aligned(64);
  458. static bool pi_test_and_set_on(struct pi_desc *pi_desc)
  459. {
  460. return test_and_set_bit(POSTED_INTR_ON,
  461. (unsigned long *)&pi_desc->control);
  462. }
  463. static bool pi_test_and_clear_on(struct pi_desc *pi_desc)
  464. {
  465. return test_and_clear_bit(POSTED_INTR_ON,
  466. (unsigned long *)&pi_desc->control);
  467. }
  468. static int pi_test_and_set_pir(int vector, struct pi_desc *pi_desc)
  469. {
  470. return test_and_set_bit(vector, (unsigned long *)pi_desc->pir);
  471. }
  472. static inline void pi_clear_sn(struct pi_desc *pi_desc)
  473. {
  474. return clear_bit(POSTED_INTR_SN,
  475. (unsigned long *)&pi_desc->control);
  476. }
  477. static inline void pi_set_sn(struct pi_desc *pi_desc)
  478. {
  479. return set_bit(POSTED_INTR_SN,
  480. (unsigned long *)&pi_desc->control);
  481. }
  482. static inline void pi_clear_on(struct pi_desc *pi_desc)
  483. {
  484. clear_bit(POSTED_INTR_ON,
  485. (unsigned long *)&pi_desc->control);
  486. }
  487. static inline int pi_test_on(struct pi_desc *pi_desc)
  488. {
  489. return test_bit(POSTED_INTR_ON,
  490. (unsigned long *)&pi_desc->control);
  491. }
  492. static inline int pi_test_sn(struct pi_desc *pi_desc)
  493. {
  494. return test_bit(POSTED_INTR_SN,
  495. (unsigned long *)&pi_desc->control);
  496. }
  497. struct vcpu_vmx {
  498. struct kvm_vcpu vcpu;
  499. unsigned long host_rsp;
  500. u8 fail;
  501. bool nmi_known_unmasked;
  502. u32 exit_intr_info;
  503. u32 idt_vectoring_info;
  504. ulong rflags;
  505. struct shared_msr_entry *guest_msrs;
  506. int nmsrs;
  507. int save_nmsrs;
  508. unsigned long host_idt_base;
  509. #ifdef CONFIG_X86_64
  510. u64 msr_host_kernel_gs_base;
  511. u64 msr_guest_kernel_gs_base;
  512. #endif
  513. u32 vm_entry_controls_shadow;
  514. u32 vm_exit_controls_shadow;
  515. /*
  516. * loaded_vmcs points to the VMCS currently used in this vcpu. For a
  517. * non-nested (L1) guest, it always points to vmcs01. For a nested
  518. * guest (L2), it points to a different VMCS.
  519. */
  520. struct loaded_vmcs vmcs01;
  521. struct loaded_vmcs *loaded_vmcs;
  522. bool __launched; /* temporary, used in vmx_vcpu_run */
  523. struct msr_autoload {
  524. unsigned nr;
  525. struct vmx_msr_entry guest[NR_AUTOLOAD_MSRS];
  526. struct vmx_msr_entry host[NR_AUTOLOAD_MSRS];
  527. } msr_autoload;
  528. struct {
  529. int loaded;
  530. u16 fs_sel, gs_sel, ldt_sel;
  531. #ifdef CONFIG_X86_64
  532. u16 ds_sel, es_sel;
  533. #endif
  534. int gs_ldt_reload_needed;
  535. int fs_reload_needed;
  536. u64 msr_host_bndcfgs;
  537. unsigned long vmcs_host_cr4; /* May not match real cr4 */
  538. } host_state;
  539. struct {
  540. int vm86_active;
  541. ulong save_rflags;
  542. struct kvm_segment segs[8];
  543. } rmode;
  544. struct {
  545. u32 bitmask; /* 4 bits per segment (1 bit per field) */
  546. struct kvm_save_segment {
  547. u16 selector;
  548. unsigned long base;
  549. u32 limit;
  550. u32 ar;
  551. } seg[8];
  552. } segment_cache;
  553. int vpid;
  554. bool emulation_required;
  555. /* Support for vnmi-less CPUs */
  556. int soft_vnmi_blocked;
  557. ktime_t entry_time;
  558. s64 vnmi_blocked_time;
  559. u32 exit_reason;
  560. /* Posted interrupt descriptor */
  561. struct pi_desc pi_desc;
  562. /* Support for a guest hypervisor (nested VMX) */
  563. struct nested_vmx nested;
  564. /* Dynamic PLE window. */
  565. int ple_window;
  566. bool ple_window_dirty;
  567. /* Support for PML */
  568. #define PML_ENTITY_NUM 512
  569. struct page *pml_pg;
  570. /* apic deadline value in host tsc */
  571. u64 hv_deadline_tsc;
  572. u64 current_tsc_ratio;
  573. bool guest_pkru_valid;
  574. u32 guest_pkru;
  575. u32 host_pkru;
  576. /*
  577. * Only bits masked by msr_ia32_feature_control_valid_bits can be set in
  578. * msr_ia32_feature_control. FEATURE_CONTROL_LOCKED is always included
  579. * in msr_ia32_feature_control_valid_bits.
  580. */
  581. u64 msr_ia32_feature_control;
  582. u64 msr_ia32_feature_control_valid_bits;
  583. };
  584. enum segment_cache_field {
  585. SEG_FIELD_SEL = 0,
  586. SEG_FIELD_BASE = 1,
  587. SEG_FIELD_LIMIT = 2,
  588. SEG_FIELD_AR = 3,
  589. SEG_FIELD_NR = 4
  590. };
  591. static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
  592. {
  593. return container_of(vcpu, struct vcpu_vmx, vcpu);
  594. }
  595. static struct pi_desc *vcpu_to_pi_desc(struct kvm_vcpu *vcpu)
  596. {
  597. return &(to_vmx(vcpu)->pi_desc);
  598. }
  599. #define VMCS12_OFFSET(x) offsetof(struct vmcs12, x)
  600. #define FIELD(number, name) [number] = VMCS12_OFFSET(name)
  601. #define FIELD64(number, name) [number] = VMCS12_OFFSET(name), \
  602. [number##_HIGH] = VMCS12_OFFSET(name)+4
  603. static unsigned long shadow_read_only_fields[] = {
  604. /*
  605. * We do NOT shadow fields that are modified when L0
  606. * traps and emulates any vmx instruction (e.g. VMPTRLD,
  607. * VMXON...) executed by L1.
  608. * For example, VM_INSTRUCTION_ERROR is read
  609. * by L1 if a vmx instruction fails (part of the error path).
  610. * Note the code assumes this logic. If for some reason
  611. * we start shadowing these fields then we need to
  612. * force a shadow sync when L0 emulates vmx instructions
  613. * (e.g. force a sync if VM_INSTRUCTION_ERROR is modified
  614. * by nested_vmx_failValid)
  615. */
  616. VM_EXIT_REASON,
  617. VM_EXIT_INTR_INFO,
  618. VM_EXIT_INSTRUCTION_LEN,
  619. IDT_VECTORING_INFO_FIELD,
  620. IDT_VECTORING_ERROR_CODE,
  621. VM_EXIT_INTR_ERROR_CODE,
  622. EXIT_QUALIFICATION,
  623. GUEST_LINEAR_ADDRESS,
  624. GUEST_PHYSICAL_ADDRESS
  625. };
  626. static int max_shadow_read_only_fields =
  627. ARRAY_SIZE(shadow_read_only_fields);
  628. static unsigned long shadow_read_write_fields[] = {
  629. TPR_THRESHOLD,
  630. GUEST_RIP,
  631. GUEST_RSP,
  632. GUEST_CR0,
  633. GUEST_CR3,
  634. GUEST_CR4,
  635. GUEST_INTERRUPTIBILITY_INFO,
  636. GUEST_RFLAGS,
  637. GUEST_CS_SELECTOR,
  638. GUEST_CS_AR_BYTES,
  639. GUEST_CS_LIMIT,
  640. GUEST_CS_BASE,
  641. GUEST_ES_BASE,
  642. GUEST_BNDCFGS,
  643. CR0_GUEST_HOST_MASK,
  644. CR0_READ_SHADOW,
  645. CR4_READ_SHADOW,
  646. TSC_OFFSET,
  647. EXCEPTION_BITMAP,
  648. CPU_BASED_VM_EXEC_CONTROL,
  649. VM_ENTRY_EXCEPTION_ERROR_CODE,
  650. VM_ENTRY_INTR_INFO_FIELD,
  651. VM_ENTRY_INSTRUCTION_LEN,
  652. VM_ENTRY_EXCEPTION_ERROR_CODE,
  653. HOST_FS_BASE,
  654. HOST_GS_BASE,
  655. HOST_FS_SELECTOR,
  656. HOST_GS_SELECTOR
  657. };
  658. static int max_shadow_read_write_fields =
  659. ARRAY_SIZE(shadow_read_write_fields);
  660. static const unsigned short vmcs_field_to_offset_table[] = {
  661. FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id),
  662. FIELD(POSTED_INTR_NV, posted_intr_nv),
  663. FIELD(GUEST_ES_SELECTOR, guest_es_selector),
  664. FIELD(GUEST_CS_SELECTOR, guest_cs_selector),
  665. FIELD(GUEST_SS_SELECTOR, guest_ss_selector),
  666. FIELD(GUEST_DS_SELECTOR, guest_ds_selector),
  667. FIELD(GUEST_FS_SELECTOR, guest_fs_selector),
  668. FIELD(GUEST_GS_SELECTOR, guest_gs_selector),
  669. FIELD(GUEST_LDTR_SELECTOR, guest_ldtr_selector),
  670. FIELD(GUEST_TR_SELECTOR, guest_tr_selector),
  671. FIELD(GUEST_INTR_STATUS, guest_intr_status),
  672. FIELD(HOST_ES_SELECTOR, host_es_selector),
  673. FIELD(HOST_CS_SELECTOR, host_cs_selector),
  674. FIELD(HOST_SS_SELECTOR, host_ss_selector),
  675. FIELD(HOST_DS_SELECTOR, host_ds_selector),
  676. FIELD(HOST_FS_SELECTOR, host_fs_selector),
  677. FIELD(HOST_GS_SELECTOR, host_gs_selector),
  678. FIELD(HOST_TR_SELECTOR, host_tr_selector),
  679. FIELD64(IO_BITMAP_A, io_bitmap_a),
  680. FIELD64(IO_BITMAP_B, io_bitmap_b),
  681. FIELD64(MSR_BITMAP, msr_bitmap),
  682. FIELD64(VM_EXIT_MSR_STORE_ADDR, vm_exit_msr_store_addr),
  683. FIELD64(VM_EXIT_MSR_LOAD_ADDR, vm_exit_msr_load_addr),
  684. FIELD64(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr),
  685. FIELD64(TSC_OFFSET, tsc_offset),
  686. FIELD64(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr),
  687. FIELD64(APIC_ACCESS_ADDR, apic_access_addr),
  688. FIELD64(POSTED_INTR_DESC_ADDR, posted_intr_desc_addr),
  689. FIELD64(EPT_POINTER, ept_pointer),
  690. FIELD64(EOI_EXIT_BITMAP0, eoi_exit_bitmap0),
  691. FIELD64(EOI_EXIT_BITMAP1, eoi_exit_bitmap1),
  692. FIELD64(EOI_EXIT_BITMAP2, eoi_exit_bitmap2),
  693. FIELD64(EOI_EXIT_BITMAP3, eoi_exit_bitmap3),
  694. FIELD64(XSS_EXIT_BITMAP, xss_exit_bitmap),
  695. FIELD64(GUEST_PHYSICAL_ADDRESS, guest_physical_address),
  696. FIELD64(VMCS_LINK_POINTER, vmcs_link_pointer),
  697. FIELD64(GUEST_IA32_DEBUGCTL, guest_ia32_debugctl),
  698. FIELD64(GUEST_IA32_PAT, guest_ia32_pat),
  699. FIELD64(GUEST_IA32_EFER, guest_ia32_efer),
  700. FIELD64(GUEST_IA32_PERF_GLOBAL_CTRL, guest_ia32_perf_global_ctrl),
  701. FIELD64(GUEST_PDPTR0, guest_pdptr0),
  702. FIELD64(GUEST_PDPTR1, guest_pdptr1),
  703. FIELD64(GUEST_PDPTR2, guest_pdptr2),
  704. FIELD64(GUEST_PDPTR3, guest_pdptr3),
  705. FIELD64(GUEST_BNDCFGS, guest_bndcfgs),
  706. FIELD64(HOST_IA32_PAT, host_ia32_pat),
  707. FIELD64(HOST_IA32_EFER, host_ia32_efer),
  708. FIELD64(HOST_IA32_PERF_GLOBAL_CTRL, host_ia32_perf_global_ctrl),
  709. FIELD(PIN_BASED_VM_EXEC_CONTROL, pin_based_vm_exec_control),
  710. FIELD(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control),
  711. FIELD(EXCEPTION_BITMAP, exception_bitmap),
  712. FIELD(PAGE_FAULT_ERROR_CODE_MASK, page_fault_error_code_mask),
  713. FIELD(PAGE_FAULT_ERROR_CODE_MATCH, page_fault_error_code_match),
  714. FIELD(CR3_TARGET_COUNT, cr3_target_count),
  715. FIELD(VM_EXIT_CONTROLS, vm_exit_controls),
  716. FIELD(VM_EXIT_MSR_STORE_COUNT, vm_exit_msr_store_count),
  717. FIELD(VM_EXIT_MSR_LOAD_COUNT, vm_exit_msr_load_count),
  718. FIELD(VM_ENTRY_CONTROLS, vm_entry_controls),
  719. FIELD(VM_ENTRY_MSR_LOAD_COUNT, vm_entry_msr_load_count),
  720. FIELD(VM_ENTRY_INTR_INFO_FIELD, vm_entry_intr_info_field),
  721. FIELD(VM_ENTRY_EXCEPTION_ERROR_CODE, vm_entry_exception_error_code),
  722. FIELD(VM_ENTRY_INSTRUCTION_LEN, vm_entry_instruction_len),
  723. FIELD(TPR_THRESHOLD, tpr_threshold),
  724. FIELD(SECONDARY_VM_EXEC_CONTROL, secondary_vm_exec_control),
  725. FIELD(VM_INSTRUCTION_ERROR, vm_instruction_error),
  726. FIELD(VM_EXIT_REASON, vm_exit_reason),
  727. FIELD(VM_EXIT_INTR_INFO, vm_exit_intr_info),
  728. FIELD(VM_EXIT_INTR_ERROR_CODE, vm_exit_intr_error_code),
  729. FIELD(IDT_VECTORING_INFO_FIELD, idt_vectoring_info_field),
  730. FIELD(IDT_VECTORING_ERROR_CODE, idt_vectoring_error_code),
  731. FIELD(VM_EXIT_INSTRUCTION_LEN, vm_exit_instruction_len),
  732. FIELD(VMX_INSTRUCTION_INFO, vmx_instruction_info),
  733. FIELD(GUEST_ES_LIMIT, guest_es_limit),
  734. FIELD(GUEST_CS_LIMIT, guest_cs_limit),
  735. FIELD(GUEST_SS_LIMIT, guest_ss_limit),
  736. FIELD(GUEST_DS_LIMIT, guest_ds_limit),
  737. FIELD(GUEST_FS_LIMIT, guest_fs_limit),
  738. FIELD(GUEST_GS_LIMIT, guest_gs_limit),
  739. FIELD(GUEST_LDTR_LIMIT, guest_ldtr_limit),
  740. FIELD(GUEST_TR_LIMIT, guest_tr_limit),
  741. FIELD(GUEST_GDTR_LIMIT, guest_gdtr_limit),
  742. FIELD(GUEST_IDTR_LIMIT, guest_idtr_limit),
  743. FIELD(GUEST_ES_AR_BYTES, guest_es_ar_bytes),
  744. FIELD(GUEST_CS_AR_BYTES, guest_cs_ar_bytes),
  745. FIELD(GUEST_SS_AR_BYTES, guest_ss_ar_bytes),
  746. FIELD(GUEST_DS_AR_BYTES, guest_ds_ar_bytes),
  747. FIELD(GUEST_FS_AR_BYTES, guest_fs_ar_bytes),
  748. FIELD(GUEST_GS_AR_BYTES, guest_gs_ar_bytes),
  749. FIELD(GUEST_LDTR_AR_BYTES, guest_ldtr_ar_bytes),
  750. FIELD(GUEST_TR_AR_BYTES, guest_tr_ar_bytes),
  751. FIELD(GUEST_INTERRUPTIBILITY_INFO, guest_interruptibility_info),
  752. FIELD(GUEST_ACTIVITY_STATE, guest_activity_state),
  753. FIELD(GUEST_SYSENTER_CS, guest_sysenter_cs),
  754. FIELD(HOST_IA32_SYSENTER_CS, host_ia32_sysenter_cs),
  755. FIELD(VMX_PREEMPTION_TIMER_VALUE, vmx_preemption_timer_value),
  756. FIELD(CR0_GUEST_HOST_MASK, cr0_guest_host_mask),
  757. FIELD(CR4_GUEST_HOST_MASK, cr4_guest_host_mask),
  758. FIELD(CR0_READ_SHADOW, cr0_read_shadow),
  759. FIELD(CR4_READ_SHADOW, cr4_read_shadow),
  760. FIELD(CR3_TARGET_VALUE0, cr3_target_value0),
  761. FIELD(CR3_TARGET_VALUE1, cr3_target_value1),
  762. FIELD(CR3_TARGET_VALUE2, cr3_target_value2),
  763. FIELD(CR3_TARGET_VALUE3, cr3_target_value3),
  764. FIELD(EXIT_QUALIFICATION, exit_qualification),
  765. FIELD(GUEST_LINEAR_ADDRESS, guest_linear_address),
  766. FIELD(GUEST_CR0, guest_cr0),
  767. FIELD(GUEST_CR3, guest_cr3),
  768. FIELD(GUEST_CR4, guest_cr4),
  769. FIELD(GUEST_ES_BASE, guest_es_base),
  770. FIELD(GUEST_CS_BASE, guest_cs_base),
  771. FIELD(GUEST_SS_BASE, guest_ss_base),
  772. FIELD(GUEST_DS_BASE, guest_ds_base),
  773. FIELD(GUEST_FS_BASE, guest_fs_base),
  774. FIELD(GUEST_GS_BASE, guest_gs_base),
  775. FIELD(GUEST_LDTR_BASE, guest_ldtr_base),
  776. FIELD(GUEST_TR_BASE, guest_tr_base),
  777. FIELD(GUEST_GDTR_BASE, guest_gdtr_base),
  778. FIELD(GUEST_IDTR_BASE, guest_idtr_base),
  779. FIELD(GUEST_DR7, guest_dr7),
  780. FIELD(GUEST_RSP, guest_rsp),
  781. FIELD(GUEST_RIP, guest_rip),
  782. FIELD(GUEST_RFLAGS, guest_rflags),
  783. FIELD(GUEST_PENDING_DBG_EXCEPTIONS, guest_pending_dbg_exceptions),
  784. FIELD(GUEST_SYSENTER_ESP, guest_sysenter_esp),
  785. FIELD(GUEST_SYSENTER_EIP, guest_sysenter_eip),
  786. FIELD(HOST_CR0, host_cr0),
  787. FIELD(HOST_CR3, host_cr3),
  788. FIELD(HOST_CR4, host_cr4),
  789. FIELD(HOST_FS_BASE, host_fs_base),
  790. FIELD(HOST_GS_BASE, host_gs_base),
  791. FIELD(HOST_TR_BASE, host_tr_base),
  792. FIELD(HOST_GDTR_BASE, host_gdtr_base),
  793. FIELD(HOST_IDTR_BASE, host_idtr_base),
  794. FIELD(HOST_IA32_SYSENTER_ESP, host_ia32_sysenter_esp),
  795. FIELD(HOST_IA32_SYSENTER_EIP, host_ia32_sysenter_eip),
  796. FIELD(HOST_RSP, host_rsp),
  797. FIELD(HOST_RIP, host_rip),
  798. };
  799. static inline short vmcs_field_to_offset(unsigned long field)
  800. {
  801. BUILD_BUG_ON(ARRAY_SIZE(vmcs_field_to_offset_table) > SHRT_MAX);
  802. if (field >= ARRAY_SIZE(vmcs_field_to_offset_table) ||
  803. vmcs_field_to_offset_table[field] == 0)
  804. return -ENOENT;
  805. return vmcs_field_to_offset_table[field];
  806. }
  807. static inline struct vmcs12 *get_vmcs12(struct kvm_vcpu *vcpu)
  808. {
  809. return to_vmx(vcpu)->nested.cached_vmcs12;
  810. }
  811. static struct page *nested_get_page(struct kvm_vcpu *vcpu, gpa_t addr)
  812. {
  813. struct page *page = kvm_vcpu_gfn_to_page(vcpu, addr >> PAGE_SHIFT);
  814. if (is_error_page(page))
  815. return NULL;
  816. return page;
  817. }
  818. static void nested_release_page(struct page *page)
  819. {
  820. kvm_release_page_dirty(page);
  821. }
  822. static void nested_release_page_clean(struct page *page)
  823. {
  824. kvm_release_page_clean(page);
  825. }
  826. static unsigned long nested_ept_get_cr3(struct kvm_vcpu *vcpu);
  827. static u64 construct_eptp(unsigned long root_hpa);
  828. static void kvm_cpu_vmxon(u64 addr);
  829. static void kvm_cpu_vmxoff(void);
  830. static bool vmx_xsaves_supported(void);
  831. static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr);
  832. static void vmx_set_segment(struct kvm_vcpu *vcpu,
  833. struct kvm_segment *var, int seg);
  834. static void vmx_get_segment(struct kvm_vcpu *vcpu,
  835. struct kvm_segment *var, int seg);
  836. static bool guest_state_valid(struct kvm_vcpu *vcpu);
  837. static u32 vmx_segment_access_rights(struct kvm_segment *var);
  838. static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx);
  839. static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx);
  840. static int alloc_identity_pagetable(struct kvm *kvm);
  841. static DEFINE_PER_CPU(struct vmcs *, vmxarea);
  842. static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
  843. /*
  844. * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
  845. * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it.
  846. */
  847. static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu);
  848. static DEFINE_PER_CPU(struct desc_ptr, host_gdt);
  849. /*
  850. * We maintian a per-CPU linked-list of vCPU, so in wakeup_handler() we
  851. * can find which vCPU should be waken up.
  852. */
  853. static DEFINE_PER_CPU(struct list_head, blocked_vcpu_on_cpu);
  854. static DEFINE_PER_CPU(spinlock_t, blocked_vcpu_on_cpu_lock);
  855. enum {
  856. VMX_IO_BITMAP_A,
  857. VMX_IO_BITMAP_B,
  858. VMX_MSR_BITMAP_LEGACY,
  859. VMX_MSR_BITMAP_LONGMODE,
  860. VMX_MSR_BITMAP_LEGACY_X2APIC_APICV,
  861. VMX_MSR_BITMAP_LONGMODE_X2APIC_APICV,
  862. VMX_MSR_BITMAP_LEGACY_X2APIC,
  863. VMX_MSR_BITMAP_LONGMODE_X2APIC,
  864. VMX_VMREAD_BITMAP,
  865. VMX_VMWRITE_BITMAP,
  866. VMX_BITMAP_NR
  867. };
  868. static unsigned long *vmx_bitmap[VMX_BITMAP_NR];
  869. #define vmx_io_bitmap_a (vmx_bitmap[VMX_IO_BITMAP_A])
  870. #define vmx_io_bitmap_b (vmx_bitmap[VMX_IO_BITMAP_B])
  871. #define vmx_msr_bitmap_legacy (vmx_bitmap[VMX_MSR_BITMAP_LEGACY])
  872. #define vmx_msr_bitmap_longmode (vmx_bitmap[VMX_MSR_BITMAP_LONGMODE])
  873. #define vmx_msr_bitmap_legacy_x2apic_apicv (vmx_bitmap[VMX_MSR_BITMAP_LEGACY_X2APIC_APICV])
  874. #define vmx_msr_bitmap_longmode_x2apic_apicv (vmx_bitmap[VMX_MSR_BITMAP_LONGMODE_X2APIC_APICV])
  875. #define vmx_msr_bitmap_legacy_x2apic (vmx_bitmap[VMX_MSR_BITMAP_LEGACY_X2APIC])
  876. #define vmx_msr_bitmap_longmode_x2apic (vmx_bitmap[VMX_MSR_BITMAP_LONGMODE_X2APIC])
  877. #define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP])
  878. #define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP])
  879. static bool cpu_has_load_ia32_efer;
  880. static bool cpu_has_load_perf_global_ctrl;
  881. static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
  882. static DEFINE_SPINLOCK(vmx_vpid_lock);
  883. static struct vmcs_config {
  884. int size;
  885. int order;
  886. u32 basic_cap;
  887. u32 revision_id;
  888. u32 pin_based_exec_ctrl;
  889. u32 cpu_based_exec_ctrl;
  890. u32 cpu_based_2nd_exec_ctrl;
  891. u32 vmexit_ctrl;
  892. u32 vmentry_ctrl;
  893. } vmcs_config;
  894. static struct vmx_capability {
  895. u32 ept;
  896. u32 vpid;
  897. } vmx_capability;
  898. #define VMX_SEGMENT_FIELD(seg) \
  899. [VCPU_SREG_##seg] = { \
  900. .selector = GUEST_##seg##_SELECTOR, \
  901. .base = GUEST_##seg##_BASE, \
  902. .limit = GUEST_##seg##_LIMIT, \
  903. .ar_bytes = GUEST_##seg##_AR_BYTES, \
  904. }
  905. static const struct kvm_vmx_segment_field {
  906. unsigned selector;
  907. unsigned base;
  908. unsigned limit;
  909. unsigned ar_bytes;
  910. } kvm_vmx_segment_fields[] = {
  911. VMX_SEGMENT_FIELD(CS),
  912. VMX_SEGMENT_FIELD(DS),
  913. VMX_SEGMENT_FIELD(ES),
  914. VMX_SEGMENT_FIELD(FS),
  915. VMX_SEGMENT_FIELD(GS),
  916. VMX_SEGMENT_FIELD(SS),
  917. VMX_SEGMENT_FIELD(TR),
  918. VMX_SEGMENT_FIELD(LDTR),
  919. };
  920. static u64 host_efer;
  921. static void ept_save_pdptrs(struct kvm_vcpu *vcpu);
  922. /*
  923. * Keep MSR_STAR at the end, as setup_msrs() will try to optimize it
  924. * away by decrementing the array size.
  925. */
  926. static const u32 vmx_msr_index[] = {
  927. #ifdef CONFIG_X86_64
  928. MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
  929. #endif
  930. MSR_EFER, MSR_TSC_AUX, MSR_STAR,
  931. };
  932. static inline bool is_exception_n(u32 intr_info, u8 vector)
  933. {
  934. return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
  935. INTR_INFO_VALID_MASK)) ==
  936. (INTR_TYPE_HARD_EXCEPTION | vector | INTR_INFO_VALID_MASK);
  937. }
  938. static inline bool is_debug(u32 intr_info)
  939. {
  940. return is_exception_n(intr_info, DB_VECTOR);
  941. }
  942. static inline bool is_breakpoint(u32 intr_info)
  943. {
  944. return is_exception_n(intr_info, BP_VECTOR);
  945. }
  946. static inline bool is_page_fault(u32 intr_info)
  947. {
  948. return is_exception_n(intr_info, PF_VECTOR);
  949. }
  950. static inline bool is_no_device(u32 intr_info)
  951. {
  952. return is_exception_n(intr_info, NM_VECTOR);
  953. }
  954. static inline bool is_invalid_opcode(u32 intr_info)
  955. {
  956. return is_exception_n(intr_info, UD_VECTOR);
  957. }
  958. static inline bool is_external_interrupt(u32 intr_info)
  959. {
  960. return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
  961. == (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
  962. }
  963. static inline bool is_machine_check(u32 intr_info)
  964. {
  965. return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
  966. INTR_INFO_VALID_MASK)) ==
  967. (INTR_TYPE_HARD_EXCEPTION | MC_VECTOR | INTR_INFO_VALID_MASK);
  968. }
  969. static inline bool cpu_has_vmx_msr_bitmap(void)
  970. {
  971. return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS;
  972. }
  973. static inline bool cpu_has_vmx_tpr_shadow(void)
  974. {
  975. return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW;
  976. }
  977. static inline bool cpu_need_tpr_shadow(struct kvm_vcpu *vcpu)
  978. {
  979. return cpu_has_vmx_tpr_shadow() && lapic_in_kernel(vcpu);
  980. }
  981. static inline bool cpu_has_secondary_exec_ctrls(void)
  982. {
  983. return vmcs_config.cpu_based_exec_ctrl &
  984. CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
  985. }
  986. static inline bool cpu_has_vmx_virtualize_apic_accesses(void)
  987. {
  988. return vmcs_config.cpu_based_2nd_exec_ctrl &
  989. SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
  990. }
  991. static inline bool cpu_has_vmx_virtualize_x2apic_mode(void)
  992. {
  993. return vmcs_config.cpu_based_2nd_exec_ctrl &
  994. SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
  995. }
  996. static inline bool cpu_has_vmx_apic_register_virt(void)
  997. {
  998. return vmcs_config.cpu_based_2nd_exec_ctrl &
  999. SECONDARY_EXEC_APIC_REGISTER_VIRT;
  1000. }
  1001. static inline bool cpu_has_vmx_virtual_intr_delivery(void)
  1002. {
  1003. return vmcs_config.cpu_based_2nd_exec_ctrl &
  1004. SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY;
  1005. }
  1006. /*
  1007. * Comment's format: document - errata name - stepping - processor name.
  1008. * Refer from
  1009. * https://www.virtualbox.org/svn/vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp
  1010. */
  1011. static u32 vmx_preemption_cpu_tfms[] = {
  1012. /* 323344.pdf - BA86 - D0 - Xeon 7500 Series */
  1013. 0x000206E6,
  1014. /* 323056.pdf - AAX65 - C2 - Xeon L3406 */
  1015. /* 322814.pdf - AAT59 - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */
  1016. /* 322911.pdf - AAU65 - C2 - i5-600, i3-500 Desktop and Pentium G6950 */
  1017. 0x00020652,
  1018. /* 322911.pdf - AAU65 - K0 - i5-600, i3-500 Desktop and Pentium G6950 */
  1019. 0x00020655,
  1020. /* 322373.pdf - AAO95 - B1 - Xeon 3400 Series */
  1021. /* 322166.pdf - AAN92 - B1 - i7-800 and i5-700 Desktop */
  1022. /*
  1023. * 320767.pdf - AAP86 - B1 -
  1024. * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile
  1025. */
  1026. 0x000106E5,
  1027. /* 321333.pdf - AAM126 - C0 - Xeon 3500 */
  1028. 0x000106A0,
  1029. /* 321333.pdf - AAM126 - C1 - Xeon 3500 */
  1030. 0x000106A1,
  1031. /* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */
  1032. 0x000106A4,
  1033. /* 321333.pdf - AAM126 - D0 - Xeon 3500 */
  1034. /* 321324.pdf - AAK139 - D0 - Xeon 5500 */
  1035. /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */
  1036. 0x000106A5,
  1037. };
  1038. static inline bool cpu_has_broken_vmx_preemption_timer(void)
  1039. {
  1040. u32 eax = cpuid_eax(0x00000001), i;
  1041. /* Clear the reserved bits */
  1042. eax &= ~(0x3U << 14 | 0xfU << 28);
  1043. for (i = 0; i < ARRAY_SIZE(vmx_preemption_cpu_tfms); i++)
  1044. if (eax == vmx_preemption_cpu_tfms[i])
  1045. return true;
  1046. return false;
  1047. }
  1048. static inline bool cpu_has_vmx_preemption_timer(void)
  1049. {
  1050. return vmcs_config.pin_based_exec_ctrl &
  1051. PIN_BASED_VMX_PREEMPTION_TIMER;
  1052. }
  1053. static inline bool cpu_has_vmx_posted_intr(void)
  1054. {
  1055. return IS_ENABLED(CONFIG_X86_LOCAL_APIC) &&
  1056. vmcs_config.pin_based_exec_ctrl & PIN_BASED_POSTED_INTR;
  1057. }
  1058. static inline bool cpu_has_vmx_apicv(void)
  1059. {
  1060. return cpu_has_vmx_apic_register_virt() &&
  1061. cpu_has_vmx_virtual_intr_delivery() &&
  1062. cpu_has_vmx_posted_intr();
  1063. }
  1064. static inline bool cpu_has_vmx_flexpriority(void)
  1065. {
  1066. return cpu_has_vmx_tpr_shadow() &&
  1067. cpu_has_vmx_virtualize_apic_accesses();
  1068. }
  1069. static inline bool cpu_has_vmx_ept_execute_only(void)
  1070. {
  1071. return vmx_capability.ept & VMX_EPT_EXECUTE_ONLY_BIT;
  1072. }
  1073. static inline bool cpu_has_vmx_ept_2m_page(void)
  1074. {
  1075. return vmx_capability.ept & VMX_EPT_2MB_PAGE_BIT;
  1076. }
  1077. static inline bool cpu_has_vmx_ept_1g_page(void)
  1078. {
  1079. return vmx_capability.ept & VMX_EPT_1GB_PAGE_BIT;
  1080. }
  1081. static inline bool cpu_has_vmx_ept_4levels(void)
  1082. {
  1083. return vmx_capability.ept & VMX_EPT_PAGE_WALK_4_BIT;
  1084. }
  1085. static inline bool cpu_has_vmx_ept_ad_bits(void)
  1086. {
  1087. return vmx_capability.ept & VMX_EPT_AD_BIT;
  1088. }
  1089. static inline bool cpu_has_vmx_invept_context(void)
  1090. {
  1091. return vmx_capability.ept & VMX_EPT_EXTENT_CONTEXT_BIT;
  1092. }
  1093. static inline bool cpu_has_vmx_invept_global(void)
  1094. {
  1095. return vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT;
  1096. }
  1097. static inline bool cpu_has_vmx_invvpid_single(void)
  1098. {
  1099. return vmx_capability.vpid & VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT;
  1100. }
  1101. static inline bool cpu_has_vmx_invvpid_global(void)
  1102. {
  1103. return vmx_capability.vpid & VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT;
  1104. }
  1105. static inline bool cpu_has_vmx_ept(void)
  1106. {
  1107. return vmcs_config.cpu_based_2nd_exec_ctrl &
  1108. SECONDARY_EXEC_ENABLE_EPT;
  1109. }
  1110. static inline bool cpu_has_vmx_unrestricted_guest(void)
  1111. {
  1112. return vmcs_config.cpu_based_2nd_exec_ctrl &
  1113. SECONDARY_EXEC_UNRESTRICTED_GUEST;
  1114. }
  1115. static inline bool cpu_has_vmx_ple(void)
  1116. {
  1117. return vmcs_config.cpu_based_2nd_exec_ctrl &
  1118. SECONDARY_EXEC_PAUSE_LOOP_EXITING;
  1119. }
  1120. static inline bool cpu_has_vmx_basic_inout(void)
  1121. {
  1122. return (((u64)vmcs_config.basic_cap << 32) & VMX_BASIC_INOUT);
  1123. }
  1124. static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu)
  1125. {
  1126. return flexpriority_enabled && lapic_in_kernel(vcpu);
  1127. }
  1128. static inline bool cpu_has_vmx_vpid(void)
  1129. {
  1130. return vmcs_config.cpu_based_2nd_exec_ctrl &
  1131. SECONDARY_EXEC_ENABLE_VPID;
  1132. }
  1133. static inline bool cpu_has_vmx_rdtscp(void)
  1134. {
  1135. return vmcs_config.cpu_based_2nd_exec_ctrl &
  1136. SECONDARY_EXEC_RDTSCP;
  1137. }
  1138. static inline bool cpu_has_vmx_invpcid(void)
  1139. {
  1140. return vmcs_config.cpu_based_2nd_exec_ctrl &
  1141. SECONDARY_EXEC_ENABLE_INVPCID;
  1142. }
  1143. static inline bool cpu_has_virtual_nmis(void)
  1144. {
  1145. return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS;
  1146. }
  1147. static inline bool cpu_has_vmx_wbinvd_exit(void)
  1148. {
  1149. return vmcs_config.cpu_based_2nd_exec_ctrl &
  1150. SECONDARY_EXEC_WBINVD_EXITING;
  1151. }
  1152. static inline bool cpu_has_vmx_shadow_vmcs(void)
  1153. {
  1154. u64 vmx_msr;
  1155. rdmsrl(MSR_IA32_VMX_MISC, vmx_msr);
  1156. /* check if the cpu supports writing r/o exit information fields */
  1157. if (!(vmx_msr & MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS))
  1158. return false;
  1159. return vmcs_config.cpu_based_2nd_exec_ctrl &
  1160. SECONDARY_EXEC_SHADOW_VMCS;
  1161. }
  1162. static inline bool cpu_has_vmx_pml(void)
  1163. {
  1164. return vmcs_config.cpu_based_2nd_exec_ctrl & SECONDARY_EXEC_ENABLE_PML;
  1165. }
  1166. static inline bool cpu_has_vmx_tsc_scaling(void)
  1167. {
  1168. return vmcs_config.cpu_based_2nd_exec_ctrl &
  1169. SECONDARY_EXEC_TSC_SCALING;
  1170. }
  1171. static inline bool report_flexpriority(void)
  1172. {
  1173. return flexpriority_enabled;
  1174. }
  1175. static inline bool nested_cpu_has(struct vmcs12 *vmcs12, u32 bit)
  1176. {
  1177. return vmcs12->cpu_based_vm_exec_control & bit;
  1178. }
  1179. static inline bool nested_cpu_has2(struct vmcs12 *vmcs12, u32 bit)
  1180. {
  1181. return (vmcs12->cpu_based_vm_exec_control &
  1182. CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
  1183. (vmcs12->secondary_vm_exec_control & bit);
  1184. }
  1185. static inline bool nested_cpu_has_virtual_nmis(struct vmcs12 *vmcs12)
  1186. {
  1187. return vmcs12->pin_based_vm_exec_control & PIN_BASED_VIRTUAL_NMIS;
  1188. }
  1189. static inline bool nested_cpu_has_preemption_timer(struct vmcs12 *vmcs12)
  1190. {
  1191. return vmcs12->pin_based_vm_exec_control &
  1192. PIN_BASED_VMX_PREEMPTION_TIMER;
  1193. }
  1194. static inline int nested_cpu_has_ept(struct vmcs12 *vmcs12)
  1195. {
  1196. return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_EPT);
  1197. }
  1198. static inline bool nested_cpu_has_xsaves(struct vmcs12 *vmcs12)
  1199. {
  1200. return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES) &&
  1201. vmx_xsaves_supported();
  1202. }
  1203. static inline bool nested_cpu_has_virt_x2apic_mode(struct vmcs12 *vmcs12)
  1204. {
  1205. return nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE);
  1206. }
  1207. static inline bool nested_cpu_has_vpid(struct vmcs12 *vmcs12)
  1208. {
  1209. return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_VPID);
  1210. }
  1211. static inline bool nested_cpu_has_apic_reg_virt(struct vmcs12 *vmcs12)
  1212. {
  1213. return nested_cpu_has2(vmcs12, SECONDARY_EXEC_APIC_REGISTER_VIRT);
  1214. }
  1215. static inline bool nested_cpu_has_vid(struct vmcs12 *vmcs12)
  1216. {
  1217. return nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
  1218. }
  1219. static inline bool nested_cpu_has_posted_intr(struct vmcs12 *vmcs12)
  1220. {
  1221. return vmcs12->pin_based_vm_exec_control & PIN_BASED_POSTED_INTR;
  1222. }
  1223. static inline bool is_nmi(u32 intr_info)
  1224. {
  1225. return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
  1226. == (INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK);
  1227. }
  1228. static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
  1229. u32 exit_intr_info,
  1230. unsigned long exit_qualification);
  1231. static void nested_vmx_entry_failure(struct kvm_vcpu *vcpu,
  1232. struct vmcs12 *vmcs12,
  1233. u32 reason, unsigned long qualification);
  1234. static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
  1235. {
  1236. int i;
  1237. for (i = 0; i < vmx->nmsrs; ++i)
  1238. if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)
  1239. return i;
  1240. return -1;
  1241. }
  1242. static inline void __invvpid(int ext, u16 vpid, gva_t gva)
  1243. {
  1244. struct {
  1245. u64 vpid : 16;
  1246. u64 rsvd : 48;
  1247. u64 gva;
  1248. } operand = { vpid, 0, gva };
  1249. asm volatile (__ex(ASM_VMX_INVVPID)
  1250. /* CF==1 or ZF==1 --> rc = -1 */
  1251. "; ja 1f ; ud2 ; 1:"
  1252. : : "a"(&operand), "c"(ext) : "cc", "memory");
  1253. }
  1254. static inline void __invept(int ext, u64 eptp, gpa_t gpa)
  1255. {
  1256. struct {
  1257. u64 eptp, gpa;
  1258. } operand = {eptp, gpa};
  1259. asm volatile (__ex(ASM_VMX_INVEPT)
  1260. /* CF==1 or ZF==1 --> rc = -1 */
  1261. "; ja 1f ; ud2 ; 1:\n"
  1262. : : "a" (&operand), "c" (ext) : "cc", "memory");
  1263. }
  1264. static struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
  1265. {
  1266. int i;
  1267. i = __find_msr_index(vmx, msr);
  1268. if (i >= 0)
  1269. return &vmx->guest_msrs[i];
  1270. return NULL;
  1271. }
  1272. static void vmcs_clear(struct vmcs *vmcs)
  1273. {
  1274. u64 phys_addr = __pa(vmcs);
  1275. u8 error;
  1276. asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) "; setna %0"
  1277. : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
  1278. : "cc", "memory");
  1279. if (error)
  1280. printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
  1281. vmcs, phys_addr);
  1282. }
  1283. static inline void loaded_vmcs_init(struct loaded_vmcs *loaded_vmcs)
  1284. {
  1285. vmcs_clear(loaded_vmcs->vmcs);
  1286. if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched)
  1287. vmcs_clear(loaded_vmcs->shadow_vmcs);
  1288. loaded_vmcs->cpu = -1;
  1289. loaded_vmcs->launched = 0;
  1290. }
  1291. static void vmcs_load(struct vmcs *vmcs)
  1292. {
  1293. u64 phys_addr = __pa(vmcs);
  1294. u8 error;
  1295. asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) "; setna %0"
  1296. : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
  1297. : "cc", "memory");
  1298. if (error)
  1299. printk(KERN_ERR "kvm: vmptrld %p/%llx failed\n",
  1300. vmcs, phys_addr);
  1301. }
  1302. #ifdef CONFIG_KEXEC_CORE
  1303. /*
  1304. * This bitmap is used to indicate whether the vmclear
  1305. * operation is enabled on all cpus. All disabled by
  1306. * default.
  1307. */
  1308. static cpumask_t crash_vmclear_enabled_bitmap = CPU_MASK_NONE;
  1309. static inline void crash_enable_local_vmclear(int cpu)
  1310. {
  1311. cpumask_set_cpu(cpu, &crash_vmclear_enabled_bitmap);
  1312. }
  1313. static inline void crash_disable_local_vmclear(int cpu)
  1314. {
  1315. cpumask_clear_cpu(cpu, &crash_vmclear_enabled_bitmap);
  1316. }
  1317. static inline int crash_local_vmclear_enabled(int cpu)
  1318. {
  1319. return cpumask_test_cpu(cpu, &crash_vmclear_enabled_bitmap);
  1320. }
  1321. static void crash_vmclear_local_loaded_vmcss(void)
  1322. {
  1323. int cpu = raw_smp_processor_id();
  1324. struct loaded_vmcs *v;
  1325. if (!crash_local_vmclear_enabled(cpu))
  1326. return;
  1327. list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu),
  1328. loaded_vmcss_on_cpu_link)
  1329. vmcs_clear(v->vmcs);
  1330. }
  1331. #else
  1332. static inline void crash_enable_local_vmclear(int cpu) { }
  1333. static inline void crash_disable_local_vmclear(int cpu) { }
  1334. #endif /* CONFIG_KEXEC_CORE */
  1335. static void __loaded_vmcs_clear(void *arg)
  1336. {
  1337. struct loaded_vmcs *loaded_vmcs = arg;
  1338. int cpu = raw_smp_processor_id();
  1339. if (loaded_vmcs->cpu != cpu)
  1340. return; /* vcpu migration can race with cpu offline */
  1341. if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs)
  1342. per_cpu(current_vmcs, cpu) = NULL;
  1343. crash_disable_local_vmclear(cpu);
  1344. list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link);
  1345. /*
  1346. * we should ensure updating loaded_vmcs->loaded_vmcss_on_cpu_link
  1347. * is before setting loaded_vmcs->vcpu to -1 which is done in
  1348. * loaded_vmcs_init. Otherwise, other cpu can see vcpu = -1 fist
  1349. * then adds the vmcs into percpu list before it is deleted.
  1350. */
  1351. smp_wmb();
  1352. loaded_vmcs_init(loaded_vmcs);
  1353. crash_enable_local_vmclear(cpu);
  1354. }
  1355. static void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs)
  1356. {
  1357. int cpu = loaded_vmcs->cpu;
  1358. if (cpu != -1)
  1359. smp_call_function_single(cpu,
  1360. __loaded_vmcs_clear, loaded_vmcs, 1);
  1361. }
  1362. static inline void vpid_sync_vcpu_single(int vpid)
  1363. {
  1364. if (vpid == 0)
  1365. return;
  1366. if (cpu_has_vmx_invvpid_single())
  1367. __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vpid, 0);
  1368. }
  1369. static inline void vpid_sync_vcpu_global(void)
  1370. {
  1371. if (cpu_has_vmx_invvpid_global())
  1372. __invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0);
  1373. }
  1374. static inline void vpid_sync_context(int vpid)
  1375. {
  1376. if (cpu_has_vmx_invvpid_single())
  1377. vpid_sync_vcpu_single(vpid);
  1378. else
  1379. vpid_sync_vcpu_global();
  1380. }
  1381. static inline void ept_sync_global(void)
  1382. {
  1383. if (cpu_has_vmx_invept_global())
  1384. __invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
  1385. }
  1386. static inline void ept_sync_context(u64 eptp)
  1387. {
  1388. if (enable_ept) {
  1389. if (cpu_has_vmx_invept_context())
  1390. __invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
  1391. else
  1392. ept_sync_global();
  1393. }
  1394. }
  1395. static __always_inline void vmcs_check16(unsigned long field)
  1396. {
  1397. BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
  1398. "16-bit accessor invalid for 64-bit field");
  1399. BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
  1400. "16-bit accessor invalid for 64-bit high field");
  1401. BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
  1402. "16-bit accessor invalid for 32-bit high field");
  1403. BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
  1404. "16-bit accessor invalid for natural width field");
  1405. }
  1406. static __always_inline void vmcs_check32(unsigned long field)
  1407. {
  1408. BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
  1409. "32-bit accessor invalid for 16-bit field");
  1410. BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
  1411. "32-bit accessor invalid for natural width field");
  1412. }
  1413. static __always_inline void vmcs_check64(unsigned long field)
  1414. {
  1415. BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
  1416. "64-bit accessor invalid for 16-bit field");
  1417. BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
  1418. "64-bit accessor invalid for 64-bit high field");
  1419. BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
  1420. "64-bit accessor invalid for 32-bit field");
  1421. BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
  1422. "64-bit accessor invalid for natural width field");
  1423. }
  1424. static __always_inline void vmcs_checkl(unsigned long field)
  1425. {
  1426. BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
  1427. "Natural width accessor invalid for 16-bit field");
  1428. BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
  1429. "Natural width accessor invalid for 64-bit field");
  1430. BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
  1431. "Natural width accessor invalid for 64-bit high field");
  1432. BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
  1433. "Natural width accessor invalid for 32-bit field");
  1434. }
  1435. static __always_inline unsigned long __vmcs_readl(unsigned long field)
  1436. {
  1437. unsigned long value;
  1438. asm volatile (__ex_clear(ASM_VMX_VMREAD_RDX_RAX, "%0")
  1439. : "=a"(value) : "d"(field) : "cc");
  1440. return value;
  1441. }
  1442. static __always_inline u16 vmcs_read16(unsigned long field)
  1443. {
  1444. vmcs_check16(field);
  1445. return __vmcs_readl(field);
  1446. }
  1447. static __always_inline u32 vmcs_read32(unsigned long field)
  1448. {
  1449. vmcs_check32(field);
  1450. return __vmcs_readl(field);
  1451. }
  1452. static __always_inline u64 vmcs_read64(unsigned long field)
  1453. {
  1454. vmcs_check64(field);
  1455. #ifdef CONFIG_X86_64
  1456. return __vmcs_readl(field);
  1457. #else
  1458. return __vmcs_readl(field) | ((u64)__vmcs_readl(field+1) << 32);
  1459. #endif
  1460. }
  1461. static __always_inline unsigned long vmcs_readl(unsigned long field)
  1462. {
  1463. vmcs_checkl(field);
  1464. return __vmcs_readl(field);
  1465. }
  1466. static noinline void vmwrite_error(unsigned long field, unsigned long value)
  1467. {
  1468. printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
  1469. field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
  1470. dump_stack();
  1471. }
  1472. static __always_inline void __vmcs_writel(unsigned long field, unsigned long value)
  1473. {
  1474. u8 error;
  1475. asm volatile (__ex(ASM_VMX_VMWRITE_RAX_RDX) "; setna %0"
  1476. : "=q"(error) : "a"(value), "d"(field) : "cc");
  1477. if (unlikely(error))
  1478. vmwrite_error(field, value);
  1479. }
  1480. static __always_inline void vmcs_write16(unsigned long field, u16 value)
  1481. {
  1482. vmcs_check16(field);
  1483. __vmcs_writel(field, value);
  1484. }
  1485. static __always_inline void vmcs_write32(unsigned long field, u32 value)
  1486. {
  1487. vmcs_check32(field);
  1488. __vmcs_writel(field, value);
  1489. }
  1490. static __always_inline void vmcs_write64(unsigned long field, u64 value)
  1491. {
  1492. vmcs_check64(field);
  1493. __vmcs_writel(field, value);
  1494. #ifndef CONFIG_X86_64
  1495. asm volatile ("");
  1496. __vmcs_writel(field+1, value >> 32);
  1497. #endif
  1498. }
  1499. static __always_inline void vmcs_writel(unsigned long field, unsigned long value)
  1500. {
  1501. vmcs_checkl(field);
  1502. __vmcs_writel(field, value);
  1503. }
  1504. static __always_inline void vmcs_clear_bits(unsigned long field, u32 mask)
  1505. {
  1506. BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
  1507. "vmcs_clear_bits does not support 64-bit fields");
  1508. __vmcs_writel(field, __vmcs_readl(field) & ~mask);
  1509. }
  1510. static __always_inline void vmcs_set_bits(unsigned long field, u32 mask)
  1511. {
  1512. BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
  1513. "vmcs_set_bits does not support 64-bit fields");
  1514. __vmcs_writel(field, __vmcs_readl(field) | mask);
  1515. }
  1516. static inline void vm_entry_controls_reset_shadow(struct vcpu_vmx *vmx)
  1517. {
  1518. vmx->vm_entry_controls_shadow = vmcs_read32(VM_ENTRY_CONTROLS);
  1519. }
  1520. static inline void vm_entry_controls_init(struct vcpu_vmx *vmx, u32 val)
  1521. {
  1522. vmcs_write32(VM_ENTRY_CONTROLS, val);
  1523. vmx->vm_entry_controls_shadow = val;
  1524. }
  1525. static inline void vm_entry_controls_set(struct vcpu_vmx *vmx, u32 val)
  1526. {
  1527. if (vmx->vm_entry_controls_shadow != val)
  1528. vm_entry_controls_init(vmx, val);
  1529. }
  1530. static inline u32 vm_entry_controls_get(struct vcpu_vmx *vmx)
  1531. {
  1532. return vmx->vm_entry_controls_shadow;
  1533. }
  1534. static inline void vm_entry_controls_setbit(struct vcpu_vmx *vmx, u32 val)
  1535. {
  1536. vm_entry_controls_set(vmx, vm_entry_controls_get(vmx) | val);
  1537. }
  1538. static inline void vm_entry_controls_clearbit(struct vcpu_vmx *vmx, u32 val)
  1539. {
  1540. vm_entry_controls_set(vmx, vm_entry_controls_get(vmx) & ~val);
  1541. }
  1542. static inline void vm_exit_controls_reset_shadow(struct vcpu_vmx *vmx)
  1543. {
  1544. vmx->vm_exit_controls_shadow = vmcs_read32(VM_EXIT_CONTROLS);
  1545. }
  1546. static inline void vm_exit_controls_init(struct vcpu_vmx *vmx, u32 val)
  1547. {
  1548. vmcs_write32(VM_EXIT_CONTROLS, val);
  1549. vmx->vm_exit_controls_shadow = val;
  1550. }
  1551. static inline void vm_exit_controls_set(struct vcpu_vmx *vmx, u32 val)
  1552. {
  1553. if (vmx->vm_exit_controls_shadow != val)
  1554. vm_exit_controls_init(vmx, val);
  1555. }
  1556. static inline u32 vm_exit_controls_get(struct vcpu_vmx *vmx)
  1557. {
  1558. return vmx->vm_exit_controls_shadow;
  1559. }
  1560. static inline void vm_exit_controls_setbit(struct vcpu_vmx *vmx, u32 val)
  1561. {
  1562. vm_exit_controls_set(vmx, vm_exit_controls_get(vmx) | val);
  1563. }
  1564. static inline void vm_exit_controls_clearbit(struct vcpu_vmx *vmx, u32 val)
  1565. {
  1566. vm_exit_controls_set(vmx, vm_exit_controls_get(vmx) & ~val);
  1567. }
  1568. static void vmx_segment_cache_clear(struct vcpu_vmx *vmx)
  1569. {
  1570. vmx->segment_cache.bitmask = 0;
  1571. }
  1572. static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg,
  1573. unsigned field)
  1574. {
  1575. bool ret;
  1576. u32 mask = 1 << (seg * SEG_FIELD_NR + field);
  1577. if (!(vmx->vcpu.arch.regs_avail & (1 << VCPU_EXREG_SEGMENTS))) {
  1578. vmx->vcpu.arch.regs_avail |= (1 << VCPU_EXREG_SEGMENTS);
  1579. vmx->segment_cache.bitmask = 0;
  1580. }
  1581. ret = vmx->segment_cache.bitmask & mask;
  1582. vmx->segment_cache.bitmask |= mask;
  1583. return ret;
  1584. }
  1585. static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg)
  1586. {
  1587. u16 *p = &vmx->segment_cache.seg[seg].selector;
  1588. if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL))
  1589. *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector);
  1590. return *p;
  1591. }
  1592. static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg)
  1593. {
  1594. ulong *p = &vmx->segment_cache.seg[seg].base;
  1595. if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE))
  1596. *p = vmcs_readl(kvm_vmx_segment_fields[seg].base);
  1597. return *p;
  1598. }
  1599. static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg)
  1600. {
  1601. u32 *p = &vmx->segment_cache.seg[seg].limit;
  1602. if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT))
  1603. *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit);
  1604. return *p;
  1605. }
  1606. static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg)
  1607. {
  1608. u32 *p = &vmx->segment_cache.seg[seg].ar;
  1609. if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR))
  1610. *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes);
  1611. return *p;
  1612. }
  1613. static void update_exception_bitmap(struct kvm_vcpu *vcpu)
  1614. {
  1615. u32 eb;
  1616. eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
  1617. (1u << NM_VECTOR) | (1u << DB_VECTOR) | (1u << AC_VECTOR);
  1618. if ((vcpu->guest_debug &
  1619. (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
  1620. (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
  1621. eb |= 1u << BP_VECTOR;
  1622. if (to_vmx(vcpu)->rmode.vm86_active)
  1623. eb = ~0;
  1624. if (enable_ept)
  1625. eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
  1626. if (vcpu->fpu_active)
  1627. eb &= ~(1u << NM_VECTOR);
  1628. /* When we are running a nested L2 guest and L1 specified for it a
  1629. * certain exception bitmap, we must trap the same exceptions and pass
  1630. * them to L1. When running L2, we will only handle the exceptions
  1631. * specified above if L1 did not want them.
  1632. */
  1633. if (is_guest_mode(vcpu))
  1634. eb |= get_vmcs12(vcpu)->exception_bitmap;
  1635. vmcs_write32(EXCEPTION_BITMAP, eb);
  1636. }
  1637. static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx,
  1638. unsigned long entry, unsigned long exit)
  1639. {
  1640. vm_entry_controls_clearbit(vmx, entry);
  1641. vm_exit_controls_clearbit(vmx, exit);
  1642. }
  1643. static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
  1644. {
  1645. unsigned i;
  1646. struct msr_autoload *m = &vmx->msr_autoload;
  1647. switch (msr) {
  1648. case MSR_EFER:
  1649. if (cpu_has_load_ia32_efer) {
  1650. clear_atomic_switch_msr_special(vmx,
  1651. VM_ENTRY_LOAD_IA32_EFER,
  1652. VM_EXIT_LOAD_IA32_EFER);
  1653. return;
  1654. }
  1655. break;
  1656. case MSR_CORE_PERF_GLOBAL_CTRL:
  1657. if (cpu_has_load_perf_global_ctrl) {
  1658. clear_atomic_switch_msr_special(vmx,
  1659. VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
  1660. VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
  1661. return;
  1662. }
  1663. break;
  1664. }
  1665. for (i = 0; i < m->nr; ++i)
  1666. if (m->guest[i].index == msr)
  1667. break;
  1668. if (i == m->nr)
  1669. return;
  1670. --m->nr;
  1671. m->guest[i] = m->guest[m->nr];
  1672. m->host[i] = m->host[m->nr];
  1673. vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
  1674. vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
  1675. }
  1676. static void add_atomic_switch_msr_special(struct vcpu_vmx *vmx,
  1677. unsigned long entry, unsigned long exit,
  1678. unsigned long guest_val_vmcs, unsigned long host_val_vmcs,
  1679. u64 guest_val, u64 host_val)
  1680. {
  1681. vmcs_write64(guest_val_vmcs, guest_val);
  1682. vmcs_write64(host_val_vmcs, host_val);
  1683. vm_entry_controls_setbit(vmx, entry);
  1684. vm_exit_controls_setbit(vmx, exit);
  1685. }
  1686. static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
  1687. u64 guest_val, u64 host_val)
  1688. {
  1689. unsigned i;
  1690. struct msr_autoload *m = &vmx->msr_autoload;
  1691. switch (msr) {
  1692. case MSR_EFER:
  1693. if (cpu_has_load_ia32_efer) {
  1694. add_atomic_switch_msr_special(vmx,
  1695. VM_ENTRY_LOAD_IA32_EFER,
  1696. VM_EXIT_LOAD_IA32_EFER,
  1697. GUEST_IA32_EFER,
  1698. HOST_IA32_EFER,
  1699. guest_val, host_val);
  1700. return;
  1701. }
  1702. break;
  1703. case MSR_CORE_PERF_GLOBAL_CTRL:
  1704. if (cpu_has_load_perf_global_ctrl) {
  1705. add_atomic_switch_msr_special(vmx,
  1706. VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
  1707. VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL,
  1708. GUEST_IA32_PERF_GLOBAL_CTRL,
  1709. HOST_IA32_PERF_GLOBAL_CTRL,
  1710. guest_val, host_val);
  1711. return;
  1712. }
  1713. break;
  1714. case MSR_IA32_PEBS_ENABLE:
  1715. /* PEBS needs a quiescent period after being disabled (to write
  1716. * a record). Disabling PEBS through VMX MSR swapping doesn't
  1717. * provide that period, so a CPU could write host's record into
  1718. * guest's memory.
  1719. */
  1720. wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
  1721. }
  1722. for (i = 0; i < m->nr; ++i)
  1723. if (m->guest[i].index == msr)
  1724. break;
  1725. if (i == NR_AUTOLOAD_MSRS) {
  1726. printk_once(KERN_WARNING "Not enough msr switch entries. "
  1727. "Can't add msr %x\n", msr);
  1728. return;
  1729. } else if (i == m->nr) {
  1730. ++m->nr;
  1731. vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
  1732. vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
  1733. }
  1734. m->guest[i].index = msr;
  1735. m->guest[i].value = guest_val;
  1736. m->host[i].index = msr;
  1737. m->host[i].value = host_val;
  1738. }
  1739. static void reload_tss(void)
  1740. {
  1741. /*
  1742. * VT restores TR but not its size. Useless.
  1743. */
  1744. struct desc_ptr *gdt = this_cpu_ptr(&host_gdt);
  1745. struct desc_struct *descs;
  1746. descs = (void *)gdt->address;
  1747. descs[GDT_ENTRY_TSS].type = 9; /* available TSS */
  1748. load_TR_desc();
  1749. }
  1750. static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
  1751. {
  1752. u64 guest_efer = vmx->vcpu.arch.efer;
  1753. u64 ignore_bits = 0;
  1754. if (!enable_ept) {
  1755. /*
  1756. * NX is needed to handle CR0.WP=1, CR4.SMEP=1. Testing
  1757. * host CPUID is more efficient than testing guest CPUID
  1758. * or CR4. Host SMEP is anyway a requirement for guest SMEP.
  1759. */
  1760. if (boot_cpu_has(X86_FEATURE_SMEP))
  1761. guest_efer |= EFER_NX;
  1762. else if (!(guest_efer & EFER_NX))
  1763. ignore_bits |= EFER_NX;
  1764. }
  1765. /*
  1766. * LMA and LME handled by hardware; SCE meaningless outside long mode.
  1767. */
  1768. ignore_bits |= EFER_SCE;
  1769. #ifdef CONFIG_X86_64
  1770. ignore_bits |= EFER_LMA | EFER_LME;
  1771. /* SCE is meaningful only in long mode on Intel */
  1772. if (guest_efer & EFER_LMA)
  1773. ignore_bits &= ~(u64)EFER_SCE;
  1774. #endif
  1775. clear_atomic_switch_msr(vmx, MSR_EFER);
  1776. /*
  1777. * On EPT, we can't emulate NX, so we must switch EFER atomically.
  1778. * On CPUs that support "load IA32_EFER", always switch EFER
  1779. * atomically, since it's faster than switching it manually.
  1780. */
  1781. if (cpu_has_load_ia32_efer ||
  1782. (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX))) {
  1783. if (!(guest_efer & EFER_LMA))
  1784. guest_efer &= ~EFER_LME;
  1785. if (guest_efer != host_efer)
  1786. add_atomic_switch_msr(vmx, MSR_EFER,
  1787. guest_efer, host_efer);
  1788. return false;
  1789. } else {
  1790. guest_efer &= ~ignore_bits;
  1791. guest_efer |= host_efer & ignore_bits;
  1792. vmx->guest_msrs[efer_offset].data = guest_efer;
  1793. vmx->guest_msrs[efer_offset].mask = ~ignore_bits;
  1794. return true;
  1795. }
  1796. }
  1797. static unsigned long segment_base(u16 selector)
  1798. {
  1799. struct desc_ptr *gdt = this_cpu_ptr(&host_gdt);
  1800. struct desc_struct *d;
  1801. unsigned long table_base;
  1802. unsigned long v;
  1803. if (!(selector & ~3))
  1804. return 0;
  1805. table_base = gdt->address;
  1806. if (selector & 4) { /* from ldt */
  1807. u16 ldt_selector = kvm_read_ldt();
  1808. if (!(ldt_selector & ~3))
  1809. return 0;
  1810. table_base = segment_base(ldt_selector);
  1811. }
  1812. d = (struct desc_struct *)(table_base + (selector & ~7));
  1813. v = get_desc_base(d);
  1814. #ifdef CONFIG_X86_64
  1815. if (d->s == 0 && (d->type == 2 || d->type == 9 || d->type == 11))
  1816. v |= ((unsigned long)((struct ldttss_desc64 *)d)->base3) << 32;
  1817. #endif
  1818. return v;
  1819. }
  1820. static inline unsigned long kvm_read_tr_base(void)
  1821. {
  1822. u16 tr;
  1823. asm("str %0" : "=g"(tr));
  1824. return segment_base(tr);
  1825. }
  1826. static void vmx_save_host_state(struct kvm_vcpu *vcpu)
  1827. {
  1828. struct vcpu_vmx *vmx = to_vmx(vcpu);
  1829. int i;
  1830. if (vmx->host_state.loaded)
  1831. return;
  1832. vmx->host_state.loaded = 1;
  1833. /*
  1834. * Set host fs and gs selectors. Unfortunately, 22.2.3 does not
  1835. * allow segment selectors with cpl > 0 or ti == 1.
  1836. */
  1837. vmx->host_state.ldt_sel = kvm_read_ldt();
  1838. vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel;
  1839. savesegment(fs, vmx->host_state.fs_sel);
  1840. if (!(vmx->host_state.fs_sel & 7)) {
  1841. vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel);
  1842. vmx->host_state.fs_reload_needed = 0;
  1843. } else {
  1844. vmcs_write16(HOST_FS_SELECTOR, 0);
  1845. vmx->host_state.fs_reload_needed = 1;
  1846. }
  1847. savesegment(gs, vmx->host_state.gs_sel);
  1848. if (!(vmx->host_state.gs_sel & 7))
  1849. vmcs_write16(HOST_GS_SELECTOR, vmx->host_state.gs_sel);
  1850. else {
  1851. vmcs_write16(HOST_GS_SELECTOR, 0);
  1852. vmx->host_state.gs_ldt_reload_needed = 1;
  1853. }
  1854. #ifdef CONFIG_X86_64
  1855. savesegment(ds, vmx->host_state.ds_sel);
  1856. savesegment(es, vmx->host_state.es_sel);
  1857. #endif
  1858. #ifdef CONFIG_X86_64
  1859. vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE));
  1860. vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE));
  1861. #else
  1862. vmcs_writel(HOST_FS_BASE, segment_base(vmx->host_state.fs_sel));
  1863. vmcs_writel(HOST_GS_BASE, segment_base(vmx->host_state.gs_sel));
  1864. #endif
  1865. #ifdef CONFIG_X86_64
  1866. rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
  1867. if (is_long_mode(&vmx->vcpu))
  1868. wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
  1869. #endif
  1870. if (boot_cpu_has(X86_FEATURE_MPX))
  1871. rdmsrl(MSR_IA32_BNDCFGS, vmx->host_state.msr_host_bndcfgs);
  1872. for (i = 0; i < vmx->save_nmsrs; ++i)
  1873. kvm_set_shared_msr(vmx->guest_msrs[i].index,
  1874. vmx->guest_msrs[i].data,
  1875. vmx->guest_msrs[i].mask);
  1876. }
  1877. static void __vmx_load_host_state(struct vcpu_vmx *vmx)
  1878. {
  1879. if (!vmx->host_state.loaded)
  1880. return;
  1881. ++vmx->vcpu.stat.host_state_reload;
  1882. vmx->host_state.loaded = 0;
  1883. #ifdef CONFIG_X86_64
  1884. if (is_long_mode(&vmx->vcpu))
  1885. rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
  1886. #endif
  1887. if (vmx->host_state.gs_ldt_reload_needed) {
  1888. kvm_load_ldt(vmx->host_state.ldt_sel);
  1889. #ifdef CONFIG_X86_64
  1890. load_gs_index(vmx->host_state.gs_sel);
  1891. #else
  1892. loadsegment(gs, vmx->host_state.gs_sel);
  1893. #endif
  1894. }
  1895. if (vmx->host_state.fs_reload_needed)
  1896. loadsegment(fs, vmx->host_state.fs_sel);
  1897. #ifdef CONFIG_X86_64
  1898. if (unlikely(vmx->host_state.ds_sel | vmx->host_state.es_sel)) {
  1899. loadsegment(ds, vmx->host_state.ds_sel);
  1900. loadsegment(es, vmx->host_state.es_sel);
  1901. }
  1902. #endif
  1903. reload_tss();
  1904. #ifdef CONFIG_X86_64
  1905. wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
  1906. #endif
  1907. if (vmx->host_state.msr_host_bndcfgs)
  1908. wrmsrl(MSR_IA32_BNDCFGS, vmx->host_state.msr_host_bndcfgs);
  1909. load_gdt(this_cpu_ptr(&host_gdt));
  1910. }
  1911. static void vmx_load_host_state(struct vcpu_vmx *vmx)
  1912. {
  1913. preempt_disable();
  1914. __vmx_load_host_state(vmx);
  1915. preempt_enable();
  1916. }
  1917. static void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
  1918. {
  1919. struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
  1920. struct pi_desc old, new;
  1921. unsigned int dest;
  1922. if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
  1923. !irq_remapping_cap(IRQ_POSTING_CAP) ||
  1924. !kvm_vcpu_apicv_active(vcpu))
  1925. return;
  1926. do {
  1927. old.control = new.control = pi_desc->control;
  1928. /*
  1929. * If 'nv' field is POSTED_INTR_WAKEUP_VECTOR, there
  1930. * are two possible cases:
  1931. * 1. After running 'pre_block', context switch
  1932. * happened. For this case, 'sn' was set in
  1933. * vmx_vcpu_put(), so we need to clear it here.
  1934. * 2. After running 'pre_block', we were blocked,
  1935. * and woken up by some other guy. For this case,
  1936. * we don't need to do anything, 'pi_post_block'
  1937. * will do everything for us. However, we cannot
  1938. * check whether it is case #1 or case #2 here
  1939. * (maybe, not needed), so we also clear sn here,
  1940. * I think it is not a big deal.
  1941. */
  1942. if (pi_desc->nv != POSTED_INTR_WAKEUP_VECTOR) {
  1943. if (vcpu->cpu != cpu) {
  1944. dest = cpu_physical_id(cpu);
  1945. if (x2apic_enabled())
  1946. new.ndst = dest;
  1947. else
  1948. new.ndst = (dest << 8) & 0xFF00;
  1949. }
  1950. /* set 'NV' to 'notification vector' */
  1951. new.nv = POSTED_INTR_VECTOR;
  1952. }
  1953. /* Allow posting non-urgent interrupts */
  1954. new.sn = 0;
  1955. } while (cmpxchg(&pi_desc->control, old.control,
  1956. new.control) != old.control);
  1957. }
  1958. static void decache_tsc_multiplier(struct vcpu_vmx *vmx)
  1959. {
  1960. vmx->current_tsc_ratio = vmx->vcpu.arch.tsc_scaling_ratio;
  1961. vmcs_write64(TSC_MULTIPLIER, vmx->current_tsc_ratio);
  1962. }
  1963. /*
  1964. * Switches to specified vcpu, until a matching vcpu_put(), but assumes
  1965. * vcpu mutex is already taken.
  1966. */
  1967. static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  1968. {
  1969. struct vcpu_vmx *vmx = to_vmx(vcpu);
  1970. u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
  1971. bool already_loaded = vmx->loaded_vmcs->cpu == cpu;
  1972. if (!vmm_exclusive)
  1973. kvm_cpu_vmxon(phys_addr);
  1974. else if (!already_loaded)
  1975. loaded_vmcs_clear(vmx->loaded_vmcs);
  1976. if (!already_loaded) {
  1977. local_irq_disable();
  1978. crash_disable_local_vmclear(cpu);
  1979. /*
  1980. * Read loaded_vmcs->cpu should be before fetching
  1981. * loaded_vmcs->loaded_vmcss_on_cpu_link.
  1982. * See the comments in __loaded_vmcs_clear().
  1983. */
  1984. smp_rmb();
  1985. list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link,
  1986. &per_cpu(loaded_vmcss_on_cpu, cpu));
  1987. crash_enable_local_vmclear(cpu);
  1988. local_irq_enable();
  1989. }
  1990. if (per_cpu(current_vmcs, cpu) != vmx->loaded_vmcs->vmcs) {
  1991. per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
  1992. vmcs_load(vmx->loaded_vmcs->vmcs);
  1993. }
  1994. if (!already_loaded) {
  1995. struct desc_ptr *gdt = this_cpu_ptr(&host_gdt);
  1996. unsigned long sysenter_esp;
  1997. kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
  1998. /*
  1999. * Linux uses per-cpu TSS and GDT, so set these when switching
  2000. * processors.
  2001. */
  2002. vmcs_writel(HOST_TR_BASE, kvm_read_tr_base()); /* 22.2.4 */
  2003. vmcs_writel(HOST_GDTR_BASE, gdt->address); /* 22.2.4 */
  2004. rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
  2005. vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
  2006. vmx->loaded_vmcs->cpu = cpu;
  2007. }
  2008. /* Setup TSC multiplier */
  2009. if (kvm_has_tsc_control &&
  2010. vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio)
  2011. decache_tsc_multiplier(vmx);
  2012. vmx_vcpu_pi_load(vcpu, cpu);
  2013. vmx->host_pkru = read_pkru();
  2014. }
  2015. static void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu)
  2016. {
  2017. struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
  2018. if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
  2019. !irq_remapping_cap(IRQ_POSTING_CAP) ||
  2020. !kvm_vcpu_apicv_active(vcpu))
  2021. return;
  2022. /* Set SN when the vCPU is preempted */
  2023. if (vcpu->preempted)
  2024. pi_set_sn(pi_desc);
  2025. }
  2026. static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
  2027. {
  2028. vmx_vcpu_pi_put(vcpu);
  2029. __vmx_load_host_state(to_vmx(vcpu));
  2030. if (!vmm_exclusive) {
  2031. __loaded_vmcs_clear(to_vmx(vcpu)->loaded_vmcs);
  2032. vcpu->cpu = -1;
  2033. kvm_cpu_vmxoff();
  2034. }
  2035. }
  2036. static void vmx_fpu_activate(struct kvm_vcpu *vcpu)
  2037. {
  2038. ulong cr0;
  2039. if (vcpu->fpu_active)
  2040. return;
  2041. vcpu->fpu_active = 1;
  2042. cr0 = vmcs_readl(GUEST_CR0);
  2043. cr0 &= ~(X86_CR0_TS | X86_CR0_MP);
  2044. cr0 |= kvm_read_cr0_bits(vcpu, X86_CR0_TS | X86_CR0_MP);
  2045. vmcs_writel(GUEST_CR0, cr0);
  2046. update_exception_bitmap(vcpu);
  2047. vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
  2048. if (is_guest_mode(vcpu))
  2049. vcpu->arch.cr0_guest_owned_bits &=
  2050. ~get_vmcs12(vcpu)->cr0_guest_host_mask;
  2051. vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
  2052. }
  2053. static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu);
  2054. /*
  2055. * Return the cr0 value that a nested guest would read. This is a combination
  2056. * of the real cr0 used to run the guest (guest_cr0), and the bits shadowed by
  2057. * its hypervisor (cr0_read_shadow).
  2058. */
  2059. static inline unsigned long nested_read_cr0(struct vmcs12 *fields)
  2060. {
  2061. return (fields->guest_cr0 & ~fields->cr0_guest_host_mask) |
  2062. (fields->cr0_read_shadow & fields->cr0_guest_host_mask);
  2063. }
  2064. static inline unsigned long nested_read_cr4(struct vmcs12 *fields)
  2065. {
  2066. return (fields->guest_cr4 & ~fields->cr4_guest_host_mask) |
  2067. (fields->cr4_read_shadow & fields->cr4_guest_host_mask);
  2068. }
  2069. static void vmx_fpu_deactivate(struct kvm_vcpu *vcpu)
  2070. {
  2071. /* Note that there is no vcpu->fpu_active = 0 here. The caller must
  2072. * set this *before* calling this function.
  2073. */
  2074. vmx_decache_cr0_guest_bits(vcpu);
  2075. vmcs_set_bits(GUEST_CR0, X86_CR0_TS | X86_CR0_MP);
  2076. update_exception_bitmap(vcpu);
  2077. vcpu->arch.cr0_guest_owned_bits = 0;
  2078. vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
  2079. if (is_guest_mode(vcpu)) {
  2080. /*
  2081. * L1's specified read shadow might not contain the TS bit,
  2082. * so now that we turned on shadowing of this bit, we need to
  2083. * set this bit of the shadow. Like in nested_vmx_run we need
  2084. * nested_read_cr0(vmcs12), but vmcs12->guest_cr0 is not yet
  2085. * up-to-date here because we just decached cr0.TS (and we'll
  2086. * only update vmcs12->guest_cr0 on nested exit).
  2087. */
  2088. struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
  2089. vmcs12->guest_cr0 = (vmcs12->guest_cr0 & ~X86_CR0_TS) |
  2090. (vcpu->arch.cr0 & X86_CR0_TS);
  2091. vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12));
  2092. } else
  2093. vmcs_writel(CR0_READ_SHADOW, vcpu->arch.cr0);
  2094. }
  2095. static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
  2096. {
  2097. unsigned long rflags, save_rflags;
  2098. if (!test_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail)) {
  2099. __set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
  2100. rflags = vmcs_readl(GUEST_RFLAGS);
  2101. if (to_vmx(vcpu)->rmode.vm86_active) {
  2102. rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
  2103. save_rflags = to_vmx(vcpu)->rmode.save_rflags;
  2104. rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
  2105. }
  2106. to_vmx(vcpu)->rflags = rflags;
  2107. }
  2108. return to_vmx(vcpu)->rflags;
  2109. }
  2110. static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
  2111. {
  2112. __set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
  2113. to_vmx(vcpu)->rflags = rflags;
  2114. if (to_vmx(vcpu)->rmode.vm86_active) {
  2115. to_vmx(vcpu)->rmode.save_rflags = rflags;
  2116. rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
  2117. }
  2118. vmcs_writel(GUEST_RFLAGS, rflags);
  2119. }
  2120. static u32 vmx_get_pkru(struct kvm_vcpu *vcpu)
  2121. {
  2122. return to_vmx(vcpu)->guest_pkru;
  2123. }
  2124. static u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu)
  2125. {
  2126. u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
  2127. int ret = 0;
  2128. if (interruptibility & GUEST_INTR_STATE_STI)
  2129. ret |= KVM_X86_SHADOW_INT_STI;
  2130. if (interruptibility & GUEST_INTR_STATE_MOV_SS)
  2131. ret |= KVM_X86_SHADOW_INT_MOV_SS;
  2132. return ret;
  2133. }
  2134. static void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
  2135. {
  2136. u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
  2137. u32 interruptibility = interruptibility_old;
  2138. interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);
  2139. if (mask & KVM_X86_SHADOW_INT_MOV_SS)
  2140. interruptibility |= GUEST_INTR_STATE_MOV_SS;
  2141. else if (mask & KVM_X86_SHADOW_INT_STI)
  2142. interruptibility |= GUEST_INTR_STATE_STI;
  2143. if ((interruptibility != interruptibility_old))
  2144. vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
  2145. }
  2146. static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
  2147. {
  2148. unsigned long rip;
  2149. rip = kvm_rip_read(vcpu);
  2150. rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
  2151. kvm_rip_write(vcpu, rip);
  2152. /* skipping an emulated instruction also counts */
  2153. vmx_set_interrupt_shadow(vcpu, 0);
  2154. }
  2155. /*
  2156. * KVM wants to inject page-faults which it got to the guest. This function
  2157. * checks whether in a nested guest, we need to inject them to L1 or L2.
  2158. */
  2159. static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned nr)
  2160. {
  2161. struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
  2162. if (!(vmcs12->exception_bitmap & (1u << nr)))
  2163. return 0;
  2164. nested_vmx_vmexit(vcpu, to_vmx(vcpu)->exit_reason,
  2165. vmcs_read32(VM_EXIT_INTR_INFO),
  2166. vmcs_readl(EXIT_QUALIFICATION));
  2167. return 1;
  2168. }
  2169. static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
  2170. bool has_error_code, u32 error_code,
  2171. bool reinject)
  2172. {
  2173. struct vcpu_vmx *vmx = to_vmx(vcpu);
  2174. u32 intr_info = nr | INTR_INFO_VALID_MASK;
  2175. if (!reinject && is_guest_mode(vcpu) &&
  2176. nested_vmx_check_exception(vcpu, nr))
  2177. return;
  2178. if (has_error_code) {
  2179. vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
  2180. intr_info |= INTR_INFO_DELIVER_CODE_MASK;
  2181. }
  2182. if (vmx->rmode.vm86_active) {
  2183. int inc_eip = 0;
  2184. if (kvm_exception_is_soft(nr))
  2185. inc_eip = vcpu->arch.event_exit_inst_len;
  2186. if (kvm_inject_realmode_interrupt(vcpu, nr, inc_eip) != EMULATE_DONE)
  2187. kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
  2188. return;
  2189. }
  2190. if (kvm_exception_is_soft(nr)) {
  2191. vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
  2192. vmx->vcpu.arch.event_exit_inst_len);
  2193. intr_info |= INTR_TYPE_SOFT_EXCEPTION;
  2194. } else
  2195. intr_info |= INTR_TYPE_HARD_EXCEPTION;
  2196. vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
  2197. }
  2198. static bool vmx_rdtscp_supported(void)
  2199. {
  2200. return cpu_has_vmx_rdtscp();
  2201. }
  2202. static bool vmx_invpcid_supported(void)
  2203. {
  2204. return cpu_has_vmx_invpcid() && enable_ept;
  2205. }
  2206. /*
  2207. * Swap MSR entry in host/guest MSR entry array.
  2208. */
  2209. static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
  2210. {
  2211. struct shared_msr_entry tmp;
  2212. tmp = vmx->guest_msrs[to];
  2213. vmx->guest_msrs[to] = vmx->guest_msrs[from];
  2214. vmx->guest_msrs[from] = tmp;
  2215. }
  2216. static void vmx_set_msr_bitmap(struct kvm_vcpu *vcpu)
  2217. {
  2218. unsigned long *msr_bitmap;
  2219. if (is_guest_mode(vcpu))
  2220. msr_bitmap = to_vmx(vcpu)->nested.msr_bitmap;
  2221. else if (cpu_has_secondary_exec_ctrls() &&
  2222. (vmcs_read32(SECONDARY_VM_EXEC_CONTROL) &
  2223. SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) {
  2224. if (enable_apicv && kvm_vcpu_apicv_active(vcpu)) {
  2225. if (is_long_mode(vcpu))
  2226. msr_bitmap = vmx_msr_bitmap_longmode_x2apic_apicv;
  2227. else
  2228. msr_bitmap = vmx_msr_bitmap_legacy_x2apic_apicv;
  2229. } else {
  2230. if (is_long_mode(vcpu))
  2231. msr_bitmap = vmx_msr_bitmap_longmode_x2apic;
  2232. else
  2233. msr_bitmap = vmx_msr_bitmap_legacy_x2apic;
  2234. }
  2235. } else {
  2236. if (is_long_mode(vcpu))
  2237. msr_bitmap = vmx_msr_bitmap_longmode;
  2238. else
  2239. msr_bitmap = vmx_msr_bitmap_legacy;
  2240. }
  2241. vmcs_write64(MSR_BITMAP, __pa(msr_bitmap));
  2242. }
  2243. /*
  2244. * Set up the vmcs to automatically save and restore system
  2245. * msrs. Don't touch the 64-bit msrs if the guest is in legacy
  2246. * mode, as fiddling with msrs is very expensive.
  2247. */
  2248. static void setup_msrs(struct vcpu_vmx *vmx)
  2249. {
  2250. int save_nmsrs, index;
  2251. save_nmsrs = 0;
  2252. #ifdef CONFIG_X86_64
  2253. if (is_long_mode(&vmx->vcpu)) {
  2254. index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
  2255. if (index >= 0)
  2256. move_msr_up(vmx, index, save_nmsrs++);
  2257. index = __find_msr_index(vmx, MSR_LSTAR);
  2258. if (index >= 0)
  2259. move_msr_up(vmx, index, save_nmsrs++);
  2260. index = __find_msr_index(vmx, MSR_CSTAR);
  2261. if (index >= 0)
  2262. move_msr_up(vmx, index, save_nmsrs++);
  2263. index = __find_msr_index(vmx, MSR_TSC_AUX);
  2264. if (index >= 0 && guest_cpuid_has_rdtscp(&vmx->vcpu))
  2265. move_msr_up(vmx, index, save_nmsrs++);
  2266. /*
  2267. * MSR_STAR is only needed on long mode guests, and only
  2268. * if efer.sce is enabled.
  2269. */
  2270. index = __find_msr_index(vmx, MSR_STAR);
  2271. if ((index >= 0) && (vmx->vcpu.arch.efer & EFER_SCE))
  2272. move_msr_up(vmx, index, save_nmsrs++);
  2273. }
  2274. #endif
  2275. index = __find_msr_index(vmx, MSR_EFER);
  2276. if (index >= 0 && update_transition_efer(vmx, index))
  2277. move_msr_up(vmx, index, save_nmsrs++);
  2278. vmx->save_nmsrs = save_nmsrs;
  2279. if (cpu_has_vmx_msr_bitmap())
  2280. vmx_set_msr_bitmap(&vmx->vcpu);
  2281. }
  2282. /*
  2283. * reads and returns guest's timestamp counter "register"
  2284. * guest_tsc = (host_tsc * tsc multiplier) >> 48 + tsc_offset
  2285. * -- Intel TSC Scaling for Virtualization White Paper, sec 1.3
  2286. */
  2287. static u64 guest_read_tsc(struct kvm_vcpu *vcpu)
  2288. {
  2289. u64 host_tsc, tsc_offset;
  2290. host_tsc = rdtsc();
  2291. tsc_offset = vmcs_read64(TSC_OFFSET);
  2292. return kvm_scale_tsc(vcpu, host_tsc) + tsc_offset;
  2293. }
  2294. /*
  2295. * writes 'offset' into guest's timestamp counter offset register
  2296. */
  2297. static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
  2298. {
  2299. if (is_guest_mode(vcpu)) {
  2300. /*
  2301. * We're here if L1 chose not to trap WRMSR to TSC. According
  2302. * to the spec, this should set L1's TSC; The offset that L1
  2303. * set for L2 remains unchanged, and still needs to be added
  2304. * to the newly set TSC to get L2's TSC.
  2305. */
  2306. struct vmcs12 *vmcs12;
  2307. /* recalculate vmcs02.TSC_OFFSET: */
  2308. vmcs12 = get_vmcs12(vcpu);
  2309. vmcs_write64(TSC_OFFSET, offset +
  2310. (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETING) ?
  2311. vmcs12->tsc_offset : 0));
  2312. } else {
  2313. trace_kvm_write_tsc_offset(vcpu->vcpu_id,
  2314. vmcs_read64(TSC_OFFSET), offset);
  2315. vmcs_write64(TSC_OFFSET, offset);
  2316. }
  2317. }
  2318. static bool guest_cpuid_has_vmx(struct kvm_vcpu *vcpu)
  2319. {
  2320. struct kvm_cpuid_entry2 *best = kvm_find_cpuid_entry(vcpu, 1, 0);
  2321. return best && (best->ecx & (1 << (X86_FEATURE_VMX & 31)));
  2322. }
  2323. /*
  2324. * nested_vmx_allowed() checks whether a guest should be allowed to use VMX
  2325. * instructions and MSRs (i.e., nested VMX). Nested VMX is disabled for
  2326. * all guests if the "nested" module option is off, and can also be disabled
  2327. * for a single guest by disabling its VMX cpuid bit.
  2328. */
  2329. static inline bool nested_vmx_allowed(struct kvm_vcpu *vcpu)
  2330. {
  2331. return nested && guest_cpuid_has_vmx(vcpu);
  2332. }
  2333. /*
  2334. * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be
  2335. * returned for the various VMX controls MSRs when nested VMX is enabled.
  2336. * The same values should also be used to verify that vmcs12 control fields are
  2337. * valid during nested entry from L1 to L2.
  2338. * Each of these control msrs has a low and high 32-bit half: A low bit is on
  2339. * if the corresponding bit in the (32-bit) control field *must* be on, and a
  2340. * bit in the high half is on if the corresponding bit in the control field
  2341. * may be on. See also vmx_control_verify().
  2342. */
  2343. static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx)
  2344. {
  2345. /*
  2346. * Note that as a general rule, the high half of the MSRs (bits in
  2347. * the control fields which may be 1) should be initialized by the
  2348. * intersection of the underlying hardware's MSR (i.e., features which
  2349. * can be supported) and the list of features we want to expose -
  2350. * because they are known to be properly supported in our code.
  2351. * Also, usually, the low half of the MSRs (bits which must be 1) can
  2352. * be set to 0, meaning that L1 may turn off any of these bits. The
  2353. * reason is that if one of these bits is necessary, it will appear
  2354. * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control
  2355. * fields of vmcs01 and vmcs02, will turn these bits off - and
  2356. * nested_vmx_exit_handled() will not pass related exits to L1.
  2357. * These rules have exceptions below.
  2358. */
  2359. /* pin-based controls */
  2360. rdmsr(MSR_IA32_VMX_PINBASED_CTLS,
  2361. vmx->nested.nested_vmx_pinbased_ctls_low,
  2362. vmx->nested.nested_vmx_pinbased_ctls_high);
  2363. vmx->nested.nested_vmx_pinbased_ctls_low |=
  2364. PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
  2365. vmx->nested.nested_vmx_pinbased_ctls_high &=
  2366. PIN_BASED_EXT_INTR_MASK |
  2367. PIN_BASED_NMI_EXITING |
  2368. PIN_BASED_VIRTUAL_NMIS;
  2369. vmx->nested.nested_vmx_pinbased_ctls_high |=
  2370. PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
  2371. PIN_BASED_VMX_PREEMPTION_TIMER;
  2372. if (kvm_vcpu_apicv_active(&vmx->vcpu))
  2373. vmx->nested.nested_vmx_pinbased_ctls_high |=
  2374. PIN_BASED_POSTED_INTR;
  2375. /* exit controls */
  2376. rdmsr(MSR_IA32_VMX_EXIT_CTLS,
  2377. vmx->nested.nested_vmx_exit_ctls_low,
  2378. vmx->nested.nested_vmx_exit_ctls_high);
  2379. vmx->nested.nested_vmx_exit_ctls_low =
  2380. VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
  2381. vmx->nested.nested_vmx_exit_ctls_high &=
  2382. #ifdef CONFIG_X86_64
  2383. VM_EXIT_HOST_ADDR_SPACE_SIZE |
  2384. #endif
  2385. VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT;
  2386. vmx->nested.nested_vmx_exit_ctls_high |=
  2387. VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |
  2388. VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER |
  2389. VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT;
  2390. if (kvm_mpx_supported())
  2391. vmx->nested.nested_vmx_exit_ctls_high |= VM_EXIT_CLEAR_BNDCFGS;
  2392. /* We support free control of debug control saving. */
  2393. vmx->nested.nested_vmx_exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS;
  2394. /* entry controls */
  2395. rdmsr(MSR_IA32_VMX_ENTRY_CTLS,
  2396. vmx->nested.nested_vmx_entry_ctls_low,
  2397. vmx->nested.nested_vmx_entry_ctls_high);
  2398. vmx->nested.nested_vmx_entry_ctls_low =
  2399. VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
  2400. vmx->nested.nested_vmx_entry_ctls_high &=
  2401. #ifdef CONFIG_X86_64
  2402. VM_ENTRY_IA32E_MODE |
  2403. #endif
  2404. VM_ENTRY_LOAD_IA32_PAT;
  2405. vmx->nested.nested_vmx_entry_ctls_high |=
  2406. (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER);
  2407. if (kvm_mpx_supported())
  2408. vmx->nested.nested_vmx_entry_ctls_high |= VM_ENTRY_LOAD_BNDCFGS;
  2409. /* We support free control of debug control loading. */
  2410. vmx->nested.nested_vmx_entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS;
  2411. /* cpu-based controls */
  2412. rdmsr(MSR_IA32_VMX_PROCBASED_CTLS,
  2413. vmx->nested.nested_vmx_procbased_ctls_low,
  2414. vmx->nested.nested_vmx_procbased_ctls_high);
  2415. vmx->nested.nested_vmx_procbased_ctls_low =
  2416. CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
  2417. vmx->nested.nested_vmx_procbased_ctls_high &=
  2418. CPU_BASED_VIRTUAL_INTR_PENDING |
  2419. CPU_BASED_VIRTUAL_NMI_PENDING | CPU_BASED_USE_TSC_OFFSETING |
  2420. CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING |
  2421. CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING |
  2422. CPU_BASED_CR3_STORE_EXITING |
  2423. #ifdef CONFIG_X86_64
  2424. CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING |
  2425. #endif
  2426. CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING |
  2427. CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG |
  2428. CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING |
  2429. CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING |
  2430. CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
  2431. /*
  2432. * We can allow some features even when not supported by the
  2433. * hardware. For example, L1 can specify an MSR bitmap - and we
  2434. * can use it to avoid exits to L1 - even when L0 runs L2
  2435. * without MSR bitmaps.
  2436. */
  2437. vmx->nested.nested_vmx_procbased_ctls_high |=
  2438. CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
  2439. CPU_BASED_USE_MSR_BITMAPS;
  2440. /* We support free control of CR3 access interception. */
  2441. vmx->nested.nested_vmx_procbased_ctls_low &=
  2442. ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING);
  2443. /* secondary cpu-based controls */
  2444. rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2,
  2445. vmx->nested.nested_vmx_secondary_ctls_low,
  2446. vmx->nested.nested_vmx_secondary_ctls_high);
  2447. vmx->nested.nested_vmx_secondary_ctls_low = 0;
  2448. vmx->nested.nested_vmx_secondary_ctls_high &=
  2449. SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
  2450. SECONDARY_EXEC_RDTSCP |
  2451. SECONDARY_EXEC_DESC |
  2452. SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
  2453. SECONDARY_EXEC_ENABLE_VPID |
  2454. SECONDARY_EXEC_APIC_REGISTER_VIRT |
  2455. SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
  2456. SECONDARY_EXEC_WBINVD_EXITING |
  2457. SECONDARY_EXEC_XSAVES;
  2458. if (enable_ept) {
  2459. /* nested EPT: emulate EPT also to L1 */
  2460. vmx->nested.nested_vmx_secondary_ctls_high |=
  2461. SECONDARY_EXEC_ENABLE_EPT;
  2462. vmx->nested.nested_vmx_ept_caps = VMX_EPT_PAGE_WALK_4_BIT |
  2463. VMX_EPTP_WB_BIT | VMX_EPT_2MB_PAGE_BIT |
  2464. VMX_EPT_INVEPT_BIT;
  2465. if (cpu_has_vmx_ept_execute_only())
  2466. vmx->nested.nested_vmx_ept_caps |=
  2467. VMX_EPT_EXECUTE_ONLY_BIT;
  2468. vmx->nested.nested_vmx_ept_caps &= vmx_capability.ept;
  2469. vmx->nested.nested_vmx_ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT |
  2470. VMX_EPT_EXTENT_CONTEXT_BIT;
  2471. } else
  2472. vmx->nested.nested_vmx_ept_caps = 0;
  2473. /*
  2474. * Old versions of KVM use the single-context version without
  2475. * checking for support, so declare that it is supported even
  2476. * though it is treated as global context. The alternative is
  2477. * not failing the single-context invvpid, and it is worse.
  2478. */
  2479. if (enable_vpid)
  2480. vmx->nested.nested_vmx_vpid_caps = VMX_VPID_INVVPID_BIT |
  2481. VMX_VPID_EXTENT_SUPPORTED_MASK;
  2482. else
  2483. vmx->nested.nested_vmx_vpid_caps = 0;
  2484. if (enable_unrestricted_guest)
  2485. vmx->nested.nested_vmx_secondary_ctls_high |=
  2486. SECONDARY_EXEC_UNRESTRICTED_GUEST;
  2487. /* miscellaneous data */
  2488. rdmsr(MSR_IA32_VMX_MISC,
  2489. vmx->nested.nested_vmx_misc_low,
  2490. vmx->nested.nested_vmx_misc_high);
  2491. vmx->nested.nested_vmx_misc_low &= VMX_MISC_SAVE_EFER_LMA;
  2492. vmx->nested.nested_vmx_misc_low |=
  2493. VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE |
  2494. VMX_MISC_ACTIVITY_HLT;
  2495. vmx->nested.nested_vmx_misc_high = 0;
  2496. /*
  2497. * This MSR reports some information about VMX support. We
  2498. * should return information about the VMX we emulate for the
  2499. * guest, and the VMCS structure we give it - not about the
  2500. * VMX support of the underlying hardware.
  2501. */
  2502. vmx->nested.nested_vmx_basic =
  2503. VMCS12_REVISION |
  2504. VMX_BASIC_TRUE_CTLS |
  2505. ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) |
  2506. (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT);
  2507. if (cpu_has_vmx_basic_inout())
  2508. vmx->nested.nested_vmx_basic |= VMX_BASIC_INOUT;
  2509. /*
  2510. * These MSRs specify bits which the guest must keep fixed on
  2511. * while L1 is in VMXON mode (in L1's root mode, or running an L2).
  2512. * We picked the standard core2 setting.
  2513. */
  2514. #define VMXON_CR0_ALWAYSON (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE)
  2515. #define VMXON_CR4_ALWAYSON X86_CR4_VMXE
  2516. vmx->nested.nested_vmx_cr0_fixed0 = VMXON_CR0_ALWAYSON;
  2517. vmx->nested.nested_vmx_cr4_fixed0 = VMXON_CR4_ALWAYSON;
  2518. /* These MSRs specify bits which the guest must keep fixed off. */
  2519. rdmsrl(MSR_IA32_VMX_CR0_FIXED1, vmx->nested.nested_vmx_cr0_fixed1);
  2520. rdmsrl(MSR_IA32_VMX_CR4_FIXED1, vmx->nested.nested_vmx_cr4_fixed1);
  2521. /* highest index: VMX_PREEMPTION_TIMER_VALUE */
  2522. vmx->nested.nested_vmx_vmcs_enum = 0x2e;
  2523. }
  2524. /*
  2525. * if fixed0[i] == 1: val[i] must be 1
  2526. * if fixed1[i] == 0: val[i] must be 0
  2527. */
  2528. static inline bool fixed_bits_valid(u64 val, u64 fixed0, u64 fixed1)
  2529. {
  2530. return ((val & fixed1) | fixed0) == val;
  2531. }
  2532. static inline bool vmx_control_verify(u32 control, u32 low, u32 high)
  2533. {
  2534. return fixed_bits_valid(control, low, high);
  2535. }
  2536. static inline u64 vmx_control_msr(u32 low, u32 high)
  2537. {
  2538. return low | ((u64)high << 32);
  2539. }
  2540. static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask)
  2541. {
  2542. superset &= mask;
  2543. subset &= mask;
  2544. return (superset | subset) == superset;
  2545. }
  2546. static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data)
  2547. {
  2548. const u64 feature_and_reserved =
  2549. /* feature (except bit 48; see below) */
  2550. BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) |
  2551. /* reserved */
  2552. BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56);
  2553. u64 vmx_basic = vmx->nested.nested_vmx_basic;
  2554. if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved))
  2555. return -EINVAL;
  2556. /*
  2557. * KVM does not emulate a version of VMX that constrains physical
  2558. * addresses of VMX structures (e.g. VMCS) to 32-bits.
  2559. */
  2560. if (data & BIT_ULL(48))
  2561. return -EINVAL;
  2562. if (vmx_basic_vmcs_revision_id(vmx_basic) !=
  2563. vmx_basic_vmcs_revision_id(data))
  2564. return -EINVAL;
  2565. if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data))
  2566. return -EINVAL;
  2567. vmx->nested.nested_vmx_basic = data;
  2568. return 0;
  2569. }
  2570. static int
  2571. vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
  2572. {
  2573. u64 supported;
  2574. u32 *lowp, *highp;
  2575. switch (msr_index) {
  2576. case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
  2577. lowp = &vmx->nested.nested_vmx_pinbased_ctls_low;
  2578. highp = &vmx->nested.nested_vmx_pinbased_ctls_high;
  2579. break;
  2580. case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
  2581. lowp = &vmx->nested.nested_vmx_procbased_ctls_low;
  2582. highp = &vmx->nested.nested_vmx_procbased_ctls_high;
  2583. break;
  2584. case MSR_IA32_VMX_TRUE_EXIT_CTLS:
  2585. lowp = &vmx->nested.nested_vmx_exit_ctls_low;
  2586. highp = &vmx->nested.nested_vmx_exit_ctls_high;
  2587. break;
  2588. case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
  2589. lowp = &vmx->nested.nested_vmx_entry_ctls_low;
  2590. highp = &vmx->nested.nested_vmx_entry_ctls_high;
  2591. break;
  2592. case MSR_IA32_VMX_PROCBASED_CTLS2:
  2593. lowp = &vmx->nested.nested_vmx_secondary_ctls_low;
  2594. highp = &vmx->nested.nested_vmx_secondary_ctls_high;
  2595. break;
  2596. default:
  2597. BUG();
  2598. }
  2599. supported = vmx_control_msr(*lowp, *highp);
  2600. /* Check must-be-1 bits are still 1. */
  2601. if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0)))
  2602. return -EINVAL;
  2603. /* Check must-be-0 bits are still 0. */
  2604. if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32)))
  2605. return -EINVAL;
  2606. *lowp = data;
  2607. *highp = data >> 32;
  2608. return 0;
  2609. }
  2610. static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data)
  2611. {
  2612. const u64 feature_and_reserved_bits =
  2613. /* feature */
  2614. BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) |
  2615. BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) |
  2616. /* reserved */
  2617. GENMASK_ULL(13, 9) | BIT_ULL(31);
  2618. u64 vmx_misc;
  2619. vmx_misc = vmx_control_msr(vmx->nested.nested_vmx_misc_low,
  2620. vmx->nested.nested_vmx_misc_high);
  2621. if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits))
  2622. return -EINVAL;
  2623. if ((vmx->nested.nested_vmx_pinbased_ctls_high &
  2624. PIN_BASED_VMX_PREEMPTION_TIMER) &&
  2625. vmx_misc_preemption_timer_rate(data) !=
  2626. vmx_misc_preemption_timer_rate(vmx_misc))
  2627. return -EINVAL;
  2628. if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc))
  2629. return -EINVAL;
  2630. if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc))
  2631. return -EINVAL;
  2632. if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc))
  2633. return -EINVAL;
  2634. vmx->nested.nested_vmx_misc_low = data;
  2635. vmx->nested.nested_vmx_misc_high = data >> 32;
  2636. return 0;
  2637. }
  2638. static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data)
  2639. {
  2640. u64 vmx_ept_vpid_cap;
  2641. vmx_ept_vpid_cap = vmx_control_msr(vmx->nested.nested_vmx_ept_caps,
  2642. vmx->nested.nested_vmx_vpid_caps);
  2643. /* Every bit is either reserved or a feature bit. */
  2644. if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL))
  2645. return -EINVAL;
  2646. vmx->nested.nested_vmx_ept_caps = data;
  2647. vmx->nested.nested_vmx_vpid_caps = data >> 32;
  2648. return 0;
  2649. }
  2650. static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
  2651. {
  2652. u64 *msr;
  2653. switch (msr_index) {
  2654. case MSR_IA32_VMX_CR0_FIXED0:
  2655. msr = &vmx->nested.nested_vmx_cr0_fixed0;
  2656. break;
  2657. case MSR_IA32_VMX_CR4_FIXED0:
  2658. msr = &vmx->nested.nested_vmx_cr4_fixed0;
  2659. break;
  2660. default:
  2661. BUG();
  2662. }
  2663. /*
  2664. * 1 bits (which indicates bits which "must-be-1" during VMX operation)
  2665. * must be 1 in the restored value.
  2666. */
  2667. if (!is_bitwise_subset(data, *msr, -1ULL))
  2668. return -EINVAL;
  2669. *msr = data;
  2670. return 0;
  2671. }
  2672. /*
  2673. * Called when userspace is restoring VMX MSRs.
  2674. *
  2675. * Returns 0 on success, non-0 otherwise.
  2676. */
  2677. static int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
  2678. {
  2679. struct vcpu_vmx *vmx = to_vmx(vcpu);
  2680. switch (msr_index) {
  2681. case MSR_IA32_VMX_BASIC:
  2682. return vmx_restore_vmx_basic(vmx, data);
  2683. case MSR_IA32_VMX_PINBASED_CTLS:
  2684. case MSR_IA32_VMX_PROCBASED_CTLS:
  2685. case MSR_IA32_VMX_EXIT_CTLS:
  2686. case MSR_IA32_VMX_ENTRY_CTLS:
  2687. /*
  2688. * The "non-true" VMX capability MSRs are generated from the
  2689. * "true" MSRs, so we do not support restoring them directly.
  2690. *
  2691. * If userspace wants to emulate VMX_BASIC[55]=0, userspace
  2692. * should restore the "true" MSRs with the must-be-1 bits
  2693. * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND
  2694. * DEFAULT SETTINGS".
  2695. */
  2696. return -EINVAL;
  2697. case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
  2698. case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
  2699. case MSR_IA32_VMX_TRUE_EXIT_CTLS:
  2700. case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
  2701. case MSR_IA32_VMX_PROCBASED_CTLS2:
  2702. return vmx_restore_control_msr(vmx, msr_index, data);
  2703. case MSR_IA32_VMX_MISC:
  2704. return vmx_restore_vmx_misc(vmx, data);
  2705. case MSR_IA32_VMX_CR0_FIXED0:
  2706. case MSR_IA32_VMX_CR4_FIXED0:
  2707. return vmx_restore_fixed0_msr(vmx, msr_index, data);
  2708. case MSR_IA32_VMX_CR0_FIXED1:
  2709. case MSR_IA32_VMX_CR4_FIXED1:
  2710. /*
  2711. * These MSRs are generated based on the vCPU's CPUID, so we
  2712. * do not support restoring them directly.
  2713. */
  2714. return -EINVAL;
  2715. case MSR_IA32_VMX_EPT_VPID_CAP:
  2716. return vmx_restore_vmx_ept_vpid_cap(vmx, data);
  2717. case MSR_IA32_VMX_VMCS_ENUM:
  2718. vmx->nested.nested_vmx_vmcs_enum = data;
  2719. return 0;
  2720. default:
  2721. /*
  2722. * The rest of the VMX capability MSRs do not support restore.
  2723. */
  2724. return -EINVAL;
  2725. }
  2726. }
  2727. /* Returns 0 on success, non-0 otherwise. */
  2728. static int vmx_get_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
  2729. {
  2730. struct vcpu_vmx *vmx = to_vmx(vcpu);
  2731. switch (msr_index) {
  2732. case MSR_IA32_VMX_BASIC:
  2733. *pdata = vmx->nested.nested_vmx_basic;
  2734. break;
  2735. case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
  2736. case MSR_IA32_VMX_PINBASED_CTLS:
  2737. *pdata = vmx_control_msr(
  2738. vmx->nested.nested_vmx_pinbased_ctls_low,
  2739. vmx->nested.nested_vmx_pinbased_ctls_high);
  2740. if (msr_index == MSR_IA32_VMX_PINBASED_CTLS)
  2741. *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
  2742. break;
  2743. case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
  2744. case MSR_IA32_VMX_PROCBASED_CTLS:
  2745. *pdata = vmx_control_msr(
  2746. vmx->nested.nested_vmx_procbased_ctls_low,
  2747. vmx->nested.nested_vmx_procbased_ctls_high);
  2748. if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS)
  2749. *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
  2750. break;
  2751. case MSR_IA32_VMX_TRUE_EXIT_CTLS:
  2752. case MSR_IA32_VMX_EXIT_CTLS:
  2753. *pdata = vmx_control_msr(
  2754. vmx->nested.nested_vmx_exit_ctls_low,
  2755. vmx->nested.nested_vmx_exit_ctls_high);
  2756. if (msr_index == MSR_IA32_VMX_EXIT_CTLS)
  2757. *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
  2758. break;
  2759. case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
  2760. case MSR_IA32_VMX_ENTRY_CTLS:
  2761. *pdata = vmx_control_msr(
  2762. vmx->nested.nested_vmx_entry_ctls_low,
  2763. vmx->nested.nested_vmx_entry_ctls_high);
  2764. if (msr_index == MSR_IA32_VMX_ENTRY_CTLS)
  2765. *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
  2766. break;
  2767. case MSR_IA32_VMX_MISC:
  2768. *pdata = vmx_control_msr(
  2769. vmx->nested.nested_vmx_misc_low,
  2770. vmx->nested.nested_vmx_misc_high);
  2771. break;
  2772. case MSR_IA32_VMX_CR0_FIXED0:
  2773. *pdata = vmx->nested.nested_vmx_cr0_fixed0;
  2774. break;
  2775. case MSR_IA32_VMX_CR0_FIXED1:
  2776. *pdata = vmx->nested.nested_vmx_cr0_fixed1;
  2777. break;
  2778. case MSR_IA32_VMX_CR4_FIXED0:
  2779. *pdata = vmx->nested.nested_vmx_cr4_fixed0;
  2780. break;
  2781. case MSR_IA32_VMX_CR4_FIXED1:
  2782. *pdata = vmx->nested.nested_vmx_cr4_fixed1;
  2783. break;
  2784. case MSR_IA32_VMX_VMCS_ENUM:
  2785. *pdata = vmx->nested.nested_vmx_vmcs_enum;
  2786. break;
  2787. case MSR_IA32_VMX_PROCBASED_CTLS2:
  2788. *pdata = vmx_control_msr(
  2789. vmx->nested.nested_vmx_secondary_ctls_low,
  2790. vmx->nested.nested_vmx_secondary_ctls_high);
  2791. break;
  2792. case MSR_IA32_VMX_EPT_VPID_CAP:
  2793. *pdata = vmx->nested.nested_vmx_ept_caps |
  2794. ((u64)vmx->nested.nested_vmx_vpid_caps << 32);
  2795. break;
  2796. default:
  2797. return 1;
  2798. }
  2799. return 0;
  2800. }
  2801. static inline bool vmx_feature_control_msr_valid(struct kvm_vcpu *vcpu,
  2802. uint64_t val)
  2803. {
  2804. uint64_t valid_bits = to_vmx(vcpu)->msr_ia32_feature_control_valid_bits;
  2805. return !(val & ~valid_bits);
  2806. }
  2807. /*
  2808. * Reads an msr value (of 'msr_index') into 'pdata'.
  2809. * Returns 0 on success, non-0 otherwise.
  2810. * Assumes vcpu_load() was already called.
  2811. */
  2812. static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
  2813. {
  2814. struct shared_msr_entry *msr;
  2815. switch (msr_info->index) {
  2816. #ifdef CONFIG_X86_64
  2817. case MSR_FS_BASE:
  2818. msr_info->data = vmcs_readl(GUEST_FS_BASE);
  2819. break;
  2820. case MSR_GS_BASE:
  2821. msr_info->data = vmcs_readl(GUEST_GS_BASE);
  2822. break;
  2823. case MSR_KERNEL_GS_BASE:
  2824. vmx_load_host_state(to_vmx(vcpu));
  2825. msr_info->data = to_vmx(vcpu)->msr_guest_kernel_gs_base;
  2826. break;
  2827. #endif
  2828. case MSR_EFER:
  2829. return kvm_get_msr_common(vcpu, msr_info);
  2830. case MSR_IA32_TSC:
  2831. msr_info->data = guest_read_tsc(vcpu);
  2832. break;
  2833. case MSR_IA32_SYSENTER_CS:
  2834. msr_info->data = vmcs_read32(GUEST_SYSENTER_CS);
  2835. break;
  2836. case MSR_IA32_SYSENTER_EIP:
  2837. msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP);
  2838. break;
  2839. case MSR_IA32_SYSENTER_ESP:
  2840. msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP);
  2841. break;
  2842. case MSR_IA32_BNDCFGS:
  2843. if (!kvm_mpx_supported())
  2844. return 1;
  2845. msr_info->data = vmcs_read64(GUEST_BNDCFGS);
  2846. break;
  2847. case MSR_IA32_MCG_EXT_CTL:
  2848. if (!msr_info->host_initiated &&
  2849. !(to_vmx(vcpu)->msr_ia32_feature_control &
  2850. FEATURE_CONTROL_LMCE))
  2851. return 1;
  2852. msr_info->data = vcpu->arch.mcg_ext_ctl;
  2853. break;
  2854. case MSR_IA32_FEATURE_CONTROL:
  2855. msr_info->data = to_vmx(vcpu)->msr_ia32_feature_control;
  2856. break;
  2857. case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
  2858. if (!nested_vmx_allowed(vcpu))
  2859. return 1;
  2860. return vmx_get_vmx_msr(vcpu, msr_info->index, &msr_info->data);
  2861. case MSR_IA32_XSS:
  2862. if (!vmx_xsaves_supported())
  2863. return 1;
  2864. msr_info->data = vcpu->arch.ia32_xss;
  2865. break;
  2866. case MSR_TSC_AUX:
  2867. if (!guest_cpuid_has_rdtscp(vcpu) && !msr_info->host_initiated)
  2868. return 1;
  2869. /* Otherwise falls through */
  2870. default:
  2871. msr = find_msr_entry(to_vmx(vcpu), msr_info->index);
  2872. if (msr) {
  2873. msr_info->data = msr->data;
  2874. break;
  2875. }
  2876. return kvm_get_msr_common(vcpu, msr_info);
  2877. }
  2878. return 0;
  2879. }
  2880. static void vmx_leave_nested(struct kvm_vcpu *vcpu);
  2881. /*
  2882. * Writes msr value into into the appropriate "register".
  2883. * Returns 0 on success, non-0 otherwise.
  2884. * Assumes vcpu_load() was already called.
  2885. */
  2886. static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
  2887. {
  2888. struct vcpu_vmx *vmx = to_vmx(vcpu);
  2889. struct shared_msr_entry *msr;
  2890. int ret = 0;
  2891. u32 msr_index = msr_info->index;
  2892. u64 data = msr_info->data;
  2893. switch (msr_index) {
  2894. case MSR_EFER:
  2895. ret = kvm_set_msr_common(vcpu, msr_info);
  2896. break;
  2897. #ifdef CONFIG_X86_64
  2898. case MSR_FS_BASE:
  2899. vmx_segment_cache_clear(vmx);
  2900. vmcs_writel(GUEST_FS_BASE, data);
  2901. break;
  2902. case MSR_GS_BASE:
  2903. vmx_segment_cache_clear(vmx);
  2904. vmcs_writel(GUEST_GS_BASE, data);
  2905. break;
  2906. case MSR_KERNEL_GS_BASE:
  2907. vmx_load_host_state(vmx);
  2908. vmx->msr_guest_kernel_gs_base = data;
  2909. break;
  2910. #endif
  2911. case MSR_IA32_SYSENTER_CS:
  2912. vmcs_write32(GUEST_SYSENTER_CS, data);
  2913. break;
  2914. case MSR_IA32_SYSENTER_EIP:
  2915. vmcs_writel(GUEST_SYSENTER_EIP, data);
  2916. break;
  2917. case MSR_IA32_SYSENTER_ESP:
  2918. vmcs_writel(GUEST_SYSENTER_ESP, data);
  2919. break;
  2920. case MSR_IA32_BNDCFGS:
  2921. if (!kvm_mpx_supported())
  2922. return 1;
  2923. vmcs_write64(GUEST_BNDCFGS, data);
  2924. break;
  2925. case MSR_IA32_TSC:
  2926. kvm_write_tsc(vcpu, msr_info);
  2927. break;
  2928. case MSR_IA32_CR_PAT:
  2929. if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
  2930. if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
  2931. return 1;
  2932. vmcs_write64(GUEST_IA32_PAT, data);
  2933. vcpu->arch.pat = data;
  2934. break;
  2935. }
  2936. ret = kvm_set_msr_common(vcpu, msr_info);
  2937. break;
  2938. case MSR_IA32_TSC_ADJUST:
  2939. ret = kvm_set_msr_common(vcpu, msr_info);
  2940. break;
  2941. case MSR_IA32_MCG_EXT_CTL:
  2942. if ((!msr_info->host_initiated &&
  2943. !(to_vmx(vcpu)->msr_ia32_feature_control &
  2944. FEATURE_CONTROL_LMCE)) ||
  2945. (data & ~MCG_EXT_CTL_LMCE_EN))
  2946. return 1;
  2947. vcpu->arch.mcg_ext_ctl = data;
  2948. break;
  2949. case MSR_IA32_FEATURE_CONTROL:
  2950. if (!vmx_feature_control_msr_valid(vcpu, data) ||
  2951. (to_vmx(vcpu)->msr_ia32_feature_control &
  2952. FEATURE_CONTROL_LOCKED && !msr_info->host_initiated))
  2953. return 1;
  2954. vmx->msr_ia32_feature_control = data;
  2955. if (msr_info->host_initiated && data == 0)
  2956. vmx_leave_nested(vcpu);
  2957. break;
  2958. case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
  2959. if (!msr_info->host_initiated)
  2960. return 1; /* they are read-only */
  2961. if (!nested_vmx_allowed(vcpu))
  2962. return 1;
  2963. return vmx_set_vmx_msr(vcpu, msr_index, data);
  2964. case MSR_IA32_XSS:
  2965. if (!vmx_xsaves_supported())
  2966. return 1;
  2967. /*
  2968. * The only supported bit as of Skylake is bit 8, but
  2969. * it is not supported on KVM.
  2970. */
  2971. if (data != 0)
  2972. return 1;
  2973. vcpu->arch.ia32_xss = data;
  2974. if (vcpu->arch.ia32_xss != host_xss)
  2975. add_atomic_switch_msr(vmx, MSR_IA32_XSS,
  2976. vcpu->arch.ia32_xss, host_xss);
  2977. else
  2978. clear_atomic_switch_msr(vmx, MSR_IA32_XSS);
  2979. break;
  2980. case MSR_TSC_AUX:
  2981. if (!guest_cpuid_has_rdtscp(vcpu) && !msr_info->host_initiated)
  2982. return 1;
  2983. /* Check reserved bit, higher 32 bits should be zero */
  2984. if ((data >> 32) != 0)
  2985. return 1;
  2986. /* Otherwise falls through */
  2987. default:
  2988. msr = find_msr_entry(vmx, msr_index);
  2989. if (msr) {
  2990. u64 old_msr_data = msr->data;
  2991. msr->data = data;
  2992. if (msr - vmx->guest_msrs < vmx->save_nmsrs) {
  2993. preempt_disable();
  2994. ret = kvm_set_shared_msr(msr->index, msr->data,
  2995. msr->mask);
  2996. preempt_enable();
  2997. if (ret)
  2998. msr->data = old_msr_data;
  2999. }
  3000. break;
  3001. }
  3002. ret = kvm_set_msr_common(vcpu, msr_info);
  3003. }
  3004. return ret;
  3005. }
  3006. static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
  3007. {
  3008. __set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
  3009. switch (reg) {
  3010. case VCPU_REGS_RSP:
  3011. vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
  3012. break;
  3013. case VCPU_REGS_RIP:
  3014. vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
  3015. break;
  3016. case VCPU_EXREG_PDPTR:
  3017. if (enable_ept)
  3018. ept_save_pdptrs(vcpu);
  3019. break;
  3020. default:
  3021. break;
  3022. }
  3023. }
  3024. static __init int cpu_has_kvm_support(void)
  3025. {
  3026. return cpu_has_vmx();
  3027. }
  3028. static __init int vmx_disabled_by_bios(void)
  3029. {
  3030. u64 msr;
  3031. rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
  3032. if (msr & FEATURE_CONTROL_LOCKED) {
  3033. /* launched w/ TXT and VMX disabled */
  3034. if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
  3035. && tboot_enabled())
  3036. return 1;
  3037. /* launched w/o TXT and VMX only enabled w/ TXT */
  3038. if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
  3039. && (msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
  3040. && !tboot_enabled()) {
  3041. printk(KERN_WARNING "kvm: disable TXT in the BIOS or "
  3042. "activate TXT before enabling KVM\n");
  3043. return 1;
  3044. }
  3045. /* launched w/o TXT and VMX disabled */
  3046. if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
  3047. && !tboot_enabled())
  3048. return 1;
  3049. }
  3050. return 0;
  3051. }
  3052. static void kvm_cpu_vmxon(u64 addr)
  3053. {
  3054. intel_pt_handle_vmx(1);
  3055. asm volatile (ASM_VMX_VMXON_RAX
  3056. : : "a"(&addr), "m"(addr)
  3057. : "memory", "cc");
  3058. }
  3059. static int hardware_enable(void)
  3060. {
  3061. int cpu = raw_smp_processor_id();
  3062. u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
  3063. u64 old, test_bits;
  3064. if (cr4_read_shadow() & X86_CR4_VMXE)
  3065. return -EBUSY;
  3066. INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu));
  3067. INIT_LIST_HEAD(&per_cpu(blocked_vcpu_on_cpu, cpu));
  3068. spin_lock_init(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
  3069. /*
  3070. * Now we can enable the vmclear operation in kdump
  3071. * since the loaded_vmcss_on_cpu list on this cpu
  3072. * has been initialized.
  3073. *
  3074. * Though the cpu is not in VMX operation now, there
  3075. * is no problem to enable the vmclear operation
  3076. * for the loaded_vmcss_on_cpu list is empty!
  3077. */
  3078. crash_enable_local_vmclear(cpu);
  3079. rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
  3080. test_bits = FEATURE_CONTROL_LOCKED;
  3081. test_bits |= FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
  3082. if (tboot_enabled())
  3083. test_bits |= FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX;
  3084. if ((old & test_bits) != test_bits) {
  3085. /* enable and lock */
  3086. wrmsrl(MSR_IA32_FEATURE_CONTROL, old | test_bits);
  3087. }
  3088. cr4_set_bits(X86_CR4_VMXE);
  3089. if (vmm_exclusive) {
  3090. kvm_cpu_vmxon(phys_addr);
  3091. ept_sync_global();
  3092. }
  3093. native_store_gdt(this_cpu_ptr(&host_gdt));
  3094. return 0;
  3095. }
  3096. static void vmclear_local_loaded_vmcss(void)
  3097. {
  3098. int cpu = raw_smp_processor_id();
  3099. struct loaded_vmcs *v, *n;
  3100. list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu),
  3101. loaded_vmcss_on_cpu_link)
  3102. __loaded_vmcs_clear(v);
  3103. }
  3104. /* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
  3105. * tricks.
  3106. */
  3107. static void kvm_cpu_vmxoff(void)
  3108. {
  3109. asm volatile (__ex(ASM_VMX_VMXOFF) : : : "cc");
  3110. intel_pt_handle_vmx(0);
  3111. }
  3112. static void hardware_disable(void)
  3113. {
  3114. if (vmm_exclusive) {
  3115. vmclear_local_loaded_vmcss();
  3116. kvm_cpu_vmxoff();
  3117. }
  3118. cr4_clear_bits(X86_CR4_VMXE);
  3119. }
  3120. static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
  3121. u32 msr, u32 *result)
  3122. {
  3123. u32 vmx_msr_low, vmx_msr_high;
  3124. u32 ctl = ctl_min | ctl_opt;
  3125. rdmsr(msr, vmx_msr_low, vmx_msr_high);
  3126. ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
  3127. ctl |= vmx_msr_low; /* bit == 1 in low word ==> must be one */
  3128. /* Ensure minimum (required) set of control bits are supported. */
  3129. if (ctl_min & ~ctl)
  3130. return -EIO;
  3131. *result = ctl;
  3132. return 0;
  3133. }
  3134. static __init bool allow_1_setting(u32 msr, u32 ctl)
  3135. {
  3136. u32 vmx_msr_low, vmx_msr_high;
  3137. rdmsr(msr, vmx_msr_low, vmx_msr_high);
  3138. return vmx_msr_high & ctl;
  3139. }
  3140. static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
  3141. {
  3142. u32 vmx_msr_low, vmx_msr_high;
  3143. u32 min, opt, min2, opt2;
  3144. u32 _pin_based_exec_control = 0;
  3145. u32 _cpu_based_exec_control = 0;
  3146. u32 _cpu_based_2nd_exec_control = 0;
  3147. u32 _vmexit_control = 0;
  3148. u32 _vmentry_control = 0;
  3149. min = CPU_BASED_HLT_EXITING |
  3150. #ifdef CONFIG_X86_64
  3151. CPU_BASED_CR8_LOAD_EXITING |
  3152. CPU_BASED_CR8_STORE_EXITING |
  3153. #endif
  3154. CPU_BASED_CR3_LOAD_EXITING |
  3155. CPU_BASED_CR3_STORE_EXITING |
  3156. CPU_BASED_USE_IO_BITMAPS |
  3157. CPU_BASED_MOV_DR_EXITING |
  3158. CPU_BASED_USE_TSC_OFFSETING |
  3159. CPU_BASED_MWAIT_EXITING |
  3160. CPU_BASED_MONITOR_EXITING |
  3161. CPU_BASED_INVLPG_EXITING |
  3162. CPU_BASED_RDPMC_EXITING;
  3163. opt = CPU_BASED_TPR_SHADOW |
  3164. CPU_BASED_USE_MSR_BITMAPS |
  3165. CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
  3166. if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
  3167. &_cpu_based_exec_control) < 0)
  3168. return -EIO;
  3169. #ifdef CONFIG_X86_64
  3170. if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
  3171. _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
  3172. ~CPU_BASED_CR8_STORE_EXITING;
  3173. #endif
  3174. if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
  3175. min2 = 0;
  3176. opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
  3177. SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
  3178. SECONDARY_EXEC_WBINVD_EXITING |
  3179. SECONDARY_EXEC_ENABLE_VPID |
  3180. SECONDARY_EXEC_ENABLE_EPT |
  3181. SECONDARY_EXEC_UNRESTRICTED_GUEST |
  3182. SECONDARY_EXEC_PAUSE_LOOP_EXITING |
  3183. SECONDARY_EXEC_RDTSCP |
  3184. SECONDARY_EXEC_ENABLE_INVPCID |
  3185. SECONDARY_EXEC_APIC_REGISTER_VIRT |
  3186. SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
  3187. SECONDARY_EXEC_SHADOW_VMCS |
  3188. SECONDARY_EXEC_XSAVES |
  3189. SECONDARY_EXEC_ENABLE_PML |
  3190. SECONDARY_EXEC_TSC_SCALING;
  3191. if (adjust_vmx_controls(min2, opt2,
  3192. MSR_IA32_VMX_PROCBASED_CTLS2,
  3193. &_cpu_based_2nd_exec_control) < 0)
  3194. return -EIO;
  3195. }
  3196. #ifndef CONFIG_X86_64
  3197. if (!(_cpu_based_2nd_exec_control &
  3198. SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
  3199. _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
  3200. #endif
  3201. if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
  3202. _cpu_based_2nd_exec_control &= ~(
  3203. SECONDARY_EXEC_APIC_REGISTER_VIRT |
  3204. SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
  3205. SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
  3206. if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
  3207. /* CR3 accesses and invlpg don't need to cause VM Exits when EPT
  3208. enabled */
  3209. _cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
  3210. CPU_BASED_CR3_STORE_EXITING |
  3211. CPU_BASED_INVLPG_EXITING);
  3212. rdmsr(MSR_IA32_VMX_EPT_VPID_CAP,
  3213. vmx_capability.ept, vmx_capability.vpid);
  3214. }
  3215. min = VM_EXIT_SAVE_DEBUG_CONTROLS | VM_EXIT_ACK_INTR_ON_EXIT;
  3216. #ifdef CONFIG_X86_64
  3217. min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
  3218. #endif
  3219. opt = VM_EXIT_SAVE_IA32_PAT | VM_EXIT_LOAD_IA32_PAT |
  3220. VM_EXIT_CLEAR_BNDCFGS;
  3221. if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
  3222. &_vmexit_control) < 0)
  3223. return -EIO;
  3224. min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
  3225. opt = PIN_BASED_VIRTUAL_NMIS | PIN_BASED_POSTED_INTR |
  3226. PIN_BASED_VMX_PREEMPTION_TIMER;
  3227. if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
  3228. &_pin_based_exec_control) < 0)
  3229. return -EIO;
  3230. if (cpu_has_broken_vmx_preemption_timer())
  3231. _pin_based_exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
  3232. if (!(_cpu_based_2nd_exec_control &
  3233. SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY))
  3234. _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR;
  3235. min = VM_ENTRY_LOAD_DEBUG_CONTROLS;
  3236. opt = VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_BNDCFGS;
  3237. if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
  3238. &_vmentry_control) < 0)
  3239. return -EIO;
  3240. rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
  3241. /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
  3242. if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
  3243. return -EIO;
  3244. #ifdef CONFIG_X86_64
  3245. /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
  3246. if (vmx_msr_high & (1u<<16))
  3247. return -EIO;
  3248. #endif
  3249. /* Require Write-Back (WB) memory type for VMCS accesses. */
  3250. if (((vmx_msr_high >> 18) & 15) != 6)
  3251. return -EIO;
  3252. vmcs_conf->size = vmx_msr_high & 0x1fff;
  3253. vmcs_conf->order = get_order(vmcs_conf->size);
  3254. vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff;
  3255. vmcs_conf->revision_id = vmx_msr_low;
  3256. vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
  3257. vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
  3258. vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
  3259. vmcs_conf->vmexit_ctrl = _vmexit_control;
  3260. vmcs_conf->vmentry_ctrl = _vmentry_control;
  3261. cpu_has_load_ia32_efer =
  3262. allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
  3263. VM_ENTRY_LOAD_IA32_EFER)
  3264. && allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
  3265. VM_EXIT_LOAD_IA32_EFER);
  3266. cpu_has_load_perf_global_ctrl =
  3267. allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
  3268. VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
  3269. && allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
  3270. VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
  3271. /*
  3272. * Some cpus support VM_ENTRY_(LOAD|SAVE)_IA32_PERF_GLOBAL_CTRL
  3273. * but due to errata below it can't be used. Workaround is to use
  3274. * msr load mechanism to switch IA32_PERF_GLOBAL_CTRL.
  3275. *
  3276. * VM Exit May Incorrectly Clear IA32_PERF_GLOBAL_CTRL [34:32]
  3277. *
  3278. * AAK155 (model 26)
  3279. * AAP115 (model 30)
  3280. * AAT100 (model 37)
  3281. * BC86,AAY89,BD102 (model 44)
  3282. * BA97 (model 46)
  3283. *
  3284. */
  3285. if (cpu_has_load_perf_global_ctrl && boot_cpu_data.x86 == 0x6) {
  3286. switch (boot_cpu_data.x86_model) {
  3287. case 26:
  3288. case 30:
  3289. case 37:
  3290. case 44:
  3291. case 46:
  3292. cpu_has_load_perf_global_ctrl = false;
  3293. printk_once(KERN_WARNING"kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL "
  3294. "does not work properly. Using workaround\n");
  3295. break;
  3296. default:
  3297. break;
  3298. }
  3299. }
  3300. if (boot_cpu_has(X86_FEATURE_XSAVES))
  3301. rdmsrl(MSR_IA32_XSS, host_xss);
  3302. return 0;
  3303. }
  3304. static struct vmcs *alloc_vmcs_cpu(int cpu)
  3305. {
  3306. int node = cpu_to_node(cpu);
  3307. struct page *pages;
  3308. struct vmcs *vmcs;
  3309. pages = __alloc_pages_node(node, GFP_KERNEL, vmcs_config.order);
  3310. if (!pages)
  3311. return NULL;
  3312. vmcs = page_address(pages);
  3313. memset(vmcs, 0, vmcs_config.size);
  3314. vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */
  3315. return vmcs;
  3316. }
  3317. static struct vmcs *alloc_vmcs(void)
  3318. {
  3319. return alloc_vmcs_cpu(raw_smp_processor_id());
  3320. }
  3321. static void free_vmcs(struct vmcs *vmcs)
  3322. {
  3323. free_pages((unsigned long)vmcs, vmcs_config.order);
  3324. }
  3325. /*
  3326. * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded
  3327. */
  3328. static void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
  3329. {
  3330. if (!loaded_vmcs->vmcs)
  3331. return;
  3332. loaded_vmcs_clear(loaded_vmcs);
  3333. free_vmcs(loaded_vmcs->vmcs);
  3334. loaded_vmcs->vmcs = NULL;
  3335. WARN_ON(loaded_vmcs->shadow_vmcs != NULL);
  3336. }
  3337. static void free_kvm_area(void)
  3338. {
  3339. int cpu;
  3340. for_each_possible_cpu(cpu) {
  3341. free_vmcs(per_cpu(vmxarea, cpu));
  3342. per_cpu(vmxarea, cpu) = NULL;
  3343. }
  3344. }
  3345. static void init_vmcs_shadow_fields(void)
  3346. {
  3347. int i, j;
  3348. /* No checks for read only fields yet */
  3349. for (i = j = 0; i < max_shadow_read_write_fields; i++) {
  3350. switch (shadow_read_write_fields[i]) {
  3351. case GUEST_BNDCFGS:
  3352. if (!kvm_mpx_supported())
  3353. continue;
  3354. break;
  3355. default:
  3356. break;
  3357. }
  3358. if (j < i)
  3359. shadow_read_write_fields[j] =
  3360. shadow_read_write_fields[i];
  3361. j++;
  3362. }
  3363. max_shadow_read_write_fields = j;
  3364. /* shadowed fields guest access without vmexit */
  3365. for (i = 0; i < max_shadow_read_write_fields; i++) {
  3366. clear_bit(shadow_read_write_fields[i],
  3367. vmx_vmwrite_bitmap);
  3368. clear_bit(shadow_read_write_fields[i],
  3369. vmx_vmread_bitmap);
  3370. }
  3371. for (i = 0; i < max_shadow_read_only_fields; i++)
  3372. clear_bit(shadow_read_only_fields[i],
  3373. vmx_vmread_bitmap);
  3374. }
  3375. static __init int alloc_kvm_area(void)
  3376. {
  3377. int cpu;
  3378. for_each_possible_cpu(cpu) {
  3379. struct vmcs *vmcs;
  3380. vmcs = alloc_vmcs_cpu(cpu);
  3381. if (!vmcs) {
  3382. free_kvm_area();
  3383. return -ENOMEM;
  3384. }
  3385. per_cpu(vmxarea, cpu) = vmcs;
  3386. }
  3387. return 0;
  3388. }
  3389. static bool emulation_required(struct kvm_vcpu *vcpu)
  3390. {
  3391. return emulate_invalid_guest_state && !guest_state_valid(vcpu);
  3392. }
  3393. static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg,
  3394. struct kvm_segment *save)
  3395. {
  3396. if (!emulate_invalid_guest_state) {
  3397. /*
  3398. * CS and SS RPL should be equal during guest entry according
  3399. * to VMX spec, but in reality it is not always so. Since vcpu
  3400. * is in the middle of the transition from real mode to
  3401. * protected mode it is safe to assume that RPL 0 is a good
  3402. * default value.
  3403. */
  3404. if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS)
  3405. save->selector &= ~SEGMENT_RPL_MASK;
  3406. save->dpl = save->selector & SEGMENT_RPL_MASK;
  3407. save->s = 1;
  3408. }
  3409. vmx_set_segment(vcpu, save, seg);
  3410. }
  3411. static void enter_pmode(struct kvm_vcpu *vcpu)
  3412. {
  3413. unsigned long flags;
  3414. struct vcpu_vmx *vmx = to_vmx(vcpu);
  3415. /*
  3416. * Update real mode segment cache. It may be not up-to-date if sement
  3417. * register was written while vcpu was in a guest mode.
  3418. */
  3419. vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
  3420. vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
  3421. vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
  3422. vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
  3423. vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
  3424. vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
  3425. vmx->rmode.vm86_active = 0;
  3426. vmx_segment_cache_clear(vmx);
  3427. vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
  3428. flags = vmcs_readl(GUEST_RFLAGS);
  3429. flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
  3430. flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
  3431. vmcs_writel(GUEST_RFLAGS, flags);
  3432. vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
  3433. (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
  3434. update_exception_bitmap(vcpu);
  3435. fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
  3436. fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
  3437. fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
  3438. fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
  3439. fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
  3440. fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
  3441. }
  3442. static void fix_rmode_seg(int seg, struct kvm_segment *save)
  3443. {
  3444. const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
  3445. struct kvm_segment var = *save;
  3446. var.dpl = 0x3;
  3447. if (seg == VCPU_SREG_CS)
  3448. var.type = 0x3;
  3449. if (!emulate_invalid_guest_state) {
  3450. var.selector = var.base >> 4;
  3451. var.base = var.base & 0xffff0;
  3452. var.limit = 0xffff;
  3453. var.g = 0;
  3454. var.db = 0;
  3455. var.present = 1;
  3456. var.s = 1;
  3457. var.l = 0;
  3458. var.unusable = 0;
  3459. var.type = 0x3;
  3460. var.avl = 0;
  3461. if (save->base & 0xf)
  3462. printk_once(KERN_WARNING "kvm: segment base is not "
  3463. "paragraph aligned when entering "
  3464. "protected mode (seg=%d)", seg);
  3465. }
  3466. vmcs_write16(sf->selector, var.selector);
  3467. vmcs_write32(sf->base, var.base);
  3468. vmcs_write32(sf->limit, var.limit);
  3469. vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var));
  3470. }
  3471. static void enter_rmode(struct kvm_vcpu *vcpu)
  3472. {
  3473. unsigned long flags;
  3474. struct vcpu_vmx *vmx = to_vmx(vcpu);
  3475. vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
  3476. vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
  3477. vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
  3478. vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
  3479. vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
  3480. vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
  3481. vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
  3482. vmx->rmode.vm86_active = 1;
  3483. /*
  3484. * Very old userspace does not call KVM_SET_TSS_ADDR before entering
  3485. * vcpu. Warn the user that an update is overdue.
  3486. */
  3487. if (!vcpu->kvm->arch.tss_addr)
  3488. printk_once(KERN_WARNING "kvm: KVM_SET_TSS_ADDR need to be "
  3489. "called before entering vcpu\n");
  3490. vmx_segment_cache_clear(vmx);
  3491. vmcs_writel(GUEST_TR_BASE, vcpu->kvm->arch.tss_addr);
  3492. vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
  3493. vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
  3494. flags = vmcs_readl(GUEST_RFLAGS);
  3495. vmx->rmode.save_rflags = flags;
  3496. flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
  3497. vmcs_writel(GUEST_RFLAGS, flags);
  3498. vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
  3499. update_exception_bitmap(vcpu);
  3500. fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
  3501. fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
  3502. fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
  3503. fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
  3504. fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
  3505. fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
  3506. kvm_mmu_reset_context(vcpu);
  3507. }
  3508. static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
  3509. {
  3510. struct vcpu_vmx *vmx = to_vmx(vcpu);
  3511. struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);
  3512. if (!msr)
  3513. return;
  3514. /*
  3515. * Force kernel_gs_base reloading before EFER changes, as control
  3516. * of this msr depends on is_long_mode().
  3517. */
  3518. vmx_load_host_state(to_vmx(vcpu));
  3519. vcpu->arch.efer = efer;
  3520. if (efer & EFER_LMA) {
  3521. vm_entry_controls_setbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
  3522. msr->data = efer;
  3523. } else {
  3524. vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
  3525. msr->data = efer & ~EFER_LME;
  3526. }
  3527. setup_msrs(vmx);
  3528. }
  3529. #ifdef CONFIG_X86_64
  3530. static void enter_lmode(struct kvm_vcpu *vcpu)
  3531. {
  3532. u32 guest_tr_ar;
  3533. vmx_segment_cache_clear(to_vmx(vcpu));
  3534. guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
  3535. if ((guest_tr_ar & VMX_AR_TYPE_MASK) != VMX_AR_TYPE_BUSY_64_TSS) {
  3536. pr_debug_ratelimited("%s: tss fixup for long mode. \n",
  3537. __func__);
  3538. vmcs_write32(GUEST_TR_AR_BYTES,
  3539. (guest_tr_ar & ~VMX_AR_TYPE_MASK)
  3540. | VMX_AR_TYPE_BUSY_64_TSS);
  3541. }
  3542. vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA);
  3543. }
  3544. static void exit_lmode(struct kvm_vcpu *vcpu)
  3545. {
  3546. vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
  3547. vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA);
  3548. }
  3549. #endif
  3550. static inline void __vmx_flush_tlb(struct kvm_vcpu *vcpu, int vpid)
  3551. {
  3552. vpid_sync_context(vpid);
  3553. if (enable_ept) {
  3554. if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
  3555. return;
  3556. ept_sync_context(construct_eptp(vcpu->arch.mmu.root_hpa));
  3557. }
  3558. }
  3559. static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
  3560. {
  3561. __vmx_flush_tlb(vcpu, to_vmx(vcpu)->vpid);
  3562. }
  3563. static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
  3564. {
  3565. ulong cr0_guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;
  3566. vcpu->arch.cr0 &= ~cr0_guest_owned_bits;
  3567. vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits;
  3568. }
  3569. static void vmx_decache_cr3(struct kvm_vcpu *vcpu)
  3570. {
  3571. if (enable_ept && is_paging(vcpu))
  3572. vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
  3573. __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
  3574. }
  3575. static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
  3576. {
  3577. ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
  3578. vcpu->arch.cr4 &= ~cr4_guest_owned_bits;
  3579. vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & cr4_guest_owned_bits;
  3580. }
  3581. static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
  3582. {
  3583. struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
  3584. if (!test_bit(VCPU_EXREG_PDPTR,
  3585. (unsigned long *)&vcpu->arch.regs_dirty))
  3586. return;
  3587. if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
  3588. vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]);
  3589. vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]);
  3590. vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]);
  3591. vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]);
  3592. }
  3593. }
  3594. static void ept_save_pdptrs(struct kvm_vcpu *vcpu)
  3595. {
  3596. struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
  3597. if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
  3598. mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
  3599. mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
  3600. mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
  3601. mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
  3602. }
  3603. __set_bit(VCPU_EXREG_PDPTR,
  3604. (unsigned long *)&vcpu->arch.regs_avail);
  3605. __set_bit(VCPU_EXREG_PDPTR,
  3606. (unsigned long *)&vcpu->arch.regs_dirty);
  3607. }
  3608. static bool nested_guest_cr0_valid(struct kvm_vcpu *vcpu, unsigned long val)
  3609. {
  3610. u64 fixed0 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed0;
  3611. u64 fixed1 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed1;
  3612. struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
  3613. if (to_vmx(vcpu)->nested.nested_vmx_secondary_ctls_high &
  3614. SECONDARY_EXEC_UNRESTRICTED_GUEST &&
  3615. nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST))
  3616. fixed0 &= ~(X86_CR0_PE | X86_CR0_PG);
  3617. return fixed_bits_valid(val, fixed0, fixed1);
  3618. }
  3619. static bool nested_host_cr0_valid(struct kvm_vcpu *vcpu, unsigned long val)
  3620. {
  3621. u64 fixed0 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed0;
  3622. u64 fixed1 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed1;
  3623. return fixed_bits_valid(val, fixed0, fixed1);
  3624. }
  3625. static bool nested_cr4_valid(struct kvm_vcpu *vcpu, unsigned long val)
  3626. {
  3627. u64 fixed0 = to_vmx(vcpu)->nested.nested_vmx_cr4_fixed0;
  3628. u64 fixed1 = to_vmx(vcpu)->nested.nested_vmx_cr4_fixed1;
  3629. return fixed_bits_valid(val, fixed0, fixed1);
  3630. }
  3631. /* No difference in the restrictions on guest and host CR4 in VMX operation. */
  3632. #define nested_guest_cr4_valid nested_cr4_valid
  3633. #define nested_host_cr4_valid nested_cr4_valid
  3634. static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
  3635. static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
  3636. unsigned long cr0,
  3637. struct kvm_vcpu *vcpu)
  3638. {
  3639. if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
  3640. vmx_decache_cr3(vcpu);
  3641. if (!(cr0 & X86_CR0_PG)) {
  3642. /* From paging/starting to nonpaging */
  3643. vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
  3644. vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) |
  3645. (CPU_BASED_CR3_LOAD_EXITING |
  3646. CPU_BASED_CR3_STORE_EXITING));
  3647. vcpu->arch.cr0 = cr0;
  3648. vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
  3649. } else if (!is_paging(vcpu)) {
  3650. /* From nonpaging to paging */
  3651. vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
  3652. vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) &
  3653. ~(CPU_BASED_CR3_LOAD_EXITING |
  3654. CPU_BASED_CR3_STORE_EXITING));
  3655. vcpu->arch.cr0 = cr0;
  3656. vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
  3657. }
  3658. if (!(cr0 & X86_CR0_WP))
  3659. *hw_cr0 &= ~X86_CR0_WP;
  3660. }
  3661. static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
  3662. {
  3663. struct vcpu_vmx *vmx = to_vmx(vcpu);
  3664. unsigned long hw_cr0;
  3665. hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK);
  3666. if (enable_unrestricted_guest)
  3667. hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
  3668. else {
  3669. hw_cr0 |= KVM_VM_CR0_ALWAYS_ON;
  3670. if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
  3671. enter_pmode(vcpu);
  3672. if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
  3673. enter_rmode(vcpu);
  3674. }
  3675. #ifdef CONFIG_X86_64
  3676. if (vcpu->arch.efer & EFER_LME) {
  3677. if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
  3678. enter_lmode(vcpu);
  3679. if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
  3680. exit_lmode(vcpu);
  3681. }
  3682. #endif
  3683. if (enable_ept)
  3684. ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);
  3685. if (!vcpu->fpu_active)
  3686. hw_cr0 |= X86_CR0_TS | X86_CR0_MP;
  3687. vmcs_writel(CR0_READ_SHADOW, cr0);
  3688. vmcs_writel(GUEST_CR0, hw_cr0);
  3689. vcpu->arch.cr0 = cr0;
  3690. /* depends on vcpu->arch.cr0 to be set to a new value */
  3691. vmx->emulation_required = emulation_required(vcpu);
  3692. }
  3693. static u64 construct_eptp(unsigned long root_hpa)
  3694. {
  3695. u64 eptp;
  3696. /* TODO write the value reading from MSR */
  3697. eptp = VMX_EPT_DEFAULT_MT |
  3698. VMX_EPT_DEFAULT_GAW << VMX_EPT_GAW_EPTP_SHIFT;
  3699. if (enable_ept_ad_bits)
  3700. eptp |= VMX_EPT_AD_ENABLE_BIT;
  3701. eptp |= (root_hpa & PAGE_MASK);
  3702. return eptp;
  3703. }
  3704. static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
  3705. {
  3706. unsigned long guest_cr3;
  3707. u64 eptp;
  3708. guest_cr3 = cr3;
  3709. if (enable_ept) {
  3710. eptp = construct_eptp(cr3);
  3711. vmcs_write64(EPT_POINTER, eptp);
  3712. if (is_paging(vcpu) || is_guest_mode(vcpu))
  3713. guest_cr3 = kvm_read_cr3(vcpu);
  3714. else
  3715. guest_cr3 = vcpu->kvm->arch.ept_identity_map_addr;
  3716. ept_load_pdptrs(vcpu);
  3717. }
  3718. vmx_flush_tlb(vcpu);
  3719. vmcs_writel(GUEST_CR3, guest_cr3);
  3720. }
  3721. static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
  3722. {
  3723. /*
  3724. * Pass through host's Machine Check Enable value to hw_cr4, which
  3725. * is in force while we are in guest mode. Do not let guests control
  3726. * this bit, even if host CR4.MCE == 0.
  3727. */
  3728. unsigned long hw_cr4 =
  3729. (cr4_read_shadow() & X86_CR4_MCE) |
  3730. (cr4 & ~X86_CR4_MCE) |
  3731. (to_vmx(vcpu)->rmode.vm86_active ?
  3732. KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON);
  3733. if (cr4 & X86_CR4_VMXE) {
  3734. /*
  3735. * To use VMXON (and later other VMX instructions), a guest
  3736. * must first be able to turn on cr4.VMXE (see handle_vmon()).
  3737. * So basically the check on whether to allow nested VMX
  3738. * is here.
  3739. */
  3740. if (!nested_vmx_allowed(vcpu))
  3741. return 1;
  3742. }
  3743. if (to_vmx(vcpu)->nested.vmxon && !nested_cr4_valid(vcpu, cr4))
  3744. return 1;
  3745. vcpu->arch.cr4 = cr4;
  3746. if (enable_ept) {
  3747. if (!is_paging(vcpu)) {
  3748. hw_cr4 &= ~X86_CR4_PAE;
  3749. hw_cr4 |= X86_CR4_PSE;
  3750. } else if (!(cr4 & X86_CR4_PAE)) {
  3751. hw_cr4 &= ~X86_CR4_PAE;
  3752. }
  3753. }
  3754. if (!enable_unrestricted_guest && !is_paging(vcpu))
  3755. /*
  3756. * SMEP/SMAP/PKU is disabled if CPU is in non-paging mode in
  3757. * hardware. To emulate this behavior, SMEP/SMAP/PKU needs
  3758. * to be manually disabled when guest switches to non-paging
  3759. * mode.
  3760. *
  3761. * If !enable_unrestricted_guest, the CPU is always running
  3762. * with CR0.PG=1 and CR4 needs to be modified.
  3763. * If enable_unrestricted_guest, the CPU automatically
  3764. * disables SMEP/SMAP/PKU when the guest sets CR0.PG=0.
  3765. */
  3766. hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
  3767. vmcs_writel(CR4_READ_SHADOW, cr4);
  3768. vmcs_writel(GUEST_CR4, hw_cr4);
  3769. return 0;
  3770. }
  3771. static void vmx_get_segment(struct kvm_vcpu *vcpu,
  3772. struct kvm_segment *var, int seg)
  3773. {
  3774. struct vcpu_vmx *vmx = to_vmx(vcpu);
  3775. u32 ar;
  3776. if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
  3777. *var = vmx->rmode.segs[seg];
  3778. if (seg == VCPU_SREG_TR
  3779. || var->selector == vmx_read_guest_seg_selector(vmx, seg))
  3780. return;
  3781. var->base = vmx_read_guest_seg_base(vmx, seg);
  3782. var->selector = vmx_read_guest_seg_selector(vmx, seg);
  3783. return;
  3784. }
  3785. var->base = vmx_read_guest_seg_base(vmx, seg);
  3786. var->limit = vmx_read_guest_seg_limit(vmx, seg);
  3787. var->selector = vmx_read_guest_seg_selector(vmx, seg);
  3788. ar = vmx_read_guest_seg_ar(vmx, seg);
  3789. var->unusable = (ar >> 16) & 1;
  3790. var->type = ar & 15;
  3791. var->s = (ar >> 4) & 1;
  3792. var->dpl = (ar >> 5) & 3;
  3793. /*
  3794. * Some userspaces do not preserve unusable property. Since usable
  3795. * segment has to be present according to VMX spec we can use present
  3796. * property to amend userspace bug by making unusable segment always
  3797. * nonpresent. vmx_segment_access_rights() already marks nonpresent
  3798. * segment as unusable.
  3799. */
  3800. var->present = !var->unusable;
  3801. var->avl = (ar >> 12) & 1;
  3802. var->l = (ar >> 13) & 1;
  3803. var->db = (ar >> 14) & 1;
  3804. var->g = (ar >> 15) & 1;
  3805. }
  3806. static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
  3807. {
  3808. struct kvm_segment s;
  3809. if (to_vmx(vcpu)->rmode.vm86_active) {
  3810. vmx_get_segment(vcpu, &s, seg);
  3811. return s.base;
  3812. }
  3813. return vmx_read_guest_seg_base(to_vmx(vcpu), seg);
  3814. }
  3815. static int vmx_get_cpl(struct kvm_vcpu *vcpu)
  3816. {
  3817. struct vcpu_vmx *vmx = to_vmx(vcpu);
  3818. if (unlikely(vmx->rmode.vm86_active))
  3819. return 0;
  3820. else {
  3821. int ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS);
  3822. return VMX_AR_DPL(ar);
  3823. }
  3824. }
  3825. static u32 vmx_segment_access_rights(struct kvm_segment *var)
  3826. {
  3827. u32 ar;
  3828. if (var->unusable || !var->present)
  3829. ar = 1 << 16;
  3830. else {
  3831. ar = var->type & 15;
  3832. ar |= (var->s & 1) << 4;
  3833. ar |= (var->dpl & 3) << 5;
  3834. ar |= (var->present & 1) << 7;
  3835. ar |= (var->avl & 1) << 12;
  3836. ar |= (var->l & 1) << 13;
  3837. ar |= (var->db & 1) << 14;
  3838. ar |= (var->g & 1) << 15;
  3839. }
  3840. return ar;
  3841. }
  3842. static void vmx_set_segment(struct kvm_vcpu *vcpu,
  3843. struct kvm_segment *var, int seg)
  3844. {
  3845. struct vcpu_vmx *vmx = to_vmx(vcpu);
  3846. const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
  3847. vmx_segment_cache_clear(vmx);
  3848. if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
  3849. vmx->rmode.segs[seg] = *var;
  3850. if (seg == VCPU_SREG_TR)
  3851. vmcs_write16(sf->selector, var->selector);
  3852. else if (var->s)
  3853. fix_rmode_seg(seg, &vmx->rmode.segs[seg]);
  3854. goto out;
  3855. }
  3856. vmcs_writel(sf->base, var->base);
  3857. vmcs_write32(sf->limit, var->limit);
  3858. vmcs_write16(sf->selector, var->selector);
  3859. /*
  3860. * Fix the "Accessed" bit in AR field of segment registers for older
  3861. * qemu binaries.
  3862. * IA32 arch specifies that at the time of processor reset the
  3863. * "Accessed" bit in the AR field of segment registers is 1. And qemu
  3864. * is setting it to 0 in the userland code. This causes invalid guest
  3865. * state vmexit when "unrestricted guest" mode is turned on.
  3866. * Fix for this setup issue in cpu_reset is being pushed in the qemu
  3867. * tree. Newer qemu binaries with that qemu fix would not need this
  3868. * kvm hack.
  3869. */
  3870. if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
  3871. var->type |= 0x1; /* Accessed */
  3872. vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var));
  3873. out:
  3874. vmx->emulation_required = emulation_required(vcpu);
  3875. }
  3876. static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
  3877. {
  3878. u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS);
  3879. *db = (ar >> 14) & 1;
  3880. *l = (ar >> 13) & 1;
  3881. }
  3882. static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
  3883. {
  3884. dt->size = vmcs_read32(GUEST_IDTR_LIMIT);
  3885. dt->address = vmcs_readl(GUEST_IDTR_BASE);
  3886. }
  3887. static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
  3888. {
  3889. vmcs_write32(GUEST_IDTR_LIMIT, dt->size);
  3890. vmcs_writel(GUEST_IDTR_BASE, dt->address);
  3891. }
  3892. static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
  3893. {
  3894. dt->size = vmcs_read32(GUEST_GDTR_LIMIT);
  3895. dt->address = vmcs_readl(GUEST_GDTR_BASE);
  3896. }
  3897. static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
  3898. {
  3899. vmcs_write32(GUEST_GDTR_LIMIT, dt->size);
  3900. vmcs_writel(GUEST_GDTR_BASE, dt->address);
  3901. }
  3902. static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
  3903. {
  3904. struct kvm_segment var;
  3905. u32 ar;
  3906. vmx_get_segment(vcpu, &var, seg);
  3907. var.dpl = 0x3;
  3908. if (seg == VCPU_SREG_CS)
  3909. var.type = 0x3;
  3910. ar = vmx_segment_access_rights(&var);
  3911. if (var.base != (var.selector << 4))
  3912. return false;
  3913. if (var.limit != 0xffff)
  3914. return false;
  3915. if (ar != 0xf3)
  3916. return false;
  3917. return true;
  3918. }
  3919. static bool code_segment_valid(struct kvm_vcpu *vcpu)
  3920. {
  3921. struct kvm_segment cs;
  3922. unsigned int cs_rpl;
  3923. vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
  3924. cs_rpl = cs.selector & SEGMENT_RPL_MASK;
  3925. if (cs.unusable)
  3926. return false;
  3927. if (~cs.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_ACCESSES_MASK))
  3928. return false;
  3929. if (!cs.s)
  3930. return false;
  3931. if (cs.type & VMX_AR_TYPE_WRITEABLE_MASK) {
  3932. if (cs.dpl > cs_rpl)
  3933. return false;
  3934. } else {
  3935. if (cs.dpl != cs_rpl)
  3936. return false;
  3937. }
  3938. if (!cs.present)
  3939. return false;
  3940. /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
  3941. return true;
  3942. }
  3943. static bool stack_segment_valid(struct kvm_vcpu *vcpu)
  3944. {
  3945. struct kvm_segment ss;
  3946. unsigned int ss_rpl;
  3947. vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
  3948. ss_rpl = ss.selector & SEGMENT_RPL_MASK;
  3949. if (ss.unusable)
  3950. return true;
  3951. if (ss.type != 3 && ss.type != 7)
  3952. return false;
  3953. if (!ss.s)
  3954. return false;
  3955. if (ss.dpl != ss_rpl) /* DPL != RPL */
  3956. return false;
  3957. if (!ss.present)
  3958. return false;
  3959. return true;
  3960. }
  3961. static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
  3962. {
  3963. struct kvm_segment var;
  3964. unsigned int rpl;
  3965. vmx_get_segment(vcpu, &var, seg);
  3966. rpl = var.selector & SEGMENT_RPL_MASK;
  3967. if (var.unusable)
  3968. return true;
  3969. if (!var.s)
  3970. return false;
  3971. if (!var.present)
  3972. return false;
  3973. if (~var.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_WRITEABLE_MASK)) {
  3974. if (var.dpl < rpl) /* DPL < RPL */
  3975. return false;
  3976. }
  3977. /* TODO: Add other members to kvm_segment_field to allow checking for other access
  3978. * rights flags
  3979. */
  3980. return true;
  3981. }
  3982. static bool tr_valid(struct kvm_vcpu *vcpu)
  3983. {
  3984. struct kvm_segment tr;
  3985. vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);
  3986. if (tr.unusable)
  3987. return false;
  3988. if (tr.selector & SEGMENT_TI_MASK) /* TI = 1 */
  3989. return false;
  3990. if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
  3991. return false;
  3992. if (!tr.present)
  3993. return false;
  3994. return true;
  3995. }
  3996. static bool ldtr_valid(struct kvm_vcpu *vcpu)
  3997. {
  3998. struct kvm_segment ldtr;
  3999. vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);
  4000. if (ldtr.unusable)
  4001. return true;
  4002. if (ldtr.selector & SEGMENT_TI_MASK) /* TI = 1 */
  4003. return false;
  4004. if (ldtr.type != 2)
  4005. return false;
  4006. if (!ldtr.present)
  4007. return false;
  4008. return true;
  4009. }
  4010. static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
  4011. {
  4012. struct kvm_segment cs, ss;
  4013. vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
  4014. vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
  4015. return ((cs.selector & SEGMENT_RPL_MASK) ==
  4016. (ss.selector & SEGMENT_RPL_MASK));
  4017. }
  4018. /*
  4019. * Check if guest state is valid. Returns true if valid, false if
  4020. * not.
  4021. * We assume that registers are always usable
  4022. */
  4023. static bool guest_state_valid(struct kvm_vcpu *vcpu)
  4024. {
  4025. if (enable_unrestricted_guest)
  4026. return true;
  4027. /* real mode guest state checks */
  4028. if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) {
  4029. if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
  4030. return false;
  4031. if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
  4032. return false;
  4033. if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
  4034. return false;
  4035. if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
  4036. return false;
  4037. if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
  4038. return false;
  4039. if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
  4040. return false;
  4041. } else {
  4042. /* protected mode guest state checks */
  4043. if (!cs_ss_rpl_check(vcpu))
  4044. return false;
  4045. if (!code_segment_valid(vcpu))
  4046. return false;
  4047. if (!stack_segment_valid(vcpu))
  4048. return false;
  4049. if (!data_segment_valid(vcpu, VCPU_SREG_DS))
  4050. return false;
  4051. if (!data_segment_valid(vcpu, VCPU_SREG_ES))
  4052. return false;
  4053. if (!data_segment_valid(vcpu, VCPU_SREG_FS))
  4054. return false;
  4055. if (!data_segment_valid(vcpu, VCPU_SREG_GS))
  4056. return false;
  4057. if (!tr_valid(vcpu))
  4058. return false;
  4059. if (!ldtr_valid(vcpu))
  4060. return false;
  4061. }
  4062. /* TODO:
  4063. * - Add checks on RIP
  4064. * - Add checks on RFLAGS
  4065. */
  4066. return true;
  4067. }
  4068. static int init_rmode_tss(struct kvm *kvm)
  4069. {
  4070. gfn_t fn;
  4071. u16 data = 0;
  4072. int idx, r;
  4073. idx = srcu_read_lock(&kvm->srcu);
  4074. fn = kvm->arch.tss_addr >> PAGE_SHIFT;
  4075. r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
  4076. if (r < 0)
  4077. goto out;
  4078. data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
  4079. r = kvm_write_guest_page(kvm, fn++, &data,
  4080. TSS_IOPB_BASE_OFFSET, sizeof(u16));
  4081. if (r < 0)
  4082. goto out;
  4083. r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
  4084. if (r < 0)
  4085. goto out;
  4086. r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
  4087. if (r < 0)
  4088. goto out;
  4089. data = ~0;
  4090. r = kvm_write_guest_page(kvm, fn, &data,
  4091. RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
  4092. sizeof(u8));
  4093. out:
  4094. srcu_read_unlock(&kvm->srcu, idx);
  4095. return r;
  4096. }
  4097. static int init_rmode_identity_map(struct kvm *kvm)
  4098. {
  4099. int i, idx, r = 0;
  4100. kvm_pfn_t identity_map_pfn;
  4101. u32 tmp;
  4102. if (!enable_ept)
  4103. return 0;
  4104. /* Protect kvm->arch.ept_identity_pagetable_done. */
  4105. mutex_lock(&kvm->slots_lock);
  4106. if (likely(kvm->arch.ept_identity_pagetable_done))
  4107. goto out2;
  4108. identity_map_pfn = kvm->arch.ept_identity_map_addr >> PAGE_SHIFT;
  4109. r = alloc_identity_pagetable(kvm);
  4110. if (r < 0)
  4111. goto out2;
  4112. idx = srcu_read_lock(&kvm->srcu);
  4113. r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
  4114. if (r < 0)
  4115. goto out;
  4116. /* Set up identity-mapping pagetable for EPT in real mode */
  4117. for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
  4118. tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
  4119. _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
  4120. r = kvm_write_guest_page(kvm, identity_map_pfn,
  4121. &tmp, i * sizeof(tmp), sizeof(tmp));
  4122. if (r < 0)
  4123. goto out;
  4124. }
  4125. kvm->arch.ept_identity_pagetable_done = true;
  4126. out:
  4127. srcu_read_unlock(&kvm->srcu, idx);
  4128. out2:
  4129. mutex_unlock(&kvm->slots_lock);
  4130. return r;
  4131. }
  4132. static void seg_setup(int seg)
  4133. {
  4134. const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
  4135. unsigned int ar;
  4136. vmcs_write16(sf->selector, 0);
  4137. vmcs_writel(sf->base, 0);
  4138. vmcs_write32(sf->limit, 0xffff);
  4139. ar = 0x93;
  4140. if (seg == VCPU_SREG_CS)
  4141. ar |= 0x08; /* code segment */
  4142. vmcs_write32(sf->ar_bytes, ar);
  4143. }
  4144. static int alloc_apic_access_page(struct kvm *kvm)
  4145. {
  4146. struct page *page;
  4147. int r = 0;
  4148. mutex_lock(&kvm->slots_lock);
  4149. if (kvm->arch.apic_access_page_done)
  4150. goto out;
  4151. r = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
  4152. APIC_DEFAULT_PHYS_BASE, PAGE_SIZE);
  4153. if (r)
  4154. goto out;
  4155. page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
  4156. if (is_error_page(page)) {
  4157. r = -EFAULT;
  4158. goto out;
  4159. }
  4160. /*
  4161. * Do not pin the page in memory, so that memory hot-unplug
  4162. * is able to migrate it.
  4163. */
  4164. put_page(page);
  4165. kvm->arch.apic_access_page_done = true;
  4166. out:
  4167. mutex_unlock(&kvm->slots_lock);
  4168. return r;
  4169. }
  4170. static int alloc_identity_pagetable(struct kvm *kvm)
  4171. {
  4172. /* Called with kvm->slots_lock held. */
  4173. int r = 0;
  4174. BUG_ON(kvm->arch.ept_identity_pagetable_done);
  4175. r = __x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
  4176. kvm->arch.ept_identity_map_addr, PAGE_SIZE);
  4177. return r;
  4178. }
  4179. static int allocate_vpid(void)
  4180. {
  4181. int vpid;
  4182. if (!enable_vpid)
  4183. return 0;
  4184. spin_lock(&vmx_vpid_lock);
  4185. vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
  4186. if (vpid < VMX_NR_VPIDS)
  4187. __set_bit(vpid, vmx_vpid_bitmap);
  4188. else
  4189. vpid = 0;
  4190. spin_unlock(&vmx_vpid_lock);
  4191. return vpid;
  4192. }
  4193. static void free_vpid(int vpid)
  4194. {
  4195. if (!enable_vpid || vpid == 0)
  4196. return;
  4197. spin_lock(&vmx_vpid_lock);
  4198. __clear_bit(vpid, vmx_vpid_bitmap);
  4199. spin_unlock(&vmx_vpid_lock);
  4200. }
  4201. #define MSR_TYPE_R 1
  4202. #define MSR_TYPE_W 2
  4203. static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
  4204. u32 msr, int type)
  4205. {
  4206. int f = sizeof(unsigned long);
  4207. if (!cpu_has_vmx_msr_bitmap())
  4208. return;
  4209. /*
  4210. * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
  4211. * have the write-low and read-high bitmap offsets the wrong way round.
  4212. * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
  4213. */
  4214. if (msr <= 0x1fff) {
  4215. if (type & MSR_TYPE_R)
  4216. /* read-low */
  4217. __clear_bit(msr, msr_bitmap + 0x000 / f);
  4218. if (type & MSR_TYPE_W)
  4219. /* write-low */
  4220. __clear_bit(msr, msr_bitmap + 0x800 / f);
  4221. } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
  4222. msr &= 0x1fff;
  4223. if (type & MSR_TYPE_R)
  4224. /* read-high */
  4225. __clear_bit(msr, msr_bitmap + 0x400 / f);
  4226. if (type & MSR_TYPE_W)
  4227. /* write-high */
  4228. __clear_bit(msr, msr_bitmap + 0xc00 / f);
  4229. }
  4230. }
  4231. /*
  4232. * If a msr is allowed by L0, we should check whether it is allowed by L1.
  4233. * The corresponding bit will be cleared unless both of L0 and L1 allow it.
  4234. */
  4235. static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1,
  4236. unsigned long *msr_bitmap_nested,
  4237. u32 msr, int type)
  4238. {
  4239. int f = sizeof(unsigned long);
  4240. if (!cpu_has_vmx_msr_bitmap()) {
  4241. WARN_ON(1);
  4242. return;
  4243. }
  4244. /*
  4245. * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
  4246. * have the write-low and read-high bitmap offsets the wrong way round.
  4247. * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
  4248. */
  4249. if (msr <= 0x1fff) {
  4250. if (type & MSR_TYPE_R &&
  4251. !test_bit(msr, msr_bitmap_l1 + 0x000 / f))
  4252. /* read-low */
  4253. __clear_bit(msr, msr_bitmap_nested + 0x000 / f);
  4254. if (type & MSR_TYPE_W &&
  4255. !test_bit(msr, msr_bitmap_l1 + 0x800 / f))
  4256. /* write-low */
  4257. __clear_bit(msr, msr_bitmap_nested + 0x800 / f);
  4258. } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
  4259. msr &= 0x1fff;
  4260. if (type & MSR_TYPE_R &&
  4261. !test_bit(msr, msr_bitmap_l1 + 0x400 / f))
  4262. /* read-high */
  4263. __clear_bit(msr, msr_bitmap_nested + 0x400 / f);
  4264. if (type & MSR_TYPE_W &&
  4265. !test_bit(msr, msr_bitmap_l1 + 0xc00 / f))
  4266. /* write-high */
  4267. __clear_bit(msr, msr_bitmap_nested + 0xc00 / f);
  4268. }
  4269. }
  4270. static void vmx_disable_intercept_for_msr(u32 msr, bool longmode_only)
  4271. {
  4272. if (!longmode_only)
  4273. __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy,
  4274. msr, MSR_TYPE_R | MSR_TYPE_W);
  4275. __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode,
  4276. msr, MSR_TYPE_R | MSR_TYPE_W);
  4277. }
  4278. static void vmx_disable_intercept_msr_x2apic(u32 msr, int type, bool apicv_active)
  4279. {
  4280. if (apicv_active) {
  4281. __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic_apicv,
  4282. msr, type);
  4283. __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic_apicv,
  4284. msr, type);
  4285. } else {
  4286. __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic,
  4287. msr, type);
  4288. __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic,
  4289. msr, type);
  4290. }
  4291. }
  4292. static bool vmx_get_enable_apicv(void)
  4293. {
  4294. return enable_apicv;
  4295. }
  4296. static int vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu)
  4297. {
  4298. struct vcpu_vmx *vmx = to_vmx(vcpu);
  4299. int max_irr;
  4300. void *vapic_page;
  4301. u16 status;
  4302. if (vmx->nested.pi_desc &&
  4303. vmx->nested.pi_pending) {
  4304. vmx->nested.pi_pending = false;
  4305. if (!pi_test_and_clear_on(vmx->nested.pi_desc))
  4306. return 0;
  4307. max_irr = find_last_bit(
  4308. (unsigned long *)vmx->nested.pi_desc->pir, 256);
  4309. if (max_irr == 256)
  4310. return 0;
  4311. vapic_page = kmap(vmx->nested.virtual_apic_page);
  4312. if (!vapic_page) {
  4313. WARN_ON(1);
  4314. return -ENOMEM;
  4315. }
  4316. __kvm_apic_update_irr(vmx->nested.pi_desc->pir, vapic_page);
  4317. kunmap(vmx->nested.virtual_apic_page);
  4318. status = vmcs_read16(GUEST_INTR_STATUS);
  4319. if ((u8)max_irr > ((u8)status & 0xff)) {
  4320. status &= ~0xff;
  4321. status |= (u8)max_irr;
  4322. vmcs_write16(GUEST_INTR_STATUS, status);
  4323. }
  4324. }
  4325. return 0;
  4326. }
  4327. static inline bool kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu)
  4328. {
  4329. #ifdef CONFIG_SMP
  4330. if (vcpu->mode == IN_GUEST_MODE) {
  4331. struct vcpu_vmx *vmx = to_vmx(vcpu);
  4332. /*
  4333. * Currently, we don't support urgent interrupt,
  4334. * all interrupts are recognized as non-urgent
  4335. * interrupt, so we cannot post interrupts when
  4336. * 'SN' is set.
  4337. *
  4338. * If the vcpu is in guest mode, it means it is
  4339. * running instead of being scheduled out and
  4340. * waiting in the run queue, and that's the only
  4341. * case when 'SN' is set currently, warning if
  4342. * 'SN' is set.
  4343. */
  4344. WARN_ON_ONCE(pi_test_sn(&vmx->pi_desc));
  4345. apic->send_IPI_mask(get_cpu_mask(vcpu->cpu),
  4346. POSTED_INTR_VECTOR);
  4347. return true;
  4348. }
  4349. #endif
  4350. return false;
  4351. }
  4352. static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu,
  4353. int vector)
  4354. {
  4355. struct vcpu_vmx *vmx = to_vmx(vcpu);
  4356. if (is_guest_mode(vcpu) &&
  4357. vector == vmx->nested.posted_intr_nv) {
  4358. /* the PIR and ON have been set by L1. */
  4359. kvm_vcpu_trigger_posted_interrupt(vcpu);
  4360. /*
  4361. * If a posted intr is not recognized by hardware,
  4362. * we will accomplish it in the next vmentry.
  4363. */
  4364. vmx->nested.pi_pending = true;
  4365. kvm_make_request(KVM_REQ_EVENT, vcpu);
  4366. return 0;
  4367. }
  4368. return -1;
  4369. }
  4370. /*
  4371. * Send interrupt to vcpu via posted interrupt way.
  4372. * 1. If target vcpu is running(non-root mode), send posted interrupt
  4373. * notification to vcpu and hardware will sync PIR to vIRR atomically.
  4374. * 2. If target vcpu isn't running(root mode), kick it to pick up the
  4375. * interrupt from PIR in next vmentry.
  4376. */
  4377. static void vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector)
  4378. {
  4379. struct vcpu_vmx *vmx = to_vmx(vcpu);
  4380. int r;
  4381. r = vmx_deliver_nested_posted_interrupt(vcpu, vector);
  4382. if (!r)
  4383. return;
  4384. if (pi_test_and_set_pir(vector, &vmx->pi_desc))
  4385. return;
  4386. r = pi_test_and_set_on(&vmx->pi_desc);
  4387. kvm_make_request(KVM_REQ_EVENT, vcpu);
  4388. if (r || !kvm_vcpu_trigger_posted_interrupt(vcpu))
  4389. kvm_vcpu_kick(vcpu);
  4390. }
  4391. static void vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
  4392. {
  4393. struct vcpu_vmx *vmx = to_vmx(vcpu);
  4394. if (!pi_test_on(&vmx->pi_desc))
  4395. return;
  4396. pi_clear_on(&vmx->pi_desc);
  4397. /*
  4398. * IOMMU can write to PIR.ON, so the barrier matters even on UP.
  4399. * But on x86 this is just a compiler barrier anyway.
  4400. */
  4401. smp_mb__after_atomic();
  4402. kvm_apic_update_irr(vcpu, vmx->pi_desc.pir);
  4403. }
  4404. /*
  4405. * Set up the vmcs's constant host-state fields, i.e., host-state fields that
  4406. * will not change in the lifetime of the guest.
  4407. * Note that host-state that does change is set elsewhere. E.g., host-state
  4408. * that is set differently for each CPU is set in vmx_vcpu_load(), not here.
  4409. */
  4410. static void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
  4411. {
  4412. u32 low32, high32;
  4413. unsigned long tmpl;
  4414. struct desc_ptr dt;
  4415. unsigned long cr0, cr4;
  4416. cr0 = read_cr0();
  4417. WARN_ON(cr0 & X86_CR0_TS);
  4418. vmcs_writel(HOST_CR0, cr0); /* 22.2.3 */
  4419. vmcs_writel(HOST_CR3, read_cr3()); /* 22.2.3 FIXME: shadow tables */
  4420. /* Save the most likely value for this task's CR4 in the VMCS. */
  4421. cr4 = cr4_read_shadow();
  4422. vmcs_writel(HOST_CR4, cr4); /* 22.2.3, 22.2.5 */
  4423. vmx->host_state.vmcs_host_cr4 = cr4;
  4424. vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS); /* 22.2.4 */
  4425. #ifdef CONFIG_X86_64
  4426. /*
  4427. * Load null selectors, so we can avoid reloading them in
  4428. * __vmx_load_host_state(), in case userspace uses the null selectors
  4429. * too (the expected case).
  4430. */
  4431. vmcs_write16(HOST_DS_SELECTOR, 0);
  4432. vmcs_write16(HOST_ES_SELECTOR, 0);
  4433. #else
  4434. vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS); /* 22.2.4 */
  4435. vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS); /* 22.2.4 */
  4436. #endif
  4437. vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS); /* 22.2.4 */
  4438. vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8); /* 22.2.4 */
  4439. native_store_idt(&dt);
  4440. vmcs_writel(HOST_IDTR_BASE, dt.address); /* 22.2.4 */
  4441. vmx->host_idt_base = dt.address;
  4442. vmcs_writel(HOST_RIP, vmx_return); /* 22.2.5 */
  4443. rdmsr(MSR_IA32_SYSENTER_CS, low32, high32);
  4444. vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
  4445. rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl);
  4446. vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl); /* 22.2.3 */
  4447. if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
  4448. rdmsr(MSR_IA32_CR_PAT, low32, high32);
  4449. vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32));
  4450. }
  4451. }
  4452. static void set_cr4_guest_host_mask(struct vcpu_vmx *vmx)
  4453. {
  4454. vmx->vcpu.arch.cr4_guest_owned_bits = KVM_CR4_GUEST_OWNED_BITS;
  4455. if (enable_ept)
  4456. vmx->vcpu.arch.cr4_guest_owned_bits |= X86_CR4_PGE;
  4457. if (is_guest_mode(&vmx->vcpu))
  4458. vmx->vcpu.arch.cr4_guest_owned_bits &=
  4459. ~get_vmcs12(&vmx->vcpu)->cr4_guest_host_mask;
  4460. vmcs_writel(CR4_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr4_guest_owned_bits);
  4461. }
  4462. static u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx)
  4463. {
  4464. u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl;
  4465. if (!kvm_vcpu_apicv_active(&vmx->vcpu))
  4466. pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR;
  4467. /* Enable the preemption timer dynamically */
  4468. pin_based_exec_ctrl &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
  4469. return pin_based_exec_ctrl;
  4470. }
  4471. static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
  4472. {
  4473. struct vcpu_vmx *vmx = to_vmx(vcpu);
  4474. vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, vmx_pin_based_exec_ctrl(vmx));
  4475. if (cpu_has_secondary_exec_ctrls()) {
  4476. if (kvm_vcpu_apicv_active(vcpu))
  4477. vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,
  4478. SECONDARY_EXEC_APIC_REGISTER_VIRT |
  4479. SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
  4480. else
  4481. vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,
  4482. SECONDARY_EXEC_APIC_REGISTER_VIRT |
  4483. SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
  4484. }
  4485. if (cpu_has_vmx_msr_bitmap())
  4486. vmx_set_msr_bitmap(vcpu);
  4487. }
  4488. static u32 vmx_exec_control(struct vcpu_vmx *vmx)
  4489. {
  4490. u32 exec_control = vmcs_config.cpu_based_exec_ctrl;
  4491. if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)
  4492. exec_control &= ~CPU_BASED_MOV_DR_EXITING;
  4493. if (!cpu_need_tpr_shadow(&vmx->vcpu)) {
  4494. exec_control &= ~CPU_BASED_TPR_SHADOW;
  4495. #ifdef CONFIG_X86_64
  4496. exec_control |= CPU_BASED_CR8_STORE_EXITING |
  4497. CPU_BASED_CR8_LOAD_EXITING;
  4498. #endif
  4499. }
  4500. if (!enable_ept)
  4501. exec_control |= CPU_BASED_CR3_STORE_EXITING |
  4502. CPU_BASED_CR3_LOAD_EXITING |
  4503. CPU_BASED_INVLPG_EXITING;
  4504. return exec_control;
  4505. }
  4506. static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx)
  4507. {
  4508. u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
  4509. if (!cpu_need_virtualize_apic_accesses(&vmx->vcpu))
  4510. exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
  4511. if (vmx->vpid == 0)
  4512. exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
  4513. if (!enable_ept) {
  4514. exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
  4515. enable_unrestricted_guest = 0;
  4516. /* Enable INVPCID for non-ept guests may cause performance regression. */
  4517. exec_control &= ~SECONDARY_EXEC_ENABLE_INVPCID;
  4518. }
  4519. if (!enable_unrestricted_guest)
  4520. exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
  4521. if (!ple_gap)
  4522. exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
  4523. if (!kvm_vcpu_apicv_active(&vmx->vcpu))
  4524. exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT |
  4525. SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
  4526. exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
  4527. /* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD
  4528. (handle_vmptrld).
  4529. We can NOT enable shadow_vmcs here because we don't have yet
  4530. a current VMCS12
  4531. */
  4532. exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
  4533. if (!enable_pml)
  4534. exec_control &= ~SECONDARY_EXEC_ENABLE_PML;
  4535. return exec_control;
  4536. }
  4537. static void ept_set_mmio_spte_mask(void)
  4538. {
  4539. /*
  4540. * EPT Misconfigurations can be generated if the value of bits 2:0
  4541. * of an EPT paging-structure entry is 110b (write/execute).
  4542. * Also, magic bits (0x3ull << 62) is set to quickly identify mmio
  4543. * spte.
  4544. */
  4545. kvm_mmu_set_mmio_spte_mask((0x3ull << 62) | 0x6ull);
  4546. }
  4547. #define VMX_XSS_EXIT_BITMAP 0
  4548. /*
  4549. * Sets up the vmcs for emulated real mode.
  4550. */
  4551. static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
  4552. {
  4553. #ifdef CONFIG_X86_64
  4554. unsigned long a;
  4555. #endif
  4556. int i;
  4557. /* I/O */
  4558. vmcs_write64(IO_BITMAP_A, __pa(vmx_io_bitmap_a));
  4559. vmcs_write64(IO_BITMAP_B, __pa(vmx_io_bitmap_b));
  4560. if (enable_shadow_vmcs) {
  4561. vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap));
  4562. vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap));
  4563. }
  4564. if (cpu_has_vmx_msr_bitmap())
  4565. vmcs_write64(MSR_BITMAP, __pa(vmx_msr_bitmap_legacy));
  4566. vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
  4567. /* Control */
  4568. vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, vmx_pin_based_exec_ctrl(vmx));
  4569. vmx->hv_deadline_tsc = -1;
  4570. vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, vmx_exec_control(vmx));
  4571. if (cpu_has_secondary_exec_ctrls()) {
  4572. vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
  4573. vmx_secondary_exec_control(vmx));
  4574. }
  4575. if (kvm_vcpu_apicv_active(&vmx->vcpu)) {
  4576. vmcs_write64(EOI_EXIT_BITMAP0, 0);
  4577. vmcs_write64(EOI_EXIT_BITMAP1, 0);
  4578. vmcs_write64(EOI_EXIT_BITMAP2, 0);
  4579. vmcs_write64(EOI_EXIT_BITMAP3, 0);
  4580. vmcs_write16(GUEST_INTR_STATUS, 0);
  4581. vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
  4582. vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc)));
  4583. }
  4584. if (ple_gap) {
  4585. vmcs_write32(PLE_GAP, ple_gap);
  4586. vmx->ple_window = ple_window;
  4587. vmx->ple_window_dirty = true;
  4588. }
  4589. vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
  4590. vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
  4591. vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */
  4592. vmcs_write16(HOST_FS_SELECTOR, 0); /* 22.2.4 */
  4593. vmcs_write16(HOST_GS_SELECTOR, 0); /* 22.2.4 */
  4594. vmx_set_constant_host_state(vmx);
  4595. #ifdef CONFIG_X86_64
  4596. rdmsrl(MSR_FS_BASE, a);
  4597. vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */
  4598. rdmsrl(MSR_GS_BASE, a);
  4599. vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */
  4600. #else
  4601. vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
  4602. vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
  4603. #endif
  4604. vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
  4605. vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
  4606. vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host));
  4607. vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
  4608. vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest));
  4609. if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT)
  4610. vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
  4611. for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i) {
  4612. u32 index = vmx_msr_index[i];
  4613. u32 data_low, data_high;
  4614. int j = vmx->nmsrs;
  4615. if (rdmsr_safe(index, &data_low, &data_high) < 0)
  4616. continue;
  4617. if (wrmsr_safe(index, data_low, data_high) < 0)
  4618. continue;
  4619. vmx->guest_msrs[j].index = i;
  4620. vmx->guest_msrs[j].data = 0;
  4621. vmx->guest_msrs[j].mask = -1ull;
  4622. ++vmx->nmsrs;
  4623. }
  4624. vm_exit_controls_init(vmx, vmcs_config.vmexit_ctrl);
  4625. /* 22.2.1, 20.8.1 */
  4626. vm_entry_controls_init(vmx, vmcs_config.vmentry_ctrl);
  4627. vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL);
  4628. set_cr4_guest_host_mask(vmx);
  4629. if (vmx_xsaves_supported())
  4630. vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP);
  4631. if (enable_pml) {
  4632. ASSERT(vmx->pml_pg);
  4633. vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
  4634. vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
  4635. }
  4636. return 0;
  4637. }
  4638. static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
  4639. {
  4640. struct vcpu_vmx *vmx = to_vmx(vcpu);
  4641. struct msr_data apic_base_msr;
  4642. u64 cr0;
  4643. vmx->rmode.vm86_active = 0;
  4644. vmx->soft_vnmi_blocked = 0;
  4645. vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
  4646. kvm_set_cr8(vcpu, 0);
  4647. if (!init_event) {
  4648. apic_base_msr.data = APIC_DEFAULT_PHYS_BASE |
  4649. MSR_IA32_APICBASE_ENABLE;
  4650. if (kvm_vcpu_is_reset_bsp(vcpu))
  4651. apic_base_msr.data |= MSR_IA32_APICBASE_BSP;
  4652. apic_base_msr.host_initiated = true;
  4653. kvm_set_apic_base(vcpu, &apic_base_msr);
  4654. }
  4655. vmx_segment_cache_clear(vmx);
  4656. seg_setup(VCPU_SREG_CS);
  4657. vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
  4658. vmcs_writel(GUEST_CS_BASE, 0xffff0000ul);
  4659. seg_setup(VCPU_SREG_DS);
  4660. seg_setup(VCPU_SREG_ES);
  4661. seg_setup(VCPU_SREG_FS);
  4662. seg_setup(VCPU_SREG_GS);
  4663. seg_setup(VCPU_SREG_SS);
  4664. vmcs_write16(GUEST_TR_SELECTOR, 0);
  4665. vmcs_writel(GUEST_TR_BASE, 0);
  4666. vmcs_write32(GUEST_TR_LIMIT, 0xffff);
  4667. vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
  4668. vmcs_write16(GUEST_LDTR_SELECTOR, 0);
  4669. vmcs_writel(GUEST_LDTR_BASE, 0);
  4670. vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
  4671. vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);
  4672. if (!init_event) {
  4673. vmcs_write32(GUEST_SYSENTER_CS, 0);
  4674. vmcs_writel(GUEST_SYSENTER_ESP, 0);
  4675. vmcs_writel(GUEST_SYSENTER_EIP, 0);
  4676. vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
  4677. }
  4678. vmcs_writel(GUEST_RFLAGS, 0x02);
  4679. kvm_rip_write(vcpu, 0xfff0);
  4680. vmcs_writel(GUEST_GDTR_BASE, 0);
  4681. vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
  4682. vmcs_writel(GUEST_IDTR_BASE, 0);
  4683. vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
  4684. vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
  4685. vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
  4686. vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0);
  4687. setup_msrs(vmx);
  4688. vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */
  4689. if (cpu_has_vmx_tpr_shadow() && !init_event) {
  4690. vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
  4691. if (cpu_need_tpr_shadow(vcpu))
  4692. vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
  4693. __pa(vcpu->arch.apic->regs));
  4694. vmcs_write32(TPR_THRESHOLD, 0);
  4695. }
  4696. kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
  4697. if (kvm_vcpu_apicv_active(vcpu))
  4698. memset(&vmx->pi_desc, 0, sizeof(struct pi_desc));
  4699. if (vmx->vpid != 0)
  4700. vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
  4701. cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
  4702. vmx->vcpu.arch.cr0 = cr0;
  4703. vmx_set_cr0(vcpu, cr0); /* enter rmode */
  4704. vmx_set_cr4(vcpu, 0);
  4705. vmx_set_efer(vcpu, 0);
  4706. vmx_fpu_activate(vcpu);
  4707. update_exception_bitmap(vcpu);
  4708. vpid_sync_context(vmx->vpid);
  4709. }
  4710. /*
  4711. * In nested virtualization, check if L1 asked to exit on external interrupts.
  4712. * For most existing hypervisors, this will always return true.
  4713. */
  4714. static bool nested_exit_on_intr(struct kvm_vcpu *vcpu)
  4715. {
  4716. return get_vmcs12(vcpu)->pin_based_vm_exec_control &
  4717. PIN_BASED_EXT_INTR_MASK;
  4718. }
  4719. /*
  4720. * In nested virtualization, check if L1 has set
  4721. * VM_EXIT_ACK_INTR_ON_EXIT
  4722. */
  4723. static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu)
  4724. {
  4725. return get_vmcs12(vcpu)->vm_exit_controls &
  4726. VM_EXIT_ACK_INTR_ON_EXIT;
  4727. }
  4728. static bool nested_exit_on_nmi(struct kvm_vcpu *vcpu)
  4729. {
  4730. return get_vmcs12(vcpu)->pin_based_vm_exec_control &
  4731. PIN_BASED_NMI_EXITING;
  4732. }
  4733. static void enable_irq_window(struct kvm_vcpu *vcpu)
  4734. {
  4735. u32 cpu_based_vm_exec_control;
  4736. cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
  4737. cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING;
  4738. vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
  4739. }
  4740. static void enable_nmi_window(struct kvm_vcpu *vcpu)
  4741. {
  4742. u32 cpu_based_vm_exec_control;
  4743. if (!cpu_has_virtual_nmis() ||
  4744. vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
  4745. enable_irq_window(vcpu);
  4746. return;
  4747. }
  4748. cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
  4749. cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_NMI_PENDING;
  4750. vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
  4751. }
  4752. static void vmx_inject_irq(struct kvm_vcpu *vcpu)
  4753. {
  4754. struct vcpu_vmx *vmx = to_vmx(vcpu);
  4755. uint32_t intr;
  4756. int irq = vcpu->arch.interrupt.nr;
  4757. trace_kvm_inj_virq(irq);
  4758. ++vcpu->stat.irq_injections;
  4759. if (vmx->rmode.vm86_active) {
  4760. int inc_eip = 0;
  4761. if (vcpu->arch.interrupt.soft)
  4762. inc_eip = vcpu->arch.event_exit_inst_len;
  4763. if (kvm_inject_realmode_interrupt(vcpu, irq, inc_eip) != EMULATE_DONE)
  4764. kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
  4765. return;
  4766. }
  4767. intr = irq | INTR_INFO_VALID_MASK;
  4768. if (vcpu->arch.interrupt.soft) {
  4769. intr |= INTR_TYPE_SOFT_INTR;
  4770. vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
  4771. vmx->vcpu.arch.event_exit_inst_len);
  4772. } else
  4773. intr |= INTR_TYPE_EXT_INTR;
  4774. vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
  4775. }
  4776. static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
  4777. {
  4778. struct vcpu_vmx *vmx = to_vmx(vcpu);
  4779. if (!is_guest_mode(vcpu)) {
  4780. if (!cpu_has_virtual_nmis()) {
  4781. /*
  4782. * Tracking the NMI-blocked state in software is built upon
  4783. * finding the next open IRQ window. This, in turn, depends on
  4784. * well-behaving guests: They have to keep IRQs disabled at
  4785. * least as long as the NMI handler runs. Otherwise we may
  4786. * cause NMI nesting, maybe breaking the guest. But as this is
  4787. * highly unlikely, we can live with the residual risk.
  4788. */
  4789. vmx->soft_vnmi_blocked = 1;
  4790. vmx->vnmi_blocked_time = 0;
  4791. }
  4792. ++vcpu->stat.nmi_injections;
  4793. vmx->nmi_known_unmasked = false;
  4794. }
  4795. if (vmx->rmode.vm86_active) {
  4796. if (kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0) != EMULATE_DONE)
  4797. kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
  4798. return;
  4799. }
  4800. vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
  4801. INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
  4802. }
  4803. static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
  4804. {
  4805. if (!cpu_has_virtual_nmis())
  4806. return to_vmx(vcpu)->soft_vnmi_blocked;
  4807. if (to_vmx(vcpu)->nmi_known_unmasked)
  4808. return false;
  4809. return vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_NMI;
  4810. }
  4811. static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
  4812. {
  4813. struct vcpu_vmx *vmx = to_vmx(vcpu);
  4814. if (!cpu_has_virtual_nmis()) {
  4815. if (vmx->soft_vnmi_blocked != masked) {
  4816. vmx->soft_vnmi_blocked = masked;
  4817. vmx->vnmi_blocked_time = 0;
  4818. }
  4819. } else {
  4820. vmx->nmi_known_unmasked = !masked;
  4821. if (masked)
  4822. vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
  4823. GUEST_INTR_STATE_NMI);
  4824. else
  4825. vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
  4826. GUEST_INTR_STATE_NMI);
  4827. }
  4828. }
  4829. static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
  4830. {
  4831. if (to_vmx(vcpu)->nested.nested_run_pending)
  4832. return 0;
  4833. if (!cpu_has_virtual_nmis() && to_vmx(vcpu)->soft_vnmi_blocked)
  4834. return 0;
  4835. return !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
  4836. (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI
  4837. | GUEST_INTR_STATE_NMI));
  4838. }
  4839. static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu)
  4840. {
  4841. return (!to_vmx(vcpu)->nested.nested_run_pending &&
  4842. vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
  4843. !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
  4844. (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
  4845. }
  4846. static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
  4847. {
  4848. int ret;
  4849. ret = x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, addr,
  4850. PAGE_SIZE * 3);
  4851. if (ret)
  4852. return ret;
  4853. kvm->arch.tss_addr = addr;
  4854. return init_rmode_tss(kvm);
  4855. }
  4856. static bool rmode_exception(struct kvm_vcpu *vcpu, int vec)
  4857. {
  4858. switch (vec) {
  4859. case BP_VECTOR:
  4860. /*
  4861. * Update instruction length as we may reinject the exception
  4862. * from user space while in guest debugging mode.
  4863. */
  4864. to_vmx(vcpu)->vcpu.arch.event_exit_inst_len =
  4865. vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
  4866. if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
  4867. return false;
  4868. /* fall through */
  4869. case DB_VECTOR:
  4870. if (vcpu->guest_debug &
  4871. (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
  4872. return false;
  4873. /* fall through */
  4874. case DE_VECTOR:
  4875. case OF_VECTOR:
  4876. case BR_VECTOR:
  4877. case UD_VECTOR:
  4878. case DF_VECTOR:
  4879. case SS_VECTOR:
  4880. case GP_VECTOR:
  4881. case MF_VECTOR:
  4882. return true;
  4883. break;
  4884. }
  4885. return false;
  4886. }
  4887. static int handle_rmode_exception(struct kvm_vcpu *vcpu,
  4888. int vec, u32 err_code)
  4889. {
  4890. /*
  4891. * Instruction with address size override prefix opcode 0x67
  4892. * Cause the #SS fault with 0 error code in VM86 mode.
  4893. */
  4894. if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) {
  4895. if (emulate_instruction(vcpu, 0) == EMULATE_DONE) {
  4896. if (vcpu->arch.halt_request) {
  4897. vcpu->arch.halt_request = 0;
  4898. return kvm_vcpu_halt(vcpu);
  4899. }
  4900. return 1;
  4901. }
  4902. return 0;
  4903. }
  4904. /*
  4905. * Forward all other exceptions that are valid in real mode.
  4906. * FIXME: Breaks guest debugging in real mode, needs to be fixed with
  4907. * the required debugging infrastructure rework.
  4908. */
  4909. kvm_queue_exception(vcpu, vec);
  4910. return 1;
  4911. }
  4912. /*
  4913. * Trigger machine check on the host. We assume all the MSRs are already set up
  4914. * by the CPU and that we still run on the same CPU as the MCE occurred on.
  4915. * We pass a fake environment to the machine check handler because we want
  4916. * the guest to be always treated like user space, no matter what context
  4917. * it used internally.
  4918. */
  4919. static void kvm_machine_check(void)
  4920. {
  4921. #if defined(CONFIG_X86_MCE) && defined(CONFIG_X86_64)
  4922. struct pt_regs regs = {
  4923. .cs = 3, /* Fake ring 3 no matter what the guest ran on */
  4924. .flags = X86_EFLAGS_IF,
  4925. };
  4926. do_machine_check(&regs, 0);
  4927. #endif
  4928. }
  4929. static int handle_machine_check(struct kvm_vcpu *vcpu)
  4930. {
  4931. /* already handled by vcpu_run */
  4932. return 1;
  4933. }
  4934. static int handle_exception(struct kvm_vcpu *vcpu)
  4935. {
  4936. struct vcpu_vmx *vmx = to_vmx(vcpu);
  4937. struct kvm_run *kvm_run = vcpu->run;
  4938. u32 intr_info, ex_no, error_code;
  4939. unsigned long cr2, rip, dr6;
  4940. u32 vect_info;
  4941. enum emulation_result er;
  4942. vect_info = vmx->idt_vectoring_info;
  4943. intr_info = vmx->exit_intr_info;
  4944. if (is_machine_check(intr_info))
  4945. return handle_machine_check(vcpu);
  4946. if (is_nmi(intr_info))
  4947. return 1; /* already handled by vmx_vcpu_run() */
  4948. if (is_no_device(intr_info)) {
  4949. vmx_fpu_activate(vcpu);
  4950. return 1;
  4951. }
  4952. if (is_invalid_opcode(intr_info)) {
  4953. if (is_guest_mode(vcpu)) {
  4954. kvm_queue_exception(vcpu, UD_VECTOR);
  4955. return 1;
  4956. }
  4957. er = emulate_instruction(vcpu, EMULTYPE_TRAP_UD);
  4958. if (er != EMULATE_DONE)
  4959. kvm_queue_exception(vcpu, UD_VECTOR);
  4960. return 1;
  4961. }
  4962. error_code = 0;
  4963. if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
  4964. error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
  4965. /*
  4966. * The #PF with PFEC.RSVD = 1 indicates the guest is accessing
  4967. * MMIO, it is better to report an internal error.
  4968. * See the comments in vmx_handle_exit.
  4969. */
  4970. if ((vect_info & VECTORING_INFO_VALID_MASK) &&
  4971. !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) {
  4972. vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
  4973. vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
  4974. vcpu->run->internal.ndata = 3;
  4975. vcpu->run->internal.data[0] = vect_info;
  4976. vcpu->run->internal.data[1] = intr_info;
  4977. vcpu->run->internal.data[2] = error_code;
  4978. return 0;
  4979. }
  4980. if (is_page_fault(intr_info)) {
  4981. /* EPT won't cause page fault directly */
  4982. BUG_ON(enable_ept);
  4983. cr2 = vmcs_readl(EXIT_QUALIFICATION);
  4984. trace_kvm_page_fault(cr2, error_code);
  4985. if (kvm_event_needs_reinjection(vcpu))
  4986. kvm_mmu_unprotect_page_virt(vcpu, cr2);
  4987. return kvm_mmu_page_fault(vcpu, cr2, error_code, NULL, 0);
  4988. }
  4989. ex_no = intr_info & INTR_INFO_VECTOR_MASK;
  4990. if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no))
  4991. return handle_rmode_exception(vcpu, ex_no, error_code);
  4992. switch (ex_no) {
  4993. case AC_VECTOR:
  4994. kvm_queue_exception_e(vcpu, AC_VECTOR, error_code);
  4995. return 1;
  4996. case DB_VECTOR:
  4997. dr6 = vmcs_readl(EXIT_QUALIFICATION);
  4998. if (!(vcpu->guest_debug &
  4999. (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
  5000. vcpu->arch.dr6 &= ~15;
  5001. vcpu->arch.dr6 |= dr6 | DR6_RTM;
  5002. if (!(dr6 & ~DR6_RESERVED)) /* icebp */
  5003. skip_emulated_instruction(vcpu);
  5004. kvm_queue_exception(vcpu, DB_VECTOR);
  5005. return 1;
  5006. }
  5007. kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1;
  5008. kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
  5009. /* fall through */
  5010. case BP_VECTOR:
  5011. /*
  5012. * Update instruction length as we may reinject #BP from
  5013. * user space while in guest debugging mode. Reading it for
  5014. * #DB as well causes no harm, it is not used in that case.
  5015. */
  5016. vmx->vcpu.arch.event_exit_inst_len =
  5017. vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
  5018. kvm_run->exit_reason = KVM_EXIT_DEBUG;
  5019. rip = kvm_rip_read(vcpu);
  5020. kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip;
  5021. kvm_run->debug.arch.exception = ex_no;
  5022. break;
  5023. default:
  5024. kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
  5025. kvm_run->ex.exception = ex_no;
  5026. kvm_run->ex.error_code = error_code;
  5027. break;
  5028. }
  5029. return 0;
  5030. }
  5031. static int handle_external_interrupt(struct kvm_vcpu *vcpu)
  5032. {
  5033. ++vcpu->stat.irq_exits;
  5034. return 1;
  5035. }
  5036. static int handle_triple_fault(struct kvm_vcpu *vcpu)
  5037. {
  5038. vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
  5039. return 0;
  5040. }
  5041. static int handle_io(struct kvm_vcpu *vcpu)
  5042. {
  5043. unsigned long exit_qualification;
  5044. int size, in, string, ret;
  5045. unsigned port;
  5046. exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
  5047. string = (exit_qualification & 16) != 0;
  5048. in = (exit_qualification & 8) != 0;
  5049. ++vcpu->stat.io_exits;
  5050. if (string || in)
  5051. return emulate_instruction(vcpu, 0) == EMULATE_DONE;
  5052. port = exit_qualification >> 16;
  5053. size = (exit_qualification & 7) + 1;
  5054. ret = kvm_skip_emulated_instruction(vcpu);
  5055. /*
  5056. * TODO: we might be squashing a KVM_GUESTDBG_SINGLESTEP-triggered
  5057. * KVM_EXIT_DEBUG here.
  5058. */
  5059. return kvm_fast_pio_out(vcpu, size, port) && ret;
  5060. }
  5061. static void
  5062. vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
  5063. {
  5064. /*
  5065. * Patch in the VMCALL instruction:
  5066. */
  5067. hypercall[0] = 0x0f;
  5068. hypercall[1] = 0x01;
  5069. hypercall[2] = 0xc1;
  5070. }
  5071. /* called to set cr0 as appropriate for a mov-to-cr0 exit. */
  5072. static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val)
  5073. {
  5074. if (is_guest_mode(vcpu)) {
  5075. struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
  5076. unsigned long orig_val = val;
  5077. /*
  5078. * We get here when L2 changed cr0 in a way that did not change
  5079. * any of L1's shadowed bits (see nested_vmx_exit_handled_cr),
  5080. * but did change L0 shadowed bits. So we first calculate the
  5081. * effective cr0 value that L1 would like to write into the
  5082. * hardware. It consists of the L2-owned bits from the new
  5083. * value combined with the L1-owned bits from L1's guest_cr0.
  5084. */
  5085. val = (val & ~vmcs12->cr0_guest_host_mask) |
  5086. (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask);
  5087. if (!nested_guest_cr0_valid(vcpu, val))
  5088. return 1;
  5089. if (kvm_set_cr0(vcpu, val))
  5090. return 1;
  5091. vmcs_writel(CR0_READ_SHADOW, orig_val);
  5092. return 0;
  5093. } else {
  5094. if (to_vmx(vcpu)->nested.vmxon &&
  5095. !nested_host_cr0_valid(vcpu, val))
  5096. return 1;
  5097. return kvm_set_cr0(vcpu, val);
  5098. }
  5099. }
  5100. static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val)
  5101. {
  5102. if (is_guest_mode(vcpu)) {
  5103. struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
  5104. unsigned long orig_val = val;
  5105. /* analogously to handle_set_cr0 */
  5106. val = (val & ~vmcs12->cr4_guest_host_mask) |
  5107. (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask);
  5108. if (kvm_set_cr4(vcpu, val))
  5109. return 1;
  5110. vmcs_writel(CR4_READ_SHADOW, orig_val);
  5111. return 0;
  5112. } else
  5113. return kvm_set_cr4(vcpu, val);
  5114. }
  5115. /* called to set cr0 as appropriate for clts instruction exit. */
  5116. static void handle_clts(struct kvm_vcpu *vcpu)
  5117. {
  5118. if (is_guest_mode(vcpu)) {
  5119. /*
  5120. * We get here when L2 did CLTS, and L1 didn't shadow CR0.TS
  5121. * but we did (!fpu_active). We need to keep GUEST_CR0.TS on,
  5122. * just pretend it's off (also in arch.cr0 for fpu_activate).
  5123. */
  5124. vmcs_writel(CR0_READ_SHADOW,
  5125. vmcs_readl(CR0_READ_SHADOW) & ~X86_CR0_TS);
  5126. vcpu->arch.cr0 &= ~X86_CR0_TS;
  5127. } else
  5128. vmx_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
  5129. }
  5130. static int handle_cr(struct kvm_vcpu *vcpu)
  5131. {
  5132. unsigned long exit_qualification, val;
  5133. int cr;
  5134. int reg;
  5135. int err;
  5136. int ret;
  5137. exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
  5138. cr = exit_qualification & 15;
  5139. reg = (exit_qualification >> 8) & 15;
  5140. switch ((exit_qualification >> 4) & 3) {
  5141. case 0: /* mov to cr */
  5142. val = kvm_register_readl(vcpu, reg);
  5143. trace_kvm_cr_write(cr, val);
  5144. switch (cr) {
  5145. case 0:
  5146. err = handle_set_cr0(vcpu, val);
  5147. return kvm_complete_insn_gp(vcpu, err);
  5148. case 3:
  5149. err = kvm_set_cr3(vcpu, val);
  5150. return kvm_complete_insn_gp(vcpu, err);
  5151. case 4:
  5152. err = handle_set_cr4(vcpu, val);
  5153. return kvm_complete_insn_gp(vcpu, err);
  5154. case 8: {
  5155. u8 cr8_prev = kvm_get_cr8(vcpu);
  5156. u8 cr8 = (u8)val;
  5157. err = kvm_set_cr8(vcpu, cr8);
  5158. ret = kvm_complete_insn_gp(vcpu, err);
  5159. if (lapic_in_kernel(vcpu))
  5160. return ret;
  5161. if (cr8_prev <= cr8)
  5162. return ret;
  5163. /*
  5164. * TODO: we might be squashing a
  5165. * KVM_GUESTDBG_SINGLESTEP-triggered
  5166. * KVM_EXIT_DEBUG here.
  5167. */
  5168. vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
  5169. return 0;
  5170. }
  5171. }
  5172. break;
  5173. case 2: /* clts */
  5174. handle_clts(vcpu);
  5175. trace_kvm_cr_write(0, kvm_read_cr0(vcpu));
  5176. vmx_fpu_activate(vcpu);
  5177. return kvm_skip_emulated_instruction(vcpu);
  5178. case 1: /*mov from cr*/
  5179. switch (cr) {
  5180. case 3:
  5181. val = kvm_read_cr3(vcpu);
  5182. kvm_register_write(vcpu, reg, val);
  5183. trace_kvm_cr_read(cr, val);
  5184. return kvm_skip_emulated_instruction(vcpu);
  5185. case 8:
  5186. val = kvm_get_cr8(vcpu);
  5187. kvm_register_write(vcpu, reg, val);
  5188. trace_kvm_cr_read(cr, val);
  5189. return kvm_skip_emulated_instruction(vcpu);
  5190. }
  5191. break;
  5192. case 3: /* lmsw */
  5193. val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
  5194. trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
  5195. kvm_lmsw(vcpu, val);
  5196. return kvm_skip_emulated_instruction(vcpu);
  5197. default:
  5198. break;
  5199. }
  5200. vcpu->run->exit_reason = 0;
  5201. vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
  5202. (int)(exit_qualification >> 4) & 3, cr);
  5203. return 0;
  5204. }
  5205. static int handle_dr(struct kvm_vcpu *vcpu)
  5206. {
  5207. unsigned long exit_qualification;
  5208. int dr, dr7, reg;
  5209. exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
  5210. dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
  5211. /* First, if DR does not exist, trigger UD */
  5212. if (!kvm_require_dr(vcpu, dr))
  5213. return 1;
  5214. /* Do not handle if the CPL > 0, will trigger GP on re-entry */
  5215. if (!kvm_require_cpl(vcpu, 0))
  5216. return 1;
  5217. dr7 = vmcs_readl(GUEST_DR7);
  5218. if (dr7 & DR7_GD) {
  5219. /*
  5220. * As the vm-exit takes precedence over the debug trap, we
  5221. * need to emulate the latter, either for the host or the
  5222. * guest debugging itself.
  5223. */
  5224. if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
  5225. vcpu->run->debug.arch.dr6 = vcpu->arch.dr6;
  5226. vcpu->run->debug.arch.dr7 = dr7;
  5227. vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu);
  5228. vcpu->run->debug.arch.exception = DB_VECTOR;
  5229. vcpu->run->exit_reason = KVM_EXIT_DEBUG;
  5230. return 0;
  5231. } else {
  5232. vcpu->arch.dr6 &= ~15;
  5233. vcpu->arch.dr6 |= DR6_BD | DR6_RTM;
  5234. kvm_queue_exception(vcpu, DB_VECTOR);
  5235. return 1;
  5236. }
  5237. }
  5238. if (vcpu->guest_debug == 0) {
  5239. vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL,
  5240. CPU_BASED_MOV_DR_EXITING);
  5241. /*
  5242. * No more DR vmexits; force a reload of the debug registers
  5243. * and reenter on this instruction. The next vmexit will
  5244. * retrieve the full state of the debug registers.
  5245. */
  5246. vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
  5247. return 1;
  5248. }
  5249. reg = DEBUG_REG_ACCESS_REG(exit_qualification);
  5250. if (exit_qualification & TYPE_MOV_FROM_DR) {
  5251. unsigned long val;
  5252. if (kvm_get_dr(vcpu, dr, &val))
  5253. return 1;
  5254. kvm_register_write(vcpu, reg, val);
  5255. } else
  5256. if (kvm_set_dr(vcpu, dr, kvm_register_readl(vcpu, reg)))
  5257. return 1;
  5258. return kvm_skip_emulated_instruction(vcpu);
  5259. }
  5260. static u64 vmx_get_dr6(struct kvm_vcpu *vcpu)
  5261. {
  5262. return vcpu->arch.dr6;
  5263. }
  5264. static void vmx_set_dr6(struct kvm_vcpu *vcpu, unsigned long val)
  5265. {
  5266. }
  5267. static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
  5268. {
  5269. get_debugreg(vcpu->arch.db[0], 0);
  5270. get_debugreg(vcpu->arch.db[1], 1);
  5271. get_debugreg(vcpu->arch.db[2], 2);
  5272. get_debugreg(vcpu->arch.db[3], 3);
  5273. get_debugreg(vcpu->arch.dr6, 6);
  5274. vcpu->arch.dr7 = vmcs_readl(GUEST_DR7);
  5275. vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
  5276. vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL, CPU_BASED_MOV_DR_EXITING);
  5277. }
  5278. static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
  5279. {
  5280. vmcs_writel(GUEST_DR7, val);
  5281. }
  5282. static int handle_cpuid(struct kvm_vcpu *vcpu)
  5283. {
  5284. return kvm_emulate_cpuid(vcpu);
  5285. }
  5286. static int handle_rdmsr(struct kvm_vcpu *vcpu)
  5287. {
  5288. u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
  5289. struct msr_data msr_info;
  5290. msr_info.index = ecx;
  5291. msr_info.host_initiated = false;
  5292. if (vmx_get_msr(vcpu, &msr_info)) {
  5293. trace_kvm_msr_read_ex(ecx);
  5294. kvm_inject_gp(vcpu, 0);
  5295. return 1;
  5296. }
  5297. trace_kvm_msr_read(ecx, msr_info.data);
  5298. /* FIXME: handling of bits 32:63 of rax, rdx */
  5299. vcpu->arch.regs[VCPU_REGS_RAX] = msr_info.data & -1u;
  5300. vcpu->arch.regs[VCPU_REGS_RDX] = (msr_info.data >> 32) & -1u;
  5301. return kvm_skip_emulated_instruction(vcpu);
  5302. }
  5303. static int handle_wrmsr(struct kvm_vcpu *vcpu)
  5304. {
  5305. struct msr_data msr;
  5306. u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
  5307. u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u)
  5308. | ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32);
  5309. msr.data = data;
  5310. msr.index = ecx;
  5311. msr.host_initiated = false;
  5312. if (kvm_set_msr(vcpu, &msr) != 0) {
  5313. trace_kvm_msr_write_ex(ecx, data);
  5314. kvm_inject_gp(vcpu, 0);
  5315. return 1;
  5316. }
  5317. trace_kvm_msr_write(ecx, data);
  5318. return kvm_skip_emulated_instruction(vcpu);
  5319. }
  5320. static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
  5321. {
  5322. kvm_make_request(KVM_REQ_EVENT, vcpu);
  5323. return 1;
  5324. }
  5325. static int handle_interrupt_window(struct kvm_vcpu *vcpu)
  5326. {
  5327. u32 cpu_based_vm_exec_control;
  5328. /* clear pending irq */
  5329. cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
  5330. cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
  5331. vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
  5332. kvm_make_request(KVM_REQ_EVENT, vcpu);
  5333. ++vcpu->stat.irq_window_exits;
  5334. return 1;
  5335. }
  5336. static int handle_halt(struct kvm_vcpu *vcpu)
  5337. {
  5338. return kvm_emulate_halt(vcpu);
  5339. }
  5340. static int handle_vmcall(struct kvm_vcpu *vcpu)
  5341. {
  5342. return kvm_emulate_hypercall(vcpu);
  5343. }
  5344. static int handle_invd(struct kvm_vcpu *vcpu)
  5345. {
  5346. return emulate_instruction(vcpu, 0) == EMULATE_DONE;
  5347. }
  5348. static int handle_invlpg(struct kvm_vcpu *vcpu)
  5349. {
  5350. unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
  5351. kvm_mmu_invlpg(vcpu, exit_qualification);
  5352. return kvm_skip_emulated_instruction(vcpu);
  5353. }
  5354. static int handle_rdpmc(struct kvm_vcpu *vcpu)
  5355. {
  5356. int err;
  5357. err = kvm_rdpmc(vcpu);
  5358. return kvm_complete_insn_gp(vcpu, err);
  5359. }
  5360. static int handle_wbinvd(struct kvm_vcpu *vcpu)
  5361. {
  5362. return kvm_emulate_wbinvd(vcpu);
  5363. }
  5364. static int handle_xsetbv(struct kvm_vcpu *vcpu)
  5365. {
  5366. u64 new_bv = kvm_read_edx_eax(vcpu);
  5367. u32 index = kvm_register_read(vcpu, VCPU_REGS_RCX);
  5368. if (kvm_set_xcr(vcpu, index, new_bv) == 0)
  5369. return kvm_skip_emulated_instruction(vcpu);
  5370. return 1;
  5371. }
  5372. static int handle_xsaves(struct kvm_vcpu *vcpu)
  5373. {
  5374. kvm_skip_emulated_instruction(vcpu);
  5375. WARN(1, "this should never happen\n");
  5376. return 1;
  5377. }
  5378. static int handle_xrstors(struct kvm_vcpu *vcpu)
  5379. {
  5380. kvm_skip_emulated_instruction(vcpu);
  5381. WARN(1, "this should never happen\n");
  5382. return 1;
  5383. }
  5384. static int handle_apic_access(struct kvm_vcpu *vcpu)
  5385. {
  5386. if (likely(fasteoi)) {
  5387. unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
  5388. int access_type, offset;
  5389. access_type = exit_qualification & APIC_ACCESS_TYPE;
  5390. offset = exit_qualification & APIC_ACCESS_OFFSET;
  5391. /*
  5392. * Sane guest uses MOV to write EOI, with written value
  5393. * not cared. So make a short-circuit here by avoiding
  5394. * heavy instruction emulation.
  5395. */
  5396. if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) &&
  5397. (offset == APIC_EOI)) {
  5398. kvm_lapic_set_eoi(vcpu);
  5399. return kvm_skip_emulated_instruction(vcpu);
  5400. }
  5401. }
  5402. return emulate_instruction(vcpu, 0) == EMULATE_DONE;
  5403. }
  5404. static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu)
  5405. {
  5406. unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
  5407. int vector = exit_qualification & 0xff;
  5408. /* EOI-induced VM exit is trap-like and thus no need to adjust IP */
  5409. kvm_apic_set_eoi_accelerated(vcpu, vector);
  5410. return 1;
  5411. }
  5412. static int handle_apic_write(struct kvm_vcpu *vcpu)
  5413. {
  5414. unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
  5415. u32 offset = exit_qualification & 0xfff;
  5416. /* APIC-write VM exit is trap-like and thus no need to adjust IP */
  5417. kvm_apic_write_nodecode(vcpu, offset);
  5418. return 1;
  5419. }
  5420. static int handle_task_switch(struct kvm_vcpu *vcpu)
  5421. {
  5422. struct vcpu_vmx *vmx = to_vmx(vcpu);
  5423. unsigned long exit_qualification;
  5424. bool has_error_code = false;
  5425. u32 error_code = 0;
  5426. u16 tss_selector;
  5427. int reason, type, idt_v, idt_index;
  5428. idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
  5429. idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK);
  5430. type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
  5431. exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
  5432. reason = (u32)exit_qualification >> 30;
  5433. if (reason == TASK_SWITCH_GATE && idt_v) {
  5434. switch (type) {
  5435. case INTR_TYPE_NMI_INTR:
  5436. vcpu->arch.nmi_injected = false;
  5437. vmx_set_nmi_mask(vcpu, true);
  5438. break;
  5439. case INTR_TYPE_EXT_INTR:
  5440. case INTR_TYPE_SOFT_INTR:
  5441. kvm_clear_interrupt_queue(vcpu);
  5442. break;
  5443. case INTR_TYPE_HARD_EXCEPTION:
  5444. if (vmx->idt_vectoring_info &
  5445. VECTORING_INFO_DELIVER_CODE_MASK) {
  5446. has_error_code = true;
  5447. error_code =
  5448. vmcs_read32(IDT_VECTORING_ERROR_CODE);
  5449. }
  5450. /* fall through */
  5451. case INTR_TYPE_SOFT_EXCEPTION:
  5452. kvm_clear_exception_queue(vcpu);
  5453. break;
  5454. default:
  5455. break;
  5456. }
  5457. }
  5458. tss_selector = exit_qualification;
  5459. if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
  5460. type != INTR_TYPE_EXT_INTR &&
  5461. type != INTR_TYPE_NMI_INTR))
  5462. skip_emulated_instruction(vcpu);
  5463. if (kvm_task_switch(vcpu, tss_selector,
  5464. type == INTR_TYPE_SOFT_INTR ? idt_index : -1, reason,
  5465. has_error_code, error_code) == EMULATE_FAIL) {
  5466. vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
  5467. vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
  5468. vcpu->run->internal.ndata = 0;
  5469. return 0;
  5470. }
  5471. /*
  5472. * TODO: What about debug traps on tss switch?
  5473. * Are we supposed to inject them and update dr6?
  5474. */
  5475. return 1;
  5476. }
  5477. static int handle_ept_violation(struct kvm_vcpu *vcpu)
  5478. {
  5479. unsigned long exit_qualification;
  5480. gpa_t gpa;
  5481. u32 error_code;
  5482. int gla_validity;
  5483. exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
  5484. gla_validity = (exit_qualification >> 7) & 0x3;
  5485. if (gla_validity == 0x2) {
  5486. printk(KERN_ERR "EPT: Handling EPT violation failed!\n");
  5487. printk(KERN_ERR "EPT: GPA: 0x%lx, GVA: 0x%lx\n",
  5488. (long unsigned int)vmcs_read64(GUEST_PHYSICAL_ADDRESS),
  5489. vmcs_readl(GUEST_LINEAR_ADDRESS));
  5490. printk(KERN_ERR "EPT: Exit qualification is 0x%lx\n",
  5491. (long unsigned int)exit_qualification);
  5492. vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
  5493. vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_VIOLATION;
  5494. return 0;
  5495. }
  5496. /*
  5497. * EPT violation happened while executing iret from NMI,
  5498. * "blocked by NMI" bit has to be set before next VM entry.
  5499. * There are errata that may cause this bit to not be set:
  5500. * AAK134, BY25.
  5501. */
  5502. if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
  5503. cpu_has_virtual_nmis() &&
  5504. (exit_qualification & INTR_INFO_UNBLOCK_NMI))
  5505. vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI);
  5506. gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
  5507. trace_kvm_page_fault(gpa, exit_qualification);
  5508. /* it is a read fault? */
  5509. error_code = (exit_qualification << 2) & PFERR_USER_MASK;
  5510. /* it is a write fault? */
  5511. error_code |= exit_qualification & PFERR_WRITE_MASK;
  5512. /* It is a fetch fault? */
  5513. error_code |= (exit_qualification << 2) & PFERR_FETCH_MASK;
  5514. /* ept page table is present? */
  5515. error_code |= (exit_qualification & 0x38) != 0;
  5516. vcpu->arch.exit_qualification = exit_qualification;
  5517. return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
  5518. }
  5519. static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
  5520. {
  5521. int ret;
  5522. gpa_t gpa;
  5523. gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
  5524. if (!kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) {
  5525. trace_kvm_fast_mmio(gpa);
  5526. return kvm_skip_emulated_instruction(vcpu);
  5527. }
  5528. ret = handle_mmio_page_fault(vcpu, gpa, true);
  5529. if (likely(ret == RET_MMIO_PF_EMULATE))
  5530. return x86_emulate_instruction(vcpu, gpa, 0, NULL, 0) ==
  5531. EMULATE_DONE;
  5532. if (unlikely(ret == RET_MMIO_PF_INVALID))
  5533. return kvm_mmu_page_fault(vcpu, gpa, 0, NULL, 0);
  5534. if (unlikely(ret == RET_MMIO_PF_RETRY))
  5535. return 1;
  5536. /* It is the real ept misconfig */
  5537. WARN_ON(1);
  5538. vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
  5539. vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_MISCONFIG;
  5540. return 0;
  5541. }
  5542. static int handle_nmi_window(struct kvm_vcpu *vcpu)
  5543. {
  5544. u32 cpu_based_vm_exec_control;
  5545. /* clear pending NMI */
  5546. cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
  5547. cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
  5548. vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
  5549. ++vcpu->stat.nmi_window_exits;
  5550. kvm_make_request(KVM_REQ_EVENT, vcpu);
  5551. return 1;
  5552. }
  5553. static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
  5554. {
  5555. struct vcpu_vmx *vmx = to_vmx(vcpu);
  5556. enum emulation_result err = EMULATE_DONE;
  5557. int ret = 1;
  5558. u32 cpu_exec_ctrl;
  5559. bool intr_window_requested;
  5560. unsigned count = 130;
  5561. cpu_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
  5562. intr_window_requested = cpu_exec_ctrl & CPU_BASED_VIRTUAL_INTR_PENDING;
  5563. while (vmx->emulation_required && count-- != 0) {
  5564. if (intr_window_requested && vmx_interrupt_allowed(vcpu))
  5565. return handle_interrupt_window(&vmx->vcpu);
  5566. if (test_bit(KVM_REQ_EVENT, &vcpu->requests))
  5567. return 1;
  5568. err = emulate_instruction(vcpu, EMULTYPE_NO_REEXECUTE);
  5569. if (err == EMULATE_USER_EXIT) {
  5570. ++vcpu->stat.mmio_exits;
  5571. ret = 0;
  5572. goto out;
  5573. }
  5574. if (err != EMULATE_DONE) {
  5575. vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
  5576. vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
  5577. vcpu->run->internal.ndata = 0;
  5578. return 0;
  5579. }
  5580. if (vcpu->arch.halt_request) {
  5581. vcpu->arch.halt_request = 0;
  5582. ret = kvm_vcpu_halt(vcpu);
  5583. goto out;
  5584. }
  5585. if (signal_pending(current))
  5586. goto out;
  5587. if (need_resched())
  5588. schedule();
  5589. }
  5590. out:
  5591. return ret;
  5592. }
  5593. static int __grow_ple_window(int val)
  5594. {
  5595. if (ple_window_grow < 1)
  5596. return ple_window;
  5597. val = min(val, ple_window_actual_max);
  5598. if (ple_window_grow < ple_window)
  5599. val *= ple_window_grow;
  5600. else
  5601. val += ple_window_grow;
  5602. return val;
  5603. }
  5604. static int __shrink_ple_window(int val, int modifier, int minimum)
  5605. {
  5606. if (modifier < 1)
  5607. return ple_window;
  5608. if (modifier < ple_window)
  5609. val /= modifier;
  5610. else
  5611. val -= modifier;
  5612. return max(val, minimum);
  5613. }
  5614. static void grow_ple_window(struct kvm_vcpu *vcpu)
  5615. {
  5616. struct vcpu_vmx *vmx = to_vmx(vcpu);
  5617. int old = vmx->ple_window;
  5618. vmx->ple_window = __grow_ple_window(old);
  5619. if (vmx->ple_window != old)
  5620. vmx->ple_window_dirty = true;
  5621. trace_kvm_ple_window_grow(vcpu->vcpu_id, vmx->ple_window, old);
  5622. }
  5623. static void shrink_ple_window(struct kvm_vcpu *vcpu)
  5624. {
  5625. struct vcpu_vmx *vmx = to_vmx(vcpu);
  5626. int old = vmx->ple_window;
  5627. vmx->ple_window = __shrink_ple_window(old,
  5628. ple_window_shrink, ple_window);
  5629. if (vmx->ple_window != old)
  5630. vmx->ple_window_dirty = true;
  5631. trace_kvm_ple_window_shrink(vcpu->vcpu_id, vmx->ple_window, old);
  5632. }
  5633. /*
  5634. * ple_window_actual_max is computed to be one grow_ple_window() below
  5635. * ple_window_max. (See __grow_ple_window for the reason.)
  5636. * This prevents overflows, because ple_window_max is int.
  5637. * ple_window_max effectively rounded down to a multiple of ple_window_grow in
  5638. * this process.
  5639. * ple_window_max is also prevented from setting vmx->ple_window < ple_window.
  5640. */
  5641. static void update_ple_window_actual_max(void)
  5642. {
  5643. ple_window_actual_max =
  5644. __shrink_ple_window(max(ple_window_max, ple_window),
  5645. ple_window_grow, INT_MIN);
  5646. }
  5647. /*
  5648. * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
  5649. */
  5650. static void wakeup_handler(void)
  5651. {
  5652. struct kvm_vcpu *vcpu;
  5653. int cpu = smp_processor_id();
  5654. spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
  5655. list_for_each_entry(vcpu, &per_cpu(blocked_vcpu_on_cpu, cpu),
  5656. blocked_vcpu_list) {
  5657. struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
  5658. if (pi_test_on(pi_desc) == 1)
  5659. kvm_vcpu_kick(vcpu);
  5660. }
  5661. spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
  5662. }
  5663. static __init int hardware_setup(void)
  5664. {
  5665. int r = -ENOMEM, i, msr;
  5666. rdmsrl_safe(MSR_EFER, &host_efer);
  5667. for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i)
  5668. kvm_define_shared_msr(i, vmx_msr_index[i]);
  5669. for (i = 0; i < VMX_BITMAP_NR; i++) {
  5670. vmx_bitmap[i] = (unsigned long *)__get_free_page(GFP_KERNEL);
  5671. if (!vmx_bitmap[i])
  5672. goto out;
  5673. }
  5674. vmx_io_bitmap_b = (unsigned long *)__get_free_page(GFP_KERNEL);
  5675. memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE);
  5676. memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE);
  5677. /*
  5678. * Allow direct access to the PC debug port (it is often used for I/O
  5679. * delays, but the vmexits simply slow things down).
  5680. */
  5681. memset(vmx_io_bitmap_a, 0xff, PAGE_SIZE);
  5682. clear_bit(0x80, vmx_io_bitmap_a);
  5683. memset(vmx_io_bitmap_b, 0xff, PAGE_SIZE);
  5684. memset(vmx_msr_bitmap_legacy, 0xff, PAGE_SIZE);
  5685. memset(vmx_msr_bitmap_longmode, 0xff, PAGE_SIZE);
  5686. if (setup_vmcs_config(&vmcs_config) < 0) {
  5687. r = -EIO;
  5688. goto out;
  5689. }
  5690. if (boot_cpu_has(X86_FEATURE_NX))
  5691. kvm_enable_efer_bits(EFER_NX);
  5692. if (!cpu_has_vmx_vpid())
  5693. enable_vpid = 0;
  5694. if (!cpu_has_vmx_shadow_vmcs())
  5695. enable_shadow_vmcs = 0;
  5696. if (enable_shadow_vmcs)
  5697. init_vmcs_shadow_fields();
  5698. if (!cpu_has_vmx_ept() ||
  5699. !cpu_has_vmx_ept_4levels()) {
  5700. enable_ept = 0;
  5701. enable_unrestricted_guest = 0;
  5702. enable_ept_ad_bits = 0;
  5703. }
  5704. if (!cpu_has_vmx_ept_ad_bits())
  5705. enable_ept_ad_bits = 0;
  5706. if (!cpu_has_vmx_unrestricted_guest())
  5707. enable_unrestricted_guest = 0;
  5708. if (!cpu_has_vmx_flexpriority())
  5709. flexpriority_enabled = 0;
  5710. /*
  5711. * set_apic_access_page_addr() is used to reload apic access
  5712. * page upon invalidation. No need to do anything if not
  5713. * using the APIC_ACCESS_ADDR VMCS field.
  5714. */
  5715. if (!flexpriority_enabled)
  5716. kvm_x86_ops->set_apic_access_page_addr = NULL;
  5717. if (!cpu_has_vmx_tpr_shadow())
  5718. kvm_x86_ops->update_cr8_intercept = NULL;
  5719. if (enable_ept && !cpu_has_vmx_ept_2m_page())
  5720. kvm_disable_largepages();
  5721. if (!cpu_has_vmx_ple())
  5722. ple_gap = 0;
  5723. if (!cpu_has_vmx_apicv())
  5724. enable_apicv = 0;
  5725. if (cpu_has_vmx_tsc_scaling()) {
  5726. kvm_has_tsc_control = true;
  5727. kvm_max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX;
  5728. kvm_tsc_scaling_ratio_frac_bits = 48;
  5729. }
  5730. vmx_disable_intercept_for_msr(MSR_FS_BASE, false);
  5731. vmx_disable_intercept_for_msr(MSR_GS_BASE, false);
  5732. vmx_disable_intercept_for_msr(MSR_KERNEL_GS_BASE, true);
  5733. vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_CS, false);
  5734. vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_ESP, false);
  5735. vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_EIP, false);
  5736. vmx_disable_intercept_for_msr(MSR_IA32_BNDCFGS, true);
  5737. memcpy(vmx_msr_bitmap_legacy_x2apic_apicv,
  5738. vmx_msr_bitmap_legacy, PAGE_SIZE);
  5739. memcpy(vmx_msr_bitmap_longmode_x2apic_apicv,
  5740. vmx_msr_bitmap_longmode, PAGE_SIZE);
  5741. memcpy(vmx_msr_bitmap_legacy_x2apic,
  5742. vmx_msr_bitmap_legacy, PAGE_SIZE);
  5743. memcpy(vmx_msr_bitmap_longmode_x2apic,
  5744. vmx_msr_bitmap_longmode, PAGE_SIZE);
  5745. set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
  5746. for (msr = 0x800; msr <= 0x8ff; msr++) {
  5747. if (msr == 0x839 /* TMCCT */)
  5748. continue;
  5749. vmx_disable_intercept_msr_x2apic(msr, MSR_TYPE_R, true);
  5750. }
  5751. /*
  5752. * TPR reads and writes can be virtualized even if virtual interrupt
  5753. * delivery is not in use.
  5754. */
  5755. vmx_disable_intercept_msr_x2apic(0x808, MSR_TYPE_W, true);
  5756. vmx_disable_intercept_msr_x2apic(0x808, MSR_TYPE_R | MSR_TYPE_W, false);
  5757. /* EOI */
  5758. vmx_disable_intercept_msr_x2apic(0x80b, MSR_TYPE_W, true);
  5759. /* SELF-IPI */
  5760. vmx_disable_intercept_msr_x2apic(0x83f, MSR_TYPE_W, true);
  5761. if (enable_ept) {
  5762. kvm_mmu_set_mask_ptes(VMX_EPT_READABLE_MASK,
  5763. (enable_ept_ad_bits) ? VMX_EPT_ACCESS_BIT : 0ull,
  5764. (enable_ept_ad_bits) ? VMX_EPT_DIRTY_BIT : 0ull,
  5765. 0ull, VMX_EPT_EXECUTABLE_MASK,
  5766. cpu_has_vmx_ept_execute_only() ?
  5767. 0ull : VMX_EPT_READABLE_MASK);
  5768. ept_set_mmio_spte_mask();
  5769. kvm_enable_tdp();
  5770. } else
  5771. kvm_disable_tdp();
  5772. update_ple_window_actual_max();
  5773. /*
  5774. * Only enable PML when hardware supports PML feature, and both EPT
  5775. * and EPT A/D bit features are enabled -- PML depends on them to work.
  5776. */
  5777. if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml())
  5778. enable_pml = 0;
  5779. if (!enable_pml) {
  5780. kvm_x86_ops->slot_enable_log_dirty = NULL;
  5781. kvm_x86_ops->slot_disable_log_dirty = NULL;
  5782. kvm_x86_ops->flush_log_dirty = NULL;
  5783. kvm_x86_ops->enable_log_dirty_pt_masked = NULL;
  5784. }
  5785. if (cpu_has_vmx_preemption_timer() && enable_preemption_timer) {
  5786. u64 vmx_msr;
  5787. rdmsrl(MSR_IA32_VMX_MISC, vmx_msr);
  5788. cpu_preemption_timer_multi =
  5789. vmx_msr & VMX_MISC_PREEMPTION_TIMER_RATE_MASK;
  5790. } else {
  5791. kvm_x86_ops->set_hv_timer = NULL;
  5792. kvm_x86_ops->cancel_hv_timer = NULL;
  5793. }
  5794. kvm_set_posted_intr_wakeup_handler(wakeup_handler);
  5795. kvm_mce_cap_supported |= MCG_LMCE_P;
  5796. return alloc_kvm_area();
  5797. out:
  5798. for (i = 0; i < VMX_BITMAP_NR; i++)
  5799. free_page((unsigned long)vmx_bitmap[i]);
  5800. return r;
  5801. }
  5802. static __exit void hardware_unsetup(void)
  5803. {
  5804. int i;
  5805. for (i = 0; i < VMX_BITMAP_NR; i++)
  5806. free_page((unsigned long)vmx_bitmap[i]);
  5807. free_kvm_area();
  5808. }
  5809. /*
  5810. * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
  5811. * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
  5812. */
  5813. static int handle_pause(struct kvm_vcpu *vcpu)
  5814. {
  5815. if (ple_gap)
  5816. grow_ple_window(vcpu);
  5817. kvm_vcpu_on_spin(vcpu);
  5818. return kvm_skip_emulated_instruction(vcpu);
  5819. }
  5820. static int handle_nop(struct kvm_vcpu *vcpu)
  5821. {
  5822. return kvm_skip_emulated_instruction(vcpu);
  5823. }
  5824. static int handle_mwait(struct kvm_vcpu *vcpu)
  5825. {
  5826. printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
  5827. return handle_nop(vcpu);
  5828. }
  5829. static int handle_monitor_trap(struct kvm_vcpu *vcpu)
  5830. {
  5831. return 1;
  5832. }
  5833. static int handle_monitor(struct kvm_vcpu *vcpu)
  5834. {
  5835. printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
  5836. return handle_nop(vcpu);
  5837. }
  5838. /*
  5839. * To run an L2 guest, we need a vmcs02 based on the L1-specified vmcs12.
  5840. * We could reuse a single VMCS for all the L2 guests, but we also want the
  5841. * option to allocate a separate vmcs02 for each separate loaded vmcs12 - this
  5842. * allows keeping them loaded on the processor, and in the future will allow
  5843. * optimizations where prepare_vmcs02 doesn't need to set all the fields on
  5844. * every entry if they never change.
  5845. * So we keep, in vmx->nested.vmcs02_pool, a cache of size VMCS02_POOL_SIZE
  5846. * (>=0) with a vmcs02 for each recently loaded vmcs12s, most recent first.
  5847. *
  5848. * The following functions allocate and free a vmcs02 in this pool.
  5849. */
  5850. /* Get a VMCS from the pool to use as vmcs02 for the current vmcs12. */
  5851. static struct loaded_vmcs *nested_get_current_vmcs02(struct vcpu_vmx *vmx)
  5852. {
  5853. struct vmcs02_list *item;
  5854. list_for_each_entry(item, &vmx->nested.vmcs02_pool, list)
  5855. if (item->vmptr == vmx->nested.current_vmptr) {
  5856. list_move(&item->list, &vmx->nested.vmcs02_pool);
  5857. return &item->vmcs02;
  5858. }
  5859. if (vmx->nested.vmcs02_num >= max(VMCS02_POOL_SIZE, 1)) {
  5860. /* Recycle the least recently used VMCS. */
  5861. item = list_last_entry(&vmx->nested.vmcs02_pool,
  5862. struct vmcs02_list, list);
  5863. item->vmptr = vmx->nested.current_vmptr;
  5864. list_move(&item->list, &vmx->nested.vmcs02_pool);
  5865. return &item->vmcs02;
  5866. }
  5867. /* Create a new VMCS */
  5868. item = kmalloc(sizeof(struct vmcs02_list), GFP_KERNEL);
  5869. if (!item)
  5870. return NULL;
  5871. item->vmcs02.vmcs = alloc_vmcs();
  5872. item->vmcs02.shadow_vmcs = NULL;
  5873. if (!item->vmcs02.vmcs) {
  5874. kfree(item);
  5875. return NULL;
  5876. }
  5877. loaded_vmcs_init(&item->vmcs02);
  5878. item->vmptr = vmx->nested.current_vmptr;
  5879. list_add(&(item->list), &(vmx->nested.vmcs02_pool));
  5880. vmx->nested.vmcs02_num++;
  5881. return &item->vmcs02;
  5882. }
  5883. /* Free and remove from pool a vmcs02 saved for a vmcs12 (if there is one) */
  5884. static void nested_free_vmcs02(struct vcpu_vmx *vmx, gpa_t vmptr)
  5885. {
  5886. struct vmcs02_list *item;
  5887. list_for_each_entry(item, &vmx->nested.vmcs02_pool, list)
  5888. if (item->vmptr == vmptr) {
  5889. free_loaded_vmcs(&item->vmcs02);
  5890. list_del(&item->list);
  5891. kfree(item);
  5892. vmx->nested.vmcs02_num--;
  5893. return;
  5894. }
  5895. }
  5896. /*
  5897. * Free all VMCSs saved for this vcpu, except the one pointed by
  5898. * vmx->loaded_vmcs. We must be running L1, so vmx->loaded_vmcs
  5899. * must be &vmx->vmcs01.
  5900. */
  5901. static void nested_free_all_saved_vmcss(struct vcpu_vmx *vmx)
  5902. {
  5903. struct vmcs02_list *item, *n;
  5904. WARN_ON(vmx->loaded_vmcs != &vmx->vmcs01);
  5905. list_for_each_entry_safe(item, n, &vmx->nested.vmcs02_pool, list) {
  5906. /*
  5907. * Something will leak if the above WARN triggers. Better than
  5908. * a use-after-free.
  5909. */
  5910. if (vmx->loaded_vmcs == &item->vmcs02)
  5911. continue;
  5912. free_loaded_vmcs(&item->vmcs02);
  5913. list_del(&item->list);
  5914. kfree(item);
  5915. vmx->nested.vmcs02_num--;
  5916. }
  5917. }
  5918. /*
  5919. * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(),
  5920. * set the success or error code of an emulated VMX instruction, as specified
  5921. * by Vol 2B, VMX Instruction Reference, "Conventions".
  5922. */
  5923. static void nested_vmx_succeed(struct kvm_vcpu *vcpu)
  5924. {
  5925. vmx_set_rflags(vcpu, vmx_get_rflags(vcpu)
  5926. & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
  5927. X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF));
  5928. }
  5929. static void nested_vmx_failInvalid(struct kvm_vcpu *vcpu)
  5930. {
  5931. vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
  5932. & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
  5933. X86_EFLAGS_SF | X86_EFLAGS_OF))
  5934. | X86_EFLAGS_CF);
  5935. }
  5936. static void nested_vmx_failValid(struct kvm_vcpu *vcpu,
  5937. u32 vm_instruction_error)
  5938. {
  5939. if (to_vmx(vcpu)->nested.current_vmptr == -1ull) {
  5940. /*
  5941. * failValid writes the error number to the current VMCS, which
  5942. * can't be done there isn't a current VMCS.
  5943. */
  5944. nested_vmx_failInvalid(vcpu);
  5945. return;
  5946. }
  5947. vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
  5948. & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
  5949. X86_EFLAGS_SF | X86_EFLAGS_OF))
  5950. | X86_EFLAGS_ZF);
  5951. get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error;
  5952. /*
  5953. * We don't need to force a shadow sync because
  5954. * VM_INSTRUCTION_ERROR is not shadowed
  5955. */
  5956. }
  5957. static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator)
  5958. {
  5959. /* TODO: not to reset guest simply here. */
  5960. kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
  5961. pr_debug_ratelimited("kvm: nested vmx abort, indicator %d\n", indicator);
  5962. }
  5963. static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer)
  5964. {
  5965. struct vcpu_vmx *vmx =
  5966. container_of(timer, struct vcpu_vmx, nested.preemption_timer);
  5967. vmx->nested.preemption_timer_expired = true;
  5968. kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu);
  5969. kvm_vcpu_kick(&vmx->vcpu);
  5970. return HRTIMER_NORESTART;
  5971. }
  5972. /*
  5973. * Decode the memory-address operand of a vmx instruction, as recorded on an
  5974. * exit caused by such an instruction (run by a guest hypervisor).
  5975. * On success, returns 0. When the operand is invalid, returns 1 and throws
  5976. * #UD or #GP.
  5977. */
  5978. static int get_vmx_mem_address(struct kvm_vcpu *vcpu,
  5979. unsigned long exit_qualification,
  5980. u32 vmx_instruction_info, bool wr, gva_t *ret)
  5981. {
  5982. gva_t off;
  5983. bool exn;
  5984. struct kvm_segment s;
  5985. /*
  5986. * According to Vol. 3B, "Information for VM Exits Due to Instruction
  5987. * Execution", on an exit, vmx_instruction_info holds most of the
  5988. * addressing components of the operand. Only the displacement part
  5989. * is put in exit_qualification (see 3B, "Basic VM-Exit Information").
  5990. * For how an actual address is calculated from all these components,
  5991. * refer to Vol. 1, "Operand Addressing".
  5992. */
  5993. int scaling = vmx_instruction_info & 3;
  5994. int addr_size = (vmx_instruction_info >> 7) & 7;
  5995. bool is_reg = vmx_instruction_info & (1u << 10);
  5996. int seg_reg = (vmx_instruction_info >> 15) & 7;
  5997. int index_reg = (vmx_instruction_info >> 18) & 0xf;
  5998. bool index_is_valid = !(vmx_instruction_info & (1u << 22));
  5999. int base_reg = (vmx_instruction_info >> 23) & 0xf;
  6000. bool base_is_valid = !(vmx_instruction_info & (1u << 27));
  6001. if (is_reg) {
  6002. kvm_queue_exception(vcpu, UD_VECTOR);
  6003. return 1;
  6004. }
  6005. /* Addr = segment_base + offset */
  6006. /* offset = base + [index * scale] + displacement */
  6007. off = exit_qualification; /* holds the displacement */
  6008. if (base_is_valid)
  6009. off += kvm_register_read(vcpu, base_reg);
  6010. if (index_is_valid)
  6011. off += kvm_register_read(vcpu, index_reg)<<scaling;
  6012. vmx_get_segment(vcpu, &s, seg_reg);
  6013. *ret = s.base + off;
  6014. if (addr_size == 1) /* 32 bit */
  6015. *ret &= 0xffffffff;
  6016. /* Checks for #GP/#SS exceptions. */
  6017. exn = false;
  6018. if (is_long_mode(vcpu)) {
  6019. /* Long mode: #GP(0)/#SS(0) if the memory address is in a
  6020. * non-canonical form. This is the only check on the memory
  6021. * destination for long mode!
  6022. */
  6023. exn = is_noncanonical_address(*ret);
  6024. } else if (is_protmode(vcpu)) {
  6025. /* Protected mode: apply checks for segment validity in the
  6026. * following order:
  6027. * - segment type check (#GP(0) may be thrown)
  6028. * - usability check (#GP(0)/#SS(0))
  6029. * - limit check (#GP(0)/#SS(0))
  6030. */
  6031. if (wr)
  6032. /* #GP(0) if the destination operand is located in a
  6033. * read-only data segment or any code segment.
  6034. */
  6035. exn = ((s.type & 0xa) == 0 || (s.type & 8));
  6036. else
  6037. /* #GP(0) if the source operand is located in an
  6038. * execute-only code segment
  6039. */
  6040. exn = ((s.type & 0xa) == 8);
  6041. if (exn) {
  6042. kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
  6043. return 1;
  6044. }
  6045. /* Protected mode: #GP(0)/#SS(0) if the segment is unusable.
  6046. */
  6047. exn = (s.unusable != 0);
  6048. /* Protected mode: #GP(0)/#SS(0) if the memory
  6049. * operand is outside the segment limit.
  6050. */
  6051. exn = exn || (off + sizeof(u64) > s.limit);
  6052. }
  6053. if (exn) {
  6054. kvm_queue_exception_e(vcpu,
  6055. seg_reg == VCPU_SREG_SS ?
  6056. SS_VECTOR : GP_VECTOR,
  6057. 0);
  6058. return 1;
  6059. }
  6060. return 0;
  6061. }
  6062. /*
  6063. * This function performs the various checks including
  6064. * - if it's 4KB aligned
  6065. * - No bits beyond the physical address width are set
  6066. * - Returns 0 on success or else 1
  6067. * (Intel SDM Section 30.3)
  6068. */
  6069. static int nested_vmx_check_vmptr(struct kvm_vcpu *vcpu, int exit_reason,
  6070. gpa_t *vmpointer)
  6071. {
  6072. gva_t gva;
  6073. gpa_t vmptr;
  6074. struct x86_exception e;
  6075. struct page *page;
  6076. struct vcpu_vmx *vmx = to_vmx(vcpu);
  6077. int maxphyaddr = cpuid_maxphyaddr(vcpu);
  6078. if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
  6079. vmcs_read32(VMX_INSTRUCTION_INFO), false, &gva))
  6080. return 1;
  6081. if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva, &vmptr,
  6082. sizeof(vmptr), &e)) {
  6083. kvm_inject_page_fault(vcpu, &e);
  6084. return 1;
  6085. }
  6086. switch (exit_reason) {
  6087. case EXIT_REASON_VMON:
  6088. /*
  6089. * SDM 3: 24.11.5
  6090. * The first 4 bytes of VMXON region contain the supported
  6091. * VMCS revision identifier
  6092. *
  6093. * Note - IA32_VMX_BASIC[48] will never be 1
  6094. * for the nested case;
  6095. * which replaces physical address width with 32
  6096. *
  6097. */
  6098. if (!PAGE_ALIGNED(vmptr) || (vmptr >> maxphyaddr)) {
  6099. nested_vmx_failInvalid(vcpu);
  6100. return kvm_skip_emulated_instruction(vcpu);
  6101. }
  6102. page = nested_get_page(vcpu, vmptr);
  6103. if (page == NULL ||
  6104. *(u32 *)kmap(page) != VMCS12_REVISION) {
  6105. nested_vmx_failInvalid(vcpu);
  6106. kunmap(page);
  6107. return kvm_skip_emulated_instruction(vcpu);
  6108. }
  6109. kunmap(page);
  6110. vmx->nested.vmxon_ptr = vmptr;
  6111. break;
  6112. case EXIT_REASON_VMCLEAR:
  6113. if (!PAGE_ALIGNED(vmptr) || (vmptr >> maxphyaddr)) {
  6114. nested_vmx_failValid(vcpu,
  6115. VMXERR_VMCLEAR_INVALID_ADDRESS);
  6116. return kvm_skip_emulated_instruction(vcpu);
  6117. }
  6118. if (vmptr == vmx->nested.vmxon_ptr) {
  6119. nested_vmx_failValid(vcpu,
  6120. VMXERR_VMCLEAR_VMXON_POINTER);
  6121. return kvm_skip_emulated_instruction(vcpu);
  6122. }
  6123. break;
  6124. case EXIT_REASON_VMPTRLD:
  6125. if (!PAGE_ALIGNED(vmptr) || (vmptr >> maxphyaddr)) {
  6126. nested_vmx_failValid(vcpu,
  6127. VMXERR_VMPTRLD_INVALID_ADDRESS);
  6128. return kvm_skip_emulated_instruction(vcpu);
  6129. }
  6130. if (vmptr == vmx->nested.vmxon_ptr) {
  6131. nested_vmx_failValid(vcpu,
  6132. VMXERR_VMPTRLD_VMXON_POINTER);
  6133. return kvm_skip_emulated_instruction(vcpu);
  6134. }
  6135. break;
  6136. default:
  6137. return 1; /* shouldn't happen */
  6138. }
  6139. if (vmpointer)
  6140. *vmpointer = vmptr;
  6141. return 0;
  6142. }
  6143. /*
  6144. * Emulate the VMXON instruction.
  6145. * Currently, we just remember that VMX is active, and do not save or even
  6146. * inspect the argument to VMXON (the so-called "VMXON pointer") because we
  6147. * do not currently need to store anything in that guest-allocated memory
  6148. * region. Consequently, VMCLEAR and VMPTRLD also do not verify that the their
  6149. * argument is different from the VMXON pointer (which the spec says they do).
  6150. */
  6151. static int handle_vmon(struct kvm_vcpu *vcpu)
  6152. {
  6153. struct kvm_segment cs;
  6154. struct vcpu_vmx *vmx = to_vmx(vcpu);
  6155. struct vmcs *shadow_vmcs;
  6156. const u64 VMXON_NEEDED_FEATURES = FEATURE_CONTROL_LOCKED
  6157. | FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
  6158. /* The Intel VMX Instruction Reference lists a bunch of bits that
  6159. * are prerequisite to running VMXON, most notably cr4.VMXE must be
  6160. * set to 1 (see vmx_set_cr4() for when we allow the guest to set this).
  6161. * Otherwise, we should fail with #UD. We test these now:
  6162. */
  6163. if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE) ||
  6164. !kvm_read_cr0_bits(vcpu, X86_CR0_PE) ||
  6165. (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) {
  6166. kvm_queue_exception(vcpu, UD_VECTOR);
  6167. return 1;
  6168. }
  6169. vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
  6170. if (is_long_mode(vcpu) && !cs.l) {
  6171. kvm_queue_exception(vcpu, UD_VECTOR);
  6172. return 1;
  6173. }
  6174. if (vmx_get_cpl(vcpu)) {
  6175. kvm_inject_gp(vcpu, 0);
  6176. return 1;
  6177. }
  6178. if (nested_vmx_check_vmptr(vcpu, EXIT_REASON_VMON, NULL))
  6179. return 1;
  6180. if (vmx->nested.vmxon) {
  6181. nested_vmx_failValid(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION);
  6182. return kvm_skip_emulated_instruction(vcpu);
  6183. }
  6184. if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES)
  6185. != VMXON_NEEDED_FEATURES) {
  6186. kvm_inject_gp(vcpu, 0);
  6187. return 1;
  6188. }
  6189. if (cpu_has_vmx_msr_bitmap()) {
  6190. vmx->nested.msr_bitmap =
  6191. (unsigned long *)__get_free_page(GFP_KERNEL);
  6192. if (!vmx->nested.msr_bitmap)
  6193. goto out_msr_bitmap;
  6194. }
  6195. vmx->nested.cached_vmcs12 = kmalloc(VMCS12_SIZE, GFP_KERNEL);
  6196. if (!vmx->nested.cached_vmcs12)
  6197. goto out_cached_vmcs12;
  6198. if (enable_shadow_vmcs) {
  6199. shadow_vmcs = alloc_vmcs();
  6200. if (!shadow_vmcs)
  6201. goto out_shadow_vmcs;
  6202. /* mark vmcs as shadow */
  6203. shadow_vmcs->revision_id |= (1u << 31);
  6204. /* init shadow vmcs */
  6205. vmcs_clear(shadow_vmcs);
  6206. vmx->vmcs01.shadow_vmcs = shadow_vmcs;
  6207. }
  6208. INIT_LIST_HEAD(&(vmx->nested.vmcs02_pool));
  6209. vmx->nested.vmcs02_num = 0;
  6210. hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC,
  6211. HRTIMER_MODE_REL_PINNED);
  6212. vmx->nested.preemption_timer.function = vmx_preemption_timer_fn;
  6213. vmx->nested.vmxon = true;
  6214. nested_vmx_succeed(vcpu);
  6215. return kvm_skip_emulated_instruction(vcpu);
  6216. out_shadow_vmcs:
  6217. kfree(vmx->nested.cached_vmcs12);
  6218. out_cached_vmcs12:
  6219. free_page((unsigned long)vmx->nested.msr_bitmap);
  6220. out_msr_bitmap:
  6221. return -ENOMEM;
  6222. }
  6223. /*
  6224. * Intel's VMX Instruction Reference specifies a common set of prerequisites
  6225. * for running VMX instructions (except VMXON, whose prerequisites are
  6226. * slightly different). It also specifies what exception to inject otherwise.
  6227. */
  6228. static int nested_vmx_check_permission(struct kvm_vcpu *vcpu)
  6229. {
  6230. struct kvm_segment cs;
  6231. struct vcpu_vmx *vmx = to_vmx(vcpu);
  6232. if (!vmx->nested.vmxon) {
  6233. kvm_queue_exception(vcpu, UD_VECTOR);
  6234. return 0;
  6235. }
  6236. vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
  6237. if ((vmx_get_rflags(vcpu) & X86_EFLAGS_VM) ||
  6238. (is_long_mode(vcpu) && !cs.l)) {
  6239. kvm_queue_exception(vcpu, UD_VECTOR);
  6240. return 0;
  6241. }
  6242. if (vmx_get_cpl(vcpu)) {
  6243. kvm_inject_gp(vcpu, 0);
  6244. return 0;
  6245. }
  6246. return 1;
  6247. }
  6248. static inline void nested_release_vmcs12(struct vcpu_vmx *vmx)
  6249. {
  6250. if (vmx->nested.current_vmptr == -1ull)
  6251. return;
  6252. /* current_vmptr and current_vmcs12 are always set/reset together */
  6253. if (WARN_ON(vmx->nested.current_vmcs12 == NULL))
  6254. return;
  6255. if (enable_shadow_vmcs) {
  6256. /* copy to memory all shadowed fields in case
  6257. they were modified */
  6258. copy_shadow_to_vmcs12(vmx);
  6259. vmx->nested.sync_shadow_vmcs = false;
  6260. vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,
  6261. SECONDARY_EXEC_SHADOW_VMCS);
  6262. vmcs_write64(VMCS_LINK_POINTER, -1ull);
  6263. }
  6264. vmx->nested.posted_intr_nv = -1;
  6265. /* Flush VMCS12 to guest memory */
  6266. memcpy(vmx->nested.current_vmcs12, vmx->nested.cached_vmcs12,
  6267. VMCS12_SIZE);
  6268. kunmap(vmx->nested.current_vmcs12_page);
  6269. nested_release_page(vmx->nested.current_vmcs12_page);
  6270. vmx->nested.current_vmptr = -1ull;
  6271. vmx->nested.current_vmcs12 = NULL;
  6272. }
  6273. /*
  6274. * Free whatever needs to be freed from vmx->nested when L1 goes down, or
  6275. * just stops using VMX.
  6276. */
  6277. static void free_nested(struct vcpu_vmx *vmx)
  6278. {
  6279. if (!vmx->nested.vmxon)
  6280. return;
  6281. vmx->nested.vmxon = false;
  6282. free_vpid(vmx->nested.vpid02);
  6283. nested_release_vmcs12(vmx);
  6284. if (vmx->nested.msr_bitmap) {
  6285. free_page((unsigned long)vmx->nested.msr_bitmap);
  6286. vmx->nested.msr_bitmap = NULL;
  6287. }
  6288. if (enable_shadow_vmcs) {
  6289. vmcs_clear(vmx->vmcs01.shadow_vmcs);
  6290. free_vmcs(vmx->vmcs01.shadow_vmcs);
  6291. vmx->vmcs01.shadow_vmcs = NULL;
  6292. }
  6293. kfree(vmx->nested.cached_vmcs12);
  6294. /* Unpin physical memory we referred to in current vmcs02 */
  6295. if (vmx->nested.apic_access_page) {
  6296. nested_release_page(vmx->nested.apic_access_page);
  6297. vmx->nested.apic_access_page = NULL;
  6298. }
  6299. if (vmx->nested.virtual_apic_page) {
  6300. nested_release_page(vmx->nested.virtual_apic_page);
  6301. vmx->nested.virtual_apic_page = NULL;
  6302. }
  6303. if (vmx->nested.pi_desc_page) {
  6304. kunmap(vmx->nested.pi_desc_page);
  6305. nested_release_page(vmx->nested.pi_desc_page);
  6306. vmx->nested.pi_desc_page = NULL;
  6307. vmx->nested.pi_desc = NULL;
  6308. }
  6309. nested_free_all_saved_vmcss(vmx);
  6310. }
  6311. /* Emulate the VMXOFF instruction */
  6312. static int handle_vmoff(struct kvm_vcpu *vcpu)
  6313. {
  6314. if (!nested_vmx_check_permission(vcpu))
  6315. return 1;
  6316. free_nested(to_vmx(vcpu));
  6317. nested_vmx_succeed(vcpu);
  6318. return kvm_skip_emulated_instruction(vcpu);
  6319. }
  6320. /* Emulate the VMCLEAR instruction */
  6321. static int handle_vmclear(struct kvm_vcpu *vcpu)
  6322. {
  6323. struct vcpu_vmx *vmx = to_vmx(vcpu);
  6324. gpa_t vmptr;
  6325. struct vmcs12 *vmcs12;
  6326. struct page *page;
  6327. if (!nested_vmx_check_permission(vcpu))
  6328. return 1;
  6329. if (nested_vmx_check_vmptr(vcpu, EXIT_REASON_VMCLEAR, &vmptr))
  6330. return 1;
  6331. if (vmptr == vmx->nested.current_vmptr)
  6332. nested_release_vmcs12(vmx);
  6333. page = nested_get_page(vcpu, vmptr);
  6334. if (page == NULL) {
  6335. /*
  6336. * For accurate processor emulation, VMCLEAR beyond available
  6337. * physical memory should do nothing at all. However, it is
  6338. * possible that a nested vmx bug, not a guest hypervisor bug,
  6339. * resulted in this case, so let's shut down before doing any
  6340. * more damage:
  6341. */
  6342. kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
  6343. return 1;
  6344. }
  6345. vmcs12 = kmap(page);
  6346. vmcs12->launch_state = 0;
  6347. kunmap(page);
  6348. nested_release_page(page);
  6349. nested_free_vmcs02(vmx, vmptr);
  6350. nested_vmx_succeed(vcpu);
  6351. return kvm_skip_emulated_instruction(vcpu);
  6352. }
  6353. static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch);
  6354. /* Emulate the VMLAUNCH instruction */
  6355. static int handle_vmlaunch(struct kvm_vcpu *vcpu)
  6356. {
  6357. return nested_vmx_run(vcpu, true);
  6358. }
  6359. /* Emulate the VMRESUME instruction */
  6360. static int handle_vmresume(struct kvm_vcpu *vcpu)
  6361. {
  6362. return nested_vmx_run(vcpu, false);
  6363. }
  6364. enum vmcs_field_type {
  6365. VMCS_FIELD_TYPE_U16 = 0,
  6366. VMCS_FIELD_TYPE_U64 = 1,
  6367. VMCS_FIELD_TYPE_U32 = 2,
  6368. VMCS_FIELD_TYPE_NATURAL_WIDTH = 3
  6369. };
  6370. static inline int vmcs_field_type(unsigned long field)
  6371. {
  6372. if (0x1 & field) /* the *_HIGH fields are all 32 bit */
  6373. return VMCS_FIELD_TYPE_U32;
  6374. return (field >> 13) & 0x3 ;
  6375. }
  6376. static inline int vmcs_field_readonly(unsigned long field)
  6377. {
  6378. return (((field >> 10) & 0x3) == 1);
  6379. }
  6380. /*
  6381. * Read a vmcs12 field. Since these can have varying lengths and we return
  6382. * one type, we chose the biggest type (u64) and zero-extend the return value
  6383. * to that size. Note that the caller, handle_vmread, might need to use only
  6384. * some of the bits we return here (e.g., on 32-bit guests, only 32 bits of
  6385. * 64-bit fields are to be returned).
  6386. */
  6387. static inline int vmcs12_read_any(struct kvm_vcpu *vcpu,
  6388. unsigned long field, u64 *ret)
  6389. {
  6390. short offset = vmcs_field_to_offset(field);
  6391. char *p;
  6392. if (offset < 0)
  6393. return offset;
  6394. p = ((char *)(get_vmcs12(vcpu))) + offset;
  6395. switch (vmcs_field_type(field)) {
  6396. case VMCS_FIELD_TYPE_NATURAL_WIDTH:
  6397. *ret = *((natural_width *)p);
  6398. return 0;
  6399. case VMCS_FIELD_TYPE_U16:
  6400. *ret = *((u16 *)p);
  6401. return 0;
  6402. case VMCS_FIELD_TYPE_U32:
  6403. *ret = *((u32 *)p);
  6404. return 0;
  6405. case VMCS_FIELD_TYPE_U64:
  6406. *ret = *((u64 *)p);
  6407. return 0;
  6408. default:
  6409. WARN_ON(1);
  6410. return -ENOENT;
  6411. }
  6412. }
  6413. static inline int vmcs12_write_any(struct kvm_vcpu *vcpu,
  6414. unsigned long field, u64 field_value){
  6415. short offset = vmcs_field_to_offset(field);
  6416. char *p = ((char *) get_vmcs12(vcpu)) + offset;
  6417. if (offset < 0)
  6418. return offset;
  6419. switch (vmcs_field_type(field)) {
  6420. case VMCS_FIELD_TYPE_U16:
  6421. *(u16 *)p = field_value;
  6422. return 0;
  6423. case VMCS_FIELD_TYPE_U32:
  6424. *(u32 *)p = field_value;
  6425. return 0;
  6426. case VMCS_FIELD_TYPE_U64:
  6427. *(u64 *)p = field_value;
  6428. return 0;
  6429. case VMCS_FIELD_TYPE_NATURAL_WIDTH:
  6430. *(natural_width *)p = field_value;
  6431. return 0;
  6432. default:
  6433. WARN_ON(1);
  6434. return -ENOENT;
  6435. }
  6436. }
  6437. static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx)
  6438. {
  6439. int i;
  6440. unsigned long field;
  6441. u64 field_value;
  6442. struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
  6443. const unsigned long *fields = shadow_read_write_fields;
  6444. const int num_fields = max_shadow_read_write_fields;
  6445. preempt_disable();
  6446. vmcs_load(shadow_vmcs);
  6447. for (i = 0; i < num_fields; i++) {
  6448. field = fields[i];
  6449. switch (vmcs_field_type(field)) {
  6450. case VMCS_FIELD_TYPE_U16:
  6451. field_value = vmcs_read16(field);
  6452. break;
  6453. case VMCS_FIELD_TYPE_U32:
  6454. field_value = vmcs_read32(field);
  6455. break;
  6456. case VMCS_FIELD_TYPE_U64:
  6457. field_value = vmcs_read64(field);
  6458. break;
  6459. case VMCS_FIELD_TYPE_NATURAL_WIDTH:
  6460. field_value = vmcs_readl(field);
  6461. break;
  6462. default:
  6463. WARN_ON(1);
  6464. continue;
  6465. }
  6466. vmcs12_write_any(&vmx->vcpu, field, field_value);
  6467. }
  6468. vmcs_clear(shadow_vmcs);
  6469. vmcs_load(vmx->loaded_vmcs->vmcs);
  6470. preempt_enable();
  6471. }
  6472. static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx)
  6473. {
  6474. const unsigned long *fields[] = {
  6475. shadow_read_write_fields,
  6476. shadow_read_only_fields
  6477. };
  6478. const int max_fields[] = {
  6479. max_shadow_read_write_fields,
  6480. max_shadow_read_only_fields
  6481. };
  6482. int i, q;
  6483. unsigned long field;
  6484. u64 field_value = 0;
  6485. struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
  6486. vmcs_load(shadow_vmcs);
  6487. for (q = 0; q < ARRAY_SIZE(fields); q++) {
  6488. for (i = 0; i < max_fields[q]; i++) {
  6489. field = fields[q][i];
  6490. vmcs12_read_any(&vmx->vcpu, field, &field_value);
  6491. switch (vmcs_field_type(field)) {
  6492. case VMCS_FIELD_TYPE_U16:
  6493. vmcs_write16(field, (u16)field_value);
  6494. break;
  6495. case VMCS_FIELD_TYPE_U32:
  6496. vmcs_write32(field, (u32)field_value);
  6497. break;
  6498. case VMCS_FIELD_TYPE_U64:
  6499. vmcs_write64(field, (u64)field_value);
  6500. break;
  6501. case VMCS_FIELD_TYPE_NATURAL_WIDTH:
  6502. vmcs_writel(field, (long)field_value);
  6503. break;
  6504. default:
  6505. WARN_ON(1);
  6506. break;
  6507. }
  6508. }
  6509. }
  6510. vmcs_clear(shadow_vmcs);
  6511. vmcs_load(vmx->loaded_vmcs->vmcs);
  6512. }
  6513. /*
  6514. * VMX instructions which assume a current vmcs12 (i.e., that VMPTRLD was
  6515. * used before) all generate the same failure when it is missing.
  6516. */
  6517. static int nested_vmx_check_vmcs12(struct kvm_vcpu *vcpu)
  6518. {
  6519. struct vcpu_vmx *vmx = to_vmx(vcpu);
  6520. if (vmx->nested.current_vmptr == -1ull) {
  6521. nested_vmx_failInvalid(vcpu);
  6522. return 0;
  6523. }
  6524. return 1;
  6525. }
  6526. static int handle_vmread(struct kvm_vcpu *vcpu)
  6527. {
  6528. unsigned long field;
  6529. u64 field_value;
  6530. unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
  6531. u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
  6532. gva_t gva = 0;
  6533. if (!nested_vmx_check_permission(vcpu))
  6534. return 1;
  6535. if (!nested_vmx_check_vmcs12(vcpu))
  6536. return kvm_skip_emulated_instruction(vcpu);
  6537. /* Decode instruction info and find the field to read */
  6538. field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
  6539. /* Read the field, zero-extended to a u64 field_value */
  6540. if (vmcs12_read_any(vcpu, field, &field_value) < 0) {
  6541. nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
  6542. return kvm_skip_emulated_instruction(vcpu);
  6543. }
  6544. /*
  6545. * Now copy part of this value to register or memory, as requested.
  6546. * Note that the number of bits actually copied is 32 or 64 depending
  6547. * on the guest's mode (32 or 64 bit), not on the given field's length.
  6548. */
  6549. if (vmx_instruction_info & (1u << 10)) {
  6550. kvm_register_writel(vcpu, (((vmx_instruction_info) >> 3) & 0xf),
  6551. field_value);
  6552. } else {
  6553. if (get_vmx_mem_address(vcpu, exit_qualification,
  6554. vmx_instruction_info, true, &gva))
  6555. return 1;
  6556. /* _system ok, as nested_vmx_check_permission verified cpl=0 */
  6557. kvm_write_guest_virt_system(&vcpu->arch.emulate_ctxt, gva,
  6558. &field_value, (is_long_mode(vcpu) ? 8 : 4), NULL);
  6559. }
  6560. nested_vmx_succeed(vcpu);
  6561. return kvm_skip_emulated_instruction(vcpu);
  6562. }
  6563. static int handle_vmwrite(struct kvm_vcpu *vcpu)
  6564. {
  6565. unsigned long field;
  6566. gva_t gva;
  6567. unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
  6568. u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
  6569. /* The value to write might be 32 or 64 bits, depending on L1's long
  6570. * mode, and eventually we need to write that into a field of several
  6571. * possible lengths. The code below first zero-extends the value to 64
  6572. * bit (field_value), and then copies only the appropriate number of
  6573. * bits into the vmcs12 field.
  6574. */
  6575. u64 field_value = 0;
  6576. struct x86_exception e;
  6577. if (!nested_vmx_check_permission(vcpu))
  6578. return 1;
  6579. if (!nested_vmx_check_vmcs12(vcpu))
  6580. return kvm_skip_emulated_instruction(vcpu);
  6581. if (vmx_instruction_info & (1u << 10))
  6582. field_value = kvm_register_readl(vcpu,
  6583. (((vmx_instruction_info) >> 3) & 0xf));
  6584. else {
  6585. if (get_vmx_mem_address(vcpu, exit_qualification,
  6586. vmx_instruction_info, false, &gva))
  6587. return 1;
  6588. if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva,
  6589. &field_value, (is_64_bit_mode(vcpu) ? 8 : 4), &e)) {
  6590. kvm_inject_page_fault(vcpu, &e);
  6591. return 1;
  6592. }
  6593. }
  6594. field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
  6595. if (vmcs_field_readonly(field)) {
  6596. nested_vmx_failValid(vcpu,
  6597. VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT);
  6598. return kvm_skip_emulated_instruction(vcpu);
  6599. }
  6600. if (vmcs12_write_any(vcpu, field, field_value) < 0) {
  6601. nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
  6602. return kvm_skip_emulated_instruction(vcpu);
  6603. }
  6604. nested_vmx_succeed(vcpu);
  6605. return kvm_skip_emulated_instruction(vcpu);
  6606. }
  6607. /* Emulate the VMPTRLD instruction */
  6608. static int handle_vmptrld(struct kvm_vcpu *vcpu)
  6609. {
  6610. struct vcpu_vmx *vmx = to_vmx(vcpu);
  6611. gpa_t vmptr;
  6612. if (!nested_vmx_check_permission(vcpu))
  6613. return 1;
  6614. if (nested_vmx_check_vmptr(vcpu, EXIT_REASON_VMPTRLD, &vmptr))
  6615. return 1;
  6616. if (vmx->nested.current_vmptr != vmptr) {
  6617. struct vmcs12 *new_vmcs12;
  6618. struct page *page;
  6619. page = nested_get_page(vcpu, vmptr);
  6620. if (page == NULL) {
  6621. nested_vmx_failInvalid(vcpu);
  6622. return kvm_skip_emulated_instruction(vcpu);
  6623. }
  6624. new_vmcs12 = kmap(page);
  6625. if (new_vmcs12->revision_id != VMCS12_REVISION) {
  6626. kunmap(page);
  6627. nested_release_page_clean(page);
  6628. nested_vmx_failValid(vcpu,
  6629. VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
  6630. return kvm_skip_emulated_instruction(vcpu);
  6631. }
  6632. nested_release_vmcs12(vmx);
  6633. vmx->nested.current_vmptr = vmptr;
  6634. vmx->nested.current_vmcs12 = new_vmcs12;
  6635. vmx->nested.current_vmcs12_page = page;
  6636. /*
  6637. * Load VMCS12 from guest memory since it is not already
  6638. * cached.
  6639. */
  6640. memcpy(vmx->nested.cached_vmcs12,
  6641. vmx->nested.current_vmcs12, VMCS12_SIZE);
  6642. if (enable_shadow_vmcs) {
  6643. vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,
  6644. SECONDARY_EXEC_SHADOW_VMCS);
  6645. vmcs_write64(VMCS_LINK_POINTER,
  6646. __pa(vmx->vmcs01.shadow_vmcs));
  6647. vmx->nested.sync_shadow_vmcs = true;
  6648. }
  6649. }
  6650. nested_vmx_succeed(vcpu);
  6651. return kvm_skip_emulated_instruction(vcpu);
  6652. }
  6653. /* Emulate the VMPTRST instruction */
  6654. static int handle_vmptrst(struct kvm_vcpu *vcpu)
  6655. {
  6656. unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
  6657. u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
  6658. gva_t vmcs_gva;
  6659. struct x86_exception e;
  6660. if (!nested_vmx_check_permission(vcpu))
  6661. return 1;
  6662. if (get_vmx_mem_address(vcpu, exit_qualification,
  6663. vmx_instruction_info, true, &vmcs_gva))
  6664. return 1;
  6665. /* ok to use *_system, as nested_vmx_check_permission verified cpl=0 */
  6666. if (kvm_write_guest_virt_system(&vcpu->arch.emulate_ctxt, vmcs_gva,
  6667. (void *)&to_vmx(vcpu)->nested.current_vmptr,
  6668. sizeof(u64), &e)) {
  6669. kvm_inject_page_fault(vcpu, &e);
  6670. return 1;
  6671. }
  6672. nested_vmx_succeed(vcpu);
  6673. return kvm_skip_emulated_instruction(vcpu);
  6674. }
  6675. /* Emulate the INVEPT instruction */
  6676. static int handle_invept(struct kvm_vcpu *vcpu)
  6677. {
  6678. struct vcpu_vmx *vmx = to_vmx(vcpu);
  6679. u32 vmx_instruction_info, types;
  6680. unsigned long type;
  6681. gva_t gva;
  6682. struct x86_exception e;
  6683. struct {
  6684. u64 eptp, gpa;
  6685. } operand;
  6686. if (!(vmx->nested.nested_vmx_secondary_ctls_high &
  6687. SECONDARY_EXEC_ENABLE_EPT) ||
  6688. !(vmx->nested.nested_vmx_ept_caps & VMX_EPT_INVEPT_BIT)) {
  6689. kvm_queue_exception(vcpu, UD_VECTOR);
  6690. return 1;
  6691. }
  6692. if (!nested_vmx_check_permission(vcpu))
  6693. return 1;
  6694. if (!kvm_read_cr0_bits(vcpu, X86_CR0_PE)) {
  6695. kvm_queue_exception(vcpu, UD_VECTOR);
  6696. return 1;
  6697. }
  6698. vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
  6699. type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
  6700. types = (vmx->nested.nested_vmx_ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6;
  6701. if (type >= 32 || !(types & (1 << type))) {
  6702. nested_vmx_failValid(vcpu,
  6703. VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
  6704. return kvm_skip_emulated_instruction(vcpu);
  6705. }
  6706. /* According to the Intel VMX instruction reference, the memory
  6707. * operand is read even if it isn't needed (e.g., for type==global)
  6708. */
  6709. if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
  6710. vmx_instruction_info, false, &gva))
  6711. return 1;
  6712. if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva, &operand,
  6713. sizeof(operand), &e)) {
  6714. kvm_inject_page_fault(vcpu, &e);
  6715. return 1;
  6716. }
  6717. switch (type) {
  6718. case VMX_EPT_EXTENT_GLOBAL:
  6719. /*
  6720. * TODO: track mappings and invalidate
  6721. * single context requests appropriately
  6722. */
  6723. case VMX_EPT_EXTENT_CONTEXT:
  6724. kvm_mmu_sync_roots(vcpu);
  6725. kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
  6726. nested_vmx_succeed(vcpu);
  6727. break;
  6728. default:
  6729. BUG_ON(1);
  6730. break;
  6731. }
  6732. return kvm_skip_emulated_instruction(vcpu);
  6733. }
  6734. static int handle_invvpid(struct kvm_vcpu *vcpu)
  6735. {
  6736. struct vcpu_vmx *vmx = to_vmx(vcpu);
  6737. u32 vmx_instruction_info;
  6738. unsigned long type, types;
  6739. gva_t gva;
  6740. struct x86_exception e;
  6741. int vpid;
  6742. if (!(vmx->nested.nested_vmx_secondary_ctls_high &
  6743. SECONDARY_EXEC_ENABLE_VPID) ||
  6744. !(vmx->nested.nested_vmx_vpid_caps & VMX_VPID_INVVPID_BIT)) {
  6745. kvm_queue_exception(vcpu, UD_VECTOR);
  6746. return 1;
  6747. }
  6748. if (!nested_vmx_check_permission(vcpu))
  6749. return 1;
  6750. vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
  6751. type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
  6752. types = (vmx->nested.nested_vmx_vpid_caps &
  6753. VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8;
  6754. if (type >= 32 || !(types & (1 << type))) {
  6755. nested_vmx_failValid(vcpu,
  6756. VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
  6757. return kvm_skip_emulated_instruction(vcpu);
  6758. }
  6759. /* according to the intel vmx instruction reference, the memory
  6760. * operand is read even if it isn't needed (e.g., for type==global)
  6761. */
  6762. if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
  6763. vmx_instruction_info, false, &gva))
  6764. return 1;
  6765. if (kvm_read_guest_virt(&vcpu->arch.emulate_ctxt, gva, &vpid,
  6766. sizeof(u32), &e)) {
  6767. kvm_inject_page_fault(vcpu, &e);
  6768. return 1;
  6769. }
  6770. switch (type) {
  6771. case VMX_VPID_EXTENT_INDIVIDUAL_ADDR:
  6772. case VMX_VPID_EXTENT_SINGLE_CONTEXT:
  6773. case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL:
  6774. if (!vpid) {
  6775. nested_vmx_failValid(vcpu,
  6776. VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
  6777. return kvm_skip_emulated_instruction(vcpu);
  6778. }
  6779. break;
  6780. case VMX_VPID_EXTENT_ALL_CONTEXT:
  6781. break;
  6782. default:
  6783. WARN_ON_ONCE(1);
  6784. return kvm_skip_emulated_instruction(vcpu);
  6785. }
  6786. __vmx_flush_tlb(vcpu, vmx->nested.vpid02);
  6787. nested_vmx_succeed(vcpu);
  6788. return kvm_skip_emulated_instruction(vcpu);
  6789. }
  6790. static int handle_pml_full(struct kvm_vcpu *vcpu)
  6791. {
  6792. unsigned long exit_qualification;
  6793. trace_kvm_pml_full(vcpu->vcpu_id);
  6794. exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
  6795. /*
  6796. * PML buffer FULL happened while executing iret from NMI,
  6797. * "blocked by NMI" bit has to be set before next VM entry.
  6798. */
  6799. if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
  6800. cpu_has_virtual_nmis() &&
  6801. (exit_qualification & INTR_INFO_UNBLOCK_NMI))
  6802. vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
  6803. GUEST_INTR_STATE_NMI);
  6804. /*
  6805. * PML buffer already flushed at beginning of VMEXIT. Nothing to do
  6806. * here.., and there's no userspace involvement needed for PML.
  6807. */
  6808. return 1;
  6809. }
  6810. static int handle_preemption_timer(struct kvm_vcpu *vcpu)
  6811. {
  6812. kvm_lapic_expired_hv_timer(vcpu);
  6813. return 1;
  6814. }
  6815. /*
  6816. * The exit handlers return 1 if the exit was handled fully and guest execution
  6817. * may resume. Otherwise they set the kvm_run parameter to indicate what needs
  6818. * to be done to userspace and return 0.
  6819. */
  6820. static int (*const kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
  6821. [EXIT_REASON_EXCEPTION_NMI] = handle_exception,
  6822. [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt,
  6823. [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault,
  6824. [EXIT_REASON_NMI_WINDOW] = handle_nmi_window,
  6825. [EXIT_REASON_IO_INSTRUCTION] = handle_io,
  6826. [EXIT_REASON_CR_ACCESS] = handle_cr,
  6827. [EXIT_REASON_DR_ACCESS] = handle_dr,
  6828. [EXIT_REASON_CPUID] = handle_cpuid,
  6829. [EXIT_REASON_MSR_READ] = handle_rdmsr,
  6830. [EXIT_REASON_MSR_WRITE] = handle_wrmsr,
  6831. [EXIT_REASON_PENDING_INTERRUPT] = handle_interrupt_window,
  6832. [EXIT_REASON_HLT] = handle_halt,
  6833. [EXIT_REASON_INVD] = handle_invd,
  6834. [EXIT_REASON_INVLPG] = handle_invlpg,
  6835. [EXIT_REASON_RDPMC] = handle_rdpmc,
  6836. [EXIT_REASON_VMCALL] = handle_vmcall,
  6837. [EXIT_REASON_VMCLEAR] = handle_vmclear,
  6838. [EXIT_REASON_VMLAUNCH] = handle_vmlaunch,
  6839. [EXIT_REASON_VMPTRLD] = handle_vmptrld,
  6840. [EXIT_REASON_VMPTRST] = handle_vmptrst,
  6841. [EXIT_REASON_VMREAD] = handle_vmread,
  6842. [EXIT_REASON_VMRESUME] = handle_vmresume,
  6843. [EXIT_REASON_VMWRITE] = handle_vmwrite,
  6844. [EXIT_REASON_VMOFF] = handle_vmoff,
  6845. [EXIT_REASON_VMON] = handle_vmon,
  6846. [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold,
  6847. [EXIT_REASON_APIC_ACCESS] = handle_apic_access,
  6848. [EXIT_REASON_APIC_WRITE] = handle_apic_write,
  6849. [EXIT_REASON_EOI_INDUCED] = handle_apic_eoi_induced,
  6850. [EXIT_REASON_WBINVD] = handle_wbinvd,
  6851. [EXIT_REASON_XSETBV] = handle_xsetbv,
  6852. [EXIT_REASON_TASK_SWITCH] = handle_task_switch,
  6853. [EXIT_REASON_MCE_DURING_VMENTRY] = handle_machine_check,
  6854. [EXIT_REASON_EPT_VIOLATION] = handle_ept_violation,
  6855. [EXIT_REASON_EPT_MISCONFIG] = handle_ept_misconfig,
  6856. [EXIT_REASON_PAUSE_INSTRUCTION] = handle_pause,
  6857. [EXIT_REASON_MWAIT_INSTRUCTION] = handle_mwait,
  6858. [EXIT_REASON_MONITOR_TRAP_FLAG] = handle_monitor_trap,
  6859. [EXIT_REASON_MONITOR_INSTRUCTION] = handle_monitor,
  6860. [EXIT_REASON_INVEPT] = handle_invept,
  6861. [EXIT_REASON_INVVPID] = handle_invvpid,
  6862. [EXIT_REASON_XSAVES] = handle_xsaves,
  6863. [EXIT_REASON_XRSTORS] = handle_xrstors,
  6864. [EXIT_REASON_PML_FULL] = handle_pml_full,
  6865. [EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer,
  6866. };
  6867. static const int kvm_vmx_max_exit_handlers =
  6868. ARRAY_SIZE(kvm_vmx_exit_handlers);
  6869. static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu,
  6870. struct vmcs12 *vmcs12)
  6871. {
  6872. unsigned long exit_qualification;
  6873. gpa_t bitmap, last_bitmap;
  6874. unsigned int port;
  6875. int size;
  6876. u8 b;
  6877. if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
  6878. return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING);
  6879. exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
  6880. port = exit_qualification >> 16;
  6881. size = (exit_qualification & 7) + 1;
  6882. last_bitmap = (gpa_t)-1;
  6883. b = -1;
  6884. while (size > 0) {
  6885. if (port < 0x8000)
  6886. bitmap = vmcs12->io_bitmap_a;
  6887. else if (port < 0x10000)
  6888. bitmap = vmcs12->io_bitmap_b;
  6889. else
  6890. return true;
  6891. bitmap += (port & 0x7fff) / 8;
  6892. if (last_bitmap != bitmap)
  6893. if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1))
  6894. return true;
  6895. if (b & (1 << (port & 7)))
  6896. return true;
  6897. port++;
  6898. size--;
  6899. last_bitmap = bitmap;
  6900. }
  6901. return false;
  6902. }
  6903. /*
  6904. * Return 1 if we should exit from L2 to L1 to handle an MSR access access,
  6905. * rather than handle it ourselves in L0. I.e., check whether L1 expressed
  6906. * disinterest in the current event (read or write a specific MSR) by using an
  6907. * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps.
  6908. */
  6909. static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
  6910. struct vmcs12 *vmcs12, u32 exit_reason)
  6911. {
  6912. u32 msr_index = vcpu->arch.regs[VCPU_REGS_RCX];
  6913. gpa_t bitmap;
  6914. if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
  6915. return true;
  6916. /*
  6917. * The MSR_BITMAP page is divided into four 1024-byte bitmaps,
  6918. * for the four combinations of read/write and low/high MSR numbers.
  6919. * First we need to figure out which of the four to use:
  6920. */
  6921. bitmap = vmcs12->msr_bitmap;
  6922. if (exit_reason == EXIT_REASON_MSR_WRITE)
  6923. bitmap += 2048;
  6924. if (msr_index >= 0xc0000000) {
  6925. msr_index -= 0xc0000000;
  6926. bitmap += 1024;
  6927. }
  6928. /* Then read the msr_index'th bit from this bitmap: */
  6929. if (msr_index < 1024*8) {
  6930. unsigned char b;
  6931. if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1))
  6932. return true;
  6933. return 1 & (b >> (msr_index & 7));
  6934. } else
  6935. return true; /* let L1 handle the wrong parameter */
  6936. }
  6937. /*
  6938. * Return 1 if we should exit from L2 to L1 to handle a CR access exit,
  6939. * rather than handle it ourselves in L0. I.e., check if L1 wanted to
  6940. * intercept (via guest_host_mask etc.) the current event.
  6941. */
  6942. static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
  6943. struct vmcs12 *vmcs12)
  6944. {
  6945. unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
  6946. int cr = exit_qualification & 15;
  6947. int reg = (exit_qualification >> 8) & 15;
  6948. unsigned long val = kvm_register_readl(vcpu, reg);
  6949. switch ((exit_qualification >> 4) & 3) {
  6950. case 0: /* mov to cr */
  6951. switch (cr) {
  6952. case 0:
  6953. if (vmcs12->cr0_guest_host_mask &
  6954. (val ^ vmcs12->cr0_read_shadow))
  6955. return true;
  6956. break;
  6957. case 3:
  6958. if ((vmcs12->cr3_target_count >= 1 &&
  6959. vmcs12->cr3_target_value0 == val) ||
  6960. (vmcs12->cr3_target_count >= 2 &&
  6961. vmcs12->cr3_target_value1 == val) ||
  6962. (vmcs12->cr3_target_count >= 3 &&
  6963. vmcs12->cr3_target_value2 == val) ||
  6964. (vmcs12->cr3_target_count >= 4 &&
  6965. vmcs12->cr3_target_value3 == val))
  6966. return false;
  6967. if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING))
  6968. return true;
  6969. break;
  6970. case 4:
  6971. if (vmcs12->cr4_guest_host_mask &
  6972. (vmcs12->cr4_read_shadow ^ val))
  6973. return true;
  6974. break;
  6975. case 8:
  6976. if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING))
  6977. return true;
  6978. break;
  6979. }
  6980. break;
  6981. case 2: /* clts */
  6982. if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) &&
  6983. (vmcs12->cr0_read_shadow & X86_CR0_TS))
  6984. return true;
  6985. break;
  6986. case 1: /* mov from cr */
  6987. switch (cr) {
  6988. case 3:
  6989. if (vmcs12->cpu_based_vm_exec_control &
  6990. CPU_BASED_CR3_STORE_EXITING)
  6991. return true;
  6992. break;
  6993. case 8:
  6994. if (vmcs12->cpu_based_vm_exec_control &
  6995. CPU_BASED_CR8_STORE_EXITING)
  6996. return true;
  6997. break;
  6998. }
  6999. break;
  7000. case 3: /* lmsw */
  7001. /*
  7002. * lmsw can change bits 1..3 of cr0, and only set bit 0 of
  7003. * cr0. Other attempted changes are ignored, with no exit.
  7004. */
  7005. if (vmcs12->cr0_guest_host_mask & 0xe &
  7006. (val ^ vmcs12->cr0_read_shadow))
  7007. return true;
  7008. if ((vmcs12->cr0_guest_host_mask & 0x1) &&
  7009. !(vmcs12->cr0_read_shadow & 0x1) &&
  7010. (val & 0x1))
  7011. return true;
  7012. break;
  7013. }
  7014. return false;
  7015. }
  7016. /*
  7017. * Return 1 if we should exit from L2 to L1 to handle an exit, or 0 if we
  7018. * should handle it ourselves in L0 (and then continue L2). Only call this
  7019. * when in is_guest_mode (L2).
  7020. */
  7021. static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
  7022. {
  7023. u32 intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
  7024. struct vcpu_vmx *vmx = to_vmx(vcpu);
  7025. struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
  7026. u32 exit_reason = vmx->exit_reason;
  7027. trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason,
  7028. vmcs_readl(EXIT_QUALIFICATION),
  7029. vmx->idt_vectoring_info,
  7030. intr_info,
  7031. vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
  7032. KVM_ISA_VMX);
  7033. if (vmx->nested.nested_run_pending)
  7034. return false;
  7035. if (unlikely(vmx->fail)) {
  7036. pr_info_ratelimited("%s failed vm entry %x\n", __func__,
  7037. vmcs_read32(VM_INSTRUCTION_ERROR));
  7038. return true;
  7039. }
  7040. switch (exit_reason) {
  7041. case EXIT_REASON_EXCEPTION_NMI:
  7042. if (is_nmi(intr_info))
  7043. return false;
  7044. else if (is_page_fault(intr_info))
  7045. return enable_ept;
  7046. else if (is_no_device(intr_info) &&
  7047. !(vmcs12->guest_cr0 & X86_CR0_TS))
  7048. return false;
  7049. else if (is_debug(intr_info) &&
  7050. vcpu->guest_debug &
  7051. (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
  7052. return false;
  7053. else if (is_breakpoint(intr_info) &&
  7054. vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
  7055. return false;
  7056. return vmcs12->exception_bitmap &
  7057. (1u << (intr_info & INTR_INFO_VECTOR_MASK));
  7058. case EXIT_REASON_EXTERNAL_INTERRUPT:
  7059. return false;
  7060. case EXIT_REASON_TRIPLE_FAULT:
  7061. return true;
  7062. case EXIT_REASON_PENDING_INTERRUPT:
  7063. return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_INTR_PENDING);
  7064. case EXIT_REASON_NMI_WINDOW:
  7065. return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING);
  7066. case EXIT_REASON_TASK_SWITCH:
  7067. return true;
  7068. case EXIT_REASON_CPUID:
  7069. if (kvm_register_read(vcpu, VCPU_REGS_RAX) == 0xa)
  7070. return false;
  7071. return true;
  7072. case EXIT_REASON_HLT:
  7073. return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING);
  7074. case EXIT_REASON_INVD:
  7075. return true;
  7076. case EXIT_REASON_INVLPG:
  7077. return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
  7078. case EXIT_REASON_RDPMC:
  7079. return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING);
  7080. case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP:
  7081. return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING);
  7082. case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR:
  7083. case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD:
  7084. case EXIT_REASON_VMPTRST: case EXIT_REASON_VMREAD:
  7085. case EXIT_REASON_VMRESUME: case EXIT_REASON_VMWRITE:
  7086. case EXIT_REASON_VMOFF: case EXIT_REASON_VMON:
  7087. case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID:
  7088. /*
  7089. * VMX instructions trap unconditionally. This allows L1 to
  7090. * emulate them for its L2 guest, i.e., allows 3-level nesting!
  7091. */
  7092. return true;
  7093. case EXIT_REASON_CR_ACCESS:
  7094. return nested_vmx_exit_handled_cr(vcpu, vmcs12);
  7095. case EXIT_REASON_DR_ACCESS:
  7096. return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING);
  7097. case EXIT_REASON_IO_INSTRUCTION:
  7098. return nested_vmx_exit_handled_io(vcpu, vmcs12);
  7099. case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR:
  7100. return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC);
  7101. case EXIT_REASON_MSR_READ:
  7102. case EXIT_REASON_MSR_WRITE:
  7103. return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason);
  7104. case EXIT_REASON_INVALID_STATE:
  7105. return true;
  7106. case EXIT_REASON_MWAIT_INSTRUCTION:
  7107. return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING);
  7108. case EXIT_REASON_MONITOR_TRAP_FLAG:
  7109. return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_TRAP_FLAG);
  7110. case EXIT_REASON_MONITOR_INSTRUCTION:
  7111. return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING);
  7112. case EXIT_REASON_PAUSE_INSTRUCTION:
  7113. return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) ||
  7114. nested_cpu_has2(vmcs12,
  7115. SECONDARY_EXEC_PAUSE_LOOP_EXITING);
  7116. case EXIT_REASON_MCE_DURING_VMENTRY:
  7117. return false;
  7118. case EXIT_REASON_TPR_BELOW_THRESHOLD:
  7119. return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW);
  7120. case EXIT_REASON_APIC_ACCESS:
  7121. return nested_cpu_has2(vmcs12,
  7122. SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
  7123. case EXIT_REASON_APIC_WRITE:
  7124. case EXIT_REASON_EOI_INDUCED:
  7125. /* apic_write and eoi_induced should exit unconditionally. */
  7126. return true;
  7127. case EXIT_REASON_EPT_VIOLATION:
  7128. /*
  7129. * L0 always deals with the EPT violation. If nested EPT is
  7130. * used, and the nested mmu code discovers that the address is
  7131. * missing in the guest EPT table (EPT12), the EPT violation
  7132. * will be injected with nested_ept_inject_page_fault()
  7133. */
  7134. return false;
  7135. case EXIT_REASON_EPT_MISCONFIG:
  7136. /*
  7137. * L2 never uses directly L1's EPT, but rather L0's own EPT
  7138. * table (shadow on EPT) or a merged EPT table that L0 built
  7139. * (EPT on EPT). So any problems with the structure of the
  7140. * table is L0's fault.
  7141. */
  7142. return false;
  7143. case EXIT_REASON_WBINVD:
  7144. return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING);
  7145. case EXIT_REASON_XSETBV:
  7146. return true;
  7147. case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS:
  7148. /*
  7149. * This should never happen, since it is not possible to
  7150. * set XSS to a non-zero value---neither in L1 nor in L2.
  7151. * If if it were, XSS would have to be checked against
  7152. * the XSS exit bitmap in vmcs12.
  7153. */
  7154. return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES);
  7155. case EXIT_REASON_PREEMPTION_TIMER:
  7156. return false;
  7157. default:
  7158. return true;
  7159. }
  7160. }
  7161. static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
  7162. {
  7163. *info1 = vmcs_readl(EXIT_QUALIFICATION);
  7164. *info2 = vmcs_read32(VM_EXIT_INTR_INFO);
  7165. }
  7166. static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx)
  7167. {
  7168. if (vmx->pml_pg) {
  7169. __free_page(vmx->pml_pg);
  7170. vmx->pml_pg = NULL;
  7171. }
  7172. }
  7173. static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu)
  7174. {
  7175. struct vcpu_vmx *vmx = to_vmx(vcpu);
  7176. u64 *pml_buf;
  7177. u16 pml_idx;
  7178. pml_idx = vmcs_read16(GUEST_PML_INDEX);
  7179. /* Do nothing if PML buffer is empty */
  7180. if (pml_idx == (PML_ENTITY_NUM - 1))
  7181. return;
  7182. /* PML index always points to next available PML buffer entity */
  7183. if (pml_idx >= PML_ENTITY_NUM)
  7184. pml_idx = 0;
  7185. else
  7186. pml_idx++;
  7187. pml_buf = page_address(vmx->pml_pg);
  7188. for (; pml_idx < PML_ENTITY_NUM; pml_idx++) {
  7189. u64 gpa;
  7190. gpa = pml_buf[pml_idx];
  7191. WARN_ON(gpa & (PAGE_SIZE - 1));
  7192. kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT);
  7193. }
  7194. /* reset PML index */
  7195. vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
  7196. }
  7197. /*
  7198. * Flush all vcpus' PML buffer and update logged GPAs to dirty_bitmap.
  7199. * Called before reporting dirty_bitmap to userspace.
  7200. */
  7201. static void kvm_flush_pml_buffers(struct kvm *kvm)
  7202. {
  7203. int i;
  7204. struct kvm_vcpu *vcpu;
  7205. /*
  7206. * We only need to kick vcpu out of guest mode here, as PML buffer
  7207. * is flushed at beginning of all VMEXITs, and it's obvious that only
  7208. * vcpus running in guest are possible to have unflushed GPAs in PML
  7209. * buffer.
  7210. */
  7211. kvm_for_each_vcpu(i, vcpu, kvm)
  7212. kvm_vcpu_kick(vcpu);
  7213. }
  7214. static void vmx_dump_sel(char *name, uint32_t sel)
  7215. {
  7216. pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n",
  7217. name, vmcs_read32(sel),
  7218. vmcs_read32(sel + GUEST_ES_AR_BYTES - GUEST_ES_SELECTOR),
  7219. vmcs_read32(sel + GUEST_ES_LIMIT - GUEST_ES_SELECTOR),
  7220. vmcs_readl(sel + GUEST_ES_BASE - GUEST_ES_SELECTOR));
  7221. }
  7222. static void vmx_dump_dtsel(char *name, uint32_t limit)
  7223. {
  7224. pr_err("%s limit=0x%08x, base=0x%016lx\n",
  7225. name, vmcs_read32(limit),
  7226. vmcs_readl(limit + GUEST_GDTR_BASE - GUEST_GDTR_LIMIT));
  7227. }
  7228. static void dump_vmcs(void)
  7229. {
  7230. u32 vmentry_ctl = vmcs_read32(VM_ENTRY_CONTROLS);
  7231. u32 vmexit_ctl = vmcs_read32(VM_EXIT_CONTROLS);
  7232. u32 cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
  7233. u32 pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL);
  7234. u32 secondary_exec_control = 0;
  7235. unsigned long cr4 = vmcs_readl(GUEST_CR4);
  7236. u64 efer = vmcs_read64(GUEST_IA32_EFER);
  7237. int i, n;
  7238. if (cpu_has_secondary_exec_ctrls())
  7239. secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
  7240. pr_err("*** Guest State ***\n");
  7241. pr_err("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
  7242. vmcs_readl(GUEST_CR0), vmcs_readl(CR0_READ_SHADOW),
  7243. vmcs_readl(CR0_GUEST_HOST_MASK));
  7244. pr_err("CR4: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
  7245. cr4, vmcs_readl(CR4_READ_SHADOW), vmcs_readl(CR4_GUEST_HOST_MASK));
  7246. pr_err("CR3 = 0x%016lx\n", vmcs_readl(GUEST_CR3));
  7247. if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT) &&
  7248. (cr4 & X86_CR4_PAE) && !(efer & EFER_LMA))
  7249. {
  7250. pr_err("PDPTR0 = 0x%016llx PDPTR1 = 0x%016llx\n",
  7251. vmcs_read64(GUEST_PDPTR0), vmcs_read64(GUEST_PDPTR1));
  7252. pr_err("PDPTR2 = 0x%016llx PDPTR3 = 0x%016llx\n",
  7253. vmcs_read64(GUEST_PDPTR2), vmcs_read64(GUEST_PDPTR3));
  7254. }
  7255. pr_err("RSP = 0x%016lx RIP = 0x%016lx\n",
  7256. vmcs_readl(GUEST_RSP), vmcs_readl(GUEST_RIP));
  7257. pr_err("RFLAGS=0x%08lx DR7 = 0x%016lx\n",
  7258. vmcs_readl(GUEST_RFLAGS), vmcs_readl(GUEST_DR7));
  7259. pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
  7260. vmcs_readl(GUEST_SYSENTER_ESP),
  7261. vmcs_read32(GUEST_SYSENTER_CS), vmcs_readl(GUEST_SYSENTER_EIP));
  7262. vmx_dump_sel("CS: ", GUEST_CS_SELECTOR);
  7263. vmx_dump_sel("DS: ", GUEST_DS_SELECTOR);
  7264. vmx_dump_sel("SS: ", GUEST_SS_SELECTOR);
  7265. vmx_dump_sel("ES: ", GUEST_ES_SELECTOR);
  7266. vmx_dump_sel("FS: ", GUEST_FS_SELECTOR);
  7267. vmx_dump_sel("GS: ", GUEST_GS_SELECTOR);
  7268. vmx_dump_dtsel("GDTR:", GUEST_GDTR_LIMIT);
  7269. vmx_dump_sel("LDTR:", GUEST_LDTR_SELECTOR);
  7270. vmx_dump_dtsel("IDTR:", GUEST_IDTR_LIMIT);
  7271. vmx_dump_sel("TR: ", GUEST_TR_SELECTOR);
  7272. if ((vmexit_ctl & (VM_EXIT_SAVE_IA32_PAT | VM_EXIT_SAVE_IA32_EFER)) ||
  7273. (vmentry_ctl & (VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_IA32_EFER)))
  7274. pr_err("EFER = 0x%016llx PAT = 0x%016llx\n",
  7275. efer, vmcs_read64(GUEST_IA32_PAT));
  7276. pr_err("DebugCtl = 0x%016llx DebugExceptions = 0x%016lx\n",
  7277. vmcs_read64(GUEST_IA32_DEBUGCTL),
  7278. vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS));
  7279. if (vmentry_ctl & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
  7280. pr_err("PerfGlobCtl = 0x%016llx\n",
  7281. vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL));
  7282. if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS)
  7283. pr_err("BndCfgS = 0x%016llx\n", vmcs_read64(GUEST_BNDCFGS));
  7284. pr_err("Interruptibility = %08x ActivityState = %08x\n",
  7285. vmcs_read32(GUEST_INTERRUPTIBILITY_INFO),
  7286. vmcs_read32(GUEST_ACTIVITY_STATE));
  7287. if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)
  7288. pr_err("InterruptStatus = %04x\n",
  7289. vmcs_read16(GUEST_INTR_STATUS));
  7290. pr_err("*** Host State ***\n");
  7291. pr_err("RIP = 0x%016lx RSP = 0x%016lx\n",
  7292. vmcs_readl(HOST_RIP), vmcs_readl(HOST_RSP));
  7293. pr_err("CS=%04x SS=%04x DS=%04x ES=%04x FS=%04x GS=%04x TR=%04x\n",
  7294. vmcs_read16(HOST_CS_SELECTOR), vmcs_read16(HOST_SS_SELECTOR),
  7295. vmcs_read16(HOST_DS_SELECTOR), vmcs_read16(HOST_ES_SELECTOR),
  7296. vmcs_read16(HOST_FS_SELECTOR), vmcs_read16(HOST_GS_SELECTOR),
  7297. vmcs_read16(HOST_TR_SELECTOR));
  7298. pr_err("FSBase=%016lx GSBase=%016lx TRBase=%016lx\n",
  7299. vmcs_readl(HOST_FS_BASE), vmcs_readl(HOST_GS_BASE),
  7300. vmcs_readl(HOST_TR_BASE));
  7301. pr_err("GDTBase=%016lx IDTBase=%016lx\n",
  7302. vmcs_readl(HOST_GDTR_BASE), vmcs_readl(HOST_IDTR_BASE));
  7303. pr_err("CR0=%016lx CR3=%016lx CR4=%016lx\n",
  7304. vmcs_readl(HOST_CR0), vmcs_readl(HOST_CR3),
  7305. vmcs_readl(HOST_CR4));
  7306. pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
  7307. vmcs_readl(HOST_IA32_SYSENTER_ESP),
  7308. vmcs_read32(HOST_IA32_SYSENTER_CS),
  7309. vmcs_readl(HOST_IA32_SYSENTER_EIP));
  7310. if (vmexit_ctl & (VM_EXIT_LOAD_IA32_PAT | VM_EXIT_LOAD_IA32_EFER))
  7311. pr_err("EFER = 0x%016llx PAT = 0x%016llx\n",
  7312. vmcs_read64(HOST_IA32_EFER),
  7313. vmcs_read64(HOST_IA32_PAT));
  7314. if (vmexit_ctl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
  7315. pr_err("PerfGlobCtl = 0x%016llx\n",
  7316. vmcs_read64(HOST_IA32_PERF_GLOBAL_CTRL));
  7317. pr_err("*** Control State ***\n");
  7318. pr_err("PinBased=%08x CPUBased=%08x SecondaryExec=%08x\n",
  7319. pin_based_exec_ctrl, cpu_based_exec_ctrl, secondary_exec_control);
  7320. pr_err("EntryControls=%08x ExitControls=%08x\n", vmentry_ctl, vmexit_ctl);
  7321. pr_err("ExceptionBitmap=%08x PFECmask=%08x PFECmatch=%08x\n",
  7322. vmcs_read32(EXCEPTION_BITMAP),
  7323. vmcs_read32(PAGE_FAULT_ERROR_CODE_MASK),
  7324. vmcs_read32(PAGE_FAULT_ERROR_CODE_MATCH));
  7325. pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x\n",
  7326. vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
  7327. vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE),
  7328. vmcs_read32(VM_ENTRY_INSTRUCTION_LEN));
  7329. pr_err("VMExit: intr_info=%08x errcode=%08x ilen=%08x\n",
  7330. vmcs_read32(VM_EXIT_INTR_INFO),
  7331. vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
  7332. vmcs_read32(VM_EXIT_INSTRUCTION_LEN));
  7333. pr_err(" reason=%08x qualification=%016lx\n",
  7334. vmcs_read32(VM_EXIT_REASON), vmcs_readl(EXIT_QUALIFICATION));
  7335. pr_err("IDTVectoring: info=%08x errcode=%08x\n",
  7336. vmcs_read32(IDT_VECTORING_INFO_FIELD),
  7337. vmcs_read32(IDT_VECTORING_ERROR_CODE));
  7338. pr_err("TSC Offset = 0x%016llx\n", vmcs_read64(TSC_OFFSET));
  7339. if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING)
  7340. pr_err("TSC Multiplier = 0x%016llx\n",
  7341. vmcs_read64(TSC_MULTIPLIER));
  7342. if (cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW)
  7343. pr_err("TPR Threshold = 0x%02x\n", vmcs_read32(TPR_THRESHOLD));
  7344. if (pin_based_exec_ctrl & PIN_BASED_POSTED_INTR)
  7345. pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV));
  7346. if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT))
  7347. pr_err("EPT pointer = 0x%016llx\n", vmcs_read64(EPT_POINTER));
  7348. n = vmcs_read32(CR3_TARGET_COUNT);
  7349. for (i = 0; i + 1 < n; i += 4)
  7350. pr_err("CR3 target%u=%016lx target%u=%016lx\n",
  7351. i, vmcs_readl(CR3_TARGET_VALUE0 + i * 2),
  7352. i + 1, vmcs_readl(CR3_TARGET_VALUE0 + i * 2 + 2));
  7353. if (i < n)
  7354. pr_err("CR3 target%u=%016lx\n",
  7355. i, vmcs_readl(CR3_TARGET_VALUE0 + i * 2));
  7356. if (secondary_exec_control & SECONDARY_EXEC_PAUSE_LOOP_EXITING)
  7357. pr_err("PLE Gap=%08x Window=%08x\n",
  7358. vmcs_read32(PLE_GAP), vmcs_read32(PLE_WINDOW));
  7359. if (secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID)
  7360. pr_err("Virtual processor ID = 0x%04x\n",
  7361. vmcs_read16(VIRTUAL_PROCESSOR_ID));
  7362. }
  7363. /*
  7364. * The guest has exited. See if we can fix it or if we need userspace
  7365. * assistance.
  7366. */
  7367. static int vmx_handle_exit(struct kvm_vcpu *vcpu)
  7368. {
  7369. struct vcpu_vmx *vmx = to_vmx(vcpu);
  7370. u32 exit_reason = vmx->exit_reason;
  7371. u32 vectoring_info = vmx->idt_vectoring_info;
  7372. trace_kvm_exit(exit_reason, vcpu, KVM_ISA_VMX);
  7373. /*
  7374. * Flush logged GPAs PML buffer, this will make dirty_bitmap more
  7375. * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before
  7376. * querying dirty_bitmap, we only need to kick all vcpus out of guest
  7377. * mode as if vcpus is in root mode, the PML buffer must has been
  7378. * flushed already.
  7379. */
  7380. if (enable_pml)
  7381. vmx_flush_pml_buffer(vcpu);
  7382. /* If guest state is invalid, start emulating */
  7383. if (vmx->emulation_required)
  7384. return handle_invalid_guest_state(vcpu);
  7385. if (is_guest_mode(vcpu) && nested_vmx_exit_handled(vcpu)) {
  7386. nested_vmx_vmexit(vcpu, exit_reason,
  7387. vmcs_read32(VM_EXIT_INTR_INFO),
  7388. vmcs_readl(EXIT_QUALIFICATION));
  7389. return 1;
  7390. }
  7391. if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) {
  7392. dump_vmcs();
  7393. vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
  7394. vcpu->run->fail_entry.hardware_entry_failure_reason
  7395. = exit_reason;
  7396. return 0;
  7397. }
  7398. if (unlikely(vmx->fail)) {
  7399. vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
  7400. vcpu->run->fail_entry.hardware_entry_failure_reason
  7401. = vmcs_read32(VM_INSTRUCTION_ERROR);
  7402. return 0;
  7403. }
  7404. /*
  7405. * Note:
  7406. * Do not try to fix EXIT_REASON_EPT_MISCONFIG if it caused by
  7407. * delivery event since it indicates guest is accessing MMIO.
  7408. * The vm-exit can be triggered again after return to guest that
  7409. * will cause infinite loop.
  7410. */
  7411. if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
  7412. (exit_reason != EXIT_REASON_EXCEPTION_NMI &&
  7413. exit_reason != EXIT_REASON_EPT_VIOLATION &&
  7414. exit_reason != EXIT_REASON_PML_FULL &&
  7415. exit_reason != EXIT_REASON_TASK_SWITCH)) {
  7416. vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
  7417. vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV;
  7418. vcpu->run->internal.ndata = 2;
  7419. vcpu->run->internal.data[0] = vectoring_info;
  7420. vcpu->run->internal.data[1] = exit_reason;
  7421. return 0;
  7422. }
  7423. if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked &&
  7424. !(is_guest_mode(vcpu) && nested_cpu_has_virtual_nmis(
  7425. get_vmcs12(vcpu))))) {
  7426. if (vmx_interrupt_allowed(vcpu)) {
  7427. vmx->soft_vnmi_blocked = 0;
  7428. } else if (vmx->vnmi_blocked_time > 1000000000LL &&
  7429. vcpu->arch.nmi_pending) {
  7430. /*
  7431. * This CPU don't support us in finding the end of an
  7432. * NMI-blocked window if the guest runs with IRQs
  7433. * disabled. So we pull the trigger after 1 s of
  7434. * futile waiting, but inform the user about this.
  7435. */
  7436. printk(KERN_WARNING "%s: Breaking out of NMI-blocked "
  7437. "state on VCPU %d after 1 s timeout\n",
  7438. __func__, vcpu->vcpu_id);
  7439. vmx->soft_vnmi_blocked = 0;
  7440. }
  7441. }
  7442. if (exit_reason < kvm_vmx_max_exit_handlers
  7443. && kvm_vmx_exit_handlers[exit_reason])
  7444. return kvm_vmx_exit_handlers[exit_reason](vcpu);
  7445. else {
  7446. WARN_ONCE(1, "vmx: unexpected exit reason 0x%x\n", exit_reason);
  7447. kvm_queue_exception(vcpu, UD_VECTOR);
  7448. return 1;
  7449. }
  7450. }
  7451. static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
  7452. {
  7453. struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
  7454. if (is_guest_mode(vcpu) &&
  7455. nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
  7456. return;
  7457. if (irr == -1 || tpr < irr) {
  7458. vmcs_write32(TPR_THRESHOLD, 0);
  7459. return;
  7460. }
  7461. vmcs_write32(TPR_THRESHOLD, irr);
  7462. }
  7463. static void vmx_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
  7464. {
  7465. u32 sec_exec_control;
  7466. /* Postpone execution until vmcs01 is the current VMCS. */
  7467. if (is_guest_mode(vcpu)) {
  7468. to_vmx(vcpu)->nested.change_vmcs01_virtual_x2apic_mode = true;
  7469. return;
  7470. }
  7471. if (!cpu_has_vmx_virtualize_x2apic_mode())
  7472. return;
  7473. if (!cpu_need_tpr_shadow(vcpu))
  7474. return;
  7475. sec_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
  7476. if (set) {
  7477. sec_exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
  7478. sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
  7479. } else {
  7480. sec_exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
  7481. sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
  7482. }
  7483. vmcs_write32(SECONDARY_VM_EXEC_CONTROL, sec_exec_control);
  7484. vmx_set_msr_bitmap(vcpu);
  7485. }
  7486. static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu, hpa_t hpa)
  7487. {
  7488. struct vcpu_vmx *vmx = to_vmx(vcpu);
  7489. /*
  7490. * Currently we do not handle the nested case where L2 has an
  7491. * APIC access page of its own; that page is still pinned.
  7492. * Hence, we skip the case where the VCPU is in guest mode _and_
  7493. * L1 prepared an APIC access page for L2.
  7494. *
  7495. * For the case where L1 and L2 share the same APIC access page
  7496. * (flexpriority=Y but SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES clear
  7497. * in the vmcs12), this function will only update either the vmcs01
  7498. * or the vmcs02. If the former, the vmcs02 will be updated by
  7499. * prepare_vmcs02. If the latter, the vmcs01 will be updated in
  7500. * the next L2->L1 exit.
  7501. */
  7502. if (!is_guest_mode(vcpu) ||
  7503. !nested_cpu_has2(get_vmcs12(&vmx->vcpu),
  7504. SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
  7505. vmcs_write64(APIC_ACCESS_ADDR, hpa);
  7506. }
  7507. static void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
  7508. {
  7509. u16 status;
  7510. u8 old;
  7511. if (max_isr == -1)
  7512. max_isr = 0;
  7513. status = vmcs_read16(GUEST_INTR_STATUS);
  7514. old = status >> 8;
  7515. if (max_isr != old) {
  7516. status &= 0xff;
  7517. status |= max_isr << 8;
  7518. vmcs_write16(GUEST_INTR_STATUS, status);
  7519. }
  7520. }
  7521. static void vmx_set_rvi(int vector)
  7522. {
  7523. u16 status;
  7524. u8 old;
  7525. if (vector == -1)
  7526. vector = 0;
  7527. status = vmcs_read16(GUEST_INTR_STATUS);
  7528. old = (u8)status & 0xff;
  7529. if ((u8)vector != old) {
  7530. status &= ~0xff;
  7531. status |= (u8)vector;
  7532. vmcs_write16(GUEST_INTR_STATUS, status);
  7533. }
  7534. }
  7535. static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
  7536. {
  7537. if (!is_guest_mode(vcpu)) {
  7538. vmx_set_rvi(max_irr);
  7539. return;
  7540. }
  7541. if (max_irr == -1)
  7542. return;
  7543. /*
  7544. * In guest mode. If a vmexit is needed, vmx_check_nested_events
  7545. * handles it.
  7546. */
  7547. if (nested_exit_on_intr(vcpu))
  7548. return;
  7549. /*
  7550. * Else, fall back to pre-APICv interrupt injection since L2
  7551. * is run without virtual interrupt delivery.
  7552. */
  7553. if (!kvm_event_needs_reinjection(vcpu) &&
  7554. vmx_interrupt_allowed(vcpu)) {
  7555. kvm_queue_interrupt(vcpu, max_irr, false);
  7556. vmx_inject_irq(vcpu);
  7557. }
  7558. }
  7559. static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
  7560. {
  7561. if (!kvm_vcpu_apicv_active(vcpu))
  7562. return;
  7563. vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]);
  7564. vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]);
  7565. vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]);
  7566. vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
  7567. }
  7568. static void vmx_complete_atomic_exit(struct vcpu_vmx *vmx)
  7569. {
  7570. u32 exit_intr_info;
  7571. if (!(vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY
  7572. || vmx->exit_reason == EXIT_REASON_EXCEPTION_NMI))
  7573. return;
  7574. vmx->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
  7575. exit_intr_info = vmx->exit_intr_info;
  7576. /* Handle machine checks before interrupts are enabled */
  7577. if (is_machine_check(exit_intr_info))
  7578. kvm_machine_check();
  7579. /* We need to handle NMIs before interrupts are enabled */
  7580. if (is_nmi(exit_intr_info)) {
  7581. kvm_before_handle_nmi(&vmx->vcpu);
  7582. asm("int $2");
  7583. kvm_after_handle_nmi(&vmx->vcpu);
  7584. }
  7585. }
  7586. static void vmx_handle_external_intr(struct kvm_vcpu *vcpu)
  7587. {
  7588. u32 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
  7589. register void *__sp asm(_ASM_SP);
  7590. if ((exit_intr_info & (INTR_INFO_VALID_MASK | INTR_INFO_INTR_TYPE_MASK))
  7591. == (INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR)) {
  7592. unsigned int vector;
  7593. unsigned long entry;
  7594. gate_desc *desc;
  7595. struct vcpu_vmx *vmx = to_vmx(vcpu);
  7596. #ifdef CONFIG_X86_64
  7597. unsigned long tmp;
  7598. #endif
  7599. vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
  7600. desc = (gate_desc *)vmx->host_idt_base + vector;
  7601. entry = gate_offset(*desc);
  7602. asm volatile(
  7603. #ifdef CONFIG_X86_64
  7604. "mov %%" _ASM_SP ", %[sp]\n\t"
  7605. "and $0xfffffffffffffff0, %%" _ASM_SP "\n\t"
  7606. "push $%c[ss]\n\t"
  7607. "push %[sp]\n\t"
  7608. #endif
  7609. "pushf\n\t"
  7610. __ASM_SIZE(push) " $%c[cs]\n\t"
  7611. "call *%[entry]\n\t"
  7612. :
  7613. #ifdef CONFIG_X86_64
  7614. [sp]"=&r"(tmp),
  7615. #endif
  7616. "+r"(__sp)
  7617. :
  7618. [entry]"r"(entry),
  7619. [ss]"i"(__KERNEL_DS),
  7620. [cs]"i"(__KERNEL_CS)
  7621. );
  7622. }
  7623. }
  7624. static bool vmx_has_high_real_mode_segbase(void)
  7625. {
  7626. return enable_unrestricted_guest || emulate_invalid_guest_state;
  7627. }
  7628. static bool vmx_mpx_supported(void)
  7629. {
  7630. return (vmcs_config.vmexit_ctrl & VM_EXIT_CLEAR_BNDCFGS) &&
  7631. (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_BNDCFGS);
  7632. }
  7633. static bool vmx_xsaves_supported(void)
  7634. {
  7635. return vmcs_config.cpu_based_2nd_exec_ctrl &
  7636. SECONDARY_EXEC_XSAVES;
  7637. }
  7638. static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx)
  7639. {
  7640. u32 exit_intr_info;
  7641. bool unblock_nmi;
  7642. u8 vector;
  7643. bool idtv_info_valid;
  7644. idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK;
  7645. if (cpu_has_virtual_nmis()) {
  7646. if (vmx->nmi_known_unmasked)
  7647. return;
  7648. /*
  7649. * Can't use vmx->exit_intr_info since we're not sure what
  7650. * the exit reason is.
  7651. */
  7652. exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
  7653. unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
  7654. vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
  7655. /*
  7656. * SDM 3: 27.7.1.2 (September 2008)
  7657. * Re-set bit "block by NMI" before VM entry if vmexit caused by
  7658. * a guest IRET fault.
  7659. * SDM 3: 23.2.2 (September 2008)
  7660. * Bit 12 is undefined in any of the following cases:
  7661. * If the VM exit sets the valid bit in the IDT-vectoring
  7662. * information field.
  7663. * If the VM exit is due to a double fault.
  7664. */
  7665. if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
  7666. vector != DF_VECTOR && !idtv_info_valid)
  7667. vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
  7668. GUEST_INTR_STATE_NMI);
  7669. else
  7670. vmx->nmi_known_unmasked =
  7671. !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO)
  7672. & GUEST_INTR_STATE_NMI);
  7673. } else if (unlikely(vmx->soft_vnmi_blocked))
  7674. vmx->vnmi_blocked_time +=
  7675. ktime_to_ns(ktime_sub(ktime_get(), vmx->entry_time));
  7676. }
  7677. static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu,
  7678. u32 idt_vectoring_info,
  7679. int instr_len_field,
  7680. int error_code_field)
  7681. {
  7682. u8 vector;
  7683. int type;
  7684. bool idtv_info_valid;
  7685. idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
  7686. vcpu->arch.nmi_injected = false;
  7687. kvm_clear_exception_queue(vcpu);
  7688. kvm_clear_interrupt_queue(vcpu);
  7689. if (!idtv_info_valid)
  7690. return;
  7691. kvm_make_request(KVM_REQ_EVENT, vcpu);
  7692. vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
  7693. type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
  7694. switch (type) {
  7695. case INTR_TYPE_NMI_INTR:
  7696. vcpu->arch.nmi_injected = true;
  7697. /*
  7698. * SDM 3: 27.7.1.2 (September 2008)
  7699. * Clear bit "block by NMI" before VM entry if a NMI
  7700. * delivery faulted.
  7701. */
  7702. vmx_set_nmi_mask(vcpu, false);
  7703. break;
  7704. case INTR_TYPE_SOFT_EXCEPTION:
  7705. vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
  7706. /* fall through */
  7707. case INTR_TYPE_HARD_EXCEPTION:
  7708. if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
  7709. u32 err = vmcs_read32(error_code_field);
  7710. kvm_requeue_exception_e(vcpu, vector, err);
  7711. } else
  7712. kvm_requeue_exception(vcpu, vector);
  7713. break;
  7714. case INTR_TYPE_SOFT_INTR:
  7715. vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
  7716. /* fall through */
  7717. case INTR_TYPE_EXT_INTR:
  7718. kvm_queue_interrupt(vcpu, vector, type == INTR_TYPE_SOFT_INTR);
  7719. break;
  7720. default:
  7721. break;
  7722. }
  7723. }
  7724. static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
  7725. {
  7726. __vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info,
  7727. VM_EXIT_INSTRUCTION_LEN,
  7728. IDT_VECTORING_ERROR_CODE);
  7729. }
  7730. static void vmx_cancel_injection(struct kvm_vcpu *vcpu)
  7731. {
  7732. __vmx_complete_interrupts(vcpu,
  7733. vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
  7734. VM_ENTRY_INSTRUCTION_LEN,
  7735. VM_ENTRY_EXCEPTION_ERROR_CODE);
  7736. vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
  7737. }
  7738. static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx)
  7739. {
  7740. int i, nr_msrs;
  7741. struct perf_guest_switch_msr *msrs;
  7742. msrs = perf_guest_get_msrs(&nr_msrs);
  7743. if (!msrs)
  7744. return;
  7745. for (i = 0; i < nr_msrs; i++)
  7746. if (msrs[i].host == msrs[i].guest)
  7747. clear_atomic_switch_msr(vmx, msrs[i].msr);
  7748. else
  7749. add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest,
  7750. msrs[i].host);
  7751. }
  7752. static void vmx_arm_hv_timer(struct kvm_vcpu *vcpu)
  7753. {
  7754. struct vcpu_vmx *vmx = to_vmx(vcpu);
  7755. u64 tscl;
  7756. u32 delta_tsc;
  7757. if (vmx->hv_deadline_tsc == -1)
  7758. return;
  7759. tscl = rdtsc();
  7760. if (vmx->hv_deadline_tsc > tscl)
  7761. /* sure to be 32 bit only because checked on set_hv_timer */
  7762. delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >>
  7763. cpu_preemption_timer_multi);
  7764. else
  7765. delta_tsc = 0;
  7766. vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, delta_tsc);
  7767. }
  7768. static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
  7769. {
  7770. struct vcpu_vmx *vmx = to_vmx(vcpu);
  7771. unsigned long debugctlmsr, cr4;
  7772. /* Record the guest's net vcpu time for enforced NMI injections. */
  7773. if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked))
  7774. vmx->entry_time = ktime_get();
  7775. /* Don't enter VMX if guest state is invalid, let the exit handler
  7776. start emulation until we arrive back to a valid state */
  7777. if (vmx->emulation_required)
  7778. return;
  7779. if (vmx->ple_window_dirty) {
  7780. vmx->ple_window_dirty = false;
  7781. vmcs_write32(PLE_WINDOW, vmx->ple_window);
  7782. }
  7783. if (vmx->nested.sync_shadow_vmcs) {
  7784. copy_vmcs12_to_shadow(vmx);
  7785. vmx->nested.sync_shadow_vmcs = false;
  7786. }
  7787. if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty))
  7788. vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
  7789. if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
  7790. vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);
  7791. cr4 = cr4_read_shadow();
  7792. if (unlikely(cr4 != vmx->host_state.vmcs_host_cr4)) {
  7793. vmcs_writel(HOST_CR4, cr4);
  7794. vmx->host_state.vmcs_host_cr4 = cr4;
  7795. }
  7796. /* When single-stepping over STI and MOV SS, we must clear the
  7797. * corresponding interruptibility bits in the guest state. Otherwise
  7798. * vmentry fails as it then expects bit 14 (BS) in pending debug
  7799. * exceptions being set, but that's not correct for the guest debugging
  7800. * case. */
  7801. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
  7802. vmx_set_interrupt_shadow(vcpu, 0);
  7803. if (vmx->guest_pkru_valid)
  7804. __write_pkru(vmx->guest_pkru);
  7805. atomic_switch_perf_msrs(vmx);
  7806. debugctlmsr = get_debugctlmsr();
  7807. vmx_arm_hv_timer(vcpu);
  7808. vmx->__launched = vmx->loaded_vmcs->launched;
  7809. asm(
  7810. /* Store host registers */
  7811. "push %%" _ASM_DX "; push %%" _ASM_BP ";"
  7812. "push %%" _ASM_CX " \n\t" /* placeholder for guest rcx */
  7813. "push %%" _ASM_CX " \n\t"
  7814. "cmp %%" _ASM_SP ", %c[host_rsp](%0) \n\t"
  7815. "je 1f \n\t"
  7816. "mov %%" _ASM_SP ", %c[host_rsp](%0) \n\t"
  7817. __ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t"
  7818. "1: \n\t"
  7819. /* Reload cr2 if changed */
  7820. "mov %c[cr2](%0), %%" _ASM_AX " \n\t"
  7821. "mov %%cr2, %%" _ASM_DX " \n\t"
  7822. "cmp %%" _ASM_AX ", %%" _ASM_DX " \n\t"
  7823. "je 2f \n\t"
  7824. "mov %%" _ASM_AX", %%cr2 \n\t"
  7825. "2: \n\t"
  7826. /* Check if vmlaunch of vmresume is needed */
  7827. "cmpl $0, %c[launched](%0) \n\t"
  7828. /* Load guest registers. Don't clobber flags. */
  7829. "mov %c[rax](%0), %%" _ASM_AX " \n\t"
  7830. "mov %c[rbx](%0), %%" _ASM_BX " \n\t"
  7831. "mov %c[rdx](%0), %%" _ASM_DX " \n\t"
  7832. "mov %c[rsi](%0), %%" _ASM_SI " \n\t"
  7833. "mov %c[rdi](%0), %%" _ASM_DI " \n\t"
  7834. "mov %c[rbp](%0), %%" _ASM_BP " \n\t"
  7835. #ifdef CONFIG_X86_64
  7836. "mov %c[r8](%0), %%r8 \n\t"
  7837. "mov %c[r9](%0), %%r9 \n\t"
  7838. "mov %c[r10](%0), %%r10 \n\t"
  7839. "mov %c[r11](%0), %%r11 \n\t"
  7840. "mov %c[r12](%0), %%r12 \n\t"
  7841. "mov %c[r13](%0), %%r13 \n\t"
  7842. "mov %c[r14](%0), %%r14 \n\t"
  7843. "mov %c[r15](%0), %%r15 \n\t"
  7844. #endif
  7845. "mov %c[rcx](%0), %%" _ASM_CX " \n\t" /* kills %0 (ecx) */
  7846. /* Enter guest mode */
  7847. "jne 1f \n\t"
  7848. __ex(ASM_VMX_VMLAUNCH) "\n\t"
  7849. "jmp 2f \n\t"
  7850. "1: " __ex(ASM_VMX_VMRESUME) "\n\t"
  7851. "2: "
  7852. /* Save guest registers, load host registers, keep flags */
  7853. "mov %0, %c[wordsize](%%" _ASM_SP ") \n\t"
  7854. "pop %0 \n\t"
  7855. "mov %%" _ASM_AX ", %c[rax](%0) \n\t"
  7856. "mov %%" _ASM_BX ", %c[rbx](%0) \n\t"
  7857. __ASM_SIZE(pop) " %c[rcx](%0) \n\t"
  7858. "mov %%" _ASM_DX ", %c[rdx](%0) \n\t"
  7859. "mov %%" _ASM_SI ", %c[rsi](%0) \n\t"
  7860. "mov %%" _ASM_DI ", %c[rdi](%0) \n\t"
  7861. "mov %%" _ASM_BP ", %c[rbp](%0) \n\t"
  7862. #ifdef CONFIG_X86_64
  7863. "mov %%r8, %c[r8](%0) \n\t"
  7864. "mov %%r9, %c[r9](%0) \n\t"
  7865. "mov %%r10, %c[r10](%0) \n\t"
  7866. "mov %%r11, %c[r11](%0) \n\t"
  7867. "mov %%r12, %c[r12](%0) \n\t"
  7868. "mov %%r13, %c[r13](%0) \n\t"
  7869. "mov %%r14, %c[r14](%0) \n\t"
  7870. "mov %%r15, %c[r15](%0) \n\t"
  7871. #endif
  7872. "mov %%cr2, %%" _ASM_AX " \n\t"
  7873. "mov %%" _ASM_AX ", %c[cr2](%0) \n\t"
  7874. "pop %%" _ASM_BP "; pop %%" _ASM_DX " \n\t"
  7875. "setbe %c[fail](%0) \n\t"
  7876. ".pushsection .rodata \n\t"
  7877. ".global vmx_return \n\t"
  7878. "vmx_return: " _ASM_PTR " 2b \n\t"
  7879. ".popsection"
  7880. : : "c"(vmx), "d"((unsigned long)HOST_RSP),
  7881. [launched]"i"(offsetof(struct vcpu_vmx, __launched)),
  7882. [fail]"i"(offsetof(struct vcpu_vmx, fail)),
  7883. [host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)),
  7884. [rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])),
  7885. [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])),
  7886. [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])),
  7887. [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])),
  7888. [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])),
  7889. [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])),
  7890. [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])),
  7891. #ifdef CONFIG_X86_64
  7892. [r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])),
  7893. [r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])),
  7894. [r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])),
  7895. [r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])),
  7896. [r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])),
  7897. [r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])),
  7898. [r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])),
  7899. [r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])),
  7900. #endif
  7901. [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)),
  7902. [wordsize]"i"(sizeof(ulong))
  7903. : "cc", "memory"
  7904. #ifdef CONFIG_X86_64
  7905. , "rax", "rbx", "rdi", "rsi"
  7906. , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
  7907. #else
  7908. , "eax", "ebx", "edi", "esi"
  7909. #endif
  7910. );
  7911. /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
  7912. if (debugctlmsr)
  7913. update_debugctlmsr(debugctlmsr);
  7914. #ifndef CONFIG_X86_64
  7915. /*
  7916. * The sysexit path does not restore ds/es, so we must set them to
  7917. * a reasonable value ourselves.
  7918. *
  7919. * We can't defer this to vmx_load_host_state() since that function
  7920. * may be executed in interrupt context, which saves and restore segments
  7921. * around it, nullifying its effect.
  7922. */
  7923. loadsegment(ds, __USER_DS);
  7924. loadsegment(es, __USER_DS);
  7925. #endif
  7926. vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
  7927. | (1 << VCPU_EXREG_RFLAGS)
  7928. | (1 << VCPU_EXREG_PDPTR)
  7929. | (1 << VCPU_EXREG_SEGMENTS)
  7930. | (1 << VCPU_EXREG_CR3));
  7931. vcpu->arch.regs_dirty = 0;
  7932. vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
  7933. vmx->loaded_vmcs->launched = 1;
  7934. vmx->exit_reason = vmcs_read32(VM_EXIT_REASON);
  7935. /*
  7936. * eager fpu is enabled if PKEY is supported and CR4 is switched
  7937. * back on host, so it is safe to read guest PKRU from current
  7938. * XSAVE.
  7939. */
  7940. if (boot_cpu_has(X86_FEATURE_OSPKE)) {
  7941. vmx->guest_pkru = __read_pkru();
  7942. if (vmx->guest_pkru != vmx->host_pkru) {
  7943. vmx->guest_pkru_valid = true;
  7944. __write_pkru(vmx->host_pkru);
  7945. } else
  7946. vmx->guest_pkru_valid = false;
  7947. }
  7948. /*
  7949. * the KVM_REQ_EVENT optimization bit is only on for one entry, and if
  7950. * we did not inject a still-pending event to L1 now because of
  7951. * nested_run_pending, we need to re-enable this bit.
  7952. */
  7953. if (vmx->nested.nested_run_pending)
  7954. kvm_make_request(KVM_REQ_EVENT, vcpu);
  7955. vmx->nested.nested_run_pending = 0;
  7956. vmx_complete_atomic_exit(vmx);
  7957. vmx_recover_nmi_blocking(vmx);
  7958. vmx_complete_interrupts(vmx);
  7959. }
  7960. static void vmx_load_vmcs01(struct kvm_vcpu *vcpu)
  7961. {
  7962. struct vcpu_vmx *vmx = to_vmx(vcpu);
  7963. int cpu;
  7964. if (vmx->loaded_vmcs == &vmx->vmcs01)
  7965. return;
  7966. cpu = get_cpu();
  7967. vmx->loaded_vmcs = &vmx->vmcs01;
  7968. vmx_vcpu_put(vcpu);
  7969. vmx_vcpu_load(vcpu, cpu);
  7970. vcpu->cpu = cpu;
  7971. put_cpu();
  7972. }
  7973. /*
  7974. * Ensure that the current vmcs of the logical processor is the
  7975. * vmcs01 of the vcpu before calling free_nested().
  7976. */
  7977. static void vmx_free_vcpu_nested(struct kvm_vcpu *vcpu)
  7978. {
  7979. struct vcpu_vmx *vmx = to_vmx(vcpu);
  7980. int r;
  7981. r = vcpu_load(vcpu);
  7982. BUG_ON(r);
  7983. vmx_load_vmcs01(vcpu);
  7984. free_nested(vmx);
  7985. vcpu_put(vcpu);
  7986. }
  7987. static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
  7988. {
  7989. struct vcpu_vmx *vmx = to_vmx(vcpu);
  7990. if (enable_pml)
  7991. vmx_destroy_pml_buffer(vmx);
  7992. free_vpid(vmx->vpid);
  7993. leave_guest_mode(vcpu);
  7994. vmx_free_vcpu_nested(vcpu);
  7995. free_loaded_vmcs(vmx->loaded_vmcs);
  7996. kfree(vmx->guest_msrs);
  7997. kvm_vcpu_uninit(vcpu);
  7998. kmem_cache_free(kvm_vcpu_cache, vmx);
  7999. }
  8000. static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
  8001. {
  8002. int err;
  8003. struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
  8004. int cpu;
  8005. if (!vmx)
  8006. return ERR_PTR(-ENOMEM);
  8007. vmx->vpid = allocate_vpid();
  8008. err = kvm_vcpu_init(&vmx->vcpu, kvm, id);
  8009. if (err)
  8010. goto free_vcpu;
  8011. err = -ENOMEM;
  8012. /*
  8013. * If PML is turned on, failure on enabling PML just results in failure
  8014. * of creating the vcpu, therefore we can simplify PML logic (by
  8015. * avoiding dealing with cases, such as enabling PML partially on vcpus
  8016. * for the guest, etc.
  8017. */
  8018. if (enable_pml) {
  8019. vmx->pml_pg = alloc_page(GFP_KERNEL | __GFP_ZERO);
  8020. if (!vmx->pml_pg)
  8021. goto uninit_vcpu;
  8022. }
  8023. vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
  8024. BUILD_BUG_ON(ARRAY_SIZE(vmx_msr_index) * sizeof(vmx->guest_msrs[0])
  8025. > PAGE_SIZE);
  8026. if (!vmx->guest_msrs)
  8027. goto free_pml;
  8028. vmx->loaded_vmcs = &vmx->vmcs01;
  8029. vmx->loaded_vmcs->vmcs = alloc_vmcs();
  8030. vmx->loaded_vmcs->shadow_vmcs = NULL;
  8031. if (!vmx->loaded_vmcs->vmcs)
  8032. goto free_msrs;
  8033. if (!vmm_exclusive)
  8034. kvm_cpu_vmxon(__pa(per_cpu(vmxarea, raw_smp_processor_id())));
  8035. loaded_vmcs_init(vmx->loaded_vmcs);
  8036. if (!vmm_exclusive)
  8037. kvm_cpu_vmxoff();
  8038. cpu = get_cpu();
  8039. vmx_vcpu_load(&vmx->vcpu, cpu);
  8040. vmx->vcpu.cpu = cpu;
  8041. err = vmx_vcpu_setup(vmx);
  8042. vmx_vcpu_put(&vmx->vcpu);
  8043. put_cpu();
  8044. if (err)
  8045. goto free_vmcs;
  8046. if (cpu_need_virtualize_apic_accesses(&vmx->vcpu)) {
  8047. err = alloc_apic_access_page(kvm);
  8048. if (err)
  8049. goto free_vmcs;
  8050. }
  8051. if (enable_ept) {
  8052. if (!kvm->arch.ept_identity_map_addr)
  8053. kvm->arch.ept_identity_map_addr =
  8054. VMX_EPT_IDENTITY_PAGETABLE_ADDR;
  8055. err = init_rmode_identity_map(kvm);
  8056. if (err)
  8057. goto free_vmcs;
  8058. }
  8059. if (nested) {
  8060. nested_vmx_setup_ctls_msrs(vmx);
  8061. vmx->nested.vpid02 = allocate_vpid();
  8062. }
  8063. vmx->nested.posted_intr_nv = -1;
  8064. vmx->nested.current_vmptr = -1ull;
  8065. vmx->nested.current_vmcs12 = NULL;
  8066. vmx->msr_ia32_feature_control_valid_bits = FEATURE_CONTROL_LOCKED;
  8067. return &vmx->vcpu;
  8068. free_vmcs:
  8069. free_vpid(vmx->nested.vpid02);
  8070. free_loaded_vmcs(vmx->loaded_vmcs);
  8071. free_msrs:
  8072. kfree(vmx->guest_msrs);
  8073. free_pml:
  8074. vmx_destroy_pml_buffer(vmx);
  8075. uninit_vcpu:
  8076. kvm_vcpu_uninit(&vmx->vcpu);
  8077. free_vcpu:
  8078. free_vpid(vmx->vpid);
  8079. kmem_cache_free(kvm_vcpu_cache, vmx);
  8080. return ERR_PTR(err);
  8081. }
  8082. static void __init vmx_check_processor_compat(void *rtn)
  8083. {
  8084. struct vmcs_config vmcs_conf;
  8085. *(int *)rtn = 0;
  8086. if (setup_vmcs_config(&vmcs_conf) < 0)
  8087. *(int *)rtn = -EIO;
  8088. if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
  8089. printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
  8090. smp_processor_id());
  8091. *(int *)rtn = -EIO;
  8092. }
  8093. }
  8094. static int get_ept_level(void)
  8095. {
  8096. return VMX_EPT_DEFAULT_GAW + 1;
  8097. }
  8098. static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
  8099. {
  8100. u8 cache;
  8101. u64 ipat = 0;
  8102. /* For VT-d and EPT combination
  8103. * 1. MMIO: always map as UC
  8104. * 2. EPT with VT-d:
  8105. * a. VT-d without snooping control feature: can't guarantee the
  8106. * result, try to trust guest.
  8107. * b. VT-d with snooping control feature: snooping control feature of
  8108. * VT-d engine can guarantee the cache correctness. Just set it
  8109. * to WB to keep consistent with host. So the same as item 3.
  8110. * 3. EPT without VT-d: always map as WB and set IPAT=1 to keep
  8111. * consistent with host MTRR
  8112. */
  8113. if (is_mmio) {
  8114. cache = MTRR_TYPE_UNCACHABLE;
  8115. goto exit;
  8116. }
  8117. if (!kvm_arch_has_noncoherent_dma(vcpu->kvm)) {
  8118. ipat = VMX_EPT_IPAT_BIT;
  8119. cache = MTRR_TYPE_WRBACK;
  8120. goto exit;
  8121. }
  8122. if (kvm_read_cr0(vcpu) & X86_CR0_CD) {
  8123. ipat = VMX_EPT_IPAT_BIT;
  8124. if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
  8125. cache = MTRR_TYPE_WRBACK;
  8126. else
  8127. cache = MTRR_TYPE_UNCACHABLE;
  8128. goto exit;
  8129. }
  8130. cache = kvm_mtrr_get_guest_memory_type(vcpu, gfn);
  8131. exit:
  8132. return (cache << VMX_EPT_MT_EPTE_SHIFT) | ipat;
  8133. }
  8134. static int vmx_get_lpage_level(void)
  8135. {
  8136. if (enable_ept && !cpu_has_vmx_ept_1g_page())
  8137. return PT_DIRECTORY_LEVEL;
  8138. else
  8139. /* For shadow and EPT supported 1GB page */
  8140. return PT_PDPE_LEVEL;
  8141. }
  8142. static void vmcs_set_secondary_exec_control(u32 new_ctl)
  8143. {
  8144. /*
  8145. * These bits in the secondary execution controls field
  8146. * are dynamic, the others are mostly based on the hypervisor
  8147. * architecture and the guest's CPUID. Do not touch the
  8148. * dynamic bits.
  8149. */
  8150. u32 mask =
  8151. SECONDARY_EXEC_SHADOW_VMCS |
  8152. SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
  8153. SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
  8154. u32 cur_ctl = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
  8155. vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
  8156. (new_ctl & ~mask) | (cur_ctl & mask));
  8157. }
  8158. /*
  8159. * Generate MSR_IA32_VMX_CR{0,4}_FIXED1 according to CPUID. Only set bits
  8160. * (indicating "allowed-1") if they are supported in the guest's CPUID.
  8161. */
  8162. static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu)
  8163. {
  8164. struct vcpu_vmx *vmx = to_vmx(vcpu);
  8165. struct kvm_cpuid_entry2 *entry;
  8166. vmx->nested.nested_vmx_cr0_fixed1 = 0xffffffff;
  8167. vmx->nested.nested_vmx_cr4_fixed1 = X86_CR4_PCE;
  8168. #define cr4_fixed1_update(_cr4_mask, _reg, _cpuid_mask) do { \
  8169. if (entry && (entry->_reg & (_cpuid_mask))) \
  8170. vmx->nested.nested_vmx_cr4_fixed1 |= (_cr4_mask); \
  8171. } while (0)
  8172. entry = kvm_find_cpuid_entry(vcpu, 0x1, 0);
  8173. cr4_fixed1_update(X86_CR4_VME, edx, bit(X86_FEATURE_VME));
  8174. cr4_fixed1_update(X86_CR4_PVI, edx, bit(X86_FEATURE_VME));
  8175. cr4_fixed1_update(X86_CR4_TSD, edx, bit(X86_FEATURE_TSC));
  8176. cr4_fixed1_update(X86_CR4_DE, edx, bit(X86_FEATURE_DE));
  8177. cr4_fixed1_update(X86_CR4_PSE, edx, bit(X86_FEATURE_PSE));
  8178. cr4_fixed1_update(X86_CR4_PAE, edx, bit(X86_FEATURE_PAE));
  8179. cr4_fixed1_update(X86_CR4_MCE, edx, bit(X86_FEATURE_MCE));
  8180. cr4_fixed1_update(X86_CR4_PGE, edx, bit(X86_FEATURE_PGE));
  8181. cr4_fixed1_update(X86_CR4_OSFXSR, edx, bit(X86_FEATURE_FXSR));
  8182. cr4_fixed1_update(X86_CR4_OSXMMEXCPT, edx, bit(X86_FEATURE_XMM));
  8183. cr4_fixed1_update(X86_CR4_VMXE, ecx, bit(X86_FEATURE_VMX));
  8184. cr4_fixed1_update(X86_CR4_SMXE, ecx, bit(X86_FEATURE_SMX));
  8185. cr4_fixed1_update(X86_CR4_PCIDE, ecx, bit(X86_FEATURE_PCID));
  8186. cr4_fixed1_update(X86_CR4_OSXSAVE, ecx, bit(X86_FEATURE_XSAVE));
  8187. entry = kvm_find_cpuid_entry(vcpu, 0x7, 0);
  8188. cr4_fixed1_update(X86_CR4_FSGSBASE, ebx, bit(X86_FEATURE_FSGSBASE));
  8189. cr4_fixed1_update(X86_CR4_SMEP, ebx, bit(X86_FEATURE_SMEP));
  8190. cr4_fixed1_update(X86_CR4_SMAP, ebx, bit(X86_FEATURE_SMAP));
  8191. cr4_fixed1_update(X86_CR4_PKE, ecx, bit(X86_FEATURE_PKU));
  8192. /* TODO: Use X86_CR4_UMIP and X86_FEATURE_UMIP macros */
  8193. cr4_fixed1_update(bit(11), ecx, bit(2));
  8194. #undef cr4_fixed1_update
  8195. }
  8196. static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
  8197. {
  8198. struct kvm_cpuid_entry2 *best;
  8199. struct vcpu_vmx *vmx = to_vmx(vcpu);
  8200. u32 secondary_exec_ctl = vmx_secondary_exec_control(vmx);
  8201. if (vmx_rdtscp_supported()) {
  8202. bool rdtscp_enabled = guest_cpuid_has_rdtscp(vcpu);
  8203. if (!rdtscp_enabled)
  8204. secondary_exec_ctl &= ~SECONDARY_EXEC_RDTSCP;
  8205. if (nested) {
  8206. if (rdtscp_enabled)
  8207. vmx->nested.nested_vmx_secondary_ctls_high |=
  8208. SECONDARY_EXEC_RDTSCP;
  8209. else
  8210. vmx->nested.nested_vmx_secondary_ctls_high &=
  8211. ~SECONDARY_EXEC_RDTSCP;
  8212. }
  8213. }
  8214. /* Exposing INVPCID only when PCID is exposed */
  8215. best = kvm_find_cpuid_entry(vcpu, 0x7, 0);
  8216. if (vmx_invpcid_supported() &&
  8217. (!best || !(best->ebx & bit(X86_FEATURE_INVPCID)) ||
  8218. !guest_cpuid_has_pcid(vcpu))) {
  8219. secondary_exec_ctl &= ~SECONDARY_EXEC_ENABLE_INVPCID;
  8220. if (best)
  8221. best->ebx &= ~bit(X86_FEATURE_INVPCID);
  8222. }
  8223. if (cpu_has_secondary_exec_ctrls())
  8224. vmcs_set_secondary_exec_control(secondary_exec_ctl);
  8225. if (nested_vmx_allowed(vcpu))
  8226. to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |=
  8227. FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
  8228. else
  8229. to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &=
  8230. ~FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
  8231. if (nested_vmx_allowed(vcpu))
  8232. nested_vmx_cr_fixed1_bits_update(vcpu);
  8233. }
  8234. static void vmx_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
  8235. {
  8236. if (func == 1 && nested)
  8237. entry->ecx |= bit(X86_FEATURE_VMX);
  8238. }
  8239. static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
  8240. struct x86_exception *fault)
  8241. {
  8242. struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
  8243. u32 exit_reason;
  8244. if (fault->error_code & PFERR_RSVD_MASK)
  8245. exit_reason = EXIT_REASON_EPT_MISCONFIG;
  8246. else
  8247. exit_reason = EXIT_REASON_EPT_VIOLATION;
  8248. nested_vmx_vmexit(vcpu, exit_reason, 0, vcpu->arch.exit_qualification);
  8249. vmcs12->guest_physical_address = fault->address;
  8250. }
  8251. /* Callbacks for nested_ept_init_mmu_context: */
  8252. static unsigned long nested_ept_get_cr3(struct kvm_vcpu *vcpu)
  8253. {
  8254. /* return the page table to be shadowed - in our case, EPT12 */
  8255. return get_vmcs12(vcpu)->ept_pointer;
  8256. }
  8257. static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
  8258. {
  8259. WARN_ON(mmu_is_nested(vcpu));
  8260. kvm_init_shadow_ept_mmu(vcpu,
  8261. to_vmx(vcpu)->nested.nested_vmx_ept_caps &
  8262. VMX_EPT_EXECUTE_ONLY_BIT);
  8263. vcpu->arch.mmu.set_cr3 = vmx_set_cr3;
  8264. vcpu->arch.mmu.get_cr3 = nested_ept_get_cr3;
  8265. vcpu->arch.mmu.inject_page_fault = nested_ept_inject_page_fault;
  8266. vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu;
  8267. }
  8268. static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
  8269. {
  8270. vcpu->arch.walk_mmu = &vcpu->arch.mmu;
  8271. }
  8272. static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
  8273. u16 error_code)
  8274. {
  8275. bool inequality, bit;
  8276. bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0;
  8277. inequality =
  8278. (error_code & vmcs12->page_fault_error_code_mask) !=
  8279. vmcs12->page_fault_error_code_match;
  8280. return inequality ^ bit;
  8281. }
  8282. static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu,
  8283. struct x86_exception *fault)
  8284. {
  8285. struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
  8286. WARN_ON(!is_guest_mode(vcpu));
  8287. if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code))
  8288. nested_vmx_vmexit(vcpu, to_vmx(vcpu)->exit_reason,
  8289. vmcs_read32(VM_EXIT_INTR_INFO),
  8290. vmcs_readl(EXIT_QUALIFICATION));
  8291. else
  8292. kvm_inject_page_fault(vcpu, fault);
  8293. }
  8294. static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu,
  8295. struct vmcs12 *vmcs12)
  8296. {
  8297. struct vcpu_vmx *vmx = to_vmx(vcpu);
  8298. int maxphyaddr = cpuid_maxphyaddr(vcpu);
  8299. if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
  8300. if (!PAGE_ALIGNED(vmcs12->apic_access_addr) ||
  8301. vmcs12->apic_access_addr >> maxphyaddr)
  8302. return false;
  8303. /*
  8304. * Translate L1 physical address to host physical
  8305. * address for vmcs02. Keep the page pinned, so this
  8306. * physical address remains valid. We keep a reference
  8307. * to it so we can release it later.
  8308. */
  8309. if (vmx->nested.apic_access_page) /* shouldn't happen */
  8310. nested_release_page(vmx->nested.apic_access_page);
  8311. vmx->nested.apic_access_page =
  8312. nested_get_page(vcpu, vmcs12->apic_access_addr);
  8313. }
  8314. if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
  8315. if (!PAGE_ALIGNED(vmcs12->virtual_apic_page_addr) ||
  8316. vmcs12->virtual_apic_page_addr >> maxphyaddr)
  8317. return false;
  8318. if (vmx->nested.virtual_apic_page) /* shouldn't happen */
  8319. nested_release_page(vmx->nested.virtual_apic_page);
  8320. vmx->nested.virtual_apic_page =
  8321. nested_get_page(vcpu, vmcs12->virtual_apic_page_addr);
  8322. /*
  8323. * Failing the vm entry is _not_ what the processor does
  8324. * but it's basically the only possibility we have.
  8325. * We could still enter the guest if CR8 load exits are
  8326. * enabled, CR8 store exits are enabled, and virtualize APIC
  8327. * access is disabled; in this case the processor would never
  8328. * use the TPR shadow and we could simply clear the bit from
  8329. * the execution control. But such a configuration is useless,
  8330. * so let's keep the code simple.
  8331. */
  8332. if (!vmx->nested.virtual_apic_page)
  8333. return false;
  8334. }
  8335. if (nested_cpu_has_posted_intr(vmcs12)) {
  8336. if (!IS_ALIGNED(vmcs12->posted_intr_desc_addr, 64) ||
  8337. vmcs12->posted_intr_desc_addr >> maxphyaddr)
  8338. return false;
  8339. if (vmx->nested.pi_desc_page) { /* shouldn't happen */
  8340. kunmap(vmx->nested.pi_desc_page);
  8341. nested_release_page(vmx->nested.pi_desc_page);
  8342. }
  8343. vmx->nested.pi_desc_page =
  8344. nested_get_page(vcpu, vmcs12->posted_intr_desc_addr);
  8345. if (!vmx->nested.pi_desc_page)
  8346. return false;
  8347. vmx->nested.pi_desc =
  8348. (struct pi_desc *)kmap(vmx->nested.pi_desc_page);
  8349. if (!vmx->nested.pi_desc) {
  8350. nested_release_page_clean(vmx->nested.pi_desc_page);
  8351. return false;
  8352. }
  8353. vmx->nested.pi_desc =
  8354. (struct pi_desc *)((void *)vmx->nested.pi_desc +
  8355. (unsigned long)(vmcs12->posted_intr_desc_addr &
  8356. (PAGE_SIZE - 1)));
  8357. }
  8358. return true;
  8359. }
  8360. static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu)
  8361. {
  8362. u64 preemption_timeout = get_vmcs12(vcpu)->vmx_preemption_timer_value;
  8363. struct vcpu_vmx *vmx = to_vmx(vcpu);
  8364. if (vcpu->arch.virtual_tsc_khz == 0)
  8365. return;
  8366. /* Make sure short timeouts reliably trigger an immediate vmexit.
  8367. * hrtimer_start does not guarantee this. */
  8368. if (preemption_timeout <= 1) {
  8369. vmx_preemption_timer_fn(&vmx->nested.preemption_timer);
  8370. return;
  8371. }
  8372. preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
  8373. preemption_timeout *= 1000000;
  8374. do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz);
  8375. hrtimer_start(&vmx->nested.preemption_timer,
  8376. ns_to_ktime(preemption_timeout), HRTIMER_MODE_REL);
  8377. }
  8378. static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu,
  8379. struct vmcs12 *vmcs12)
  8380. {
  8381. int maxphyaddr;
  8382. u64 addr;
  8383. if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
  8384. return 0;
  8385. if (vmcs12_read_any(vcpu, MSR_BITMAP, &addr)) {
  8386. WARN_ON(1);
  8387. return -EINVAL;
  8388. }
  8389. maxphyaddr = cpuid_maxphyaddr(vcpu);
  8390. if (!PAGE_ALIGNED(vmcs12->msr_bitmap) ||
  8391. ((addr + PAGE_SIZE) >> maxphyaddr))
  8392. return -EINVAL;
  8393. return 0;
  8394. }
  8395. /*
  8396. * Merge L0's and L1's MSR bitmap, return false to indicate that
  8397. * we do not use the hardware.
  8398. */
  8399. static inline bool nested_vmx_merge_msr_bitmap(struct kvm_vcpu *vcpu,
  8400. struct vmcs12 *vmcs12)
  8401. {
  8402. int msr;
  8403. struct page *page;
  8404. unsigned long *msr_bitmap_l1;
  8405. unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.msr_bitmap;
  8406. /* This shortcut is ok because we support only x2APIC MSRs so far. */
  8407. if (!nested_cpu_has_virt_x2apic_mode(vmcs12))
  8408. return false;
  8409. page = nested_get_page(vcpu, vmcs12->msr_bitmap);
  8410. if (!page) {
  8411. WARN_ON(1);
  8412. return false;
  8413. }
  8414. msr_bitmap_l1 = (unsigned long *)kmap(page);
  8415. if (!msr_bitmap_l1) {
  8416. nested_release_page_clean(page);
  8417. WARN_ON(1);
  8418. return false;
  8419. }
  8420. memset(msr_bitmap_l0, 0xff, PAGE_SIZE);
  8421. if (nested_cpu_has_virt_x2apic_mode(vmcs12)) {
  8422. if (nested_cpu_has_apic_reg_virt(vmcs12))
  8423. for (msr = 0x800; msr <= 0x8ff; msr++)
  8424. nested_vmx_disable_intercept_for_msr(
  8425. msr_bitmap_l1, msr_bitmap_l0,
  8426. msr, MSR_TYPE_R);
  8427. nested_vmx_disable_intercept_for_msr(
  8428. msr_bitmap_l1, msr_bitmap_l0,
  8429. APIC_BASE_MSR + (APIC_TASKPRI >> 4),
  8430. MSR_TYPE_R | MSR_TYPE_W);
  8431. if (nested_cpu_has_vid(vmcs12)) {
  8432. nested_vmx_disable_intercept_for_msr(
  8433. msr_bitmap_l1, msr_bitmap_l0,
  8434. APIC_BASE_MSR + (APIC_EOI >> 4),
  8435. MSR_TYPE_W);
  8436. nested_vmx_disable_intercept_for_msr(
  8437. msr_bitmap_l1, msr_bitmap_l0,
  8438. APIC_BASE_MSR + (APIC_SELF_IPI >> 4),
  8439. MSR_TYPE_W);
  8440. }
  8441. }
  8442. kunmap(page);
  8443. nested_release_page_clean(page);
  8444. return true;
  8445. }
  8446. static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu,
  8447. struct vmcs12 *vmcs12)
  8448. {
  8449. if (!nested_cpu_has_virt_x2apic_mode(vmcs12) &&
  8450. !nested_cpu_has_apic_reg_virt(vmcs12) &&
  8451. !nested_cpu_has_vid(vmcs12) &&
  8452. !nested_cpu_has_posted_intr(vmcs12))
  8453. return 0;
  8454. /*
  8455. * If virtualize x2apic mode is enabled,
  8456. * virtualize apic access must be disabled.
  8457. */
  8458. if (nested_cpu_has_virt_x2apic_mode(vmcs12) &&
  8459. nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
  8460. return -EINVAL;
  8461. /*
  8462. * If virtual interrupt delivery is enabled,
  8463. * we must exit on external interrupts.
  8464. */
  8465. if (nested_cpu_has_vid(vmcs12) &&
  8466. !nested_exit_on_intr(vcpu))
  8467. return -EINVAL;
  8468. /*
  8469. * bits 15:8 should be zero in posted_intr_nv,
  8470. * the descriptor address has been already checked
  8471. * in nested_get_vmcs12_pages.
  8472. */
  8473. if (nested_cpu_has_posted_intr(vmcs12) &&
  8474. (!nested_cpu_has_vid(vmcs12) ||
  8475. !nested_exit_intr_ack_set(vcpu) ||
  8476. vmcs12->posted_intr_nv & 0xff00))
  8477. return -EINVAL;
  8478. /* tpr shadow is needed by all apicv features. */
  8479. if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
  8480. return -EINVAL;
  8481. return 0;
  8482. }
  8483. static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
  8484. unsigned long count_field,
  8485. unsigned long addr_field)
  8486. {
  8487. int maxphyaddr;
  8488. u64 count, addr;
  8489. if (vmcs12_read_any(vcpu, count_field, &count) ||
  8490. vmcs12_read_any(vcpu, addr_field, &addr)) {
  8491. WARN_ON(1);
  8492. return -EINVAL;
  8493. }
  8494. if (count == 0)
  8495. return 0;
  8496. maxphyaddr = cpuid_maxphyaddr(vcpu);
  8497. if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr ||
  8498. (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr) {
  8499. pr_debug_ratelimited(
  8500. "nVMX: invalid MSR switch (0x%lx, %d, %llu, 0x%08llx)",
  8501. addr_field, maxphyaddr, count, addr);
  8502. return -EINVAL;
  8503. }
  8504. return 0;
  8505. }
  8506. static int nested_vmx_check_msr_switch_controls(struct kvm_vcpu *vcpu,
  8507. struct vmcs12 *vmcs12)
  8508. {
  8509. if (vmcs12->vm_exit_msr_load_count == 0 &&
  8510. vmcs12->vm_exit_msr_store_count == 0 &&
  8511. vmcs12->vm_entry_msr_load_count == 0)
  8512. return 0; /* Fast path */
  8513. if (nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_LOAD_COUNT,
  8514. VM_EXIT_MSR_LOAD_ADDR) ||
  8515. nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_STORE_COUNT,
  8516. VM_EXIT_MSR_STORE_ADDR) ||
  8517. nested_vmx_check_msr_switch(vcpu, VM_ENTRY_MSR_LOAD_COUNT,
  8518. VM_ENTRY_MSR_LOAD_ADDR))
  8519. return -EINVAL;
  8520. return 0;
  8521. }
  8522. static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu,
  8523. struct vmx_msr_entry *e)
  8524. {
  8525. /* x2APIC MSR accesses are not allowed */
  8526. if (vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8)
  8527. return -EINVAL;
  8528. if (e->index == MSR_IA32_UCODE_WRITE || /* SDM Table 35-2 */
  8529. e->index == MSR_IA32_UCODE_REV)
  8530. return -EINVAL;
  8531. if (e->reserved != 0)
  8532. return -EINVAL;
  8533. return 0;
  8534. }
  8535. static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu,
  8536. struct vmx_msr_entry *e)
  8537. {
  8538. if (e->index == MSR_FS_BASE ||
  8539. e->index == MSR_GS_BASE ||
  8540. e->index == MSR_IA32_SMM_MONITOR_CTL || /* SMM is not supported */
  8541. nested_vmx_msr_check_common(vcpu, e))
  8542. return -EINVAL;
  8543. return 0;
  8544. }
  8545. static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu,
  8546. struct vmx_msr_entry *e)
  8547. {
  8548. if (e->index == MSR_IA32_SMBASE || /* SMM is not supported */
  8549. nested_vmx_msr_check_common(vcpu, e))
  8550. return -EINVAL;
  8551. return 0;
  8552. }
  8553. /*
  8554. * Load guest's/host's msr at nested entry/exit.
  8555. * return 0 for success, entry index for failure.
  8556. */
  8557. static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
  8558. {
  8559. u32 i;
  8560. struct vmx_msr_entry e;
  8561. struct msr_data msr;
  8562. msr.host_initiated = false;
  8563. for (i = 0; i < count; i++) {
  8564. if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e),
  8565. &e, sizeof(e))) {
  8566. pr_debug_ratelimited(
  8567. "%s cannot read MSR entry (%u, 0x%08llx)\n",
  8568. __func__, i, gpa + i * sizeof(e));
  8569. goto fail;
  8570. }
  8571. if (nested_vmx_load_msr_check(vcpu, &e)) {
  8572. pr_debug_ratelimited(
  8573. "%s check failed (%u, 0x%x, 0x%x)\n",
  8574. __func__, i, e.index, e.reserved);
  8575. goto fail;
  8576. }
  8577. msr.index = e.index;
  8578. msr.data = e.value;
  8579. if (kvm_set_msr(vcpu, &msr)) {
  8580. pr_debug_ratelimited(
  8581. "%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
  8582. __func__, i, e.index, e.value);
  8583. goto fail;
  8584. }
  8585. }
  8586. return 0;
  8587. fail:
  8588. return i + 1;
  8589. }
  8590. static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
  8591. {
  8592. u32 i;
  8593. struct vmx_msr_entry e;
  8594. for (i = 0; i < count; i++) {
  8595. struct msr_data msr_info;
  8596. if (kvm_vcpu_read_guest(vcpu,
  8597. gpa + i * sizeof(e),
  8598. &e, 2 * sizeof(u32))) {
  8599. pr_debug_ratelimited(
  8600. "%s cannot read MSR entry (%u, 0x%08llx)\n",
  8601. __func__, i, gpa + i * sizeof(e));
  8602. return -EINVAL;
  8603. }
  8604. if (nested_vmx_store_msr_check(vcpu, &e)) {
  8605. pr_debug_ratelimited(
  8606. "%s check failed (%u, 0x%x, 0x%x)\n",
  8607. __func__, i, e.index, e.reserved);
  8608. return -EINVAL;
  8609. }
  8610. msr_info.host_initiated = false;
  8611. msr_info.index = e.index;
  8612. if (kvm_get_msr(vcpu, &msr_info)) {
  8613. pr_debug_ratelimited(
  8614. "%s cannot read MSR (%u, 0x%x)\n",
  8615. __func__, i, e.index);
  8616. return -EINVAL;
  8617. }
  8618. if (kvm_vcpu_write_guest(vcpu,
  8619. gpa + i * sizeof(e) +
  8620. offsetof(struct vmx_msr_entry, value),
  8621. &msr_info.data, sizeof(msr_info.data))) {
  8622. pr_debug_ratelimited(
  8623. "%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
  8624. __func__, i, e.index, msr_info.data);
  8625. return -EINVAL;
  8626. }
  8627. }
  8628. return 0;
  8629. }
  8630. static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val)
  8631. {
  8632. unsigned long invalid_mask;
  8633. invalid_mask = (~0ULL) << cpuid_maxphyaddr(vcpu);
  8634. return (val & invalid_mask) == 0;
  8635. }
  8636. /*
  8637. * Load guest's/host's cr3 at nested entry/exit. nested_ept is true if we are
  8638. * emulating VM entry into a guest with EPT enabled.
  8639. * Returns 0 on success, 1 on failure. Invalid state exit qualification code
  8640. * is assigned to entry_failure_code on failure.
  8641. */
  8642. static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept,
  8643. unsigned long *entry_failure_code)
  8644. {
  8645. if (cr3 != kvm_read_cr3(vcpu) || (!nested_ept && pdptrs_changed(vcpu))) {
  8646. if (!nested_cr3_valid(vcpu, cr3)) {
  8647. *entry_failure_code = ENTRY_FAIL_DEFAULT;
  8648. return 1;
  8649. }
  8650. /*
  8651. * If PAE paging and EPT are both on, CR3 is not used by the CPU and
  8652. * must not be dereferenced.
  8653. */
  8654. if (!is_long_mode(vcpu) && is_pae(vcpu) && is_paging(vcpu) &&
  8655. !nested_ept) {
  8656. if (!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) {
  8657. *entry_failure_code = ENTRY_FAIL_PDPTE;
  8658. return 1;
  8659. }
  8660. }
  8661. vcpu->arch.cr3 = cr3;
  8662. __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
  8663. }
  8664. kvm_mmu_reset_context(vcpu);
  8665. return 0;
  8666. }
  8667. /*
  8668. * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested
  8669. * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it
  8670. * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2
  8671. * guest in a way that will both be appropriate to L1's requests, and our
  8672. * needs. In addition to modifying the active vmcs (which is vmcs02), this
  8673. * function also has additional necessary side-effects, like setting various
  8674. * vcpu->arch fields.
  8675. * Returns 0 on success, 1 on failure. Invalid state exit qualification code
  8676. * is assigned to entry_failure_code on failure.
  8677. */
  8678. static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
  8679. unsigned long *entry_failure_code)
  8680. {
  8681. struct vcpu_vmx *vmx = to_vmx(vcpu);
  8682. u32 exec_control;
  8683. bool nested_ept_enabled = false;
  8684. vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
  8685. vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector);
  8686. vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector);
  8687. vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector);
  8688. vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector);
  8689. vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector);
  8690. vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector);
  8691. vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector);
  8692. vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit);
  8693. vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit);
  8694. vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit);
  8695. vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit);
  8696. vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit);
  8697. vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit);
  8698. vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit);
  8699. vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit);
  8700. vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit);
  8701. vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit);
  8702. vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes);
  8703. vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes);
  8704. vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes);
  8705. vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes);
  8706. vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes);
  8707. vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes);
  8708. vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes);
  8709. vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes);
  8710. vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base);
  8711. vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base);
  8712. vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base);
  8713. vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base);
  8714. vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base);
  8715. vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base);
  8716. vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base);
  8717. vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base);
  8718. vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base);
  8719. vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base);
  8720. if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) {
  8721. kvm_set_dr(vcpu, 7, vmcs12->guest_dr7);
  8722. vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl);
  8723. } else {
  8724. kvm_set_dr(vcpu, 7, vcpu->arch.dr7);
  8725. vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl);
  8726. }
  8727. vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
  8728. vmcs12->vm_entry_intr_info_field);
  8729. vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE,
  8730. vmcs12->vm_entry_exception_error_code);
  8731. vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
  8732. vmcs12->vm_entry_instruction_len);
  8733. vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,
  8734. vmcs12->guest_interruptibility_info);
  8735. vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs);
  8736. vmx_set_rflags(vcpu, vmcs12->guest_rflags);
  8737. vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
  8738. vmcs12->guest_pending_dbg_exceptions);
  8739. vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp);
  8740. vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip);
  8741. if (nested_cpu_has_xsaves(vmcs12))
  8742. vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap);
  8743. vmcs_write64(VMCS_LINK_POINTER, -1ull);
  8744. exec_control = vmcs12->pin_based_vm_exec_control;
  8745. /* Preemption timer setting is only taken from vmcs01. */
  8746. exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
  8747. exec_control |= vmcs_config.pin_based_exec_ctrl;
  8748. if (vmx->hv_deadline_tsc == -1)
  8749. exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
  8750. /* Posted interrupts setting is only taken from vmcs12. */
  8751. if (nested_cpu_has_posted_intr(vmcs12)) {
  8752. /*
  8753. * Note that we use L0's vector here and in
  8754. * vmx_deliver_nested_posted_interrupt.
  8755. */
  8756. vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv;
  8757. vmx->nested.pi_pending = false;
  8758. vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
  8759. vmcs_write64(POSTED_INTR_DESC_ADDR,
  8760. page_to_phys(vmx->nested.pi_desc_page) +
  8761. (unsigned long)(vmcs12->posted_intr_desc_addr &
  8762. (PAGE_SIZE - 1)));
  8763. } else
  8764. exec_control &= ~PIN_BASED_POSTED_INTR;
  8765. vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, exec_control);
  8766. vmx->nested.preemption_timer_expired = false;
  8767. if (nested_cpu_has_preemption_timer(vmcs12))
  8768. vmx_start_preemption_timer(vcpu);
  8769. /*
  8770. * Whether page-faults are trapped is determined by a combination of
  8771. * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF.
  8772. * If enable_ept, L0 doesn't care about page faults and we should
  8773. * set all of these to L1's desires. However, if !enable_ept, L0 does
  8774. * care about (at least some) page faults, and because it is not easy
  8775. * (if at all possible?) to merge L0 and L1's desires, we simply ask
  8776. * to exit on each and every L2 page fault. This is done by setting
  8777. * MASK=MATCH=0 and (see below) EB.PF=1.
  8778. * Note that below we don't need special code to set EB.PF beyond the
  8779. * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept,
  8780. * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when
  8781. * !enable_ept, EB.PF is 1, so the "or" will always be 1.
  8782. *
  8783. * A problem with this approach (when !enable_ept) is that L1 may be
  8784. * injected with more page faults than it asked for. This could have
  8785. * caused problems, but in practice existing hypervisors don't care.
  8786. * To fix this, we will need to emulate the PFEC checking (on the L1
  8787. * page tables), using walk_addr(), when injecting PFs to L1.
  8788. */
  8789. vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK,
  8790. enable_ept ? vmcs12->page_fault_error_code_mask : 0);
  8791. vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH,
  8792. enable_ept ? vmcs12->page_fault_error_code_match : 0);
  8793. if (cpu_has_secondary_exec_ctrls()) {
  8794. exec_control = vmx_secondary_exec_control(vmx);
  8795. /* Take the following fields only from vmcs12 */
  8796. exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
  8797. SECONDARY_EXEC_RDTSCP |
  8798. SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
  8799. SECONDARY_EXEC_APIC_REGISTER_VIRT);
  8800. if (nested_cpu_has(vmcs12,
  8801. CPU_BASED_ACTIVATE_SECONDARY_CONTROLS))
  8802. exec_control |= vmcs12->secondary_vm_exec_control;
  8803. if (exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) {
  8804. /*
  8805. * If translation failed, no matter: This feature asks
  8806. * to exit when accessing the given address, and if it
  8807. * can never be accessed, this feature won't do
  8808. * anything anyway.
  8809. */
  8810. if (!vmx->nested.apic_access_page)
  8811. exec_control &=
  8812. ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
  8813. else
  8814. vmcs_write64(APIC_ACCESS_ADDR,
  8815. page_to_phys(vmx->nested.apic_access_page));
  8816. } else if (!(nested_cpu_has_virt_x2apic_mode(vmcs12)) &&
  8817. cpu_need_virtualize_apic_accesses(&vmx->vcpu)) {
  8818. exec_control |=
  8819. SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
  8820. kvm_vcpu_reload_apic_access_page(vcpu);
  8821. }
  8822. if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) {
  8823. vmcs_write64(EOI_EXIT_BITMAP0,
  8824. vmcs12->eoi_exit_bitmap0);
  8825. vmcs_write64(EOI_EXIT_BITMAP1,
  8826. vmcs12->eoi_exit_bitmap1);
  8827. vmcs_write64(EOI_EXIT_BITMAP2,
  8828. vmcs12->eoi_exit_bitmap2);
  8829. vmcs_write64(EOI_EXIT_BITMAP3,
  8830. vmcs12->eoi_exit_bitmap3);
  8831. vmcs_write16(GUEST_INTR_STATUS,
  8832. vmcs12->guest_intr_status);
  8833. }
  8834. nested_ept_enabled = (exec_control & SECONDARY_EXEC_ENABLE_EPT) != 0;
  8835. vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
  8836. }
  8837. /*
  8838. * Set host-state according to L0's settings (vmcs12 is irrelevant here)
  8839. * Some constant fields are set here by vmx_set_constant_host_state().
  8840. * Other fields are different per CPU, and will be set later when
  8841. * vmx_vcpu_load() is called, and when vmx_save_host_state() is called.
  8842. */
  8843. vmx_set_constant_host_state(vmx);
  8844. /*
  8845. * Set the MSR load/store lists to match L0's settings.
  8846. */
  8847. vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
  8848. vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
  8849. vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host));
  8850. vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
  8851. vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest));
  8852. /*
  8853. * HOST_RSP is normally set correctly in vmx_vcpu_run() just before
  8854. * entry, but only if the current (host) sp changed from the value
  8855. * we wrote last (vmx->host_rsp). This cache is no longer relevant
  8856. * if we switch vmcs, and rather than hold a separate cache per vmcs,
  8857. * here we just force the write to happen on entry.
  8858. */
  8859. vmx->host_rsp = 0;
  8860. exec_control = vmx_exec_control(vmx); /* L0's desires */
  8861. exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
  8862. exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
  8863. exec_control &= ~CPU_BASED_TPR_SHADOW;
  8864. exec_control |= vmcs12->cpu_based_vm_exec_control;
  8865. if (exec_control & CPU_BASED_TPR_SHADOW) {
  8866. vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
  8867. page_to_phys(vmx->nested.virtual_apic_page));
  8868. vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold);
  8869. }
  8870. if (cpu_has_vmx_msr_bitmap() &&
  8871. exec_control & CPU_BASED_USE_MSR_BITMAPS &&
  8872. nested_vmx_merge_msr_bitmap(vcpu, vmcs12))
  8873. ; /* MSR_BITMAP will be set by following vmx_set_efer. */
  8874. else
  8875. exec_control &= ~CPU_BASED_USE_MSR_BITMAPS;
  8876. /*
  8877. * Merging of IO bitmap not currently supported.
  8878. * Rather, exit every time.
  8879. */
  8880. exec_control &= ~CPU_BASED_USE_IO_BITMAPS;
  8881. exec_control |= CPU_BASED_UNCOND_IO_EXITING;
  8882. vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control);
  8883. /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the
  8884. * bitwise-or of what L1 wants to trap for L2, and what we want to
  8885. * trap. Note that CR0.TS also needs updating - we do this later.
  8886. */
  8887. update_exception_bitmap(vcpu);
  8888. vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask;
  8889. vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
  8890. /* L2->L1 exit controls are emulated - the hardware exit is to L0 so
  8891. * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER
  8892. * bits are further modified by vmx_set_efer() below.
  8893. */
  8894. vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
  8895. /* vmcs12's VM_ENTRY_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE are
  8896. * emulated by vmx_set_efer(), below.
  8897. */
  8898. vm_entry_controls_init(vmx,
  8899. (vmcs12->vm_entry_controls & ~VM_ENTRY_LOAD_IA32_EFER &
  8900. ~VM_ENTRY_IA32E_MODE) |
  8901. (vmcs_config.vmentry_ctrl & ~VM_ENTRY_IA32E_MODE));
  8902. if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) {
  8903. vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat);
  8904. vcpu->arch.pat = vmcs12->guest_ia32_pat;
  8905. } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT)
  8906. vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
  8907. set_cr4_guest_host_mask(vmx);
  8908. if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)
  8909. vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs);
  8910. if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING)
  8911. vmcs_write64(TSC_OFFSET,
  8912. vcpu->arch.tsc_offset + vmcs12->tsc_offset);
  8913. else
  8914. vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
  8915. if (kvm_has_tsc_control)
  8916. decache_tsc_multiplier(vmx);
  8917. if (enable_vpid) {
  8918. /*
  8919. * There is no direct mapping between vpid02 and vpid12, the
  8920. * vpid02 is per-vCPU for L0 and reused while the value of
  8921. * vpid12 is changed w/ one invvpid during nested vmentry.
  8922. * The vpid12 is allocated by L1 for L2, so it will not
  8923. * influence global bitmap(for vpid01 and vpid02 allocation)
  8924. * even if spawn a lot of nested vCPUs.
  8925. */
  8926. if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) {
  8927. vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02);
  8928. if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
  8929. vmx->nested.last_vpid = vmcs12->virtual_processor_id;
  8930. __vmx_flush_tlb(vcpu, to_vmx(vcpu)->nested.vpid02);
  8931. }
  8932. } else {
  8933. vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
  8934. vmx_flush_tlb(vcpu);
  8935. }
  8936. }
  8937. if (nested_cpu_has_ept(vmcs12)) {
  8938. kvm_mmu_unload(vcpu);
  8939. nested_ept_init_mmu_context(vcpu);
  8940. }
  8941. /*
  8942. * This sets GUEST_CR0 to vmcs12->guest_cr0, with possibly a modified
  8943. * TS bit (for lazy fpu) and bits which we consider mandatory enabled.
  8944. * The CR0_READ_SHADOW is what L2 should have expected to read given
  8945. * the specifications by L1; It's not enough to take
  8946. * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we
  8947. * have more bits than L1 expected.
  8948. */
  8949. vmx_set_cr0(vcpu, vmcs12->guest_cr0);
  8950. vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12));
  8951. vmx_set_cr4(vcpu, vmcs12->guest_cr4);
  8952. vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12));
  8953. if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)
  8954. vcpu->arch.efer = vmcs12->guest_ia32_efer;
  8955. else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE)
  8956. vcpu->arch.efer |= (EFER_LMA | EFER_LME);
  8957. else
  8958. vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
  8959. /* Note: modifies VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */
  8960. vmx_set_efer(vcpu, vcpu->arch.efer);
  8961. /* Shadow page tables on either EPT or shadow page tables. */
  8962. if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_ept_enabled,
  8963. entry_failure_code))
  8964. return 1;
  8965. kvm_mmu_reset_context(vcpu);
  8966. if (!enable_ept)
  8967. vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested;
  8968. /*
  8969. * L1 may access the L2's PDPTR, so save them to construct vmcs12
  8970. */
  8971. if (enable_ept) {
  8972. vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
  8973. vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
  8974. vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
  8975. vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
  8976. }
  8977. kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->guest_rsp);
  8978. kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->guest_rip);
  8979. return 0;
  8980. }
  8981. /*
  8982. * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1
  8983. * for running an L2 nested guest.
  8984. */
  8985. static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
  8986. {
  8987. struct vmcs12 *vmcs12;
  8988. struct vcpu_vmx *vmx = to_vmx(vcpu);
  8989. int cpu;
  8990. struct loaded_vmcs *vmcs02;
  8991. bool ia32e;
  8992. u32 msr_entry_idx;
  8993. unsigned long exit_qualification;
  8994. if (!nested_vmx_check_permission(vcpu))
  8995. return 1;
  8996. if (!nested_vmx_check_vmcs12(vcpu))
  8997. goto out;
  8998. vmcs12 = get_vmcs12(vcpu);
  8999. if (enable_shadow_vmcs)
  9000. copy_shadow_to_vmcs12(vmx);
  9001. /*
  9002. * The nested entry process starts with enforcing various prerequisites
  9003. * on vmcs12 as required by the Intel SDM, and act appropriately when
  9004. * they fail: As the SDM explains, some conditions should cause the
  9005. * instruction to fail, while others will cause the instruction to seem
  9006. * to succeed, but return an EXIT_REASON_INVALID_STATE.
  9007. * To speed up the normal (success) code path, we should avoid checking
  9008. * for misconfigurations which will anyway be caught by the processor
  9009. * when using the merged vmcs02.
  9010. */
  9011. if (vmcs12->launch_state == launch) {
  9012. nested_vmx_failValid(vcpu,
  9013. launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS
  9014. : VMXERR_VMRESUME_NONLAUNCHED_VMCS);
  9015. goto out;
  9016. }
  9017. if (vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE &&
  9018. vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT) {
  9019. nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
  9020. goto out;
  9021. }
  9022. if (!nested_get_vmcs12_pages(vcpu, vmcs12)) {
  9023. nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
  9024. goto out;
  9025. }
  9026. if (nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12)) {
  9027. nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
  9028. goto out;
  9029. }
  9030. if (nested_vmx_check_apicv_controls(vcpu, vmcs12)) {
  9031. nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
  9032. goto out;
  9033. }
  9034. if (nested_vmx_check_msr_switch_controls(vcpu, vmcs12)) {
  9035. nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
  9036. goto out;
  9037. }
  9038. if (!vmx_control_verify(vmcs12->cpu_based_vm_exec_control,
  9039. vmx->nested.nested_vmx_procbased_ctls_low,
  9040. vmx->nested.nested_vmx_procbased_ctls_high) ||
  9041. !vmx_control_verify(vmcs12->secondary_vm_exec_control,
  9042. vmx->nested.nested_vmx_secondary_ctls_low,
  9043. vmx->nested.nested_vmx_secondary_ctls_high) ||
  9044. !vmx_control_verify(vmcs12->pin_based_vm_exec_control,
  9045. vmx->nested.nested_vmx_pinbased_ctls_low,
  9046. vmx->nested.nested_vmx_pinbased_ctls_high) ||
  9047. !vmx_control_verify(vmcs12->vm_exit_controls,
  9048. vmx->nested.nested_vmx_exit_ctls_low,
  9049. vmx->nested.nested_vmx_exit_ctls_high) ||
  9050. !vmx_control_verify(vmcs12->vm_entry_controls,
  9051. vmx->nested.nested_vmx_entry_ctls_low,
  9052. vmx->nested.nested_vmx_entry_ctls_high))
  9053. {
  9054. nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
  9055. goto out;
  9056. }
  9057. if (!nested_host_cr0_valid(vcpu, vmcs12->host_cr0) ||
  9058. !nested_host_cr4_valid(vcpu, vmcs12->host_cr4) ||
  9059. !nested_cr3_valid(vcpu, vmcs12->host_cr3)) {
  9060. nested_vmx_failValid(vcpu,
  9061. VMXERR_ENTRY_INVALID_HOST_STATE_FIELD);
  9062. goto out;
  9063. }
  9064. if (!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0) ||
  9065. !nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4)) {
  9066. nested_vmx_entry_failure(vcpu, vmcs12,
  9067. EXIT_REASON_INVALID_STATE, ENTRY_FAIL_DEFAULT);
  9068. return 1;
  9069. }
  9070. if (vmcs12->vmcs_link_pointer != -1ull) {
  9071. nested_vmx_entry_failure(vcpu, vmcs12,
  9072. EXIT_REASON_INVALID_STATE, ENTRY_FAIL_VMCS_LINK_PTR);
  9073. return 1;
  9074. }
  9075. /*
  9076. * If the load IA32_EFER VM-entry control is 1, the following checks
  9077. * are performed on the field for the IA32_EFER MSR:
  9078. * - Bits reserved in the IA32_EFER MSR must be 0.
  9079. * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of
  9080. * the IA-32e mode guest VM-exit control. It must also be identical
  9081. * to bit 8 (LME) if bit 31 in the CR0 field (corresponding to
  9082. * CR0.PG) is 1.
  9083. */
  9084. if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER) {
  9085. ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0;
  9086. if (!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer) ||
  9087. ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA) ||
  9088. ((vmcs12->guest_cr0 & X86_CR0_PG) &&
  9089. ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME))) {
  9090. nested_vmx_entry_failure(vcpu, vmcs12,
  9091. EXIT_REASON_INVALID_STATE, ENTRY_FAIL_DEFAULT);
  9092. return 1;
  9093. }
  9094. }
  9095. /*
  9096. * If the load IA32_EFER VM-exit control is 1, bits reserved in the
  9097. * IA32_EFER MSR must be 0 in the field for that register. In addition,
  9098. * the values of the LMA and LME bits in the field must each be that of
  9099. * the host address-space size VM-exit control.
  9100. */
  9101. if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) {
  9102. ia32e = (vmcs12->vm_exit_controls &
  9103. VM_EXIT_HOST_ADDR_SPACE_SIZE) != 0;
  9104. if (!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer) ||
  9105. ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA) ||
  9106. ia32e != !!(vmcs12->host_ia32_efer & EFER_LME)) {
  9107. nested_vmx_entry_failure(vcpu, vmcs12,
  9108. EXIT_REASON_INVALID_STATE, ENTRY_FAIL_DEFAULT);
  9109. return 1;
  9110. }
  9111. }
  9112. /*
  9113. * We're finally done with prerequisite checking, and can start with
  9114. * the nested entry.
  9115. */
  9116. vmcs02 = nested_get_current_vmcs02(vmx);
  9117. if (!vmcs02)
  9118. return -ENOMEM;
  9119. /*
  9120. * After this point, the trap flag no longer triggers a singlestep trap
  9121. * on the vm entry instructions. Don't call
  9122. * kvm_skip_emulated_instruction.
  9123. */
  9124. skip_emulated_instruction(vcpu);
  9125. enter_guest_mode(vcpu);
  9126. if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS))
  9127. vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
  9128. cpu = get_cpu();
  9129. vmx->loaded_vmcs = vmcs02;
  9130. vmx_vcpu_put(vcpu);
  9131. vmx_vcpu_load(vcpu, cpu);
  9132. vcpu->cpu = cpu;
  9133. put_cpu();
  9134. vmx_segment_cache_clear(vmx);
  9135. if (prepare_vmcs02(vcpu, vmcs12, &exit_qualification)) {
  9136. leave_guest_mode(vcpu);
  9137. vmx_load_vmcs01(vcpu);
  9138. nested_vmx_entry_failure(vcpu, vmcs12,
  9139. EXIT_REASON_INVALID_STATE, exit_qualification);
  9140. return 1;
  9141. }
  9142. msr_entry_idx = nested_vmx_load_msr(vcpu,
  9143. vmcs12->vm_entry_msr_load_addr,
  9144. vmcs12->vm_entry_msr_load_count);
  9145. if (msr_entry_idx) {
  9146. leave_guest_mode(vcpu);
  9147. vmx_load_vmcs01(vcpu);
  9148. nested_vmx_entry_failure(vcpu, vmcs12,
  9149. EXIT_REASON_MSR_LOAD_FAIL, msr_entry_idx);
  9150. return 1;
  9151. }
  9152. vmcs12->launch_state = 1;
  9153. if (vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT)
  9154. return kvm_vcpu_halt(vcpu);
  9155. vmx->nested.nested_run_pending = 1;
  9156. /*
  9157. * Note no nested_vmx_succeed or nested_vmx_fail here. At this point
  9158. * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet
  9159. * returned as far as L1 is concerned. It will only return (and set
  9160. * the success flag) when L2 exits (see nested_vmx_vmexit()).
  9161. */
  9162. return 1;
  9163. out:
  9164. return kvm_skip_emulated_instruction(vcpu);
  9165. }
  9166. /*
  9167. * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date
  9168. * because L2 may have changed some cr0 bits directly (CRO_GUEST_HOST_MASK).
  9169. * This function returns the new value we should put in vmcs12.guest_cr0.
  9170. * It's not enough to just return the vmcs02 GUEST_CR0. Rather,
  9171. * 1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now
  9172. * available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0
  9173. * didn't trap the bit, because if L1 did, so would L0).
  9174. * 2. Bits that L1 asked to trap (and therefore L0 also did) could not have
  9175. * been modified by L2, and L1 knows it. So just leave the old value of
  9176. * the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0
  9177. * isn't relevant, because if L0 traps this bit it can set it to anything.
  9178. * 3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have
  9179. * changed these bits, and therefore they need to be updated, but L0
  9180. * didn't necessarily allow them to be changed in GUEST_CR0 - and rather
  9181. * put them in vmcs02 CR0_READ_SHADOW. So take these bits from there.
  9182. */
  9183. static inline unsigned long
  9184. vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
  9185. {
  9186. return
  9187. /*1*/ (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) |
  9188. /*2*/ (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) |
  9189. /*3*/ (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask |
  9190. vcpu->arch.cr0_guest_owned_bits));
  9191. }
  9192. static inline unsigned long
  9193. vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
  9194. {
  9195. return
  9196. /*1*/ (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) |
  9197. /*2*/ (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) |
  9198. /*3*/ (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask |
  9199. vcpu->arch.cr4_guest_owned_bits));
  9200. }
  9201. static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu,
  9202. struct vmcs12 *vmcs12)
  9203. {
  9204. u32 idt_vectoring;
  9205. unsigned int nr;
  9206. if (vcpu->arch.exception.pending && vcpu->arch.exception.reinject) {
  9207. nr = vcpu->arch.exception.nr;
  9208. idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
  9209. if (kvm_exception_is_soft(nr)) {
  9210. vmcs12->vm_exit_instruction_len =
  9211. vcpu->arch.event_exit_inst_len;
  9212. idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION;
  9213. } else
  9214. idt_vectoring |= INTR_TYPE_HARD_EXCEPTION;
  9215. if (vcpu->arch.exception.has_error_code) {
  9216. idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK;
  9217. vmcs12->idt_vectoring_error_code =
  9218. vcpu->arch.exception.error_code;
  9219. }
  9220. vmcs12->idt_vectoring_info_field = idt_vectoring;
  9221. } else if (vcpu->arch.nmi_injected) {
  9222. vmcs12->idt_vectoring_info_field =
  9223. INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR;
  9224. } else if (vcpu->arch.interrupt.pending) {
  9225. nr = vcpu->arch.interrupt.nr;
  9226. idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
  9227. if (vcpu->arch.interrupt.soft) {
  9228. idt_vectoring |= INTR_TYPE_SOFT_INTR;
  9229. vmcs12->vm_entry_instruction_len =
  9230. vcpu->arch.event_exit_inst_len;
  9231. } else
  9232. idt_vectoring |= INTR_TYPE_EXT_INTR;
  9233. vmcs12->idt_vectoring_info_field = idt_vectoring;
  9234. }
  9235. }
  9236. static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr)
  9237. {
  9238. struct vcpu_vmx *vmx = to_vmx(vcpu);
  9239. if (nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) &&
  9240. vmx->nested.preemption_timer_expired) {
  9241. if (vmx->nested.nested_run_pending)
  9242. return -EBUSY;
  9243. nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0);
  9244. return 0;
  9245. }
  9246. if (vcpu->arch.nmi_pending && nested_exit_on_nmi(vcpu)) {
  9247. if (vmx->nested.nested_run_pending ||
  9248. vcpu->arch.interrupt.pending)
  9249. return -EBUSY;
  9250. nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
  9251. NMI_VECTOR | INTR_TYPE_NMI_INTR |
  9252. INTR_INFO_VALID_MASK, 0);
  9253. /*
  9254. * The NMI-triggered VM exit counts as injection:
  9255. * clear this one and block further NMIs.
  9256. */
  9257. vcpu->arch.nmi_pending = 0;
  9258. vmx_set_nmi_mask(vcpu, true);
  9259. return 0;
  9260. }
  9261. if ((kvm_cpu_has_interrupt(vcpu) || external_intr) &&
  9262. nested_exit_on_intr(vcpu)) {
  9263. if (vmx->nested.nested_run_pending)
  9264. return -EBUSY;
  9265. nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0);
  9266. return 0;
  9267. }
  9268. return vmx_complete_nested_posted_interrupt(vcpu);
  9269. }
  9270. static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu)
  9271. {
  9272. ktime_t remaining =
  9273. hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer);
  9274. u64 value;
  9275. if (ktime_to_ns(remaining) <= 0)
  9276. return 0;
  9277. value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz;
  9278. do_div(value, 1000000);
  9279. return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
  9280. }
  9281. /*
  9282. * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits
  9283. * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12),
  9284. * and this function updates it to reflect the changes to the guest state while
  9285. * L2 was running (and perhaps made some exits which were handled directly by L0
  9286. * without going back to L1), and to reflect the exit reason.
  9287. * Note that we do not have to copy here all VMCS fields, just those that
  9288. * could have changed by the L2 guest or the exit - i.e., the guest-state and
  9289. * exit-information fields only. Other fields are modified by L1 with VMWRITE,
  9290. * which already writes to vmcs12 directly.
  9291. */
  9292. static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
  9293. u32 exit_reason, u32 exit_intr_info,
  9294. unsigned long exit_qualification)
  9295. {
  9296. /* update guest state fields: */
  9297. vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12);
  9298. vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12);
  9299. vmcs12->guest_rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
  9300. vmcs12->guest_rip = kvm_register_read(vcpu, VCPU_REGS_RIP);
  9301. vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS);
  9302. vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR);
  9303. vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR);
  9304. vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR);
  9305. vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR);
  9306. vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR);
  9307. vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR);
  9308. vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR);
  9309. vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR);
  9310. vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT);
  9311. vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT);
  9312. vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT);
  9313. vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT);
  9314. vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT);
  9315. vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT);
  9316. vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT);
  9317. vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT);
  9318. vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT);
  9319. vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT);
  9320. vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES);
  9321. vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES);
  9322. vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES);
  9323. vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES);
  9324. vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES);
  9325. vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES);
  9326. vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES);
  9327. vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES);
  9328. vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE);
  9329. vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE);
  9330. vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE);
  9331. vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE);
  9332. vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE);
  9333. vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE);
  9334. vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE);
  9335. vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE);
  9336. vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE);
  9337. vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE);
  9338. vmcs12->guest_interruptibility_info =
  9339. vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
  9340. vmcs12->guest_pending_dbg_exceptions =
  9341. vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);
  9342. if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
  9343. vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT;
  9344. else
  9345. vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE;
  9346. if (nested_cpu_has_preemption_timer(vmcs12)) {
  9347. if (vmcs12->vm_exit_controls &
  9348. VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)
  9349. vmcs12->vmx_preemption_timer_value =
  9350. vmx_get_preemption_timer_value(vcpu);
  9351. hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer);
  9352. }
  9353. /*
  9354. * In some cases (usually, nested EPT), L2 is allowed to change its
  9355. * own CR3 without exiting. If it has changed it, we must keep it.
  9356. * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined
  9357. * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12.
  9358. *
  9359. * Additionally, restore L2's PDPTR to vmcs12.
  9360. */
  9361. if (enable_ept) {
  9362. vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3);
  9363. vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0);
  9364. vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1);
  9365. vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2);
  9366. vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3);
  9367. }
  9368. if (nested_cpu_has_ept(vmcs12))
  9369. vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS);
  9370. if (nested_cpu_has_vid(vmcs12))
  9371. vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS);
  9372. vmcs12->vm_entry_controls =
  9373. (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) |
  9374. (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE);
  9375. if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) {
  9376. kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7);
  9377. vmcs12->guest_ia32_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
  9378. }
  9379. /* TODO: These cannot have changed unless we have MSR bitmaps and
  9380. * the relevant bit asks not to trap the change */
  9381. if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT)
  9382. vmcs12->guest_ia32_pat = vmcs_read64(GUEST_IA32_PAT);
  9383. if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER)
  9384. vmcs12->guest_ia32_efer = vcpu->arch.efer;
  9385. vmcs12->guest_sysenter_cs = vmcs_read32(GUEST_SYSENTER_CS);
  9386. vmcs12->guest_sysenter_esp = vmcs_readl(GUEST_SYSENTER_ESP);
  9387. vmcs12->guest_sysenter_eip = vmcs_readl(GUEST_SYSENTER_EIP);
  9388. if (kvm_mpx_supported())
  9389. vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS);
  9390. if (nested_cpu_has_xsaves(vmcs12))
  9391. vmcs12->xss_exit_bitmap = vmcs_read64(XSS_EXIT_BITMAP);
  9392. /* update exit information fields: */
  9393. vmcs12->vm_exit_reason = exit_reason;
  9394. vmcs12->exit_qualification = exit_qualification;
  9395. vmcs12->vm_exit_intr_info = exit_intr_info;
  9396. if ((vmcs12->vm_exit_intr_info &
  9397. (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) ==
  9398. (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK))
  9399. vmcs12->vm_exit_intr_error_code =
  9400. vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
  9401. vmcs12->idt_vectoring_info_field = 0;
  9402. vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
  9403. vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
  9404. if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) {
  9405. /* vm_entry_intr_info_field is cleared on exit. Emulate this
  9406. * instead of reading the real value. */
  9407. vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK;
  9408. /*
  9409. * Transfer the event that L0 or L1 may wanted to inject into
  9410. * L2 to IDT_VECTORING_INFO_FIELD.
  9411. */
  9412. vmcs12_save_pending_event(vcpu, vmcs12);
  9413. }
  9414. /*
  9415. * Drop what we picked up for L2 via vmx_complete_interrupts. It is
  9416. * preserved above and would only end up incorrectly in L1.
  9417. */
  9418. vcpu->arch.nmi_injected = false;
  9419. kvm_clear_exception_queue(vcpu);
  9420. kvm_clear_interrupt_queue(vcpu);
  9421. }
  9422. /*
  9423. * A part of what we need to when the nested L2 guest exits and we want to
  9424. * run its L1 parent, is to reset L1's guest state to the host state specified
  9425. * in vmcs12.
  9426. * This function is to be called not only on normal nested exit, but also on
  9427. * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry
  9428. * Failures During or After Loading Guest State").
  9429. * This function should be called when the active VMCS is L1's (vmcs01).
  9430. */
  9431. static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
  9432. struct vmcs12 *vmcs12)
  9433. {
  9434. struct kvm_segment seg;
  9435. unsigned long entry_failure_code;
  9436. if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER)
  9437. vcpu->arch.efer = vmcs12->host_ia32_efer;
  9438. else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
  9439. vcpu->arch.efer |= (EFER_LMA | EFER_LME);
  9440. else
  9441. vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
  9442. vmx_set_efer(vcpu, vcpu->arch.efer);
  9443. kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->host_rsp);
  9444. kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->host_rip);
  9445. vmx_set_rflags(vcpu, X86_EFLAGS_FIXED);
  9446. /*
  9447. * Note that calling vmx_set_cr0 is important, even if cr0 hasn't
  9448. * actually changed, because it depends on the current state of
  9449. * fpu_active (which may have changed).
  9450. * Note that vmx_set_cr0 refers to efer set above.
  9451. */
  9452. vmx_set_cr0(vcpu, vmcs12->host_cr0);
  9453. /*
  9454. * If we did fpu_activate()/fpu_deactivate() during L2's run, we need
  9455. * to apply the same changes to L1's vmcs. We just set cr0 correctly,
  9456. * but we also need to update cr0_guest_host_mask and exception_bitmap.
  9457. */
  9458. update_exception_bitmap(vcpu);
  9459. vcpu->arch.cr0_guest_owned_bits = (vcpu->fpu_active ? X86_CR0_TS : 0);
  9460. vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
  9461. /*
  9462. * Note that CR4_GUEST_HOST_MASK is already set in the original vmcs01
  9463. * (KVM doesn't change it)- no reason to call set_cr4_guest_host_mask();
  9464. */
  9465. vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
  9466. kvm_set_cr4(vcpu, vmcs12->host_cr4);
  9467. nested_ept_uninit_mmu_context(vcpu);
  9468. /*
  9469. * Only PDPTE load can fail as the value of cr3 was checked on entry and
  9470. * couldn't have changed.
  9471. */
  9472. if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &entry_failure_code))
  9473. nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL);
  9474. if (!enable_ept)
  9475. vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
  9476. if (enable_vpid) {
  9477. /*
  9478. * Trivially support vpid by letting L2s share their parent
  9479. * L1's vpid. TODO: move to a more elaborate solution, giving
  9480. * each L2 its own vpid and exposing the vpid feature to L1.
  9481. */
  9482. vmx_flush_tlb(vcpu);
  9483. }
  9484. vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs);
  9485. vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp);
  9486. vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip);
  9487. vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base);
  9488. vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base);
  9489. /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1. */
  9490. if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS)
  9491. vmcs_write64(GUEST_BNDCFGS, 0);
  9492. if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) {
  9493. vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat);
  9494. vcpu->arch.pat = vmcs12->host_ia32_pat;
  9495. }
  9496. if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
  9497. vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL,
  9498. vmcs12->host_ia32_perf_global_ctrl);
  9499. /* Set L1 segment info according to Intel SDM
  9500. 27.5.2 Loading Host Segment and Descriptor-Table Registers */
  9501. seg = (struct kvm_segment) {
  9502. .base = 0,
  9503. .limit = 0xFFFFFFFF,
  9504. .selector = vmcs12->host_cs_selector,
  9505. .type = 11,
  9506. .present = 1,
  9507. .s = 1,
  9508. .g = 1
  9509. };
  9510. if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
  9511. seg.l = 1;
  9512. else
  9513. seg.db = 1;
  9514. vmx_set_segment(vcpu, &seg, VCPU_SREG_CS);
  9515. seg = (struct kvm_segment) {
  9516. .base = 0,
  9517. .limit = 0xFFFFFFFF,
  9518. .type = 3,
  9519. .present = 1,
  9520. .s = 1,
  9521. .db = 1,
  9522. .g = 1
  9523. };
  9524. seg.selector = vmcs12->host_ds_selector;
  9525. vmx_set_segment(vcpu, &seg, VCPU_SREG_DS);
  9526. seg.selector = vmcs12->host_es_selector;
  9527. vmx_set_segment(vcpu, &seg, VCPU_SREG_ES);
  9528. seg.selector = vmcs12->host_ss_selector;
  9529. vmx_set_segment(vcpu, &seg, VCPU_SREG_SS);
  9530. seg.selector = vmcs12->host_fs_selector;
  9531. seg.base = vmcs12->host_fs_base;
  9532. vmx_set_segment(vcpu, &seg, VCPU_SREG_FS);
  9533. seg.selector = vmcs12->host_gs_selector;
  9534. seg.base = vmcs12->host_gs_base;
  9535. vmx_set_segment(vcpu, &seg, VCPU_SREG_GS);
  9536. seg = (struct kvm_segment) {
  9537. .base = vmcs12->host_tr_base,
  9538. .limit = 0x67,
  9539. .selector = vmcs12->host_tr_selector,
  9540. .type = 11,
  9541. .present = 1
  9542. };
  9543. vmx_set_segment(vcpu, &seg, VCPU_SREG_TR);
  9544. kvm_set_dr(vcpu, 7, 0x400);
  9545. vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
  9546. if (cpu_has_vmx_msr_bitmap())
  9547. vmx_set_msr_bitmap(vcpu);
  9548. if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr,
  9549. vmcs12->vm_exit_msr_load_count))
  9550. nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
  9551. }
  9552. /*
  9553. * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1
  9554. * and modify vmcs12 to make it see what it would expect to see there if
  9555. * L2 was its real guest. Must only be called when in L2 (is_guest_mode())
  9556. */
  9557. static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
  9558. u32 exit_intr_info,
  9559. unsigned long exit_qualification)
  9560. {
  9561. struct vcpu_vmx *vmx = to_vmx(vcpu);
  9562. struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
  9563. u32 vm_inst_error = 0;
  9564. /* trying to cancel vmlaunch/vmresume is a bug */
  9565. WARN_ON_ONCE(vmx->nested.nested_run_pending);
  9566. leave_guest_mode(vcpu);
  9567. prepare_vmcs12(vcpu, vmcs12, exit_reason, exit_intr_info,
  9568. exit_qualification);
  9569. if (nested_vmx_store_msr(vcpu, vmcs12->vm_exit_msr_store_addr,
  9570. vmcs12->vm_exit_msr_store_count))
  9571. nested_vmx_abort(vcpu, VMX_ABORT_SAVE_GUEST_MSR_FAIL);
  9572. if (unlikely(vmx->fail))
  9573. vm_inst_error = vmcs_read32(VM_INSTRUCTION_ERROR);
  9574. vmx_load_vmcs01(vcpu);
  9575. if ((exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT)
  9576. && nested_exit_intr_ack_set(vcpu)) {
  9577. int irq = kvm_cpu_get_interrupt(vcpu);
  9578. WARN_ON(irq < 0);
  9579. vmcs12->vm_exit_intr_info = irq |
  9580. INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR;
  9581. }
  9582. trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason,
  9583. vmcs12->exit_qualification,
  9584. vmcs12->idt_vectoring_info_field,
  9585. vmcs12->vm_exit_intr_info,
  9586. vmcs12->vm_exit_intr_error_code,
  9587. KVM_ISA_VMX);
  9588. vm_entry_controls_reset_shadow(vmx);
  9589. vm_exit_controls_reset_shadow(vmx);
  9590. vmx_segment_cache_clear(vmx);
  9591. /* if no vmcs02 cache requested, remove the one we used */
  9592. if (VMCS02_POOL_SIZE == 0)
  9593. nested_free_vmcs02(vmx, vmx->nested.current_vmptr);
  9594. load_vmcs12_host_state(vcpu, vmcs12);
  9595. /* Update any VMCS fields that might have changed while L2 ran */
  9596. vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
  9597. vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
  9598. vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
  9599. if (vmx->hv_deadline_tsc == -1)
  9600. vmcs_clear_bits(PIN_BASED_VM_EXEC_CONTROL,
  9601. PIN_BASED_VMX_PREEMPTION_TIMER);
  9602. else
  9603. vmcs_set_bits(PIN_BASED_VM_EXEC_CONTROL,
  9604. PIN_BASED_VMX_PREEMPTION_TIMER);
  9605. if (kvm_has_tsc_control)
  9606. decache_tsc_multiplier(vmx);
  9607. if (vmx->nested.change_vmcs01_virtual_x2apic_mode) {
  9608. vmx->nested.change_vmcs01_virtual_x2apic_mode = false;
  9609. vmx_set_virtual_x2apic_mode(vcpu,
  9610. vcpu->arch.apic_base & X2APIC_ENABLE);
  9611. }
  9612. /* This is needed for same reason as it was needed in prepare_vmcs02 */
  9613. vmx->host_rsp = 0;
  9614. /* Unpin physical memory we referred to in vmcs02 */
  9615. if (vmx->nested.apic_access_page) {
  9616. nested_release_page(vmx->nested.apic_access_page);
  9617. vmx->nested.apic_access_page = NULL;
  9618. }
  9619. if (vmx->nested.virtual_apic_page) {
  9620. nested_release_page(vmx->nested.virtual_apic_page);
  9621. vmx->nested.virtual_apic_page = NULL;
  9622. }
  9623. if (vmx->nested.pi_desc_page) {
  9624. kunmap(vmx->nested.pi_desc_page);
  9625. nested_release_page(vmx->nested.pi_desc_page);
  9626. vmx->nested.pi_desc_page = NULL;
  9627. vmx->nested.pi_desc = NULL;
  9628. }
  9629. /*
  9630. * We are now running in L2, mmu_notifier will force to reload the
  9631. * page's hpa for L2 vmcs. Need to reload it for L1 before entering L1.
  9632. */
  9633. kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
  9634. /*
  9635. * Exiting from L2 to L1, we're now back to L1 which thinks it just
  9636. * finished a VMLAUNCH or VMRESUME instruction, so we need to set the
  9637. * success or failure flag accordingly.
  9638. */
  9639. if (unlikely(vmx->fail)) {
  9640. vmx->fail = 0;
  9641. nested_vmx_failValid(vcpu, vm_inst_error);
  9642. } else
  9643. nested_vmx_succeed(vcpu);
  9644. if (enable_shadow_vmcs)
  9645. vmx->nested.sync_shadow_vmcs = true;
  9646. /* in case we halted in L2 */
  9647. vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
  9648. }
  9649. /*
  9650. * Forcibly leave nested mode in order to be able to reset the VCPU later on.
  9651. */
  9652. static void vmx_leave_nested(struct kvm_vcpu *vcpu)
  9653. {
  9654. if (is_guest_mode(vcpu))
  9655. nested_vmx_vmexit(vcpu, -1, 0, 0);
  9656. free_nested(to_vmx(vcpu));
  9657. }
  9658. /*
  9659. * L1's failure to enter L2 is a subset of a normal exit, as explained in
  9660. * 23.7 "VM-entry failures during or after loading guest state" (this also
  9661. * lists the acceptable exit-reason and exit-qualification parameters).
  9662. * It should only be called before L2 actually succeeded to run, and when
  9663. * vmcs01 is current (it doesn't leave_guest_mode() or switch vmcss).
  9664. */
  9665. static void nested_vmx_entry_failure(struct kvm_vcpu *vcpu,
  9666. struct vmcs12 *vmcs12,
  9667. u32 reason, unsigned long qualification)
  9668. {
  9669. load_vmcs12_host_state(vcpu, vmcs12);
  9670. vmcs12->vm_exit_reason = reason | VMX_EXIT_REASONS_FAILED_VMENTRY;
  9671. vmcs12->exit_qualification = qualification;
  9672. nested_vmx_succeed(vcpu);
  9673. if (enable_shadow_vmcs)
  9674. to_vmx(vcpu)->nested.sync_shadow_vmcs = true;
  9675. }
  9676. static int vmx_check_intercept(struct kvm_vcpu *vcpu,
  9677. struct x86_instruction_info *info,
  9678. enum x86_intercept_stage stage)
  9679. {
  9680. return X86EMUL_CONTINUE;
  9681. }
  9682. #ifdef CONFIG_X86_64
  9683. /* (a << shift) / divisor, return 1 if overflow otherwise 0 */
  9684. static inline int u64_shl_div_u64(u64 a, unsigned int shift,
  9685. u64 divisor, u64 *result)
  9686. {
  9687. u64 low = a << shift, high = a >> (64 - shift);
  9688. /* To avoid the overflow on divq */
  9689. if (high >= divisor)
  9690. return 1;
  9691. /* Low hold the result, high hold rem which is discarded */
  9692. asm("divq %2\n\t" : "=a" (low), "=d" (high) :
  9693. "rm" (divisor), "0" (low), "1" (high));
  9694. *result = low;
  9695. return 0;
  9696. }
  9697. static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc)
  9698. {
  9699. struct vcpu_vmx *vmx = to_vmx(vcpu);
  9700. u64 tscl = rdtsc();
  9701. u64 guest_tscl = kvm_read_l1_tsc(vcpu, tscl);
  9702. u64 delta_tsc = max(guest_deadline_tsc, guest_tscl) - guest_tscl;
  9703. /* Convert to host delta tsc if tsc scaling is enabled */
  9704. if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio &&
  9705. u64_shl_div_u64(delta_tsc,
  9706. kvm_tsc_scaling_ratio_frac_bits,
  9707. vcpu->arch.tsc_scaling_ratio,
  9708. &delta_tsc))
  9709. return -ERANGE;
  9710. /*
  9711. * If the delta tsc can't fit in the 32 bit after the multi shift,
  9712. * we can't use the preemption timer.
  9713. * It's possible that it fits on later vmentries, but checking
  9714. * on every vmentry is costly so we just use an hrtimer.
  9715. */
  9716. if (delta_tsc >> (cpu_preemption_timer_multi + 32))
  9717. return -ERANGE;
  9718. vmx->hv_deadline_tsc = tscl + delta_tsc;
  9719. vmcs_set_bits(PIN_BASED_VM_EXEC_CONTROL,
  9720. PIN_BASED_VMX_PREEMPTION_TIMER);
  9721. return 0;
  9722. }
  9723. static void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu)
  9724. {
  9725. struct vcpu_vmx *vmx = to_vmx(vcpu);
  9726. vmx->hv_deadline_tsc = -1;
  9727. vmcs_clear_bits(PIN_BASED_VM_EXEC_CONTROL,
  9728. PIN_BASED_VMX_PREEMPTION_TIMER);
  9729. }
  9730. #endif
  9731. static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu)
  9732. {
  9733. if (ple_gap)
  9734. shrink_ple_window(vcpu);
  9735. }
  9736. static void vmx_slot_enable_log_dirty(struct kvm *kvm,
  9737. struct kvm_memory_slot *slot)
  9738. {
  9739. kvm_mmu_slot_leaf_clear_dirty(kvm, slot);
  9740. kvm_mmu_slot_largepage_remove_write_access(kvm, slot);
  9741. }
  9742. static void vmx_slot_disable_log_dirty(struct kvm *kvm,
  9743. struct kvm_memory_slot *slot)
  9744. {
  9745. kvm_mmu_slot_set_dirty(kvm, slot);
  9746. }
  9747. static void vmx_flush_log_dirty(struct kvm *kvm)
  9748. {
  9749. kvm_flush_pml_buffers(kvm);
  9750. }
  9751. static void vmx_enable_log_dirty_pt_masked(struct kvm *kvm,
  9752. struct kvm_memory_slot *memslot,
  9753. gfn_t offset, unsigned long mask)
  9754. {
  9755. kvm_mmu_clear_dirty_pt_masked(kvm, memslot, offset, mask);
  9756. }
  9757. /*
  9758. * This routine does the following things for vCPU which is going
  9759. * to be blocked if VT-d PI is enabled.
  9760. * - Store the vCPU to the wakeup list, so when interrupts happen
  9761. * we can find the right vCPU to wake up.
  9762. * - Change the Posted-interrupt descriptor as below:
  9763. * 'NDST' <-- vcpu->pre_pcpu
  9764. * 'NV' <-- POSTED_INTR_WAKEUP_VECTOR
  9765. * - If 'ON' is set during this process, which means at least one
  9766. * interrupt is posted for this vCPU, we cannot block it, in
  9767. * this case, return 1, otherwise, return 0.
  9768. *
  9769. */
  9770. static int pi_pre_block(struct kvm_vcpu *vcpu)
  9771. {
  9772. unsigned long flags;
  9773. unsigned int dest;
  9774. struct pi_desc old, new;
  9775. struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
  9776. if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
  9777. !irq_remapping_cap(IRQ_POSTING_CAP) ||
  9778. !kvm_vcpu_apicv_active(vcpu))
  9779. return 0;
  9780. vcpu->pre_pcpu = vcpu->cpu;
  9781. spin_lock_irqsave(&per_cpu(blocked_vcpu_on_cpu_lock,
  9782. vcpu->pre_pcpu), flags);
  9783. list_add_tail(&vcpu->blocked_vcpu_list,
  9784. &per_cpu(blocked_vcpu_on_cpu,
  9785. vcpu->pre_pcpu));
  9786. spin_unlock_irqrestore(&per_cpu(blocked_vcpu_on_cpu_lock,
  9787. vcpu->pre_pcpu), flags);
  9788. do {
  9789. old.control = new.control = pi_desc->control;
  9790. /*
  9791. * We should not block the vCPU if
  9792. * an interrupt is posted for it.
  9793. */
  9794. if (pi_test_on(pi_desc) == 1) {
  9795. spin_lock_irqsave(&per_cpu(blocked_vcpu_on_cpu_lock,
  9796. vcpu->pre_pcpu), flags);
  9797. list_del(&vcpu->blocked_vcpu_list);
  9798. spin_unlock_irqrestore(
  9799. &per_cpu(blocked_vcpu_on_cpu_lock,
  9800. vcpu->pre_pcpu), flags);
  9801. vcpu->pre_pcpu = -1;
  9802. return 1;
  9803. }
  9804. WARN((pi_desc->sn == 1),
  9805. "Warning: SN field of posted-interrupts "
  9806. "is set before blocking\n");
  9807. /*
  9808. * Since vCPU can be preempted during this process,
  9809. * vcpu->cpu could be different with pre_pcpu, we
  9810. * need to set pre_pcpu as the destination of wakeup
  9811. * notification event, then we can find the right vCPU
  9812. * to wakeup in wakeup handler if interrupts happen
  9813. * when the vCPU is in blocked state.
  9814. */
  9815. dest = cpu_physical_id(vcpu->pre_pcpu);
  9816. if (x2apic_enabled())
  9817. new.ndst = dest;
  9818. else
  9819. new.ndst = (dest << 8) & 0xFF00;
  9820. /* set 'NV' to 'wakeup vector' */
  9821. new.nv = POSTED_INTR_WAKEUP_VECTOR;
  9822. } while (cmpxchg(&pi_desc->control, old.control,
  9823. new.control) != old.control);
  9824. return 0;
  9825. }
  9826. static int vmx_pre_block(struct kvm_vcpu *vcpu)
  9827. {
  9828. if (pi_pre_block(vcpu))
  9829. return 1;
  9830. if (kvm_lapic_hv_timer_in_use(vcpu))
  9831. kvm_lapic_switch_to_sw_timer(vcpu);
  9832. return 0;
  9833. }
  9834. static void pi_post_block(struct kvm_vcpu *vcpu)
  9835. {
  9836. struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
  9837. struct pi_desc old, new;
  9838. unsigned int dest;
  9839. unsigned long flags;
  9840. if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
  9841. !irq_remapping_cap(IRQ_POSTING_CAP) ||
  9842. !kvm_vcpu_apicv_active(vcpu))
  9843. return;
  9844. do {
  9845. old.control = new.control = pi_desc->control;
  9846. dest = cpu_physical_id(vcpu->cpu);
  9847. if (x2apic_enabled())
  9848. new.ndst = dest;
  9849. else
  9850. new.ndst = (dest << 8) & 0xFF00;
  9851. /* Allow posting non-urgent interrupts */
  9852. new.sn = 0;
  9853. /* set 'NV' to 'notification vector' */
  9854. new.nv = POSTED_INTR_VECTOR;
  9855. } while (cmpxchg(&pi_desc->control, old.control,
  9856. new.control) != old.control);
  9857. if(vcpu->pre_pcpu != -1) {
  9858. spin_lock_irqsave(
  9859. &per_cpu(blocked_vcpu_on_cpu_lock,
  9860. vcpu->pre_pcpu), flags);
  9861. list_del(&vcpu->blocked_vcpu_list);
  9862. spin_unlock_irqrestore(
  9863. &per_cpu(blocked_vcpu_on_cpu_lock,
  9864. vcpu->pre_pcpu), flags);
  9865. vcpu->pre_pcpu = -1;
  9866. }
  9867. }
  9868. static void vmx_post_block(struct kvm_vcpu *vcpu)
  9869. {
  9870. if (kvm_x86_ops->set_hv_timer)
  9871. kvm_lapic_switch_to_hv_timer(vcpu);
  9872. pi_post_block(vcpu);
  9873. }
  9874. /*
  9875. * vmx_update_pi_irte - set IRTE for Posted-Interrupts
  9876. *
  9877. * @kvm: kvm
  9878. * @host_irq: host irq of the interrupt
  9879. * @guest_irq: gsi of the interrupt
  9880. * @set: set or unset PI
  9881. * returns 0 on success, < 0 on failure
  9882. */
  9883. static int vmx_update_pi_irte(struct kvm *kvm, unsigned int host_irq,
  9884. uint32_t guest_irq, bool set)
  9885. {
  9886. struct kvm_kernel_irq_routing_entry *e;
  9887. struct kvm_irq_routing_table *irq_rt;
  9888. struct kvm_lapic_irq irq;
  9889. struct kvm_vcpu *vcpu;
  9890. struct vcpu_data vcpu_info;
  9891. int idx, ret = -EINVAL;
  9892. if (!kvm_arch_has_assigned_device(kvm) ||
  9893. !irq_remapping_cap(IRQ_POSTING_CAP) ||
  9894. !kvm_vcpu_apicv_active(kvm->vcpus[0]))
  9895. return 0;
  9896. idx = srcu_read_lock(&kvm->irq_srcu);
  9897. irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
  9898. BUG_ON(guest_irq >= irq_rt->nr_rt_entries);
  9899. hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
  9900. if (e->type != KVM_IRQ_ROUTING_MSI)
  9901. continue;
  9902. /*
  9903. * VT-d PI cannot support posting multicast/broadcast
  9904. * interrupts to a vCPU, we still use interrupt remapping
  9905. * for these kind of interrupts.
  9906. *
  9907. * For lowest-priority interrupts, we only support
  9908. * those with single CPU as the destination, e.g. user
  9909. * configures the interrupts via /proc/irq or uses
  9910. * irqbalance to make the interrupts single-CPU.
  9911. *
  9912. * We will support full lowest-priority interrupt later.
  9913. */
  9914. kvm_set_msi_irq(kvm, e, &irq);
  9915. if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu)) {
  9916. /*
  9917. * Make sure the IRTE is in remapped mode if
  9918. * we don't handle it in posted mode.
  9919. */
  9920. ret = irq_set_vcpu_affinity(host_irq, NULL);
  9921. if (ret < 0) {
  9922. printk(KERN_INFO
  9923. "failed to back to remapped mode, irq: %u\n",
  9924. host_irq);
  9925. goto out;
  9926. }
  9927. continue;
  9928. }
  9929. vcpu_info.pi_desc_addr = __pa(vcpu_to_pi_desc(vcpu));
  9930. vcpu_info.vector = irq.vector;
  9931. trace_kvm_pi_irte_update(vcpu->vcpu_id, host_irq, e->gsi,
  9932. vcpu_info.vector, vcpu_info.pi_desc_addr, set);
  9933. if (set)
  9934. ret = irq_set_vcpu_affinity(host_irq, &vcpu_info);
  9935. else {
  9936. /* suppress notification event before unposting */
  9937. pi_set_sn(vcpu_to_pi_desc(vcpu));
  9938. ret = irq_set_vcpu_affinity(host_irq, NULL);
  9939. pi_clear_sn(vcpu_to_pi_desc(vcpu));
  9940. }
  9941. if (ret < 0) {
  9942. printk(KERN_INFO "%s: failed to update PI IRTE\n",
  9943. __func__);
  9944. goto out;
  9945. }
  9946. }
  9947. ret = 0;
  9948. out:
  9949. srcu_read_unlock(&kvm->irq_srcu, idx);
  9950. return ret;
  9951. }
  9952. static void vmx_setup_mce(struct kvm_vcpu *vcpu)
  9953. {
  9954. if (vcpu->arch.mcg_cap & MCG_LMCE_P)
  9955. to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |=
  9956. FEATURE_CONTROL_LMCE;
  9957. else
  9958. to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &=
  9959. ~FEATURE_CONTROL_LMCE;
  9960. }
  9961. static struct kvm_x86_ops vmx_x86_ops __ro_after_init = {
  9962. .cpu_has_kvm_support = cpu_has_kvm_support,
  9963. .disabled_by_bios = vmx_disabled_by_bios,
  9964. .hardware_setup = hardware_setup,
  9965. .hardware_unsetup = hardware_unsetup,
  9966. .check_processor_compatibility = vmx_check_processor_compat,
  9967. .hardware_enable = hardware_enable,
  9968. .hardware_disable = hardware_disable,
  9969. .cpu_has_accelerated_tpr = report_flexpriority,
  9970. .cpu_has_high_real_mode_segbase = vmx_has_high_real_mode_segbase,
  9971. .vcpu_create = vmx_create_vcpu,
  9972. .vcpu_free = vmx_free_vcpu,
  9973. .vcpu_reset = vmx_vcpu_reset,
  9974. .prepare_guest_switch = vmx_save_host_state,
  9975. .vcpu_load = vmx_vcpu_load,
  9976. .vcpu_put = vmx_vcpu_put,
  9977. .update_bp_intercept = update_exception_bitmap,
  9978. .get_msr = vmx_get_msr,
  9979. .set_msr = vmx_set_msr,
  9980. .get_segment_base = vmx_get_segment_base,
  9981. .get_segment = vmx_get_segment,
  9982. .set_segment = vmx_set_segment,
  9983. .get_cpl = vmx_get_cpl,
  9984. .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
  9985. .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits,
  9986. .decache_cr3 = vmx_decache_cr3,
  9987. .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
  9988. .set_cr0 = vmx_set_cr0,
  9989. .set_cr3 = vmx_set_cr3,
  9990. .set_cr4 = vmx_set_cr4,
  9991. .set_efer = vmx_set_efer,
  9992. .get_idt = vmx_get_idt,
  9993. .set_idt = vmx_set_idt,
  9994. .get_gdt = vmx_get_gdt,
  9995. .set_gdt = vmx_set_gdt,
  9996. .get_dr6 = vmx_get_dr6,
  9997. .set_dr6 = vmx_set_dr6,
  9998. .set_dr7 = vmx_set_dr7,
  9999. .sync_dirty_debug_regs = vmx_sync_dirty_debug_regs,
  10000. .cache_reg = vmx_cache_reg,
  10001. .get_rflags = vmx_get_rflags,
  10002. .set_rflags = vmx_set_rflags,
  10003. .get_pkru = vmx_get_pkru,
  10004. .fpu_activate = vmx_fpu_activate,
  10005. .fpu_deactivate = vmx_fpu_deactivate,
  10006. .tlb_flush = vmx_flush_tlb,
  10007. .run = vmx_vcpu_run,
  10008. .handle_exit = vmx_handle_exit,
  10009. .skip_emulated_instruction = skip_emulated_instruction,
  10010. .set_interrupt_shadow = vmx_set_interrupt_shadow,
  10011. .get_interrupt_shadow = vmx_get_interrupt_shadow,
  10012. .patch_hypercall = vmx_patch_hypercall,
  10013. .set_irq = vmx_inject_irq,
  10014. .set_nmi = vmx_inject_nmi,
  10015. .queue_exception = vmx_queue_exception,
  10016. .cancel_injection = vmx_cancel_injection,
  10017. .interrupt_allowed = vmx_interrupt_allowed,
  10018. .nmi_allowed = vmx_nmi_allowed,
  10019. .get_nmi_mask = vmx_get_nmi_mask,
  10020. .set_nmi_mask = vmx_set_nmi_mask,
  10021. .enable_nmi_window = enable_nmi_window,
  10022. .enable_irq_window = enable_irq_window,
  10023. .update_cr8_intercept = update_cr8_intercept,
  10024. .set_virtual_x2apic_mode = vmx_set_virtual_x2apic_mode,
  10025. .set_apic_access_page_addr = vmx_set_apic_access_page_addr,
  10026. .get_enable_apicv = vmx_get_enable_apicv,
  10027. .refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
  10028. .load_eoi_exitmap = vmx_load_eoi_exitmap,
  10029. .hwapic_irr_update = vmx_hwapic_irr_update,
  10030. .hwapic_isr_update = vmx_hwapic_isr_update,
  10031. .sync_pir_to_irr = vmx_sync_pir_to_irr,
  10032. .deliver_posted_interrupt = vmx_deliver_posted_interrupt,
  10033. .set_tss_addr = vmx_set_tss_addr,
  10034. .get_tdp_level = get_ept_level,
  10035. .get_mt_mask = vmx_get_mt_mask,
  10036. .get_exit_info = vmx_get_exit_info,
  10037. .get_lpage_level = vmx_get_lpage_level,
  10038. .cpuid_update = vmx_cpuid_update,
  10039. .rdtscp_supported = vmx_rdtscp_supported,
  10040. .invpcid_supported = vmx_invpcid_supported,
  10041. .set_supported_cpuid = vmx_set_supported_cpuid,
  10042. .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
  10043. .write_tsc_offset = vmx_write_tsc_offset,
  10044. .set_tdp_cr3 = vmx_set_cr3,
  10045. .check_intercept = vmx_check_intercept,
  10046. .handle_external_intr = vmx_handle_external_intr,
  10047. .mpx_supported = vmx_mpx_supported,
  10048. .xsaves_supported = vmx_xsaves_supported,
  10049. .check_nested_events = vmx_check_nested_events,
  10050. .sched_in = vmx_sched_in,
  10051. .slot_enable_log_dirty = vmx_slot_enable_log_dirty,
  10052. .slot_disable_log_dirty = vmx_slot_disable_log_dirty,
  10053. .flush_log_dirty = vmx_flush_log_dirty,
  10054. .enable_log_dirty_pt_masked = vmx_enable_log_dirty_pt_masked,
  10055. .pre_block = vmx_pre_block,
  10056. .post_block = vmx_post_block,
  10057. .pmu_ops = &intel_pmu_ops,
  10058. .update_pi_irte = vmx_update_pi_irte,
  10059. #ifdef CONFIG_X86_64
  10060. .set_hv_timer = vmx_set_hv_timer,
  10061. .cancel_hv_timer = vmx_cancel_hv_timer,
  10062. #endif
  10063. .setup_mce = vmx_setup_mce,
  10064. };
  10065. static int __init vmx_init(void)
  10066. {
  10067. int r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx),
  10068. __alignof__(struct vcpu_vmx), THIS_MODULE);
  10069. if (r)
  10070. return r;
  10071. #ifdef CONFIG_KEXEC_CORE
  10072. rcu_assign_pointer(crash_vmclear_loaded_vmcss,
  10073. crash_vmclear_local_loaded_vmcss);
  10074. #endif
  10075. return 0;
  10076. }
  10077. static void __exit vmx_exit(void)
  10078. {
  10079. #ifdef CONFIG_KEXEC_CORE
  10080. RCU_INIT_POINTER(crash_vmclear_loaded_vmcss, NULL);
  10081. synchronize_rcu();
  10082. #endif
  10083. kvm_exit();
  10084. }
  10085. module_init(vmx_init)
  10086. module_exit(vmx_exit)