cmd_parser.c 85 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883
  1. /*
  2. * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice (including the next
  12. * paragraph) shall be included in all copies or substantial portions of the
  13. * Software.
  14. *
  15. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18. * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20. * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21. * SOFTWARE.
  22. *
  23. * Authors:
  24. * Ke Yu
  25. * Kevin Tian <kevin.tian@intel.com>
  26. * Zhiyuan Lv <zhiyuan.lv@intel.com>
  27. *
  28. * Contributors:
  29. * Min He <min.he@intel.com>
  30. * Ping Gao <ping.a.gao@intel.com>
  31. * Tina Zhang <tina.zhang@intel.com>
  32. * Yulei Zhang <yulei.zhang@intel.com>
  33. * Zhi Wang <zhi.a.wang@intel.com>
  34. *
  35. */
  36. #include <linux/slab.h>
  37. #include "i915_drv.h"
  38. #include "gvt.h"
  39. #include "i915_pvinfo.h"
  40. #include "trace.h"
  41. #define INVALID_OP (~0U)
  42. #define OP_LEN_MI 9
  43. #define OP_LEN_2D 10
  44. #define OP_LEN_3D_MEDIA 16
  45. #define OP_LEN_MFX_VC 16
  46. #define OP_LEN_VEBOX 16
  47. #define CMD_TYPE(cmd) (((cmd) >> 29) & 7)
  48. struct sub_op_bits {
  49. int hi;
  50. int low;
  51. };
  52. struct decode_info {
  53. char *name;
  54. int op_len;
  55. int nr_sub_op;
  56. struct sub_op_bits *sub_op;
  57. };
  58. #define MAX_CMD_BUDGET 0x7fffffff
  59. #define MI_WAIT_FOR_PLANE_C_FLIP_PENDING (1<<15)
  60. #define MI_WAIT_FOR_PLANE_B_FLIP_PENDING (1<<9)
  61. #define MI_WAIT_FOR_PLANE_A_FLIP_PENDING (1<<1)
  62. #define MI_WAIT_FOR_SPRITE_C_FLIP_PENDING (1<<20)
  63. #define MI_WAIT_FOR_SPRITE_B_FLIP_PENDING (1<<10)
  64. #define MI_WAIT_FOR_SPRITE_A_FLIP_PENDING (1<<2)
  65. /* Render Command Map */
  66. /* MI_* command Opcode (28:23) */
  67. #define OP_MI_NOOP 0x0
  68. #define OP_MI_SET_PREDICATE 0x1 /* HSW+ */
  69. #define OP_MI_USER_INTERRUPT 0x2
  70. #define OP_MI_WAIT_FOR_EVENT 0x3
  71. #define OP_MI_FLUSH 0x4
  72. #define OP_MI_ARB_CHECK 0x5
  73. #define OP_MI_RS_CONTROL 0x6 /* HSW+ */
  74. #define OP_MI_REPORT_HEAD 0x7
  75. #define OP_MI_ARB_ON_OFF 0x8
  76. #define OP_MI_URB_ATOMIC_ALLOC 0x9 /* HSW+ */
  77. #define OP_MI_BATCH_BUFFER_END 0xA
  78. #define OP_MI_SUSPEND_FLUSH 0xB
  79. #define OP_MI_PREDICATE 0xC /* IVB+ */
  80. #define OP_MI_TOPOLOGY_FILTER 0xD /* IVB+ */
  81. #define OP_MI_SET_APPID 0xE /* IVB+ */
  82. #define OP_MI_RS_CONTEXT 0xF /* HSW+ */
  83. #define OP_MI_LOAD_SCAN_LINES_INCL 0x12 /* HSW+ */
  84. #define OP_MI_DISPLAY_FLIP 0x14
  85. #define OP_MI_SEMAPHORE_MBOX 0x16
  86. #define OP_MI_SET_CONTEXT 0x18
  87. #define OP_MI_MATH 0x1A
  88. #define OP_MI_URB_CLEAR 0x19
  89. #define OP_MI_SEMAPHORE_SIGNAL 0x1B /* BDW+ */
  90. #define OP_MI_SEMAPHORE_WAIT 0x1C /* BDW+ */
  91. #define OP_MI_STORE_DATA_IMM 0x20
  92. #define OP_MI_STORE_DATA_INDEX 0x21
  93. #define OP_MI_LOAD_REGISTER_IMM 0x22
  94. #define OP_MI_UPDATE_GTT 0x23
  95. #define OP_MI_STORE_REGISTER_MEM 0x24
  96. #define OP_MI_FLUSH_DW 0x26
  97. #define OP_MI_CLFLUSH 0x27
  98. #define OP_MI_REPORT_PERF_COUNT 0x28
  99. #define OP_MI_LOAD_REGISTER_MEM 0x29 /* HSW+ */
  100. #define OP_MI_LOAD_REGISTER_REG 0x2A /* HSW+ */
  101. #define OP_MI_RS_STORE_DATA_IMM 0x2B /* HSW+ */
  102. #define OP_MI_LOAD_URB_MEM 0x2C /* HSW+ */
  103. #define OP_MI_STORE_URM_MEM 0x2D /* HSW+ */
  104. #define OP_MI_2E 0x2E /* BDW+ */
  105. #define OP_MI_2F 0x2F /* BDW+ */
  106. #define OP_MI_BATCH_BUFFER_START 0x31
  107. /* Bit definition for dword 0 */
  108. #define _CMDBIT_BB_START_IN_PPGTT (1UL << 8)
  109. #define OP_MI_CONDITIONAL_BATCH_BUFFER_END 0x36
  110. #define BATCH_BUFFER_ADDR_MASK ((1UL << 32) - (1U << 2))
  111. #define BATCH_BUFFER_ADDR_HIGH_MASK ((1UL << 16) - (1U))
  112. #define BATCH_BUFFER_ADR_SPACE_BIT(x) (((x) >> 8) & 1U)
  113. #define BATCH_BUFFER_2ND_LEVEL_BIT(x) ((x) >> 22 & 1U)
  114. /* 2D command: Opcode (28:22) */
  115. #define OP_2D(x) ((2<<7) | x)
  116. #define OP_XY_SETUP_BLT OP_2D(0x1)
  117. #define OP_XY_SETUP_CLIP_BLT OP_2D(0x3)
  118. #define OP_XY_SETUP_MONO_PATTERN_SL_BLT OP_2D(0x11)
  119. #define OP_XY_PIXEL_BLT OP_2D(0x24)
  120. #define OP_XY_SCANLINES_BLT OP_2D(0x25)
  121. #define OP_XY_TEXT_BLT OP_2D(0x26)
  122. #define OP_XY_TEXT_IMMEDIATE_BLT OP_2D(0x31)
  123. #define OP_XY_COLOR_BLT OP_2D(0x50)
  124. #define OP_XY_PAT_BLT OP_2D(0x51)
  125. #define OP_XY_MONO_PAT_BLT OP_2D(0x52)
  126. #define OP_XY_SRC_COPY_BLT OP_2D(0x53)
  127. #define OP_XY_MONO_SRC_COPY_BLT OP_2D(0x54)
  128. #define OP_XY_FULL_BLT OP_2D(0x55)
  129. #define OP_XY_FULL_MONO_SRC_BLT OP_2D(0x56)
  130. #define OP_XY_FULL_MONO_PATTERN_BLT OP_2D(0x57)
  131. #define OP_XY_FULL_MONO_PATTERN_MONO_SRC_BLT OP_2D(0x58)
  132. #define OP_XY_MONO_PAT_FIXED_BLT OP_2D(0x59)
  133. #define OP_XY_MONO_SRC_COPY_IMMEDIATE_BLT OP_2D(0x71)
  134. #define OP_XY_PAT_BLT_IMMEDIATE OP_2D(0x72)
  135. #define OP_XY_SRC_COPY_CHROMA_BLT OP_2D(0x73)
  136. #define OP_XY_FULL_IMMEDIATE_PATTERN_BLT OP_2D(0x74)
  137. #define OP_XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT OP_2D(0x75)
  138. #define OP_XY_PAT_CHROMA_BLT OP_2D(0x76)
  139. #define OP_XY_PAT_CHROMA_BLT_IMMEDIATE OP_2D(0x77)
  140. /* 3D/Media Command: Pipeline Type(28:27) Opcode(26:24) Sub Opcode(23:16) */
  141. #define OP_3D_MEDIA(sub_type, opcode, sub_opcode) \
  142. ((3 << 13) | ((sub_type) << 11) | ((opcode) << 8) | (sub_opcode))
  143. #define OP_STATE_PREFETCH OP_3D_MEDIA(0x0, 0x0, 0x03)
  144. #define OP_STATE_BASE_ADDRESS OP_3D_MEDIA(0x0, 0x1, 0x01)
  145. #define OP_STATE_SIP OP_3D_MEDIA(0x0, 0x1, 0x02)
  146. #define OP_3D_MEDIA_0_1_4 OP_3D_MEDIA(0x0, 0x1, 0x04)
  147. #define OP_3DSTATE_VF_STATISTICS_GM45 OP_3D_MEDIA(0x1, 0x0, 0x0B)
  148. #define OP_PIPELINE_SELECT OP_3D_MEDIA(0x1, 0x1, 0x04)
  149. #define OP_MEDIA_VFE_STATE OP_3D_MEDIA(0x2, 0x0, 0x0)
  150. #define OP_MEDIA_CURBE_LOAD OP_3D_MEDIA(0x2, 0x0, 0x1)
  151. #define OP_MEDIA_INTERFACE_DESCRIPTOR_LOAD OP_3D_MEDIA(0x2, 0x0, 0x2)
  152. #define OP_MEDIA_GATEWAY_STATE OP_3D_MEDIA(0x2, 0x0, 0x3)
  153. #define OP_MEDIA_STATE_FLUSH OP_3D_MEDIA(0x2, 0x0, 0x4)
  154. #define OP_MEDIA_OBJECT OP_3D_MEDIA(0x2, 0x1, 0x0)
  155. #define OP_MEDIA_OBJECT_PRT OP_3D_MEDIA(0x2, 0x1, 0x2)
  156. #define OP_MEDIA_OBJECT_WALKER OP_3D_MEDIA(0x2, 0x1, 0x3)
  157. #define OP_GPGPU_WALKER OP_3D_MEDIA(0x2, 0x1, 0x5)
  158. #define OP_3DSTATE_CLEAR_PARAMS OP_3D_MEDIA(0x3, 0x0, 0x04) /* IVB+ */
  159. #define OP_3DSTATE_DEPTH_BUFFER OP_3D_MEDIA(0x3, 0x0, 0x05) /* IVB+ */
  160. #define OP_3DSTATE_STENCIL_BUFFER OP_3D_MEDIA(0x3, 0x0, 0x06) /* IVB+ */
  161. #define OP_3DSTATE_HIER_DEPTH_BUFFER OP_3D_MEDIA(0x3, 0x0, 0x07) /* IVB+ */
  162. #define OP_3DSTATE_VERTEX_BUFFERS OP_3D_MEDIA(0x3, 0x0, 0x08)
  163. #define OP_3DSTATE_VERTEX_ELEMENTS OP_3D_MEDIA(0x3, 0x0, 0x09)
  164. #define OP_3DSTATE_INDEX_BUFFER OP_3D_MEDIA(0x3, 0x0, 0x0A)
  165. #define OP_3DSTATE_VF_STATISTICS OP_3D_MEDIA(0x3, 0x0, 0x0B)
  166. #define OP_3DSTATE_VF OP_3D_MEDIA(0x3, 0x0, 0x0C) /* HSW+ */
  167. #define OP_3DSTATE_CC_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x0E)
  168. #define OP_3DSTATE_SCISSOR_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x0F)
  169. #define OP_3DSTATE_VS OP_3D_MEDIA(0x3, 0x0, 0x10)
  170. #define OP_3DSTATE_GS OP_3D_MEDIA(0x3, 0x0, 0x11)
  171. #define OP_3DSTATE_CLIP OP_3D_MEDIA(0x3, 0x0, 0x12)
  172. #define OP_3DSTATE_SF OP_3D_MEDIA(0x3, 0x0, 0x13)
  173. #define OP_3DSTATE_WM OP_3D_MEDIA(0x3, 0x0, 0x14)
  174. #define OP_3DSTATE_CONSTANT_VS OP_3D_MEDIA(0x3, 0x0, 0x15)
  175. #define OP_3DSTATE_CONSTANT_GS OP_3D_MEDIA(0x3, 0x0, 0x16)
  176. #define OP_3DSTATE_CONSTANT_PS OP_3D_MEDIA(0x3, 0x0, 0x17)
  177. #define OP_3DSTATE_SAMPLE_MASK OP_3D_MEDIA(0x3, 0x0, 0x18)
  178. #define OP_3DSTATE_CONSTANT_HS OP_3D_MEDIA(0x3, 0x0, 0x19) /* IVB+ */
  179. #define OP_3DSTATE_CONSTANT_DS OP_3D_MEDIA(0x3, 0x0, 0x1A) /* IVB+ */
  180. #define OP_3DSTATE_HS OP_3D_MEDIA(0x3, 0x0, 0x1B) /* IVB+ */
  181. #define OP_3DSTATE_TE OP_3D_MEDIA(0x3, 0x0, 0x1C) /* IVB+ */
  182. #define OP_3DSTATE_DS OP_3D_MEDIA(0x3, 0x0, 0x1D) /* IVB+ */
  183. #define OP_3DSTATE_STREAMOUT OP_3D_MEDIA(0x3, 0x0, 0x1E) /* IVB+ */
  184. #define OP_3DSTATE_SBE OP_3D_MEDIA(0x3, 0x0, 0x1F) /* IVB+ */
  185. #define OP_3DSTATE_PS OP_3D_MEDIA(0x3, 0x0, 0x20) /* IVB+ */
  186. #define OP_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP OP_3D_MEDIA(0x3, 0x0, 0x21) /* IVB+ */
  187. #define OP_3DSTATE_VIEWPORT_STATE_POINTERS_CC OP_3D_MEDIA(0x3, 0x0, 0x23) /* IVB+ */
  188. #define OP_3DSTATE_BLEND_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x24) /* IVB+ */
  189. #define OP_3DSTATE_DEPTH_STENCIL_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x25) /* IVB+ */
  190. #define OP_3DSTATE_BINDING_TABLE_POINTERS_VS OP_3D_MEDIA(0x3, 0x0, 0x26) /* IVB+ */
  191. #define OP_3DSTATE_BINDING_TABLE_POINTERS_HS OP_3D_MEDIA(0x3, 0x0, 0x27) /* IVB+ */
  192. #define OP_3DSTATE_BINDING_TABLE_POINTERS_DS OP_3D_MEDIA(0x3, 0x0, 0x28) /* IVB+ */
  193. #define OP_3DSTATE_BINDING_TABLE_POINTERS_GS OP_3D_MEDIA(0x3, 0x0, 0x29) /* IVB+ */
  194. #define OP_3DSTATE_BINDING_TABLE_POINTERS_PS OP_3D_MEDIA(0x3, 0x0, 0x2A) /* IVB+ */
  195. #define OP_3DSTATE_SAMPLER_STATE_POINTERS_VS OP_3D_MEDIA(0x3, 0x0, 0x2B) /* IVB+ */
  196. #define OP_3DSTATE_SAMPLER_STATE_POINTERS_HS OP_3D_MEDIA(0x3, 0x0, 0x2C) /* IVB+ */
  197. #define OP_3DSTATE_SAMPLER_STATE_POINTERS_DS OP_3D_MEDIA(0x3, 0x0, 0x2D) /* IVB+ */
  198. #define OP_3DSTATE_SAMPLER_STATE_POINTERS_GS OP_3D_MEDIA(0x3, 0x0, 0x2E) /* IVB+ */
  199. #define OP_3DSTATE_SAMPLER_STATE_POINTERS_PS OP_3D_MEDIA(0x3, 0x0, 0x2F) /* IVB+ */
  200. #define OP_3DSTATE_URB_VS OP_3D_MEDIA(0x3, 0x0, 0x30) /* IVB+ */
  201. #define OP_3DSTATE_URB_HS OP_3D_MEDIA(0x3, 0x0, 0x31) /* IVB+ */
  202. #define OP_3DSTATE_URB_DS OP_3D_MEDIA(0x3, 0x0, 0x32) /* IVB+ */
  203. #define OP_3DSTATE_URB_GS OP_3D_MEDIA(0x3, 0x0, 0x33) /* IVB+ */
  204. #define OP_3DSTATE_GATHER_CONSTANT_VS OP_3D_MEDIA(0x3, 0x0, 0x34) /* HSW+ */
  205. #define OP_3DSTATE_GATHER_CONSTANT_GS OP_3D_MEDIA(0x3, 0x0, 0x35) /* HSW+ */
  206. #define OP_3DSTATE_GATHER_CONSTANT_HS OP_3D_MEDIA(0x3, 0x0, 0x36) /* HSW+ */
  207. #define OP_3DSTATE_GATHER_CONSTANT_DS OP_3D_MEDIA(0x3, 0x0, 0x37) /* HSW+ */
  208. #define OP_3DSTATE_GATHER_CONSTANT_PS OP_3D_MEDIA(0x3, 0x0, 0x38) /* HSW+ */
  209. #define OP_3DSTATE_DX9_CONSTANTF_VS OP_3D_MEDIA(0x3, 0x0, 0x39) /* HSW+ */
  210. #define OP_3DSTATE_DX9_CONSTANTF_PS OP_3D_MEDIA(0x3, 0x0, 0x3A) /* HSW+ */
  211. #define OP_3DSTATE_DX9_CONSTANTI_VS OP_3D_MEDIA(0x3, 0x0, 0x3B) /* HSW+ */
  212. #define OP_3DSTATE_DX9_CONSTANTI_PS OP_3D_MEDIA(0x3, 0x0, 0x3C) /* HSW+ */
  213. #define OP_3DSTATE_DX9_CONSTANTB_VS OP_3D_MEDIA(0x3, 0x0, 0x3D) /* HSW+ */
  214. #define OP_3DSTATE_DX9_CONSTANTB_PS OP_3D_MEDIA(0x3, 0x0, 0x3E) /* HSW+ */
  215. #define OP_3DSTATE_DX9_LOCAL_VALID_VS OP_3D_MEDIA(0x3, 0x0, 0x3F) /* HSW+ */
  216. #define OP_3DSTATE_DX9_LOCAL_VALID_PS OP_3D_MEDIA(0x3, 0x0, 0x40) /* HSW+ */
  217. #define OP_3DSTATE_DX9_GENERATE_ACTIVE_VS OP_3D_MEDIA(0x3, 0x0, 0x41) /* HSW+ */
  218. #define OP_3DSTATE_DX9_GENERATE_ACTIVE_PS OP_3D_MEDIA(0x3, 0x0, 0x42) /* HSW+ */
  219. #define OP_3DSTATE_BINDING_TABLE_EDIT_VS OP_3D_MEDIA(0x3, 0x0, 0x43) /* HSW+ */
  220. #define OP_3DSTATE_BINDING_TABLE_EDIT_GS OP_3D_MEDIA(0x3, 0x0, 0x44) /* HSW+ */
  221. #define OP_3DSTATE_BINDING_TABLE_EDIT_HS OP_3D_MEDIA(0x3, 0x0, 0x45) /* HSW+ */
  222. #define OP_3DSTATE_BINDING_TABLE_EDIT_DS OP_3D_MEDIA(0x3, 0x0, 0x46) /* HSW+ */
  223. #define OP_3DSTATE_BINDING_TABLE_EDIT_PS OP_3D_MEDIA(0x3, 0x0, 0x47) /* HSW+ */
  224. #define OP_3DSTATE_VF_INSTANCING OP_3D_MEDIA(0x3, 0x0, 0x49) /* BDW+ */
  225. #define OP_3DSTATE_VF_SGVS OP_3D_MEDIA(0x3, 0x0, 0x4A) /* BDW+ */
  226. #define OP_3DSTATE_VF_TOPOLOGY OP_3D_MEDIA(0x3, 0x0, 0x4B) /* BDW+ */
  227. #define OP_3DSTATE_WM_CHROMAKEY OP_3D_MEDIA(0x3, 0x0, 0x4C) /* BDW+ */
  228. #define OP_3DSTATE_PS_BLEND OP_3D_MEDIA(0x3, 0x0, 0x4D) /* BDW+ */
  229. #define OP_3DSTATE_WM_DEPTH_STENCIL OP_3D_MEDIA(0x3, 0x0, 0x4E) /* BDW+ */
  230. #define OP_3DSTATE_PS_EXTRA OP_3D_MEDIA(0x3, 0x0, 0x4F) /* BDW+ */
  231. #define OP_3DSTATE_RASTER OP_3D_MEDIA(0x3, 0x0, 0x50) /* BDW+ */
  232. #define OP_3DSTATE_SBE_SWIZ OP_3D_MEDIA(0x3, 0x0, 0x51) /* BDW+ */
  233. #define OP_3DSTATE_WM_HZ_OP OP_3D_MEDIA(0x3, 0x0, 0x52) /* BDW+ */
  234. #define OP_3DSTATE_COMPONENT_PACKING OP_3D_MEDIA(0x3, 0x0, 0x55) /* SKL+ */
  235. #define OP_3DSTATE_DRAWING_RECTANGLE OP_3D_MEDIA(0x3, 0x1, 0x00)
  236. #define OP_3DSTATE_SAMPLER_PALETTE_LOAD0 OP_3D_MEDIA(0x3, 0x1, 0x02)
  237. #define OP_3DSTATE_CHROMA_KEY OP_3D_MEDIA(0x3, 0x1, 0x04)
  238. #define OP_SNB_3DSTATE_DEPTH_BUFFER OP_3D_MEDIA(0x3, 0x1, 0x05)
  239. #define OP_3DSTATE_POLY_STIPPLE_OFFSET OP_3D_MEDIA(0x3, 0x1, 0x06)
  240. #define OP_3DSTATE_POLY_STIPPLE_PATTERN OP_3D_MEDIA(0x3, 0x1, 0x07)
  241. #define OP_3DSTATE_LINE_STIPPLE OP_3D_MEDIA(0x3, 0x1, 0x08)
  242. #define OP_3DSTATE_AA_LINE_PARAMS OP_3D_MEDIA(0x3, 0x1, 0x0A)
  243. #define OP_3DSTATE_GS_SVB_INDEX OP_3D_MEDIA(0x3, 0x1, 0x0B)
  244. #define OP_3DSTATE_SAMPLER_PALETTE_LOAD1 OP_3D_MEDIA(0x3, 0x1, 0x0C)
  245. #define OP_3DSTATE_MULTISAMPLE_BDW OP_3D_MEDIA(0x3, 0x0, 0x0D)
  246. #define OP_SNB_3DSTATE_STENCIL_BUFFER OP_3D_MEDIA(0x3, 0x1, 0x0E)
  247. #define OP_SNB_3DSTATE_HIER_DEPTH_BUFFER OP_3D_MEDIA(0x3, 0x1, 0x0F)
  248. #define OP_SNB_3DSTATE_CLEAR_PARAMS OP_3D_MEDIA(0x3, 0x1, 0x10)
  249. #define OP_3DSTATE_MONOFILTER_SIZE OP_3D_MEDIA(0x3, 0x1, 0x11)
  250. #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_VS OP_3D_MEDIA(0x3, 0x1, 0x12) /* IVB+ */
  251. #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_HS OP_3D_MEDIA(0x3, 0x1, 0x13) /* IVB+ */
  252. #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_DS OP_3D_MEDIA(0x3, 0x1, 0x14) /* IVB+ */
  253. #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_GS OP_3D_MEDIA(0x3, 0x1, 0x15) /* IVB+ */
  254. #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_PS OP_3D_MEDIA(0x3, 0x1, 0x16) /* IVB+ */
  255. #define OP_3DSTATE_SO_DECL_LIST OP_3D_MEDIA(0x3, 0x1, 0x17)
  256. #define OP_3DSTATE_SO_BUFFER OP_3D_MEDIA(0x3, 0x1, 0x18)
  257. #define OP_3DSTATE_BINDING_TABLE_POOL_ALLOC OP_3D_MEDIA(0x3, 0x1, 0x19) /* HSW+ */
  258. #define OP_3DSTATE_GATHER_POOL_ALLOC OP_3D_MEDIA(0x3, 0x1, 0x1A) /* HSW+ */
  259. #define OP_3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC OP_3D_MEDIA(0x3, 0x1, 0x1B) /* HSW+ */
  260. #define OP_3DSTATE_SAMPLE_PATTERN OP_3D_MEDIA(0x3, 0x1, 0x1C)
  261. #define OP_PIPE_CONTROL OP_3D_MEDIA(0x3, 0x2, 0x00)
  262. #define OP_3DPRIMITIVE OP_3D_MEDIA(0x3, 0x3, 0x00)
  263. /* VCCP Command Parser */
  264. /*
  265. * Below MFX and VBE cmd definition is from vaapi intel driver project (BSD License)
  266. * git://anongit.freedesktop.org/vaapi/intel-driver
  267. * src/i965_defines.h
  268. *
  269. */
  270. #define OP_MFX(pipeline, op, sub_opa, sub_opb) \
  271. (3 << 13 | \
  272. (pipeline) << 11 | \
  273. (op) << 8 | \
  274. (sub_opa) << 5 | \
  275. (sub_opb))
  276. #define OP_MFX_PIPE_MODE_SELECT OP_MFX(2, 0, 0, 0) /* ALL */
  277. #define OP_MFX_SURFACE_STATE OP_MFX(2, 0, 0, 1) /* ALL */
  278. #define OP_MFX_PIPE_BUF_ADDR_STATE OP_MFX(2, 0, 0, 2) /* ALL */
  279. #define OP_MFX_IND_OBJ_BASE_ADDR_STATE OP_MFX(2, 0, 0, 3) /* ALL */
  280. #define OP_MFX_BSP_BUF_BASE_ADDR_STATE OP_MFX(2, 0, 0, 4) /* ALL */
  281. #define OP_2_0_0_5 OP_MFX(2, 0, 0, 5) /* ALL */
  282. #define OP_MFX_STATE_POINTER OP_MFX(2, 0, 0, 6) /* ALL */
  283. #define OP_MFX_QM_STATE OP_MFX(2, 0, 0, 7) /* IVB+ */
  284. #define OP_MFX_FQM_STATE OP_MFX(2, 0, 0, 8) /* IVB+ */
  285. #define OP_MFX_PAK_INSERT_OBJECT OP_MFX(2, 0, 2, 8) /* IVB+ */
  286. #define OP_MFX_STITCH_OBJECT OP_MFX(2, 0, 2, 0xA) /* IVB+ */
  287. #define OP_MFD_IT_OBJECT OP_MFX(2, 0, 1, 9) /* ALL */
  288. #define OP_MFX_WAIT OP_MFX(1, 0, 0, 0) /* IVB+ */
  289. #define OP_MFX_AVC_IMG_STATE OP_MFX(2, 1, 0, 0) /* ALL */
  290. #define OP_MFX_AVC_QM_STATE OP_MFX(2, 1, 0, 1) /* ALL */
  291. #define OP_MFX_AVC_DIRECTMODE_STATE OP_MFX(2, 1, 0, 2) /* ALL */
  292. #define OP_MFX_AVC_SLICE_STATE OP_MFX(2, 1, 0, 3) /* ALL */
  293. #define OP_MFX_AVC_REF_IDX_STATE OP_MFX(2, 1, 0, 4) /* ALL */
  294. #define OP_MFX_AVC_WEIGHTOFFSET_STATE OP_MFX(2, 1, 0, 5) /* ALL */
  295. #define OP_MFD_AVC_PICID_STATE OP_MFX(2, 1, 1, 5) /* HSW+ */
  296. #define OP_MFD_AVC_DPB_STATE OP_MFX(2, 1, 1, 6) /* IVB+ */
  297. #define OP_MFD_AVC_SLICEADDR OP_MFX(2, 1, 1, 7) /* IVB+ */
  298. #define OP_MFD_AVC_BSD_OBJECT OP_MFX(2, 1, 1, 8) /* ALL */
  299. #define OP_MFC_AVC_PAK_OBJECT OP_MFX(2, 1, 2, 9) /* ALL */
  300. #define OP_MFX_VC1_PRED_PIPE_STATE OP_MFX(2, 2, 0, 1) /* ALL */
  301. #define OP_MFX_VC1_DIRECTMODE_STATE OP_MFX(2, 2, 0, 2) /* ALL */
  302. #define OP_MFD_VC1_SHORT_PIC_STATE OP_MFX(2, 2, 1, 0) /* IVB+ */
  303. #define OP_MFD_VC1_LONG_PIC_STATE OP_MFX(2, 2, 1, 1) /* IVB+ */
  304. #define OP_MFD_VC1_BSD_OBJECT OP_MFX(2, 2, 1, 8) /* ALL */
  305. #define OP_MFX_MPEG2_PIC_STATE OP_MFX(2, 3, 0, 0) /* ALL */
  306. #define OP_MFX_MPEG2_QM_STATE OP_MFX(2, 3, 0, 1) /* ALL */
  307. #define OP_MFD_MPEG2_BSD_OBJECT OP_MFX(2, 3, 1, 8) /* ALL */
  308. #define OP_MFC_MPEG2_SLICEGROUP_STATE OP_MFX(2, 3, 2, 3) /* ALL */
  309. #define OP_MFC_MPEG2_PAK_OBJECT OP_MFX(2, 3, 2, 9) /* ALL */
  310. #define OP_MFX_2_6_0_0 OP_MFX(2, 6, 0, 0) /* IVB+ */
  311. #define OP_MFX_2_6_0_8 OP_MFX(2, 6, 0, 8) /* IVB+ */
  312. #define OP_MFX_2_6_0_9 OP_MFX(2, 6, 0, 9) /* IVB+ */
  313. #define OP_MFX_JPEG_PIC_STATE OP_MFX(2, 7, 0, 0)
  314. #define OP_MFX_JPEG_HUFF_TABLE_STATE OP_MFX(2, 7, 0, 2)
  315. #define OP_MFD_JPEG_BSD_OBJECT OP_MFX(2, 7, 1, 8)
  316. #define OP_VEB(pipeline, op, sub_opa, sub_opb) \
  317. (3 << 13 | \
  318. (pipeline) << 11 | \
  319. (op) << 8 | \
  320. (sub_opa) << 5 | \
  321. (sub_opb))
  322. #define OP_VEB_SURFACE_STATE OP_VEB(2, 4, 0, 0)
  323. #define OP_VEB_STATE OP_VEB(2, 4, 0, 2)
  324. #define OP_VEB_DNDI_IECP_STATE OP_VEB(2, 4, 0, 3)
  325. struct parser_exec_state;
  326. typedef int (*parser_cmd_handler)(struct parser_exec_state *s);
  327. #define GVT_CMD_HASH_BITS 7
  328. /* which DWords need address fix */
  329. #define ADDR_FIX_1(x1) (1 << (x1))
  330. #define ADDR_FIX_2(x1, x2) (ADDR_FIX_1(x1) | ADDR_FIX_1(x2))
  331. #define ADDR_FIX_3(x1, x2, x3) (ADDR_FIX_1(x1) | ADDR_FIX_2(x2, x3))
  332. #define ADDR_FIX_4(x1, x2, x3, x4) (ADDR_FIX_1(x1) | ADDR_FIX_3(x2, x3, x4))
  333. #define ADDR_FIX_5(x1, x2, x3, x4, x5) (ADDR_FIX_1(x1) | ADDR_FIX_4(x2, x3, x4, x5))
  334. struct cmd_info {
  335. char *name;
  336. u32 opcode;
  337. #define F_LEN_MASK (1U<<0)
  338. #define F_LEN_CONST 1U
  339. #define F_LEN_VAR 0U
  340. /*
  341. * command has its own ip advance logic
  342. * e.g. MI_BATCH_START, MI_BATCH_END
  343. */
  344. #define F_IP_ADVANCE_CUSTOM (1<<1)
  345. #define F_POST_HANDLE (1<<2)
  346. u32 flag;
  347. #define R_RCS (1 << RCS)
  348. #define R_VCS1 (1 << VCS)
  349. #define R_VCS2 (1 << VCS2)
  350. #define R_VCS (R_VCS1 | R_VCS2)
  351. #define R_BCS (1 << BCS)
  352. #define R_VECS (1 << VECS)
  353. #define R_ALL (R_RCS | R_VCS | R_BCS | R_VECS)
  354. /* rings that support this cmd: BLT/RCS/VCS/VECS */
  355. uint16_t rings;
  356. /* devices that support this cmd: SNB/IVB/HSW/... */
  357. uint16_t devices;
  358. /* which DWords are address that need fix up.
  359. * bit 0 means a 32-bit non address operand in command
  360. * bit 1 means address operand, which could be 32-bit
  361. * or 64-bit depending on different architectures.(
  362. * defined by "gmadr_bytes_in_cmd" in intel_gvt.
  363. * No matter the address length, each address only takes
  364. * one bit in the bitmap.
  365. */
  366. uint16_t addr_bitmap;
  367. /* flag == F_LEN_CONST : command length
  368. * flag == F_LEN_VAR : length bias bits
  369. * Note: length is in DWord
  370. */
  371. uint8_t len;
  372. parser_cmd_handler handler;
  373. };
  374. struct cmd_entry {
  375. struct hlist_node hlist;
  376. struct cmd_info *info;
  377. };
  378. enum {
  379. RING_BUFFER_INSTRUCTION,
  380. BATCH_BUFFER_INSTRUCTION,
  381. BATCH_BUFFER_2ND_LEVEL,
  382. };
  383. enum {
  384. GTT_BUFFER,
  385. PPGTT_BUFFER
  386. };
  387. struct parser_exec_state {
  388. struct intel_vgpu *vgpu;
  389. int ring_id;
  390. int buf_type;
  391. /* batch buffer address type */
  392. int buf_addr_type;
  393. /* graphics memory address of ring buffer start */
  394. unsigned long ring_start;
  395. unsigned long ring_size;
  396. unsigned long ring_head;
  397. unsigned long ring_tail;
  398. /* instruction graphics memory address */
  399. unsigned long ip_gma;
  400. /* mapped va of the instr_gma */
  401. void *ip_va;
  402. void *rb_va;
  403. void *ret_bb_va;
  404. /* next instruction when return from batch buffer to ring buffer */
  405. unsigned long ret_ip_gma_ring;
  406. /* next instruction when return from 2nd batch buffer to batch buffer */
  407. unsigned long ret_ip_gma_bb;
  408. /* batch buffer address type (GTT or PPGTT)
  409. * used when ret from 2nd level batch buffer
  410. */
  411. int saved_buf_addr_type;
  412. struct cmd_info *info;
  413. struct intel_vgpu_workload *workload;
  414. };
  415. #define gmadr_dw_number(s) \
  416. (s->vgpu->gvt->device_info.gmadr_bytes_in_cmd >> 2)
  417. static unsigned long bypass_scan_mask = 0;
  418. /* ring ALL, type = 0 */
  419. static struct sub_op_bits sub_op_mi[] = {
  420. {31, 29},
  421. {28, 23},
  422. };
  423. static struct decode_info decode_info_mi = {
  424. "MI",
  425. OP_LEN_MI,
  426. ARRAY_SIZE(sub_op_mi),
  427. sub_op_mi,
  428. };
  429. /* ring RCS, command type 2 */
  430. static struct sub_op_bits sub_op_2d[] = {
  431. {31, 29},
  432. {28, 22},
  433. };
  434. static struct decode_info decode_info_2d = {
  435. "2D",
  436. OP_LEN_2D,
  437. ARRAY_SIZE(sub_op_2d),
  438. sub_op_2d,
  439. };
  440. /* ring RCS, command type 3 */
  441. static struct sub_op_bits sub_op_3d_media[] = {
  442. {31, 29},
  443. {28, 27},
  444. {26, 24},
  445. {23, 16},
  446. };
  447. static struct decode_info decode_info_3d_media = {
  448. "3D_Media",
  449. OP_LEN_3D_MEDIA,
  450. ARRAY_SIZE(sub_op_3d_media),
  451. sub_op_3d_media,
  452. };
  453. /* ring VCS, command type 3 */
  454. static struct sub_op_bits sub_op_mfx_vc[] = {
  455. {31, 29},
  456. {28, 27},
  457. {26, 24},
  458. {23, 21},
  459. {20, 16},
  460. };
  461. static struct decode_info decode_info_mfx_vc = {
  462. "MFX_VC",
  463. OP_LEN_MFX_VC,
  464. ARRAY_SIZE(sub_op_mfx_vc),
  465. sub_op_mfx_vc,
  466. };
  467. /* ring VECS, command type 3 */
  468. static struct sub_op_bits sub_op_vebox[] = {
  469. {31, 29},
  470. {28, 27},
  471. {26, 24},
  472. {23, 21},
  473. {20, 16},
  474. };
  475. static struct decode_info decode_info_vebox = {
  476. "VEBOX",
  477. OP_LEN_VEBOX,
  478. ARRAY_SIZE(sub_op_vebox),
  479. sub_op_vebox,
  480. };
  481. static struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = {
  482. [RCS] = {
  483. &decode_info_mi,
  484. NULL,
  485. NULL,
  486. &decode_info_3d_media,
  487. NULL,
  488. NULL,
  489. NULL,
  490. NULL,
  491. },
  492. [VCS] = {
  493. &decode_info_mi,
  494. NULL,
  495. NULL,
  496. &decode_info_mfx_vc,
  497. NULL,
  498. NULL,
  499. NULL,
  500. NULL,
  501. },
  502. [BCS] = {
  503. &decode_info_mi,
  504. NULL,
  505. &decode_info_2d,
  506. NULL,
  507. NULL,
  508. NULL,
  509. NULL,
  510. NULL,
  511. },
  512. [VECS] = {
  513. &decode_info_mi,
  514. NULL,
  515. NULL,
  516. &decode_info_vebox,
  517. NULL,
  518. NULL,
  519. NULL,
  520. NULL,
  521. },
  522. [VCS2] = {
  523. &decode_info_mi,
  524. NULL,
  525. NULL,
  526. &decode_info_mfx_vc,
  527. NULL,
  528. NULL,
  529. NULL,
  530. NULL,
  531. },
  532. };
  533. static inline u32 get_opcode(u32 cmd, int ring_id)
  534. {
  535. struct decode_info *d_info;
  536. d_info = ring_decode_info[ring_id][CMD_TYPE(cmd)];
  537. if (d_info == NULL)
  538. return INVALID_OP;
  539. return cmd >> (32 - d_info->op_len);
  540. }
  541. static inline struct cmd_info *find_cmd_entry(struct intel_gvt *gvt,
  542. unsigned int opcode, int ring_id)
  543. {
  544. struct cmd_entry *e;
  545. hash_for_each_possible(gvt->cmd_table, e, hlist, opcode) {
  546. if ((opcode == e->info->opcode) &&
  547. (e->info->rings & (1 << ring_id)))
  548. return e->info;
  549. }
  550. return NULL;
  551. }
  552. static inline struct cmd_info *get_cmd_info(struct intel_gvt *gvt,
  553. u32 cmd, int ring_id)
  554. {
  555. u32 opcode;
  556. opcode = get_opcode(cmd, ring_id);
  557. if (opcode == INVALID_OP)
  558. return NULL;
  559. return find_cmd_entry(gvt, opcode, ring_id);
  560. }
  561. static inline u32 sub_op_val(u32 cmd, u32 hi, u32 low)
  562. {
  563. return (cmd >> low) & ((1U << (hi - low + 1)) - 1);
  564. }
  565. static inline void print_opcode(u32 cmd, int ring_id)
  566. {
  567. struct decode_info *d_info;
  568. int i;
  569. d_info = ring_decode_info[ring_id][CMD_TYPE(cmd)];
  570. if (d_info == NULL)
  571. return;
  572. gvt_dbg_cmd("opcode=0x%x %s sub_ops:",
  573. cmd >> (32 - d_info->op_len), d_info->name);
  574. for (i = 0; i < d_info->nr_sub_op; i++)
  575. pr_err("0x%x ", sub_op_val(cmd, d_info->sub_op[i].hi,
  576. d_info->sub_op[i].low));
  577. pr_err("\n");
  578. }
  579. static inline u32 *cmd_ptr(struct parser_exec_state *s, int index)
  580. {
  581. return s->ip_va + (index << 2);
  582. }
  583. static inline u32 cmd_val(struct parser_exec_state *s, int index)
  584. {
  585. return *cmd_ptr(s, index);
  586. }
  587. static void parser_exec_state_dump(struct parser_exec_state *s)
  588. {
  589. int cnt = 0;
  590. int i;
  591. gvt_dbg_cmd(" vgpu%d RING%d: ring_start(%08lx) ring_end(%08lx)"
  592. " ring_head(%08lx) ring_tail(%08lx)\n", s->vgpu->id,
  593. s->ring_id, s->ring_start, s->ring_start + s->ring_size,
  594. s->ring_head, s->ring_tail);
  595. gvt_dbg_cmd(" %s %s ip_gma(%08lx) ",
  596. s->buf_type == RING_BUFFER_INSTRUCTION ?
  597. "RING_BUFFER" : "BATCH_BUFFER",
  598. s->buf_addr_type == GTT_BUFFER ?
  599. "GTT" : "PPGTT", s->ip_gma);
  600. if (s->ip_va == NULL) {
  601. gvt_dbg_cmd(" ip_va(NULL)");
  602. return;
  603. }
  604. gvt_dbg_cmd(" ip_va=%p: %08x %08x %08x %08x\n",
  605. s->ip_va, cmd_val(s, 0), cmd_val(s, 1),
  606. cmd_val(s, 2), cmd_val(s, 3));
  607. print_opcode(cmd_val(s, 0), s->ring_id);
  608. /* print the whole page to trace */
  609. pr_err(" ip_va=%p: %08x %08x %08x %08x\n",
  610. s->ip_va, cmd_val(s, 0), cmd_val(s, 1),
  611. cmd_val(s, 2), cmd_val(s, 3));
  612. s->ip_va = (u32 *)((((u64)s->ip_va) >> 12) << 12);
  613. while (cnt < 1024) {
  614. pr_err("ip_va=%p: ", s->ip_va);
  615. for (i = 0; i < 8; i++)
  616. pr_err("%08x ", cmd_val(s, i));
  617. pr_err("\n");
  618. s->ip_va += 8 * sizeof(u32);
  619. cnt += 8;
  620. }
  621. }
  622. static inline void update_ip_va(struct parser_exec_state *s)
  623. {
  624. unsigned long len = 0;
  625. if (WARN_ON(s->ring_head == s->ring_tail))
  626. return;
  627. if (s->buf_type == RING_BUFFER_INSTRUCTION) {
  628. unsigned long ring_top = s->ring_start + s->ring_size;
  629. if (s->ring_head > s->ring_tail) {
  630. if (s->ip_gma >= s->ring_head && s->ip_gma < ring_top)
  631. len = (s->ip_gma - s->ring_head);
  632. else if (s->ip_gma >= s->ring_start &&
  633. s->ip_gma <= s->ring_tail)
  634. len = (ring_top - s->ring_head) +
  635. (s->ip_gma - s->ring_start);
  636. } else
  637. len = (s->ip_gma - s->ring_head);
  638. s->ip_va = s->rb_va + len;
  639. } else {/* shadow batch buffer */
  640. s->ip_va = s->ret_bb_va;
  641. }
  642. }
  643. static inline int ip_gma_set(struct parser_exec_state *s,
  644. unsigned long ip_gma)
  645. {
  646. WARN_ON(!IS_ALIGNED(ip_gma, 4));
  647. s->ip_gma = ip_gma;
  648. update_ip_va(s);
  649. return 0;
  650. }
  651. static inline int ip_gma_advance(struct parser_exec_state *s,
  652. unsigned int dw_len)
  653. {
  654. s->ip_gma += (dw_len << 2);
  655. if (s->buf_type == RING_BUFFER_INSTRUCTION) {
  656. if (s->ip_gma >= s->ring_start + s->ring_size)
  657. s->ip_gma -= s->ring_size;
  658. update_ip_va(s);
  659. } else {
  660. s->ip_va += (dw_len << 2);
  661. }
  662. return 0;
  663. }
  664. static inline int get_cmd_length(struct cmd_info *info, u32 cmd)
  665. {
  666. if ((info->flag & F_LEN_MASK) == F_LEN_CONST)
  667. return info->len;
  668. else
  669. return (cmd & ((1U << info->len) - 1)) + 2;
  670. return 0;
  671. }
  672. static inline int cmd_length(struct parser_exec_state *s)
  673. {
  674. return get_cmd_length(s->info, cmd_val(s, 0));
  675. }
  676. /* do not remove this, some platform may need clflush here */
  677. #define patch_value(s, addr, val) do { \
  678. *addr = val; \
  679. } while (0)
  680. static bool is_shadowed_mmio(unsigned int offset)
  681. {
  682. bool ret = false;
  683. if ((offset == 0x2168) || /*BB current head register UDW */
  684. (offset == 0x2140) || /*BB current header register */
  685. (offset == 0x211c) || /*second BB header register UDW */
  686. (offset == 0x2114)) { /*second BB header register UDW */
  687. ret = true;
  688. }
  689. return ret;
  690. }
  691. static inline bool is_force_nonpriv_mmio(unsigned int offset)
  692. {
  693. return (offset >= 0x24d0 && offset < 0x2500);
  694. }
  695. static int force_nonpriv_reg_handler(struct parser_exec_state *s,
  696. unsigned int offset, unsigned int index)
  697. {
  698. struct intel_gvt *gvt = s->vgpu->gvt;
  699. unsigned int data = cmd_val(s, index + 1);
  700. if (!intel_gvt_in_force_nonpriv_whitelist(gvt, data)) {
  701. gvt_err("Unexpected forcenonpriv 0x%x LRI write, value=0x%x\n",
  702. offset, data);
  703. return -EPERM;
  704. }
  705. return 0;
  706. }
  707. static int cmd_reg_handler(struct parser_exec_state *s,
  708. unsigned int offset, unsigned int index, char *cmd)
  709. {
  710. struct intel_vgpu *vgpu = s->vgpu;
  711. struct intel_gvt *gvt = vgpu->gvt;
  712. if (offset + 4 > gvt->device_info.mmio_size) {
  713. gvt_vgpu_err("%s access to (%x) outside of MMIO range\n",
  714. cmd, offset);
  715. return -EFAULT;
  716. }
  717. if (!intel_gvt_mmio_is_cmd_access(gvt, offset)) {
  718. gvt_vgpu_err("%s access to non-render register (%x)\n",
  719. cmd, offset);
  720. return 0;
  721. }
  722. if (is_shadowed_mmio(offset)) {
  723. gvt_vgpu_err("found access of shadowed MMIO %x\n", offset);
  724. return 0;
  725. }
  726. if (is_force_nonpriv_mmio(offset) &&
  727. force_nonpriv_reg_handler(s, offset, index))
  728. return -EPERM;
  729. if (offset == i915_mmio_reg_offset(DERRMR) ||
  730. offset == i915_mmio_reg_offset(FORCEWAKE_MT)) {
  731. /* Writing to HW VGT_PVINFO_PAGE offset will be discarded */
  732. patch_value(s, cmd_ptr(s, index), VGT_PVINFO_PAGE);
  733. }
  734. /* TODO: Update the global mask if this MMIO is a masked-MMIO */
  735. intel_gvt_mmio_set_cmd_accessed(gvt, offset);
  736. return 0;
  737. }
  738. #define cmd_reg(s, i) \
  739. (cmd_val(s, i) & GENMASK(22, 2))
  740. #define cmd_reg_inhibit(s, i) \
  741. (cmd_val(s, i) & GENMASK(22, 18))
  742. #define cmd_gma(s, i) \
  743. (cmd_val(s, i) & GENMASK(31, 2))
  744. #define cmd_gma_hi(s, i) \
  745. (cmd_val(s, i) & GENMASK(15, 0))
  746. static int cmd_handler_lri(struct parser_exec_state *s)
  747. {
  748. int i, ret = 0;
  749. int cmd_len = cmd_length(s);
  750. struct intel_gvt *gvt = s->vgpu->gvt;
  751. for (i = 1; i < cmd_len; i += 2) {
  752. if (IS_BROADWELL(gvt->dev_priv) &&
  753. (s->ring_id != RCS)) {
  754. if (s->ring_id == BCS &&
  755. cmd_reg(s, i) ==
  756. i915_mmio_reg_offset(DERRMR))
  757. ret |= 0;
  758. else
  759. ret |= (cmd_reg_inhibit(s, i)) ?
  760. -EBADRQC : 0;
  761. }
  762. if (ret)
  763. break;
  764. ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lri");
  765. if (ret)
  766. break;
  767. }
  768. return ret;
  769. }
  770. static int cmd_handler_lrr(struct parser_exec_state *s)
  771. {
  772. int i, ret = 0;
  773. int cmd_len = cmd_length(s);
  774. for (i = 1; i < cmd_len; i += 2) {
  775. if (IS_BROADWELL(s->vgpu->gvt->dev_priv))
  776. ret |= ((cmd_reg_inhibit(s, i) ||
  777. (cmd_reg_inhibit(s, i + 1)))) ?
  778. -EBADRQC : 0;
  779. if (ret)
  780. break;
  781. ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrr-src");
  782. if (ret)
  783. break;
  784. ret |= cmd_reg_handler(s, cmd_reg(s, i + 1), i, "lrr-dst");
  785. if (ret)
  786. break;
  787. }
  788. return ret;
  789. }
  790. static inline int cmd_address_audit(struct parser_exec_state *s,
  791. unsigned long guest_gma, int op_size, bool index_mode);
  792. static int cmd_handler_lrm(struct parser_exec_state *s)
  793. {
  794. struct intel_gvt *gvt = s->vgpu->gvt;
  795. int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd;
  796. unsigned long gma;
  797. int i, ret = 0;
  798. int cmd_len = cmd_length(s);
  799. for (i = 1; i < cmd_len;) {
  800. if (IS_BROADWELL(gvt->dev_priv))
  801. ret |= (cmd_reg_inhibit(s, i)) ? -EBADRQC : 0;
  802. if (ret)
  803. break;
  804. ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrm");
  805. if (ret)
  806. break;
  807. if (cmd_val(s, 0) & (1 << 22)) {
  808. gma = cmd_gma(s, i + 1);
  809. if (gmadr_bytes == 8)
  810. gma |= (cmd_gma_hi(s, i + 2)) << 32;
  811. ret |= cmd_address_audit(s, gma, sizeof(u32), false);
  812. if (ret)
  813. break;
  814. }
  815. i += gmadr_dw_number(s) + 1;
  816. }
  817. return ret;
  818. }
  819. static int cmd_handler_srm(struct parser_exec_state *s)
  820. {
  821. int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
  822. unsigned long gma;
  823. int i, ret = 0;
  824. int cmd_len = cmd_length(s);
  825. for (i = 1; i < cmd_len;) {
  826. ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "srm");
  827. if (ret)
  828. break;
  829. if (cmd_val(s, 0) & (1 << 22)) {
  830. gma = cmd_gma(s, i + 1);
  831. if (gmadr_bytes == 8)
  832. gma |= (cmd_gma_hi(s, i + 2)) << 32;
  833. ret |= cmd_address_audit(s, gma, sizeof(u32), false);
  834. if (ret)
  835. break;
  836. }
  837. i += gmadr_dw_number(s) + 1;
  838. }
  839. return ret;
  840. }
  841. struct cmd_interrupt_event {
  842. int pipe_control_notify;
  843. int mi_flush_dw;
  844. int mi_user_interrupt;
  845. };
  846. static struct cmd_interrupt_event cmd_interrupt_events[] = {
  847. [RCS] = {
  848. .pipe_control_notify = RCS_PIPE_CONTROL,
  849. .mi_flush_dw = INTEL_GVT_EVENT_RESERVED,
  850. .mi_user_interrupt = RCS_MI_USER_INTERRUPT,
  851. },
  852. [BCS] = {
  853. .pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
  854. .mi_flush_dw = BCS_MI_FLUSH_DW,
  855. .mi_user_interrupt = BCS_MI_USER_INTERRUPT,
  856. },
  857. [VCS] = {
  858. .pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
  859. .mi_flush_dw = VCS_MI_FLUSH_DW,
  860. .mi_user_interrupt = VCS_MI_USER_INTERRUPT,
  861. },
  862. [VCS2] = {
  863. .pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
  864. .mi_flush_dw = VCS2_MI_FLUSH_DW,
  865. .mi_user_interrupt = VCS2_MI_USER_INTERRUPT,
  866. },
  867. [VECS] = {
  868. .pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
  869. .mi_flush_dw = VECS_MI_FLUSH_DW,
  870. .mi_user_interrupt = VECS_MI_USER_INTERRUPT,
  871. },
  872. };
  873. static int cmd_handler_pipe_control(struct parser_exec_state *s)
  874. {
  875. int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
  876. unsigned long gma;
  877. bool index_mode = false;
  878. unsigned int post_sync;
  879. int ret = 0;
  880. post_sync = (cmd_val(s, 1) & PIPE_CONTROL_POST_SYNC_OP_MASK) >> 14;
  881. /* LRI post sync */
  882. if (cmd_val(s, 1) & PIPE_CONTROL_MMIO_WRITE)
  883. ret = cmd_reg_handler(s, cmd_reg(s, 2), 1, "pipe_ctrl");
  884. /* post sync */
  885. else if (post_sync) {
  886. if (post_sync == 2)
  887. ret = cmd_reg_handler(s, 0x2350, 1, "pipe_ctrl");
  888. else if (post_sync == 3)
  889. ret = cmd_reg_handler(s, 0x2358, 1, "pipe_ctrl");
  890. else if (post_sync == 1) {
  891. /* check ggtt*/
  892. if ((cmd_val(s, 1) & PIPE_CONTROL_GLOBAL_GTT_IVB)) {
  893. gma = cmd_val(s, 2) & GENMASK(31, 3);
  894. if (gmadr_bytes == 8)
  895. gma |= (cmd_gma_hi(s, 3)) << 32;
  896. /* Store Data Index */
  897. if (cmd_val(s, 1) & (1 << 21))
  898. index_mode = true;
  899. ret |= cmd_address_audit(s, gma, sizeof(u64),
  900. index_mode);
  901. }
  902. }
  903. }
  904. if (ret)
  905. return ret;
  906. if (cmd_val(s, 1) & PIPE_CONTROL_NOTIFY)
  907. set_bit(cmd_interrupt_events[s->ring_id].pipe_control_notify,
  908. s->workload->pending_events);
  909. return 0;
  910. }
  911. static int cmd_handler_mi_user_interrupt(struct parser_exec_state *s)
  912. {
  913. set_bit(cmd_interrupt_events[s->ring_id].mi_user_interrupt,
  914. s->workload->pending_events);
  915. return 0;
  916. }
  917. static int cmd_advance_default(struct parser_exec_state *s)
  918. {
  919. return ip_gma_advance(s, cmd_length(s));
  920. }
  921. static int cmd_handler_mi_batch_buffer_end(struct parser_exec_state *s)
  922. {
  923. int ret;
  924. if (s->buf_type == BATCH_BUFFER_2ND_LEVEL) {
  925. s->buf_type = BATCH_BUFFER_INSTRUCTION;
  926. ret = ip_gma_set(s, s->ret_ip_gma_bb);
  927. s->buf_addr_type = s->saved_buf_addr_type;
  928. } else {
  929. s->buf_type = RING_BUFFER_INSTRUCTION;
  930. s->buf_addr_type = GTT_BUFFER;
  931. if (s->ret_ip_gma_ring >= s->ring_start + s->ring_size)
  932. s->ret_ip_gma_ring -= s->ring_size;
  933. ret = ip_gma_set(s, s->ret_ip_gma_ring);
  934. }
  935. return ret;
  936. }
  937. struct mi_display_flip_command_info {
  938. int pipe;
  939. int plane;
  940. int event;
  941. i915_reg_t stride_reg;
  942. i915_reg_t ctrl_reg;
  943. i915_reg_t surf_reg;
  944. u64 stride_val;
  945. u64 tile_val;
  946. u64 surf_val;
  947. bool async_flip;
  948. };
  949. struct plane_code_mapping {
  950. int pipe;
  951. int plane;
  952. int event;
  953. };
  954. static int gen8_decode_mi_display_flip(struct parser_exec_state *s,
  955. struct mi_display_flip_command_info *info)
  956. {
  957. struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
  958. struct plane_code_mapping gen8_plane_code[] = {
  959. [0] = {PIPE_A, PLANE_A, PRIMARY_A_FLIP_DONE},
  960. [1] = {PIPE_B, PLANE_A, PRIMARY_B_FLIP_DONE},
  961. [2] = {PIPE_A, PLANE_B, SPRITE_A_FLIP_DONE},
  962. [3] = {PIPE_B, PLANE_B, SPRITE_B_FLIP_DONE},
  963. [4] = {PIPE_C, PLANE_A, PRIMARY_C_FLIP_DONE},
  964. [5] = {PIPE_C, PLANE_B, SPRITE_C_FLIP_DONE},
  965. };
  966. u32 dword0, dword1, dword2;
  967. u32 v;
  968. dword0 = cmd_val(s, 0);
  969. dword1 = cmd_val(s, 1);
  970. dword2 = cmd_val(s, 2);
  971. v = (dword0 & GENMASK(21, 19)) >> 19;
  972. if (WARN_ON(v >= ARRAY_SIZE(gen8_plane_code)))
  973. return -EBADRQC;
  974. info->pipe = gen8_plane_code[v].pipe;
  975. info->plane = gen8_plane_code[v].plane;
  976. info->event = gen8_plane_code[v].event;
  977. info->stride_val = (dword1 & GENMASK(15, 6)) >> 6;
  978. info->tile_val = (dword1 & 0x1);
  979. info->surf_val = (dword2 & GENMASK(31, 12)) >> 12;
  980. info->async_flip = ((dword2 & GENMASK(1, 0)) == 0x1);
  981. if (info->plane == PLANE_A) {
  982. info->ctrl_reg = DSPCNTR(info->pipe);
  983. info->stride_reg = DSPSTRIDE(info->pipe);
  984. info->surf_reg = DSPSURF(info->pipe);
  985. } else if (info->plane == PLANE_B) {
  986. info->ctrl_reg = SPRCTL(info->pipe);
  987. info->stride_reg = SPRSTRIDE(info->pipe);
  988. info->surf_reg = SPRSURF(info->pipe);
  989. } else {
  990. WARN_ON(1);
  991. return -EBADRQC;
  992. }
  993. return 0;
  994. }
  995. static int skl_decode_mi_display_flip(struct parser_exec_state *s,
  996. struct mi_display_flip_command_info *info)
  997. {
  998. struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
  999. struct intel_vgpu *vgpu = s->vgpu;
  1000. u32 dword0 = cmd_val(s, 0);
  1001. u32 dword1 = cmd_val(s, 1);
  1002. u32 dword2 = cmd_val(s, 2);
  1003. u32 plane = (dword0 & GENMASK(12, 8)) >> 8;
  1004. info->plane = PRIMARY_PLANE;
  1005. switch (plane) {
  1006. case MI_DISPLAY_FLIP_SKL_PLANE_1_A:
  1007. info->pipe = PIPE_A;
  1008. info->event = PRIMARY_A_FLIP_DONE;
  1009. break;
  1010. case MI_DISPLAY_FLIP_SKL_PLANE_1_B:
  1011. info->pipe = PIPE_B;
  1012. info->event = PRIMARY_B_FLIP_DONE;
  1013. break;
  1014. case MI_DISPLAY_FLIP_SKL_PLANE_1_C:
  1015. info->pipe = PIPE_C;
  1016. info->event = PRIMARY_C_FLIP_DONE;
  1017. break;
  1018. case MI_DISPLAY_FLIP_SKL_PLANE_2_A:
  1019. info->pipe = PIPE_A;
  1020. info->event = SPRITE_A_FLIP_DONE;
  1021. info->plane = SPRITE_PLANE;
  1022. break;
  1023. case MI_DISPLAY_FLIP_SKL_PLANE_2_B:
  1024. info->pipe = PIPE_B;
  1025. info->event = SPRITE_B_FLIP_DONE;
  1026. info->plane = SPRITE_PLANE;
  1027. break;
  1028. case MI_DISPLAY_FLIP_SKL_PLANE_2_C:
  1029. info->pipe = PIPE_C;
  1030. info->event = SPRITE_C_FLIP_DONE;
  1031. info->plane = SPRITE_PLANE;
  1032. break;
  1033. default:
  1034. gvt_vgpu_err("unknown plane code %d\n", plane);
  1035. return -EBADRQC;
  1036. }
  1037. info->stride_val = (dword1 & GENMASK(15, 6)) >> 6;
  1038. info->tile_val = (dword1 & GENMASK(2, 0));
  1039. info->surf_val = (dword2 & GENMASK(31, 12)) >> 12;
  1040. info->async_flip = ((dword2 & GENMASK(1, 0)) == 0x1);
  1041. info->ctrl_reg = DSPCNTR(info->pipe);
  1042. info->stride_reg = DSPSTRIDE(info->pipe);
  1043. info->surf_reg = DSPSURF(info->pipe);
  1044. return 0;
  1045. }
  1046. static int gen8_check_mi_display_flip(struct parser_exec_state *s,
  1047. struct mi_display_flip_command_info *info)
  1048. {
  1049. struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
  1050. u32 stride, tile;
  1051. if (!info->async_flip)
  1052. return 0;
  1053. if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
  1054. stride = vgpu_vreg(s->vgpu, info->stride_reg) & GENMASK(9, 0);
  1055. tile = (vgpu_vreg(s->vgpu, info->ctrl_reg) &
  1056. GENMASK(12, 10)) >> 10;
  1057. } else {
  1058. stride = (vgpu_vreg(s->vgpu, info->stride_reg) &
  1059. GENMASK(15, 6)) >> 6;
  1060. tile = (vgpu_vreg(s->vgpu, info->ctrl_reg) & (1 << 10)) >> 10;
  1061. }
  1062. if (stride != info->stride_val)
  1063. gvt_dbg_cmd("cannot change stride during async flip\n");
  1064. if (tile != info->tile_val)
  1065. gvt_dbg_cmd("cannot change tile during async flip\n");
  1066. return 0;
  1067. }
  1068. static int gen8_update_plane_mmio_from_mi_display_flip(
  1069. struct parser_exec_state *s,
  1070. struct mi_display_flip_command_info *info)
  1071. {
  1072. struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
  1073. struct intel_vgpu *vgpu = s->vgpu;
  1074. set_mask_bits(&vgpu_vreg(vgpu, info->surf_reg), GENMASK(31, 12),
  1075. info->surf_val << 12);
  1076. if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
  1077. set_mask_bits(&vgpu_vreg(vgpu, info->stride_reg), GENMASK(9, 0),
  1078. info->stride_val);
  1079. set_mask_bits(&vgpu_vreg(vgpu, info->ctrl_reg), GENMASK(12, 10),
  1080. info->tile_val << 10);
  1081. } else {
  1082. set_mask_bits(&vgpu_vreg(vgpu, info->stride_reg), GENMASK(15, 6),
  1083. info->stride_val << 6);
  1084. set_mask_bits(&vgpu_vreg(vgpu, info->ctrl_reg), GENMASK(10, 10),
  1085. info->tile_val << 10);
  1086. }
  1087. vgpu_vreg(vgpu, PIPE_FRMCOUNT_G4X(info->pipe))++;
  1088. intel_vgpu_trigger_virtual_event(vgpu, info->event);
  1089. return 0;
  1090. }
  1091. static int decode_mi_display_flip(struct parser_exec_state *s,
  1092. struct mi_display_flip_command_info *info)
  1093. {
  1094. struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
  1095. if (IS_BROADWELL(dev_priv))
  1096. return gen8_decode_mi_display_flip(s, info);
  1097. if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
  1098. return skl_decode_mi_display_flip(s, info);
  1099. return -ENODEV;
  1100. }
  1101. static int check_mi_display_flip(struct parser_exec_state *s,
  1102. struct mi_display_flip_command_info *info)
  1103. {
  1104. struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
  1105. if (IS_BROADWELL(dev_priv)
  1106. || IS_SKYLAKE(dev_priv)
  1107. || IS_KABYLAKE(dev_priv))
  1108. return gen8_check_mi_display_flip(s, info);
  1109. return -ENODEV;
  1110. }
  1111. static int update_plane_mmio_from_mi_display_flip(
  1112. struct parser_exec_state *s,
  1113. struct mi_display_flip_command_info *info)
  1114. {
  1115. struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
  1116. if (IS_BROADWELL(dev_priv)
  1117. || IS_SKYLAKE(dev_priv)
  1118. || IS_KABYLAKE(dev_priv))
  1119. return gen8_update_plane_mmio_from_mi_display_flip(s, info);
  1120. return -ENODEV;
  1121. }
  1122. static int cmd_handler_mi_display_flip(struct parser_exec_state *s)
  1123. {
  1124. struct mi_display_flip_command_info info;
  1125. struct intel_vgpu *vgpu = s->vgpu;
  1126. int ret;
  1127. int i;
  1128. int len = cmd_length(s);
  1129. ret = decode_mi_display_flip(s, &info);
  1130. if (ret) {
  1131. gvt_vgpu_err("fail to decode MI display flip command\n");
  1132. return ret;
  1133. }
  1134. ret = check_mi_display_flip(s, &info);
  1135. if (ret) {
  1136. gvt_vgpu_err("invalid MI display flip command\n");
  1137. return ret;
  1138. }
  1139. ret = update_plane_mmio_from_mi_display_flip(s, &info);
  1140. if (ret) {
  1141. gvt_vgpu_err("fail to update plane mmio\n");
  1142. return ret;
  1143. }
  1144. for (i = 0; i < len; i++)
  1145. patch_value(s, cmd_ptr(s, i), MI_NOOP);
  1146. return 0;
  1147. }
  1148. static bool is_wait_for_flip_pending(u32 cmd)
  1149. {
  1150. return cmd & (MI_WAIT_FOR_PLANE_A_FLIP_PENDING |
  1151. MI_WAIT_FOR_PLANE_B_FLIP_PENDING |
  1152. MI_WAIT_FOR_PLANE_C_FLIP_PENDING |
  1153. MI_WAIT_FOR_SPRITE_A_FLIP_PENDING |
  1154. MI_WAIT_FOR_SPRITE_B_FLIP_PENDING |
  1155. MI_WAIT_FOR_SPRITE_C_FLIP_PENDING);
  1156. }
  1157. static int cmd_handler_mi_wait_for_event(struct parser_exec_state *s)
  1158. {
  1159. u32 cmd = cmd_val(s, 0);
  1160. if (!is_wait_for_flip_pending(cmd))
  1161. return 0;
  1162. patch_value(s, cmd_ptr(s, 0), MI_NOOP);
  1163. return 0;
  1164. }
  1165. static unsigned long get_gma_bb_from_cmd(struct parser_exec_state *s, int index)
  1166. {
  1167. unsigned long addr;
  1168. unsigned long gma_high, gma_low;
  1169. struct intel_vgpu *vgpu = s->vgpu;
  1170. int gmadr_bytes = vgpu->gvt->device_info.gmadr_bytes_in_cmd;
  1171. if (WARN_ON(gmadr_bytes != 4 && gmadr_bytes != 8)) {
  1172. gvt_vgpu_err("invalid gma bytes %d\n", gmadr_bytes);
  1173. return INTEL_GVT_INVALID_ADDR;
  1174. }
  1175. gma_low = cmd_val(s, index) & BATCH_BUFFER_ADDR_MASK;
  1176. if (gmadr_bytes == 4) {
  1177. addr = gma_low;
  1178. } else {
  1179. gma_high = cmd_val(s, index + 1) & BATCH_BUFFER_ADDR_HIGH_MASK;
  1180. addr = (((unsigned long)gma_high) << 32) | gma_low;
  1181. }
  1182. return addr;
  1183. }
  1184. static inline int cmd_address_audit(struct parser_exec_state *s,
  1185. unsigned long guest_gma, int op_size, bool index_mode)
  1186. {
  1187. struct intel_vgpu *vgpu = s->vgpu;
  1188. u32 max_surface_size = vgpu->gvt->device_info.max_surface_size;
  1189. int i;
  1190. int ret;
  1191. if (op_size > max_surface_size) {
  1192. gvt_vgpu_err("command address audit fail name %s\n",
  1193. s->info->name);
  1194. return -EFAULT;
  1195. }
  1196. if (index_mode) {
  1197. if (guest_gma >= I915_GTT_PAGE_SIZE / sizeof(u64)) {
  1198. ret = -EFAULT;
  1199. goto err;
  1200. }
  1201. } else if (!intel_gvt_ggtt_validate_range(vgpu, guest_gma, op_size)) {
  1202. ret = -EFAULT;
  1203. goto err;
  1204. }
  1205. return 0;
  1206. err:
  1207. gvt_vgpu_err("cmd_parser: Malicious %s detected, addr=0x%lx, len=%d!\n",
  1208. s->info->name, guest_gma, op_size);
  1209. pr_err("cmd dump: ");
  1210. for (i = 0; i < cmd_length(s); i++) {
  1211. if (!(i % 4))
  1212. pr_err("\n%08x ", cmd_val(s, i));
  1213. else
  1214. pr_err("%08x ", cmd_val(s, i));
  1215. }
  1216. pr_err("\nvgpu%d: aperture 0x%llx - 0x%llx, hidden 0x%llx - 0x%llx\n",
  1217. vgpu->id,
  1218. vgpu_aperture_gmadr_base(vgpu),
  1219. vgpu_aperture_gmadr_end(vgpu),
  1220. vgpu_hidden_gmadr_base(vgpu),
  1221. vgpu_hidden_gmadr_end(vgpu));
  1222. return ret;
  1223. }
  1224. static int cmd_handler_mi_store_data_imm(struct parser_exec_state *s)
  1225. {
  1226. int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
  1227. int op_size = (cmd_length(s) - 3) * sizeof(u32);
  1228. int core_id = (cmd_val(s, 2) & (1 << 0)) ? 1 : 0;
  1229. unsigned long gma, gma_low, gma_high;
  1230. int ret = 0;
  1231. /* check ppggt */
  1232. if (!(cmd_val(s, 0) & (1 << 22)))
  1233. return 0;
  1234. gma = cmd_val(s, 2) & GENMASK(31, 2);
  1235. if (gmadr_bytes == 8) {
  1236. gma_low = cmd_val(s, 1) & GENMASK(31, 2);
  1237. gma_high = cmd_val(s, 2) & GENMASK(15, 0);
  1238. gma = (gma_high << 32) | gma_low;
  1239. core_id = (cmd_val(s, 1) & (1 << 0)) ? 1 : 0;
  1240. }
  1241. ret = cmd_address_audit(s, gma + op_size * core_id, op_size, false);
  1242. return ret;
  1243. }
  1244. static inline int unexpected_cmd(struct parser_exec_state *s)
  1245. {
  1246. struct intel_vgpu *vgpu = s->vgpu;
  1247. gvt_vgpu_err("Unexpected %s in command buffer!\n", s->info->name);
  1248. return -EBADRQC;
  1249. }
  1250. static int cmd_handler_mi_semaphore_wait(struct parser_exec_state *s)
  1251. {
  1252. return unexpected_cmd(s);
  1253. }
  1254. static int cmd_handler_mi_report_perf_count(struct parser_exec_state *s)
  1255. {
  1256. return unexpected_cmd(s);
  1257. }
  1258. static int cmd_handler_mi_op_2e(struct parser_exec_state *s)
  1259. {
  1260. return unexpected_cmd(s);
  1261. }
  1262. static int cmd_handler_mi_op_2f(struct parser_exec_state *s)
  1263. {
  1264. int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
  1265. int op_size = (1 << ((cmd_val(s, 0) & GENMASK(20, 19)) >> 19)) *
  1266. sizeof(u32);
  1267. unsigned long gma, gma_high;
  1268. int ret = 0;
  1269. if (!(cmd_val(s, 0) & (1 << 22)))
  1270. return ret;
  1271. gma = cmd_val(s, 1) & GENMASK(31, 2);
  1272. if (gmadr_bytes == 8) {
  1273. gma_high = cmd_val(s, 2) & GENMASK(15, 0);
  1274. gma = (gma_high << 32) | gma;
  1275. }
  1276. ret = cmd_address_audit(s, gma, op_size, false);
  1277. return ret;
  1278. }
  1279. static int cmd_handler_mi_store_data_index(struct parser_exec_state *s)
  1280. {
  1281. return unexpected_cmd(s);
  1282. }
  1283. static int cmd_handler_mi_clflush(struct parser_exec_state *s)
  1284. {
  1285. return unexpected_cmd(s);
  1286. }
  1287. static int cmd_handler_mi_conditional_batch_buffer_end(
  1288. struct parser_exec_state *s)
  1289. {
  1290. return unexpected_cmd(s);
  1291. }
  1292. static int cmd_handler_mi_update_gtt(struct parser_exec_state *s)
  1293. {
  1294. return unexpected_cmd(s);
  1295. }
  1296. static int cmd_handler_mi_flush_dw(struct parser_exec_state *s)
  1297. {
  1298. int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
  1299. unsigned long gma;
  1300. bool index_mode = false;
  1301. int ret = 0;
  1302. /* Check post-sync and ppgtt bit */
  1303. if (((cmd_val(s, 0) >> 14) & 0x3) && (cmd_val(s, 1) & (1 << 2))) {
  1304. gma = cmd_val(s, 1) & GENMASK(31, 3);
  1305. if (gmadr_bytes == 8)
  1306. gma |= (cmd_val(s, 2) & GENMASK(15, 0)) << 32;
  1307. /* Store Data Index */
  1308. if (cmd_val(s, 0) & (1 << 21))
  1309. index_mode = true;
  1310. ret = cmd_address_audit(s, gma, sizeof(u64), index_mode);
  1311. }
  1312. /* Check notify bit */
  1313. if ((cmd_val(s, 0) & (1 << 8)))
  1314. set_bit(cmd_interrupt_events[s->ring_id].mi_flush_dw,
  1315. s->workload->pending_events);
  1316. return ret;
  1317. }
  1318. static void addr_type_update_snb(struct parser_exec_state *s)
  1319. {
  1320. if ((s->buf_type == RING_BUFFER_INSTRUCTION) &&
  1321. (BATCH_BUFFER_ADR_SPACE_BIT(cmd_val(s, 0)) == 1)) {
  1322. s->buf_addr_type = PPGTT_BUFFER;
  1323. }
  1324. }
  1325. static int copy_gma_to_hva(struct intel_vgpu *vgpu, struct intel_vgpu_mm *mm,
  1326. unsigned long gma, unsigned long end_gma, void *va)
  1327. {
  1328. unsigned long copy_len, offset;
  1329. unsigned long len = 0;
  1330. unsigned long gpa;
  1331. while (gma != end_gma) {
  1332. gpa = intel_vgpu_gma_to_gpa(mm, gma);
  1333. if (gpa == INTEL_GVT_INVALID_ADDR) {
  1334. gvt_vgpu_err("invalid gma address: %lx\n", gma);
  1335. return -EFAULT;
  1336. }
  1337. offset = gma & (I915_GTT_PAGE_SIZE - 1);
  1338. copy_len = (end_gma - gma) >= (I915_GTT_PAGE_SIZE - offset) ?
  1339. I915_GTT_PAGE_SIZE - offset : end_gma - gma;
  1340. intel_gvt_hypervisor_read_gpa(vgpu, gpa, va + len, copy_len);
  1341. len += copy_len;
  1342. gma += copy_len;
  1343. }
  1344. return len;
  1345. }
  1346. /*
  1347. * Check whether a batch buffer needs to be scanned. Currently
  1348. * the only criteria is based on privilege.
  1349. */
  1350. static int batch_buffer_needs_scan(struct parser_exec_state *s)
  1351. {
  1352. struct intel_gvt *gvt = s->vgpu->gvt;
  1353. if (IS_BROADWELL(gvt->dev_priv) || IS_SKYLAKE(gvt->dev_priv)
  1354. || IS_KABYLAKE(gvt->dev_priv)) {
  1355. /* BDW decides privilege based on address space */
  1356. if (cmd_val(s, 0) & (1 << 8))
  1357. return 0;
  1358. }
  1359. return 1;
  1360. }
  1361. static int find_bb_size(struct parser_exec_state *s, unsigned long *bb_size)
  1362. {
  1363. unsigned long gma = 0;
  1364. struct cmd_info *info;
  1365. uint32_t cmd_len = 0;
  1366. bool bb_end = false;
  1367. struct intel_vgpu *vgpu = s->vgpu;
  1368. u32 cmd;
  1369. *bb_size = 0;
  1370. /* get the start gm address of the batch buffer */
  1371. gma = get_gma_bb_from_cmd(s, 1);
  1372. if (gma == INTEL_GVT_INVALID_ADDR)
  1373. return -EFAULT;
  1374. cmd = cmd_val(s, 0);
  1375. info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
  1376. if (info == NULL) {
  1377. gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n",
  1378. cmd, get_opcode(cmd, s->ring_id));
  1379. return -EBADRQC;
  1380. }
  1381. do {
  1382. if (copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm,
  1383. gma, gma + 4, &cmd) < 0)
  1384. return -EFAULT;
  1385. info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
  1386. if (info == NULL) {
  1387. gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n",
  1388. cmd, get_opcode(cmd, s->ring_id));
  1389. return -EBADRQC;
  1390. }
  1391. if (info->opcode == OP_MI_BATCH_BUFFER_END) {
  1392. bb_end = true;
  1393. } else if (info->opcode == OP_MI_BATCH_BUFFER_START) {
  1394. if (BATCH_BUFFER_2ND_LEVEL_BIT(cmd) == 0)
  1395. /* chained batch buffer */
  1396. bb_end = true;
  1397. }
  1398. cmd_len = get_cmd_length(info, cmd) << 2;
  1399. *bb_size += cmd_len;
  1400. gma += cmd_len;
  1401. } while (!bb_end);
  1402. return 0;
  1403. }
  1404. static int perform_bb_shadow(struct parser_exec_state *s)
  1405. {
  1406. struct intel_vgpu *vgpu = s->vgpu;
  1407. struct intel_vgpu_shadow_bb *bb;
  1408. unsigned long gma = 0;
  1409. unsigned long bb_size;
  1410. int ret = 0;
  1411. /* get the start gm address of the batch buffer */
  1412. gma = get_gma_bb_from_cmd(s, 1);
  1413. if (gma == INTEL_GVT_INVALID_ADDR)
  1414. return -EFAULT;
  1415. ret = find_bb_size(s, &bb_size);
  1416. if (ret)
  1417. return ret;
  1418. bb = kzalloc(sizeof(*bb), GFP_KERNEL);
  1419. if (!bb)
  1420. return -ENOMEM;
  1421. bb->obj = i915_gem_object_create(s->vgpu->gvt->dev_priv,
  1422. roundup(bb_size, PAGE_SIZE));
  1423. if (IS_ERR(bb->obj)) {
  1424. ret = PTR_ERR(bb->obj);
  1425. goto err_free_bb;
  1426. }
  1427. ret = i915_gem_obj_prepare_shmem_write(bb->obj, &bb->clflush);
  1428. if (ret)
  1429. goto err_free_obj;
  1430. bb->va = i915_gem_object_pin_map(bb->obj, I915_MAP_WB);
  1431. if (IS_ERR(bb->va)) {
  1432. ret = PTR_ERR(bb->va);
  1433. goto err_finish_shmem_access;
  1434. }
  1435. if (bb->clflush & CLFLUSH_BEFORE) {
  1436. drm_clflush_virt_range(bb->va, bb->obj->base.size);
  1437. bb->clflush &= ~CLFLUSH_BEFORE;
  1438. }
  1439. ret = copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm,
  1440. gma, gma + bb_size,
  1441. bb->va);
  1442. if (ret < 0) {
  1443. gvt_vgpu_err("fail to copy guest ring buffer\n");
  1444. ret = -EFAULT;
  1445. goto err_unmap;
  1446. }
  1447. INIT_LIST_HEAD(&bb->list);
  1448. list_add(&bb->list, &s->workload->shadow_bb);
  1449. bb->accessing = true;
  1450. bb->bb_start_cmd_va = s->ip_va;
  1451. /*
  1452. * ip_va saves the virtual address of the shadow batch buffer, while
  1453. * ip_gma saves the graphics address of the original batch buffer.
  1454. * As the shadow batch buffer is just a copy from the originial one,
  1455. * it should be right to use shadow batch buffer'va and original batch
  1456. * buffer's gma in pair. After all, we don't want to pin the shadow
  1457. * buffer here (too early).
  1458. */
  1459. s->ip_va = bb->va;
  1460. s->ip_gma = gma;
  1461. return 0;
  1462. err_unmap:
  1463. i915_gem_object_unpin_map(bb->obj);
  1464. err_finish_shmem_access:
  1465. i915_gem_obj_finish_shmem_access(bb->obj);
  1466. err_free_obj:
  1467. i915_gem_object_put(bb->obj);
  1468. err_free_bb:
  1469. kfree(bb);
  1470. return ret;
  1471. }
  1472. static int cmd_handler_mi_batch_buffer_start(struct parser_exec_state *s)
  1473. {
  1474. bool second_level;
  1475. int ret = 0;
  1476. struct intel_vgpu *vgpu = s->vgpu;
  1477. if (s->buf_type == BATCH_BUFFER_2ND_LEVEL) {
  1478. gvt_vgpu_err("Found MI_BATCH_BUFFER_START in 2nd level BB\n");
  1479. return -EFAULT;
  1480. }
  1481. second_level = BATCH_BUFFER_2ND_LEVEL_BIT(cmd_val(s, 0)) == 1;
  1482. if (second_level && (s->buf_type != BATCH_BUFFER_INSTRUCTION)) {
  1483. gvt_vgpu_err("Jumping to 2nd level BB from RB is not allowed\n");
  1484. return -EFAULT;
  1485. }
  1486. s->saved_buf_addr_type = s->buf_addr_type;
  1487. addr_type_update_snb(s);
  1488. if (s->buf_type == RING_BUFFER_INSTRUCTION) {
  1489. s->ret_ip_gma_ring = s->ip_gma + cmd_length(s) * sizeof(u32);
  1490. s->buf_type = BATCH_BUFFER_INSTRUCTION;
  1491. } else if (second_level) {
  1492. s->buf_type = BATCH_BUFFER_2ND_LEVEL;
  1493. s->ret_ip_gma_bb = s->ip_gma + cmd_length(s) * sizeof(u32);
  1494. s->ret_bb_va = s->ip_va + cmd_length(s) * sizeof(u32);
  1495. }
  1496. if (batch_buffer_needs_scan(s)) {
  1497. ret = perform_bb_shadow(s);
  1498. if (ret < 0)
  1499. gvt_vgpu_err("invalid shadow batch buffer\n");
  1500. } else {
  1501. /* emulate a batch buffer end to do return right */
  1502. ret = cmd_handler_mi_batch_buffer_end(s);
  1503. if (ret < 0)
  1504. return ret;
  1505. }
  1506. return ret;
  1507. }
  1508. static struct cmd_info cmd_info[] = {
  1509. {"MI_NOOP", OP_MI_NOOP, F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL},
  1510. {"MI_SET_PREDICATE", OP_MI_SET_PREDICATE, F_LEN_CONST, R_ALL, D_ALL,
  1511. 0, 1, NULL},
  1512. {"MI_USER_INTERRUPT", OP_MI_USER_INTERRUPT, F_LEN_CONST, R_ALL, D_ALL,
  1513. 0, 1, cmd_handler_mi_user_interrupt},
  1514. {"MI_WAIT_FOR_EVENT", OP_MI_WAIT_FOR_EVENT, F_LEN_CONST, R_RCS | R_BCS,
  1515. D_ALL, 0, 1, cmd_handler_mi_wait_for_event},
  1516. {"MI_FLUSH", OP_MI_FLUSH, F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL},
  1517. {"MI_ARB_CHECK", OP_MI_ARB_CHECK, F_LEN_CONST, R_ALL, D_ALL, 0, 1,
  1518. NULL},
  1519. {"MI_RS_CONTROL", OP_MI_RS_CONTROL, F_LEN_CONST, R_RCS, D_ALL, 0, 1,
  1520. NULL},
  1521. {"MI_REPORT_HEAD", OP_MI_REPORT_HEAD, F_LEN_CONST, R_ALL, D_ALL, 0, 1,
  1522. NULL},
  1523. {"MI_ARB_ON_OFF", OP_MI_ARB_ON_OFF, F_LEN_CONST, R_ALL, D_ALL, 0, 1,
  1524. NULL},
  1525. {"MI_URB_ATOMIC_ALLOC", OP_MI_URB_ATOMIC_ALLOC, F_LEN_CONST, R_RCS,
  1526. D_ALL, 0, 1, NULL},
  1527. {"MI_BATCH_BUFFER_END", OP_MI_BATCH_BUFFER_END,
  1528. F_IP_ADVANCE_CUSTOM | F_LEN_CONST, R_ALL, D_ALL, 0, 1,
  1529. cmd_handler_mi_batch_buffer_end},
  1530. {"MI_SUSPEND_FLUSH", OP_MI_SUSPEND_FLUSH, F_LEN_CONST, R_ALL, D_ALL,
  1531. 0, 1, NULL},
  1532. {"MI_PREDICATE", OP_MI_PREDICATE, F_LEN_CONST, R_RCS, D_ALL, 0, 1,
  1533. NULL},
  1534. {"MI_TOPOLOGY_FILTER", OP_MI_TOPOLOGY_FILTER, F_LEN_CONST, R_ALL,
  1535. D_ALL, 0, 1, NULL},
  1536. {"MI_SET_APPID", OP_MI_SET_APPID, F_LEN_CONST, R_ALL, D_ALL, 0, 1,
  1537. NULL},
  1538. {"MI_RS_CONTEXT", OP_MI_RS_CONTEXT, F_LEN_CONST, R_RCS, D_ALL, 0, 1,
  1539. NULL},
  1540. {"MI_DISPLAY_FLIP", OP_MI_DISPLAY_FLIP, F_LEN_VAR | F_POST_HANDLE,
  1541. R_RCS | R_BCS, D_ALL, 0, 8, cmd_handler_mi_display_flip},
  1542. {"MI_SEMAPHORE_MBOX", OP_MI_SEMAPHORE_MBOX, F_LEN_VAR, R_ALL, D_ALL,
  1543. 0, 8, NULL},
  1544. {"MI_MATH", OP_MI_MATH, F_LEN_VAR, R_ALL, D_ALL, 0, 8, NULL},
  1545. {"MI_URB_CLEAR", OP_MI_URB_CLEAR, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1546. {"ME_SEMAPHORE_SIGNAL", OP_MI_SEMAPHORE_SIGNAL, F_LEN_VAR, R_ALL,
  1547. D_BDW_PLUS, 0, 8, NULL},
  1548. {"ME_SEMAPHORE_WAIT", OP_MI_SEMAPHORE_WAIT, F_LEN_VAR, R_ALL, D_BDW_PLUS,
  1549. ADDR_FIX_1(2), 8, cmd_handler_mi_semaphore_wait},
  1550. {"MI_STORE_DATA_IMM", OP_MI_STORE_DATA_IMM, F_LEN_VAR, R_ALL, D_BDW_PLUS,
  1551. ADDR_FIX_1(1), 10, cmd_handler_mi_store_data_imm},
  1552. {"MI_STORE_DATA_INDEX", OP_MI_STORE_DATA_INDEX, F_LEN_VAR, R_ALL, D_ALL,
  1553. 0, 8, cmd_handler_mi_store_data_index},
  1554. {"MI_LOAD_REGISTER_IMM", OP_MI_LOAD_REGISTER_IMM, F_LEN_VAR, R_ALL,
  1555. D_ALL, 0, 8, cmd_handler_lri},
  1556. {"MI_UPDATE_GTT", OP_MI_UPDATE_GTT, F_LEN_VAR, R_ALL, D_BDW_PLUS, 0, 10,
  1557. cmd_handler_mi_update_gtt},
  1558. {"MI_STORE_REGISTER_MEM", OP_MI_STORE_REGISTER_MEM, F_LEN_VAR, R_ALL,
  1559. D_ALL, ADDR_FIX_1(2), 8, cmd_handler_srm},
  1560. {"MI_FLUSH_DW", OP_MI_FLUSH_DW, F_LEN_VAR, R_ALL, D_ALL, 0, 6,
  1561. cmd_handler_mi_flush_dw},
  1562. {"MI_CLFLUSH", OP_MI_CLFLUSH, F_LEN_VAR, R_ALL, D_ALL, ADDR_FIX_1(1),
  1563. 10, cmd_handler_mi_clflush},
  1564. {"MI_REPORT_PERF_COUNT", OP_MI_REPORT_PERF_COUNT, F_LEN_VAR, R_ALL,
  1565. D_ALL, ADDR_FIX_1(1), 6, cmd_handler_mi_report_perf_count},
  1566. {"MI_LOAD_REGISTER_MEM", OP_MI_LOAD_REGISTER_MEM, F_LEN_VAR, R_ALL,
  1567. D_ALL, ADDR_FIX_1(2), 8, cmd_handler_lrm},
  1568. {"MI_LOAD_REGISTER_REG", OP_MI_LOAD_REGISTER_REG, F_LEN_VAR, R_ALL,
  1569. D_ALL, 0, 8, cmd_handler_lrr},
  1570. {"MI_RS_STORE_DATA_IMM", OP_MI_RS_STORE_DATA_IMM, F_LEN_VAR, R_RCS,
  1571. D_ALL, 0, 8, NULL},
  1572. {"MI_LOAD_URB_MEM", OP_MI_LOAD_URB_MEM, F_LEN_VAR, R_RCS, D_ALL,
  1573. ADDR_FIX_1(2), 8, NULL},
  1574. {"MI_STORE_URM_MEM", OP_MI_STORE_URM_MEM, F_LEN_VAR, R_RCS, D_ALL,
  1575. ADDR_FIX_1(2), 8, NULL},
  1576. {"MI_OP_2E", OP_MI_2E, F_LEN_VAR, R_ALL, D_BDW_PLUS, ADDR_FIX_2(1, 2),
  1577. 8, cmd_handler_mi_op_2e},
  1578. {"MI_OP_2F", OP_MI_2F, F_LEN_VAR, R_ALL, D_BDW_PLUS, ADDR_FIX_1(1),
  1579. 8, cmd_handler_mi_op_2f},
  1580. {"MI_BATCH_BUFFER_START", OP_MI_BATCH_BUFFER_START,
  1581. F_IP_ADVANCE_CUSTOM, R_ALL, D_ALL, 0, 8,
  1582. cmd_handler_mi_batch_buffer_start},
  1583. {"MI_CONDITIONAL_BATCH_BUFFER_END", OP_MI_CONDITIONAL_BATCH_BUFFER_END,
  1584. F_LEN_VAR, R_ALL, D_ALL, ADDR_FIX_1(2), 8,
  1585. cmd_handler_mi_conditional_batch_buffer_end},
  1586. {"MI_LOAD_SCAN_LINES_INCL", OP_MI_LOAD_SCAN_LINES_INCL, F_LEN_CONST,
  1587. R_RCS | R_BCS, D_ALL, 0, 2, NULL},
  1588. {"XY_SETUP_BLT", OP_XY_SETUP_BLT, F_LEN_VAR, R_BCS, D_ALL,
  1589. ADDR_FIX_2(4, 7), 8, NULL},
  1590. {"XY_SETUP_CLIP_BLT", OP_XY_SETUP_CLIP_BLT, F_LEN_VAR, R_BCS, D_ALL,
  1591. 0, 8, NULL},
  1592. {"XY_SETUP_MONO_PATTERN_SL_BLT", OP_XY_SETUP_MONO_PATTERN_SL_BLT,
  1593. F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL},
  1594. {"XY_PIXEL_BLT", OP_XY_PIXEL_BLT, F_LEN_VAR, R_BCS, D_ALL, 0, 8, NULL},
  1595. {"XY_SCANLINES_BLT", OP_XY_SCANLINES_BLT, F_LEN_VAR, R_BCS, D_ALL,
  1596. 0, 8, NULL},
  1597. {"XY_TEXT_BLT", OP_XY_TEXT_BLT, F_LEN_VAR, R_BCS, D_ALL,
  1598. ADDR_FIX_1(3), 8, NULL},
  1599. {"XY_TEXT_IMMEDIATE_BLT", OP_XY_TEXT_IMMEDIATE_BLT, F_LEN_VAR, R_BCS,
  1600. D_ALL, 0, 8, NULL},
  1601. {"XY_COLOR_BLT", OP_XY_COLOR_BLT, F_LEN_VAR, R_BCS, D_ALL,
  1602. ADDR_FIX_1(4), 8, NULL},
  1603. {"XY_PAT_BLT", OP_XY_PAT_BLT, F_LEN_VAR, R_BCS, D_ALL,
  1604. ADDR_FIX_2(4, 5), 8, NULL},
  1605. {"XY_MONO_PAT_BLT", OP_XY_MONO_PAT_BLT, F_LEN_VAR, R_BCS, D_ALL,
  1606. ADDR_FIX_1(4), 8, NULL},
  1607. {"XY_SRC_COPY_BLT", OP_XY_SRC_COPY_BLT, F_LEN_VAR, R_BCS, D_ALL,
  1608. ADDR_FIX_2(4, 7), 8, NULL},
  1609. {"XY_MONO_SRC_COPY_BLT", OP_XY_MONO_SRC_COPY_BLT, F_LEN_VAR, R_BCS,
  1610. D_ALL, ADDR_FIX_2(4, 5), 8, NULL},
  1611. {"XY_FULL_BLT", OP_XY_FULL_BLT, F_LEN_VAR, R_BCS, D_ALL, 0, 8, NULL},
  1612. {"XY_FULL_MONO_SRC_BLT", OP_XY_FULL_MONO_SRC_BLT, F_LEN_VAR, R_BCS,
  1613. D_ALL, ADDR_FIX_3(4, 5, 8), 8, NULL},
  1614. {"XY_FULL_MONO_PATTERN_BLT", OP_XY_FULL_MONO_PATTERN_BLT, F_LEN_VAR,
  1615. R_BCS, D_ALL, ADDR_FIX_2(4, 7), 8, NULL},
  1616. {"XY_FULL_MONO_PATTERN_MONO_SRC_BLT",
  1617. OP_XY_FULL_MONO_PATTERN_MONO_SRC_BLT,
  1618. F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 5), 8, NULL},
  1619. {"XY_MONO_PAT_FIXED_BLT", OP_XY_MONO_PAT_FIXED_BLT, F_LEN_VAR, R_BCS,
  1620. D_ALL, ADDR_FIX_1(4), 8, NULL},
  1621. {"XY_MONO_SRC_COPY_IMMEDIATE_BLT", OP_XY_MONO_SRC_COPY_IMMEDIATE_BLT,
  1622. F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL},
  1623. {"XY_PAT_BLT_IMMEDIATE", OP_XY_PAT_BLT_IMMEDIATE, F_LEN_VAR, R_BCS,
  1624. D_ALL, ADDR_FIX_1(4), 8, NULL},
  1625. {"XY_SRC_COPY_CHROMA_BLT", OP_XY_SRC_COPY_CHROMA_BLT, F_LEN_VAR, R_BCS,
  1626. D_ALL, ADDR_FIX_2(4, 7), 8, NULL},
  1627. {"XY_FULL_IMMEDIATE_PATTERN_BLT", OP_XY_FULL_IMMEDIATE_PATTERN_BLT,
  1628. F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 7), 8, NULL},
  1629. {"XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT",
  1630. OP_XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT,
  1631. F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 5), 8, NULL},
  1632. {"XY_PAT_CHROMA_BLT", OP_XY_PAT_CHROMA_BLT, F_LEN_VAR, R_BCS, D_ALL,
  1633. ADDR_FIX_2(4, 5), 8, NULL},
  1634. {"XY_PAT_CHROMA_BLT_IMMEDIATE", OP_XY_PAT_CHROMA_BLT_IMMEDIATE,
  1635. F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL},
  1636. {"3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP",
  1637. OP_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP,
  1638. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1639. {"3DSTATE_VIEWPORT_STATE_POINTERS_CC",
  1640. OP_3DSTATE_VIEWPORT_STATE_POINTERS_CC,
  1641. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1642. {"3DSTATE_BLEND_STATE_POINTERS",
  1643. OP_3DSTATE_BLEND_STATE_POINTERS,
  1644. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1645. {"3DSTATE_DEPTH_STENCIL_STATE_POINTERS",
  1646. OP_3DSTATE_DEPTH_STENCIL_STATE_POINTERS,
  1647. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1648. {"3DSTATE_BINDING_TABLE_POINTERS_VS",
  1649. OP_3DSTATE_BINDING_TABLE_POINTERS_VS,
  1650. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1651. {"3DSTATE_BINDING_TABLE_POINTERS_HS",
  1652. OP_3DSTATE_BINDING_TABLE_POINTERS_HS,
  1653. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1654. {"3DSTATE_BINDING_TABLE_POINTERS_DS",
  1655. OP_3DSTATE_BINDING_TABLE_POINTERS_DS,
  1656. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1657. {"3DSTATE_BINDING_TABLE_POINTERS_GS",
  1658. OP_3DSTATE_BINDING_TABLE_POINTERS_GS,
  1659. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1660. {"3DSTATE_BINDING_TABLE_POINTERS_PS",
  1661. OP_3DSTATE_BINDING_TABLE_POINTERS_PS,
  1662. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1663. {"3DSTATE_SAMPLER_STATE_POINTERS_VS",
  1664. OP_3DSTATE_SAMPLER_STATE_POINTERS_VS,
  1665. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1666. {"3DSTATE_SAMPLER_STATE_POINTERS_HS",
  1667. OP_3DSTATE_SAMPLER_STATE_POINTERS_HS,
  1668. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1669. {"3DSTATE_SAMPLER_STATE_POINTERS_DS",
  1670. OP_3DSTATE_SAMPLER_STATE_POINTERS_DS,
  1671. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1672. {"3DSTATE_SAMPLER_STATE_POINTERS_GS",
  1673. OP_3DSTATE_SAMPLER_STATE_POINTERS_GS,
  1674. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1675. {"3DSTATE_SAMPLER_STATE_POINTERS_PS",
  1676. OP_3DSTATE_SAMPLER_STATE_POINTERS_PS,
  1677. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1678. {"3DSTATE_URB_VS", OP_3DSTATE_URB_VS, F_LEN_VAR, R_RCS, D_ALL,
  1679. 0, 8, NULL},
  1680. {"3DSTATE_URB_HS", OP_3DSTATE_URB_HS, F_LEN_VAR, R_RCS, D_ALL,
  1681. 0, 8, NULL},
  1682. {"3DSTATE_URB_DS", OP_3DSTATE_URB_DS, F_LEN_VAR, R_RCS, D_ALL,
  1683. 0, 8, NULL},
  1684. {"3DSTATE_URB_GS", OP_3DSTATE_URB_GS, F_LEN_VAR, R_RCS, D_ALL,
  1685. 0, 8, NULL},
  1686. {"3DSTATE_GATHER_CONSTANT_VS", OP_3DSTATE_GATHER_CONSTANT_VS,
  1687. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1688. {"3DSTATE_GATHER_CONSTANT_GS", OP_3DSTATE_GATHER_CONSTANT_GS,
  1689. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1690. {"3DSTATE_GATHER_CONSTANT_HS", OP_3DSTATE_GATHER_CONSTANT_HS,
  1691. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1692. {"3DSTATE_GATHER_CONSTANT_DS", OP_3DSTATE_GATHER_CONSTANT_DS,
  1693. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1694. {"3DSTATE_GATHER_CONSTANT_PS", OP_3DSTATE_GATHER_CONSTANT_PS,
  1695. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1696. {"3DSTATE_DX9_CONSTANTF_VS", OP_3DSTATE_DX9_CONSTANTF_VS,
  1697. F_LEN_VAR, R_RCS, D_ALL, 0, 11, NULL},
  1698. {"3DSTATE_DX9_CONSTANTF_PS", OP_3DSTATE_DX9_CONSTANTF_PS,
  1699. F_LEN_VAR, R_RCS, D_ALL, 0, 11, NULL},
  1700. {"3DSTATE_DX9_CONSTANTI_VS", OP_3DSTATE_DX9_CONSTANTI_VS,
  1701. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1702. {"3DSTATE_DX9_CONSTANTI_PS", OP_3DSTATE_DX9_CONSTANTI_PS,
  1703. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1704. {"3DSTATE_DX9_CONSTANTB_VS", OP_3DSTATE_DX9_CONSTANTB_VS,
  1705. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1706. {"3DSTATE_DX9_CONSTANTB_PS", OP_3DSTATE_DX9_CONSTANTB_PS,
  1707. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1708. {"3DSTATE_DX9_LOCAL_VALID_VS", OP_3DSTATE_DX9_LOCAL_VALID_VS,
  1709. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1710. {"3DSTATE_DX9_LOCAL_VALID_PS", OP_3DSTATE_DX9_LOCAL_VALID_PS,
  1711. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1712. {"3DSTATE_DX9_GENERATE_ACTIVE_VS", OP_3DSTATE_DX9_GENERATE_ACTIVE_VS,
  1713. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1714. {"3DSTATE_DX9_GENERATE_ACTIVE_PS", OP_3DSTATE_DX9_GENERATE_ACTIVE_PS,
  1715. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1716. {"3DSTATE_BINDING_TABLE_EDIT_VS", OP_3DSTATE_BINDING_TABLE_EDIT_VS,
  1717. F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
  1718. {"3DSTATE_BINDING_TABLE_EDIT_GS", OP_3DSTATE_BINDING_TABLE_EDIT_GS,
  1719. F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
  1720. {"3DSTATE_BINDING_TABLE_EDIT_HS", OP_3DSTATE_BINDING_TABLE_EDIT_HS,
  1721. F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
  1722. {"3DSTATE_BINDING_TABLE_EDIT_DS", OP_3DSTATE_BINDING_TABLE_EDIT_DS,
  1723. F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
  1724. {"3DSTATE_BINDING_TABLE_EDIT_PS", OP_3DSTATE_BINDING_TABLE_EDIT_PS,
  1725. F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
  1726. {"3DSTATE_VF_INSTANCING", OP_3DSTATE_VF_INSTANCING, F_LEN_VAR, R_RCS,
  1727. D_BDW_PLUS, 0, 8, NULL},
  1728. {"3DSTATE_VF_SGVS", OP_3DSTATE_VF_SGVS, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8,
  1729. NULL},
  1730. {"3DSTATE_VF_TOPOLOGY", OP_3DSTATE_VF_TOPOLOGY, F_LEN_VAR, R_RCS,
  1731. D_BDW_PLUS, 0, 8, NULL},
  1732. {"3DSTATE_WM_CHROMAKEY", OP_3DSTATE_WM_CHROMAKEY, F_LEN_VAR, R_RCS,
  1733. D_BDW_PLUS, 0, 8, NULL},
  1734. {"3DSTATE_PS_BLEND", OP_3DSTATE_PS_BLEND, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0,
  1735. 8, NULL},
  1736. {"3DSTATE_WM_DEPTH_STENCIL", OP_3DSTATE_WM_DEPTH_STENCIL, F_LEN_VAR,
  1737. R_RCS, D_BDW_PLUS, 0, 8, NULL},
  1738. {"3DSTATE_PS_EXTRA", OP_3DSTATE_PS_EXTRA, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0,
  1739. 8, NULL},
  1740. {"3DSTATE_RASTER", OP_3DSTATE_RASTER, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8,
  1741. NULL},
  1742. {"3DSTATE_SBE_SWIZ", OP_3DSTATE_SBE_SWIZ, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8,
  1743. NULL},
  1744. {"3DSTATE_WM_HZ_OP", OP_3DSTATE_WM_HZ_OP, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8,
  1745. NULL},
  1746. {"3DSTATE_VERTEX_BUFFERS", OP_3DSTATE_VERTEX_BUFFERS, F_LEN_VAR, R_RCS,
  1747. D_BDW_PLUS, 0, 8, NULL},
  1748. {"3DSTATE_VERTEX_ELEMENTS", OP_3DSTATE_VERTEX_ELEMENTS, F_LEN_VAR,
  1749. R_RCS, D_ALL, 0, 8, NULL},
  1750. {"3DSTATE_INDEX_BUFFER", OP_3DSTATE_INDEX_BUFFER, F_LEN_VAR, R_RCS,
  1751. D_BDW_PLUS, ADDR_FIX_1(2), 8, NULL},
  1752. {"3DSTATE_VF_STATISTICS", OP_3DSTATE_VF_STATISTICS, F_LEN_CONST,
  1753. R_RCS, D_ALL, 0, 1, NULL},
  1754. {"3DSTATE_VF", OP_3DSTATE_VF, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1755. {"3DSTATE_CC_STATE_POINTERS", OP_3DSTATE_CC_STATE_POINTERS, F_LEN_VAR,
  1756. R_RCS, D_ALL, 0, 8, NULL},
  1757. {"3DSTATE_SCISSOR_STATE_POINTERS", OP_3DSTATE_SCISSOR_STATE_POINTERS,
  1758. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1759. {"3DSTATE_GS", OP_3DSTATE_GS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1760. {"3DSTATE_CLIP", OP_3DSTATE_CLIP, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1761. {"3DSTATE_WM", OP_3DSTATE_WM, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1762. {"3DSTATE_CONSTANT_GS", OP_3DSTATE_CONSTANT_GS, F_LEN_VAR, R_RCS,
  1763. D_BDW_PLUS, 0, 8, NULL},
  1764. {"3DSTATE_CONSTANT_PS", OP_3DSTATE_CONSTANT_PS, F_LEN_VAR, R_RCS,
  1765. D_BDW_PLUS, 0, 8, NULL},
  1766. {"3DSTATE_SAMPLE_MASK", OP_3DSTATE_SAMPLE_MASK, F_LEN_VAR, R_RCS,
  1767. D_ALL, 0, 8, NULL},
  1768. {"3DSTATE_CONSTANT_HS", OP_3DSTATE_CONSTANT_HS, F_LEN_VAR, R_RCS,
  1769. D_BDW_PLUS, 0, 8, NULL},
  1770. {"3DSTATE_CONSTANT_DS", OP_3DSTATE_CONSTANT_DS, F_LEN_VAR, R_RCS,
  1771. D_BDW_PLUS, 0, 8, NULL},
  1772. {"3DSTATE_HS", OP_3DSTATE_HS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1773. {"3DSTATE_TE", OP_3DSTATE_TE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1774. {"3DSTATE_DS", OP_3DSTATE_DS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1775. {"3DSTATE_STREAMOUT", OP_3DSTATE_STREAMOUT, F_LEN_VAR, R_RCS,
  1776. D_ALL, 0, 8, NULL},
  1777. {"3DSTATE_SBE", OP_3DSTATE_SBE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1778. {"3DSTATE_PS", OP_3DSTATE_PS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1779. {"3DSTATE_DRAWING_RECTANGLE", OP_3DSTATE_DRAWING_RECTANGLE, F_LEN_VAR,
  1780. R_RCS, D_ALL, 0, 8, NULL},
  1781. {"3DSTATE_SAMPLER_PALETTE_LOAD0", OP_3DSTATE_SAMPLER_PALETTE_LOAD0,
  1782. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1783. {"3DSTATE_CHROMA_KEY", OP_3DSTATE_CHROMA_KEY, F_LEN_VAR, R_RCS, D_ALL,
  1784. 0, 8, NULL},
  1785. {"3DSTATE_DEPTH_BUFFER", OP_3DSTATE_DEPTH_BUFFER, F_LEN_VAR, R_RCS,
  1786. D_ALL, ADDR_FIX_1(2), 8, NULL},
  1787. {"3DSTATE_POLY_STIPPLE_OFFSET", OP_3DSTATE_POLY_STIPPLE_OFFSET,
  1788. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1789. {"3DSTATE_POLY_STIPPLE_PATTERN", OP_3DSTATE_POLY_STIPPLE_PATTERN,
  1790. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1791. {"3DSTATE_LINE_STIPPLE", OP_3DSTATE_LINE_STIPPLE, F_LEN_VAR, R_RCS,
  1792. D_ALL, 0, 8, NULL},
  1793. {"3DSTATE_AA_LINE_PARAMS", OP_3DSTATE_AA_LINE_PARAMS, F_LEN_VAR, R_RCS,
  1794. D_ALL, 0, 8, NULL},
  1795. {"3DSTATE_GS_SVB_INDEX", OP_3DSTATE_GS_SVB_INDEX, F_LEN_VAR, R_RCS,
  1796. D_ALL, 0, 8, NULL},
  1797. {"3DSTATE_SAMPLER_PALETTE_LOAD1", OP_3DSTATE_SAMPLER_PALETTE_LOAD1,
  1798. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1799. {"3DSTATE_MULTISAMPLE", OP_3DSTATE_MULTISAMPLE_BDW, F_LEN_VAR, R_RCS,
  1800. D_BDW_PLUS, 0, 8, NULL},
  1801. {"3DSTATE_STENCIL_BUFFER", OP_3DSTATE_STENCIL_BUFFER, F_LEN_VAR, R_RCS,
  1802. D_ALL, ADDR_FIX_1(2), 8, NULL},
  1803. {"3DSTATE_HIER_DEPTH_BUFFER", OP_3DSTATE_HIER_DEPTH_BUFFER, F_LEN_VAR,
  1804. R_RCS, D_ALL, ADDR_FIX_1(2), 8, NULL},
  1805. {"3DSTATE_CLEAR_PARAMS", OP_3DSTATE_CLEAR_PARAMS, F_LEN_VAR,
  1806. R_RCS, D_ALL, 0, 8, NULL},
  1807. {"3DSTATE_PUSH_CONSTANT_ALLOC_VS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_VS,
  1808. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1809. {"3DSTATE_PUSH_CONSTANT_ALLOC_HS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_HS,
  1810. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1811. {"3DSTATE_PUSH_CONSTANT_ALLOC_DS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_DS,
  1812. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1813. {"3DSTATE_PUSH_CONSTANT_ALLOC_GS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_GS,
  1814. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1815. {"3DSTATE_PUSH_CONSTANT_ALLOC_PS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_PS,
  1816. F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1817. {"3DSTATE_MONOFILTER_SIZE", OP_3DSTATE_MONOFILTER_SIZE, F_LEN_VAR,
  1818. R_RCS, D_ALL, 0, 8, NULL},
  1819. {"3DSTATE_SO_DECL_LIST", OP_3DSTATE_SO_DECL_LIST, F_LEN_VAR, R_RCS,
  1820. D_ALL, 0, 9, NULL},
  1821. {"3DSTATE_SO_BUFFER", OP_3DSTATE_SO_BUFFER, F_LEN_VAR, R_RCS, D_BDW_PLUS,
  1822. ADDR_FIX_2(2, 4), 8, NULL},
  1823. {"3DSTATE_BINDING_TABLE_POOL_ALLOC",
  1824. OP_3DSTATE_BINDING_TABLE_POOL_ALLOC,
  1825. F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL},
  1826. {"3DSTATE_GATHER_POOL_ALLOC", OP_3DSTATE_GATHER_POOL_ALLOC,
  1827. F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL},
  1828. {"3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC",
  1829. OP_3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC,
  1830. F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL},
  1831. {"3DSTATE_SAMPLE_PATTERN", OP_3DSTATE_SAMPLE_PATTERN, F_LEN_VAR, R_RCS,
  1832. D_BDW_PLUS, 0, 8, NULL},
  1833. {"PIPE_CONTROL", OP_PIPE_CONTROL, F_LEN_VAR, R_RCS, D_ALL,
  1834. ADDR_FIX_1(2), 8, cmd_handler_pipe_control},
  1835. {"3DPRIMITIVE", OP_3DPRIMITIVE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1836. {"PIPELINE_SELECT", OP_PIPELINE_SELECT, F_LEN_CONST, R_RCS, D_ALL, 0,
  1837. 1, NULL},
  1838. {"STATE_PREFETCH", OP_STATE_PREFETCH, F_LEN_VAR, R_RCS, D_ALL,
  1839. ADDR_FIX_1(1), 8, NULL},
  1840. {"STATE_SIP", OP_STATE_SIP, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1841. {"STATE_BASE_ADDRESS", OP_STATE_BASE_ADDRESS, F_LEN_VAR, R_RCS, D_BDW_PLUS,
  1842. ADDR_FIX_5(1, 3, 4, 5, 6), 8, NULL},
  1843. {"OP_3D_MEDIA_0_1_4", OP_3D_MEDIA_0_1_4, F_LEN_VAR, R_RCS, D_ALL,
  1844. ADDR_FIX_1(1), 8, NULL},
  1845. {"3DSTATE_VS", OP_3DSTATE_VS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1846. {"3DSTATE_SF", OP_3DSTATE_SF, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
  1847. {"3DSTATE_CONSTANT_VS", OP_3DSTATE_CONSTANT_VS, F_LEN_VAR, R_RCS, D_BDW_PLUS,
  1848. 0, 8, NULL},
  1849. {"3DSTATE_COMPONENT_PACKING", OP_3DSTATE_COMPONENT_PACKING, F_LEN_VAR, R_RCS,
  1850. D_SKL_PLUS, 0, 8, NULL},
  1851. {"MEDIA_INTERFACE_DESCRIPTOR_LOAD", OP_MEDIA_INTERFACE_DESCRIPTOR_LOAD,
  1852. F_LEN_VAR, R_RCS, D_ALL, 0, 16, NULL},
  1853. {"MEDIA_GATEWAY_STATE", OP_MEDIA_GATEWAY_STATE, F_LEN_VAR, R_RCS, D_ALL,
  1854. 0, 16, NULL},
  1855. {"MEDIA_STATE_FLUSH", OP_MEDIA_STATE_FLUSH, F_LEN_VAR, R_RCS, D_ALL,
  1856. 0, 16, NULL},
  1857. {"MEDIA_OBJECT", OP_MEDIA_OBJECT, F_LEN_VAR, R_RCS, D_ALL, 0, 16, NULL},
  1858. {"MEDIA_CURBE_LOAD", OP_MEDIA_CURBE_LOAD, F_LEN_VAR, R_RCS, D_ALL,
  1859. 0, 16, NULL},
  1860. {"MEDIA_OBJECT_PRT", OP_MEDIA_OBJECT_PRT, F_LEN_VAR, R_RCS, D_ALL,
  1861. 0, 16, NULL},
  1862. {"MEDIA_OBJECT_WALKER", OP_MEDIA_OBJECT_WALKER, F_LEN_VAR, R_RCS, D_ALL,
  1863. 0, 16, NULL},
  1864. {"GPGPU_WALKER", OP_GPGPU_WALKER, F_LEN_VAR, R_RCS, D_ALL,
  1865. 0, 8, NULL},
  1866. {"MEDIA_VFE_STATE", OP_MEDIA_VFE_STATE, F_LEN_VAR, R_RCS, D_ALL, 0, 16,
  1867. NULL},
  1868. {"3DSTATE_VF_STATISTICS_GM45", OP_3DSTATE_VF_STATISTICS_GM45,
  1869. F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL},
  1870. {"MFX_PIPE_MODE_SELECT", OP_MFX_PIPE_MODE_SELECT, F_LEN_VAR,
  1871. R_VCS, D_ALL, 0, 12, NULL},
  1872. {"MFX_SURFACE_STATE", OP_MFX_SURFACE_STATE, F_LEN_VAR,
  1873. R_VCS, D_ALL, 0, 12, NULL},
  1874. {"MFX_PIPE_BUF_ADDR_STATE", OP_MFX_PIPE_BUF_ADDR_STATE, F_LEN_VAR,
  1875. R_VCS, D_BDW_PLUS, 0, 12, NULL},
  1876. {"MFX_IND_OBJ_BASE_ADDR_STATE", OP_MFX_IND_OBJ_BASE_ADDR_STATE,
  1877. F_LEN_VAR, R_VCS, D_BDW_PLUS, 0, 12, NULL},
  1878. {"MFX_BSP_BUF_BASE_ADDR_STATE", OP_MFX_BSP_BUF_BASE_ADDR_STATE,
  1879. F_LEN_VAR, R_VCS, D_BDW_PLUS, ADDR_FIX_3(1, 3, 5), 12, NULL},
  1880. {"OP_2_0_0_5", OP_2_0_0_5, F_LEN_VAR, R_VCS, D_BDW_PLUS, 0, 12, NULL},
  1881. {"MFX_STATE_POINTER", OP_MFX_STATE_POINTER, F_LEN_VAR,
  1882. R_VCS, D_ALL, 0, 12, NULL},
  1883. {"MFX_QM_STATE", OP_MFX_QM_STATE, F_LEN_VAR,
  1884. R_VCS, D_ALL, 0, 12, NULL},
  1885. {"MFX_FQM_STATE", OP_MFX_FQM_STATE, F_LEN_VAR,
  1886. R_VCS, D_ALL, 0, 12, NULL},
  1887. {"MFX_PAK_INSERT_OBJECT", OP_MFX_PAK_INSERT_OBJECT, F_LEN_VAR,
  1888. R_VCS, D_ALL, 0, 12, NULL},
  1889. {"MFX_STITCH_OBJECT", OP_MFX_STITCH_OBJECT, F_LEN_VAR,
  1890. R_VCS, D_ALL, 0, 12, NULL},
  1891. {"MFD_IT_OBJECT", OP_MFD_IT_OBJECT, F_LEN_VAR,
  1892. R_VCS, D_ALL, 0, 12, NULL},
  1893. {"MFX_WAIT", OP_MFX_WAIT, F_LEN_VAR,
  1894. R_VCS, D_ALL, 0, 6, NULL},
  1895. {"MFX_AVC_IMG_STATE", OP_MFX_AVC_IMG_STATE, F_LEN_VAR,
  1896. R_VCS, D_ALL, 0, 12, NULL},
  1897. {"MFX_AVC_QM_STATE", OP_MFX_AVC_QM_STATE, F_LEN_VAR,
  1898. R_VCS, D_ALL, 0, 12, NULL},
  1899. {"MFX_AVC_DIRECTMODE_STATE", OP_MFX_AVC_DIRECTMODE_STATE, F_LEN_VAR,
  1900. R_VCS, D_ALL, 0, 12, NULL},
  1901. {"MFX_AVC_SLICE_STATE", OP_MFX_AVC_SLICE_STATE, F_LEN_VAR,
  1902. R_VCS, D_ALL, 0, 12, NULL},
  1903. {"MFX_AVC_REF_IDX_STATE", OP_MFX_AVC_REF_IDX_STATE, F_LEN_VAR,
  1904. R_VCS, D_ALL, 0, 12, NULL},
  1905. {"MFX_AVC_WEIGHTOFFSET_STATE", OP_MFX_AVC_WEIGHTOFFSET_STATE, F_LEN_VAR,
  1906. R_VCS, D_ALL, 0, 12, NULL},
  1907. {"MFD_AVC_PICID_STATE", OP_MFD_AVC_PICID_STATE, F_LEN_VAR,
  1908. R_VCS, D_ALL, 0, 12, NULL},
  1909. {"MFD_AVC_DPB_STATE", OP_MFD_AVC_DPB_STATE, F_LEN_VAR,
  1910. R_VCS, D_ALL, 0, 12, NULL},
  1911. {"MFD_AVC_BSD_OBJECT", OP_MFD_AVC_BSD_OBJECT, F_LEN_VAR,
  1912. R_VCS, D_ALL, 0, 12, NULL},
  1913. {"MFD_AVC_SLICEADDR", OP_MFD_AVC_SLICEADDR, F_LEN_VAR,
  1914. R_VCS, D_ALL, ADDR_FIX_1(2), 12, NULL},
  1915. {"MFC_AVC_PAK_OBJECT", OP_MFC_AVC_PAK_OBJECT, F_LEN_VAR,
  1916. R_VCS, D_ALL, 0, 12, NULL},
  1917. {"MFX_VC1_PRED_PIPE_STATE", OP_MFX_VC1_PRED_PIPE_STATE, F_LEN_VAR,
  1918. R_VCS, D_ALL, 0, 12, NULL},
  1919. {"MFX_VC1_DIRECTMODE_STATE", OP_MFX_VC1_DIRECTMODE_STATE, F_LEN_VAR,
  1920. R_VCS, D_ALL, 0, 12, NULL},
  1921. {"MFD_VC1_SHORT_PIC_STATE", OP_MFD_VC1_SHORT_PIC_STATE, F_LEN_VAR,
  1922. R_VCS, D_ALL, 0, 12, NULL},
  1923. {"MFD_VC1_LONG_PIC_STATE", OP_MFD_VC1_LONG_PIC_STATE, F_LEN_VAR,
  1924. R_VCS, D_ALL, 0, 12, NULL},
  1925. {"MFD_VC1_BSD_OBJECT", OP_MFD_VC1_BSD_OBJECT, F_LEN_VAR,
  1926. R_VCS, D_ALL, 0, 12, NULL},
  1927. {"MFC_MPEG2_SLICEGROUP_STATE", OP_MFC_MPEG2_SLICEGROUP_STATE, F_LEN_VAR,
  1928. R_VCS, D_ALL, 0, 12, NULL},
  1929. {"MFC_MPEG2_PAK_OBJECT", OP_MFC_MPEG2_PAK_OBJECT, F_LEN_VAR,
  1930. R_VCS, D_ALL, 0, 12, NULL},
  1931. {"MFX_MPEG2_PIC_STATE", OP_MFX_MPEG2_PIC_STATE, F_LEN_VAR,
  1932. R_VCS, D_ALL, 0, 12, NULL},
  1933. {"MFX_MPEG2_QM_STATE", OP_MFX_MPEG2_QM_STATE, F_LEN_VAR,
  1934. R_VCS, D_ALL, 0, 12, NULL},
  1935. {"MFD_MPEG2_BSD_OBJECT", OP_MFD_MPEG2_BSD_OBJECT, F_LEN_VAR,
  1936. R_VCS, D_ALL, 0, 12, NULL},
  1937. {"MFX_2_6_0_0", OP_MFX_2_6_0_0, F_LEN_VAR, R_VCS, D_ALL,
  1938. 0, 16, NULL},
  1939. {"MFX_2_6_0_9", OP_MFX_2_6_0_9, F_LEN_VAR, R_VCS, D_ALL, 0, 16, NULL},
  1940. {"MFX_2_6_0_8", OP_MFX_2_6_0_8, F_LEN_VAR, R_VCS, D_ALL, 0, 16, NULL},
  1941. {"MFX_JPEG_PIC_STATE", OP_MFX_JPEG_PIC_STATE, F_LEN_VAR,
  1942. R_VCS, D_ALL, 0, 12, NULL},
  1943. {"MFX_JPEG_HUFF_TABLE_STATE", OP_MFX_JPEG_HUFF_TABLE_STATE, F_LEN_VAR,
  1944. R_VCS, D_ALL, 0, 12, NULL},
  1945. {"MFD_JPEG_BSD_OBJECT", OP_MFD_JPEG_BSD_OBJECT, F_LEN_VAR,
  1946. R_VCS, D_ALL, 0, 12, NULL},
  1947. {"VEBOX_STATE", OP_VEB_STATE, F_LEN_VAR, R_VECS, D_ALL, 0, 12, NULL},
  1948. {"VEBOX_SURFACE_STATE", OP_VEB_SURFACE_STATE, F_LEN_VAR, R_VECS, D_ALL,
  1949. 0, 12, NULL},
  1950. {"VEB_DI_IECP", OP_VEB_DNDI_IECP_STATE, F_LEN_VAR, R_VECS, D_BDW_PLUS,
  1951. 0, 20, NULL},
  1952. };
  1953. static void add_cmd_entry(struct intel_gvt *gvt, struct cmd_entry *e)
  1954. {
  1955. hash_add(gvt->cmd_table, &e->hlist, e->info->opcode);
  1956. }
  1957. /* call the cmd handler, and advance ip */
  1958. static int cmd_parser_exec(struct parser_exec_state *s)
  1959. {
  1960. struct intel_vgpu *vgpu = s->vgpu;
  1961. struct cmd_info *info;
  1962. u32 cmd;
  1963. int ret = 0;
  1964. cmd = cmd_val(s, 0);
  1965. info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
  1966. if (info == NULL) {
  1967. gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n",
  1968. cmd, get_opcode(cmd, s->ring_id));
  1969. return -EBADRQC;
  1970. }
  1971. s->info = info;
  1972. trace_gvt_command(vgpu->id, s->ring_id, s->ip_gma, s->ip_va,
  1973. cmd_length(s), s->buf_type);
  1974. if (info->handler) {
  1975. ret = info->handler(s);
  1976. if (ret < 0) {
  1977. gvt_vgpu_err("%s handler error\n", info->name);
  1978. return ret;
  1979. }
  1980. }
  1981. if (!(info->flag & F_IP_ADVANCE_CUSTOM)) {
  1982. ret = cmd_advance_default(s);
  1983. if (ret) {
  1984. gvt_vgpu_err("%s IP advance error\n", info->name);
  1985. return ret;
  1986. }
  1987. }
  1988. return 0;
  1989. }
  1990. static inline bool gma_out_of_range(unsigned long gma,
  1991. unsigned long gma_head, unsigned int gma_tail)
  1992. {
  1993. if (gma_tail >= gma_head)
  1994. return (gma < gma_head) || (gma > gma_tail);
  1995. else
  1996. return (gma > gma_tail) && (gma < gma_head);
  1997. }
  1998. /* Keep the consistent return type, e.g EBADRQC for unknown
  1999. * cmd, EFAULT for invalid address, EPERM for nonpriv. later
  2000. * works as the input of VM healthy status.
  2001. */
  2002. static int command_scan(struct parser_exec_state *s,
  2003. unsigned long rb_head, unsigned long rb_tail,
  2004. unsigned long rb_start, unsigned long rb_len)
  2005. {
  2006. unsigned long gma_head, gma_tail, gma_bottom;
  2007. int ret = 0;
  2008. struct intel_vgpu *vgpu = s->vgpu;
  2009. gma_head = rb_start + rb_head;
  2010. gma_tail = rb_start + rb_tail;
  2011. gma_bottom = rb_start + rb_len;
  2012. while (s->ip_gma != gma_tail) {
  2013. if (s->buf_type == RING_BUFFER_INSTRUCTION) {
  2014. if (!(s->ip_gma >= rb_start) ||
  2015. !(s->ip_gma < gma_bottom)) {
  2016. gvt_vgpu_err("ip_gma %lx out of ring scope."
  2017. "(base:0x%lx, bottom: 0x%lx)\n",
  2018. s->ip_gma, rb_start,
  2019. gma_bottom);
  2020. parser_exec_state_dump(s);
  2021. return -EFAULT;
  2022. }
  2023. if (gma_out_of_range(s->ip_gma, gma_head, gma_tail)) {
  2024. gvt_vgpu_err("ip_gma %lx out of range."
  2025. "base 0x%lx head 0x%lx tail 0x%lx\n",
  2026. s->ip_gma, rb_start,
  2027. rb_head, rb_tail);
  2028. parser_exec_state_dump(s);
  2029. break;
  2030. }
  2031. }
  2032. ret = cmd_parser_exec(s);
  2033. if (ret) {
  2034. gvt_vgpu_err("cmd parser error\n");
  2035. parser_exec_state_dump(s);
  2036. break;
  2037. }
  2038. }
  2039. return ret;
  2040. }
  2041. static int scan_workload(struct intel_vgpu_workload *workload)
  2042. {
  2043. unsigned long gma_head, gma_tail, gma_bottom;
  2044. struct parser_exec_state s;
  2045. int ret = 0;
  2046. /* ring base is page aligned */
  2047. if (WARN_ON(!IS_ALIGNED(workload->rb_start, I915_GTT_PAGE_SIZE)))
  2048. return -EINVAL;
  2049. gma_head = workload->rb_start + workload->rb_head;
  2050. gma_tail = workload->rb_start + workload->rb_tail;
  2051. gma_bottom = workload->rb_start + _RING_CTL_BUF_SIZE(workload->rb_ctl);
  2052. s.buf_type = RING_BUFFER_INSTRUCTION;
  2053. s.buf_addr_type = GTT_BUFFER;
  2054. s.vgpu = workload->vgpu;
  2055. s.ring_id = workload->ring_id;
  2056. s.ring_start = workload->rb_start;
  2057. s.ring_size = _RING_CTL_BUF_SIZE(workload->rb_ctl);
  2058. s.ring_head = gma_head;
  2059. s.ring_tail = gma_tail;
  2060. s.rb_va = workload->shadow_ring_buffer_va;
  2061. s.workload = workload;
  2062. if ((bypass_scan_mask & (1 << workload->ring_id)) ||
  2063. gma_head == gma_tail)
  2064. return 0;
  2065. if (!intel_gvt_ggtt_validate_range(s.vgpu, s.ring_start, s.ring_size)) {
  2066. ret = -EINVAL;
  2067. goto out;
  2068. }
  2069. ret = ip_gma_set(&s, gma_head);
  2070. if (ret)
  2071. goto out;
  2072. ret = command_scan(&s, workload->rb_head, workload->rb_tail,
  2073. workload->rb_start, _RING_CTL_BUF_SIZE(workload->rb_ctl));
  2074. out:
  2075. return ret;
  2076. }
  2077. static int scan_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
  2078. {
  2079. unsigned long gma_head, gma_tail, gma_bottom, ring_size, ring_tail;
  2080. struct parser_exec_state s;
  2081. int ret = 0;
  2082. struct intel_vgpu_workload *workload = container_of(wa_ctx,
  2083. struct intel_vgpu_workload,
  2084. wa_ctx);
  2085. /* ring base is page aligned */
  2086. if (WARN_ON(!IS_ALIGNED(wa_ctx->indirect_ctx.guest_gma,
  2087. I915_GTT_PAGE_SIZE)))
  2088. return -EINVAL;
  2089. ring_tail = wa_ctx->indirect_ctx.size + 3 * sizeof(uint32_t);
  2090. ring_size = round_up(wa_ctx->indirect_ctx.size + CACHELINE_BYTES,
  2091. PAGE_SIZE);
  2092. gma_head = wa_ctx->indirect_ctx.guest_gma;
  2093. gma_tail = wa_ctx->indirect_ctx.guest_gma + ring_tail;
  2094. gma_bottom = wa_ctx->indirect_ctx.guest_gma + ring_size;
  2095. s.buf_type = RING_BUFFER_INSTRUCTION;
  2096. s.buf_addr_type = GTT_BUFFER;
  2097. s.vgpu = workload->vgpu;
  2098. s.ring_id = workload->ring_id;
  2099. s.ring_start = wa_ctx->indirect_ctx.guest_gma;
  2100. s.ring_size = ring_size;
  2101. s.ring_head = gma_head;
  2102. s.ring_tail = gma_tail;
  2103. s.rb_va = wa_ctx->indirect_ctx.shadow_va;
  2104. s.workload = workload;
  2105. if (!intel_gvt_ggtt_validate_range(s.vgpu, s.ring_start, s.ring_size)) {
  2106. ret = -EINVAL;
  2107. goto out;
  2108. }
  2109. ret = ip_gma_set(&s, gma_head);
  2110. if (ret)
  2111. goto out;
  2112. ret = command_scan(&s, 0, ring_tail,
  2113. wa_ctx->indirect_ctx.guest_gma, ring_size);
  2114. out:
  2115. return ret;
  2116. }
  2117. static int shadow_workload_ring_buffer(struct intel_vgpu_workload *workload)
  2118. {
  2119. struct intel_vgpu *vgpu = workload->vgpu;
  2120. struct intel_vgpu_submission *s = &vgpu->submission;
  2121. unsigned long gma_head, gma_tail, gma_top, guest_rb_size;
  2122. void *shadow_ring_buffer_va;
  2123. int ring_id = workload->ring_id;
  2124. int ret;
  2125. guest_rb_size = _RING_CTL_BUF_SIZE(workload->rb_ctl);
  2126. /* calculate workload ring buffer size */
  2127. workload->rb_len = (workload->rb_tail + guest_rb_size -
  2128. workload->rb_head) % guest_rb_size;
  2129. gma_head = workload->rb_start + workload->rb_head;
  2130. gma_tail = workload->rb_start + workload->rb_tail;
  2131. gma_top = workload->rb_start + guest_rb_size;
  2132. if (workload->rb_len > s->ring_scan_buffer_size[ring_id]) {
  2133. void *p;
  2134. /* realloc the new ring buffer if needed */
  2135. p = krealloc(s->ring_scan_buffer[ring_id], workload->rb_len,
  2136. GFP_KERNEL);
  2137. if (!p) {
  2138. gvt_vgpu_err("fail to re-alloc ring scan buffer\n");
  2139. return -ENOMEM;
  2140. }
  2141. s->ring_scan_buffer[ring_id] = p;
  2142. s->ring_scan_buffer_size[ring_id] = workload->rb_len;
  2143. }
  2144. shadow_ring_buffer_va = s->ring_scan_buffer[ring_id];
  2145. /* get shadow ring buffer va */
  2146. workload->shadow_ring_buffer_va = shadow_ring_buffer_va;
  2147. /* head > tail --> copy head <-> top */
  2148. if (gma_head > gma_tail) {
  2149. ret = copy_gma_to_hva(vgpu, vgpu->gtt.ggtt_mm,
  2150. gma_head, gma_top, shadow_ring_buffer_va);
  2151. if (ret < 0) {
  2152. gvt_vgpu_err("fail to copy guest ring buffer\n");
  2153. return ret;
  2154. }
  2155. shadow_ring_buffer_va += ret;
  2156. gma_head = workload->rb_start;
  2157. }
  2158. /* copy head or start <-> tail */
  2159. ret = copy_gma_to_hva(vgpu, vgpu->gtt.ggtt_mm, gma_head, gma_tail,
  2160. shadow_ring_buffer_va);
  2161. if (ret < 0) {
  2162. gvt_vgpu_err("fail to copy guest ring buffer\n");
  2163. return ret;
  2164. }
  2165. return 0;
  2166. }
  2167. int intel_gvt_scan_and_shadow_ringbuffer(struct intel_vgpu_workload *workload)
  2168. {
  2169. int ret;
  2170. struct intel_vgpu *vgpu = workload->vgpu;
  2171. ret = shadow_workload_ring_buffer(workload);
  2172. if (ret) {
  2173. gvt_vgpu_err("fail to shadow workload ring_buffer\n");
  2174. return ret;
  2175. }
  2176. ret = scan_workload(workload);
  2177. if (ret) {
  2178. gvt_vgpu_err("scan workload error\n");
  2179. return ret;
  2180. }
  2181. return 0;
  2182. }
  2183. static int shadow_indirect_ctx(struct intel_shadow_wa_ctx *wa_ctx)
  2184. {
  2185. int ctx_size = wa_ctx->indirect_ctx.size;
  2186. unsigned long guest_gma = wa_ctx->indirect_ctx.guest_gma;
  2187. struct intel_vgpu_workload *workload = container_of(wa_ctx,
  2188. struct intel_vgpu_workload,
  2189. wa_ctx);
  2190. struct intel_vgpu *vgpu = workload->vgpu;
  2191. struct drm_i915_gem_object *obj;
  2192. int ret = 0;
  2193. void *map;
  2194. obj = i915_gem_object_create(workload->vgpu->gvt->dev_priv,
  2195. roundup(ctx_size + CACHELINE_BYTES,
  2196. PAGE_SIZE));
  2197. if (IS_ERR(obj))
  2198. return PTR_ERR(obj);
  2199. /* get the va of the shadow batch buffer */
  2200. map = i915_gem_object_pin_map(obj, I915_MAP_WB);
  2201. if (IS_ERR(map)) {
  2202. gvt_vgpu_err("failed to vmap shadow indirect ctx\n");
  2203. ret = PTR_ERR(map);
  2204. goto put_obj;
  2205. }
  2206. ret = i915_gem_object_set_to_cpu_domain(obj, false);
  2207. if (ret) {
  2208. gvt_vgpu_err("failed to set shadow indirect ctx to CPU\n");
  2209. goto unmap_src;
  2210. }
  2211. ret = copy_gma_to_hva(workload->vgpu,
  2212. workload->vgpu->gtt.ggtt_mm,
  2213. guest_gma, guest_gma + ctx_size,
  2214. map);
  2215. if (ret < 0) {
  2216. gvt_vgpu_err("fail to copy guest indirect ctx\n");
  2217. goto unmap_src;
  2218. }
  2219. wa_ctx->indirect_ctx.obj = obj;
  2220. wa_ctx->indirect_ctx.shadow_va = map;
  2221. return 0;
  2222. unmap_src:
  2223. i915_gem_object_unpin_map(obj);
  2224. put_obj:
  2225. i915_gem_object_put(obj);
  2226. return ret;
  2227. }
  2228. static int combine_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
  2229. {
  2230. uint32_t per_ctx_start[CACHELINE_DWORDS] = {0};
  2231. unsigned char *bb_start_sva;
  2232. if (!wa_ctx->per_ctx.valid)
  2233. return 0;
  2234. per_ctx_start[0] = 0x18800001;
  2235. per_ctx_start[1] = wa_ctx->per_ctx.guest_gma;
  2236. bb_start_sva = (unsigned char *)wa_ctx->indirect_ctx.shadow_va +
  2237. wa_ctx->indirect_ctx.size;
  2238. memcpy(bb_start_sva, per_ctx_start, CACHELINE_BYTES);
  2239. return 0;
  2240. }
  2241. int intel_gvt_scan_and_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
  2242. {
  2243. int ret;
  2244. struct intel_vgpu_workload *workload = container_of(wa_ctx,
  2245. struct intel_vgpu_workload,
  2246. wa_ctx);
  2247. struct intel_vgpu *vgpu = workload->vgpu;
  2248. if (wa_ctx->indirect_ctx.size == 0)
  2249. return 0;
  2250. ret = shadow_indirect_ctx(wa_ctx);
  2251. if (ret) {
  2252. gvt_vgpu_err("fail to shadow indirect ctx\n");
  2253. return ret;
  2254. }
  2255. combine_wa_ctx(wa_ctx);
  2256. ret = scan_wa_ctx(wa_ctx);
  2257. if (ret) {
  2258. gvt_vgpu_err("scan wa ctx error\n");
  2259. return ret;
  2260. }
  2261. return 0;
  2262. }
  2263. static struct cmd_info *find_cmd_entry_any_ring(struct intel_gvt *gvt,
  2264. unsigned int opcode, int rings)
  2265. {
  2266. struct cmd_info *info = NULL;
  2267. unsigned int ring;
  2268. for_each_set_bit(ring, (unsigned long *)&rings, I915_NUM_ENGINES) {
  2269. info = find_cmd_entry(gvt, opcode, ring);
  2270. if (info)
  2271. break;
  2272. }
  2273. return info;
  2274. }
  2275. static int init_cmd_table(struct intel_gvt *gvt)
  2276. {
  2277. int i;
  2278. struct cmd_entry *e;
  2279. struct cmd_info *info;
  2280. unsigned int gen_type;
  2281. gen_type = intel_gvt_get_device_type(gvt);
  2282. for (i = 0; i < ARRAY_SIZE(cmd_info); i++) {
  2283. if (!(cmd_info[i].devices & gen_type))
  2284. continue;
  2285. e = kzalloc(sizeof(*e), GFP_KERNEL);
  2286. if (!e)
  2287. return -ENOMEM;
  2288. e->info = &cmd_info[i];
  2289. info = find_cmd_entry_any_ring(gvt,
  2290. e->info->opcode, e->info->rings);
  2291. if (info) {
  2292. gvt_err("%s %s duplicated\n", e->info->name,
  2293. info->name);
  2294. return -EEXIST;
  2295. }
  2296. INIT_HLIST_NODE(&e->hlist);
  2297. add_cmd_entry(gvt, e);
  2298. gvt_dbg_cmd("add %-30s op %04x flag %x devs %02x rings %02x\n",
  2299. e->info->name, e->info->opcode, e->info->flag,
  2300. e->info->devices, e->info->rings);
  2301. }
  2302. return 0;
  2303. }
  2304. static void clean_cmd_table(struct intel_gvt *gvt)
  2305. {
  2306. struct hlist_node *tmp;
  2307. struct cmd_entry *e;
  2308. int i;
  2309. hash_for_each_safe(gvt->cmd_table, i, tmp, e, hlist)
  2310. kfree(e);
  2311. hash_init(gvt->cmd_table);
  2312. }
  2313. void intel_gvt_clean_cmd_parser(struct intel_gvt *gvt)
  2314. {
  2315. clean_cmd_table(gvt);
  2316. }
  2317. int intel_gvt_init_cmd_parser(struct intel_gvt *gvt)
  2318. {
  2319. int ret;
  2320. ret = init_cmd_table(gvt);
  2321. if (ret) {
  2322. intel_gvt_clean_cmd_parser(gvt);
  2323. return ret;
  2324. }
  2325. return 0;
  2326. }