ice_main.c 102 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957
  1. // SPDX-License-Identifier: GPL-2.0
  2. /* Copyright (c) 2018, Intel Corporation. */
  3. /* Intel(R) Ethernet Connection E800 Series Linux Driver */
  4. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  5. #include "ice.h"
  6. #include "ice_lib.h"
  7. #define DRV_VERSION "0.7.2-k"
  8. #define DRV_SUMMARY "Intel(R) Ethernet Connection E800 Series Linux Driver"
  9. const char ice_drv_ver[] = DRV_VERSION;
  10. static const char ice_driver_string[] = DRV_SUMMARY;
  11. static const char ice_copyright[] = "Copyright (c) 2018, Intel Corporation.";
  12. MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  13. MODULE_DESCRIPTION(DRV_SUMMARY);
  14. MODULE_LICENSE("GPL v2");
  15. MODULE_VERSION(DRV_VERSION);
  16. static int debug = -1;
  17. module_param(debug, int, 0644);
  18. #ifndef CONFIG_DYNAMIC_DEBUG
  19. MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all), hw debug_mask (0x8XXXXXXX)");
  20. #else
  21. MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all)");
  22. #endif /* !CONFIG_DYNAMIC_DEBUG */
  23. static struct workqueue_struct *ice_wq;
  24. static const struct net_device_ops ice_netdev_ops;
  25. static void ice_pf_dis_all_vsi(struct ice_pf *pf);
  26. static void ice_rebuild(struct ice_pf *pf);
  27. static void ice_vsi_release_all(struct ice_pf *pf);
  28. static void ice_update_vsi_stats(struct ice_vsi *vsi);
  29. static void ice_update_pf_stats(struct ice_pf *pf);
  30. /**
  31. * ice_get_tx_pending - returns number of Tx descriptors not processed
  32. * @ring: the ring of descriptors
  33. */
  34. static u32 ice_get_tx_pending(struct ice_ring *ring)
  35. {
  36. u32 head, tail;
  37. head = ring->next_to_clean;
  38. tail = readl(ring->tail);
  39. if (head != tail)
  40. return (head < tail) ?
  41. tail - head : (tail + ring->count - head);
  42. return 0;
  43. }
  44. /**
  45. * ice_check_for_hang_subtask - check for and recover hung queues
  46. * @pf: pointer to PF struct
  47. */
  48. static void ice_check_for_hang_subtask(struct ice_pf *pf)
  49. {
  50. struct ice_vsi *vsi = NULL;
  51. unsigned int i;
  52. u32 v, v_idx;
  53. int packets;
  54. ice_for_each_vsi(pf, v)
  55. if (pf->vsi[v] && pf->vsi[v]->type == ICE_VSI_PF) {
  56. vsi = pf->vsi[v];
  57. break;
  58. }
  59. if (!vsi || test_bit(__ICE_DOWN, vsi->state))
  60. return;
  61. if (!(vsi->netdev && netif_carrier_ok(vsi->netdev)))
  62. return;
  63. for (i = 0; i < vsi->num_txq; i++) {
  64. struct ice_ring *tx_ring = vsi->tx_rings[i];
  65. if (tx_ring && tx_ring->desc) {
  66. int itr = ICE_ITR_NONE;
  67. /* If packet counter has not changed the queue is
  68. * likely stalled, so force an interrupt for this
  69. * queue.
  70. *
  71. * prev_pkt would be negative if there was no
  72. * pending work.
  73. */
  74. packets = tx_ring->stats.pkts & INT_MAX;
  75. if (tx_ring->tx_stats.prev_pkt == packets) {
  76. /* Trigger sw interrupt to revive the queue */
  77. v_idx = tx_ring->q_vector->v_idx;
  78. wr32(&vsi->back->hw,
  79. GLINT_DYN_CTL(vsi->hw_base_vector + v_idx),
  80. (itr << GLINT_DYN_CTL_ITR_INDX_S) |
  81. GLINT_DYN_CTL_SWINT_TRIG_M |
  82. GLINT_DYN_CTL_INTENA_MSK_M);
  83. continue;
  84. }
  85. /* Memory barrier between read of packet count and call
  86. * to ice_get_tx_pending()
  87. */
  88. smp_rmb();
  89. tx_ring->tx_stats.prev_pkt =
  90. ice_get_tx_pending(tx_ring) ? packets : -1;
  91. }
  92. }
  93. }
  94. /**
  95. * ice_add_mac_to_sync_list - creates list of mac addresses to be synced
  96. * @netdev: the net device on which the sync is happening
  97. * @addr: mac address to sync
  98. *
  99. * This is a callback function which is called by the in kernel device sync
  100. * functions (like __dev_uc_sync, __dev_mc_sync, etc). This function only
  101. * populates the tmp_sync_list, which is later used by ice_add_mac to add the
  102. * mac filters from the hardware.
  103. */
  104. static int ice_add_mac_to_sync_list(struct net_device *netdev, const u8 *addr)
  105. {
  106. struct ice_netdev_priv *np = netdev_priv(netdev);
  107. struct ice_vsi *vsi = np->vsi;
  108. if (ice_add_mac_to_list(vsi, &vsi->tmp_sync_list, addr))
  109. return -EINVAL;
  110. return 0;
  111. }
  112. /**
  113. * ice_add_mac_to_unsync_list - creates list of mac addresses to be unsynced
  114. * @netdev: the net device on which the unsync is happening
  115. * @addr: mac address to unsync
  116. *
  117. * This is a callback function which is called by the in kernel device unsync
  118. * functions (like __dev_uc_unsync, __dev_mc_unsync, etc). This function only
  119. * populates the tmp_unsync_list, which is later used by ice_remove_mac to
  120. * delete the mac filters from the hardware.
  121. */
  122. static int ice_add_mac_to_unsync_list(struct net_device *netdev, const u8 *addr)
  123. {
  124. struct ice_netdev_priv *np = netdev_priv(netdev);
  125. struct ice_vsi *vsi = np->vsi;
  126. if (ice_add_mac_to_list(vsi, &vsi->tmp_unsync_list, addr))
  127. return -EINVAL;
  128. return 0;
  129. }
  130. /**
  131. * ice_vsi_fltr_changed - check if filter state changed
  132. * @vsi: VSI to be checked
  133. *
  134. * returns true if filter state has changed, false otherwise.
  135. */
  136. static bool ice_vsi_fltr_changed(struct ice_vsi *vsi)
  137. {
  138. return test_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags) ||
  139. test_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags) ||
  140. test_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags);
  141. }
  142. /**
  143. * ice_vsi_sync_fltr - Update the VSI filter list to the HW
  144. * @vsi: ptr to the VSI
  145. *
  146. * Push any outstanding VSI filter changes through the AdminQ.
  147. */
  148. static int ice_vsi_sync_fltr(struct ice_vsi *vsi)
  149. {
  150. struct device *dev = &vsi->back->pdev->dev;
  151. struct net_device *netdev = vsi->netdev;
  152. bool promisc_forced_on = false;
  153. struct ice_pf *pf = vsi->back;
  154. struct ice_hw *hw = &pf->hw;
  155. enum ice_status status = 0;
  156. u32 changed_flags = 0;
  157. int err = 0;
  158. if (!vsi->netdev)
  159. return -EINVAL;
  160. while (test_and_set_bit(__ICE_CFG_BUSY, vsi->state))
  161. usleep_range(1000, 2000);
  162. changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
  163. vsi->current_netdev_flags = vsi->netdev->flags;
  164. INIT_LIST_HEAD(&vsi->tmp_sync_list);
  165. INIT_LIST_HEAD(&vsi->tmp_unsync_list);
  166. if (ice_vsi_fltr_changed(vsi)) {
  167. clear_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags);
  168. clear_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags);
  169. clear_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags);
  170. /* grab the netdev's addr_list_lock */
  171. netif_addr_lock_bh(netdev);
  172. __dev_uc_sync(netdev, ice_add_mac_to_sync_list,
  173. ice_add_mac_to_unsync_list);
  174. __dev_mc_sync(netdev, ice_add_mac_to_sync_list,
  175. ice_add_mac_to_unsync_list);
  176. /* our temp lists are populated. release lock */
  177. netif_addr_unlock_bh(netdev);
  178. }
  179. /* Remove mac addresses in the unsync list */
  180. status = ice_remove_mac(hw, &vsi->tmp_unsync_list);
  181. ice_free_fltr_list(dev, &vsi->tmp_unsync_list);
  182. if (status) {
  183. netdev_err(netdev, "Failed to delete MAC filters\n");
  184. /* if we failed because of alloc failures, just bail */
  185. if (status == ICE_ERR_NO_MEMORY) {
  186. err = -ENOMEM;
  187. goto out;
  188. }
  189. }
  190. /* Add mac addresses in the sync list */
  191. status = ice_add_mac(hw, &vsi->tmp_sync_list);
  192. ice_free_fltr_list(dev, &vsi->tmp_sync_list);
  193. if (status) {
  194. netdev_err(netdev, "Failed to add MAC filters\n");
  195. /* If there is no more space for new umac filters, vsi
  196. * should go into promiscuous mode. There should be some
  197. * space reserved for promiscuous filters.
  198. */
  199. if (hw->adminq.sq_last_status == ICE_AQ_RC_ENOSPC &&
  200. !test_and_set_bit(__ICE_FLTR_OVERFLOW_PROMISC,
  201. vsi->state)) {
  202. promisc_forced_on = true;
  203. netdev_warn(netdev,
  204. "Reached MAC filter limit, forcing promisc mode on VSI %d\n",
  205. vsi->vsi_num);
  206. } else {
  207. err = -EIO;
  208. goto out;
  209. }
  210. }
  211. /* check for changes in promiscuous modes */
  212. if (changed_flags & IFF_ALLMULTI)
  213. netdev_warn(netdev, "Unsupported configuration\n");
  214. if (((changed_flags & IFF_PROMISC) || promisc_forced_on) ||
  215. test_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags)) {
  216. clear_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags);
  217. if (vsi->current_netdev_flags & IFF_PROMISC) {
  218. /* Apply TX filter rule to get traffic from VMs */
  219. status = ice_cfg_dflt_vsi(hw, vsi->idx, true,
  220. ICE_FLTR_TX);
  221. if (status) {
  222. netdev_err(netdev, "Error setting default VSI %i tx rule\n",
  223. vsi->vsi_num);
  224. vsi->current_netdev_flags &= ~IFF_PROMISC;
  225. err = -EIO;
  226. goto out_promisc;
  227. }
  228. /* Apply RX filter rule to get traffic from wire */
  229. status = ice_cfg_dflt_vsi(hw, vsi->idx, true,
  230. ICE_FLTR_RX);
  231. if (status) {
  232. netdev_err(netdev, "Error setting default VSI %i rx rule\n",
  233. vsi->vsi_num);
  234. vsi->current_netdev_flags &= ~IFF_PROMISC;
  235. err = -EIO;
  236. goto out_promisc;
  237. }
  238. } else {
  239. /* Clear TX filter rule to stop traffic from VMs */
  240. status = ice_cfg_dflt_vsi(hw, vsi->idx, false,
  241. ICE_FLTR_TX);
  242. if (status) {
  243. netdev_err(netdev, "Error clearing default VSI %i tx rule\n",
  244. vsi->vsi_num);
  245. vsi->current_netdev_flags |= IFF_PROMISC;
  246. err = -EIO;
  247. goto out_promisc;
  248. }
  249. /* Clear RX filter to remove traffic from wire */
  250. status = ice_cfg_dflt_vsi(hw, vsi->idx, false,
  251. ICE_FLTR_RX);
  252. if (status) {
  253. netdev_err(netdev, "Error clearing default VSI %i rx rule\n",
  254. vsi->vsi_num);
  255. vsi->current_netdev_flags |= IFF_PROMISC;
  256. err = -EIO;
  257. goto out_promisc;
  258. }
  259. }
  260. }
  261. goto exit;
  262. out_promisc:
  263. set_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags);
  264. goto exit;
  265. out:
  266. /* if something went wrong then set the changed flag so we try again */
  267. set_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags);
  268. set_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags);
  269. exit:
  270. clear_bit(__ICE_CFG_BUSY, vsi->state);
  271. return err;
  272. }
  273. /**
  274. * ice_sync_fltr_subtask - Sync the VSI filter list with HW
  275. * @pf: board private structure
  276. */
  277. static void ice_sync_fltr_subtask(struct ice_pf *pf)
  278. {
  279. int v;
  280. if (!pf || !(test_bit(ICE_FLAG_FLTR_SYNC, pf->flags)))
  281. return;
  282. clear_bit(ICE_FLAG_FLTR_SYNC, pf->flags);
  283. for (v = 0; v < pf->num_alloc_vsi; v++)
  284. if (pf->vsi[v] && ice_vsi_fltr_changed(pf->vsi[v]) &&
  285. ice_vsi_sync_fltr(pf->vsi[v])) {
  286. /* come back and try again later */
  287. set_bit(ICE_FLAG_FLTR_SYNC, pf->flags);
  288. break;
  289. }
  290. }
  291. /**
  292. * ice_prepare_for_reset - prep for the core to reset
  293. * @pf: board private structure
  294. *
  295. * Inform or close all dependent features in prep for reset.
  296. */
  297. static void
  298. ice_prepare_for_reset(struct ice_pf *pf)
  299. {
  300. struct ice_hw *hw = &pf->hw;
  301. /* Notify VFs of impending reset */
  302. if (ice_check_sq_alive(hw, &hw->mailboxq))
  303. ice_vc_notify_reset(pf);
  304. /* disable the VSIs and their queues that are not already DOWN */
  305. ice_pf_dis_all_vsi(pf);
  306. ice_shutdown_all_ctrlq(hw);
  307. set_bit(__ICE_PREPARED_FOR_RESET, pf->state);
  308. }
  309. /**
  310. * ice_do_reset - Initiate one of many types of resets
  311. * @pf: board private structure
  312. * @reset_type: reset type requested
  313. * before this function was called.
  314. */
  315. static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type)
  316. {
  317. struct device *dev = &pf->pdev->dev;
  318. struct ice_hw *hw = &pf->hw;
  319. dev_dbg(dev, "reset_type 0x%x requested\n", reset_type);
  320. WARN_ON(in_interrupt());
  321. ice_prepare_for_reset(pf);
  322. /* trigger the reset */
  323. if (ice_reset(hw, reset_type)) {
  324. dev_err(dev, "reset %d failed\n", reset_type);
  325. set_bit(__ICE_RESET_FAILED, pf->state);
  326. clear_bit(__ICE_RESET_OICR_RECV, pf->state);
  327. clear_bit(__ICE_PREPARED_FOR_RESET, pf->state);
  328. clear_bit(__ICE_PFR_REQ, pf->state);
  329. clear_bit(__ICE_CORER_REQ, pf->state);
  330. clear_bit(__ICE_GLOBR_REQ, pf->state);
  331. return;
  332. }
  333. /* PFR is a bit of a special case because it doesn't result in an OICR
  334. * interrupt. So for PFR, rebuild after the reset and clear the reset-
  335. * associated state bits.
  336. */
  337. if (reset_type == ICE_RESET_PFR) {
  338. pf->pfr_count++;
  339. ice_rebuild(pf);
  340. clear_bit(__ICE_PREPARED_FOR_RESET, pf->state);
  341. clear_bit(__ICE_PFR_REQ, pf->state);
  342. }
  343. }
  344. /**
  345. * ice_reset_subtask - Set up for resetting the device and driver
  346. * @pf: board private structure
  347. */
  348. static void ice_reset_subtask(struct ice_pf *pf)
  349. {
  350. enum ice_reset_req reset_type = ICE_RESET_INVAL;
  351. /* When a CORER/GLOBR/EMPR is about to happen, the hardware triggers an
  352. * OICR interrupt. The OICR handler (ice_misc_intr) determines what type
  353. * of reset is pending and sets bits in pf->state indicating the reset
  354. * type and __ICE_RESET_OICR_RECV. So, if the latter bit is set
  355. * prepare for pending reset if not already (for PF software-initiated
  356. * global resets the software should already be prepared for it as
  357. * indicated by __ICE_PREPARED_FOR_RESET; for global resets initiated
  358. * by firmware or software on other PFs, that bit is not set so prepare
  359. * for the reset now), poll for reset done, rebuild and return.
  360. */
  361. if (test_bit(__ICE_RESET_OICR_RECV, pf->state)) {
  362. clear_bit(__ICE_GLOBR_RECV, pf->state);
  363. clear_bit(__ICE_CORER_RECV, pf->state);
  364. if (!test_bit(__ICE_PREPARED_FOR_RESET, pf->state))
  365. ice_prepare_for_reset(pf);
  366. /* make sure we are ready to rebuild */
  367. if (ice_check_reset(&pf->hw)) {
  368. set_bit(__ICE_RESET_FAILED, pf->state);
  369. } else {
  370. /* done with reset. start rebuild */
  371. pf->hw.reset_ongoing = false;
  372. ice_rebuild(pf);
  373. /* clear bit to resume normal operations, but
  374. * ICE_NEEDS_RESTART bit is set incase rebuild failed
  375. */
  376. clear_bit(__ICE_RESET_OICR_RECV, pf->state);
  377. clear_bit(__ICE_PREPARED_FOR_RESET, pf->state);
  378. clear_bit(__ICE_PFR_REQ, pf->state);
  379. clear_bit(__ICE_CORER_REQ, pf->state);
  380. clear_bit(__ICE_GLOBR_REQ, pf->state);
  381. }
  382. return;
  383. }
  384. /* No pending resets to finish processing. Check for new resets */
  385. if (test_bit(__ICE_PFR_REQ, pf->state))
  386. reset_type = ICE_RESET_PFR;
  387. if (test_bit(__ICE_CORER_REQ, pf->state))
  388. reset_type = ICE_RESET_CORER;
  389. if (test_bit(__ICE_GLOBR_REQ, pf->state))
  390. reset_type = ICE_RESET_GLOBR;
  391. /* If no valid reset type requested just return */
  392. if (reset_type == ICE_RESET_INVAL)
  393. return;
  394. /* reset if not already down or busy */
  395. if (!test_bit(__ICE_DOWN, pf->state) &&
  396. !test_bit(__ICE_CFG_BUSY, pf->state)) {
  397. ice_do_reset(pf, reset_type);
  398. }
  399. }
  400. /**
  401. * ice_watchdog_subtask - periodic tasks not using event driven scheduling
  402. * @pf: board private structure
  403. */
  404. static void ice_watchdog_subtask(struct ice_pf *pf)
  405. {
  406. int i;
  407. /* if interface is down do nothing */
  408. if (test_bit(__ICE_DOWN, pf->state) ||
  409. test_bit(__ICE_CFG_BUSY, pf->state))
  410. return;
  411. /* make sure we don't do these things too often */
  412. if (time_before(jiffies,
  413. pf->serv_tmr_prev + pf->serv_tmr_period))
  414. return;
  415. pf->serv_tmr_prev = jiffies;
  416. /* Update the stats for active netdevs so the network stack
  417. * can look at updated numbers whenever it cares to
  418. */
  419. ice_update_pf_stats(pf);
  420. for (i = 0; i < pf->num_alloc_vsi; i++)
  421. if (pf->vsi[i] && pf->vsi[i]->netdev)
  422. ice_update_vsi_stats(pf->vsi[i]);
  423. }
  424. /**
  425. * ice_print_link_msg - print link up or down message
  426. * @vsi: the VSI whose link status is being queried
  427. * @isup: boolean for if the link is now up or down
  428. */
  429. void ice_print_link_msg(struct ice_vsi *vsi, bool isup)
  430. {
  431. const char *speed;
  432. const char *fc;
  433. if (vsi->current_isup == isup)
  434. return;
  435. vsi->current_isup = isup;
  436. if (!isup) {
  437. netdev_info(vsi->netdev, "NIC Link is Down\n");
  438. return;
  439. }
  440. switch (vsi->port_info->phy.link_info.link_speed) {
  441. case ICE_AQ_LINK_SPEED_40GB:
  442. speed = "40 G";
  443. break;
  444. case ICE_AQ_LINK_SPEED_25GB:
  445. speed = "25 G";
  446. break;
  447. case ICE_AQ_LINK_SPEED_20GB:
  448. speed = "20 G";
  449. break;
  450. case ICE_AQ_LINK_SPEED_10GB:
  451. speed = "10 G";
  452. break;
  453. case ICE_AQ_LINK_SPEED_5GB:
  454. speed = "5 G";
  455. break;
  456. case ICE_AQ_LINK_SPEED_2500MB:
  457. speed = "2.5 G";
  458. break;
  459. case ICE_AQ_LINK_SPEED_1000MB:
  460. speed = "1 G";
  461. break;
  462. case ICE_AQ_LINK_SPEED_100MB:
  463. speed = "100 M";
  464. break;
  465. default:
  466. speed = "Unknown";
  467. break;
  468. }
  469. switch (vsi->port_info->fc.current_mode) {
  470. case ICE_FC_FULL:
  471. fc = "RX/TX";
  472. break;
  473. case ICE_FC_TX_PAUSE:
  474. fc = "TX";
  475. break;
  476. case ICE_FC_RX_PAUSE:
  477. fc = "RX";
  478. break;
  479. default:
  480. fc = "Unknown";
  481. break;
  482. }
  483. netdev_info(vsi->netdev, "NIC Link is up %sbps, Flow Control: %s\n",
  484. speed, fc);
  485. }
  486. /**
  487. * ice_init_link_events - enable/initialize link events
  488. * @pi: pointer to the port_info instance
  489. *
  490. * Returns -EIO on failure, 0 on success
  491. */
  492. static int ice_init_link_events(struct ice_port_info *pi)
  493. {
  494. u16 mask;
  495. mask = ~((u16)(ICE_AQ_LINK_EVENT_UPDOWN | ICE_AQ_LINK_EVENT_MEDIA_NA |
  496. ICE_AQ_LINK_EVENT_MODULE_QUAL_FAIL));
  497. if (ice_aq_set_event_mask(pi->hw, pi->lport, mask, NULL)) {
  498. dev_dbg(ice_hw_to_dev(pi->hw),
  499. "Failed to set link event mask for port %d\n",
  500. pi->lport);
  501. return -EIO;
  502. }
  503. if (ice_aq_get_link_info(pi, true, NULL, NULL)) {
  504. dev_dbg(ice_hw_to_dev(pi->hw),
  505. "Failed to enable link events for port %d\n",
  506. pi->lport);
  507. return -EIO;
  508. }
  509. return 0;
  510. }
  511. /**
  512. * ice_vsi_link_event - update the vsi's netdev
  513. * @vsi: the vsi on which the link event occurred
  514. * @link_up: whether or not the vsi needs to be set up or down
  515. */
  516. static void ice_vsi_link_event(struct ice_vsi *vsi, bool link_up)
  517. {
  518. if (!vsi || test_bit(__ICE_DOWN, vsi->state))
  519. return;
  520. if (vsi->type == ICE_VSI_PF) {
  521. if (!vsi->netdev) {
  522. dev_dbg(&vsi->back->pdev->dev,
  523. "vsi->netdev is not initialized!\n");
  524. return;
  525. }
  526. if (link_up) {
  527. netif_carrier_on(vsi->netdev);
  528. netif_tx_wake_all_queues(vsi->netdev);
  529. } else {
  530. netif_carrier_off(vsi->netdev);
  531. netif_tx_stop_all_queues(vsi->netdev);
  532. }
  533. }
  534. }
  535. /**
  536. * ice_link_event - process the link event
  537. * @pf: pf that the link event is associated with
  538. * @pi: port_info for the port that the link event is associated with
  539. *
  540. * Returns -EIO if ice_get_link_status() fails
  541. * Returns 0 on success
  542. */
  543. static int
  544. ice_link_event(struct ice_pf *pf, struct ice_port_info *pi)
  545. {
  546. u8 new_link_speed, old_link_speed;
  547. struct ice_phy_info *phy_info;
  548. bool new_link_same_as_old;
  549. bool new_link, old_link;
  550. u8 lport;
  551. u16 v;
  552. phy_info = &pi->phy;
  553. phy_info->link_info_old = phy_info->link_info;
  554. /* Force ice_get_link_status() to update link info */
  555. phy_info->get_link_info = true;
  556. old_link = (phy_info->link_info_old.link_info & ICE_AQ_LINK_UP);
  557. old_link_speed = phy_info->link_info_old.link_speed;
  558. lport = pi->lport;
  559. if (ice_get_link_status(pi, &new_link)) {
  560. dev_dbg(&pf->pdev->dev,
  561. "Could not get link status for port %d\n", lport);
  562. return -EIO;
  563. }
  564. new_link_speed = phy_info->link_info.link_speed;
  565. new_link_same_as_old = (new_link == old_link &&
  566. new_link_speed == old_link_speed);
  567. ice_for_each_vsi(pf, v) {
  568. struct ice_vsi *vsi = pf->vsi[v];
  569. if (!vsi || !vsi->port_info)
  570. continue;
  571. if (new_link_same_as_old &&
  572. (test_bit(__ICE_DOWN, vsi->state) ||
  573. new_link == netif_carrier_ok(vsi->netdev)))
  574. continue;
  575. if (vsi->port_info->lport == lport) {
  576. ice_print_link_msg(vsi, new_link);
  577. ice_vsi_link_event(vsi, new_link);
  578. }
  579. }
  580. ice_vc_notify_link_state(pf);
  581. return 0;
  582. }
  583. /**
  584. * ice_handle_link_event - handle link event via ARQ
  585. * @pf: pf that the link event is associated with
  586. *
  587. * Return -EINVAL if port_info is null
  588. * Return status on succes
  589. */
  590. static int ice_handle_link_event(struct ice_pf *pf)
  591. {
  592. struct ice_port_info *port_info;
  593. int status;
  594. port_info = pf->hw.port_info;
  595. if (!port_info)
  596. return -EINVAL;
  597. status = ice_link_event(pf, port_info);
  598. if (status)
  599. dev_dbg(&pf->pdev->dev,
  600. "Could not process link event, error %d\n", status);
  601. return status;
  602. }
  603. /**
  604. * __ice_clean_ctrlq - helper function to clean controlq rings
  605. * @pf: ptr to struct ice_pf
  606. * @q_type: specific Control queue type
  607. */
  608. static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type)
  609. {
  610. struct ice_rq_event_info event;
  611. struct ice_hw *hw = &pf->hw;
  612. struct ice_ctl_q_info *cq;
  613. u16 pending, i = 0;
  614. const char *qtype;
  615. u32 oldval, val;
  616. /* Do not clean control queue if/when PF reset fails */
  617. if (test_bit(__ICE_RESET_FAILED, pf->state))
  618. return 0;
  619. switch (q_type) {
  620. case ICE_CTL_Q_ADMIN:
  621. cq = &hw->adminq;
  622. qtype = "Admin";
  623. break;
  624. case ICE_CTL_Q_MAILBOX:
  625. cq = &hw->mailboxq;
  626. qtype = "Mailbox";
  627. break;
  628. default:
  629. dev_warn(&pf->pdev->dev, "Unknown control queue type 0x%x\n",
  630. q_type);
  631. return 0;
  632. }
  633. /* check for error indications - PF_xx_AxQLEN register layout for
  634. * FW/MBX/SB are identical so just use defines for PF_FW_AxQLEN.
  635. */
  636. val = rd32(hw, cq->rq.len);
  637. if (val & (PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
  638. PF_FW_ARQLEN_ARQCRIT_M)) {
  639. oldval = val;
  640. if (val & PF_FW_ARQLEN_ARQVFE_M)
  641. dev_dbg(&pf->pdev->dev,
  642. "%s Receive Queue VF Error detected\n", qtype);
  643. if (val & PF_FW_ARQLEN_ARQOVFL_M) {
  644. dev_dbg(&pf->pdev->dev,
  645. "%s Receive Queue Overflow Error detected\n",
  646. qtype);
  647. }
  648. if (val & PF_FW_ARQLEN_ARQCRIT_M)
  649. dev_dbg(&pf->pdev->dev,
  650. "%s Receive Queue Critical Error detected\n",
  651. qtype);
  652. val &= ~(PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
  653. PF_FW_ARQLEN_ARQCRIT_M);
  654. if (oldval != val)
  655. wr32(hw, cq->rq.len, val);
  656. }
  657. val = rd32(hw, cq->sq.len);
  658. if (val & (PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
  659. PF_FW_ATQLEN_ATQCRIT_M)) {
  660. oldval = val;
  661. if (val & PF_FW_ATQLEN_ATQVFE_M)
  662. dev_dbg(&pf->pdev->dev,
  663. "%s Send Queue VF Error detected\n", qtype);
  664. if (val & PF_FW_ATQLEN_ATQOVFL_M) {
  665. dev_dbg(&pf->pdev->dev,
  666. "%s Send Queue Overflow Error detected\n",
  667. qtype);
  668. }
  669. if (val & PF_FW_ATQLEN_ATQCRIT_M)
  670. dev_dbg(&pf->pdev->dev,
  671. "%s Send Queue Critical Error detected\n",
  672. qtype);
  673. val &= ~(PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
  674. PF_FW_ATQLEN_ATQCRIT_M);
  675. if (oldval != val)
  676. wr32(hw, cq->sq.len, val);
  677. }
  678. event.buf_len = cq->rq_buf_size;
  679. event.msg_buf = devm_kzalloc(&pf->pdev->dev, event.buf_len,
  680. GFP_KERNEL);
  681. if (!event.msg_buf)
  682. return 0;
  683. do {
  684. enum ice_status ret;
  685. u16 opcode;
  686. ret = ice_clean_rq_elem(hw, cq, &event, &pending);
  687. if (ret == ICE_ERR_AQ_NO_WORK)
  688. break;
  689. if (ret) {
  690. dev_err(&pf->pdev->dev,
  691. "%s Receive Queue event error %d\n", qtype,
  692. ret);
  693. break;
  694. }
  695. opcode = le16_to_cpu(event.desc.opcode);
  696. switch (opcode) {
  697. case ice_aqc_opc_get_link_status:
  698. if (ice_handle_link_event(pf))
  699. dev_err(&pf->pdev->dev,
  700. "Could not handle link event\n");
  701. break;
  702. case ice_mbx_opc_send_msg_to_pf:
  703. ice_vc_process_vf_msg(pf, &event);
  704. break;
  705. case ice_aqc_opc_fw_logging:
  706. ice_output_fw_log(hw, &event.desc, event.msg_buf);
  707. break;
  708. default:
  709. dev_dbg(&pf->pdev->dev,
  710. "%s Receive Queue unknown event 0x%04x ignored\n",
  711. qtype, opcode);
  712. break;
  713. }
  714. } while (pending && (i++ < ICE_DFLT_IRQ_WORK));
  715. devm_kfree(&pf->pdev->dev, event.msg_buf);
  716. return pending && (i == ICE_DFLT_IRQ_WORK);
  717. }
  718. /**
  719. * ice_ctrlq_pending - check if there is a difference between ntc and ntu
  720. * @hw: pointer to hardware info
  721. * @cq: control queue information
  722. *
  723. * returns true if there are pending messages in a queue, false if there aren't
  724. */
  725. static bool ice_ctrlq_pending(struct ice_hw *hw, struct ice_ctl_q_info *cq)
  726. {
  727. u16 ntu;
  728. ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
  729. return cq->rq.next_to_clean != ntu;
  730. }
  731. /**
  732. * ice_clean_adminq_subtask - clean the AdminQ rings
  733. * @pf: board private structure
  734. */
  735. static void ice_clean_adminq_subtask(struct ice_pf *pf)
  736. {
  737. struct ice_hw *hw = &pf->hw;
  738. if (!test_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state))
  739. return;
  740. if (__ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN))
  741. return;
  742. clear_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state);
  743. /* There might be a situation where new messages arrive to a control
  744. * queue between processing the last message and clearing the
  745. * EVENT_PENDING bit. So before exiting, check queue head again (using
  746. * ice_ctrlq_pending) and process new messages if any.
  747. */
  748. if (ice_ctrlq_pending(hw, &hw->adminq))
  749. __ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN);
  750. ice_flush(hw);
  751. }
  752. /**
  753. * ice_clean_mailboxq_subtask - clean the MailboxQ rings
  754. * @pf: board private structure
  755. */
  756. static void ice_clean_mailboxq_subtask(struct ice_pf *pf)
  757. {
  758. struct ice_hw *hw = &pf->hw;
  759. if (!test_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state))
  760. return;
  761. if (__ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX))
  762. return;
  763. clear_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state);
  764. if (ice_ctrlq_pending(hw, &hw->mailboxq))
  765. __ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX);
  766. ice_flush(hw);
  767. }
  768. /**
  769. * ice_service_task_schedule - schedule the service task to wake up
  770. * @pf: board private structure
  771. *
  772. * If not already scheduled, this puts the task into the work queue.
  773. */
  774. static void ice_service_task_schedule(struct ice_pf *pf)
  775. {
  776. if (!test_bit(__ICE_SERVICE_DIS, pf->state) &&
  777. !test_and_set_bit(__ICE_SERVICE_SCHED, pf->state) &&
  778. !test_bit(__ICE_NEEDS_RESTART, pf->state))
  779. queue_work(ice_wq, &pf->serv_task);
  780. }
  781. /**
  782. * ice_service_task_complete - finish up the service task
  783. * @pf: board private structure
  784. */
  785. static void ice_service_task_complete(struct ice_pf *pf)
  786. {
  787. WARN_ON(!test_bit(__ICE_SERVICE_SCHED, pf->state));
  788. /* force memory (pf->state) to sync before next service task */
  789. smp_mb__before_atomic();
  790. clear_bit(__ICE_SERVICE_SCHED, pf->state);
  791. }
  792. /**
  793. * ice_service_task_stop - stop service task and cancel works
  794. * @pf: board private structure
  795. */
  796. static void ice_service_task_stop(struct ice_pf *pf)
  797. {
  798. set_bit(__ICE_SERVICE_DIS, pf->state);
  799. if (pf->serv_tmr.function)
  800. del_timer_sync(&pf->serv_tmr);
  801. if (pf->serv_task.func)
  802. cancel_work_sync(&pf->serv_task);
  803. clear_bit(__ICE_SERVICE_SCHED, pf->state);
  804. }
  805. /**
  806. * ice_service_timer - timer callback to schedule service task
  807. * @t: pointer to timer_list
  808. */
  809. static void ice_service_timer(struct timer_list *t)
  810. {
  811. struct ice_pf *pf = from_timer(pf, t, serv_tmr);
  812. mod_timer(&pf->serv_tmr, round_jiffies(pf->serv_tmr_period + jiffies));
  813. ice_service_task_schedule(pf);
  814. }
  815. /**
  816. * ice_handle_mdd_event - handle malicious driver detect event
  817. * @pf: pointer to the PF structure
  818. *
  819. * Called from service task. OICR interrupt handler indicates MDD event
  820. */
  821. static void ice_handle_mdd_event(struct ice_pf *pf)
  822. {
  823. struct ice_hw *hw = &pf->hw;
  824. bool mdd_detected = false;
  825. u32 reg;
  826. int i;
  827. if (!test_bit(__ICE_MDD_EVENT_PENDING, pf->state))
  828. return;
  829. /* find what triggered the MDD event */
  830. reg = rd32(hw, GL_MDET_TX_PQM);
  831. if (reg & GL_MDET_TX_PQM_VALID_M) {
  832. u8 pf_num = (reg & GL_MDET_TX_PQM_PF_NUM_M) >>
  833. GL_MDET_TX_PQM_PF_NUM_S;
  834. u16 vf_num = (reg & GL_MDET_TX_PQM_VF_NUM_M) >>
  835. GL_MDET_TX_PQM_VF_NUM_S;
  836. u8 event = (reg & GL_MDET_TX_PQM_MAL_TYPE_M) >>
  837. GL_MDET_TX_PQM_MAL_TYPE_S;
  838. u16 queue = ((reg & GL_MDET_TX_PQM_QNUM_M) >>
  839. GL_MDET_TX_PQM_QNUM_S);
  840. if (netif_msg_tx_err(pf))
  841. dev_info(&pf->pdev->dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n",
  842. event, queue, pf_num, vf_num);
  843. wr32(hw, GL_MDET_TX_PQM, 0xffffffff);
  844. mdd_detected = true;
  845. }
  846. reg = rd32(hw, GL_MDET_TX_TCLAN);
  847. if (reg & GL_MDET_TX_TCLAN_VALID_M) {
  848. u8 pf_num = (reg & GL_MDET_TX_TCLAN_PF_NUM_M) >>
  849. GL_MDET_TX_TCLAN_PF_NUM_S;
  850. u16 vf_num = (reg & GL_MDET_TX_TCLAN_VF_NUM_M) >>
  851. GL_MDET_TX_TCLAN_VF_NUM_S;
  852. u8 event = (reg & GL_MDET_TX_TCLAN_MAL_TYPE_M) >>
  853. GL_MDET_TX_TCLAN_MAL_TYPE_S;
  854. u16 queue = ((reg & GL_MDET_TX_TCLAN_QNUM_M) >>
  855. GL_MDET_TX_TCLAN_QNUM_S);
  856. if (netif_msg_rx_err(pf))
  857. dev_info(&pf->pdev->dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n",
  858. event, queue, pf_num, vf_num);
  859. wr32(hw, GL_MDET_TX_TCLAN, 0xffffffff);
  860. mdd_detected = true;
  861. }
  862. reg = rd32(hw, GL_MDET_RX);
  863. if (reg & GL_MDET_RX_VALID_M) {
  864. u8 pf_num = (reg & GL_MDET_RX_PF_NUM_M) >>
  865. GL_MDET_RX_PF_NUM_S;
  866. u16 vf_num = (reg & GL_MDET_RX_VF_NUM_M) >>
  867. GL_MDET_RX_VF_NUM_S;
  868. u8 event = (reg & GL_MDET_RX_MAL_TYPE_M) >>
  869. GL_MDET_RX_MAL_TYPE_S;
  870. u16 queue = ((reg & GL_MDET_RX_QNUM_M) >>
  871. GL_MDET_RX_QNUM_S);
  872. if (netif_msg_rx_err(pf))
  873. dev_info(&pf->pdev->dev, "Malicious Driver Detection event %d on RX queue %d PF# %d VF# %d\n",
  874. event, queue, pf_num, vf_num);
  875. wr32(hw, GL_MDET_RX, 0xffffffff);
  876. mdd_detected = true;
  877. }
  878. if (mdd_detected) {
  879. bool pf_mdd_detected = false;
  880. reg = rd32(hw, PF_MDET_TX_PQM);
  881. if (reg & PF_MDET_TX_PQM_VALID_M) {
  882. wr32(hw, PF_MDET_TX_PQM, 0xFFFF);
  883. dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n");
  884. pf_mdd_detected = true;
  885. }
  886. reg = rd32(hw, PF_MDET_TX_TCLAN);
  887. if (reg & PF_MDET_TX_TCLAN_VALID_M) {
  888. wr32(hw, PF_MDET_TX_TCLAN, 0xFFFF);
  889. dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n");
  890. pf_mdd_detected = true;
  891. }
  892. reg = rd32(hw, PF_MDET_RX);
  893. if (reg & PF_MDET_RX_VALID_M) {
  894. wr32(hw, PF_MDET_RX, 0xFFFF);
  895. dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n");
  896. pf_mdd_detected = true;
  897. }
  898. /* Queue belongs to the PF initiate a reset */
  899. if (pf_mdd_detected) {
  900. set_bit(__ICE_NEEDS_RESTART, pf->state);
  901. ice_service_task_schedule(pf);
  902. }
  903. }
  904. /* see if one of the VFs needs to be reset */
  905. for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
  906. struct ice_vf *vf = &pf->vf[i];
  907. reg = rd32(hw, VP_MDET_TX_PQM(i));
  908. if (reg & VP_MDET_TX_PQM_VALID_M) {
  909. wr32(hw, VP_MDET_TX_PQM(i), 0xFFFF);
  910. vf->num_mdd_events++;
  911. dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
  912. i);
  913. }
  914. reg = rd32(hw, VP_MDET_TX_TCLAN(i));
  915. if (reg & VP_MDET_TX_TCLAN_VALID_M) {
  916. wr32(hw, VP_MDET_TX_TCLAN(i), 0xFFFF);
  917. vf->num_mdd_events++;
  918. dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
  919. i);
  920. }
  921. reg = rd32(hw, VP_MDET_TX_TDPU(i));
  922. if (reg & VP_MDET_TX_TDPU_VALID_M) {
  923. wr32(hw, VP_MDET_TX_TDPU(i), 0xFFFF);
  924. vf->num_mdd_events++;
  925. dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
  926. i);
  927. }
  928. reg = rd32(hw, VP_MDET_RX(i));
  929. if (reg & VP_MDET_RX_VALID_M) {
  930. wr32(hw, VP_MDET_RX(i), 0xFFFF);
  931. vf->num_mdd_events++;
  932. dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
  933. i);
  934. }
  935. if (vf->num_mdd_events > ICE_DFLT_NUM_MDD_EVENTS_ALLOWED) {
  936. dev_info(&pf->pdev->dev,
  937. "Too many MDD events on VF %d, disabled\n", i);
  938. dev_info(&pf->pdev->dev,
  939. "Use PF Control I/F to re-enable the VF\n");
  940. set_bit(ICE_VF_STATE_DIS, vf->vf_states);
  941. }
  942. }
  943. /* re-enable MDD interrupt cause */
  944. clear_bit(__ICE_MDD_EVENT_PENDING, pf->state);
  945. reg = rd32(hw, PFINT_OICR_ENA);
  946. reg |= PFINT_OICR_MAL_DETECT_M;
  947. wr32(hw, PFINT_OICR_ENA, reg);
  948. ice_flush(hw);
  949. }
  950. /**
  951. * ice_service_task - manage and run subtasks
  952. * @work: pointer to work_struct contained by the PF struct
  953. */
  954. static void ice_service_task(struct work_struct *work)
  955. {
  956. struct ice_pf *pf = container_of(work, struct ice_pf, serv_task);
  957. unsigned long start_time = jiffies;
  958. /* subtasks */
  959. /* process reset requests first */
  960. ice_reset_subtask(pf);
  961. /* bail if a reset/recovery cycle is pending or rebuild failed */
  962. if (ice_is_reset_in_progress(pf->state) ||
  963. test_bit(__ICE_SUSPENDED, pf->state) ||
  964. test_bit(__ICE_NEEDS_RESTART, pf->state)) {
  965. ice_service_task_complete(pf);
  966. return;
  967. }
  968. ice_check_for_hang_subtask(pf);
  969. ice_sync_fltr_subtask(pf);
  970. ice_handle_mdd_event(pf);
  971. ice_process_vflr_event(pf);
  972. ice_watchdog_subtask(pf);
  973. ice_clean_adminq_subtask(pf);
  974. ice_clean_mailboxq_subtask(pf);
  975. /* Clear __ICE_SERVICE_SCHED flag to allow scheduling next event */
  976. ice_service_task_complete(pf);
  977. /* If the tasks have taken longer than one service timer period
  978. * or there is more work to be done, reset the service timer to
  979. * schedule the service task now.
  980. */
  981. if (time_after(jiffies, (start_time + pf->serv_tmr_period)) ||
  982. test_bit(__ICE_MDD_EVENT_PENDING, pf->state) ||
  983. test_bit(__ICE_VFLR_EVENT_PENDING, pf->state) ||
  984. test_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state) ||
  985. test_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state))
  986. mod_timer(&pf->serv_tmr, jiffies);
  987. }
  988. /**
  989. * ice_set_ctrlq_len - helper function to set controlq length
  990. * @hw: pointer to the hw instance
  991. */
  992. static void ice_set_ctrlq_len(struct ice_hw *hw)
  993. {
  994. hw->adminq.num_rq_entries = ICE_AQ_LEN;
  995. hw->adminq.num_sq_entries = ICE_AQ_LEN;
  996. hw->adminq.rq_buf_size = ICE_AQ_MAX_BUF_LEN;
  997. hw->adminq.sq_buf_size = ICE_AQ_MAX_BUF_LEN;
  998. hw->mailboxq.num_rq_entries = ICE_MBXQ_LEN;
  999. hw->mailboxq.num_sq_entries = ICE_MBXQ_LEN;
  1000. hw->mailboxq.rq_buf_size = ICE_MBXQ_MAX_BUF_LEN;
  1001. hw->mailboxq.sq_buf_size = ICE_MBXQ_MAX_BUF_LEN;
  1002. }
  1003. /**
  1004. * ice_irq_affinity_notify - Callback for affinity changes
  1005. * @notify: context as to what irq was changed
  1006. * @mask: the new affinity mask
  1007. *
  1008. * This is a callback function used by the irq_set_affinity_notifier function
  1009. * so that we may register to receive changes to the irq affinity masks.
  1010. */
  1011. static void ice_irq_affinity_notify(struct irq_affinity_notify *notify,
  1012. const cpumask_t *mask)
  1013. {
  1014. struct ice_q_vector *q_vector =
  1015. container_of(notify, struct ice_q_vector, affinity_notify);
  1016. cpumask_copy(&q_vector->affinity_mask, mask);
  1017. }
  1018. /**
  1019. * ice_irq_affinity_release - Callback for affinity notifier release
  1020. * @ref: internal core kernel usage
  1021. *
  1022. * This is a callback function used by the irq_set_affinity_notifier function
  1023. * to inform the current notification subscriber that they will no longer
  1024. * receive notifications.
  1025. */
  1026. static void ice_irq_affinity_release(struct kref __always_unused *ref) {}
  1027. /**
  1028. * ice_vsi_ena_irq - Enable IRQ for the given VSI
  1029. * @vsi: the VSI being configured
  1030. */
  1031. static int ice_vsi_ena_irq(struct ice_vsi *vsi)
  1032. {
  1033. struct ice_pf *pf = vsi->back;
  1034. struct ice_hw *hw = &pf->hw;
  1035. if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
  1036. int i;
  1037. for (i = 0; i < vsi->num_q_vectors; i++)
  1038. ice_irq_dynamic_ena(hw, vsi, vsi->q_vectors[i]);
  1039. }
  1040. ice_flush(hw);
  1041. return 0;
  1042. }
  1043. /**
  1044. * ice_vsi_req_irq_msix - get MSI-X vectors from the OS for the VSI
  1045. * @vsi: the VSI being configured
  1046. * @basename: name for the vector
  1047. */
  1048. static int ice_vsi_req_irq_msix(struct ice_vsi *vsi, char *basename)
  1049. {
  1050. int q_vectors = vsi->num_q_vectors;
  1051. struct ice_pf *pf = vsi->back;
  1052. int base = vsi->sw_base_vector;
  1053. int rx_int_idx = 0;
  1054. int tx_int_idx = 0;
  1055. int vector, err;
  1056. int irq_num;
  1057. for (vector = 0; vector < q_vectors; vector++) {
  1058. struct ice_q_vector *q_vector = vsi->q_vectors[vector];
  1059. irq_num = pf->msix_entries[base + vector].vector;
  1060. if (q_vector->tx.ring && q_vector->rx.ring) {
  1061. snprintf(q_vector->name, sizeof(q_vector->name) - 1,
  1062. "%s-%s-%d", basename, "TxRx", rx_int_idx++);
  1063. tx_int_idx++;
  1064. } else if (q_vector->rx.ring) {
  1065. snprintf(q_vector->name, sizeof(q_vector->name) - 1,
  1066. "%s-%s-%d", basename, "rx", rx_int_idx++);
  1067. } else if (q_vector->tx.ring) {
  1068. snprintf(q_vector->name, sizeof(q_vector->name) - 1,
  1069. "%s-%s-%d", basename, "tx", tx_int_idx++);
  1070. } else {
  1071. /* skip this unused q_vector */
  1072. continue;
  1073. }
  1074. err = devm_request_irq(&pf->pdev->dev,
  1075. pf->msix_entries[base + vector].vector,
  1076. vsi->irq_handler, 0, q_vector->name,
  1077. q_vector);
  1078. if (err) {
  1079. netdev_err(vsi->netdev,
  1080. "MSIX request_irq failed, error: %d\n", err);
  1081. goto free_q_irqs;
  1082. }
  1083. /* register for affinity change notifications */
  1084. q_vector->affinity_notify.notify = ice_irq_affinity_notify;
  1085. q_vector->affinity_notify.release = ice_irq_affinity_release;
  1086. irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
  1087. /* assign the mask for this irq */
  1088. irq_set_affinity_hint(irq_num, &q_vector->affinity_mask);
  1089. }
  1090. vsi->irqs_ready = true;
  1091. return 0;
  1092. free_q_irqs:
  1093. while (vector) {
  1094. vector--;
  1095. irq_num = pf->msix_entries[base + vector].vector,
  1096. irq_set_affinity_notifier(irq_num, NULL);
  1097. irq_set_affinity_hint(irq_num, NULL);
  1098. devm_free_irq(&pf->pdev->dev, irq_num, &vsi->q_vectors[vector]);
  1099. }
  1100. return err;
  1101. }
  1102. /**
  1103. * ice_ena_misc_vector - enable the non-queue interrupts
  1104. * @pf: board private structure
  1105. */
  1106. static void ice_ena_misc_vector(struct ice_pf *pf)
  1107. {
  1108. struct ice_hw *hw = &pf->hw;
  1109. u32 val;
  1110. /* clear things first */
  1111. wr32(hw, PFINT_OICR_ENA, 0); /* disable all */
  1112. rd32(hw, PFINT_OICR); /* read to clear */
  1113. val = (PFINT_OICR_ECC_ERR_M |
  1114. PFINT_OICR_MAL_DETECT_M |
  1115. PFINT_OICR_GRST_M |
  1116. PFINT_OICR_PCI_EXCEPTION_M |
  1117. PFINT_OICR_VFLR_M |
  1118. PFINT_OICR_HMC_ERR_M |
  1119. PFINT_OICR_PE_CRITERR_M);
  1120. wr32(hw, PFINT_OICR_ENA, val);
  1121. /* SW_ITR_IDX = 0, but don't change INTENA */
  1122. wr32(hw, GLINT_DYN_CTL(pf->hw_oicr_idx),
  1123. GLINT_DYN_CTL_SW_ITR_INDX_M | GLINT_DYN_CTL_INTENA_MSK_M);
  1124. }
  1125. /**
  1126. * ice_misc_intr - misc interrupt handler
  1127. * @irq: interrupt number
  1128. * @data: pointer to a q_vector
  1129. */
  1130. static irqreturn_t ice_misc_intr(int __always_unused irq, void *data)
  1131. {
  1132. struct ice_pf *pf = (struct ice_pf *)data;
  1133. struct ice_hw *hw = &pf->hw;
  1134. irqreturn_t ret = IRQ_NONE;
  1135. u32 oicr, ena_mask;
  1136. set_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state);
  1137. set_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state);
  1138. oicr = rd32(hw, PFINT_OICR);
  1139. ena_mask = rd32(hw, PFINT_OICR_ENA);
  1140. if (oicr & PFINT_OICR_MAL_DETECT_M) {
  1141. ena_mask &= ~PFINT_OICR_MAL_DETECT_M;
  1142. set_bit(__ICE_MDD_EVENT_PENDING, pf->state);
  1143. }
  1144. if (oicr & PFINT_OICR_VFLR_M) {
  1145. ena_mask &= ~PFINT_OICR_VFLR_M;
  1146. set_bit(__ICE_VFLR_EVENT_PENDING, pf->state);
  1147. }
  1148. if (oicr & PFINT_OICR_GRST_M) {
  1149. u32 reset;
  1150. /* we have a reset warning */
  1151. ena_mask &= ~PFINT_OICR_GRST_M;
  1152. reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >>
  1153. GLGEN_RSTAT_RESET_TYPE_S;
  1154. if (reset == ICE_RESET_CORER)
  1155. pf->corer_count++;
  1156. else if (reset == ICE_RESET_GLOBR)
  1157. pf->globr_count++;
  1158. else if (reset == ICE_RESET_EMPR)
  1159. pf->empr_count++;
  1160. else
  1161. dev_dbg(&pf->pdev->dev, "Invalid reset type %d\n",
  1162. reset);
  1163. /* If a reset cycle isn't already in progress, we set a bit in
  1164. * pf->state so that the service task can start a reset/rebuild.
  1165. * We also make note of which reset happened so that peer
  1166. * devices/drivers can be informed.
  1167. */
  1168. if (!test_and_set_bit(__ICE_RESET_OICR_RECV, pf->state)) {
  1169. if (reset == ICE_RESET_CORER)
  1170. set_bit(__ICE_CORER_RECV, pf->state);
  1171. else if (reset == ICE_RESET_GLOBR)
  1172. set_bit(__ICE_GLOBR_RECV, pf->state);
  1173. else
  1174. set_bit(__ICE_EMPR_RECV, pf->state);
  1175. /* There are couple of different bits at play here.
  1176. * hw->reset_ongoing indicates whether the hardware is
  1177. * in reset. This is set to true when a reset interrupt
  1178. * is received and set back to false after the driver
  1179. * has determined that the hardware is out of reset.
  1180. *
  1181. * __ICE_RESET_OICR_RECV in pf->state indicates
  1182. * that a post reset rebuild is required before the
  1183. * driver is operational again. This is set above.
  1184. *
  1185. * As this is the start of the reset/rebuild cycle, set
  1186. * both to indicate that.
  1187. */
  1188. hw->reset_ongoing = true;
  1189. }
  1190. }
  1191. if (oicr & PFINT_OICR_HMC_ERR_M) {
  1192. ena_mask &= ~PFINT_OICR_HMC_ERR_M;
  1193. dev_dbg(&pf->pdev->dev,
  1194. "HMC Error interrupt - info 0x%x, data 0x%x\n",
  1195. rd32(hw, PFHMC_ERRORINFO),
  1196. rd32(hw, PFHMC_ERRORDATA));
  1197. }
  1198. /* Report and mask off any remaining unexpected interrupts */
  1199. oicr &= ena_mask;
  1200. if (oicr) {
  1201. dev_dbg(&pf->pdev->dev, "unhandled interrupt oicr=0x%08x\n",
  1202. oicr);
  1203. /* If a critical error is pending there is no choice but to
  1204. * reset the device.
  1205. */
  1206. if (oicr & (PFINT_OICR_PE_CRITERR_M |
  1207. PFINT_OICR_PCI_EXCEPTION_M |
  1208. PFINT_OICR_ECC_ERR_M)) {
  1209. set_bit(__ICE_PFR_REQ, pf->state);
  1210. ice_service_task_schedule(pf);
  1211. }
  1212. ena_mask &= ~oicr;
  1213. }
  1214. ret = IRQ_HANDLED;
  1215. /* re-enable interrupt causes that are not handled during this pass */
  1216. wr32(hw, PFINT_OICR_ENA, ena_mask);
  1217. if (!test_bit(__ICE_DOWN, pf->state)) {
  1218. ice_service_task_schedule(pf);
  1219. ice_irq_dynamic_ena(hw, NULL, NULL);
  1220. }
  1221. return ret;
  1222. }
  1223. /**
  1224. * ice_free_irq_msix_misc - Unroll misc vector setup
  1225. * @pf: board private structure
  1226. */
  1227. static void ice_free_irq_msix_misc(struct ice_pf *pf)
  1228. {
  1229. /* disable OICR interrupt */
  1230. wr32(&pf->hw, PFINT_OICR_ENA, 0);
  1231. ice_flush(&pf->hw);
  1232. if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags) && pf->msix_entries) {
  1233. synchronize_irq(pf->msix_entries[pf->sw_oicr_idx].vector);
  1234. devm_free_irq(&pf->pdev->dev,
  1235. pf->msix_entries[pf->sw_oicr_idx].vector, pf);
  1236. }
  1237. pf->num_avail_sw_msix += 1;
  1238. ice_free_res(pf->sw_irq_tracker, pf->sw_oicr_idx, ICE_RES_MISC_VEC_ID);
  1239. pf->num_avail_hw_msix += 1;
  1240. ice_free_res(pf->hw_irq_tracker, pf->hw_oicr_idx, ICE_RES_MISC_VEC_ID);
  1241. }
  1242. /**
  1243. * ice_req_irq_msix_misc - Setup the misc vector to handle non queue events
  1244. * @pf: board private structure
  1245. *
  1246. * This sets up the handler for MSIX 0, which is used to manage the
  1247. * non-queue interrupts, e.g. AdminQ and errors. This is not used
  1248. * when in MSI or Legacy interrupt mode.
  1249. */
  1250. static int ice_req_irq_msix_misc(struct ice_pf *pf)
  1251. {
  1252. struct ice_hw *hw = &pf->hw;
  1253. int oicr_idx, err = 0;
  1254. u8 itr_gran;
  1255. u32 val;
  1256. if (!pf->int_name[0])
  1257. snprintf(pf->int_name, sizeof(pf->int_name) - 1, "%s-%s:misc",
  1258. dev_driver_string(&pf->pdev->dev),
  1259. dev_name(&pf->pdev->dev));
  1260. /* Do not request IRQ but do enable OICR interrupt since settings are
  1261. * lost during reset. Note that this function is called only during
  1262. * rebuild path and not while reset is in progress.
  1263. */
  1264. if (ice_is_reset_in_progress(pf->state))
  1265. goto skip_req_irq;
  1266. /* reserve one vector in sw_irq_tracker for misc interrupts */
  1267. oicr_idx = ice_get_res(pf, pf->sw_irq_tracker, 1, ICE_RES_MISC_VEC_ID);
  1268. if (oicr_idx < 0)
  1269. return oicr_idx;
  1270. pf->num_avail_sw_msix -= 1;
  1271. pf->sw_oicr_idx = oicr_idx;
  1272. /* reserve one vector in hw_irq_tracker for misc interrupts */
  1273. oicr_idx = ice_get_res(pf, pf->hw_irq_tracker, 1, ICE_RES_MISC_VEC_ID);
  1274. if (oicr_idx < 0) {
  1275. ice_free_res(pf->sw_irq_tracker, 1, ICE_RES_MISC_VEC_ID);
  1276. pf->num_avail_sw_msix += 1;
  1277. return oicr_idx;
  1278. }
  1279. pf->num_avail_hw_msix -= 1;
  1280. pf->hw_oicr_idx = oicr_idx;
  1281. err = devm_request_irq(&pf->pdev->dev,
  1282. pf->msix_entries[pf->sw_oicr_idx].vector,
  1283. ice_misc_intr, 0, pf->int_name, pf);
  1284. if (err) {
  1285. dev_err(&pf->pdev->dev,
  1286. "devm_request_irq for %s failed: %d\n",
  1287. pf->int_name, err);
  1288. ice_free_res(pf->sw_irq_tracker, 1, ICE_RES_MISC_VEC_ID);
  1289. pf->num_avail_sw_msix += 1;
  1290. ice_free_res(pf->hw_irq_tracker, 1, ICE_RES_MISC_VEC_ID);
  1291. pf->num_avail_hw_msix += 1;
  1292. return err;
  1293. }
  1294. skip_req_irq:
  1295. ice_ena_misc_vector(pf);
  1296. val = ((pf->hw_oicr_idx & PFINT_OICR_CTL_MSIX_INDX_M) |
  1297. PFINT_OICR_CTL_CAUSE_ENA_M);
  1298. wr32(hw, PFINT_OICR_CTL, val);
  1299. /* This enables Admin queue Interrupt causes */
  1300. val = ((pf->hw_oicr_idx & PFINT_FW_CTL_MSIX_INDX_M) |
  1301. PFINT_FW_CTL_CAUSE_ENA_M);
  1302. wr32(hw, PFINT_FW_CTL, val);
  1303. /* This enables Mailbox queue Interrupt causes */
  1304. val = ((pf->hw_oicr_idx & PFINT_MBX_CTL_MSIX_INDX_M) |
  1305. PFINT_MBX_CTL_CAUSE_ENA_M);
  1306. wr32(hw, PFINT_MBX_CTL, val);
  1307. itr_gran = hw->itr_gran;
  1308. wr32(hw, GLINT_ITR(ICE_RX_ITR, pf->hw_oicr_idx),
  1309. ITR_TO_REG(ICE_ITR_8K, itr_gran));
  1310. ice_flush(hw);
  1311. ice_irq_dynamic_ena(hw, NULL, NULL);
  1312. return 0;
  1313. }
  1314. /**
  1315. * ice_napi_del - Remove NAPI handler for the VSI
  1316. * @vsi: VSI for which NAPI handler is to be removed
  1317. */
  1318. static void ice_napi_del(struct ice_vsi *vsi)
  1319. {
  1320. int v_idx;
  1321. if (!vsi->netdev)
  1322. return;
  1323. for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
  1324. netif_napi_del(&vsi->q_vectors[v_idx]->napi);
  1325. }
  1326. /**
  1327. * ice_napi_add - register NAPI handler for the VSI
  1328. * @vsi: VSI for which NAPI handler is to be registered
  1329. *
  1330. * This function is only called in the driver's load path. Registering the NAPI
  1331. * handler is done in ice_vsi_alloc_q_vector() for all other cases (i.e. resume,
  1332. * reset/rebuild, etc.)
  1333. */
  1334. static void ice_napi_add(struct ice_vsi *vsi)
  1335. {
  1336. int v_idx;
  1337. if (!vsi->netdev)
  1338. return;
  1339. for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
  1340. netif_napi_add(vsi->netdev, &vsi->q_vectors[v_idx]->napi,
  1341. ice_napi_poll, NAPI_POLL_WEIGHT);
  1342. }
  1343. /**
  1344. * ice_cfg_netdev - Allocate, configure and register a netdev
  1345. * @vsi: the VSI associated with the new netdev
  1346. *
  1347. * Returns 0 on success, negative value on failure
  1348. */
  1349. static int ice_cfg_netdev(struct ice_vsi *vsi)
  1350. {
  1351. netdev_features_t csumo_features;
  1352. netdev_features_t vlano_features;
  1353. netdev_features_t dflt_features;
  1354. netdev_features_t tso_features;
  1355. struct ice_netdev_priv *np;
  1356. struct net_device *netdev;
  1357. u8 mac_addr[ETH_ALEN];
  1358. int err;
  1359. netdev = alloc_etherdev_mqs(sizeof(struct ice_netdev_priv),
  1360. vsi->alloc_txq, vsi->alloc_rxq);
  1361. if (!netdev)
  1362. return -ENOMEM;
  1363. vsi->netdev = netdev;
  1364. np = netdev_priv(netdev);
  1365. np->vsi = vsi;
  1366. dflt_features = NETIF_F_SG |
  1367. NETIF_F_HIGHDMA |
  1368. NETIF_F_RXHASH;
  1369. csumo_features = NETIF_F_RXCSUM |
  1370. NETIF_F_IP_CSUM |
  1371. NETIF_F_IPV6_CSUM;
  1372. vlano_features = NETIF_F_HW_VLAN_CTAG_FILTER |
  1373. NETIF_F_HW_VLAN_CTAG_TX |
  1374. NETIF_F_HW_VLAN_CTAG_RX;
  1375. tso_features = NETIF_F_TSO;
  1376. /* set features that user can change */
  1377. netdev->hw_features = dflt_features | csumo_features |
  1378. vlano_features | tso_features;
  1379. /* enable features */
  1380. netdev->features |= netdev->hw_features;
  1381. /* encap and VLAN devices inherit default, csumo and tso features */
  1382. netdev->hw_enc_features |= dflt_features | csumo_features |
  1383. tso_features;
  1384. netdev->vlan_features |= dflt_features | csumo_features |
  1385. tso_features;
  1386. if (vsi->type == ICE_VSI_PF) {
  1387. SET_NETDEV_DEV(netdev, &vsi->back->pdev->dev);
  1388. ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr);
  1389. ether_addr_copy(netdev->dev_addr, mac_addr);
  1390. ether_addr_copy(netdev->perm_addr, mac_addr);
  1391. }
  1392. netdev->priv_flags |= IFF_UNICAST_FLT;
  1393. /* assign netdev_ops */
  1394. netdev->netdev_ops = &ice_netdev_ops;
  1395. /* setup watchdog timeout value to be 5 second */
  1396. netdev->watchdog_timeo = 5 * HZ;
  1397. ice_set_ethtool_ops(netdev);
  1398. netdev->min_mtu = ETH_MIN_MTU;
  1399. netdev->max_mtu = ICE_MAX_MTU;
  1400. err = register_netdev(vsi->netdev);
  1401. if (err)
  1402. return err;
  1403. netif_carrier_off(vsi->netdev);
  1404. /* make sure transmit queues start off as stopped */
  1405. netif_tx_stop_all_queues(vsi->netdev);
  1406. return 0;
  1407. }
  1408. /**
  1409. * ice_fill_rss_lut - Fill the RSS lookup table with default values
  1410. * @lut: Lookup table
  1411. * @rss_table_size: Lookup table size
  1412. * @rss_size: Range of queue number for hashing
  1413. */
  1414. void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size)
  1415. {
  1416. u16 i;
  1417. for (i = 0; i < rss_table_size; i++)
  1418. lut[i] = i % rss_size;
  1419. }
  1420. /**
  1421. * ice_pf_vsi_setup - Set up a PF VSI
  1422. * @pf: board private structure
  1423. * @pi: pointer to the port_info instance
  1424. *
  1425. * Returns pointer to the successfully allocated VSI sw struct on success,
  1426. * otherwise returns NULL on failure.
  1427. */
  1428. static struct ice_vsi *
  1429. ice_pf_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi)
  1430. {
  1431. return ice_vsi_setup(pf, pi, ICE_VSI_PF, ICE_INVAL_VFID);
  1432. }
  1433. /**
  1434. * ice_vlan_rx_add_vid - Add a vlan id filter to HW offload
  1435. * @netdev: network interface to be adjusted
  1436. * @proto: unused protocol
  1437. * @vid: vlan id to be added
  1438. *
  1439. * net_device_ops implementation for adding vlan ids
  1440. */
  1441. static int ice_vlan_rx_add_vid(struct net_device *netdev,
  1442. __always_unused __be16 proto, u16 vid)
  1443. {
  1444. struct ice_netdev_priv *np = netdev_priv(netdev);
  1445. struct ice_vsi *vsi = np->vsi;
  1446. int ret;
  1447. if (vid >= VLAN_N_VID) {
  1448. netdev_err(netdev, "VLAN id requested %d is out of range %d\n",
  1449. vid, VLAN_N_VID);
  1450. return -EINVAL;
  1451. }
  1452. if (vsi->info.pvid)
  1453. return -EINVAL;
  1454. /* Enable VLAN pruning when VLAN 0 is added */
  1455. if (unlikely(!vid)) {
  1456. ret = ice_cfg_vlan_pruning(vsi, true);
  1457. if (ret)
  1458. return ret;
  1459. }
  1460. /* Add all VLAN ids including 0 to the switch filter. VLAN id 0 is
  1461. * needed to continue allowing all untagged packets since VLAN prune
  1462. * list is applied to all packets by the switch
  1463. */
  1464. ret = ice_vsi_add_vlan(vsi, vid);
  1465. if (!ret)
  1466. set_bit(vid, vsi->active_vlans);
  1467. return ret;
  1468. }
  1469. /**
  1470. * ice_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
  1471. * @netdev: network interface to be adjusted
  1472. * @proto: unused protocol
  1473. * @vid: vlan id to be removed
  1474. *
  1475. * net_device_ops implementation for removing vlan ids
  1476. */
  1477. static int ice_vlan_rx_kill_vid(struct net_device *netdev,
  1478. __always_unused __be16 proto, u16 vid)
  1479. {
  1480. struct ice_netdev_priv *np = netdev_priv(netdev);
  1481. struct ice_vsi *vsi = np->vsi;
  1482. int status;
  1483. if (vsi->info.pvid)
  1484. return -EINVAL;
  1485. /* Make sure ice_vsi_kill_vlan is successful before updating VLAN
  1486. * information
  1487. */
  1488. status = ice_vsi_kill_vlan(vsi, vid);
  1489. if (status)
  1490. return status;
  1491. clear_bit(vid, vsi->active_vlans);
  1492. /* Disable VLAN pruning when VLAN 0 is removed */
  1493. if (unlikely(!vid))
  1494. status = ice_cfg_vlan_pruning(vsi, false);
  1495. return status;
  1496. }
  1497. /**
  1498. * ice_setup_pf_sw - Setup the HW switch on startup or after reset
  1499. * @pf: board private structure
  1500. *
  1501. * Returns 0 on success, negative value on failure
  1502. */
  1503. static int ice_setup_pf_sw(struct ice_pf *pf)
  1504. {
  1505. LIST_HEAD(tmp_add_list);
  1506. u8 broadcast[ETH_ALEN];
  1507. struct ice_vsi *vsi;
  1508. int status = 0;
  1509. if (ice_is_reset_in_progress(pf->state))
  1510. return -EBUSY;
  1511. vsi = ice_pf_vsi_setup(pf, pf->hw.port_info);
  1512. if (!vsi) {
  1513. status = -ENOMEM;
  1514. goto unroll_vsi_setup;
  1515. }
  1516. status = ice_cfg_netdev(vsi);
  1517. if (status) {
  1518. status = -ENODEV;
  1519. goto unroll_vsi_setup;
  1520. }
  1521. /* registering the NAPI handler requires both the queues and
  1522. * netdev to be created, which are done in ice_pf_vsi_setup()
  1523. * and ice_cfg_netdev() respectively
  1524. */
  1525. ice_napi_add(vsi);
  1526. /* To add a MAC filter, first add the MAC to a list and then
  1527. * pass the list to ice_add_mac.
  1528. */
  1529. /* Add a unicast MAC filter so the VSI can get its packets */
  1530. status = ice_add_mac_to_list(vsi, &tmp_add_list,
  1531. vsi->port_info->mac.perm_addr);
  1532. if (status)
  1533. goto unroll_napi_add;
  1534. /* VSI needs to receive broadcast traffic, so add the broadcast
  1535. * MAC address to the list as well.
  1536. */
  1537. eth_broadcast_addr(broadcast);
  1538. status = ice_add_mac_to_list(vsi, &tmp_add_list, broadcast);
  1539. if (status)
  1540. goto free_mac_list;
  1541. /* program MAC filters for entries in tmp_add_list */
  1542. status = ice_add_mac(&pf->hw, &tmp_add_list);
  1543. if (status) {
  1544. dev_err(&pf->pdev->dev, "Could not add MAC filters\n");
  1545. status = -ENOMEM;
  1546. goto free_mac_list;
  1547. }
  1548. ice_free_fltr_list(&pf->pdev->dev, &tmp_add_list);
  1549. return status;
  1550. free_mac_list:
  1551. ice_free_fltr_list(&pf->pdev->dev, &tmp_add_list);
  1552. unroll_napi_add:
  1553. if (vsi) {
  1554. ice_napi_del(vsi);
  1555. if (vsi->netdev) {
  1556. if (vsi->netdev->reg_state == NETREG_REGISTERED)
  1557. unregister_netdev(vsi->netdev);
  1558. free_netdev(vsi->netdev);
  1559. vsi->netdev = NULL;
  1560. }
  1561. }
  1562. unroll_vsi_setup:
  1563. if (vsi) {
  1564. ice_vsi_free_q_vectors(vsi);
  1565. ice_vsi_delete(vsi);
  1566. ice_vsi_put_qs(vsi);
  1567. pf->q_left_tx += vsi->alloc_txq;
  1568. pf->q_left_rx += vsi->alloc_rxq;
  1569. ice_vsi_clear(vsi);
  1570. }
  1571. return status;
  1572. }
  1573. /**
  1574. * ice_determine_q_usage - Calculate queue distribution
  1575. * @pf: board private structure
  1576. *
  1577. * Return -ENOMEM if we don't get enough queues for all ports
  1578. */
  1579. static void ice_determine_q_usage(struct ice_pf *pf)
  1580. {
  1581. u16 q_left_tx, q_left_rx;
  1582. q_left_tx = pf->hw.func_caps.common_cap.num_txq;
  1583. q_left_rx = pf->hw.func_caps.common_cap.num_rxq;
  1584. pf->num_lan_tx = min_t(int, q_left_tx, num_online_cpus());
  1585. /* only 1 rx queue unless RSS is enabled */
  1586. if (!test_bit(ICE_FLAG_RSS_ENA, pf->flags))
  1587. pf->num_lan_rx = 1;
  1588. else
  1589. pf->num_lan_rx = min_t(int, q_left_rx, num_online_cpus());
  1590. pf->q_left_tx = q_left_tx - pf->num_lan_tx;
  1591. pf->q_left_rx = q_left_rx - pf->num_lan_rx;
  1592. }
  1593. /**
  1594. * ice_deinit_pf - Unrolls initialziations done by ice_init_pf
  1595. * @pf: board private structure to initialize
  1596. */
  1597. static void ice_deinit_pf(struct ice_pf *pf)
  1598. {
  1599. ice_service_task_stop(pf);
  1600. mutex_destroy(&pf->sw_mutex);
  1601. mutex_destroy(&pf->avail_q_mutex);
  1602. }
  1603. /**
  1604. * ice_init_pf - Initialize general software structures (struct ice_pf)
  1605. * @pf: board private structure to initialize
  1606. */
  1607. static void ice_init_pf(struct ice_pf *pf)
  1608. {
  1609. bitmap_zero(pf->flags, ICE_PF_FLAGS_NBITS);
  1610. set_bit(ICE_FLAG_MSIX_ENA, pf->flags);
  1611. #ifdef CONFIG_PCI_IOV
  1612. if (pf->hw.func_caps.common_cap.sr_iov_1_1) {
  1613. struct ice_hw *hw = &pf->hw;
  1614. set_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags);
  1615. pf->num_vfs_supported = min_t(int, hw->func_caps.num_allocd_vfs,
  1616. ICE_MAX_VF_COUNT);
  1617. }
  1618. #endif /* CONFIG_PCI_IOV */
  1619. mutex_init(&pf->sw_mutex);
  1620. mutex_init(&pf->avail_q_mutex);
  1621. /* Clear avail_[t|r]x_qs bitmaps (set all to avail) */
  1622. mutex_lock(&pf->avail_q_mutex);
  1623. bitmap_zero(pf->avail_txqs, ICE_MAX_TXQS);
  1624. bitmap_zero(pf->avail_rxqs, ICE_MAX_RXQS);
  1625. mutex_unlock(&pf->avail_q_mutex);
  1626. if (pf->hw.func_caps.common_cap.rss_table_size)
  1627. set_bit(ICE_FLAG_RSS_ENA, pf->flags);
  1628. /* setup service timer and periodic service task */
  1629. timer_setup(&pf->serv_tmr, ice_service_timer, 0);
  1630. pf->serv_tmr_period = HZ;
  1631. INIT_WORK(&pf->serv_task, ice_service_task);
  1632. clear_bit(__ICE_SERVICE_SCHED, pf->state);
  1633. }
  1634. /**
  1635. * ice_ena_msix_range - Request a range of MSIX vectors from the OS
  1636. * @pf: board private structure
  1637. *
  1638. * compute the number of MSIX vectors required (v_budget) and request from
  1639. * the OS. Return the number of vectors reserved or negative on failure
  1640. */
  1641. static int ice_ena_msix_range(struct ice_pf *pf)
  1642. {
  1643. int v_left, v_actual, v_budget = 0;
  1644. int needed, err, i;
  1645. v_left = pf->hw.func_caps.common_cap.num_msix_vectors;
  1646. /* reserve one vector for miscellaneous handler */
  1647. needed = 1;
  1648. v_budget += needed;
  1649. v_left -= needed;
  1650. /* reserve vectors for LAN traffic */
  1651. pf->num_lan_msix = min_t(int, num_online_cpus(), v_left);
  1652. v_budget += pf->num_lan_msix;
  1653. v_left -= pf->num_lan_msix;
  1654. pf->msix_entries = devm_kcalloc(&pf->pdev->dev, v_budget,
  1655. sizeof(struct msix_entry), GFP_KERNEL);
  1656. if (!pf->msix_entries) {
  1657. err = -ENOMEM;
  1658. goto exit_err;
  1659. }
  1660. for (i = 0; i < v_budget; i++)
  1661. pf->msix_entries[i].entry = i;
  1662. /* actually reserve the vectors */
  1663. v_actual = pci_enable_msix_range(pf->pdev, pf->msix_entries,
  1664. ICE_MIN_MSIX, v_budget);
  1665. if (v_actual < 0) {
  1666. dev_err(&pf->pdev->dev, "unable to reserve MSI-X vectors\n");
  1667. err = v_actual;
  1668. goto msix_err;
  1669. }
  1670. if (v_actual < v_budget) {
  1671. dev_warn(&pf->pdev->dev,
  1672. "not enough vectors. requested = %d, obtained = %d\n",
  1673. v_budget, v_actual);
  1674. if (v_actual >= (pf->num_lan_msix + 1)) {
  1675. pf->num_avail_sw_msix = v_actual -
  1676. (pf->num_lan_msix + 1);
  1677. } else if (v_actual >= 2) {
  1678. pf->num_lan_msix = 1;
  1679. pf->num_avail_sw_msix = v_actual - 2;
  1680. } else {
  1681. pci_disable_msix(pf->pdev);
  1682. err = -ERANGE;
  1683. goto msix_err;
  1684. }
  1685. }
  1686. return v_actual;
  1687. msix_err:
  1688. devm_kfree(&pf->pdev->dev, pf->msix_entries);
  1689. goto exit_err;
  1690. exit_err:
  1691. pf->num_lan_msix = 0;
  1692. clear_bit(ICE_FLAG_MSIX_ENA, pf->flags);
  1693. return err;
  1694. }
  1695. /**
  1696. * ice_dis_msix - Disable MSI-X interrupt setup in OS
  1697. * @pf: board private structure
  1698. */
  1699. static void ice_dis_msix(struct ice_pf *pf)
  1700. {
  1701. pci_disable_msix(pf->pdev);
  1702. devm_kfree(&pf->pdev->dev, pf->msix_entries);
  1703. pf->msix_entries = NULL;
  1704. clear_bit(ICE_FLAG_MSIX_ENA, pf->flags);
  1705. }
  1706. /**
  1707. * ice_clear_interrupt_scheme - Undo things done by ice_init_interrupt_scheme
  1708. * @pf: board private structure
  1709. */
  1710. static void ice_clear_interrupt_scheme(struct ice_pf *pf)
  1711. {
  1712. if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
  1713. ice_dis_msix(pf);
  1714. if (pf->sw_irq_tracker) {
  1715. devm_kfree(&pf->pdev->dev, pf->sw_irq_tracker);
  1716. pf->sw_irq_tracker = NULL;
  1717. }
  1718. if (pf->hw_irq_tracker) {
  1719. devm_kfree(&pf->pdev->dev, pf->hw_irq_tracker);
  1720. pf->hw_irq_tracker = NULL;
  1721. }
  1722. }
  1723. /**
  1724. * ice_init_interrupt_scheme - Determine proper interrupt scheme
  1725. * @pf: board private structure to initialize
  1726. */
  1727. static int ice_init_interrupt_scheme(struct ice_pf *pf)
  1728. {
  1729. int vectors = 0, hw_vectors = 0;
  1730. ssize_t size;
  1731. if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
  1732. vectors = ice_ena_msix_range(pf);
  1733. else
  1734. return -ENODEV;
  1735. if (vectors < 0)
  1736. return vectors;
  1737. /* set up vector assignment tracking */
  1738. size = sizeof(struct ice_res_tracker) + (sizeof(u16) * vectors);
  1739. pf->sw_irq_tracker = devm_kzalloc(&pf->pdev->dev, size, GFP_KERNEL);
  1740. if (!pf->sw_irq_tracker) {
  1741. ice_dis_msix(pf);
  1742. return -ENOMEM;
  1743. }
  1744. /* populate SW interrupts pool with number of OS granted IRQs. */
  1745. pf->num_avail_sw_msix = vectors;
  1746. pf->sw_irq_tracker->num_entries = vectors;
  1747. /* set up HW vector assignment tracking */
  1748. hw_vectors = pf->hw.func_caps.common_cap.num_msix_vectors;
  1749. size = sizeof(struct ice_res_tracker) + (sizeof(u16) * hw_vectors);
  1750. pf->hw_irq_tracker = devm_kzalloc(&pf->pdev->dev, size, GFP_KERNEL);
  1751. if (!pf->hw_irq_tracker) {
  1752. ice_clear_interrupt_scheme(pf);
  1753. return -ENOMEM;
  1754. }
  1755. /* populate HW interrupts pool with number of HW supported irqs. */
  1756. pf->num_avail_hw_msix = hw_vectors;
  1757. pf->hw_irq_tracker->num_entries = hw_vectors;
  1758. return 0;
  1759. }
  1760. /**
  1761. * ice_probe - Device initialization routine
  1762. * @pdev: PCI device information struct
  1763. * @ent: entry in ice_pci_tbl
  1764. *
  1765. * Returns 0 on success, negative on failure
  1766. */
  1767. static int ice_probe(struct pci_dev *pdev,
  1768. const struct pci_device_id __always_unused *ent)
  1769. {
  1770. struct ice_pf *pf;
  1771. struct ice_hw *hw;
  1772. int err;
  1773. /* this driver uses devres, see Documentation/driver-model/devres.txt */
  1774. err = pcim_enable_device(pdev);
  1775. if (err)
  1776. return err;
  1777. err = pcim_iomap_regions(pdev, BIT(ICE_BAR0), pci_name(pdev));
  1778. if (err) {
  1779. dev_err(&pdev->dev, "BAR0 I/O map error %d\n", err);
  1780. return err;
  1781. }
  1782. pf = devm_kzalloc(&pdev->dev, sizeof(*pf), GFP_KERNEL);
  1783. if (!pf)
  1784. return -ENOMEM;
  1785. /* set up for high or low dma */
  1786. err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
  1787. if (err)
  1788. err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
  1789. if (err) {
  1790. dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err);
  1791. return err;
  1792. }
  1793. pci_enable_pcie_error_reporting(pdev);
  1794. pci_set_master(pdev);
  1795. pf->pdev = pdev;
  1796. pci_set_drvdata(pdev, pf);
  1797. set_bit(__ICE_DOWN, pf->state);
  1798. /* Disable service task until DOWN bit is cleared */
  1799. set_bit(__ICE_SERVICE_DIS, pf->state);
  1800. hw = &pf->hw;
  1801. hw->hw_addr = pcim_iomap_table(pdev)[ICE_BAR0];
  1802. hw->back = pf;
  1803. hw->vendor_id = pdev->vendor;
  1804. hw->device_id = pdev->device;
  1805. pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
  1806. hw->subsystem_vendor_id = pdev->subsystem_vendor;
  1807. hw->subsystem_device_id = pdev->subsystem_device;
  1808. hw->bus.device = PCI_SLOT(pdev->devfn);
  1809. hw->bus.func = PCI_FUNC(pdev->devfn);
  1810. ice_set_ctrlq_len(hw);
  1811. pf->msg_enable = netif_msg_init(debug, ICE_DFLT_NETIF_M);
  1812. #ifndef CONFIG_DYNAMIC_DEBUG
  1813. if (debug < -1)
  1814. hw->debug_mask = debug;
  1815. #endif
  1816. err = ice_init_hw(hw);
  1817. if (err) {
  1818. dev_err(&pdev->dev, "ice_init_hw failed: %d\n", err);
  1819. err = -EIO;
  1820. goto err_exit_unroll;
  1821. }
  1822. dev_info(&pdev->dev, "firmware %d.%d.%05d api %d.%d\n",
  1823. hw->fw_maj_ver, hw->fw_min_ver, hw->fw_build,
  1824. hw->api_maj_ver, hw->api_min_ver);
  1825. ice_init_pf(pf);
  1826. ice_determine_q_usage(pf);
  1827. pf->num_alloc_vsi = min_t(u16, ICE_MAX_VSI_ALLOC,
  1828. hw->func_caps.guaranteed_num_vsi);
  1829. if (!pf->num_alloc_vsi) {
  1830. err = -EIO;
  1831. goto err_init_pf_unroll;
  1832. }
  1833. pf->vsi = devm_kcalloc(&pdev->dev, pf->num_alloc_vsi,
  1834. sizeof(struct ice_vsi *), GFP_KERNEL);
  1835. if (!pf->vsi) {
  1836. err = -ENOMEM;
  1837. goto err_init_pf_unroll;
  1838. }
  1839. err = ice_init_interrupt_scheme(pf);
  1840. if (err) {
  1841. dev_err(&pdev->dev,
  1842. "ice_init_interrupt_scheme failed: %d\n", err);
  1843. err = -EIO;
  1844. goto err_init_interrupt_unroll;
  1845. }
  1846. /* Driver is mostly up */
  1847. clear_bit(__ICE_DOWN, pf->state);
  1848. /* In case of MSIX we are going to setup the misc vector right here
  1849. * to handle admin queue events etc. In case of legacy and MSI
  1850. * the misc functionality and queue processing is combined in
  1851. * the same vector and that gets setup at open.
  1852. */
  1853. if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
  1854. err = ice_req_irq_msix_misc(pf);
  1855. if (err) {
  1856. dev_err(&pdev->dev,
  1857. "setup of misc vector failed: %d\n", err);
  1858. goto err_init_interrupt_unroll;
  1859. }
  1860. }
  1861. /* create switch struct for the switch element created by FW on boot */
  1862. pf->first_sw = devm_kzalloc(&pdev->dev, sizeof(struct ice_sw),
  1863. GFP_KERNEL);
  1864. if (!pf->first_sw) {
  1865. err = -ENOMEM;
  1866. goto err_msix_misc_unroll;
  1867. }
  1868. if (hw->evb_veb)
  1869. pf->first_sw->bridge_mode = BRIDGE_MODE_VEB;
  1870. else
  1871. pf->first_sw->bridge_mode = BRIDGE_MODE_VEPA;
  1872. pf->first_sw->pf = pf;
  1873. /* record the sw_id available for later use */
  1874. pf->first_sw->sw_id = hw->port_info->sw_id;
  1875. err = ice_setup_pf_sw(pf);
  1876. if (err) {
  1877. dev_err(&pdev->dev,
  1878. "probe failed due to setup pf switch:%d\n", err);
  1879. goto err_alloc_sw_unroll;
  1880. }
  1881. clear_bit(__ICE_SERVICE_DIS, pf->state);
  1882. /* since everything is good, start the service timer */
  1883. mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period));
  1884. err = ice_init_link_events(pf->hw.port_info);
  1885. if (err) {
  1886. dev_err(&pdev->dev, "ice_init_link_events failed: %d\n", err);
  1887. goto err_alloc_sw_unroll;
  1888. }
  1889. return 0;
  1890. err_alloc_sw_unroll:
  1891. set_bit(__ICE_SERVICE_DIS, pf->state);
  1892. set_bit(__ICE_DOWN, pf->state);
  1893. devm_kfree(&pf->pdev->dev, pf->first_sw);
  1894. err_msix_misc_unroll:
  1895. ice_free_irq_msix_misc(pf);
  1896. err_init_interrupt_unroll:
  1897. ice_clear_interrupt_scheme(pf);
  1898. devm_kfree(&pdev->dev, pf->vsi);
  1899. err_init_pf_unroll:
  1900. ice_deinit_pf(pf);
  1901. ice_deinit_hw(hw);
  1902. err_exit_unroll:
  1903. pci_disable_pcie_error_reporting(pdev);
  1904. return err;
  1905. }
  1906. /**
  1907. * ice_remove - Device removal routine
  1908. * @pdev: PCI device information struct
  1909. */
  1910. static void ice_remove(struct pci_dev *pdev)
  1911. {
  1912. struct ice_pf *pf = pci_get_drvdata(pdev);
  1913. int i;
  1914. if (!pf)
  1915. return;
  1916. set_bit(__ICE_DOWN, pf->state);
  1917. ice_service_task_stop(pf);
  1918. if (test_bit(ICE_FLAG_SRIOV_ENA, pf->flags))
  1919. ice_free_vfs(pf);
  1920. ice_vsi_release_all(pf);
  1921. ice_free_irq_msix_misc(pf);
  1922. ice_for_each_vsi(pf, i) {
  1923. if (!pf->vsi[i])
  1924. continue;
  1925. ice_vsi_free_q_vectors(pf->vsi[i]);
  1926. }
  1927. ice_clear_interrupt_scheme(pf);
  1928. ice_deinit_pf(pf);
  1929. ice_deinit_hw(&pf->hw);
  1930. pci_disable_pcie_error_reporting(pdev);
  1931. }
  1932. /* ice_pci_tbl - PCI Device ID Table
  1933. *
  1934. * Wildcard entries (PCI_ANY_ID) should come last
  1935. * Last entry must be all 0s
  1936. *
  1937. * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  1938. * Class, Class Mask, private data (not used) }
  1939. */
  1940. static const struct pci_device_id ice_pci_tbl[] = {
  1941. { PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_BACKPLANE), 0 },
  1942. { PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_QSFP), 0 },
  1943. { PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_SFP), 0 },
  1944. /* required last entry */
  1945. { 0, }
  1946. };
  1947. MODULE_DEVICE_TABLE(pci, ice_pci_tbl);
  1948. static struct pci_driver ice_driver = {
  1949. .name = KBUILD_MODNAME,
  1950. .id_table = ice_pci_tbl,
  1951. .probe = ice_probe,
  1952. .remove = ice_remove,
  1953. .sriov_configure = ice_sriov_configure,
  1954. };
  1955. /**
  1956. * ice_module_init - Driver registration routine
  1957. *
  1958. * ice_module_init is the first routine called when the driver is
  1959. * loaded. All it does is register with the PCI subsystem.
  1960. */
  1961. static int __init ice_module_init(void)
  1962. {
  1963. int status;
  1964. pr_info("%s - version %s\n", ice_driver_string, ice_drv_ver);
  1965. pr_info("%s\n", ice_copyright);
  1966. ice_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, KBUILD_MODNAME);
  1967. if (!ice_wq) {
  1968. pr_err("Failed to create workqueue\n");
  1969. return -ENOMEM;
  1970. }
  1971. status = pci_register_driver(&ice_driver);
  1972. if (status) {
  1973. pr_err("failed to register pci driver, err %d\n", status);
  1974. destroy_workqueue(ice_wq);
  1975. }
  1976. return status;
  1977. }
  1978. module_init(ice_module_init);
  1979. /**
  1980. * ice_module_exit - Driver exit cleanup routine
  1981. *
  1982. * ice_module_exit is called just before the driver is removed
  1983. * from memory.
  1984. */
  1985. static void __exit ice_module_exit(void)
  1986. {
  1987. pci_unregister_driver(&ice_driver);
  1988. destroy_workqueue(ice_wq);
  1989. pr_info("module unloaded\n");
  1990. }
  1991. module_exit(ice_module_exit);
  1992. /**
  1993. * ice_set_mac_address - NDO callback to set mac address
  1994. * @netdev: network interface device structure
  1995. * @pi: pointer to an address structure
  1996. *
  1997. * Returns 0 on success, negative on failure
  1998. */
  1999. static int ice_set_mac_address(struct net_device *netdev, void *pi)
  2000. {
  2001. struct ice_netdev_priv *np = netdev_priv(netdev);
  2002. struct ice_vsi *vsi = np->vsi;
  2003. struct ice_pf *pf = vsi->back;
  2004. struct ice_hw *hw = &pf->hw;
  2005. struct sockaddr *addr = pi;
  2006. enum ice_status status;
  2007. LIST_HEAD(a_mac_list);
  2008. LIST_HEAD(r_mac_list);
  2009. u8 flags = 0;
  2010. int err;
  2011. u8 *mac;
  2012. mac = (u8 *)addr->sa_data;
  2013. if (!is_valid_ether_addr(mac))
  2014. return -EADDRNOTAVAIL;
  2015. if (ether_addr_equal(netdev->dev_addr, mac)) {
  2016. netdev_warn(netdev, "already using mac %pM\n", mac);
  2017. return 0;
  2018. }
  2019. if (test_bit(__ICE_DOWN, pf->state) ||
  2020. ice_is_reset_in_progress(pf->state)) {
  2021. netdev_err(netdev, "can't set mac %pM. device not ready\n",
  2022. mac);
  2023. return -EBUSY;
  2024. }
  2025. /* When we change the mac address we also have to change the mac address
  2026. * based filter rules that were created previously for the old mac
  2027. * address. So first, we remove the old filter rule using ice_remove_mac
  2028. * and then create a new filter rule using ice_add_mac. Note that for
  2029. * both these operations, we first need to form a "list" of mac
  2030. * addresses (even though in this case, we have only 1 mac address to be
  2031. * added/removed) and this done using ice_add_mac_to_list. Depending on
  2032. * the ensuing operation this "list" of mac addresses is either to be
  2033. * added or removed from the filter.
  2034. */
  2035. err = ice_add_mac_to_list(vsi, &r_mac_list, netdev->dev_addr);
  2036. if (err) {
  2037. err = -EADDRNOTAVAIL;
  2038. goto free_lists;
  2039. }
  2040. status = ice_remove_mac(hw, &r_mac_list);
  2041. if (status) {
  2042. err = -EADDRNOTAVAIL;
  2043. goto free_lists;
  2044. }
  2045. err = ice_add_mac_to_list(vsi, &a_mac_list, mac);
  2046. if (err) {
  2047. err = -EADDRNOTAVAIL;
  2048. goto free_lists;
  2049. }
  2050. status = ice_add_mac(hw, &a_mac_list);
  2051. if (status) {
  2052. err = -EADDRNOTAVAIL;
  2053. goto free_lists;
  2054. }
  2055. free_lists:
  2056. /* free list entries */
  2057. ice_free_fltr_list(&pf->pdev->dev, &r_mac_list);
  2058. ice_free_fltr_list(&pf->pdev->dev, &a_mac_list);
  2059. if (err) {
  2060. netdev_err(netdev, "can't set mac %pM. filter update failed\n",
  2061. mac);
  2062. return err;
  2063. }
  2064. /* change the netdev's mac address */
  2065. memcpy(netdev->dev_addr, mac, netdev->addr_len);
  2066. netdev_dbg(vsi->netdev, "updated mac address to %pM\n",
  2067. netdev->dev_addr);
  2068. /* write new mac address to the firmware */
  2069. flags = ICE_AQC_MAN_MAC_UPDATE_LAA_WOL;
  2070. status = ice_aq_manage_mac_write(hw, mac, flags, NULL);
  2071. if (status) {
  2072. netdev_err(netdev, "can't set mac %pM. write to firmware failed.\n",
  2073. mac);
  2074. }
  2075. return 0;
  2076. }
  2077. /**
  2078. * ice_set_rx_mode - NDO callback to set the netdev filters
  2079. * @netdev: network interface device structure
  2080. */
  2081. static void ice_set_rx_mode(struct net_device *netdev)
  2082. {
  2083. struct ice_netdev_priv *np = netdev_priv(netdev);
  2084. struct ice_vsi *vsi = np->vsi;
  2085. if (!vsi)
  2086. return;
  2087. /* Set the flags to synchronize filters
  2088. * ndo_set_rx_mode may be triggered even without a change in netdev
  2089. * flags
  2090. */
  2091. set_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags);
  2092. set_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags);
  2093. set_bit(ICE_FLAG_FLTR_SYNC, vsi->back->flags);
  2094. /* schedule our worker thread which will take care of
  2095. * applying the new filter changes
  2096. */
  2097. ice_service_task_schedule(vsi->back);
  2098. }
  2099. /**
  2100. * ice_fdb_add - add an entry to the hardware database
  2101. * @ndm: the input from the stack
  2102. * @tb: pointer to array of nladdr (unused)
  2103. * @dev: the net device pointer
  2104. * @addr: the MAC address entry being added
  2105. * @vid: VLAN id
  2106. * @flags: instructions from stack about fdb operation
  2107. */
  2108. static int ice_fdb_add(struct ndmsg *ndm, struct nlattr __always_unused *tb[],
  2109. struct net_device *dev, const unsigned char *addr,
  2110. u16 vid, u16 flags)
  2111. {
  2112. int err;
  2113. if (vid) {
  2114. netdev_err(dev, "VLANs aren't supported yet for dev_uc|mc_add()\n");
  2115. return -EINVAL;
  2116. }
  2117. if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
  2118. netdev_err(dev, "FDB only supports static addresses\n");
  2119. return -EINVAL;
  2120. }
  2121. if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
  2122. err = dev_uc_add_excl(dev, addr);
  2123. else if (is_multicast_ether_addr(addr))
  2124. err = dev_mc_add_excl(dev, addr);
  2125. else
  2126. err = -EINVAL;
  2127. /* Only return duplicate errors if NLM_F_EXCL is set */
  2128. if (err == -EEXIST && !(flags & NLM_F_EXCL))
  2129. err = 0;
  2130. return err;
  2131. }
  2132. /**
  2133. * ice_fdb_del - delete an entry from the hardware database
  2134. * @ndm: the input from the stack
  2135. * @tb: pointer to array of nladdr (unused)
  2136. * @dev: the net device pointer
  2137. * @addr: the MAC address entry being added
  2138. * @vid: VLAN id
  2139. */
  2140. static int ice_fdb_del(struct ndmsg *ndm, __always_unused struct nlattr *tb[],
  2141. struct net_device *dev, const unsigned char *addr,
  2142. __always_unused u16 vid)
  2143. {
  2144. int err;
  2145. if (ndm->ndm_state & NUD_PERMANENT) {
  2146. netdev_err(dev, "FDB only supports static addresses\n");
  2147. return -EINVAL;
  2148. }
  2149. if (is_unicast_ether_addr(addr))
  2150. err = dev_uc_del(dev, addr);
  2151. else if (is_multicast_ether_addr(addr))
  2152. err = dev_mc_del(dev, addr);
  2153. else
  2154. err = -EINVAL;
  2155. return err;
  2156. }
  2157. /**
  2158. * ice_set_features - set the netdev feature flags
  2159. * @netdev: ptr to the netdev being adjusted
  2160. * @features: the feature set that the stack is suggesting
  2161. */
  2162. static int ice_set_features(struct net_device *netdev,
  2163. netdev_features_t features)
  2164. {
  2165. struct ice_netdev_priv *np = netdev_priv(netdev);
  2166. struct ice_vsi *vsi = np->vsi;
  2167. int ret = 0;
  2168. if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
  2169. ret = ice_vsi_manage_rss_lut(vsi, true);
  2170. else if (!(features & NETIF_F_RXHASH) &&
  2171. netdev->features & NETIF_F_RXHASH)
  2172. ret = ice_vsi_manage_rss_lut(vsi, false);
  2173. if ((features & NETIF_F_HW_VLAN_CTAG_RX) &&
  2174. !(netdev->features & NETIF_F_HW_VLAN_CTAG_RX))
  2175. ret = ice_vsi_manage_vlan_stripping(vsi, true);
  2176. else if (!(features & NETIF_F_HW_VLAN_CTAG_RX) &&
  2177. (netdev->features & NETIF_F_HW_VLAN_CTAG_RX))
  2178. ret = ice_vsi_manage_vlan_stripping(vsi, false);
  2179. else if ((features & NETIF_F_HW_VLAN_CTAG_TX) &&
  2180. !(netdev->features & NETIF_F_HW_VLAN_CTAG_TX))
  2181. ret = ice_vsi_manage_vlan_insertion(vsi);
  2182. else if (!(features & NETIF_F_HW_VLAN_CTAG_TX) &&
  2183. (netdev->features & NETIF_F_HW_VLAN_CTAG_TX))
  2184. ret = ice_vsi_manage_vlan_insertion(vsi);
  2185. return ret;
  2186. }
  2187. /**
  2188. * ice_vsi_vlan_setup - Setup vlan offload properties on a VSI
  2189. * @vsi: VSI to setup vlan properties for
  2190. */
  2191. static int ice_vsi_vlan_setup(struct ice_vsi *vsi)
  2192. {
  2193. int ret = 0;
  2194. if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
  2195. ret = ice_vsi_manage_vlan_stripping(vsi, true);
  2196. if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)
  2197. ret = ice_vsi_manage_vlan_insertion(vsi);
  2198. return ret;
  2199. }
  2200. /**
  2201. * ice_restore_vlan - Reinstate VLANs when vsi/netdev comes back up
  2202. * @vsi: the VSI being brought back up
  2203. */
  2204. static int ice_restore_vlan(struct ice_vsi *vsi)
  2205. {
  2206. int err;
  2207. u16 vid;
  2208. if (!vsi->netdev)
  2209. return -EINVAL;
  2210. err = ice_vsi_vlan_setup(vsi);
  2211. if (err)
  2212. return err;
  2213. for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) {
  2214. err = ice_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q), vid);
  2215. if (err)
  2216. break;
  2217. }
  2218. return err;
  2219. }
  2220. /**
  2221. * ice_vsi_cfg - Setup the VSI
  2222. * @vsi: the VSI being configured
  2223. *
  2224. * Return 0 on success and negative value on error
  2225. */
  2226. static int ice_vsi_cfg(struct ice_vsi *vsi)
  2227. {
  2228. int err;
  2229. if (vsi->netdev) {
  2230. ice_set_rx_mode(vsi->netdev);
  2231. err = ice_restore_vlan(vsi);
  2232. if (err)
  2233. return err;
  2234. }
  2235. err = ice_vsi_cfg_txqs(vsi);
  2236. if (!err)
  2237. err = ice_vsi_cfg_rxqs(vsi);
  2238. return err;
  2239. }
  2240. /**
  2241. * ice_napi_enable_all - Enable NAPI for all q_vectors in the VSI
  2242. * @vsi: the VSI being configured
  2243. */
  2244. static void ice_napi_enable_all(struct ice_vsi *vsi)
  2245. {
  2246. int q_idx;
  2247. if (!vsi->netdev)
  2248. return;
  2249. for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
  2250. napi_enable(&vsi->q_vectors[q_idx]->napi);
  2251. }
  2252. /**
  2253. * ice_up_complete - Finish the last steps of bringing up a connection
  2254. * @vsi: The VSI being configured
  2255. *
  2256. * Return 0 on success and negative value on error
  2257. */
  2258. static int ice_up_complete(struct ice_vsi *vsi)
  2259. {
  2260. struct ice_pf *pf = vsi->back;
  2261. int err;
  2262. if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
  2263. ice_vsi_cfg_msix(vsi);
  2264. else
  2265. return -ENOTSUPP;
  2266. /* Enable only Rx rings, Tx rings were enabled by the FW when the
  2267. * Tx queue group list was configured and the context bits were
  2268. * programmed using ice_vsi_cfg_txqs
  2269. */
  2270. err = ice_vsi_start_rx_rings(vsi);
  2271. if (err)
  2272. return err;
  2273. clear_bit(__ICE_DOWN, vsi->state);
  2274. ice_napi_enable_all(vsi);
  2275. ice_vsi_ena_irq(vsi);
  2276. if (vsi->port_info &&
  2277. (vsi->port_info->phy.link_info.link_info & ICE_AQ_LINK_UP) &&
  2278. vsi->netdev) {
  2279. ice_print_link_msg(vsi, true);
  2280. netif_tx_start_all_queues(vsi->netdev);
  2281. netif_carrier_on(vsi->netdev);
  2282. }
  2283. ice_service_task_schedule(pf);
  2284. return err;
  2285. }
  2286. /**
  2287. * ice_up - Bring the connection back up after being down
  2288. * @vsi: VSI being configured
  2289. */
  2290. int ice_up(struct ice_vsi *vsi)
  2291. {
  2292. int err;
  2293. err = ice_vsi_cfg(vsi);
  2294. if (!err)
  2295. err = ice_up_complete(vsi);
  2296. return err;
  2297. }
  2298. /**
  2299. * ice_fetch_u64_stats_per_ring - get packets and bytes stats per ring
  2300. * @ring: Tx or Rx ring to read stats from
  2301. * @pkts: packets stats counter
  2302. * @bytes: bytes stats counter
  2303. *
  2304. * This function fetches stats from the ring considering the atomic operations
  2305. * that needs to be performed to read u64 values in 32 bit machine.
  2306. */
  2307. static void ice_fetch_u64_stats_per_ring(struct ice_ring *ring, u64 *pkts,
  2308. u64 *bytes)
  2309. {
  2310. unsigned int start;
  2311. *pkts = 0;
  2312. *bytes = 0;
  2313. if (!ring)
  2314. return;
  2315. do {
  2316. start = u64_stats_fetch_begin_irq(&ring->syncp);
  2317. *pkts = ring->stats.pkts;
  2318. *bytes = ring->stats.bytes;
  2319. } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
  2320. }
  2321. /**
  2322. * ice_update_vsi_ring_stats - Update VSI stats counters
  2323. * @vsi: the VSI to be updated
  2324. */
  2325. static void ice_update_vsi_ring_stats(struct ice_vsi *vsi)
  2326. {
  2327. struct rtnl_link_stats64 *vsi_stats = &vsi->net_stats;
  2328. struct ice_ring *ring;
  2329. u64 pkts, bytes;
  2330. int i;
  2331. /* reset netdev stats */
  2332. vsi_stats->tx_packets = 0;
  2333. vsi_stats->tx_bytes = 0;
  2334. vsi_stats->rx_packets = 0;
  2335. vsi_stats->rx_bytes = 0;
  2336. /* reset non-netdev (extended) stats */
  2337. vsi->tx_restart = 0;
  2338. vsi->tx_busy = 0;
  2339. vsi->tx_linearize = 0;
  2340. vsi->rx_buf_failed = 0;
  2341. vsi->rx_page_failed = 0;
  2342. rcu_read_lock();
  2343. /* update Tx rings counters */
  2344. ice_for_each_txq(vsi, i) {
  2345. ring = READ_ONCE(vsi->tx_rings[i]);
  2346. ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes);
  2347. vsi_stats->tx_packets += pkts;
  2348. vsi_stats->tx_bytes += bytes;
  2349. vsi->tx_restart += ring->tx_stats.restart_q;
  2350. vsi->tx_busy += ring->tx_stats.tx_busy;
  2351. vsi->tx_linearize += ring->tx_stats.tx_linearize;
  2352. }
  2353. /* update Rx rings counters */
  2354. ice_for_each_rxq(vsi, i) {
  2355. ring = READ_ONCE(vsi->rx_rings[i]);
  2356. ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes);
  2357. vsi_stats->rx_packets += pkts;
  2358. vsi_stats->rx_bytes += bytes;
  2359. vsi->rx_buf_failed += ring->rx_stats.alloc_buf_failed;
  2360. vsi->rx_page_failed += ring->rx_stats.alloc_page_failed;
  2361. }
  2362. rcu_read_unlock();
  2363. }
  2364. /**
  2365. * ice_update_vsi_stats - Update VSI stats counters
  2366. * @vsi: the VSI to be updated
  2367. */
  2368. static void ice_update_vsi_stats(struct ice_vsi *vsi)
  2369. {
  2370. struct rtnl_link_stats64 *cur_ns = &vsi->net_stats;
  2371. struct ice_eth_stats *cur_es = &vsi->eth_stats;
  2372. struct ice_pf *pf = vsi->back;
  2373. if (test_bit(__ICE_DOWN, vsi->state) ||
  2374. test_bit(__ICE_CFG_BUSY, pf->state))
  2375. return;
  2376. /* get stats as recorded by Tx/Rx rings */
  2377. ice_update_vsi_ring_stats(vsi);
  2378. /* get VSI stats as recorded by the hardware */
  2379. ice_update_eth_stats(vsi);
  2380. cur_ns->tx_errors = cur_es->tx_errors;
  2381. cur_ns->rx_dropped = cur_es->rx_discards;
  2382. cur_ns->tx_dropped = cur_es->tx_discards;
  2383. cur_ns->multicast = cur_es->rx_multicast;
  2384. /* update some more netdev stats if this is main VSI */
  2385. if (vsi->type == ICE_VSI_PF) {
  2386. cur_ns->rx_crc_errors = pf->stats.crc_errors;
  2387. cur_ns->rx_errors = pf->stats.crc_errors +
  2388. pf->stats.illegal_bytes;
  2389. cur_ns->rx_length_errors = pf->stats.rx_len_errors;
  2390. }
  2391. }
  2392. /**
  2393. * ice_update_pf_stats - Update PF port stats counters
  2394. * @pf: PF whose stats needs to be updated
  2395. */
  2396. static void ice_update_pf_stats(struct ice_pf *pf)
  2397. {
  2398. struct ice_hw_port_stats *prev_ps, *cur_ps;
  2399. struct ice_hw *hw = &pf->hw;
  2400. u8 pf_id;
  2401. prev_ps = &pf->stats_prev;
  2402. cur_ps = &pf->stats;
  2403. pf_id = hw->pf_id;
  2404. ice_stat_update40(hw, GLPRT_GORCH(pf_id), GLPRT_GORCL(pf_id),
  2405. pf->stat_prev_loaded, &prev_ps->eth.rx_bytes,
  2406. &cur_ps->eth.rx_bytes);
  2407. ice_stat_update40(hw, GLPRT_UPRCH(pf_id), GLPRT_UPRCL(pf_id),
  2408. pf->stat_prev_loaded, &prev_ps->eth.rx_unicast,
  2409. &cur_ps->eth.rx_unicast);
  2410. ice_stat_update40(hw, GLPRT_MPRCH(pf_id), GLPRT_MPRCL(pf_id),
  2411. pf->stat_prev_loaded, &prev_ps->eth.rx_multicast,
  2412. &cur_ps->eth.rx_multicast);
  2413. ice_stat_update40(hw, GLPRT_BPRCH(pf_id), GLPRT_BPRCL(pf_id),
  2414. pf->stat_prev_loaded, &prev_ps->eth.rx_broadcast,
  2415. &cur_ps->eth.rx_broadcast);
  2416. ice_stat_update40(hw, GLPRT_GOTCH(pf_id), GLPRT_GOTCL(pf_id),
  2417. pf->stat_prev_loaded, &prev_ps->eth.tx_bytes,
  2418. &cur_ps->eth.tx_bytes);
  2419. ice_stat_update40(hw, GLPRT_UPTCH(pf_id), GLPRT_UPTCL(pf_id),
  2420. pf->stat_prev_loaded, &prev_ps->eth.tx_unicast,
  2421. &cur_ps->eth.tx_unicast);
  2422. ice_stat_update40(hw, GLPRT_MPTCH(pf_id), GLPRT_MPTCL(pf_id),
  2423. pf->stat_prev_loaded, &prev_ps->eth.tx_multicast,
  2424. &cur_ps->eth.tx_multicast);
  2425. ice_stat_update40(hw, GLPRT_BPTCH(pf_id), GLPRT_BPTCL(pf_id),
  2426. pf->stat_prev_loaded, &prev_ps->eth.tx_broadcast,
  2427. &cur_ps->eth.tx_broadcast);
  2428. ice_stat_update32(hw, GLPRT_TDOLD(pf_id), pf->stat_prev_loaded,
  2429. &prev_ps->tx_dropped_link_down,
  2430. &cur_ps->tx_dropped_link_down);
  2431. ice_stat_update40(hw, GLPRT_PRC64H(pf_id), GLPRT_PRC64L(pf_id),
  2432. pf->stat_prev_loaded, &prev_ps->rx_size_64,
  2433. &cur_ps->rx_size_64);
  2434. ice_stat_update40(hw, GLPRT_PRC127H(pf_id), GLPRT_PRC127L(pf_id),
  2435. pf->stat_prev_loaded, &prev_ps->rx_size_127,
  2436. &cur_ps->rx_size_127);
  2437. ice_stat_update40(hw, GLPRT_PRC255H(pf_id), GLPRT_PRC255L(pf_id),
  2438. pf->stat_prev_loaded, &prev_ps->rx_size_255,
  2439. &cur_ps->rx_size_255);
  2440. ice_stat_update40(hw, GLPRT_PRC511H(pf_id), GLPRT_PRC511L(pf_id),
  2441. pf->stat_prev_loaded, &prev_ps->rx_size_511,
  2442. &cur_ps->rx_size_511);
  2443. ice_stat_update40(hw, GLPRT_PRC1023H(pf_id),
  2444. GLPRT_PRC1023L(pf_id), pf->stat_prev_loaded,
  2445. &prev_ps->rx_size_1023, &cur_ps->rx_size_1023);
  2446. ice_stat_update40(hw, GLPRT_PRC1522H(pf_id),
  2447. GLPRT_PRC1522L(pf_id), pf->stat_prev_loaded,
  2448. &prev_ps->rx_size_1522, &cur_ps->rx_size_1522);
  2449. ice_stat_update40(hw, GLPRT_PRC9522H(pf_id),
  2450. GLPRT_PRC9522L(pf_id), pf->stat_prev_loaded,
  2451. &prev_ps->rx_size_big, &cur_ps->rx_size_big);
  2452. ice_stat_update40(hw, GLPRT_PTC64H(pf_id), GLPRT_PTC64L(pf_id),
  2453. pf->stat_prev_loaded, &prev_ps->tx_size_64,
  2454. &cur_ps->tx_size_64);
  2455. ice_stat_update40(hw, GLPRT_PTC127H(pf_id), GLPRT_PTC127L(pf_id),
  2456. pf->stat_prev_loaded, &prev_ps->tx_size_127,
  2457. &cur_ps->tx_size_127);
  2458. ice_stat_update40(hw, GLPRT_PTC255H(pf_id), GLPRT_PTC255L(pf_id),
  2459. pf->stat_prev_loaded, &prev_ps->tx_size_255,
  2460. &cur_ps->tx_size_255);
  2461. ice_stat_update40(hw, GLPRT_PTC511H(pf_id), GLPRT_PTC511L(pf_id),
  2462. pf->stat_prev_loaded, &prev_ps->tx_size_511,
  2463. &cur_ps->tx_size_511);
  2464. ice_stat_update40(hw, GLPRT_PTC1023H(pf_id),
  2465. GLPRT_PTC1023L(pf_id), pf->stat_prev_loaded,
  2466. &prev_ps->tx_size_1023, &cur_ps->tx_size_1023);
  2467. ice_stat_update40(hw, GLPRT_PTC1522H(pf_id),
  2468. GLPRT_PTC1522L(pf_id), pf->stat_prev_loaded,
  2469. &prev_ps->tx_size_1522, &cur_ps->tx_size_1522);
  2470. ice_stat_update40(hw, GLPRT_PTC9522H(pf_id),
  2471. GLPRT_PTC9522L(pf_id), pf->stat_prev_loaded,
  2472. &prev_ps->tx_size_big, &cur_ps->tx_size_big);
  2473. ice_stat_update32(hw, GLPRT_LXONRXC(pf_id), pf->stat_prev_loaded,
  2474. &prev_ps->link_xon_rx, &cur_ps->link_xon_rx);
  2475. ice_stat_update32(hw, GLPRT_LXOFFRXC(pf_id), pf->stat_prev_loaded,
  2476. &prev_ps->link_xoff_rx, &cur_ps->link_xoff_rx);
  2477. ice_stat_update32(hw, GLPRT_LXONTXC(pf_id), pf->stat_prev_loaded,
  2478. &prev_ps->link_xon_tx, &cur_ps->link_xon_tx);
  2479. ice_stat_update32(hw, GLPRT_LXOFFTXC(pf_id), pf->stat_prev_loaded,
  2480. &prev_ps->link_xoff_tx, &cur_ps->link_xoff_tx);
  2481. ice_stat_update32(hw, GLPRT_CRCERRS(pf_id), pf->stat_prev_loaded,
  2482. &prev_ps->crc_errors, &cur_ps->crc_errors);
  2483. ice_stat_update32(hw, GLPRT_ILLERRC(pf_id), pf->stat_prev_loaded,
  2484. &prev_ps->illegal_bytes, &cur_ps->illegal_bytes);
  2485. ice_stat_update32(hw, GLPRT_MLFC(pf_id), pf->stat_prev_loaded,
  2486. &prev_ps->mac_local_faults,
  2487. &cur_ps->mac_local_faults);
  2488. ice_stat_update32(hw, GLPRT_MRFC(pf_id), pf->stat_prev_loaded,
  2489. &prev_ps->mac_remote_faults,
  2490. &cur_ps->mac_remote_faults);
  2491. ice_stat_update32(hw, GLPRT_RLEC(pf_id), pf->stat_prev_loaded,
  2492. &prev_ps->rx_len_errors, &cur_ps->rx_len_errors);
  2493. ice_stat_update32(hw, GLPRT_RUC(pf_id), pf->stat_prev_loaded,
  2494. &prev_ps->rx_undersize, &cur_ps->rx_undersize);
  2495. ice_stat_update32(hw, GLPRT_RFC(pf_id), pf->stat_prev_loaded,
  2496. &prev_ps->rx_fragments, &cur_ps->rx_fragments);
  2497. ice_stat_update32(hw, GLPRT_ROC(pf_id), pf->stat_prev_loaded,
  2498. &prev_ps->rx_oversize, &cur_ps->rx_oversize);
  2499. ice_stat_update32(hw, GLPRT_RJC(pf_id), pf->stat_prev_loaded,
  2500. &prev_ps->rx_jabber, &cur_ps->rx_jabber);
  2501. pf->stat_prev_loaded = true;
  2502. }
  2503. /**
  2504. * ice_get_stats64 - get statistics for network device structure
  2505. * @netdev: network interface device structure
  2506. * @stats: main device statistics structure
  2507. */
  2508. static
  2509. void ice_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
  2510. {
  2511. struct ice_netdev_priv *np = netdev_priv(netdev);
  2512. struct rtnl_link_stats64 *vsi_stats;
  2513. struct ice_vsi *vsi = np->vsi;
  2514. vsi_stats = &vsi->net_stats;
  2515. if (test_bit(__ICE_DOWN, vsi->state) || !vsi->num_txq || !vsi->num_rxq)
  2516. return;
  2517. /* netdev packet/byte stats come from ring counter. These are obtained
  2518. * by summing up ring counters (done by ice_update_vsi_ring_stats).
  2519. */
  2520. ice_update_vsi_ring_stats(vsi);
  2521. stats->tx_packets = vsi_stats->tx_packets;
  2522. stats->tx_bytes = vsi_stats->tx_bytes;
  2523. stats->rx_packets = vsi_stats->rx_packets;
  2524. stats->rx_bytes = vsi_stats->rx_bytes;
  2525. /* The rest of the stats can be read from the hardware but instead we
  2526. * just return values that the watchdog task has already obtained from
  2527. * the hardware.
  2528. */
  2529. stats->multicast = vsi_stats->multicast;
  2530. stats->tx_errors = vsi_stats->tx_errors;
  2531. stats->tx_dropped = vsi_stats->tx_dropped;
  2532. stats->rx_errors = vsi_stats->rx_errors;
  2533. stats->rx_dropped = vsi_stats->rx_dropped;
  2534. stats->rx_crc_errors = vsi_stats->rx_crc_errors;
  2535. stats->rx_length_errors = vsi_stats->rx_length_errors;
  2536. }
  2537. /**
  2538. * ice_napi_disable_all - Disable NAPI for all q_vectors in the VSI
  2539. * @vsi: VSI having NAPI disabled
  2540. */
  2541. static void ice_napi_disable_all(struct ice_vsi *vsi)
  2542. {
  2543. int q_idx;
  2544. if (!vsi->netdev)
  2545. return;
  2546. for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
  2547. napi_disable(&vsi->q_vectors[q_idx]->napi);
  2548. }
  2549. /**
  2550. * ice_down - Shutdown the connection
  2551. * @vsi: The VSI being stopped
  2552. */
  2553. int ice_down(struct ice_vsi *vsi)
  2554. {
  2555. int i, tx_err, rx_err;
  2556. /* Caller of this function is expected to set the
  2557. * vsi->state __ICE_DOWN bit
  2558. */
  2559. if (vsi->netdev) {
  2560. netif_carrier_off(vsi->netdev);
  2561. netif_tx_disable(vsi->netdev);
  2562. }
  2563. ice_vsi_dis_irq(vsi);
  2564. tx_err = ice_vsi_stop_tx_rings(vsi, ICE_NO_RESET, 0);
  2565. if (tx_err)
  2566. netdev_err(vsi->netdev,
  2567. "Failed stop Tx rings, VSI %d error %d\n",
  2568. vsi->vsi_num, tx_err);
  2569. rx_err = ice_vsi_stop_rx_rings(vsi);
  2570. if (rx_err)
  2571. netdev_err(vsi->netdev,
  2572. "Failed stop Rx rings, VSI %d error %d\n",
  2573. vsi->vsi_num, rx_err);
  2574. ice_napi_disable_all(vsi);
  2575. ice_for_each_txq(vsi, i)
  2576. ice_clean_tx_ring(vsi->tx_rings[i]);
  2577. ice_for_each_rxq(vsi, i)
  2578. ice_clean_rx_ring(vsi->rx_rings[i]);
  2579. if (tx_err || rx_err) {
  2580. netdev_err(vsi->netdev,
  2581. "Failed to close VSI 0x%04X on switch 0x%04X\n",
  2582. vsi->vsi_num, vsi->vsw->sw_id);
  2583. return -EIO;
  2584. }
  2585. return 0;
  2586. }
  2587. /**
  2588. * ice_vsi_setup_tx_rings - Allocate VSI Tx queue resources
  2589. * @vsi: VSI having resources allocated
  2590. *
  2591. * Return 0 on success, negative on failure
  2592. */
  2593. static int ice_vsi_setup_tx_rings(struct ice_vsi *vsi)
  2594. {
  2595. int i, err = 0;
  2596. if (!vsi->num_txq) {
  2597. dev_err(&vsi->back->pdev->dev, "VSI %d has 0 Tx queues\n",
  2598. vsi->vsi_num);
  2599. return -EINVAL;
  2600. }
  2601. ice_for_each_txq(vsi, i) {
  2602. vsi->tx_rings[i]->netdev = vsi->netdev;
  2603. err = ice_setup_tx_ring(vsi->tx_rings[i]);
  2604. if (err)
  2605. break;
  2606. }
  2607. return err;
  2608. }
  2609. /**
  2610. * ice_vsi_setup_rx_rings - Allocate VSI Rx queue resources
  2611. * @vsi: VSI having resources allocated
  2612. *
  2613. * Return 0 on success, negative on failure
  2614. */
  2615. static int ice_vsi_setup_rx_rings(struct ice_vsi *vsi)
  2616. {
  2617. int i, err = 0;
  2618. if (!vsi->num_rxq) {
  2619. dev_err(&vsi->back->pdev->dev, "VSI %d has 0 Rx queues\n",
  2620. vsi->vsi_num);
  2621. return -EINVAL;
  2622. }
  2623. ice_for_each_rxq(vsi, i) {
  2624. vsi->rx_rings[i]->netdev = vsi->netdev;
  2625. err = ice_setup_rx_ring(vsi->rx_rings[i]);
  2626. if (err)
  2627. break;
  2628. }
  2629. return err;
  2630. }
  2631. /**
  2632. * ice_vsi_req_irq - Request IRQ from the OS
  2633. * @vsi: The VSI IRQ is being requested for
  2634. * @basename: name for the vector
  2635. *
  2636. * Return 0 on success and a negative value on error
  2637. */
  2638. static int ice_vsi_req_irq(struct ice_vsi *vsi, char *basename)
  2639. {
  2640. struct ice_pf *pf = vsi->back;
  2641. int err = -EINVAL;
  2642. if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
  2643. err = ice_vsi_req_irq_msix(vsi, basename);
  2644. return err;
  2645. }
  2646. /**
  2647. * ice_vsi_open - Called when a network interface is made active
  2648. * @vsi: the VSI to open
  2649. *
  2650. * Initialization of the VSI
  2651. *
  2652. * Returns 0 on success, negative value on error
  2653. */
  2654. static int ice_vsi_open(struct ice_vsi *vsi)
  2655. {
  2656. char int_name[ICE_INT_NAME_STR_LEN];
  2657. struct ice_pf *pf = vsi->back;
  2658. int err;
  2659. /* allocate descriptors */
  2660. err = ice_vsi_setup_tx_rings(vsi);
  2661. if (err)
  2662. goto err_setup_tx;
  2663. err = ice_vsi_setup_rx_rings(vsi);
  2664. if (err)
  2665. goto err_setup_rx;
  2666. err = ice_vsi_cfg(vsi);
  2667. if (err)
  2668. goto err_setup_rx;
  2669. snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
  2670. dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
  2671. err = ice_vsi_req_irq(vsi, int_name);
  2672. if (err)
  2673. goto err_setup_rx;
  2674. /* Notify the stack of the actual queue counts. */
  2675. err = netif_set_real_num_tx_queues(vsi->netdev, vsi->num_txq);
  2676. if (err)
  2677. goto err_set_qs;
  2678. err = netif_set_real_num_rx_queues(vsi->netdev, vsi->num_rxq);
  2679. if (err)
  2680. goto err_set_qs;
  2681. err = ice_up_complete(vsi);
  2682. if (err)
  2683. goto err_up_complete;
  2684. return 0;
  2685. err_up_complete:
  2686. ice_down(vsi);
  2687. err_set_qs:
  2688. ice_vsi_free_irq(vsi);
  2689. err_setup_rx:
  2690. ice_vsi_free_rx_rings(vsi);
  2691. err_setup_tx:
  2692. ice_vsi_free_tx_rings(vsi);
  2693. return err;
  2694. }
  2695. /**
  2696. * ice_vsi_release_all - Delete all VSIs
  2697. * @pf: PF from which all VSIs are being removed
  2698. */
  2699. static void ice_vsi_release_all(struct ice_pf *pf)
  2700. {
  2701. int err, i;
  2702. if (!pf->vsi)
  2703. return;
  2704. for (i = 0; i < pf->num_alloc_vsi; i++) {
  2705. if (!pf->vsi[i])
  2706. continue;
  2707. err = ice_vsi_release(pf->vsi[i]);
  2708. if (err)
  2709. dev_dbg(&pf->pdev->dev,
  2710. "Failed to release pf->vsi[%d], err %d, vsi_num = %d\n",
  2711. i, err, pf->vsi[i]->vsi_num);
  2712. }
  2713. }
  2714. /**
  2715. * ice_dis_vsi - pause a VSI
  2716. * @vsi: the VSI being paused
  2717. */
  2718. static void ice_dis_vsi(struct ice_vsi *vsi)
  2719. {
  2720. if (test_bit(__ICE_DOWN, vsi->state))
  2721. return;
  2722. set_bit(__ICE_NEEDS_RESTART, vsi->state);
  2723. if (vsi->type == ICE_VSI_PF && vsi->netdev) {
  2724. if (netif_running(vsi->netdev)) {
  2725. rtnl_lock();
  2726. vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
  2727. rtnl_unlock();
  2728. } else {
  2729. ice_vsi_close(vsi);
  2730. }
  2731. }
  2732. }
  2733. /**
  2734. * ice_ena_vsi - resume a VSI
  2735. * @vsi: the VSI being resume
  2736. */
  2737. static int ice_ena_vsi(struct ice_vsi *vsi)
  2738. {
  2739. int err = 0;
  2740. if (test_and_clear_bit(__ICE_NEEDS_RESTART, vsi->state) &&
  2741. vsi->netdev) {
  2742. if (netif_running(vsi->netdev)) {
  2743. rtnl_lock();
  2744. err = vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
  2745. rtnl_unlock();
  2746. } else {
  2747. err = ice_vsi_open(vsi);
  2748. }
  2749. }
  2750. return err;
  2751. }
  2752. /**
  2753. * ice_pf_dis_all_vsi - Pause all VSIs on a PF
  2754. * @pf: the PF
  2755. */
  2756. static void ice_pf_dis_all_vsi(struct ice_pf *pf)
  2757. {
  2758. int v;
  2759. ice_for_each_vsi(pf, v)
  2760. if (pf->vsi[v])
  2761. ice_dis_vsi(pf->vsi[v]);
  2762. }
  2763. /**
  2764. * ice_pf_ena_all_vsi - Resume all VSIs on a PF
  2765. * @pf: the PF
  2766. */
  2767. static int ice_pf_ena_all_vsi(struct ice_pf *pf)
  2768. {
  2769. int v;
  2770. ice_for_each_vsi(pf, v)
  2771. if (pf->vsi[v])
  2772. if (ice_ena_vsi(pf->vsi[v]))
  2773. return -EIO;
  2774. return 0;
  2775. }
  2776. /**
  2777. * ice_vsi_rebuild_all - rebuild all VSIs in pf
  2778. * @pf: the PF
  2779. */
  2780. static int ice_vsi_rebuild_all(struct ice_pf *pf)
  2781. {
  2782. int i;
  2783. /* loop through pf->vsi array and reinit the VSI if found */
  2784. for (i = 0; i < pf->num_alloc_vsi; i++) {
  2785. int err;
  2786. if (!pf->vsi[i])
  2787. continue;
  2788. /* VF VSI rebuild isn't supported yet */
  2789. if (pf->vsi[i]->type == ICE_VSI_VF)
  2790. continue;
  2791. err = ice_vsi_rebuild(pf->vsi[i]);
  2792. if (err) {
  2793. dev_err(&pf->pdev->dev,
  2794. "VSI at index %d rebuild failed\n",
  2795. pf->vsi[i]->idx);
  2796. return err;
  2797. }
  2798. dev_info(&pf->pdev->dev,
  2799. "VSI at index %d rebuilt. vsi_num = 0x%x\n",
  2800. pf->vsi[i]->idx, pf->vsi[i]->vsi_num);
  2801. }
  2802. return 0;
  2803. }
  2804. /**
  2805. * ice_vsi_replay_all - replay all VSIs configuration in the PF
  2806. * @pf: the PF
  2807. */
  2808. static int ice_vsi_replay_all(struct ice_pf *pf)
  2809. {
  2810. struct ice_hw *hw = &pf->hw;
  2811. enum ice_status ret;
  2812. int i;
  2813. /* loop through pf->vsi array and replay the VSI if found */
  2814. for (i = 0; i < pf->num_alloc_vsi; i++) {
  2815. if (!pf->vsi[i])
  2816. continue;
  2817. ret = ice_replay_vsi(hw, pf->vsi[i]->idx);
  2818. if (ret) {
  2819. dev_err(&pf->pdev->dev,
  2820. "VSI at index %d replay failed %d\n",
  2821. pf->vsi[i]->idx, ret);
  2822. return -EIO;
  2823. }
  2824. /* Re-map HW VSI number, using VSI handle that has been
  2825. * previously validated in ice_replay_vsi() call above
  2826. */
  2827. pf->vsi[i]->vsi_num = ice_get_hw_vsi_num(hw, pf->vsi[i]->idx);
  2828. dev_info(&pf->pdev->dev,
  2829. "VSI at index %d filter replayed successfully - vsi_num %i\n",
  2830. pf->vsi[i]->idx, pf->vsi[i]->vsi_num);
  2831. }
  2832. /* Clean up replay filter after successful re-configuration */
  2833. ice_replay_post(hw);
  2834. return 0;
  2835. }
  2836. /**
  2837. * ice_rebuild - rebuild after reset
  2838. * @pf: pf to rebuild
  2839. */
  2840. static void ice_rebuild(struct ice_pf *pf)
  2841. {
  2842. struct device *dev = &pf->pdev->dev;
  2843. struct ice_hw *hw = &pf->hw;
  2844. enum ice_status ret;
  2845. int err;
  2846. if (test_bit(__ICE_DOWN, pf->state))
  2847. goto clear_recovery;
  2848. dev_dbg(dev, "rebuilding pf\n");
  2849. ret = ice_init_all_ctrlq(hw);
  2850. if (ret) {
  2851. dev_err(dev, "control queues init failed %d\n", ret);
  2852. goto err_init_ctrlq;
  2853. }
  2854. ret = ice_clear_pf_cfg(hw);
  2855. if (ret) {
  2856. dev_err(dev, "clear PF configuration failed %d\n", ret);
  2857. goto err_init_ctrlq;
  2858. }
  2859. ice_clear_pxe_mode(hw);
  2860. ret = ice_get_caps(hw);
  2861. if (ret) {
  2862. dev_err(dev, "ice_get_caps failed %d\n", ret);
  2863. goto err_init_ctrlq;
  2864. }
  2865. err = ice_sched_init_port(hw->port_info);
  2866. if (err)
  2867. goto err_sched_init_port;
  2868. /* reset search_hint of irq_trackers to 0 since interrupts are
  2869. * reclaimed and could be allocated from beginning during VSI rebuild
  2870. */
  2871. pf->sw_irq_tracker->search_hint = 0;
  2872. pf->hw_irq_tracker->search_hint = 0;
  2873. err = ice_vsi_rebuild_all(pf);
  2874. if (err) {
  2875. dev_err(dev, "ice_vsi_rebuild_all failed\n");
  2876. goto err_vsi_rebuild;
  2877. }
  2878. err = ice_update_link_info(hw->port_info);
  2879. if (err)
  2880. dev_err(&pf->pdev->dev, "Get link status error %d\n", err);
  2881. /* Replay all VSIs Configuration, including filters after reset */
  2882. if (ice_vsi_replay_all(pf)) {
  2883. dev_err(&pf->pdev->dev,
  2884. "error replaying VSI configurations with switch filter rules\n");
  2885. goto err_vsi_rebuild;
  2886. }
  2887. /* start misc vector */
  2888. if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
  2889. err = ice_req_irq_msix_misc(pf);
  2890. if (err) {
  2891. dev_err(dev, "misc vector setup failed: %d\n", err);
  2892. goto err_vsi_rebuild;
  2893. }
  2894. }
  2895. /* restart the VSIs that were rebuilt and running before the reset */
  2896. err = ice_pf_ena_all_vsi(pf);
  2897. if (err) {
  2898. dev_err(&pf->pdev->dev, "error enabling VSIs\n");
  2899. /* no need to disable VSIs in tear down path in ice_rebuild()
  2900. * since its already taken care in ice_vsi_open()
  2901. */
  2902. goto err_vsi_rebuild;
  2903. }
  2904. ice_reset_all_vfs(pf, true);
  2905. /* if we get here, reset flow is successful */
  2906. clear_bit(__ICE_RESET_FAILED, pf->state);
  2907. return;
  2908. err_vsi_rebuild:
  2909. ice_vsi_release_all(pf);
  2910. err_sched_init_port:
  2911. ice_sched_cleanup_all(hw);
  2912. err_init_ctrlq:
  2913. ice_shutdown_all_ctrlq(hw);
  2914. set_bit(__ICE_RESET_FAILED, pf->state);
  2915. clear_recovery:
  2916. /* set this bit in PF state to control service task scheduling */
  2917. set_bit(__ICE_NEEDS_RESTART, pf->state);
  2918. dev_err(dev, "Rebuild failed, unload and reload driver\n");
  2919. }
  2920. /**
  2921. * ice_change_mtu - NDO callback to change the MTU
  2922. * @netdev: network interface device structure
  2923. * @new_mtu: new value for maximum frame size
  2924. *
  2925. * Returns 0 on success, negative on failure
  2926. */
  2927. static int ice_change_mtu(struct net_device *netdev, int new_mtu)
  2928. {
  2929. struct ice_netdev_priv *np = netdev_priv(netdev);
  2930. struct ice_vsi *vsi = np->vsi;
  2931. struct ice_pf *pf = vsi->back;
  2932. u8 count = 0;
  2933. if (new_mtu == netdev->mtu) {
  2934. netdev_warn(netdev, "mtu is already %u\n", netdev->mtu);
  2935. return 0;
  2936. }
  2937. if (new_mtu < netdev->min_mtu) {
  2938. netdev_err(netdev, "new mtu invalid. min_mtu is %d\n",
  2939. netdev->min_mtu);
  2940. return -EINVAL;
  2941. } else if (new_mtu > netdev->max_mtu) {
  2942. netdev_err(netdev, "new mtu invalid. max_mtu is %d\n",
  2943. netdev->min_mtu);
  2944. return -EINVAL;
  2945. }
  2946. /* if a reset is in progress, wait for some time for it to complete */
  2947. do {
  2948. if (ice_is_reset_in_progress(pf->state)) {
  2949. count++;
  2950. usleep_range(1000, 2000);
  2951. } else {
  2952. break;
  2953. }
  2954. } while (count < 100);
  2955. if (count == 100) {
  2956. netdev_err(netdev, "can't change mtu. Device is busy\n");
  2957. return -EBUSY;
  2958. }
  2959. netdev->mtu = new_mtu;
  2960. /* if VSI is up, bring it down and then back up */
  2961. if (!test_and_set_bit(__ICE_DOWN, vsi->state)) {
  2962. int err;
  2963. err = ice_down(vsi);
  2964. if (err) {
  2965. netdev_err(netdev, "change mtu if_up err %d\n", err);
  2966. return err;
  2967. }
  2968. err = ice_up(vsi);
  2969. if (err) {
  2970. netdev_err(netdev, "change mtu if_up err %d\n", err);
  2971. return err;
  2972. }
  2973. }
  2974. netdev_dbg(netdev, "changed mtu to %d\n", new_mtu);
  2975. return 0;
  2976. }
  2977. /**
  2978. * ice_set_rss - Set RSS keys and lut
  2979. * @vsi: Pointer to VSI structure
  2980. * @seed: RSS hash seed
  2981. * @lut: Lookup table
  2982. * @lut_size: Lookup table size
  2983. *
  2984. * Returns 0 on success, negative on failure
  2985. */
  2986. int ice_set_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
  2987. {
  2988. struct ice_pf *pf = vsi->back;
  2989. struct ice_hw *hw = &pf->hw;
  2990. enum ice_status status;
  2991. if (seed) {
  2992. struct ice_aqc_get_set_rss_keys *buf =
  2993. (struct ice_aqc_get_set_rss_keys *)seed;
  2994. status = ice_aq_set_rss_key(hw, vsi->idx, buf);
  2995. if (status) {
  2996. dev_err(&pf->pdev->dev,
  2997. "Cannot set RSS key, err %d aq_err %d\n",
  2998. status, hw->adminq.rq_last_status);
  2999. return -EIO;
  3000. }
  3001. }
  3002. if (lut) {
  3003. status = ice_aq_set_rss_lut(hw, vsi->idx, vsi->rss_lut_type,
  3004. lut, lut_size);
  3005. if (status) {
  3006. dev_err(&pf->pdev->dev,
  3007. "Cannot set RSS lut, err %d aq_err %d\n",
  3008. status, hw->adminq.rq_last_status);
  3009. return -EIO;
  3010. }
  3011. }
  3012. return 0;
  3013. }
  3014. /**
  3015. * ice_get_rss - Get RSS keys and lut
  3016. * @vsi: Pointer to VSI structure
  3017. * @seed: Buffer to store the keys
  3018. * @lut: Buffer to store the lookup table entries
  3019. * @lut_size: Size of buffer to store the lookup table entries
  3020. *
  3021. * Returns 0 on success, negative on failure
  3022. */
  3023. int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
  3024. {
  3025. struct ice_pf *pf = vsi->back;
  3026. struct ice_hw *hw = &pf->hw;
  3027. enum ice_status status;
  3028. if (seed) {
  3029. struct ice_aqc_get_set_rss_keys *buf =
  3030. (struct ice_aqc_get_set_rss_keys *)seed;
  3031. status = ice_aq_get_rss_key(hw, vsi->idx, buf);
  3032. if (status) {
  3033. dev_err(&pf->pdev->dev,
  3034. "Cannot get RSS key, err %d aq_err %d\n",
  3035. status, hw->adminq.rq_last_status);
  3036. return -EIO;
  3037. }
  3038. }
  3039. if (lut) {
  3040. status = ice_aq_get_rss_lut(hw, vsi->idx, vsi->rss_lut_type,
  3041. lut, lut_size);
  3042. if (status) {
  3043. dev_err(&pf->pdev->dev,
  3044. "Cannot get RSS lut, err %d aq_err %d\n",
  3045. status, hw->adminq.rq_last_status);
  3046. return -EIO;
  3047. }
  3048. }
  3049. return 0;
  3050. }
  3051. /**
  3052. * ice_bridge_getlink - Get the hardware bridge mode
  3053. * @skb: skb buff
  3054. * @pid: process id
  3055. * @seq: RTNL message seq
  3056. * @dev: the netdev being configured
  3057. * @filter_mask: filter mask passed in
  3058. * @nlflags: netlink flags passed in
  3059. *
  3060. * Return the bridge mode (VEB/VEPA)
  3061. */
  3062. static int
  3063. ice_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
  3064. struct net_device *dev, u32 filter_mask, int nlflags)
  3065. {
  3066. struct ice_netdev_priv *np = netdev_priv(dev);
  3067. struct ice_vsi *vsi = np->vsi;
  3068. struct ice_pf *pf = vsi->back;
  3069. u16 bmode;
  3070. bmode = pf->first_sw->bridge_mode;
  3071. return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bmode, 0, 0, nlflags,
  3072. filter_mask, NULL);
  3073. }
  3074. /**
  3075. * ice_vsi_update_bridge_mode - Update VSI for switching bridge mode (VEB/VEPA)
  3076. * @vsi: Pointer to VSI structure
  3077. * @bmode: Hardware bridge mode (VEB/VEPA)
  3078. *
  3079. * Returns 0 on success, negative on failure
  3080. */
  3081. static int ice_vsi_update_bridge_mode(struct ice_vsi *vsi, u16 bmode)
  3082. {
  3083. struct device *dev = &vsi->back->pdev->dev;
  3084. struct ice_aqc_vsi_props *vsi_props;
  3085. struct ice_hw *hw = &vsi->back->hw;
  3086. struct ice_vsi_ctx ctxt = { 0 };
  3087. enum ice_status status;
  3088. vsi_props = &vsi->info;
  3089. ctxt.info = vsi->info;
  3090. if (bmode == BRIDGE_MODE_VEB)
  3091. /* change from VEPA to VEB mode */
  3092. ctxt.info.sw_flags |= ICE_AQ_VSI_SW_FLAG_ALLOW_LB;
  3093. else
  3094. /* change from VEB to VEPA mode */
  3095. ctxt.info.sw_flags &= ~ICE_AQ_VSI_SW_FLAG_ALLOW_LB;
  3096. ctxt.info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_SW_VALID);
  3097. status = ice_update_vsi(hw, vsi->idx, &ctxt, NULL);
  3098. if (status) {
  3099. dev_err(dev, "update VSI for bridge mode failed, bmode = %d err %d aq_err %d\n",
  3100. bmode, status, hw->adminq.sq_last_status);
  3101. return -EIO;
  3102. }
  3103. /* Update sw flags for book keeping */
  3104. vsi_props->sw_flags = ctxt.info.sw_flags;
  3105. return 0;
  3106. }
  3107. /**
  3108. * ice_bridge_setlink - Set the hardware bridge mode
  3109. * @dev: the netdev being configured
  3110. * @nlh: RTNL message
  3111. * @flags: bridge setlink flags
  3112. *
  3113. * Sets the bridge mode (VEB/VEPA) of the switch to which the netdev (VSI) is
  3114. * hooked up to. Iterates through the PF VSI list and sets the loopback mode (if
  3115. * not already set for all VSIs connected to this switch. And also update the
  3116. * unicast switch filter rules for the corresponding switch of the netdev.
  3117. */
  3118. static int
  3119. ice_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
  3120. u16 __always_unused flags)
  3121. {
  3122. struct ice_netdev_priv *np = netdev_priv(dev);
  3123. struct ice_pf *pf = np->vsi->back;
  3124. struct nlattr *attr, *br_spec;
  3125. struct ice_hw *hw = &pf->hw;
  3126. enum ice_status status;
  3127. struct ice_sw *pf_sw;
  3128. int rem, v, err = 0;
  3129. pf_sw = pf->first_sw;
  3130. /* find the attribute in the netlink message */
  3131. br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
  3132. nla_for_each_nested(attr, br_spec, rem) {
  3133. __u16 mode;
  3134. if (nla_type(attr) != IFLA_BRIDGE_MODE)
  3135. continue;
  3136. mode = nla_get_u16(attr);
  3137. if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
  3138. return -EINVAL;
  3139. /* Continue if bridge mode is not being flipped */
  3140. if (mode == pf_sw->bridge_mode)
  3141. continue;
  3142. /* Iterates through the PF VSI list and update the loopback
  3143. * mode of the VSI
  3144. */
  3145. ice_for_each_vsi(pf, v) {
  3146. if (!pf->vsi[v])
  3147. continue;
  3148. err = ice_vsi_update_bridge_mode(pf->vsi[v], mode);
  3149. if (err)
  3150. return err;
  3151. }
  3152. hw->evb_veb = (mode == BRIDGE_MODE_VEB);
  3153. /* Update the unicast switch filter rules for the corresponding
  3154. * switch of the netdev
  3155. */
  3156. status = ice_update_sw_rule_bridge_mode(hw);
  3157. if (status) {
  3158. netdev_err(dev, "update SW_RULE for bridge mode failed, = %d err %d aq_err %d\n",
  3159. mode, status, hw->adminq.sq_last_status);
  3160. /* revert hw->evb_veb */
  3161. hw->evb_veb = (pf_sw->bridge_mode == BRIDGE_MODE_VEB);
  3162. return -EIO;
  3163. }
  3164. pf_sw->bridge_mode = mode;
  3165. }
  3166. return 0;
  3167. }
  3168. /**
  3169. * ice_tx_timeout - Respond to a Tx Hang
  3170. * @netdev: network interface device structure
  3171. */
  3172. static void ice_tx_timeout(struct net_device *netdev)
  3173. {
  3174. struct ice_netdev_priv *np = netdev_priv(netdev);
  3175. struct ice_ring *tx_ring = NULL;
  3176. struct ice_vsi *vsi = np->vsi;
  3177. struct ice_pf *pf = vsi->back;
  3178. u32 head, val = 0, i;
  3179. int hung_queue = -1;
  3180. pf->tx_timeout_count++;
  3181. /* find the stopped queue the same way the stack does */
  3182. for (i = 0; i < netdev->num_tx_queues; i++) {
  3183. struct netdev_queue *q;
  3184. unsigned long trans_start;
  3185. q = netdev_get_tx_queue(netdev, i);
  3186. trans_start = q->trans_start;
  3187. if (netif_xmit_stopped(q) &&
  3188. time_after(jiffies,
  3189. (trans_start + netdev->watchdog_timeo))) {
  3190. hung_queue = i;
  3191. break;
  3192. }
  3193. }
  3194. if (i == netdev->num_tx_queues) {
  3195. netdev_info(netdev, "tx_timeout: no netdev hung queue found\n");
  3196. } else {
  3197. /* now that we have an index, find the tx_ring struct */
  3198. for (i = 0; i < vsi->num_txq; i++) {
  3199. if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
  3200. if (hung_queue ==
  3201. vsi->tx_rings[i]->q_index) {
  3202. tx_ring = vsi->tx_rings[i];
  3203. break;
  3204. }
  3205. }
  3206. }
  3207. }
  3208. /* Reset recovery level if enough time has elapsed after last timeout.
  3209. * Also ensure no new reset action happens before next timeout period.
  3210. */
  3211. if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ * 20)))
  3212. pf->tx_timeout_recovery_level = 1;
  3213. else if (time_before(jiffies, (pf->tx_timeout_last_recovery +
  3214. netdev->watchdog_timeo)))
  3215. return;
  3216. if (tx_ring) {
  3217. head = tx_ring->next_to_clean;
  3218. /* Read interrupt register */
  3219. if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
  3220. val = rd32(&pf->hw,
  3221. GLINT_DYN_CTL(tx_ring->q_vector->v_idx +
  3222. tx_ring->vsi->hw_base_vector));
  3223. netdev_info(netdev, "tx_timeout: VSI_num: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
  3224. vsi->vsi_num, hung_queue, tx_ring->next_to_clean,
  3225. head, tx_ring->next_to_use,
  3226. readl(tx_ring->tail), val);
  3227. }
  3228. pf->tx_timeout_last_recovery = jiffies;
  3229. netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n",
  3230. pf->tx_timeout_recovery_level, hung_queue);
  3231. switch (pf->tx_timeout_recovery_level) {
  3232. case 1:
  3233. set_bit(__ICE_PFR_REQ, pf->state);
  3234. break;
  3235. case 2:
  3236. set_bit(__ICE_CORER_REQ, pf->state);
  3237. break;
  3238. case 3:
  3239. set_bit(__ICE_GLOBR_REQ, pf->state);
  3240. break;
  3241. default:
  3242. netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in unrecoverable state.\n");
  3243. set_bit(__ICE_DOWN, pf->state);
  3244. set_bit(__ICE_NEEDS_RESTART, vsi->state);
  3245. set_bit(__ICE_SERVICE_DIS, pf->state);
  3246. break;
  3247. }
  3248. ice_service_task_schedule(pf);
  3249. pf->tx_timeout_recovery_level++;
  3250. }
  3251. /**
  3252. * ice_open - Called when a network interface becomes active
  3253. * @netdev: network interface device structure
  3254. *
  3255. * The open entry point is called when a network interface is made
  3256. * active by the system (IFF_UP). At this point all resources needed
  3257. * for transmit and receive operations are allocated, the interrupt
  3258. * handler is registered with the OS, the netdev watchdog is enabled,
  3259. * and the stack is notified that the interface is ready.
  3260. *
  3261. * Returns 0 on success, negative value on failure
  3262. */
  3263. static int ice_open(struct net_device *netdev)
  3264. {
  3265. struct ice_netdev_priv *np = netdev_priv(netdev);
  3266. struct ice_vsi *vsi = np->vsi;
  3267. int err;
  3268. if (test_bit(__ICE_NEEDS_RESTART, vsi->back->state)) {
  3269. netdev_err(netdev, "driver needs to be unloaded and reloaded\n");
  3270. return -EIO;
  3271. }
  3272. netif_carrier_off(netdev);
  3273. err = ice_vsi_open(vsi);
  3274. if (err)
  3275. netdev_err(netdev, "Failed to open VSI 0x%04X on switch 0x%04X\n",
  3276. vsi->vsi_num, vsi->vsw->sw_id);
  3277. return err;
  3278. }
  3279. /**
  3280. * ice_stop - Disables a network interface
  3281. * @netdev: network interface device structure
  3282. *
  3283. * The stop entry point is called when an interface is de-activated by the OS,
  3284. * and the netdevice enters the DOWN state. The hardware is still under the
  3285. * driver's control, but the netdev interface is disabled.
  3286. *
  3287. * Returns success only - not allowed to fail
  3288. */
  3289. static int ice_stop(struct net_device *netdev)
  3290. {
  3291. struct ice_netdev_priv *np = netdev_priv(netdev);
  3292. struct ice_vsi *vsi = np->vsi;
  3293. ice_vsi_close(vsi);
  3294. return 0;
  3295. }
  3296. /**
  3297. * ice_features_check - Validate encapsulated packet conforms to limits
  3298. * @skb: skb buffer
  3299. * @netdev: This port's netdev
  3300. * @features: Offload features that the stack believes apply
  3301. */
  3302. static netdev_features_t
  3303. ice_features_check(struct sk_buff *skb,
  3304. struct net_device __always_unused *netdev,
  3305. netdev_features_t features)
  3306. {
  3307. size_t len;
  3308. /* No point in doing any of this if neither checksum nor GSO are
  3309. * being requested for this frame. We can rule out both by just
  3310. * checking for CHECKSUM_PARTIAL
  3311. */
  3312. if (skb->ip_summed != CHECKSUM_PARTIAL)
  3313. return features;
  3314. /* We cannot support GSO if the MSS is going to be less than
  3315. * 64 bytes. If it is then we need to drop support for GSO.
  3316. */
  3317. if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
  3318. features &= ~NETIF_F_GSO_MASK;
  3319. len = skb_network_header(skb) - skb->data;
  3320. if (len & ~(ICE_TXD_MACLEN_MAX))
  3321. goto out_rm_features;
  3322. len = skb_transport_header(skb) - skb_network_header(skb);
  3323. if (len & ~(ICE_TXD_IPLEN_MAX))
  3324. goto out_rm_features;
  3325. if (skb->encapsulation) {
  3326. len = skb_inner_network_header(skb) - skb_transport_header(skb);
  3327. if (len & ~(ICE_TXD_L4LEN_MAX))
  3328. goto out_rm_features;
  3329. len = skb_inner_transport_header(skb) -
  3330. skb_inner_network_header(skb);
  3331. if (len & ~(ICE_TXD_IPLEN_MAX))
  3332. goto out_rm_features;
  3333. }
  3334. return features;
  3335. out_rm_features:
  3336. return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
  3337. }
  3338. static const struct net_device_ops ice_netdev_ops = {
  3339. .ndo_open = ice_open,
  3340. .ndo_stop = ice_stop,
  3341. .ndo_start_xmit = ice_start_xmit,
  3342. .ndo_features_check = ice_features_check,
  3343. .ndo_set_rx_mode = ice_set_rx_mode,
  3344. .ndo_set_mac_address = ice_set_mac_address,
  3345. .ndo_validate_addr = eth_validate_addr,
  3346. .ndo_change_mtu = ice_change_mtu,
  3347. .ndo_get_stats64 = ice_get_stats64,
  3348. .ndo_set_vf_spoofchk = ice_set_vf_spoofchk,
  3349. .ndo_set_vf_mac = ice_set_vf_mac,
  3350. .ndo_get_vf_config = ice_get_vf_cfg,
  3351. .ndo_set_vf_trust = ice_set_vf_trust,
  3352. .ndo_set_vf_vlan = ice_set_vf_port_vlan,
  3353. .ndo_set_vf_link_state = ice_set_vf_link_state,
  3354. .ndo_vlan_rx_add_vid = ice_vlan_rx_add_vid,
  3355. .ndo_vlan_rx_kill_vid = ice_vlan_rx_kill_vid,
  3356. .ndo_set_features = ice_set_features,
  3357. .ndo_bridge_getlink = ice_bridge_getlink,
  3358. .ndo_bridge_setlink = ice_bridge_setlink,
  3359. .ndo_fdb_add = ice_fdb_add,
  3360. .ndo_fdb_del = ice_fdb_del,
  3361. .ndo_tx_timeout = ice_tx_timeout,
  3362. };