i40e_main.c 254 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281828282838284828582868287828882898290829182928293829482958296829782988299830083018302830383048305830683078308830983108311831283138314831583168317831883198320832183228323832483258326832783288329833083318332833383348335833683378338833983408341834283438344834583468347834883498350835183528353835483558356835783588359836083618362836383648365836683678368836983708371837283738374837583768377837883798380838183828383838483858386838783888389839083918392839383948395839683978398839984008401840284038404840584068407840884098410841184128413841484158416841784188419842084218422842384248425842684278428842984308431843284338434843584368437843884398440844184428443844484458446844784488449845084518452845384548455845684578458845984608461846284638464846584668467846884698470847184728473847484758476847784788479848084818482848384848485848684878488848984908491849284938494849584968497849884998500850185028503850485058506850785088509851085118512851385148515851685178518851985208521852285238524852585268527852885298530853185328533853485358536853785388539854085418542854385448545854685478548854985508551855285538554855585568557855885598560856185628563856485658566856785688569857085718572857385748575857685778578857985808581858285838584858585868587858885898590859185928593859485958596859785988599860086018602860386048605860686078608860986108611861286138614861586168617861886198620862186228623862486258626862786288629863086318632863386348635863686378638863986408641864286438644864586468647864886498650865186528653865486558656865786588659866086618662866386648665866686678668866986708671867286738674867586768677867886798680868186828683868486858686868786888689869086918692869386948695869686978698869987008701870287038704870587068707870887098710871187128713871487158716871787188719872087218722872387248725872687278728872987308731873287338734873587368737873887398740874187428743874487458746874787488749875087518752875387548755875687578758875987608761876287638764876587668767876887698770877187728773877487758776877787788779878087818782878387848785878687878788878987908791879287938794879587968797879887998800880188028803880488058806880788088809881088118812881388148815881688178818881988208821882288238824882588268827882888298830883188328833883488358836883788388839884088418842884388448845884688478848884988508851885288538854885588568857885888598860886188628863886488658866886788688869887088718872887388748875887688778878887988808881888288838884888588868887888888898890889188928893889488958896889788988899890089018902890389048905890689078908890989108911891289138914891589168917891889198920892189228923892489258926892789288929893089318932893389348935893689378938893989408941894289438944894589468947894889498950895189528953895489558956895789588959896089618962896389648965896689678968896989708971897289738974897589768977897889798980898189828983898489858986898789888989899089918992899389948995899689978998899990009001900290039004900590069007900890099010901190129013901490159016901790189019902090219022902390249025902690279028902990309031903290339034903590369037903890399040904190429043904490459046904790489049905090519052905390549055905690579058905990609061906290639064906590669067906890699070907190729073907490759076907790789079908090819082908390849085908690879088908990909091909290939094909590969097909890999100910191029103910491059106910791089109911091119112911391149115911691179118911991209121912291239124912591269127912891299130913191329133913491359136913791389139914091419142914391449145914691479148914991509151915291539154915591569157915891599160916191629163916491659166916791689169917091719172917391749175917691779178917991809181918291839184918591869187918891899190919191929193919491959196919791989199920092019202920392049205920692079208920992109211921292139214921592169217921892199220922192229223922492259226922792289229923092319232923392349235923692379238923992409241924292439244924592469247924892499250925192529253925492559256925792589259926092619262926392649265926692679268926992709271927292739274927592769277927892799280928192829283928492859286928792889289929092919292929392949295929692979298929993009301930293039304930593069307930893099310931193129313931493159316931793189319932093219322932393249325932693279328932993309331933293339334933593369337933893399340934193429343934493459346934793489349935093519352935393549355935693579358935993609361936293639364936593669367936893699370937193729373937493759376937793789379938093819382938393849385938693879388938993909391939293939394939593969397939893999400940194029403940494059406940794089409941094119412941394149415941694179418941994209421942294239424942594269427942894299430943194329433943494359436943794389439944094419442944394449445944694479448944994509451945294539454945594569457945894599460946194629463946494659466946794689469
  1. /*******************************************************************************
  2. *
  3. * Intel Ethernet Controller XL710 Family Linux Driver
  4. * Copyright(c) 2013 - 2014 Intel Corporation.
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms and conditions of the GNU General Public License,
  8. * version 2, as published by the Free Software Foundation.
  9. *
  10. * This program is distributed in the hope it will be useful, but WITHOUT
  11. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  13. * more details.
  14. *
  15. * You should have received a copy of the GNU General Public License along
  16. * with this program. If not, see <http://www.gnu.org/licenses/>.
  17. *
  18. * The full GNU General Public License is included in this distribution in
  19. * the file called "COPYING".
  20. *
  21. * Contact Information:
  22. * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  23. * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  24. *
  25. ******************************************************************************/
  26. /* Local includes */
  27. #include "i40e.h"
  28. #include "i40e_diag.h"
  29. #ifdef CONFIG_I40E_VXLAN
  30. #include <net/vxlan.h>
  31. #endif
  32. const char i40e_driver_name[] = "i40e";
  33. static const char i40e_driver_string[] =
  34. "Intel(R) Ethernet Connection XL710 Network Driver";
  35. #define DRV_KERN "-k"
  36. #define DRV_VERSION_MAJOR 0
  37. #define DRV_VERSION_MINOR 4
  38. #define DRV_VERSION_BUILD 21
  39. #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
  40. __stringify(DRV_VERSION_MINOR) "." \
  41. __stringify(DRV_VERSION_BUILD) DRV_KERN
  42. const char i40e_driver_version_str[] = DRV_VERSION;
  43. static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation.";
  44. /* a bit of forward declarations */
  45. static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  46. static void i40e_handle_reset_warning(struct i40e_pf *pf);
  47. static int i40e_add_vsi(struct i40e_vsi *vsi);
  48. static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  49. static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
  50. static int i40e_setup_misc_vector(struct i40e_pf *pf);
  51. static void i40e_determine_queue_usage(struct i40e_pf *pf);
  52. static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  53. static void i40e_fdir_sb_setup(struct i40e_pf *pf);
  54. static int i40e_veb_get_bw_info(struct i40e_veb *veb);
  55. /* i40e_pci_tbl - PCI Device ID Table
  56. *
  57. * Last entry must be all 0s
  58. *
  59. * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  60. * Class, Class Mask, private data (not used) }
  61. */
  62. static const struct pci_device_id i40e_pci_tbl[] = {
  63. {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
  64. {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
  65. {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_A), 0},
  66. {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
  67. {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
  68. {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
  69. {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
  70. {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
  71. /* required last entry */
  72. {0, }
  73. };
  74. MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
  75. #define I40E_MAX_VF_COUNT 128
  76. static int debug = -1;
  77. module_param(debug, int, 0);
  78. MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  79. MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
  80. MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
  81. MODULE_LICENSE("GPL");
  82. MODULE_VERSION(DRV_VERSION);
  83. /**
  84. * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
  85. * @hw: pointer to the HW structure
  86. * @mem: ptr to mem struct to fill out
  87. * @size: size of memory requested
  88. * @alignment: what to align the allocation to
  89. **/
  90. int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
  91. u64 size, u32 alignment)
  92. {
  93. struct i40e_pf *pf = (struct i40e_pf *)hw->back;
  94. mem->size = ALIGN(size, alignment);
  95. mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size,
  96. &mem->pa, GFP_KERNEL);
  97. if (!mem->va)
  98. return -ENOMEM;
  99. return 0;
  100. }
  101. /**
  102. * i40e_free_dma_mem_d - OS specific memory free for shared code
  103. * @hw: pointer to the HW structure
  104. * @mem: ptr to mem struct to free
  105. **/
  106. int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
  107. {
  108. struct i40e_pf *pf = (struct i40e_pf *)hw->back;
  109. dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
  110. mem->va = NULL;
  111. mem->pa = 0;
  112. mem->size = 0;
  113. return 0;
  114. }
  115. /**
  116. * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
  117. * @hw: pointer to the HW structure
  118. * @mem: ptr to mem struct to fill out
  119. * @size: size of memory requested
  120. **/
  121. int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
  122. u32 size)
  123. {
  124. mem->size = size;
  125. mem->va = kzalloc(size, GFP_KERNEL);
  126. if (!mem->va)
  127. return -ENOMEM;
  128. return 0;
  129. }
  130. /**
  131. * i40e_free_virt_mem_d - OS specific memory free for shared code
  132. * @hw: pointer to the HW structure
  133. * @mem: ptr to mem struct to free
  134. **/
  135. int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
  136. {
  137. /* it's ok to kfree a NULL pointer */
  138. kfree(mem->va);
  139. mem->va = NULL;
  140. mem->size = 0;
  141. return 0;
  142. }
  143. /**
  144. * i40e_get_lump - find a lump of free generic resource
  145. * @pf: board private structure
  146. * @pile: the pile of resource to search
  147. * @needed: the number of items needed
  148. * @id: an owner id to stick on the items assigned
  149. *
  150. * Returns the base item index of the lump, or negative for error
  151. *
  152. * The search_hint trick and lack of advanced fit-finding only work
  153. * because we're highly likely to have all the same size lump requests.
  154. * Linear search time and any fragmentation should be minimal.
  155. **/
  156. static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
  157. u16 needed, u16 id)
  158. {
  159. int ret = -ENOMEM;
  160. int i, j;
  161. if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
  162. dev_info(&pf->pdev->dev,
  163. "param err: pile=%p needed=%d id=0x%04x\n",
  164. pile, needed, id);
  165. return -EINVAL;
  166. }
  167. /* start the linear search with an imperfect hint */
  168. i = pile->search_hint;
  169. while (i < pile->num_entries) {
  170. /* skip already allocated entries */
  171. if (pile->list[i] & I40E_PILE_VALID_BIT) {
  172. i++;
  173. continue;
  174. }
  175. /* do we have enough in this lump? */
  176. for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
  177. if (pile->list[i+j] & I40E_PILE_VALID_BIT)
  178. break;
  179. }
  180. if (j == needed) {
  181. /* there was enough, so assign it to the requestor */
  182. for (j = 0; j < needed; j++)
  183. pile->list[i+j] = id | I40E_PILE_VALID_BIT;
  184. ret = i;
  185. pile->search_hint = i + j;
  186. break;
  187. } else {
  188. /* not enough, so skip over it and continue looking */
  189. i += j;
  190. }
  191. }
  192. return ret;
  193. }
  194. /**
  195. * i40e_put_lump - return a lump of generic resource
  196. * @pile: the pile of resource to search
  197. * @index: the base item index
  198. * @id: the owner id of the items assigned
  199. *
  200. * Returns the count of items in the lump
  201. **/
  202. static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
  203. {
  204. int valid_id = (id | I40E_PILE_VALID_BIT);
  205. int count = 0;
  206. int i;
  207. if (!pile || index >= pile->num_entries)
  208. return -EINVAL;
  209. for (i = index;
  210. i < pile->num_entries && pile->list[i] == valid_id;
  211. i++) {
  212. pile->list[i] = 0;
  213. count++;
  214. }
  215. if (count && index < pile->search_hint)
  216. pile->search_hint = index;
  217. return count;
  218. }
  219. /**
  220. * i40e_service_event_schedule - Schedule the service task to wake up
  221. * @pf: board private structure
  222. *
  223. * If not already scheduled, this puts the task into the work queue
  224. **/
  225. static void i40e_service_event_schedule(struct i40e_pf *pf)
  226. {
  227. if (!test_bit(__I40E_DOWN, &pf->state) &&
  228. !test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state) &&
  229. !test_and_set_bit(__I40E_SERVICE_SCHED, &pf->state))
  230. schedule_work(&pf->service_task);
  231. }
  232. /**
  233. * i40e_tx_timeout - Respond to a Tx Hang
  234. * @netdev: network interface device structure
  235. *
  236. * If any port has noticed a Tx timeout, it is likely that the whole
  237. * device is munged, not just the one netdev port, so go for the full
  238. * reset.
  239. **/
  240. #ifdef I40E_FCOE
  241. void i40e_tx_timeout(struct net_device *netdev)
  242. #else
  243. static void i40e_tx_timeout(struct net_device *netdev)
  244. #endif
  245. {
  246. struct i40e_netdev_priv *np = netdev_priv(netdev);
  247. struct i40e_vsi *vsi = np->vsi;
  248. struct i40e_pf *pf = vsi->back;
  249. pf->tx_timeout_count++;
  250. if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
  251. pf->tx_timeout_recovery_level = 1;
  252. pf->tx_timeout_last_recovery = jiffies;
  253. netdev_info(netdev, "tx_timeout recovery level %d\n",
  254. pf->tx_timeout_recovery_level);
  255. switch (pf->tx_timeout_recovery_level) {
  256. case 0:
  257. /* disable and re-enable queues for the VSI */
  258. if (in_interrupt()) {
  259. set_bit(__I40E_REINIT_REQUESTED, &pf->state);
  260. set_bit(__I40E_REINIT_REQUESTED, &vsi->state);
  261. } else {
  262. i40e_vsi_reinit_locked(vsi);
  263. }
  264. break;
  265. case 1:
  266. set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
  267. break;
  268. case 2:
  269. set_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
  270. break;
  271. case 3:
  272. set_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
  273. break;
  274. default:
  275. netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
  276. set_bit(__I40E_DOWN_REQUESTED, &pf->state);
  277. set_bit(__I40E_DOWN_REQUESTED, &vsi->state);
  278. break;
  279. }
  280. i40e_service_event_schedule(pf);
  281. pf->tx_timeout_recovery_level++;
  282. }
  283. /**
  284. * i40e_release_rx_desc - Store the new tail and head values
  285. * @rx_ring: ring to bump
  286. * @val: new head index
  287. **/
  288. static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
  289. {
  290. rx_ring->next_to_use = val;
  291. /* Force memory writes to complete before letting h/w
  292. * know there are new descriptors to fetch. (Only
  293. * applicable for weak-ordered memory model archs,
  294. * such as IA-64).
  295. */
  296. wmb();
  297. writel(val, rx_ring->tail);
  298. }
  299. /**
  300. * i40e_get_vsi_stats_struct - Get System Network Statistics
  301. * @vsi: the VSI we care about
  302. *
  303. * Returns the address of the device statistics structure.
  304. * The statistics are actually updated from the service task.
  305. **/
  306. struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
  307. {
  308. return &vsi->net_stats;
  309. }
  310. /**
  311. * i40e_get_netdev_stats_struct - Get statistics for netdev interface
  312. * @netdev: network interface device structure
  313. *
  314. * Returns the address of the device statistics structure.
  315. * The statistics are actually updated from the service task.
  316. **/
  317. #ifdef I40E_FCOE
  318. struct rtnl_link_stats64 *i40e_get_netdev_stats_struct(
  319. struct net_device *netdev,
  320. struct rtnl_link_stats64 *stats)
  321. #else
  322. static struct rtnl_link_stats64 *i40e_get_netdev_stats_struct(
  323. struct net_device *netdev,
  324. struct rtnl_link_stats64 *stats)
  325. #endif
  326. {
  327. struct i40e_netdev_priv *np = netdev_priv(netdev);
  328. struct i40e_ring *tx_ring, *rx_ring;
  329. struct i40e_vsi *vsi = np->vsi;
  330. struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
  331. int i;
  332. if (test_bit(__I40E_DOWN, &vsi->state))
  333. return stats;
  334. if (!vsi->tx_rings)
  335. return stats;
  336. rcu_read_lock();
  337. for (i = 0; i < vsi->num_queue_pairs; i++) {
  338. u64 bytes, packets;
  339. unsigned int start;
  340. tx_ring = ACCESS_ONCE(vsi->tx_rings[i]);
  341. if (!tx_ring)
  342. continue;
  343. do {
  344. start = u64_stats_fetch_begin_irq(&tx_ring->syncp);
  345. packets = tx_ring->stats.packets;
  346. bytes = tx_ring->stats.bytes;
  347. } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start));
  348. stats->tx_packets += packets;
  349. stats->tx_bytes += bytes;
  350. rx_ring = &tx_ring[1];
  351. do {
  352. start = u64_stats_fetch_begin_irq(&rx_ring->syncp);
  353. packets = rx_ring->stats.packets;
  354. bytes = rx_ring->stats.bytes;
  355. } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start));
  356. stats->rx_packets += packets;
  357. stats->rx_bytes += bytes;
  358. }
  359. rcu_read_unlock();
  360. /* following stats updated by i40e_watchdog_subtask() */
  361. stats->multicast = vsi_stats->multicast;
  362. stats->tx_errors = vsi_stats->tx_errors;
  363. stats->tx_dropped = vsi_stats->tx_dropped;
  364. stats->rx_errors = vsi_stats->rx_errors;
  365. stats->rx_crc_errors = vsi_stats->rx_crc_errors;
  366. stats->rx_length_errors = vsi_stats->rx_length_errors;
  367. return stats;
  368. }
  369. /**
  370. * i40e_vsi_reset_stats - Resets all stats of the given vsi
  371. * @vsi: the VSI to have its stats reset
  372. **/
  373. void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
  374. {
  375. struct rtnl_link_stats64 *ns;
  376. int i;
  377. if (!vsi)
  378. return;
  379. ns = i40e_get_vsi_stats_struct(vsi);
  380. memset(ns, 0, sizeof(*ns));
  381. memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
  382. memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
  383. memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
  384. if (vsi->rx_rings && vsi->rx_rings[0]) {
  385. for (i = 0; i < vsi->num_queue_pairs; i++) {
  386. memset(&vsi->rx_rings[i]->stats, 0 ,
  387. sizeof(vsi->rx_rings[i]->stats));
  388. memset(&vsi->rx_rings[i]->rx_stats, 0 ,
  389. sizeof(vsi->rx_rings[i]->rx_stats));
  390. memset(&vsi->tx_rings[i]->stats, 0 ,
  391. sizeof(vsi->tx_rings[i]->stats));
  392. memset(&vsi->tx_rings[i]->tx_stats, 0,
  393. sizeof(vsi->tx_rings[i]->tx_stats));
  394. }
  395. }
  396. vsi->stat_offsets_loaded = false;
  397. }
  398. /**
  399. * i40e_pf_reset_stats - Reset all of the stats for the given pf
  400. * @pf: the PF to be reset
  401. **/
  402. void i40e_pf_reset_stats(struct i40e_pf *pf)
  403. {
  404. int i;
  405. memset(&pf->stats, 0, sizeof(pf->stats));
  406. memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
  407. pf->stat_offsets_loaded = false;
  408. for (i = 0; i < I40E_MAX_VEB; i++) {
  409. if (pf->veb[i]) {
  410. memset(&pf->veb[i]->stats, 0,
  411. sizeof(pf->veb[i]->stats));
  412. memset(&pf->veb[i]->stats_offsets, 0,
  413. sizeof(pf->veb[i]->stats_offsets));
  414. pf->veb[i]->stat_offsets_loaded = false;
  415. }
  416. }
  417. }
  418. /**
  419. * i40e_stat_update48 - read and update a 48 bit stat from the chip
  420. * @hw: ptr to the hardware info
  421. * @hireg: the high 32 bit reg to read
  422. * @loreg: the low 32 bit reg to read
  423. * @offset_loaded: has the initial offset been loaded yet
  424. * @offset: ptr to current offset value
  425. * @stat: ptr to the stat
  426. *
  427. * Since the device stats are not reset at PFReset, they likely will not
  428. * be zeroed when the driver starts. We'll save the first values read
  429. * and use them as offsets to be subtracted from the raw values in order
  430. * to report stats that count from zero. In the process, we also manage
  431. * the potential roll-over.
  432. **/
  433. static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
  434. bool offset_loaded, u64 *offset, u64 *stat)
  435. {
  436. u64 new_data;
  437. if (hw->device_id == I40E_DEV_ID_QEMU) {
  438. new_data = rd32(hw, loreg);
  439. new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
  440. } else {
  441. new_data = rd64(hw, loreg);
  442. }
  443. if (!offset_loaded)
  444. *offset = new_data;
  445. if (likely(new_data >= *offset))
  446. *stat = new_data - *offset;
  447. else
  448. *stat = (new_data + ((u64)1 << 48)) - *offset;
  449. *stat &= 0xFFFFFFFFFFFFULL;
  450. }
  451. /**
  452. * i40e_stat_update32 - read and update a 32 bit stat from the chip
  453. * @hw: ptr to the hardware info
  454. * @reg: the hw reg to read
  455. * @offset_loaded: has the initial offset been loaded yet
  456. * @offset: ptr to current offset value
  457. * @stat: ptr to the stat
  458. **/
  459. static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
  460. bool offset_loaded, u64 *offset, u64 *stat)
  461. {
  462. u32 new_data;
  463. new_data = rd32(hw, reg);
  464. if (!offset_loaded)
  465. *offset = new_data;
  466. if (likely(new_data >= *offset))
  467. *stat = (u32)(new_data - *offset);
  468. else
  469. *stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
  470. }
  471. /**
  472. * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
  473. * @vsi: the VSI to be updated
  474. **/
  475. void i40e_update_eth_stats(struct i40e_vsi *vsi)
  476. {
  477. int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
  478. struct i40e_pf *pf = vsi->back;
  479. struct i40e_hw *hw = &pf->hw;
  480. struct i40e_eth_stats *oes;
  481. struct i40e_eth_stats *es; /* device's eth stats */
  482. es = &vsi->eth_stats;
  483. oes = &vsi->eth_stats_offsets;
  484. /* Gather up the stats that the hw collects */
  485. i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
  486. vsi->stat_offsets_loaded,
  487. &oes->tx_errors, &es->tx_errors);
  488. i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
  489. vsi->stat_offsets_loaded,
  490. &oes->rx_discards, &es->rx_discards);
  491. i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
  492. vsi->stat_offsets_loaded,
  493. &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
  494. i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
  495. vsi->stat_offsets_loaded,
  496. &oes->tx_errors, &es->tx_errors);
  497. i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
  498. I40E_GLV_GORCL(stat_idx),
  499. vsi->stat_offsets_loaded,
  500. &oes->rx_bytes, &es->rx_bytes);
  501. i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
  502. I40E_GLV_UPRCL(stat_idx),
  503. vsi->stat_offsets_loaded,
  504. &oes->rx_unicast, &es->rx_unicast);
  505. i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
  506. I40E_GLV_MPRCL(stat_idx),
  507. vsi->stat_offsets_loaded,
  508. &oes->rx_multicast, &es->rx_multicast);
  509. i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
  510. I40E_GLV_BPRCL(stat_idx),
  511. vsi->stat_offsets_loaded,
  512. &oes->rx_broadcast, &es->rx_broadcast);
  513. i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
  514. I40E_GLV_GOTCL(stat_idx),
  515. vsi->stat_offsets_loaded,
  516. &oes->tx_bytes, &es->tx_bytes);
  517. i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
  518. I40E_GLV_UPTCL(stat_idx),
  519. vsi->stat_offsets_loaded,
  520. &oes->tx_unicast, &es->tx_unicast);
  521. i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
  522. I40E_GLV_MPTCL(stat_idx),
  523. vsi->stat_offsets_loaded,
  524. &oes->tx_multicast, &es->tx_multicast);
  525. i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
  526. I40E_GLV_BPTCL(stat_idx),
  527. vsi->stat_offsets_loaded,
  528. &oes->tx_broadcast, &es->tx_broadcast);
  529. vsi->stat_offsets_loaded = true;
  530. }
  531. /**
  532. * i40e_update_veb_stats - Update Switch component statistics
  533. * @veb: the VEB being updated
  534. **/
  535. static void i40e_update_veb_stats(struct i40e_veb *veb)
  536. {
  537. struct i40e_pf *pf = veb->pf;
  538. struct i40e_hw *hw = &pf->hw;
  539. struct i40e_eth_stats *oes;
  540. struct i40e_eth_stats *es; /* device's eth stats */
  541. int idx = 0;
  542. idx = veb->stats_idx;
  543. es = &veb->stats;
  544. oes = &veb->stats_offsets;
  545. /* Gather up the stats that the hw collects */
  546. i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
  547. veb->stat_offsets_loaded,
  548. &oes->tx_discards, &es->tx_discards);
  549. if (hw->revision_id > 0)
  550. i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
  551. veb->stat_offsets_loaded,
  552. &oes->rx_unknown_protocol,
  553. &es->rx_unknown_protocol);
  554. i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
  555. veb->stat_offsets_loaded,
  556. &oes->rx_bytes, &es->rx_bytes);
  557. i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
  558. veb->stat_offsets_loaded,
  559. &oes->rx_unicast, &es->rx_unicast);
  560. i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
  561. veb->stat_offsets_loaded,
  562. &oes->rx_multicast, &es->rx_multicast);
  563. i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
  564. veb->stat_offsets_loaded,
  565. &oes->rx_broadcast, &es->rx_broadcast);
  566. i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
  567. veb->stat_offsets_loaded,
  568. &oes->tx_bytes, &es->tx_bytes);
  569. i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
  570. veb->stat_offsets_loaded,
  571. &oes->tx_unicast, &es->tx_unicast);
  572. i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
  573. veb->stat_offsets_loaded,
  574. &oes->tx_multicast, &es->tx_multicast);
  575. i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
  576. veb->stat_offsets_loaded,
  577. &oes->tx_broadcast, &es->tx_broadcast);
  578. veb->stat_offsets_loaded = true;
  579. }
  580. #ifdef I40E_FCOE
  581. /**
  582. * i40e_update_fcoe_stats - Update FCoE-specific ethernet statistics counters.
  583. * @vsi: the VSI that is capable of doing FCoE
  584. **/
  585. static void i40e_update_fcoe_stats(struct i40e_vsi *vsi)
  586. {
  587. struct i40e_pf *pf = vsi->back;
  588. struct i40e_hw *hw = &pf->hw;
  589. struct i40e_fcoe_stats *ofs;
  590. struct i40e_fcoe_stats *fs; /* device's eth stats */
  591. int idx;
  592. if (vsi->type != I40E_VSI_FCOE)
  593. return;
  594. idx = (pf->pf_seid - I40E_BASE_PF_SEID) + I40E_FCOE_PF_STAT_OFFSET;
  595. fs = &vsi->fcoe_stats;
  596. ofs = &vsi->fcoe_stats_offsets;
  597. i40e_stat_update32(hw, I40E_GL_FCOEPRC(idx),
  598. vsi->fcoe_stat_offsets_loaded,
  599. &ofs->rx_fcoe_packets, &fs->rx_fcoe_packets);
  600. i40e_stat_update48(hw, I40E_GL_FCOEDWRCH(idx), I40E_GL_FCOEDWRCL(idx),
  601. vsi->fcoe_stat_offsets_loaded,
  602. &ofs->rx_fcoe_dwords, &fs->rx_fcoe_dwords);
  603. i40e_stat_update32(hw, I40E_GL_FCOERPDC(idx),
  604. vsi->fcoe_stat_offsets_loaded,
  605. &ofs->rx_fcoe_dropped, &fs->rx_fcoe_dropped);
  606. i40e_stat_update32(hw, I40E_GL_FCOEPTC(idx),
  607. vsi->fcoe_stat_offsets_loaded,
  608. &ofs->tx_fcoe_packets, &fs->tx_fcoe_packets);
  609. i40e_stat_update48(hw, I40E_GL_FCOEDWTCH(idx), I40E_GL_FCOEDWTCL(idx),
  610. vsi->fcoe_stat_offsets_loaded,
  611. &ofs->tx_fcoe_dwords, &fs->tx_fcoe_dwords);
  612. i40e_stat_update32(hw, I40E_GL_FCOECRC(idx),
  613. vsi->fcoe_stat_offsets_loaded,
  614. &ofs->fcoe_bad_fccrc, &fs->fcoe_bad_fccrc);
  615. i40e_stat_update32(hw, I40E_GL_FCOELAST(idx),
  616. vsi->fcoe_stat_offsets_loaded,
  617. &ofs->fcoe_last_error, &fs->fcoe_last_error);
  618. i40e_stat_update32(hw, I40E_GL_FCOEDDPC(idx),
  619. vsi->fcoe_stat_offsets_loaded,
  620. &ofs->fcoe_ddp_count, &fs->fcoe_ddp_count);
  621. vsi->fcoe_stat_offsets_loaded = true;
  622. }
  623. #endif
  624. /**
  625. * i40e_update_link_xoff_rx - Update XOFF received in link flow control mode
  626. * @pf: the corresponding PF
  627. *
  628. * Update the Rx XOFF counter (PAUSE frames) in link flow control mode
  629. **/
  630. static void i40e_update_link_xoff_rx(struct i40e_pf *pf)
  631. {
  632. struct i40e_hw_port_stats *osd = &pf->stats_offsets;
  633. struct i40e_hw_port_stats *nsd = &pf->stats;
  634. struct i40e_hw *hw = &pf->hw;
  635. u64 xoff = 0;
  636. u16 i, v;
  637. if ((hw->fc.current_mode != I40E_FC_FULL) &&
  638. (hw->fc.current_mode != I40E_FC_RX_PAUSE))
  639. return;
  640. xoff = nsd->link_xoff_rx;
  641. i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
  642. pf->stat_offsets_loaded,
  643. &osd->link_xoff_rx, &nsd->link_xoff_rx);
  644. /* No new LFC xoff rx */
  645. if (!(nsd->link_xoff_rx - xoff))
  646. return;
  647. /* Clear the __I40E_HANG_CHECK_ARMED bit for all Tx rings */
  648. for (v = 0; v < pf->num_alloc_vsi; v++) {
  649. struct i40e_vsi *vsi = pf->vsi[v];
  650. if (!vsi || !vsi->tx_rings[0])
  651. continue;
  652. for (i = 0; i < vsi->num_queue_pairs; i++) {
  653. struct i40e_ring *ring = vsi->tx_rings[i];
  654. clear_bit(__I40E_HANG_CHECK_ARMED, &ring->state);
  655. }
  656. }
  657. }
  658. /**
  659. * i40e_update_prio_xoff_rx - Update XOFF received in PFC mode
  660. * @pf: the corresponding PF
  661. *
  662. * Update the Rx XOFF counter (PAUSE frames) in PFC mode
  663. **/
  664. static void i40e_update_prio_xoff_rx(struct i40e_pf *pf)
  665. {
  666. struct i40e_hw_port_stats *osd = &pf->stats_offsets;
  667. struct i40e_hw_port_stats *nsd = &pf->stats;
  668. bool xoff[I40E_MAX_TRAFFIC_CLASS] = {false};
  669. struct i40e_dcbx_config *dcb_cfg;
  670. struct i40e_hw *hw = &pf->hw;
  671. u16 i, v;
  672. u8 tc;
  673. dcb_cfg = &hw->local_dcbx_config;
  674. /* See if DCB enabled with PFC TC */
  675. if (!(pf->flags & I40E_FLAG_DCB_ENABLED) ||
  676. !(dcb_cfg->pfc.pfcenable)) {
  677. i40e_update_link_xoff_rx(pf);
  678. return;
  679. }
  680. for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
  681. u64 prio_xoff = nsd->priority_xoff_rx[i];
  682. i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
  683. pf->stat_offsets_loaded,
  684. &osd->priority_xoff_rx[i],
  685. &nsd->priority_xoff_rx[i]);
  686. /* No new PFC xoff rx */
  687. if (!(nsd->priority_xoff_rx[i] - prio_xoff))
  688. continue;
  689. /* Get the TC for given priority */
  690. tc = dcb_cfg->etscfg.prioritytable[i];
  691. xoff[tc] = true;
  692. }
  693. /* Clear the __I40E_HANG_CHECK_ARMED bit for Tx rings */
  694. for (v = 0; v < pf->num_alloc_vsi; v++) {
  695. struct i40e_vsi *vsi = pf->vsi[v];
  696. if (!vsi || !vsi->tx_rings[0])
  697. continue;
  698. for (i = 0; i < vsi->num_queue_pairs; i++) {
  699. struct i40e_ring *ring = vsi->tx_rings[i];
  700. tc = ring->dcb_tc;
  701. if (xoff[tc])
  702. clear_bit(__I40E_HANG_CHECK_ARMED,
  703. &ring->state);
  704. }
  705. }
  706. }
  707. /**
  708. * i40e_update_vsi_stats - Update the vsi statistics counters.
  709. * @vsi: the VSI to be updated
  710. *
  711. * There are a few instances where we store the same stat in a
  712. * couple of different structs. This is partly because we have
  713. * the netdev stats that need to be filled out, which is slightly
  714. * different from the "eth_stats" defined by the chip and used in
  715. * VF communications. We sort it out here.
  716. **/
  717. static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
  718. {
  719. struct i40e_pf *pf = vsi->back;
  720. struct rtnl_link_stats64 *ons;
  721. struct rtnl_link_stats64 *ns; /* netdev stats */
  722. struct i40e_eth_stats *oes;
  723. struct i40e_eth_stats *es; /* device's eth stats */
  724. u32 tx_restart, tx_busy;
  725. u32 rx_page, rx_buf;
  726. u64 rx_p, rx_b;
  727. u64 tx_p, tx_b;
  728. u16 q;
  729. if (test_bit(__I40E_DOWN, &vsi->state) ||
  730. test_bit(__I40E_CONFIG_BUSY, &pf->state))
  731. return;
  732. ns = i40e_get_vsi_stats_struct(vsi);
  733. ons = &vsi->net_stats_offsets;
  734. es = &vsi->eth_stats;
  735. oes = &vsi->eth_stats_offsets;
  736. /* Gather up the netdev and vsi stats that the driver collects
  737. * on the fly during packet processing
  738. */
  739. rx_b = rx_p = 0;
  740. tx_b = tx_p = 0;
  741. tx_restart = tx_busy = 0;
  742. rx_page = 0;
  743. rx_buf = 0;
  744. rcu_read_lock();
  745. for (q = 0; q < vsi->num_queue_pairs; q++) {
  746. struct i40e_ring *p;
  747. u64 bytes, packets;
  748. unsigned int start;
  749. /* locate Tx ring */
  750. p = ACCESS_ONCE(vsi->tx_rings[q]);
  751. do {
  752. start = u64_stats_fetch_begin_irq(&p->syncp);
  753. packets = p->stats.packets;
  754. bytes = p->stats.bytes;
  755. } while (u64_stats_fetch_retry_irq(&p->syncp, start));
  756. tx_b += bytes;
  757. tx_p += packets;
  758. tx_restart += p->tx_stats.restart_queue;
  759. tx_busy += p->tx_stats.tx_busy;
  760. /* Rx queue is part of the same block as Tx queue */
  761. p = &p[1];
  762. do {
  763. start = u64_stats_fetch_begin_irq(&p->syncp);
  764. packets = p->stats.packets;
  765. bytes = p->stats.bytes;
  766. } while (u64_stats_fetch_retry_irq(&p->syncp, start));
  767. rx_b += bytes;
  768. rx_p += packets;
  769. rx_buf += p->rx_stats.alloc_buff_failed;
  770. rx_page += p->rx_stats.alloc_page_failed;
  771. }
  772. rcu_read_unlock();
  773. vsi->tx_restart = tx_restart;
  774. vsi->tx_busy = tx_busy;
  775. vsi->rx_page_failed = rx_page;
  776. vsi->rx_buf_failed = rx_buf;
  777. ns->rx_packets = rx_p;
  778. ns->rx_bytes = rx_b;
  779. ns->tx_packets = tx_p;
  780. ns->tx_bytes = tx_b;
  781. /* update netdev stats from eth stats */
  782. i40e_update_eth_stats(vsi);
  783. ons->tx_errors = oes->tx_errors;
  784. ns->tx_errors = es->tx_errors;
  785. ons->multicast = oes->rx_multicast;
  786. ns->multicast = es->rx_multicast;
  787. ons->rx_dropped = oes->rx_discards;
  788. ns->rx_dropped = es->rx_discards;
  789. ons->tx_dropped = oes->tx_discards;
  790. ns->tx_dropped = es->tx_discards;
  791. /* pull in a couple PF stats if this is the main vsi */
  792. if (vsi == pf->vsi[pf->lan_vsi]) {
  793. ns->rx_crc_errors = pf->stats.crc_errors;
  794. ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
  795. ns->rx_length_errors = pf->stats.rx_length_errors;
  796. }
  797. }
  798. /**
  799. * i40e_update_pf_stats - Update the pf statistics counters.
  800. * @pf: the PF to be updated
  801. **/
  802. static void i40e_update_pf_stats(struct i40e_pf *pf)
  803. {
  804. struct i40e_hw_port_stats *osd = &pf->stats_offsets;
  805. struct i40e_hw_port_stats *nsd = &pf->stats;
  806. struct i40e_hw *hw = &pf->hw;
  807. u32 val;
  808. int i;
  809. i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
  810. I40E_GLPRT_GORCL(hw->port),
  811. pf->stat_offsets_loaded,
  812. &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
  813. i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
  814. I40E_GLPRT_GOTCL(hw->port),
  815. pf->stat_offsets_loaded,
  816. &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
  817. i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
  818. pf->stat_offsets_loaded,
  819. &osd->eth.rx_discards,
  820. &nsd->eth.rx_discards);
  821. i40e_stat_update32(hw, I40E_GLPRT_TDPC(hw->port),
  822. pf->stat_offsets_loaded,
  823. &osd->eth.tx_discards,
  824. &nsd->eth.tx_discards);
  825. i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
  826. I40E_GLPRT_UPRCL(hw->port),
  827. pf->stat_offsets_loaded,
  828. &osd->eth.rx_unicast,
  829. &nsd->eth.rx_unicast);
  830. i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
  831. I40E_GLPRT_MPRCL(hw->port),
  832. pf->stat_offsets_loaded,
  833. &osd->eth.rx_multicast,
  834. &nsd->eth.rx_multicast);
  835. i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
  836. I40E_GLPRT_BPRCL(hw->port),
  837. pf->stat_offsets_loaded,
  838. &osd->eth.rx_broadcast,
  839. &nsd->eth.rx_broadcast);
  840. i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
  841. I40E_GLPRT_UPTCL(hw->port),
  842. pf->stat_offsets_loaded,
  843. &osd->eth.tx_unicast,
  844. &nsd->eth.tx_unicast);
  845. i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
  846. I40E_GLPRT_MPTCL(hw->port),
  847. pf->stat_offsets_loaded,
  848. &osd->eth.tx_multicast,
  849. &nsd->eth.tx_multicast);
  850. i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
  851. I40E_GLPRT_BPTCL(hw->port),
  852. pf->stat_offsets_loaded,
  853. &osd->eth.tx_broadcast,
  854. &nsd->eth.tx_broadcast);
  855. i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
  856. pf->stat_offsets_loaded,
  857. &osd->tx_dropped_link_down,
  858. &nsd->tx_dropped_link_down);
  859. i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
  860. pf->stat_offsets_loaded,
  861. &osd->crc_errors, &nsd->crc_errors);
  862. i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
  863. pf->stat_offsets_loaded,
  864. &osd->illegal_bytes, &nsd->illegal_bytes);
  865. i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
  866. pf->stat_offsets_loaded,
  867. &osd->mac_local_faults,
  868. &nsd->mac_local_faults);
  869. i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
  870. pf->stat_offsets_loaded,
  871. &osd->mac_remote_faults,
  872. &nsd->mac_remote_faults);
  873. i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
  874. pf->stat_offsets_loaded,
  875. &osd->rx_length_errors,
  876. &nsd->rx_length_errors);
  877. i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
  878. pf->stat_offsets_loaded,
  879. &osd->link_xon_rx, &nsd->link_xon_rx);
  880. i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
  881. pf->stat_offsets_loaded,
  882. &osd->link_xon_tx, &nsd->link_xon_tx);
  883. i40e_update_prio_xoff_rx(pf); /* handles I40E_GLPRT_LXOFFRXC */
  884. i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
  885. pf->stat_offsets_loaded,
  886. &osd->link_xoff_tx, &nsd->link_xoff_tx);
  887. for (i = 0; i < 8; i++) {
  888. i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
  889. pf->stat_offsets_loaded,
  890. &osd->priority_xon_rx[i],
  891. &nsd->priority_xon_rx[i]);
  892. i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
  893. pf->stat_offsets_loaded,
  894. &osd->priority_xon_tx[i],
  895. &nsd->priority_xon_tx[i]);
  896. i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
  897. pf->stat_offsets_loaded,
  898. &osd->priority_xoff_tx[i],
  899. &nsd->priority_xoff_tx[i]);
  900. i40e_stat_update32(hw,
  901. I40E_GLPRT_RXON2OFFCNT(hw->port, i),
  902. pf->stat_offsets_loaded,
  903. &osd->priority_xon_2_xoff[i],
  904. &nsd->priority_xon_2_xoff[i]);
  905. }
  906. i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
  907. I40E_GLPRT_PRC64L(hw->port),
  908. pf->stat_offsets_loaded,
  909. &osd->rx_size_64, &nsd->rx_size_64);
  910. i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
  911. I40E_GLPRT_PRC127L(hw->port),
  912. pf->stat_offsets_loaded,
  913. &osd->rx_size_127, &nsd->rx_size_127);
  914. i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
  915. I40E_GLPRT_PRC255L(hw->port),
  916. pf->stat_offsets_loaded,
  917. &osd->rx_size_255, &nsd->rx_size_255);
  918. i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
  919. I40E_GLPRT_PRC511L(hw->port),
  920. pf->stat_offsets_loaded,
  921. &osd->rx_size_511, &nsd->rx_size_511);
  922. i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
  923. I40E_GLPRT_PRC1023L(hw->port),
  924. pf->stat_offsets_loaded,
  925. &osd->rx_size_1023, &nsd->rx_size_1023);
  926. i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
  927. I40E_GLPRT_PRC1522L(hw->port),
  928. pf->stat_offsets_loaded,
  929. &osd->rx_size_1522, &nsd->rx_size_1522);
  930. i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
  931. I40E_GLPRT_PRC9522L(hw->port),
  932. pf->stat_offsets_loaded,
  933. &osd->rx_size_big, &nsd->rx_size_big);
  934. i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
  935. I40E_GLPRT_PTC64L(hw->port),
  936. pf->stat_offsets_loaded,
  937. &osd->tx_size_64, &nsd->tx_size_64);
  938. i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
  939. I40E_GLPRT_PTC127L(hw->port),
  940. pf->stat_offsets_loaded,
  941. &osd->tx_size_127, &nsd->tx_size_127);
  942. i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
  943. I40E_GLPRT_PTC255L(hw->port),
  944. pf->stat_offsets_loaded,
  945. &osd->tx_size_255, &nsd->tx_size_255);
  946. i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
  947. I40E_GLPRT_PTC511L(hw->port),
  948. pf->stat_offsets_loaded,
  949. &osd->tx_size_511, &nsd->tx_size_511);
  950. i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
  951. I40E_GLPRT_PTC1023L(hw->port),
  952. pf->stat_offsets_loaded,
  953. &osd->tx_size_1023, &nsd->tx_size_1023);
  954. i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
  955. I40E_GLPRT_PTC1522L(hw->port),
  956. pf->stat_offsets_loaded,
  957. &osd->tx_size_1522, &nsd->tx_size_1522);
  958. i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
  959. I40E_GLPRT_PTC9522L(hw->port),
  960. pf->stat_offsets_loaded,
  961. &osd->tx_size_big, &nsd->tx_size_big);
  962. i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
  963. pf->stat_offsets_loaded,
  964. &osd->rx_undersize, &nsd->rx_undersize);
  965. i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
  966. pf->stat_offsets_loaded,
  967. &osd->rx_fragments, &nsd->rx_fragments);
  968. i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
  969. pf->stat_offsets_loaded,
  970. &osd->rx_oversize, &nsd->rx_oversize);
  971. i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
  972. pf->stat_offsets_loaded,
  973. &osd->rx_jabber, &nsd->rx_jabber);
  974. /* FDIR stats */
  975. i40e_stat_update32(hw, I40E_GLQF_PCNT(pf->fd_atr_cnt_idx),
  976. pf->stat_offsets_loaded,
  977. &osd->fd_atr_match, &nsd->fd_atr_match);
  978. i40e_stat_update32(hw, I40E_GLQF_PCNT(pf->fd_sb_cnt_idx),
  979. pf->stat_offsets_loaded,
  980. &osd->fd_sb_match, &nsd->fd_sb_match);
  981. val = rd32(hw, I40E_PRTPM_EEE_STAT);
  982. nsd->tx_lpi_status =
  983. (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
  984. I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
  985. nsd->rx_lpi_status =
  986. (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
  987. I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
  988. i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
  989. pf->stat_offsets_loaded,
  990. &osd->tx_lpi_count, &nsd->tx_lpi_count);
  991. i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
  992. pf->stat_offsets_loaded,
  993. &osd->rx_lpi_count, &nsd->rx_lpi_count);
  994. pf->stat_offsets_loaded = true;
  995. }
  996. /**
  997. * i40e_update_stats - Update the various statistics counters.
  998. * @vsi: the VSI to be updated
  999. *
  1000. * Update the various stats for this VSI and its related entities.
  1001. **/
  1002. void i40e_update_stats(struct i40e_vsi *vsi)
  1003. {
  1004. struct i40e_pf *pf = vsi->back;
  1005. if (vsi == pf->vsi[pf->lan_vsi])
  1006. i40e_update_pf_stats(pf);
  1007. i40e_update_vsi_stats(vsi);
  1008. #ifdef I40E_FCOE
  1009. i40e_update_fcoe_stats(vsi);
  1010. #endif
  1011. }
  1012. /**
  1013. * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
  1014. * @vsi: the VSI to be searched
  1015. * @macaddr: the MAC address
  1016. * @vlan: the vlan
  1017. * @is_vf: make sure its a vf filter, else doesn't matter
  1018. * @is_netdev: make sure its a netdev filter, else doesn't matter
  1019. *
  1020. * Returns ptr to the filter object or NULL
  1021. **/
  1022. static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
  1023. u8 *macaddr, s16 vlan,
  1024. bool is_vf, bool is_netdev)
  1025. {
  1026. struct i40e_mac_filter *f;
  1027. if (!vsi || !macaddr)
  1028. return NULL;
  1029. list_for_each_entry(f, &vsi->mac_filter_list, list) {
  1030. if ((ether_addr_equal(macaddr, f->macaddr)) &&
  1031. (vlan == f->vlan) &&
  1032. (!is_vf || f->is_vf) &&
  1033. (!is_netdev || f->is_netdev))
  1034. return f;
  1035. }
  1036. return NULL;
  1037. }
  1038. /**
  1039. * i40e_find_mac - Find a mac addr in the macvlan filters list
  1040. * @vsi: the VSI to be searched
  1041. * @macaddr: the MAC address we are searching for
  1042. * @is_vf: make sure its a vf filter, else doesn't matter
  1043. * @is_netdev: make sure its a netdev filter, else doesn't matter
  1044. *
  1045. * Returns the first filter with the provided MAC address or NULL if
  1046. * MAC address was not found
  1047. **/
  1048. struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, u8 *macaddr,
  1049. bool is_vf, bool is_netdev)
  1050. {
  1051. struct i40e_mac_filter *f;
  1052. if (!vsi || !macaddr)
  1053. return NULL;
  1054. list_for_each_entry(f, &vsi->mac_filter_list, list) {
  1055. if ((ether_addr_equal(macaddr, f->macaddr)) &&
  1056. (!is_vf || f->is_vf) &&
  1057. (!is_netdev || f->is_netdev))
  1058. return f;
  1059. }
  1060. return NULL;
  1061. }
  1062. /**
  1063. * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
  1064. * @vsi: the VSI to be searched
  1065. *
  1066. * Returns true if VSI is in vlan mode or false otherwise
  1067. **/
  1068. bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
  1069. {
  1070. struct i40e_mac_filter *f;
  1071. /* Only -1 for all the filters denotes not in vlan mode
  1072. * so we have to go through all the list in order to make sure
  1073. */
  1074. list_for_each_entry(f, &vsi->mac_filter_list, list) {
  1075. if (f->vlan >= 0)
  1076. return true;
  1077. }
  1078. return false;
  1079. }
  1080. /**
  1081. * i40e_put_mac_in_vlan - Make macvlan filters from macaddrs and vlans
  1082. * @vsi: the VSI to be searched
  1083. * @macaddr: the mac address to be filtered
  1084. * @is_vf: true if it is a vf
  1085. * @is_netdev: true if it is a netdev
  1086. *
  1087. * Goes through all the macvlan filters and adds a
  1088. * macvlan filter for each unique vlan that already exists
  1089. *
  1090. * Returns first filter found on success, else NULL
  1091. **/
  1092. struct i40e_mac_filter *i40e_put_mac_in_vlan(struct i40e_vsi *vsi, u8 *macaddr,
  1093. bool is_vf, bool is_netdev)
  1094. {
  1095. struct i40e_mac_filter *f;
  1096. list_for_each_entry(f, &vsi->mac_filter_list, list) {
  1097. if (!i40e_find_filter(vsi, macaddr, f->vlan,
  1098. is_vf, is_netdev)) {
  1099. if (!i40e_add_filter(vsi, macaddr, f->vlan,
  1100. is_vf, is_netdev))
  1101. return NULL;
  1102. }
  1103. }
  1104. return list_first_entry_or_null(&vsi->mac_filter_list,
  1105. struct i40e_mac_filter, list);
  1106. }
  1107. /**
  1108. * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
  1109. * @vsi: the PF Main VSI - inappropriate for any other VSI
  1110. * @macaddr: the MAC address
  1111. **/
  1112. static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
  1113. {
  1114. struct i40e_aqc_remove_macvlan_element_data element;
  1115. struct i40e_pf *pf = vsi->back;
  1116. i40e_status aq_ret;
  1117. /* Only appropriate for the PF main VSI */
  1118. if (vsi->type != I40E_VSI_MAIN)
  1119. return;
  1120. ether_addr_copy(element.mac_addr, macaddr);
  1121. element.vlan_tag = 0;
  1122. element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
  1123. I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
  1124. aq_ret = i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
  1125. if (aq_ret)
  1126. dev_err(&pf->pdev->dev, "Could not remove default MAC-VLAN\n");
  1127. }
  1128. /**
  1129. * i40e_add_filter - Add a mac/vlan filter to the VSI
  1130. * @vsi: the VSI to be searched
  1131. * @macaddr: the MAC address
  1132. * @vlan: the vlan
  1133. * @is_vf: make sure its a vf filter, else doesn't matter
  1134. * @is_netdev: make sure its a netdev filter, else doesn't matter
  1135. *
  1136. * Returns ptr to the filter object or NULL when no memory available.
  1137. **/
  1138. struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
  1139. u8 *macaddr, s16 vlan,
  1140. bool is_vf, bool is_netdev)
  1141. {
  1142. struct i40e_mac_filter *f;
  1143. if (!vsi || !macaddr)
  1144. return NULL;
  1145. f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev);
  1146. if (!f) {
  1147. f = kzalloc(sizeof(*f), GFP_ATOMIC);
  1148. if (!f)
  1149. goto add_filter_out;
  1150. ether_addr_copy(f->macaddr, macaddr);
  1151. f->vlan = vlan;
  1152. f->changed = true;
  1153. INIT_LIST_HEAD(&f->list);
  1154. list_add(&f->list, &vsi->mac_filter_list);
  1155. }
  1156. /* increment counter and add a new flag if needed */
  1157. if (is_vf) {
  1158. if (!f->is_vf) {
  1159. f->is_vf = true;
  1160. f->counter++;
  1161. }
  1162. } else if (is_netdev) {
  1163. if (!f->is_netdev) {
  1164. f->is_netdev = true;
  1165. f->counter++;
  1166. }
  1167. } else {
  1168. f->counter++;
  1169. }
  1170. /* changed tells sync_filters_subtask to
  1171. * push the filter down to the firmware
  1172. */
  1173. if (f->changed) {
  1174. vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
  1175. vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
  1176. }
  1177. add_filter_out:
  1178. return f;
  1179. }
  1180. /**
  1181. * i40e_del_filter - Remove a mac/vlan filter from the VSI
  1182. * @vsi: the VSI to be searched
  1183. * @macaddr: the MAC address
  1184. * @vlan: the vlan
  1185. * @is_vf: make sure it's a vf filter, else doesn't matter
  1186. * @is_netdev: make sure it's a netdev filter, else doesn't matter
  1187. **/
  1188. void i40e_del_filter(struct i40e_vsi *vsi,
  1189. u8 *macaddr, s16 vlan,
  1190. bool is_vf, bool is_netdev)
  1191. {
  1192. struct i40e_mac_filter *f;
  1193. if (!vsi || !macaddr)
  1194. return;
  1195. f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev);
  1196. if (!f || f->counter == 0)
  1197. return;
  1198. if (is_vf) {
  1199. if (f->is_vf) {
  1200. f->is_vf = false;
  1201. f->counter--;
  1202. }
  1203. } else if (is_netdev) {
  1204. if (f->is_netdev) {
  1205. f->is_netdev = false;
  1206. f->counter--;
  1207. }
  1208. } else {
  1209. /* make sure we don't remove a filter in use by vf or netdev */
  1210. int min_f = 0;
  1211. min_f += (f->is_vf ? 1 : 0);
  1212. min_f += (f->is_netdev ? 1 : 0);
  1213. if (f->counter > min_f)
  1214. f->counter--;
  1215. }
  1216. /* counter == 0 tells sync_filters_subtask to
  1217. * remove the filter from the firmware's list
  1218. */
  1219. if (f->counter == 0) {
  1220. f->changed = true;
  1221. vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
  1222. vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
  1223. }
  1224. }
  1225. /**
  1226. * i40e_set_mac - NDO callback to set mac address
  1227. * @netdev: network interface device structure
  1228. * @p: pointer to an address structure
  1229. *
  1230. * Returns 0 on success, negative on failure
  1231. **/
  1232. #ifdef I40E_FCOE
  1233. int i40e_set_mac(struct net_device *netdev, void *p)
  1234. #else
  1235. static int i40e_set_mac(struct net_device *netdev, void *p)
  1236. #endif
  1237. {
  1238. struct i40e_netdev_priv *np = netdev_priv(netdev);
  1239. struct i40e_vsi *vsi = np->vsi;
  1240. struct sockaddr *addr = p;
  1241. struct i40e_mac_filter *f;
  1242. if (!is_valid_ether_addr(addr->sa_data))
  1243. return -EADDRNOTAVAIL;
  1244. netdev_info(netdev, "set mac address=%pM\n", addr->sa_data);
  1245. if (test_bit(__I40E_DOWN, &vsi->back->state) ||
  1246. test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
  1247. return -EADDRNOTAVAIL;
  1248. if (vsi->type == I40E_VSI_MAIN) {
  1249. i40e_status ret;
  1250. ret = i40e_aq_mac_address_write(&vsi->back->hw,
  1251. I40E_AQC_WRITE_TYPE_LAA_WOL,
  1252. addr->sa_data, NULL);
  1253. if (ret) {
  1254. netdev_info(netdev,
  1255. "Addr change for Main VSI failed: %d\n",
  1256. ret);
  1257. return -EADDRNOTAVAIL;
  1258. }
  1259. }
  1260. f = i40e_find_mac(vsi, addr->sa_data, false, true);
  1261. if (!f) {
  1262. /* In order to be sure to not drop any packets, add the
  1263. * new address first then delete the old one.
  1264. */
  1265. f = i40e_add_filter(vsi, addr->sa_data, I40E_VLAN_ANY,
  1266. false, false);
  1267. if (!f)
  1268. return -ENOMEM;
  1269. i40e_sync_vsi_filters(vsi);
  1270. i40e_del_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY,
  1271. false, false);
  1272. i40e_sync_vsi_filters(vsi);
  1273. }
  1274. f->is_laa = true;
  1275. if (!ether_addr_equal(netdev->dev_addr, addr->sa_data))
  1276. ether_addr_copy(netdev->dev_addr, addr->sa_data);
  1277. return 0;
  1278. }
  1279. /**
  1280. * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
  1281. * @vsi: the VSI being setup
  1282. * @ctxt: VSI context structure
  1283. * @enabled_tc: Enabled TCs bitmap
  1284. * @is_add: True if called before Add VSI
  1285. *
  1286. * Setup VSI queue mapping for enabled traffic classes.
  1287. **/
  1288. #ifdef I40E_FCOE
  1289. void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
  1290. struct i40e_vsi_context *ctxt,
  1291. u8 enabled_tc,
  1292. bool is_add)
  1293. #else
  1294. static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
  1295. struct i40e_vsi_context *ctxt,
  1296. u8 enabled_tc,
  1297. bool is_add)
  1298. #endif
  1299. {
  1300. struct i40e_pf *pf = vsi->back;
  1301. u16 sections = 0;
  1302. u8 netdev_tc = 0;
  1303. u16 numtc = 0;
  1304. u16 qcount;
  1305. u8 offset;
  1306. u16 qmap;
  1307. int i;
  1308. u16 num_tc_qps = 0;
  1309. sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
  1310. offset = 0;
  1311. if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
  1312. /* Find numtc from enabled TC bitmap */
  1313. for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
  1314. if (enabled_tc & (1 << i)) /* TC is enabled */
  1315. numtc++;
  1316. }
  1317. if (!numtc) {
  1318. dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
  1319. numtc = 1;
  1320. }
  1321. } else {
  1322. /* At least TC0 is enabled in case of non-DCB case */
  1323. numtc = 1;
  1324. }
  1325. vsi->tc_config.numtc = numtc;
  1326. vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
  1327. /* Number of queues per enabled TC */
  1328. num_tc_qps = vsi->alloc_queue_pairs/numtc;
  1329. num_tc_qps = min_t(int, num_tc_qps, I40E_MAX_QUEUES_PER_TC);
  1330. /* Setup queue offset/count for all TCs for given VSI */
  1331. for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
  1332. /* See if the given TC is enabled for the given VSI */
  1333. if (vsi->tc_config.enabled_tc & (1 << i)) { /* TC is enabled */
  1334. int pow, num_qps;
  1335. switch (vsi->type) {
  1336. case I40E_VSI_MAIN:
  1337. qcount = min_t(int, pf->rss_size, num_tc_qps);
  1338. break;
  1339. #ifdef I40E_FCOE
  1340. case I40E_VSI_FCOE:
  1341. qcount = num_tc_qps;
  1342. break;
  1343. #endif
  1344. case I40E_VSI_FDIR:
  1345. case I40E_VSI_SRIOV:
  1346. case I40E_VSI_VMDQ2:
  1347. default:
  1348. qcount = num_tc_qps;
  1349. WARN_ON(i != 0);
  1350. break;
  1351. }
  1352. vsi->tc_config.tc_info[i].qoffset = offset;
  1353. vsi->tc_config.tc_info[i].qcount = qcount;
  1354. /* find the power-of-2 of the number of queue pairs */
  1355. num_qps = qcount;
  1356. pow = 0;
  1357. while (num_qps && ((1 << pow) < qcount)) {
  1358. pow++;
  1359. num_qps >>= 1;
  1360. }
  1361. vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
  1362. qmap =
  1363. (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
  1364. (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
  1365. offset += qcount;
  1366. } else {
  1367. /* TC is not enabled so set the offset to
  1368. * default queue and allocate one queue
  1369. * for the given TC.
  1370. */
  1371. vsi->tc_config.tc_info[i].qoffset = 0;
  1372. vsi->tc_config.tc_info[i].qcount = 1;
  1373. vsi->tc_config.tc_info[i].netdev_tc = 0;
  1374. qmap = 0;
  1375. }
  1376. ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
  1377. }
  1378. /* Set actual Tx/Rx queue pairs */
  1379. vsi->num_queue_pairs = offset;
  1380. /* Scheduler section valid can only be set for ADD VSI */
  1381. if (is_add) {
  1382. sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
  1383. ctxt->info.up_enable_bits = enabled_tc;
  1384. }
  1385. if (vsi->type == I40E_VSI_SRIOV) {
  1386. ctxt->info.mapping_flags |=
  1387. cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
  1388. for (i = 0; i < vsi->num_queue_pairs; i++)
  1389. ctxt->info.queue_mapping[i] =
  1390. cpu_to_le16(vsi->base_queue + i);
  1391. } else {
  1392. ctxt->info.mapping_flags |=
  1393. cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
  1394. ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
  1395. }
  1396. ctxt->info.valid_sections |= cpu_to_le16(sections);
  1397. }
  1398. /**
  1399. * i40e_set_rx_mode - NDO callback to set the netdev filters
  1400. * @netdev: network interface device structure
  1401. **/
  1402. #ifdef I40E_FCOE
  1403. void i40e_set_rx_mode(struct net_device *netdev)
  1404. #else
  1405. static void i40e_set_rx_mode(struct net_device *netdev)
  1406. #endif
  1407. {
  1408. struct i40e_netdev_priv *np = netdev_priv(netdev);
  1409. struct i40e_mac_filter *f, *ftmp;
  1410. struct i40e_vsi *vsi = np->vsi;
  1411. struct netdev_hw_addr *uca;
  1412. struct netdev_hw_addr *mca;
  1413. struct netdev_hw_addr *ha;
  1414. /* add addr if not already in the filter list */
  1415. netdev_for_each_uc_addr(uca, netdev) {
  1416. if (!i40e_find_mac(vsi, uca->addr, false, true)) {
  1417. if (i40e_is_vsi_in_vlan(vsi))
  1418. i40e_put_mac_in_vlan(vsi, uca->addr,
  1419. false, true);
  1420. else
  1421. i40e_add_filter(vsi, uca->addr, I40E_VLAN_ANY,
  1422. false, true);
  1423. }
  1424. }
  1425. netdev_for_each_mc_addr(mca, netdev) {
  1426. if (!i40e_find_mac(vsi, mca->addr, false, true)) {
  1427. if (i40e_is_vsi_in_vlan(vsi))
  1428. i40e_put_mac_in_vlan(vsi, mca->addr,
  1429. false, true);
  1430. else
  1431. i40e_add_filter(vsi, mca->addr, I40E_VLAN_ANY,
  1432. false, true);
  1433. }
  1434. }
  1435. /* remove filter if not in netdev list */
  1436. list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
  1437. bool found = false;
  1438. if (!f->is_netdev)
  1439. continue;
  1440. if (is_multicast_ether_addr(f->macaddr)) {
  1441. netdev_for_each_mc_addr(mca, netdev) {
  1442. if (ether_addr_equal(mca->addr, f->macaddr)) {
  1443. found = true;
  1444. break;
  1445. }
  1446. }
  1447. } else {
  1448. netdev_for_each_uc_addr(uca, netdev) {
  1449. if (ether_addr_equal(uca->addr, f->macaddr)) {
  1450. found = true;
  1451. break;
  1452. }
  1453. }
  1454. for_each_dev_addr(netdev, ha) {
  1455. if (ether_addr_equal(ha->addr, f->macaddr)) {
  1456. found = true;
  1457. break;
  1458. }
  1459. }
  1460. }
  1461. if (!found)
  1462. i40e_del_filter(
  1463. vsi, f->macaddr, I40E_VLAN_ANY, false, true);
  1464. }
  1465. /* check for other flag changes */
  1466. if (vsi->current_netdev_flags != vsi->netdev->flags) {
  1467. vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
  1468. vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
  1469. }
  1470. }
  1471. /**
  1472. * i40e_sync_vsi_filters - Update the VSI filter list to the HW
  1473. * @vsi: ptr to the VSI
  1474. *
  1475. * Push any outstanding VSI filter changes through the AdminQ.
  1476. *
  1477. * Returns 0 or error value
  1478. **/
  1479. int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
  1480. {
  1481. struct i40e_mac_filter *f, *ftmp;
  1482. bool promisc_forced_on = false;
  1483. bool add_happened = false;
  1484. int filter_list_len = 0;
  1485. u32 changed_flags = 0;
  1486. i40e_status aq_ret = 0;
  1487. struct i40e_pf *pf;
  1488. int num_add = 0;
  1489. int num_del = 0;
  1490. u16 cmd_flags;
  1491. /* empty array typed pointers, kcalloc later */
  1492. struct i40e_aqc_add_macvlan_element_data *add_list;
  1493. struct i40e_aqc_remove_macvlan_element_data *del_list;
  1494. while (test_and_set_bit(__I40E_CONFIG_BUSY, &vsi->state))
  1495. usleep_range(1000, 2000);
  1496. pf = vsi->back;
  1497. if (vsi->netdev) {
  1498. changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
  1499. vsi->current_netdev_flags = vsi->netdev->flags;
  1500. }
  1501. if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
  1502. vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
  1503. filter_list_len = pf->hw.aq.asq_buf_size /
  1504. sizeof(struct i40e_aqc_remove_macvlan_element_data);
  1505. del_list = kcalloc(filter_list_len,
  1506. sizeof(struct i40e_aqc_remove_macvlan_element_data),
  1507. GFP_KERNEL);
  1508. if (!del_list)
  1509. return -ENOMEM;
  1510. list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
  1511. if (!f->changed)
  1512. continue;
  1513. if (f->counter != 0)
  1514. continue;
  1515. f->changed = false;
  1516. cmd_flags = 0;
  1517. /* add to delete list */
  1518. ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
  1519. del_list[num_del].vlan_tag =
  1520. cpu_to_le16((u16)(f->vlan ==
  1521. I40E_VLAN_ANY ? 0 : f->vlan));
  1522. cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
  1523. del_list[num_del].flags = cmd_flags;
  1524. num_del++;
  1525. /* unlink from filter list */
  1526. list_del(&f->list);
  1527. kfree(f);
  1528. /* flush a full buffer */
  1529. if (num_del == filter_list_len) {
  1530. aq_ret = i40e_aq_remove_macvlan(&pf->hw,
  1531. vsi->seid, del_list, num_del,
  1532. NULL);
  1533. num_del = 0;
  1534. memset(del_list, 0, sizeof(*del_list));
  1535. if (aq_ret &&
  1536. pf->hw.aq.asq_last_status !=
  1537. I40E_AQ_RC_ENOENT)
  1538. dev_info(&pf->pdev->dev,
  1539. "ignoring delete macvlan error, err %d, aq_err %d while flushing a full buffer\n",
  1540. aq_ret,
  1541. pf->hw.aq.asq_last_status);
  1542. }
  1543. }
  1544. if (num_del) {
  1545. aq_ret = i40e_aq_remove_macvlan(&pf->hw, vsi->seid,
  1546. del_list, num_del, NULL);
  1547. num_del = 0;
  1548. if (aq_ret &&
  1549. pf->hw.aq.asq_last_status != I40E_AQ_RC_ENOENT)
  1550. dev_info(&pf->pdev->dev,
  1551. "ignoring delete macvlan error, err %d, aq_err %d\n",
  1552. aq_ret, pf->hw.aq.asq_last_status);
  1553. }
  1554. kfree(del_list);
  1555. del_list = NULL;
  1556. /* do all the adds now */
  1557. filter_list_len = pf->hw.aq.asq_buf_size /
  1558. sizeof(struct i40e_aqc_add_macvlan_element_data),
  1559. add_list = kcalloc(filter_list_len,
  1560. sizeof(struct i40e_aqc_add_macvlan_element_data),
  1561. GFP_KERNEL);
  1562. if (!add_list)
  1563. return -ENOMEM;
  1564. list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
  1565. if (!f->changed)
  1566. continue;
  1567. if (f->counter == 0)
  1568. continue;
  1569. f->changed = false;
  1570. add_happened = true;
  1571. cmd_flags = 0;
  1572. /* add to add array */
  1573. ether_addr_copy(add_list[num_add].mac_addr, f->macaddr);
  1574. add_list[num_add].vlan_tag =
  1575. cpu_to_le16(
  1576. (u16)(f->vlan == I40E_VLAN_ANY ? 0 : f->vlan));
  1577. add_list[num_add].queue_number = 0;
  1578. cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
  1579. add_list[num_add].flags = cpu_to_le16(cmd_flags);
  1580. num_add++;
  1581. /* flush a full buffer */
  1582. if (num_add == filter_list_len) {
  1583. aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
  1584. add_list, num_add,
  1585. NULL);
  1586. num_add = 0;
  1587. if (aq_ret)
  1588. break;
  1589. memset(add_list, 0, sizeof(*add_list));
  1590. }
  1591. }
  1592. if (num_add) {
  1593. aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
  1594. add_list, num_add, NULL);
  1595. num_add = 0;
  1596. }
  1597. kfree(add_list);
  1598. add_list = NULL;
  1599. if (add_happened && (!aq_ret)) {
  1600. /* do nothing */;
  1601. } else if (add_happened && (aq_ret)) {
  1602. dev_info(&pf->pdev->dev,
  1603. "add filter failed, err %d, aq_err %d\n",
  1604. aq_ret, pf->hw.aq.asq_last_status);
  1605. if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOSPC) &&
  1606. !test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
  1607. &vsi->state)) {
  1608. promisc_forced_on = true;
  1609. set_bit(__I40E_FILTER_OVERFLOW_PROMISC,
  1610. &vsi->state);
  1611. dev_info(&pf->pdev->dev, "promiscuous mode forced on\n");
  1612. }
  1613. }
  1614. }
  1615. /* check for changes in promiscuous modes */
  1616. if (changed_flags & IFF_ALLMULTI) {
  1617. bool cur_multipromisc;
  1618. cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
  1619. aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
  1620. vsi->seid,
  1621. cur_multipromisc,
  1622. NULL);
  1623. if (aq_ret)
  1624. dev_info(&pf->pdev->dev,
  1625. "set multi promisc failed, err %d, aq_err %d\n",
  1626. aq_ret, pf->hw.aq.asq_last_status);
  1627. }
  1628. if ((changed_flags & IFF_PROMISC) || promisc_forced_on) {
  1629. bool cur_promisc;
  1630. cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
  1631. test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
  1632. &vsi->state));
  1633. aq_ret = i40e_aq_set_vsi_unicast_promiscuous(&vsi->back->hw,
  1634. vsi->seid,
  1635. cur_promisc, NULL);
  1636. if (aq_ret)
  1637. dev_info(&pf->pdev->dev,
  1638. "set uni promisc failed, err %d, aq_err %d\n",
  1639. aq_ret, pf->hw.aq.asq_last_status);
  1640. aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw,
  1641. vsi->seid,
  1642. cur_promisc, NULL);
  1643. if (aq_ret)
  1644. dev_info(&pf->pdev->dev,
  1645. "set brdcast promisc failed, err %d, aq_err %d\n",
  1646. aq_ret, pf->hw.aq.asq_last_status);
  1647. }
  1648. clear_bit(__I40E_CONFIG_BUSY, &vsi->state);
  1649. return 0;
  1650. }
  1651. /**
  1652. * i40e_sync_filters_subtask - Sync the VSI filter list with HW
  1653. * @pf: board private structure
  1654. **/
  1655. static void i40e_sync_filters_subtask(struct i40e_pf *pf)
  1656. {
  1657. int v;
  1658. if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC))
  1659. return;
  1660. pf->flags &= ~I40E_FLAG_FILTER_SYNC;
  1661. for (v = 0; v < pf->num_alloc_vsi; v++) {
  1662. if (pf->vsi[v] &&
  1663. (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED))
  1664. i40e_sync_vsi_filters(pf->vsi[v]);
  1665. }
  1666. }
  1667. /**
  1668. * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
  1669. * @netdev: network interface device structure
  1670. * @new_mtu: new value for maximum frame size
  1671. *
  1672. * Returns 0 on success, negative on failure
  1673. **/
  1674. static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
  1675. {
  1676. struct i40e_netdev_priv *np = netdev_priv(netdev);
  1677. int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
  1678. struct i40e_vsi *vsi = np->vsi;
  1679. /* MTU < 68 is an error and causes problems on some kernels */
  1680. if ((new_mtu < 68) || (max_frame > I40E_MAX_RXBUFFER))
  1681. return -EINVAL;
  1682. netdev_info(netdev, "changing MTU from %d to %d\n",
  1683. netdev->mtu, new_mtu);
  1684. netdev->mtu = new_mtu;
  1685. if (netif_running(netdev))
  1686. i40e_vsi_reinit_locked(vsi);
  1687. return 0;
  1688. }
  1689. /**
  1690. * i40e_ioctl - Access the hwtstamp interface
  1691. * @netdev: network interface device structure
  1692. * @ifr: interface request data
  1693. * @cmd: ioctl command
  1694. **/
  1695. int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
  1696. {
  1697. struct i40e_netdev_priv *np = netdev_priv(netdev);
  1698. struct i40e_pf *pf = np->vsi->back;
  1699. switch (cmd) {
  1700. case SIOCGHWTSTAMP:
  1701. return i40e_ptp_get_ts_config(pf, ifr);
  1702. case SIOCSHWTSTAMP:
  1703. return i40e_ptp_set_ts_config(pf, ifr);
  1704. default:
  1705. return -EOPNOTSUPP;
  1706. }
  1707. }
  1708. /**
  1709. * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
  1710. * @vsi: the vsi being adjusted
  1711. **/
  1712. void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
  1713. {
  1714. struct i40e_vsi_context ctxt;
  1715. i40e_status ret;
  1716. if ((vsi->info.valid_sections &
  1717. cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
  1718. ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
  1719. return; /* already enabled */
  1720. vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
  1721. vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
  1722. I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
  1723. ctxt.seid = vsi->seid;
  1724. memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
  1725. ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
  1726. if (ret) {
  1727. dev_info(&vsi->back->pdev->dev,
  1728. "%s: update vsi failed, aq_err=%d\n",
  1729. __func__, vsi->back->hw.aq.asq_last_status);
  1730. }
  1731. }
  1732. /**
  1733. * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
  1734. * @vsi: the vsi being adjusted
  1735. **/
  1736. void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
  1737. {
  1738. struct i40e_vsi_context ctxt;
  1739. i40e_status ret;
  1740. if ((vsi->info.valid_sections &
  1741. cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
  1742. ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
  1743. I40E_AQ_VSI_PVLAN_EMOD_MASK))
  1744. return; /* already disabled */
  1745. vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
  1746. vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
  1747. I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
  1748. ctxt.seid = vsi->seid;
  1749. memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
  1750. ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
  1751. if (ret) {
  1752. dev_info(&vsi->back->pdev->dev,
  1753. "%s: update vsi failed, aq_err=%d\n",
  1754. __func__, vsi->back->hw.aq.asq_last_status);
  1755. }
  1756. }
  1757. /**
  1758. * i40e_vlan_rx_register - Setup or shutdown vlan offload
  1759. * @netdev: network interface to be adjusted
  1760. * @features: netdev features to test if VLAN offload is enabled or not
  1761. **/
  1762. static void i40e_vlan_rx_register(struct net_device *netdev, u32 features)
  1763. {
  1764. struct i40e_netdev_priv *np = netdev_priv(netdev);
  1765. struct i40e_vsi *vsi = np->vsi;
  1766. if (features & NETIF_F_HW_VLAN_CTAG_RX)
  1767. i40e_vlan_stripping_enable(vsi);
  1768. else
  1769. i40e_vlan_stripping_disable(vsi);
  1770. }
  1771. /**
  1772. * i40e_vsi_add_vlan - Add vsi membership for given vlan
  1773. * @vsi: the vsi being configured
  1774. * @vid: vlan id to be added (0 = untagged only , -1 = any)
  1775. **/
  1776. int i40e_vsi_add_vlan(struct i40e_vsi *vsi, s16 vid)
  1777. {
  1778. struct i40e_mac_filter *f, *add_f;
  1779. bool is_netdev, is_vf;
  1780. is_vf = (vsi->type == I40E_VSI_SRIOV);
  1781. is_netdev = !!(vsi->netdev);
  1782. if (is_netdev) {
  1783. add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, vid,
  1784. is_vf, is_netdev);
  1785. if (!add_f) {
  1786. dev_info(&vsi->back->pdev->dev,
  1787. "Could not add vlan filter %d for %pM\n",
  1788. vid, vsi->netdev->dev_addr);
  1789. return -ENOMEM;
  1790. }
  1791. }
  1792. list_for_each_entry(f, &vsi->mac_filter_list, list) {
  1793. add_f = i40e_add_filter(vsi, f->macaddr, vid, is_vf, is_netdev);
  1794. if (!add_f) {
  1795. dev_info(&vsi->back->pdev->dev,
  1796. "Could not add vlan filter %d for %pM\n",
  1797. vid, f->macaddr);
  1798. return -ENOMEM;
  1799. }
  1800. }
  1801. /* Now if we add a vlan tag, make sure to check if it is the first
  1802. * tag (i.e. a "tag" -1 does exist) and if so replace the -1 "tag"
  1803. * with 0, so we now accept untagged and specified tagged traffic
  1804. * (and not any taged and untagged)
  1805. */
  1806. if (vid > 0) {
  1807. if (is_netdev && i40e_find_filter(vsi, vsi->netdev->dev_addr,
  1808. I40E_VLAN_ANY,
  1809. is_vf, is_netdev)) {
  1810. i40e_del_filter(vsi, vsi->netdev->dev_addr,
  1811. I40E_VLAN_ANY, is_vf, is_netdev);
  1812. add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, 0,
  1813. is_vf, is_netdev);
  1814. if (!add_f) {
  1815. dev_info(&vsi->back->pdev->dev,
  1816. "Could not add filter 0 for %pM\n",
  1817. vsi->netdev->dev_addr);
  1818. return -ENOMEM;
  1819. }
  1820. }
  1821. }
  1822. /* Do not assume that I40E_VLAN_ANY should be reset to VLAN 0 */
  1823. if (vid > 0 && !vsi->info.pvid) {
  1824. list_for_each_entry(f, &vsi->mac_filter_list, list) {
  1825. if (i40e_find_filter(vsi, f->macaddr, I40E_VLAN_ANY,
  1826. is_vf, is_netdev)) {
  1827. i40e_del_filter(vsi, f->macaddr, I40E_VLAN_ANY,
  1828. is_vf, is_netdev);
  1829. add_f = i40e_add_filter(vsi, f->macaddr,
  1830. 0, is_vf, is_netdev);
  1831. if (!add_f) {
  1832. dev_info(&vsi->back->pdev->dev,
  1833. "Could not add filter 0 for %pM\n",
  1834. f->macaddr);
  1835. return -ENOMEM;
  1836. }
  1837. }
  1838. }
  1839. }
  1840. if (test_bit(__I40E_DOWN, &vsi->back->state) ||
  1841. test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
  1842. return 0;
  1843. return i40e_sync_vsi_filters(vsi);
  1844. }
  1845. /**
  1846. * i40e_vsi_kill_vlan - Remove vsi membership for given vlan
  1847. * @vsi: the vsi being configured
  1848. * @vid: vlan id to be removed (0 = untagged only , -1 = any)
  1849. *
  1850. * Return: 0 on success or negative otherwise
  1851. **/
  1852. int i40e_vsi_kill_vlan(struct i40e_vsi *vsi, s16 vid)
  1853. {
  1854. struct net_device *netdev = vsi->netdev;
  1855. struct i40e_mac_filter *f, *add_f;
  1856. bool is_vf, is_netdev;
  1857. int filter_count = 0;
  1858. is_vf = (vsi->type == I40E_VSI_SRIOV);
  1859. is_netdev = !!(netdev);
  1860. if (is_netdev)
  1861. i40e_del_filter(vsi, netdev->dev_addr, vid, is_vf, is_netdev);
  1862. list_for_each_entry(f, &vsi->mac_filter_list, list)
  1863. i40e_del_filter(vsi, f->macaddr, vid, is_vf, is_netdev);
  1864. /* go through all the filters for this VSI and if there is only
  1865. * vid == 0 it means there are no other filters, so vid 0 must
  1866. * be replaced with -1. This signifies that we should from now
  1867. * on accept any traffic (with any tag present, or untagged)
  1868. */
  1869. list_for_each_entry(f, &vsi->mac_filter_list, list) {
  1870. if (is_netdev) {
  1871. if (f->vlan &&
  1872. ether_addr_equal(netdev->dev_addr, f->macaddr))
  1873. filter_count++;
  1874. }
  1875. if (f->vlan)
  1876. filter_count++;
  1877. }
  1878. if (!filter_count && is_netdev) {
  1879. i40e_del_filter(vsi, netdev->dev_addr, 0, is_vf, is_netdev);
  1880. f = i40e_add_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY,
  1881. is_vf, is_netdev);
  1882. if (!f) {
  1883. dev_info(&vsi->back->pdev->dev,
  1884. "Could not add filter %d for %pM\n",
  1885. I40E_VLAN_ANY, netdev->dev_addr);
  1886. return -ENOMEM;
  1887. }
  1888. }
  1889. if (!filter_count) {
  1890. list_for_each_entry(f, &vsi->mac_filter_list, list) {
  1891. i40e_del_filter(vsi, f->macaddr, 0, is_vf, is_netdev);
  1892. add_f = i40e_add_filter(vsi, f->macaddr, I40E_VLAN_ANY,
  1893. is_vf, is_netdev);
  1894. if (!add_f) {
  1895. dev_info(&vsi->back->pdev->dev,
  1896. "Could not add filter %d for %pM\n",
  1897. I40E_VLAN_ANY, f->macaddr);
  1898. return -ENOMEM;
  1899. }
  1900. }
  1901. }
  1902. if (test_bit(__I40E_DOWN, &vsi->back->state) ||
  1903. test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
  1904. return 0;
  1905. return i40e_sync_vsi_filters(vsi);
  1906. }
  1907. /**
  1908. * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
  1909. * @netdev: network interface to be adjusted
  1910. * @vid: vlan id to be added
  1911. *
  1912. * net_device_ops implementation for adding vlan ids
  1913. **/
  1914. #ifdef I40E_FCOE
  1915. int i40e_vlan_rx_add_vid(struct net_device *netdev,
  1916. __always_unused __be16 proto, u16 vid)
  1917. #else
  1918. static int i40e_vlan_rx_add_vid(struct net_device *netdev,
  1919. __always_unused __be16 proto, u16 vid)
  1920. #endif
  1921. {
  1922. struct i40e_netdev_priv *np = netdev_priv(netdev);
  1923. struct i40e_vsi *vsi = np->vsi;
  1924. int ret = 0;
  1925. if (vid > 4095)
  1926. return -EINVAL;
  1927. netdev_info(netdev, "adding %pM vid=%d\n", netdev->dev_addr, vid);
  1928. /* If the network stack called us with vid = 0 then
  1929. * it is asking to receive priority tagged packets with
  1930. * vlan id 0. Our HW receives them by default when configured
  1931. * to receive untagged packets so there is no need to add an
  1932. * extra filter for vlan 0 tagged packets.
  1933. */
  1934. if (vid)
  1935. ret = i40e_vsi_add_vlan(vsi, vid);
  1936. if (!ret && (vid < VLAN_N_VID))
  1937. set_bit(vid, vsi->active_vlans);
  1938. return ret;
  1939. }
  1940. /**
  1941. * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
  1942. * @netdev: network interface to be adjusted
  1943. * @vid: vlan id to be removed
  1944. *
  1945. * net_device_ops implementation for removing vlan ids
  1946. **/
  1947. #ifdef I40E_FCOE
  1948. int i40e_vlan_rx_kill_vid(struct net_device *netdev,
  1949. __always_unused __be16 proto, u16 vid)
  1950. #else
  1951. static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
  1952. __always_unused __be16 proto, u16 vid)
  1953. #endif
  1954. {
  1955. struct i40e_netdev_priv *np = netdev_priv(netdev);
  1956. struct i40e_vsi *vsi = np->vsi;
  1957. netdev_info(netdev, "removing %pM vid=%d\n", netdev->dev_addr, vid);
  1958. /* return code is ignored as there is nothing a user
  1959. * can do about failure to remove and a log message was
  1960. * already printed from the other function
  1961. */
  1962. i40e_vsi_kill_vlan(vsi, vid);
  1963. clear_bit(vid, vsi->active_vlans);
  1964. return 0;
  1965. }
  1966. /**
  1967. * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
  1968. * @vsi: the vsi being brought back up
  1969. **/
  1970. static void i40e_restore_vlan(struct i40e_vsi *vsi)
  1971. {
  1972. u16 vid;
  1973. if (!vsi->netdev)
  1974. return;
  1975. i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features);
  1976. for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
  1977. i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q),
  1978. vid);
  1979. }
  1980. /**
  1981. * i40e_vsi_add_pvid - Add pvid for the VSI
  1982. * @vsi: the vsi being adjusted
  1983. * @vid: the vlan id to set as a PVID
  1984. **/
  1985. int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
  1986. {
  1987. struct i40e_vsi_context ctxt;
  1988. i40e_status aq_ret;
  1989. vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
  1990. vsi->info.pvid = cpu_to_le16(vid);
  1991. vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
  1992. I40E_AQ_VSI_PVLAN_INSERT_PVID |
  1993. I40E_AQ_VSI_PVLAN_EMOD_STR;
  1994. ctxt.seid = vsi->seid;
  1995. memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
  1996. aq_ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
  1997. if (aq_ret) {
  1998. dev_info(&vsi->back->pdev->dev,
  1999. "%s: update vsi failed, aq_err=%d\n",
  2000. __func__, vsi->back->hw.aq.asq_last_status);
  2001. return -ENOENT;
  2002. }
  2003. return 0;
  2004. }
  2005. /**
  2006. * i40e_vsi_remove_pvid - Remove the pvid from the VSI
  2007. * @vsi: the vsi being adjusted
  2008. *
  2009. * Just use the vlan_rx_register() service to put it back to normal
  2010. **/
  2011. void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
  2012. {
  2013. i40e_vlan_stripping_disable(vsi);
  2014. vsi->info.pvid = 0;
  2015. }
  2016. /**
  2017. * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
  2018. * @vsi: ptr to the VSI
  2019. *
  2020. * If this function returns with an error, then it's possible one or
  2021. * more of the rings is populated (while the rest are not). It is the
  2022. * callers duty to clean those orphaned rings.
  2023. *
  2024. * Return 0 on success, negative on failure
  2025. **/
  2026. static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
  2027. {
  2028. int i, err = 0;
  2029. for (i = 0; i < vsi->num_queue_pairs && !err; i++)
  2030. err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
  2031. return err;
  2032. }
  2033. /**
  2034. * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
  2035. * @vsi: ptr to the VSI
  2036. *
  2037. * Free VSI's transmit software resources
  2038. **/
  2039. static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
  2040. {
  2041. int i;
  2042. if (!vsi->tx_rings)
  2043. return;
  2044. for (i = 0; i < vsi->num_queue_pairs; i++)
  2045. if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
  2046. i40e_free_tx_resources(vsi->tx_rings[i]);
  2047. }
  2048. /**
  2049. * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
  2050. * @vsi: ptr to the VSI
  2051. *
  2052. * If this function returns with an error, then it's possible one or
  2053. * more of the rings is populated (while the rest are not). It is the
  2054. * callers duty to clean those orphaned rings.
  2055. *
  2056. * Return 0 on success, negative on failure
  2057. **/
  2058. static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
  2059. {
  2060. int i, err = 0;
  2061. for (i = 0; i < vsi->num_queue_pairs && !err; i++)
  2062. err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
  2063. #ifdef I40E_FCOE
  2064. i40e_fcoe_setup_ddp_resources(vsi);
  2065. #endif
  2066. return err;
  2067. }
  2068. /**
  2069. * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
  2070. * @vsi: ptr to the VSI
  2071. *
  2072. * Free all receive software resources
  2073. **/
  2074. static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
  2075. {
  2076. int i;
  2077. if (!vsi->rx_rings)
  2078. return;
  2079. for (i = 0; i < vsi->num_queue_pairs; i++)
  2080. if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
  2081. i40e_free_rx_resources(vsi->rx_rings[i]);
  2082. #ifdef I40E_FCOE
  2083. i40e_fcoe_free_ddp_resources(vsi);
  2084. #endif
  2085. }
  2086. /**
  2087. * i40e_configure_tx_ring - Configure a transmit ring context and rest
  2088. * @ring: The Tx ring to configure
  2089. *
  2090. * Configure the Tx descriptor ring in the HMC context.
  2091. **/
  2092. static int i40e_configure_tx_ring(struct i40e_ring *ring)
  2093. {
  2094. struct i40e_vsi *vsi = ring->vsi;
  2095. u16 pf_q = vsi->base_queue + ring->queue_index;
  2096. struct i40e_hw *hw = &vsi->back->hw;
  2097. struct i40e_hmc_obj_txq tx_ctx;
  2098. i40e_status err = 0;
  2099. u32 qtx_ctl = 0;
  2100. /* some ATR related tx ring init */
  2101. if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
  2102. ring->atr_sample_rate = vsi->back->atr_sample_rate;
  2103. ring->atr_count = 0;
  2104. } else {
  2105. ring->atr_sample_rate = 0;
  2106. }
  2107. /* initialize XPS */
  2108. if (ring->q_vector && ring->netdev &&
  2109. vsi->tc_config.numtc <= 1 &&
  2110. !test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state))
  2111. netif_set_xps_queue(ring->netdev,
  2112. &ring->q_vector->affinity_mask,
  2113. ring->queue_index);
  2114. /* clear the context structure first */
  2115. memset(&tx_ctx, 0, sizeof(tx_ctx));
  2116. tx_ctx.new_context = 1;
  2117. tx_ctx.base = (ring->dma / 128);
  2118. tx_ctx.qlen = ring->count;
  2119. tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
  2120. I40E_FLAG_FD_ATR_ENABLED));
  2121. #ifdef I40E_FCOE
  2122. tx_ctx.fc_ena = (vsi->type == I40E_VSI_FCOE);
  2123. #endif
  2124. tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
  2125. /* FDIR VSI tx ring can still use RS bit and writebacks */
  2126. if (vsi->type != I40E_VSI_FDIR)
  2127. tx_ctx.head_wb_ena = 1;
  2128. tx_ctx.head_wb_addr = ring->dma +
  2129. (ring->count * sizeof(struct i40e_tx_desc));
  2130. /* As part of VSI creation/update, FW allocates certain
  2131. * Tx arbitration queue sets for each TC enabled for
  2132. * the VSI. The FW returns the handles to these queue
  2133. * sets as part of the response buffer to Add VSI,
  2134. * Update VSI, etc. AQ commands. It is expected that
  2135. * these queue set handles be associated with the Tx
  2136. * queues by the driver as part of the TX queue context
  2137. * initialization. This has to be done regardless of
  2138. * DCB as by default everything is mapped to TC0.
  2139. */
  2140. tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
  2141. tx_ctx.rdylist_act = 0;
  2142. /* clear the context in the HMC */
  2143. err = i40e_clear_lan_tx_queue_context(hw, pf_q);
  2144. if (err) {
  2145. dev_info(&vsi->back->pdev->dev,
  2146. "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
  2147. ring->queue_index, pf_q, err);
  2148. return -ENOMEM;
  2149. }
  2150. /* set the context in the HMC */
  2151. err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
  2152. if (err) {
  2153. dev_info(&vsi->back->pdev->dev,
  2154. "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
  2155. ring->queue_index, pf_q, err);
  2156. return -ENOMEM;
  2157. }
  2158. /* Now associate this queue with this PCI function */
  2159. if (vsi->type == I40E_VSI_VMDQ2)
  2160. qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
  2161. else
  2162. qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
  2163. qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
  2164. I40E_QTX_CTL_PF_INDX_MASK);
  2165. wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
  2166. i40e_flush(hw);
  2167. clear_bit(__I40E_HANG_CHECK_ARMED, &ring->state);
  2168. /* cache tail off for easier writes later */
  2169. ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
  2170. return 0;
  2171. }
  2172. /**
  2173. * i40e_configure_rx_ring - Configure a receive ring context
  2174. * @ring: The Rx ring to configure
  2175. *
  2176. * Configure the Rx descriptor ring in the HMC context.
  2177. **/
  2178. static int i40e_configure_rx_ring(struct i40e_ring *ring)
  2179. {
  2180. struct i40e_vsi *vsi = ring->vsi;
  2181. u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
  2182. u16 pf_q = vsi->base_queue + ring->queue_index;
  2183. struct i40e_hw *hw = &vsi->back->hw;
  2184. struct i40e_hmc_obj_rxq rx_ctx;
  2185. i40e_status err = 0;
  2186. ring->state = 0;
  2187. /* clear the context structure first */
  2188. memset(&rx_ctx, 0, sizeof(rx_ctx));
  2189. ring->rx_buf_len = vsi->rx_buf_len;
  2190. ring->rx_hdr_len = vsi->rx_hdr_len;
  2191. rx_ctx.dbuff = ring->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT;
  2192. rx_ctx.hbuff = ring->rx_hdr_len >> I40E_RXQ_CTX_HBUFF_SHIFT;
  2193. rx_ctx.base = (ring->dma / 128);
  2194. rx_ctx.qlen = ring->count;
  2195. if (vsi->back->flags & I40E_FLAG_16BYTE_RX_DESC_ENABLED) {
  2196. set_ring_16byte_desc_enabled(ring);
  2197. rx_ctx.dsize = 0;
  2198. } else {
  2199. rx_ctx.dsize = 1;
  2200. }
  2201. rx_ctx.dtype = vsi->dtype;
  2202. if (vsi->dtype) {
  2203. set_ring_ps_enabled(ring);
  2204. rx_ctx.hsplit_0 = I40E_RX_SPLIT_L2 |
  2205. I40E_RX_SPLIT_IP |
  2206. I40E_RX_SPLIT_TCP_UDP |
  2207. I40E_RX_SPLIT_SCTP;
  2208. } else {
  2209. rx_ctx.hsplit_0 = 0;
  2210. }
  2211. rx_ctx.rxmax = min_t(u16, vsi->max_frame,
  2212. (chain_len * ring->rx_buf_len));
  2213. if (hw->revision_id == 0)
  2214. rx_ctx.lrxqthresh = 0;
  2215. else
  2216. rx_ctx.lrxqthresh = 2;
  2217. rx_ctx.crcstrip = 1;
  2218. rx_ctx.l2tsel = 1;
  2219. rx_ctx.showiv = 1;
  2220. #ifdef I40E_FCOE
  2221. rx_ctx.fc_ena = (vsi->type == I40E_VSI_FCOE);
  2222. #endif
  2223. /* set the prefena field to 1 because the manual says to */
  2224. rx_ctx.prefena = 1;
  2225. /* clear the context in the HMC */
  2226. err = i40e_clear_lan_rx_queue_context(hw, pf_q);
  2227. if (err) {
  2228. dev_info(&vsi->back->pdev->dev,
  2229. "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
  2230. ring->queue_index, pf_q, err);
  2231. return -ENOMEM;
  2232. }
  2233. /* set the context in the HMC */
  2234. err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
  2235. if (err) {
  2236. dev_info(&vsi->back->pdev->dev,
  2237. "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
  2238. ring->queue_index, pf_q, err);
  2239. return -ENOMEM;
  2240. }
  2241. /* cache tail for quicker writes, and clear the reg before use */
  2242. ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
  2243. writel(0, ring->tail);
  2244. i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
  2245. return 0;
  2246. }
  2247. /**
  2248. * i40e_vsi_configure_tx - Configure the VSI for Tx
  2249. * @vsi: VSI structure describing this set of rings and resources
  2250. *
  2251. * Configure the Tx VSI for operation.
  2252. **/
  2253. static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
  2254. {
  2255. int err = 0;
  2256. u16 i;
  2257. for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
  2258. err = i40e_configure_tx_ring(vsi->tx_rings[i]);
  2259. return err;
  2260. }
  2261. /**
  2262. * i40e_vsi_configure_rx - Configure the VSI for Rx
  2263. * @vsi: the VSI being configured
  2264. *
  2265. * Configure the Rx VSI for operation.
  2266. **/
  2267. static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
  2268. {
  2269. int err = 0;
  2270. u16 i;
  2271. if (vsi->netdev && (vsi->netdev->mtu > ETH_DATA_LEN))
  2272. vsi->max_frame = vsi->netdev->mtu + ETH_HLEN
  2273. + ETH_FCS_LEN + VLAN_HLEN;
  2274. else
  2275. vsi->max_frame = I40E_RXBUFFER_2048;
  2276. /* figure out correct receive buffer length */
  2277. switch (vsi->back->flags & (I40E_FLAG_RX_1BUF_ENABLED |
  2278. I40E_FLAG_RX_PS_ENABLED)) {
  2279. case I40E_FLAG_RX_1BUF_ENABLED:
  2280. vsi->rx_hdr_len = 0;
  2281. vsi->rx_buf_len = vsi->max_frame;
  2282. vsi->dtype = I40E_RX_DTYPE_NO_SPLIT;
  2283. break;
  2284. case I40E_FLAG_RX_PS_ENABLED:
  2285. vsi->rx_hdr_len = I40E_RX_HDR_SIZE;
  2286. vsi->rx_buf_len = I40E_RXBUFFER_2048;
  2287. vsi->dtype = I40E_RX_DTYPE_HEADER_SPLIT;
  2288. break;
  2289. default:
  2290. vsi->rx_hdr_len = I40E_RX_HDR_SIZE;
  2291. vsi->rx_buf_len = I40E_RXBUFFER_2048;
  2292. vsi->dtype = I40E_RX_DTYPE_SPLIT_ALWAYS;
  2293. break;
  2294. }
  2295. #ifdef I40E_FCOE
  2296. /* setup rx buffer for FCoE */
  2297. if ((vsi->type == I40E_VSI_FCOE) &&
  2298. (vsi->back->flags & I40E_FLAG_FCOE_ENABLED)) {
  2299. vsi->rx_hdr_len = 0;
  2300. vsi->rx_buf_len = I40E_RXBUFFER_3072;
  2301. vsi->max_frame = I40E_RXBUFFER_3072;
  2302. vsi->dtype = I40E_RX_DTYPE_NO_SPLIT;
  2303. }
  2304. #endif /* I40E_FCOE */
  2305. /* round up for the chip's needs */
  2306. vsi->rx_hdr_len = ALIGN(vsi->rx_hdr_len,
  2307. (1 << I40E_RXQ_CTX_HBUFF_SHIFT));
  2308. vsi->rx_buf_len = ALIGN(vsi->rx_buf_len,
  2309. (1 << I40E_RXQ_CTX_DBUFF_SHIFT));
  2310. /* set up individual rings */
  2311. for (i = 0; i < vsi->num_queue_pairs && !err; i++)
  2312. err = i40e_configure_rx_ring(vsi->rx_rings[i]);
  2313. return err;
  2314. }
  2315. /**
  2316. * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
  2317. * @vsi: ptr to the VSI
  2318. **/
  2319. static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
  2320. {
  2321. struct i40e_ring *tx_ring, *rx_ring;
  2322. u16 qoffset, qcount;
  2323. int i, n;
  2324. if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED))
  2325. return;
  2326. for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
  2327. if (!(vsi->tc_config.enabled_tc & (1 << n)))
  2328. continue;
  2329. qoffset = vsi->tc_config.tc_info[n].qoffset;
  2330. qcount = vsi->tc_config.tc_info[n].qcount;
  2331. for (i = qoffset; i < (qoffset + qcount); i++) {
  2332. rx_ring = vsi->rx_rings[i];
  2333. tx_ring = vsi->tx_rings[i];
  2334. rx_ring->dcb_tc = n;
  2335. tx_ring->dcb_tc = n;
  2336. }
  2337. }
  2338. }
  2339. /**
  2340. * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
  2341. * @vsi: ptr to the VSI
  2342. **/
  2343. static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
  2344. {
  2345. if (vsi->netdev)
  2346. i40e_set_rx_mode(vsi->netdev);
  2347. }
  2348. /**
  2349. * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
  2350. * @vsi: Pointer to the targeted VSI
  2351. *
  2352. * This function replays the hlist on the hw where all the SB Flow Director
  2353. * filters were saved.
  2354. **/
  2355. static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
  2356. {
  2357. struct i40e_fdir_filter *filter;
  2358. struct i40e_pf *pf = vsi->back;
  2359. struct hlist_node *node;
  2360. if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
  2361. return;
  2362. hlist_for_each_entry_safe(filter, node,
  2363. &pf->fdir_filter_list, fdir_node) {
  2364. i40e_add_del_fdir(vsi, filter, true);
  2365. }
  2366. }
  2367. /**
  2368. * i40e_vsi_configure - Set up the VSI for action
  2369. * @vsi: the VSI being configured
  2370. **/
  2371. static int i40e_vsi_configure(struct i40e_vsi *vsi)
  2372. {
  2373. int err;
  2374. i40e_set_vsi_rx_mode(vsi);
  2375. i40e_restore_vlan(vsi);
  2376. i40e_vsi_config_dcb_rings(vsi);
  2377. err = i40e_vsi_configure_tx(vsi);
  2378. if (!err)
  2379. err = i40e_vsi_configure_rx(vsi);
  2380. return err;
  2381. }
  2382. /**
  2383. * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
  2384. * @vsi: the VSI being configured
  2385. **/
  2386. static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
  2387. {
  2388. struct i40e_pf *pf = vsi->back;
  2389. struct i40e_q_vector *q_vector;
  2390. struct i40e_hw *hw = &pf->hw;
  2391. u16 vector;
  2392. int i, q;
  2393. u32 val;
  2394. u32 qp;
  2395. /* The interrupt indexing is offset by 1 in the PFINT_ITRn
  2396. * and PFINT_LNKLSTn registers, e.g.:
  2397. * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts)
  2398. */
  2399. qp = vsi->base_queue;
  2400. vector = vsi->base_vector;
  2401. for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
  2402. q_vector = vsi->q_vectors[i];
  2403. q_vector->rx.itr = ITR_TO_REG(vsi->rx_itr_setting);
  2404. q_vector->rx.latency_range = I40E_LOW_LATENCY;
  2405. wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
  2406. q_vector->rx.itr);
  2407. q_vector->tx.itr = ITR_TO_REG(vsi->tx_itr_setting);
  2408. q_vector->tx.latency_range = I40E_LOW_LATENCY;
  2409. wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
  2410. q_vector->tx.itr);
  2411. /* Linked list for the queuepairs assigned to this vector */
  2412. wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
  2413. for (q = 0; q < q_vector->num_ringpairs; q++) {
  2414. val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
  2415. (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
  2416. (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
  2417. (qp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
  2418. (I40E_QUEUE_TYPE_TX
  2419. << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
  2420. wr32(hw, I40E_QINT_RQCTL(qp), val);
  2421. val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
  2422. (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
  2423. (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
  2424. ((qp+1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT)|
  2425. (I40E_QUEUE_TYPE_RX
  2426. << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
  2427. /* Terminate the linked list */
  2428. if (q == (q_vector->num_ringpairs - 1))
  2429. val |= (I40E_QUEUE_END_OF_LIST
  2430. << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
  2431. wr32(hw, I40E_QINT_TQCTL(qp), val);
  2432. qp++;
  2433. }
  2434. }
  2435. i40e_flush(hw);
  2436. }
  2437. /**
  2438. * i40e_enable_misc_int_causes - enable the non-queue interrupts
  2439. * @hw: ptr to the hardware info
  2440. **/
  2441. static void i40e_enable_misc_int_causes(struct i40e_hw *hw)
  2442. {
  2443. u32 val;
  2444. /* clear things first */
  2445. wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */
  2446. rd32(hw, I40E_PFINT_ICR0); /* read to clear */
  2447. val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
  2448. I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
  2449. I40E_PFINT_ICR0_ENA_GRST_MASK |
  2450. I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
  2451. I40E_PFINT_ICR0_ENA_GPIO_MASK |
  2452. I40E_PFINT_ICR0_ENA_TIMESYNC_MASK |
  2453. I40E_PFINT_ICR0_ENA_HMC_ERR_MASK |
  2454. I40E_PFINT_ICR0_ENA_VFLR_MASK |
  2455. I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
  2456. wr32(hw, I40E_PFINT_ICR0_ENA, val);
  2457. /* SW_ITR_IDX = 0, but don't change INTENA */
  2458. wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
  2459. I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
  2460. /* OTHER_ITR_IDX = 0 */
  2461. wr32(hw, I40E_PFINT_STAT_CTL0, 0);
  2462. }
  2463. /**
  2464. * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
  2465. * @vsi: the VSI being configured
  2466. **/
  2467. static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
  2468. {
  2469. struct i40e_q_vector *q_vector = vsi->q_vectors[0];
  2470. struct i40e_pf *pf = vsi->back;
  2471. struct i40e_hw *hw = &pf->hw;
  2472. u32 val;
  2473. /* set the ITR configuration */
  2474. q_vector->rx.itr = ITR_TO_REG(vsi->rx_itr_setting);
  2475. q_vector->rx.latency_range = I40E_LOW_LATENCY;
  2476. wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr);
  2477. q_vector->tx.itr = ITR_TO_REG(vsi->tx_itr_setting);
  2478. q_vector->tx.latency_range = I40E_LOW_LATENCY;
  2479. wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr);
  2480. i40e_enable_misc_int_causes(hw);
  2481. /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
  2482. wr32(hw, I40E_PFINT_LNKLST0, 0);
  2483. /* Associate the queue pair to the vector and enable the queue int */
  2484. val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
  2485. (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
  2486. (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
  2487. wr32(hw, I40E_QINT_RQCTL(0), val);
  2488. val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
  2489. (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
  2490. (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
  2491. wr32(hw, I40E_QINT_TQCTL(0), val);
  2492. i40e_flush(hw);
  2493. }
  2494. /**
  2495. * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
  2496. * @pf: board private structure
  2497. **/
  2498. void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
  2499. {
  2500. struct i40e_hw *hw = &pf->hw;
  2501. wr32(hw, I40E_PFINT_DYN_CTL0,
  2502. I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
  2503. i40e_flush(hw);
  2504. }
  2505. /**
  2506. * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
  2507. * @pf: board private structure
  2508. **/
  2509. void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
  2510. {
  2511. struct i40e_hw *hw = &pf->hw;
  2512. u32 val;
  2513. val = I40E_PFINT_DYN_CTL0_INTENA_MASK |
  2514. I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
  2515. (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
  2516. wr32(hw, I40E_PFINT_DYN_CTL0, val);
  2517. i40e_flush(hw);
  2518. }
  2519. /**
  2520. * i40e_irq_dynamic_enable - Enable default interrupt generation settings
  2521. * @vsi: pointer to a vsi
  2522. * @vector: enable a particular Hw Interrupt vector
  2523. **/
  2524. void i40e_irq_dynamic_enable(struct i40e_vsi *vsi, int vector)
  2525. {
  2526. struct i40e_pf *pf = vsi->back;
  2527. struct i40e_hw *hw = &pf->hw;
  2528. u32 val;
  2529. val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
  2530. I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
  2531. (I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
  2532. wr32(hw, I40E_PFINT_DYN_CTLN(vector - 1), val);
  2533. /* skip the flush */
  2534. }
  2535. /**
  2536. * i40e_irq_dynamic_disable - Disable default interrupt generation settings
  2537. * @vsi: pointer to a vsi
  2538. * @vector: enable a particular Hw Interrupt vector
  2539. **/
  2540. void i40e_irq_dynamic_disable(struct i40e_vsi *vsi, int vector)
  2541. {
  2542. struct i40e_pf *pf = vsi->back;
  2543. struct i40e_hw *hw = &pf->hw;
  2544. u32 val;
  2545. val = I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT;
  2546. wr32(hw, I40E_PFINT_DYN_CTLN(vector - 1), val);
  2547. i40e_flush(hw);
  2548. }
  2549. /**
  2550. * i40e_msix_clean_rings - MSIX mode Interrupt Handler
  2551. * @irq: interrupt number
  2552. * @data: pointer to a q_vector
  2553. **/
  2554. static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
  2555. {
  2556. struct i40e_q_vector *q_vector = data;
  2557. if (!q_vector->tx.ring && !q_vector->rx.ring)
  2558. return IRQ_HANDLED;
  2559. napi_schedule(&q_vector->napi);
  2560. return IRQ_HANDLED;
  2561. }
  2562. /**
  2563. * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
  2564. * @vsi: the VSI being configured
  2565. * @basename: name for the vector
  2566. *
  2567. * Allocates MSI-X vectors and requests interrupts from the kernel.
  2568. **/
  2569. static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
  2570. {
  2571. int q_vectors = vsi->num_q_vectors;
  2572. struct i40e_pf *pf = vsi->back;
  2573. int base = vsi->base_vector;
  2574. int rx_int_idx = 0;
  2575. int tx_int_idx = 0;
  2576. int vector, err;
  2577. for (vector = 0; vector < q_vectors; vector++) {
  2578. struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
  2579. if (q_vector->tx.ring && q_vector->rx.ring) {
  2580. snprintf(q_vector->name, sizeof(q_vector->name) - 1,
  2581. "%s-%s-%d", basename, "TxRx", rx_int_idx++);
  2582. tx_int_idx++;
  2583. } else if (q_vector->rx.ring) {
  2584. snprintf(q_vector->name, sizeof(q_vector->name) - 1,
  2585. "%s-%s-%d", basename, "rx", rx_int_idx++);
  2586. } else if (q_vector->tx.ring) {
  2587. snprintf(q_vector->name, sizeof(q_vector->name) - 1,
  2588. "%s-%s-%d", basename, "tx", tx_int_idx++);
  2589. } else {
  2590. /* skip this unused q_vector */
  2591. continue;
  2592. }
  2593. err = request_irq(pf->msix_entries[base + vector].vector,
  2594. vsi->irq_handler,
  2595. 0,
  2596. q_vector->name,
  2597. q_vector);
  2598. if (err) {
  2599. dev_info(&pf->pdev->dev,
  2600. "%s: request_irq failed, error: %d\n",
  2601. __func__, err);
  2602. goto free_queue_irqs;
  2603. }
  2604. /* assign the mask for this irq */
  2605. irq_set_affinity_hint(pf->msix_entries[base + vector].vector,
  2606. &q_vector->affinity_mask);
  2607. }
  2608. vsi->irqs_ready = true;
  2609. return 0;
  2610. free_queue_irqs:
  2611. while (vector) {
  2612. vector--;
  2613. irq_set_affinity_hint(pf->msix_entries[base + vector].vector,
  2614. NULL);
  2615. free_irq(pf->msix_entries[base + vector].vector,
  2616. &(vsi->q_vectors[vector]));
  2617. }
  2618. return err;
  2619. }
  2620. /**
  2621. * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
  2622. * @vsi: the VSI being un-configured
  2623. **/
  2624. static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
  2625. {
  2626. struct i40e_pf *pf = vsi->back;
  2627. struct i40e_hw *hw = &pf->hw;
  2628. int base = vsi->base_vector;
  2629. int i;
  2630. for (i = 0; i < vsi->num_queue_pairs; i++) {
  2631. wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), 0);
  2632. wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), 0);
  2633. }
  2634. if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
  2635. for (i = vsi->base_vector;
  2636. i < (vsi->num_q_vectors + vsi->base_vector); i++)
  2637. wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
  2638. i40e_flush(hw);
  2639. for (i = 0; i < vsi->num_q_vectors; i++)
  2640. synchronize_irq(pf->msix_entries[i + base].vector);
  2641. } else {
  2642. /* Legacy and MSI mode - this stops all interrupt handling */
  2643. wr32(hw, I40E_PFINT_ICR0_ENA, 0);
  2644. wr32(hw, I40E_PFINT_DYN_CTL0, 0);
  2645. i40e_flush(hw);
  2646. synchronize_irq(pf->pdev->irq);
  2647. }
  2648. }
  2649. /**
  2650. * i40e_vsi_enable_irq - Enable IRQ for the given VSI
  2651. * @vsi: the VSI being configured
  2652. **/
  2653. static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
  2654. {
  2655. struct i40e_pf *pf = vsi->back;
  2656. int i;
  2657. if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
  2658. for (i = vsi->base_vector;
  2659. i < (vsi->num_q_vectors + vsi->base_vector); i++)
  2660. i40e_irq_dynamic_enable(vsi, i);
  2661. } else {
  2662. i40e_irq_dynamic_enable_icr0(pf);
  2663. }
  2664. i40e_flush(&pf->hw);
  2665. return 0;
  2666. }
  2667. /**
  2668. * i40e_stop_misc_vector - Stop the vector that handles non-queue events
  2669. * @pf: board private structure
  2670. **/
  2671. static void i40e_stop_misc_vector(struct i40e_pf *pf)
  2672. {
  2673. /* Disable ICR 0 */
  2674. wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
  2675. i40e_flush(&pf->hw);
  2676. }
  2677. /**
  2678. * i40e_intr - MSI/Legacy and non-queue interrupt handler
  2679. * @irq: interrupt number
  2680. * @data: pointer to a q_vector
  2681. *
  2682. * This is the handler used for all MSI/Legacy interrupts, and deals
  2683. * with both queue and non-queue interrupts. This is also used in
  2684. * MSIX mode to handle the non-queue interrupts.
  2685. **/
  2686. static irqreturn_t i40e_intr(int irq, void *data)
  2687. {
  2688. struct i40e_pf *pf = (struct i40e_pf *)data;
  2689. struct i40e_hw *hw = &pf->hw;
  2690. irqreturn_t ret = IRQ_NONE;
  2691. u32 icr0, icr0_remaining;
  2692. u32 val, ena_mask;
  2693. icr0 = rd32(hw, I40E_PFINT_ICR0);
  2694. ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
  2695. /* if sharing a legacy IRQ, we might get called w/o an intr pending */
  2696. if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
  2697. goto enable_intr;
  2698. /* if interrupt but no bits showing, must be SWINT */
  2699. if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
  2700. (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
  2701. pf->sw_int_count++;
  2702. /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
  2703. if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
  2704. /* temporarily disable queue cause for NAPI processing */
  2705. u32 qval = rd32(hw, I40E_QINT_RQCTL(0));
  2706. qval &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
  2707. wr32(hw, I40E_QINT_RQCTL(0), qval);
  2708. qval = rd32(hw, I40E_QINT_TQCTL(0));
  2709. qval &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
  2710. wr32(hw, I40E_QINT_TQCTL(0), qval);
  2711. if (!test_bit(__I40E_DOWN, &pf->state))
  2712. napi_schedule(&pf->vsi[pf->lan_vsi]->q_vectors[0]->napi);
  2713. }
  2714. if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
  2715. ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
  2716. set_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
  2717. }
  2718. if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
  2719. ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
  2720. set_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
  2721. }
  2722. if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
  2723. ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
  2724. set_bit(__I40E_VFLR_EVENT_PENDING, &pf->state);
  2725. }
  2726. if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
  2727. if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
  2728. set_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
  2729. ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
  2730. val = rd32(hw, I40E_GLGEN_RSTAT);
  2731. val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
  2732. >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
  2733. if (val == I40E_RESET_CORER) {
  2734. pf->corer_count++;
  2735. } else if (val == I40E_RESET_GLOBR) {
  2736. pf->globr_count++;
  2737. } else if (val == I40E_RESET_EMPR) {
  2738. pf->empr_count++;
  2739. set_bit(__I40E_EMP_RESET_REQUESTED, &pf->state);
  2740. }
  2741. }
  2742. if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
  2743. icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
  2744. dev_info(&pf->pdev->dev, "HMC error interrupt\n");
  2745. }
  2746. if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
  2747. u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
  2748. if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
  2749. icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
  2750. i40e_ptp_tx_hwtstamp(pf);
  2751. }
  2752. }
  2753. /* If a critical error is pending we have no choice but to reset the
  2754. * device.
  2755. * Report and mask out any remaining unexpected interrupts.
  2756. */
  2757. icr0_remaining = icr0 & ena_mask;
  2758. if (icr0_remaining) {
  2759. dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
  2760. icr0_remaining);
  2761. if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
  2762. (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
  2763. (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
  2764. dev_info(&pf->pdev->dev, "device will be reset\n");
  2765. set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
  2766. i40e_service_event_schedule(pf);
  2767. }
  2768. ena_mask &= ~icr0_remaining;
  2769. }
  2770. ret = IRQ_HANDLED;
  2771. enable_intr:
  2772. /* re-enable interrupt causes */
  2773. wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
  2774. if (!test_bit(__I40E_DOWN, &pf->state)) {
  2775. i40e_service_event_schedule(pf);
  2776. i40e_irq_dynamic_enable_icr0(pf);
  2777. }
  2778. return ret;
  2779. }
  2780. /**
  2781. * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
  2782. * @tx_ring: tx ring to clean
  2783. * @budget: how many cleans we're allowed
  2784. *
  2785. * Returns true if there's any budget left (e.g. the clean is finished)
  2786. **/
  2787. static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
  2788. {
  2789. struct i40e_vsi *vsi = tx_ring->vsi;
  2790. u16 i = tx_ring->next_to_clean;
  2791. struct i40e_tx_buffer *tx_buf;
  2792. struct i40e_tx_desc *tx_desc;
  2793. tx_buf = &tx_ring->tx_bi[i];
  2794. tx_desc = I40E_TX_DESC(tx_ring, i);
  2795. i -= tx_ring->count;
  2796. do {
  2797. struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
  2798. /* if next_to_watch is not set then there is no work pending */
  2799. if (!eop_desc)
  2800. break;
  2801. /* prevent any other reads prior to eop_desc */
  2802. read_barrier_depends();
  2803. /* if the descriptor isn't done, no work yet to do */
  2804. if (!(eop_desc->cmd_type_offset_bsz &
  2805. cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
  2806. break;
  2807. /* clear next_to_watch to prevent false hangs */
  2808. tx_buf->next_to_watch = NULL;
  2809. tx_desc->buffer_addr = 0;
  2810. tx_desc->cmd_type_offset_bsz = 0;
  2811. /* move past filter desc */
  2812. tx_buf++;
  2813. tx_desc++;
  2814. i++;
  2815. if (unlikely(!i)) {
  2816. i -= tx_ring->count;
  2817. tx_buf = tx_ring->tx_bi;
  2818. tx_desc = I40E_TX_DESC(tx_ring, 0);
  2819. }
  2820. /* unmap skb header data */
  2821. dma_unmap_single(tx_ring->dev,
  2822. dma_unmap_addr(tx_buf, dma),
  2823. dma_unmap_len(tx_buf, len),
  2824. DMA_TO_DEVICE);
  2825. if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
  2826. kfree(tx_buf->raw_buf);
  2827. tx_buf->raw_buf = NULL;
  2828. tx_buf->tx_flags = 0;
  2829. tx_buf->next_to_watch = NULL;
  2830. dma_unmap_len_set(tx_buf, len, 0);
  2831. tx_desc->buffer_addr = 0;
  2832. tx_desc->cmd_type_offset_bsz = 0;
  2833. /* move us past the eop_desc for start of next FD desc */
  2834. tx_buf++;
  2835. tx_desc++;
  2836. i++;
  2837. if (unlikely(!i)) {
  2838. i -= tx_ring->count;
  2839. tx_buf = tx_ring->tx_bi;
  2840. tx_desc = I40E_TX_DESC(tx_ring, 0);
  2841. }
  2842. /* update budget accounting */
  2843. budget--;
  2844. } while (likely(budget));
  2845. i += tx_ring->count;
  2846. tx_ring->next_to_clean = i;
  2847. if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
  2848. i40e_irq_dynamic_enable(vsi,
  2849. tx_ring->q_vector->v_idx + vsi->base_vector);
  2850. }
  2851. return budget > 0;
  2852. }
  2853. /**
  2854. * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
  2855. * @irq: interrupt number
  2856. * @data: pointer to a q_vector
  2857. **/
  2858. static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
  2859. {
  2860. struct i40e_q_vector *q_vector = data;
  2861. struct i40e_vsi *vsi;
  2862. if (!q_vector->tx.ring)
  2863. return IRQ_HANDLED;
  2864. vsi = q_vector->tx.ring->vsi;
  2865. i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
  2866. return IRQ_HANDLED;
  2867. }
  2868. /**
  2869. * i40e_map_vector_to_qp - Assigns the queue pair to the vector
  2870. * @vsi: the VSI being configured
  2871. * @v_idx: vector index
  2872. * @qp_idx: queue pair index
  2873. **/
  2874. static void map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
  2875. {
  2876. struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
  2877. struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
  2878. struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
  2879. tx_ring->q_vector = q_vector;
  2880. tx_ring->next = q_vector->tx.ring;
  2881. q_vector->tx.ring = tx_ring;
  2882. q_vector->tx.count++;
  2883. rx_ring->q_vector = q_vector;
  2884. rx_ring->next = q_vector->rx.ring;
  2885. q_vector->rx.ring = rx_ring;
  2886. q_vector->rx.count++;
  2887. }
  2888. /**
  2889. * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
  2890. * @vsi: the VSI being configured
  2891. *
  2892. * This function maps descriptor rings to the queue-specific vectors
  2893. * we were allotted through the MSI-X enabling code. Ideally, we'd have
  2894. * one vector per queue pair, but on a constrained vector budget, we
  2895. * group the queue pairs as "efficiently" as possible.
  2896. **/
  2897. static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
  2898. {
  2899. int qp_remaining = vsi->num_queue_pairs;
  2900. int q_vectors = vsi->num_q_vectors;
  2901. int num_ringpairs;
  2902. int v_start = 0;
  2903. int qp_idx = 0;
  2904. /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
  2905. * group them so there are multiple queues per vector.
  2906. * It is also important to go through all the vectors available to be
  2907. * sure that if we don't use all the vectors, that the remaining vectors
  2908. * are cleared. This is especially important when decreasing the
  2909. * number of queues in use.
  2910. */
  2911. for (; v_start < q_vectors; v_start++) {
  2912. struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
  2913. num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
  2914. q_vector->num_ringpairs = num_ringpairs;
  2915. q_vector->rx.count = 0;
  2916. q_vector->tx.count = 0;
  2917. q_vector->rx.ring = NULL;
  2918. q_vector->tx.ring = NULL;
  2919. while (num_ringpairs--) {
  2920. map_vector_to_qp(vsi, v_start, qp_idx);
  2921. qp_idx++;
  2922. qp_remaining--;
  2923. }
  2924. }
  2925. }
  2926. /**
  2927. * i40e_vsi_request_irq - Request IRQ from the OS
  2928. * @vsi: the VSI being configured
  2929. * @basename: name for the vector
  2930. **/
  2931. static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
  2932. {
  2933. struct i40e_pf *pf = vsi->back;
  2934. int err;
  2935. if (pf->flags & I40E_FLAG_MSIX_ENABLED)
  2936. err = i40e_vsi_request_irq_msix(vsi, basename);
  2937. else if (pf->flags & I40E_FLAG_MSI_ENABLED)
  2938. err = request_irq(pf->pdev->irq, i40e_intr, 0,
  2939. pf->misc_int_name, pf);
  2940. else
  2941. err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
  2942. pf->misc_int_name, pf);
  2943. if (err)
  2944. dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
  2945. return err;
  2946. }
  2947. #ifdef CONFIG_NET_POLL_CONTROLLER
  2948. /**
  2949. * i40e_netpoll - A Polling 'interrupt'handler
  2950. * @netdev: network interface device structure
  2951. *
  2952. * This is used by netconsole to send skbs without having to re-enable
  2953. * interrupts. It's not called while the normal interrupt routine is executing.
  2954. **/
  2955. #ifdef I40E_FCOE
  2956. void i40e_netpoll(struct net_device *netdev)
  2957. #else
  2958. static void i40e_netpoll(struct net_device *netdev)
  2959. #endif
  2960. {
  2961. struct i40e_netdev_priv *np = netdev_priv(netdev);
  2962. struct i40e_vsi *vsi = np->vsi;
  2963. struct i40e_pf *pf = vsi->back;
  2964. int i;
  2965. /* if interface is down do nothing */
  2966. if (test_bit(__I40E_DOWN, &vsi->state))
  2967. return;
  2968. pf->flags |= I40E_FLAG_IN_NETPOLL;
  2969. if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
  2970. for (i = 0; i < vsi->num_q_vectors; i++)
  2971. i40e_msix_clean_rings(0, vsi->q_vectors[i]);
  2972. } else {
  2973. i40e_intr(pf->pdev->irq, netdev);
  2974. }
  2975. pf->flags &= ~I40E_FLAG_IN_NETPOLL;
  2976. }
  2977. #endif
  2978. /**
  2979. * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
  2980. * @pf: the PF being configured
  2981. * @pf_q: the PF queue
  2982. * @enable: enable or disable state of the queue
  2983. *
  2984. * This routine will wait for the given Tx queue of the PF to reach the
  2985. * enabled or disabled state.
  2986. * Returns -ETIMEDOUT in case of failing to reach the requested state after
  2987. * multiple retries; else will return 0 in case of success.
  2988. **/
  2989. static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
  2990. {
  2991. int i;
  2992. u32 tx_reg;
  2993. for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
  2994. tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
  2995. if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
  2996. break;
  2997. udelay(10);
  2998. }
  2999. if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
  3000. return -ETIMEDOUT;
  3001. return 0;
  3002. }
  3003. /**
  3004. * i40e_vsi_control_tx - Start or stop a VSI's rings
  3005. * @vsi: the VSI being configured
  3006. * @enable: start or stop the rings
  3007. **/
  3008. static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
  3009. {
  3010. struct i40e_pf *pf = vsi->back;
  3011. struct i40e_hw *hw = &pf->hw;
  3012. int i, j, pf_q, ret = 0;
  3013. u32 tx_reg;
  3014. pf_q = vsi->base_queue;
  3015. for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
  3016. /* warn the TX unit of coming changes */
  3017. i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
  3018. if (!enable)
  3019. udelay(10);
  3020. for (j = 0; j < 50; j++) {
  3021. tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
  3022. if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
  3023. ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
  3024. break;
  3025. usleep_range(1000, 2000);
  3026. }
  3027. /* Skip if the queue is already in the requested state */
  3028. if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
  3029. continue;
  3030. /* turn on/off the queue */
  3031. if (enable) {
  3032. wr32(hw, I40E_QTX_HEAD(pf_q), 0);
  3033. tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
  3034. } else {
  3035. tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
  3036. }
  3037. wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
  3038. /* wait for the change to finish */
  3039. ret = i40e_pf_txq_wait(pf, pf_q, enable);
  3040. if (ret) {
  3041. dev_info(&pf->pdev->dev,
  3042. "%s: VSI seid %d Tx ring %d %sable timeout\n",
  3043. __func__, vsi->seid, pf_q,
  3044. (enable ? "en" : "dis"));
  3045. break;
  3046. }
  3047. }
  3048. if (hw->revision_id == 0)
  3049. mdelay(50);
  3050. return ret;
  3051. }
  3052. /**
  3053. * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
  3054. * @pf: the PF being configured
  3055. * @pf_q: the PF queue
  3056. * @enable: enable or disable state of the queue
  3057. *
  3058. * This routine will wait for the given Rx queue of the PF to reach the
  3059. * enabled or disabled state.
  3060. * Returns -ETIMEDOUT in case of failing to reach the requested state after
  3061. * multiple retries; else will return 0 in case of success.
  3062. **/
  3063. static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
  3064. {
  3065. int i;
  3066. u32 rx_reg;
  3067. for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
  3068. rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
  3069. if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
  3070. break;
  3071. udelay(10);
  3072. }
  3073. if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
  3074. return -ETIMEDOUT;
  3075. return 0;
  3076. }
  3077. /**
  3078. * i40e_vsi_control_rx - Start or stop a VSI's rings
  3079. * @vsi: the VSI being configured
  3080. * @enable: start or stop the rings
  3081. **/
  3082. static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
  3083. {
  3084. struct i40e_pf *pf = vsi->back;
  3085. struct i40e_hw *hw = &pf->hw;
  3086. int i, j, pf_q, ret = 0;
  3087. u32 rx_reg;
  3088. pf_q = vsi->base_queue;
  3089. for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
  3090. for (j = 0; j < 50; j++) {
  3091. rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
  3092. if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
  3093. ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
  3094. break;
  3095. usleep_range(1000, 2000);
  3096. }
  3097. /* Skip if the queue is already in the requested state */
  3098. if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
  3099. continue;
  3100. /* turn on/off the queue */
  3101. if (enable)
  3102. rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
  3103. else
  3104. rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
  3105. wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
  3106. /* wait for the change to finish */
  3107. ret = i40e_pf_rxq_wait(pf, pf_q, enable);
  3108. if (ret) {
  3109. dev_info(&pf->pdev->dev,
  3110. "%s: VSI seid %d Rx ring %d %sable timeout\n",
  3111. __func__, vsi->seid, pf_q,
  3112. (enable ? "en" : "dis"));
  3113. break;
  3114. }
  3115. }
  3116. return ret;
  3117. }
  3118. /**
  3119. * i40e_vsi_control_rings - Start or stop a VSI's rings
  3120. * @vsi: the VSI being configured
  3121. * @enable: start or stop the rings
  3122. **/
  3123. int i40e_vsi_control_rings(struct i40e_vsi *vsi, bool request)
  3124. {
  3125. int ret = 0;
  3126. /* do rx first for enable and last for disable */
  3127. if (request) {
  3128. ret = i40e_vsi_control_rx(vsi, request);
  3129. if (ret)
  3130. return ret;
  3131. ret = i40e_vsi_control_tx(vsi, request);
  3132. } else {
  3133. /* Ignore return value, we need to shutdown whatever we can */
  3134. i40e_vsi_control_tx(vsi, request);
  3135. i40e_vsi_control_rx(vsi, request);
  3136. }
  3137. return ret;
  3138. }
  3139. /**
  3140. * i40e_vsi_free_irq - Free the irq association with the OS
  3141. * @vsi: the VSI being configured
  3142. **/
  3143. static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
  3144. {
  3145. struct i40e_pf *pf = vsi->back;
  3146. struct i40e_hw *hw = &pf->hw;
  3147. int base = vsi->base_vector;
  3148. u32 val, qp;
  3149. int i;
  3150. if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
  3151. if (!vsi->q_vectors)
  3152. return;
  3153. if (!vsi->irqs_ready)
  3154. return;
  3155. vsi->irqs_ready = false;
  3156. for (i = 0; i < vsi->num_q_vectors; i++) {
  3157. u16 vector = i + base;
  3158. /* free only the irqs that were actually requested */
  3159. if (!vsi->q_vectors[i] ||
  3160. !vsi->q_vectors[i]->num_ringpairs)
  3161. continue;
  3162. /* clear the affinity_mask in the IRQ descriptor */
  3163. irq_set_affinity_hint(pf->msix_entries[vector].vector,
  3164. NULL);
  3165. free_irq(pf->msix_entries[vector].vector,
  3166. vsi->q_vectors[i]);
  3167. /* Tear down the interrupt queue link list
  3168. *
  3169. * We know that they come in pairs and always
  3170. * the Rx first, then the Tx. To clear the
  3171. * link list, stick the EOL value into the
  3172. * next_q field of the registers.
  3173. */
  3174. val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
  3175. qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
  3176. >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
  3177. val |= I40E_QUEUE_END_OF_LIST
  3178. << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
  3179. wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
  3180. while (qp != I40E_QUEUE_END_OF_LIST) {
  3181. u32 next;
  3182. val = rd32(hw, I40E_QINT_RQCTL(qp));
  3183. val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK |
  3184. I40E_QINT_RQCTL_MSIX0_INDX_MASK |
  3185. I40E_QINT_RQCTL_CAUSE_ENA_MASK |
  3186. I40E_QINT_RQCTL_INTEVENT_MASK);
  3187. val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
  3188. I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
  3189. wr32(hw, I40E_QINT_RQCTL(qp), val);
  3190. val = rd32(hw, I40E_QINT_TQCTL(qp));
  3191. next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
  3192. >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
  3193. val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK |
  3194. I40E_QINT_TQCTL_MSIX0_INDX_MASK |
  3195. I40E_QINT_TQCTL_CAUSE_ENA_MASK |
  3196. I40E_QINT_TQCTL_INTEVENT_MASK);
  3197. val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
  3198. I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
  3199. wr32(hw, I40E_QINT_TQCTL(qp), val);
  3200. qp = next;
  3201. }
  3202. }
  3203. } else {
  3204. free_irq(pf->pdev->irq, pf);
  3205. val = rd32(hw, I40E_PFINT_LNKLST0);
  3206. qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
  3207. >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
  3208. val |= I40E_QUEUE_END_OF_LIST
  3209. << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
  3210. wr32(hw, I40E_PFINT_LNKLST0, val);
  3211. val = rd32(hw, I40E_QINT_RQCTL(qp));
  3212. val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK |
  3213. I40E_QINT_RQCTL_MSIX0_INDX_MASK |
  3214. I40E_QINT_RQCTL_CAUSE_ENA_MASK |
  3215. I40E_QINT_RQCTL_INTEVENT_MASK);
  3216. val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
  3217. I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
  3218. wr32(hw, I40E_QINT_RQCTL(qp), val);
  3219. val = rd32(hw, I40E_QINT_TQCTL(qp));
  3220. val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK |
  3221. I40E_QINT_TQCTL_MSIX0_INDX_MASK |
  3222. I40E_QINT_TQCTL_CAUSE_ENA_MASK |
  3223. I40E_QINT_TQCTL_INTEVENT_MASK);
  3224. val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
  3225. I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
  3226. wr32(hw, I40E_QINT_TQCTL(qp), val);
  3227. }
  3228. }
  3229. /**
  3230. * i40e_free_q_vector - Free memory allocated for specific interrupt vector
  3231. * @vsi: the VSI being configured
  3232. * @v_idx: Index of vector to be freed
  3233. *
  3234. * This function frees the memory allocated to the q_vector. In addition if
  3235. * NAPI is enabled it will delete any references to the NAPI struct prior
  3236. * to freeing the q_vector.
  3237. **/
  3238. static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
  3239. {
  3240. struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
  3241. struct i40e_ring *ring;
  3242. if (!q_vector)
  3243. return;
  3244. /* disassociate q_vector from rings */
  3245. i40e_for_each_ring(ring, q_vector->tx)
  3246. ring->q_vector = NULL;
  3247. i40e_for_each_ring(ring, q_vector->rx)
  3248. ring->q_vector = NULL;
  3249. /* only VSI w/ an associated netdev is set up w/ NAPI */
  3250. if (vsi->netdev)
  3251. netif_napi_del(&q_vector->napi);
  3252. vsi->q_vectors[v_idx] = NULL;
  3253. kfree_rcu(q_vector, rcu);
  3254. }
  3255. /**
  3256. * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
  3257. * @vsi: the VSI being un-configured
  3258. *
  3259. * This frees the memory allocated to the q_vectors and
  3260. * deletes references to the NAPI struct.
  3261. **/
  3262. static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
  3263. {
  3264. int v_idx;
  3265. for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
  3266. i40e_free_q_vector(vsi, v_idx);
  3267. }
  3268. /**
  3269. * i40e_reset_interrupt_capability - Disable interrupt setup in OS
  3270. * @pf: board private structure
  3271. **/
  3272. static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
  3273. {
  3274. /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
  3275. if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
  3276. pci_disable_msix(pf->pdev);
  3277. kfree(pf->msix_entries);
  3278. pf->msix_entries = NULL;
  3279. } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
  3280. pci_disable_msi(pf->pdev);
  3281. }
  3282. pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
  3283. }
  3284. /**
  3285. * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
  3286. * @pf: board private structure
  3287. *
  3288. * We go through and clear interrupt specific resources and reset the structure
  3289. * to pre-load conditions
  3290. **/
  3291. static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
  3292. {
  3293. int i;
  3294. i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
  3295. for (i = 0; i < pf->num_alloc_vsi; i++)
  3296. if (pf->vsi[i])
  3297. i40e_vsi_free_q_vectors(pf->vsi[i]);
  3298. i40e_reset_interrupt_capability(pf);
  3299. }
  3300. /**
  3301. * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
  3302. * @vsi: the VSI being configured
  3303. **/
  3304. static void i40e_napi_enable_all(struct i40e_vsi *vsi)
  3305. {
  3306. int q_idx;
  3307. if (!vsi->netdev)
  3308. return;
  3309. for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
  3310. napi_enable(&vsi->q_vectors[q_idx]->napi);
  3311. }
  3312. /**
  3313. * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
  3314. * @vsi: the VSI being configured
  3315. **/
  3316. static void i40e_napi_disable_all(struct i40e_vsi *vsi)
  3317. {
  3318. int q_idx;
  3319. if (!vsi->netdev)
  3320. return;
  3321. for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
  3322. napi_disable(&vsi->q_vectors[q_idx]->napi);
  3323. }
  3324. /**
  3325. * i40e_vsi_close - Shut down a VSI
  3326. * @vsi: the vsi to be quelled
  3327. **/
  3328. static void i40e_vsi_close(struct i40e_vsi *vsi)
  3329. {
  3330. if (!test_and_set_bit(__I40E_DOWN, &vsi->state))
  3331. i40e_down(vsi);
  3332. i40e_vsi_free_irq(vsi);
  3333. i40e_vsi_free_tx_resources(vsi);
  3334. i40e_vsi_free_rx_resources(vsi);
  3335. }
  3336. /**
  3337. * i40e_quiesce_vsi - Pause a given VSI
  3338. * @vsi: the VSI being paused
  3339. **/
  3340. static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
  3341. {
  3342. if (test_bit(__I40E_DOWN, &vsi->state))
  3343. return;
  3344. set_bit(__I40E_NEEDS_RESTART, &vsi->state);
  3345. if (vsi->netdev && netif_running(vsi->netdev)) {
  3346. vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
  3347. } else {
  3348. i40e_vsi_close(vsi);
  3349. }
  3350. }
  3351. /**
  3352. * i40e_unquiesce_vsi - Resume a given VSI
  3353. * @vsi: the VSI being resumed
  3354. **/
  3355. static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
  3356. {
  3357. if (!test_bit(__I40E_NEEDS_RESTART, &vsi->state))
  3358. return;
  3359. clear_bit(__I40E_NEEDS_RESTART, &vsi->state);
  3360. if (vsi->netdev && netif_running(vsi->netdev))
  3361. vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
  3362. else
  3363. i40e_vsi_open(vsi); /* this clears the DOWN bit */
  3364. }
  3365. /**
  3366. * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
  3367. * @pf: the PF
  3368. **/
  3369. static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
  3370. {
  3371. int v;
  3372. for (v = 0; v < pf->num_alloc_vsi; v++) {
  3373. if (pf->vsi[v])
  3374. i40e_quiesce_vsi(pf->vsi[v]);
  3375. }
  3376. }
  3377. /**
  3378. * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
  3379. * @pf: the PF
  3380. **/
  3381. static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
  3382. {
  3383. int v;
  3384. for (v = 0; v < pf->num_alloc_vsi; v++) {
  3385. if (pf->vsi[v])
  3386. i40e_unquiesce_vsi(pf->vsi[v]);
  3387. }
  3388. }
  3389. /**
  3390. * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config
  3391. * @dcbcfg: the corresponding DCBx configuration structure
  3392. *
  3393. * Return the number of TCs from given DCBx configuration
  3394. **/
  3395. static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
  3396. {
  3397. u8 num_tc = 0;
  3398. int i;
  3399. /* Scan the ETS Config Priority Table to find
  3400. * traffic class enabled for a given priority
  3401. * and use the traffic class index to get the
  3402. * number of traffic classes enabled
  3403. */
  3404. for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
  3405. if (dcbcfg->etscfg.prioritytable[i] > num_tc)
  3406. num_tc = dcbcfg->etscfg.prioritytable[i];
  3407. }
  3408. /* Traffic class index starts from zero so
  3409. * increment to return the actual count
  3410. */
  3411. return num_tc + 1;
  3412. }
  3413. /**
  3414. * i40e_dcb_get_enabled_tc - Get enabled traffic classes
  3415. * @dcbcfg: the corresponding DCBx configuration structure
  3416. *
  3417. * Query the current DCB configuration and return the number of
  3418. * traffic classes enabled from the given DCBX config
  3419. **/
  3420. static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
  3421. {
  3422. u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
  3423. u8 enabled_tc = 1;
  3424. u8 i;
  3425. for (i = 0; i < num_tc; i++)
  3426. enabled_tc |= 1 << i;
  3427. return enabled_tc;
  3428. }
  3429. /**
  3430. * i40e_pf_get_num_tc - Get enabled traffic classes for PF
  3431. * @pf: PF being queried
  3432. *
  3433. * Return number of traffic classes enabled for the given PF
  3434. **/
  3435. static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
  3436. {
  3437. struct i40e_hw *hw = &pf->hw;
  3438. u8 i, enabled_tc;
  3439. u8 num_tc = 0;
  3440. struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
  3441. /* If DCB is not enabled then always in single TC */
  3442. if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
  3443. return 1;
  3444. /* MFP mode return count of enabled TCs for this PF */
  3445. if (pf->flags & I40E_FLAG_MFP_ENABLED) {
  3446. enabled_tc = pf->hw.func_caps.enabled_tcmap;
  3447. for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
  3448. if (enabled_tc & (1 << i))
  3449. num_tc++;
  3450. }
  3451. return num_tc;
  3452. }
  3453. /* SFP mode will be enabled for all TCs on port */
  3454. return i40e_dcb_get_num_tc(dcbcfg);
  3455. }
  3456. /**
  3457. * i40e_pf_get_default_tc - Get bitmap for first enabled TC
  3458. * @pf: PF being queried
  3459. *
  3460. * Return a bitmap for first enabled traffic class for this PF.
  3461. **/
  3462. static u8 i40e_pf_get_default_tc(struct i40e_pf *pf)
  3463. {
  3464. u8 enabled_tc = pf->hw.func_caps.enabled_tcmap;
  3465. u8 i = 0;
  3466. if (!enabled_tc)
  3467. return 0x1; /* TC0 */
  3468. /* Find the first enabled TC */
  3469. for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
  3470. if (enabled_tc & (1 << i))
  3471. break;
  3472. }
  3473. return 1 << i;
  3474. }
  3475. /**
  3476. * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
  3477. * @pf: PF being queried
  3478. *
  3479. * Return a bitmap for enabled traffic classes for this PF.
  3480. **/
  3481. static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
  3482. {
  3483. /* If DCB is not enabled for this PF then just return default TC */
  3484. if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
  3485. return i40e_pf_get_default_tc(pf);
  3486. /* MFP mode will have enabled TCs set by FW */
  3487. if (pf->flags & I40E_FLAG_MFP_ENABLED)
  3488. return pf->hw.func_caps.enabled_tcmap;
  3489. /* SFP mode we want PF to be enabled for all TCs */
  3490. return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
  3491. }
  3492. /**
  3493. * i40e_vsi_get_bw_info - Query VSI BW Information
  3494. * @vsi: the VSI being queried
  3495. *
  3496. * Returns 0 on success, negative value on failure
  3497. **/
  3498. static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
  3499. {
  3500. struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
  3501. struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
  3502. struct i40e_pf *pf = vsi->back;
  3503. struct i40e_hw *hw = &pf->hw;
  3504. i40e_status aq_ret;
  3505. u32 tc_bw_max;
  3506. int i;
  3507. /* Get the VSI level BW configuration */
  3508. aq_ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
  3509. if (aq_ret) {
  3510. dev_info(&pf->pdev->dev,
  3511. "couldn't get pf vsi bw config, err %d, aq_err %d\n",
  3512. aq_ret, pf->hw.aq.asq_last_status);
  3513. return -EINVAL;
  3514. }
  3515. /* Get the VSI level BW configuration per TC */
  3516. aq_ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
  3517. NULL);
  3518. if (aq_ret) {
  3519. dev_info(&pf->pdev->dev,
  3520. "couldn't get pf vsi ets bw config, err %d, aq_err %d\n",
  3521. aq_ret, pf->hw.aq.asq_last_status);
  3522. return -EINVAL;
  3523. }
  3524. if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
  3525. dev_info(&pf->pdev->dev,
  3526. "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
  3527. bw_config.tc_valid_bits,
  3528. bw_ets_config.tc_valid_bits);
  3529. /* Still continuing */
  3530. }
  3531. vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
  3532. vsi->bw_max_quanta = bw_config.max_bw;
  3533. tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
  3534. (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
  3535. for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
  3536. vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
  3537. vsi->bw_ets_limit_credits[i] =
  3538. le16_to_cpu(bw_ets_config.credits[i]);
  3539. /* 3 bits out of 4 for each TC */
  3540. vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
  3541. }
  3542. return 0;
  3543. }
  3544. /**
  3545. * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
  3546. * @vsi: the VSI being configured
  3547. * @enabled_tc: TC bitmap
  3548. * @bw_credits: BW shared credits per TC
  3549. *
  3550. * Returns 0 on success, negative value on failure
  3551. **/
  3552. static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
  3553. u8 *bw_share)
  3554. {
  3555. struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
  3556. i40e_status aq_ret;
  3557. int i;
  3558. bw_data.tc_valid_bits = enabled_tc;
  3559. for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
  3560. bw_data.tc_bw_credits[i] = bw_share[i];
  3561. aq_ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data,
  3562. NULL);
  3563. if (aq_ret) {
  3564. dev_info(&vsi->back->pdev->dev,
  3565. "AQ command Config VSI BW allocation per TC failed = %d\n",
  3566. vsi->back->hw.aq.asq_last_status);
  3567. return -EINVAL;
  3568. }
  3569. for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
  3570. vsi->info.qs_handle[i] = bw_data.qs_handles[i];
  3571. return 0;
  3572. }
  3573. /**
  3574. * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
  3575. * @vsi: the VSI being configured
  3576. * @enabled_tc: TC map to be enabled
  3577. *
  3578. **/
  3579. static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
  3580. {
  3581. struct net_device *netdev = vsi->netdev;
  3582. struct i40e_pf *pf = vsi->back;
  3583. struct i40e_hw *hw = &pf->hw;
  3584. u8 netdev_tc = 0;
  3585. int i;
  3586. struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
  3587. if (!netdev)
  3588. return;
  3589. if (!enabled_tc) {
  3590. netdev_reset_tc(netdev);
  3591. return;
  3592. }
  3593. /* Set up actual enabled TCs on the VSI */
  3594. if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
  3595. return;
  3596. /* set per TC queues for the VSI */
  3597. for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
  3598. /* Only set TC queues for enabled tcs
  3599. *
  3600. * e.g. For a VSI that has TC0 and TC3 enabled the
  3601. * enabled_tc bitmap would be 0x00001001; the driver
  3602. * will set the numtc for netdev as 2 that will be
  3603. * referenced by the netdev layer as TC 0 and 1.
  3604. */
  3605. if (vsi->tc_config.enabled_tc & (1 << i))
  3606. netdev_set_tc_queue(netdev,
  3607. vsi->tc_config.tc_info[i].netdev_tc,
  3608. vsi->tc_config.tc_info[i].qcount,
  3609. vsi->tc_config.tc_info[i].qoffset);
  3610. }
  3611. /* Assign UP2TC map for the VSI */
  3612. for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
  3613. /* Get the actual TC# for the UP */
  3614. u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
  3615. /* Get the mapped netdev TC# for the UP */
  3616. netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc;
  3617. netdev_set_prio_tc_map(netdev, i, netdev_tc);
  3618. }
  3619. }
  3620. /**
  3621. * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
  3622. * @vsi: the VSI being configured
  3623. * @ctxt: the ctxt buffer returned from AQ VSI update param command
  3624. **/
  3625. static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
  3626. struct i40e_vsi_context *ctxt)
  3627. {
  3628. /* copy just the sections touched not the entire info
  3629. * since not all sections are valid as returned by
  3630. * update vsi params
  3631. */
  3632. vsi->info.mapping_flags = ctxt->info.mapping_flags;
  3633. memcpy(&vsi->info.queue_mapping,
  3634. &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
  3635. memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
  3636. sizeof(vsi->info.tc_mapping));
  3637. }
  3638. /**
  3639. * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
  3640. * @vsi: VSI to be configured
  3641. * @enabled_tc: TC bitmap
  3642. *
  3643. * This configures a particular VSI for TCs that are mapped to the
  3644. * given TC bitmap. It uses default bandwidth share for TCs across
  3645. * VSIs to configure TC for a particular VSI.
  3646. *
  3647. * NOTE:
  3648. * It is expected that the VSI queues have been quisced before calling
  3649. * this function.
  3650. **/
  3651. static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
  3652. {
  3653. u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
  3654. struct i40e_vsi_context ctxt;
  3655. int ret = 0;
  3656. int i;
  3657. /* Check if enabled_tc is same as existing or new TCs */
  3658. if (vsi->tc_config.enabled_tc == enabled_tc)
  3659. return ret;
  3660. /* Enable ETS TCs with equal BW Share for now across all VSIs */
  3661. for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
  3662. if (enabled_tc & (1 << i))
  3663. bw_share[i] = 1;
  3664. }
  3665. ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
  3666. if (ret) {
  3667. dev_info(&vsi->back->pdev->dev,
  3668. "Failed configuring TC map %d for VSI %d\n",
  3669. enabled_tc, vsi->seid);
  3670. goto out;
  3671. }
  3672. /* Update Queue Pairs Mapping for currently enabled UPs */
  3673. ctxt.seid = vsi->seid;
  3674. ctxt.pf_num = vsi->back->hw.pf_id;
  3675. ctxt.vf_num = 0;
  3676. ctxt.uplink_seid = vsi->uplink_seid;
  3677. memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
  3678. i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
  3679. /* Update the VSI after updating the VSI queue-mapping information */
  3680. ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
  3681. if (ret) {
  3682. dev_info(&vsi->back->pdev->dev,
  3683. "update vsi failed, aq_err=%d\n",
  3684. vsi->back->hw.aq.asq_last_status);
  3685. goto out;
  3686. }
  3687. /* update the local VSI info with updated queue map */
  3688. i40e_vsi_update_queue_map(vsi, &ctxt);
  3689. vsi->info.valid_sections = 0;
  3690. /* Update current VSI BW information */
  3691. ret = i40e_vsi_get_bw_info(vsi);
  3692. if (ret) {
  3693. dev_info(&vsi->back->pdev->dev,
  3694. "Failed updating vsi bw info, aq_err=%d\n",
  3695. vsi->back->hw.aq.asq_last_status);
  3696. goto out;
  3697. }
  3698. /* Update the netdev TC setup */
  3699. i40e_vsi_config_netdev_tc(vsi, enabled_tc);
  3700. out:
  3701. return ret;
  3702. }
  3703. /**
  3704. * i40e_veb_config_tc - Configure TCs for given VEB
  3705. * @veb: given VEB
  3706. * @enabled_tc: TC bitmap
  3707. *
  3708. * Configures given TC bitmap for VEB (switching) element
  3709. **/
  3710. int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
  3711. {
  3712. struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
  3713. struct i40e_pf *pf = veb->pf;
  3714. int ret = 0;
  3715. int i;
  3716. /* No TCs or already enabled TCs just return */
  3717. if (!enabled_tc || veb->enabled_tc == enabled_tc)
  3718. return ret;
  3719. bw_data.tc_valid_bits = enabled_tc;
  3720. /* bw_data.absolute_credits is not set (relative) */
  3721. /* Enable ETS TCs with equal BW Share for now */
  3722. for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
  3723. if (enabled_tc & (1 << i))
  3724. bw_data.tc_bw_share_credits[i] = 1;
  3725. }
  3726. ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
  3727. &bw_data, NULL);
  3728. if (ret) {
  3729. dev_info(&pf->pdev->dev,
  3730. "veb bw config failed, aq_err=%d\n",
  3731. pf->hw.aq.asq_last_status);
  3732. goto out;
  3733. }
  3734. /* Update the BW information */
  3735. ret = i40e_veb_get_bw_info(veb);
  3736. if (ret) {
  3737. dev_info(&pf->pdev->dev,
  3738. "Failed getting veb bw config, aq_err=%d\n",
  3739. pf->hw.aq.asq_last_status);
  3740. }
  3741. out:
  3742. return ret;
  3743. }
  3744. #ifdef CONFIG_I40E_DCB
  3745. /**
  3746. * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
  3747. * @pf: PF struct
  3748. *
  3749. * Reconfigure VEB/VSIs on a given PF; it is assumed that
  3750. * the caller would've quiesce all the VSIs before calling
  3751. * this function
  3752. **/
  3753. static void i40e_dcb_reconfigure(struct i40e_pf *pf)
  3754. {
  3755. u8 tc_map = 0;
  3756. int ret;
  3757. u8 v;
  3758. /* Enable the TCs available on PF to all VEBs */
  3759. tc_map = i40e_pf_get_tc_map(pf);
  3760. for (v = 0; v < I40E_MAX_VEB; v++) {
  3761. if (!pf->veb[v])
  3762. continue;
  3763. ret = i40e_veb_config_tc(pf->veb[v], tc_map);
  3764. if (ret) {
  3765. dev_info(&pf->pdev->dev,
  3766. "Failed configuring TC for VEB seid=%d\n",
  3767. pf->veb[v]->seid);
  3768. /* Will try to configure as many components */
  3769. }
  3770. }
  3771. /* Update each VSI */
  3772. for (v = 0; v < pf->num_alloc_vsi; v++) {
  3773. if (!pf->vsi[v])
  3774. continue;
  3775. /* - Enable all TCs for the LAN VSI
  3776. #ifdef I40E_FCOE
  3777. * - For FCoE VSI only enable the TC configured
  3778. * as per the APP TLV
  3779. #endif
  3780. * - For all others keep them at TC0 for now
  3781. */
  3782. if (v == pf->lan_vsi)
  3783. tc_map = i40e_pf_get_tc_map(pf);
  3784. else
  3785. tc_map = i40e_pf_get_default_tc(pf);
  3786. #ifdef I40E_FCOE
  3787. if (pf->vsi[v]->type == I40E_VSI_FCOE)
  3788. tc_map = i40e_get_fcoe_tc_map(pf);
  3789. #endif /* #ifdef I40E_FCOE */
  3790. ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
  3791. if (ret) {
  3792. dev_info(&pf->pdev->dev,
  3793. "Failed configuring TC for VSI seid=%d\n",
  3794. pf->vsi[v]->seid);
  3795. /* Will try to configure as many components */
  3796. } else {
  3797. /* Re-configure VSI vectors based on updated TC map */
  3798. i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
  3799. if (pf->vsi[v]->netdev)
  3800. i40e_dcbnl_set_all(pf->vsi[v]);
  3801. }
  3802. }
  3803. }
  3804. /**
  3805. * i40e_init_pf_dcb - Initialize DCB configuration
  3806. * @pf: PF being configured
  3807. *
  3808. * Query the current DCB configuration and cache it
  3809. * in the hardware structure
  3810. **/
  3811. static int i40e_init_pf_dcb(struct i40e_pf *pf)
  3812. {
  3813. struct i40e_hw *hw = &pf->hw;
  3814. int err = 0;
  3815. if (pf->hw.func_caps.npar_enable)
  3816. goto out;
  3817. /* Get the initial DCB configuration */
  3818. err = i40e_init_dcb(hw);
  3819. if (!err) {
  3820. /* Device/Function is not DCBX capable */
  3821. if ((!hw->func_caps.dcb) ||
  3822. (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
  3823. dev_info(&pf->pdev->dev,
  3824. "DCBX offload is not supported or is disabled for this PF.\n");
  3825. if (pf->flags & I40E_FLAG_MFP_ENABLED)
  3826. goto out;
  3827. } else {
  3828. /* When status is not DISABLED then DCBX in FW */
  3829. pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
  3830. DCB_CAP_DCBX_VER_IEEE;
  3831. pf->flags |= I40E_FLAG_DCB_CAPABLE;
  3832. /* Enable DCB tagging only when more than one TC */
  3833. if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
  3834. pf->flags |= I40E_FLAG_DCB_ENABLED;
  3835. }
  3836. } else {
  3837. dev_info(&pf->pdev->dev, "AQ Querying DCB configuration failed: %d\n",
  3838. pf->hw.aq.asq_last_status);
  3839. }
  3840. out:
  3841. return err;
  3842. }
  3843. #endif /* CONFIG_I40E_DCB */
  3844. #define SPEED_SIZE 14
  3845. #define FC_SIZE 8
  3846. /**
  3847. * i40e_print_link_message - print link up or down
  3848. * @vsi: the VSI for which link needs a message
  3849. */
  3850. static void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
  3851. {
  3852. char speed[SPEED_SIZE] = "Unknown";
  3853. char fc[FC_SIZE] = "RX/TX";
  3854. if (!isup) {
  3855. netdev_info(vsi->netdev, "NIC Link is Down\n");
  3856. return;
  3857. }
  3858. switch (vsi->back->hw.phy.link_info.link_speed) {
  3859. case I40E_LINK_SPEED_40GB:
  3860. strlcpy(speed, "40 Gbps", SPEED_SIZE);
  3861. break;
  3862. case I40E_LINK_SPEED_10GB:
  3863. strlcpy(speed, "10 Gbps", SPEED_SIZE);
  3864. break;
  3865. case I40E_LINK_SPEED_1GB:
  3866. strlcpy(speed, "1000 Mbps", SPEED_SIZE);
  3867. break;
  3868. default:
  3869. break;
  3870. }
  3871. switch (vsi->back->hw.fc.current_mode) {
  3872. case I40E_FC_FULL:
  3873. strlcpy(fc, "RX/TX", FC_SIZE);
  3874. break;
  3875. case I40E_FC_TX_PAUSE:
  3876. strlcpy(fc, "TX", FC_SIZE);
  3877. break;
  3878. case I40E_FC_RX_PAUSE:
  3879. strlcpy(fc, "RX", FC_SIZE);
  3880. break;
  3881. default:
  3882. strlcpy(fc, "None", FC_SIZE);
  3883. break;
  3884. }
  3885. netdev_info(vsi->netdev, "NIC Link is Up %s Full Duplex, Flow Control: %s\n",
  3886. speed, fc);
  3887. }
  3888. /**
  3889. * i40e_up_complete - Finish the last steps of bringing up a connection
  3890. * @vsi: the VSI being configured
  3891. **/
  3892. static int i40e_up_complete(struct i40e_vsi *vsi)
  3893. {
  3894. struct i40e_pf *pf = vsi->back;
  3895. u8 set_fc_aq_fail = 0;
  3896. int err;
  3897. /* force flow control off */
  3898. i40e_set_fc(&pf->hw, &set_fc_aq_fail, true);
  3899. if (pf->flags & I40E_FLAG_MSIX_ENABLED)
  3900. i40e_vsi_configure_msix(vsi);
  3901. else
  3902. i40e_configure_msi_and_legacy(vsi);
  3903. /* start rings */
  3904. err = i40e_vsi_control_rings(vsi, true);
  3905. if (err)
  3906. return err;
  3907. clear_bit(__I40E_DOWN, &vsi->state);
  3908. i40e_napi_enable_all(vsi);
  3909. i40e_vsi_enable_irq(vsi);
  3910. if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
  3911. (vsi->netdev)) {
  3912. i40e_print_link_message(vsi, true);
  3913. netif_tx_start_all_queues(vsi->netdev);
  3914. netif_carrier_on(vsi->netdev);
  3915. } else if (vsi->netdev) {
  3916. i40e_print_link_message(vsi, false);
  3917. }
  3918. /* replay FDIR SB filters */
  3919. if (vsi->type == I40E_VSI_FDIR)
  3920. i40e_fdir_filter_restore(vsi);
  3921. i40e_service_event_schedule(pf);
  3922. return 0;
  3923. }
  3924. /**
  3925. * i40e_vsi_reinit_locked - Reset the VSI
  3926. * @vsi: the VSI being configured
  3927. *
  3928. * Rebuild the ring structs after some configuration
  3929. * has changed, e.g. MTU size.
  3930. **/
  3931. static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
  3932. {
  3933. struct i40e_pf *pf = vsi->back;
  3934. WARN_ON(in_interrupt());
  3935. while (test_and_set_bit(__I40E_CONFIG_BUSY, &pf->state))
  3936. usleep_range(1000, 2000);
  3937. i40e_down(vsi);
  3938. /* Give a VF some time to respond to the reset. The
  3939. * two second wait is based upon the watchdog cycle in
  3940. * the VF driver.
  3941. */
  3942. if (vsi->type == I40E_VSI_SRIOV)
  3943. msleep(2000);
  3944. i40e_up(vsi);
  3945. clear_bit(__I40E_CONFIG_BUSY, &pf->state);
  3946. }
  3947. /**
  3948. * i40e_up - Bring the connection back up after being down
  3949. * @vsi: the VSI being configured
  3950. **/
  3951. int i40e_up(struct i40e_vsi *vsi)
  3952. {
  3953. int err;
  3954. err = i40e_vsi_configure(vsi);
  3955. if (!err)
  3956. err = i40e_up_complete(vsi);
  3957. return err;
  3958. }
  3959. /**
  3960. * i40e_down - Shutdown the connection processing
  3961. * @vsi: the VSI being stopped
  3962. **/
  3963. void i40e_down(struct i40e_vsi *vsi)
  3964. {
  3965. int i;
  3966. /* It is assumed that the caller of this function
  3967. * sets the vsi->state __I40E_DOWN bit.
  3968. */
  3969. if (vsi->netdev) {
  3970. netif_carrier_off(vsi->netdev);
  3971. netif_tx_disable(vsi->netdev);
  3972. }
  3973. i40e_vsi_disable_irq(vsi);
  3974. i40e_vsi_control_rings(vsi, false);
  3975. i40e_napi_disable_all(vsi);
  3976. for (i = 0; i < vsi->num_queue_pairs; i++) {
  3977. i40e_clean_tx_ring(vsi->tx_rings[i]);
  3978. i40e_clean_rx_ring(vsi->rx_rings[i]);
  3979. }
  3980. }
  3981. /**
  3982. * i40e_setup_tc - configure multiple traffic classes
  3983. * @netdev: net device to configure
  3984. * @tc: number of traffic classes to enable
  3985. **/
  3986. #ifdef I40E_FCOE
  3987. int i40e_setup_tc(struct net_device *netdev, u8 tc)
  3988. #else
  3989. static int i40e_setup_tc(struct net_device *netdev, u8 tc)
  3990. #endif
  3991. {
  3992. struct i40e_netdev_priv *np = netdev_priv(netdev);
  3993. struct i40e_vsi *vsi = np->vsi;
  3994. struct i40e_pf *pf = vsi->back;
  3995. u8 enabled_tc = 0;
  3996. int ret = -EINVAL;
  3997. int i;
  3998. /* Check if DCB enabled to continue */
  3999. if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
  4000. netdev_info(netdev, "DCB is not enabled for adapter\n");
  4001. goto exit;
  4002. }
  4003. /* Check if MFP enabled */
  4004. if (pf->flags & I40E_FLAG_MFP_ENABLED) {
  4005. netdev_info(netdev, "Configuring TC not supported in MFP mode\n");
  4006. goto exit;
  4007. }
  4008. /* Check whether tc count is within enabled limit */
  4009. if (tc > i40e_pf_get_num_tc(pf)) {
  4010. netdev_info(netdev, "TC count greater than enabled on link for adapter\n");
  4011. goto exit;
  4012. }
  4013. /* Generate TC map for number of tc requested */
  4014. for (i = 0; i < tc; i++)
  4015. enabled_tc |= (1 << i);
  4016. /* Requesting same TC configuration as already enabled */
  4017. if (enabled_tc == vsi->tc_config.enabled_tc)
  4018. return 0;
  4019. /* Quiesce VSI queues */
  4020. i40e_quiesce_vsi(vsi);
  4021. /* Configure VSI for enabled TCs */
  4022. ret = i40e_vsi_config_tc(vsi, enabled_tc);
  4023. if (ret) {
  4024. netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
  4025. vsi->seid);
  4026. goto exit;
  4027. }
  4028. /* Unquiesce VSI */
  4029. i40e_unquiesce_vsi(vsi);
  4030. exit:
  4031. return ret;
  4032. }
  4033. /**
  4034. * i40e_open - Called when a network interface is made active
  4035. * @netdev: network interface device structure
  4036. *
  4037. * The open entry point is called when a network interface is made
  4038. * active by the system (IFF_UP). At this point all resources needed
  4039. * for transmit and receive operations are allocated, the interrupt
  4040. * handler is registered with the OS, the netdev watchdog subtask is
  4041. * enabled, and the stack is notified that the interface is ready.
  4042. *
  4043. * Returns 0 on success, negative value on failure
  4044. **/
  4045. #ifdef I40E_FCOE
  4046. int i40e_open(struct net_device *netdev)
  4047. #else
  4048. static int i40e_open(struct net_device *netdev)
  4049. #endif
  4050. {
  4051. struct i40e_netdev_priv *np = netdev_priv(netdev);
  4052. struct i40e_vsi *vsi = np->vsi;
  4053. struct i40e_pf *pf = vsi->back;
  4054. int err;
  4055. /* disallow open during test or if eeprom is broken */
  4056. if (test_bit(__I40E_TESTING, &pf->state) ||
  4057. test_bit(__I40E_BAD_EEPROM, &pf->state))
  4058. return -EBUSY;
  4059. netif_carrier_off(netdev);
  4060. err = i40e_vsi_open(vsi);
  4061. if (err)
  4062. return err;
  4063. /* configure global TSO hardware offload settings */
  4064. wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
  4065. TCP_FLAG_FIN) >> 16);
  4066. wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
  4067. TCP_FLAG_FIN |
  4068. TCP_FLAG_CWR) >> 16);
  4069. wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
  4070. #ifdef CONFIG_I40E_VXLAN
  4071. vxlan_get_rx_port(netdev);
  4072. #endif
  4073. return 0;
  4074. }
  4075. /**
  4076. * i40e_vsi_open -
  4077. * @vsi: the VSI to open
  4078. *
  4079. * Finish initialization of the VSI.
  4080. *
  4081. * Returns 0 on success, negative value on failure
  4082. **/
  4083. int i40e_vsi_open(struct i40e_vsi *vsi)
  4084. {
  4085. struct i40e_pf *pf = vsi->back;
  4086. char int_name[IFNAMSIZ];
  4087. int err;
  4088. /* allocate descriptors */
  4089. err = i40e_vsi_setup_tx_resources(vsi);
  4090. if (err)
  4091. goto err_setup_tx;
  4092. err = i40e_vsi_setup_rx_resources(vsi);
  4093. if (err)
  4094. goto err_setup_rx;
  4095. err = i40e_vsi_configure(vsi);
  4096. if (err)
  4097. goto err_setup_rx;
  4098. if (vsi->netdev) {
  4099. snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
  4100. dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
  4101. err = i40e_vsi_request_irq(vsi, int_name);
  4102. if (err)
  4103. goto err_setup_rx;
  4104. /* Notify the stack of the actual queue counts. */
  4105. err = netif_set_real_num_tx_queues(vsi->netdev,
  4106. vsi->num_queue_pairs);
  4107. if (err)
  4108. goto err_set_queues;
  4109. err = netif_set_real_num_rx_queues(vsi->netdev,
  4110. vsi->num_queue_pairs);
  4111. if (err)
  4112. goto err_set_queues;
  4113. } else if (vsi->type == I40E_VSI_FDIR) {
  4114. snprintf(int_name, sizeof(int_name) - 1, "%s-fdir",
  4115. dev_driver_string(&pf->pdev->dev));
  4116. err = i40e_vsi_request_irq(vsi, int_name);
  4117. } else {
  4118. err = -EINVAL;
  4119. goto err_setup_rx;
  4120. }
  4121. err = i40e_up_complete(vsi);
  4122. if (err)
  4123. goto err_up_complete;
  4124. return 0;
  4125. err_up_complete:
  4126. i40e_down(vsi);
  4127. err_set_queues:
  4128. i40e_vsi_free_irq(vsi);
  4129. err_setup_rx:
  4130. i40e_vsi_free_rx_resources(vsi);
  4131. err_setup_tx:
  4132. i40e_vsi_free_tx_resources(vsi);
  4133. if (vsi == pf->vsi[pf->lan_vsi])
  4134. i40e_do_reset(pf, (1 << __I40E_PF_RESET_REQUESTED));
  4135. return err;
  4136. }
  4137. /**
  4138. * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
  4139. * @pf: Pointer to pf
  4140. *
  4141. * This function destroys the hlist where all the Flow Director
  4142. * filters were saved.
  4143. **/
  4144. static void i40e_fdir_filter_exit(struct i40e_pf *pf)
  4145. {
  4146. struct i40e_fdir_filter *filter;
  4147. struct hlist_node *node2;
  4148. hlist_for_each_entry_safe(filter, node2,
  4149. &pf->fdir_filter_list, fdir_node) {
  4150. hlist_del(&filter->fdir_node);
  4151. kfree(filter);
  4152. }
  4153. pf->fdir_pf_active_filters = 0;
  4154. }
  4155. /**
  4156. * i40e_close - Disables a network interface
  4157. * @netdev: network interface device structure
  4158. *
  4159. * The close entry point is called when an interface is de-activated
  4160. * by the OS. The hardware is still under the driver's control, but
  4161. * this netdev interface is disabled.
  4162. *
  4163. * Returns 0, this is not allowed to fail
  4164. **/
  4165. #ifdef I40E_FCOE
  4166. int i40e_close(struct net_device *netdev)
  4167. #else
  4168. static int i40e_close(struct net_device *netdev)
  4169. #endif
  4170. {
  4171. struct i40e_netdev_priv *np = netdev_priv(netdev);
  4172. struct i40e_vsi *vsi = np->vsi;
  4173. i40e_vsi_close(vsi);
  4174. return 0;
  4175. }
  4176. /**
  4177. * i40e_do_reset - Start a PF or Core Reset sequence
  4178. * @pf: board private structure
  4179. * @reset_flags: which reset is requested
  4180. *
  4181. * The essential difference in resets is that the PF Reset
  4182. * doesn't clear the packet buffers, doesn't reset the PE
  4183. * firmware, and doesn't bother the other PFs on the chip.
  4184. **/
  4185. void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags)
  4186. {
  4187. u32 val;
  4188. WARN_ON(in_interrupt());
  4189. if (i40e_check_asq_alive(&pf->hw))
  4190. i40e_vc_notify_reset(pf);
  4191. /* do the biggest reset indicated */
  4192. if (reset_flags & (1 << __I40E_GLOBAL_RESET_REQUESTED)) {
  4193. /* Request a Global Reset
  4194. *
  4195. * This will start the chip's countdown to the actual full
  4196. * chip reset event, and a warning interrupt to be sent
  4197. * to all PFs, including the requestor. Our handler
  4198. * for the warning interrupt will deal with the shutdown
  4199. * and recovery of the switch setup.
  4200. */
  4201. dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
  4202. val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
  4203. val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
  4204. wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
  4205. } else if (reset_flags & (1 << __I40E_CORE_RESET_REQUESTED)) {
  4206. /* Request a Core Reset
  4207. *
  4208. * Same as Global Reset, except does *not* include the MAC/PHY
  4209. */
  4210. dev_dbg(&pf->pdev->dev, "CoreR requested\n");
  4211. val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
  4212. val |= I40E_GLGEN_RTRIG_CORER_MASK;
  4213. wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
  4214. i40e_flush(&pf->hw);
  4215. } else if (reset_flags & (1 << __I40E_EMP_RESET_REQUESTED)) {
  4216. /* Request a Firmware Reset
  4217. *
  4218. * Same as Global reset, plus restarting the
  4219. * embedded firmware engine.
  4220. */
  4221. /* enable EMP Reset */
  4222. val = rd32(&pf->hw, I40E_GLGEN_RSTENA_EMP);
  4223. val |= I40E_GLGEN_RSTENA_EMP_EMP_RST_ENA_MASK;
  4224. wr32(&pf->hw, I40E_GLGEN_RSTENA_EMP, val);
  4225. /* force the reset */
  4226. val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
  4227. val |= I40E_GLGEN_RTRIG_EMPFWR_MASK;
  4228. wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
  4229. i40e_flush(&pf->hw);
  4230. } else if (reset_flags & (1 << __I40E_PF_RESET_REQUESTED)) {
  4231. /* Request a PF Reset
  4232. *
  4233. * Resets only the PF-specific registers
  4234. *
  4235. * This goes directly to the tear-down and rebuild of
  4236. * the switch, since we need to do all the recovery as
  4237. * for the Core Reset.
  4238. */
  4239. dev_dbg(&pf->pdev->dev, "PFR requested\n");
  4240. i40e_handle_reset_warning(pf);
  4241. } else if (reset_flags & (1 << __I40E_REINIT_REQUESTED)) {
  4242. int v;
  4243. /* Find the VSI(s) that requested a re-init */
  4244. dev_info(&pf->pdev->dev,
  4245. "VSI reinit requested\n");
  4246. for (v = 0; v < pf->num_alloc_vsi; v++) {
  4247. struct i40e_vsi *vsi = pf->vsi[v];
  4248. if (vsi != NULL &&
  4249. test_bit(__I40E_REINIT_REQUESTED, &vsi->state)) {
  4250. i40e_vsi_reinit_locked(pf->vsi[v]);
  4251. clear_bit(__I40E_REINIT_REQUESTED, &vsi->state);
  4252. }
  4253. }
  4254. /* no further action needed, so return now */
  4255. return;
  4256. } else if (reset_flags & (1 << __I40E_DOWN_REQUESTED)) {
  4257. int v;
  4258. /* Find the VSI(s) that needs to be brought down */
  4259. dev_info(&pf->pdev->dev, "VSI down requested\n");
  4260. for (v = 0; v < pf->num_alloc_vsi; v++) {
  4261. struct i40e_vsi *vsi = pf->vsi[v];
  4262. if (vsi != NULL &&
  4263. test_bit(__I40E_DOWN_REQUESTED, &vsi->state)) {
  4264. set_bit(__I40E_DOWN, &vsi->state);
  4265. i40e_down(vsi);
  4266. clear_bit(__I40E_DOWN_REQUESTED, &vsi->state);
  4267. }
  4268. }
  4269. /* no further action needed, so return now */
  4270. return;
  4271. } else {
  4272. dev_info(&pf->pdev->dev,
  4273. "bad reset request 0x%08x\n", reset_flags);
  4274. return;
  4275. }
  4276. }
  4277. #ifdef CONFIG_I40E_DCB
  4278. /**
  4279. * i40e_dcb_need_reconfig - Check if DCB needs reconfig
  4280. * @pf: board private structure
  4281. * @old_cfg: current DCB config
  4282. * @new_cfg: new DCB config
  4283. **/
  4284. bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
  4285. struct i40e_dcbx_config *old_cfg,
  4286. struct i40e_dcbx_config *new_cfg)
  4287. {
  4288. bool need_reconfig = false;
  4289. /* Check if ETS configuration has changed */
  4290. if (memcmp(&new_cfg->etscfg,
  4291. &old_cfg->etscfg,
  4292. sizeof(new_cfg->etscfg))) {
  4293. /* If Priority Table has changed reconfig is needed */
  4294. if (memcmp(&new_cfg->etscfg.prioritytable,
  4295. &old_cfg->etscfg.prioritytable,
  4296. sizeof(new_cfg->etscfg.prioritytable))) {
  4297. need_reconfig = true;
  4298. dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
  4299. }
  4300. if (memcmp(&new_cfg->etscfg.tcbwtable,
  4301. &old_cfg->etscfg.tcbwtable,
  4302. sizeof(new_cfg->etscfg.tcbwtable)))
  4303. dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
  4304. if (memcmp(&new_cfg->etscfg.tsatable,
  4305. &old_cfg->etscfg.tsatable,
  4306. sizeof(new_cfg->etscfg.tsatable)))
  4307. dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
  4308. }
  4309. /* Check if PFC configuration has changed */
  4310. if (memcmp(&new_cfg->pfc,
  4311. &old_cfg->pfc,
  4312. sizeof(new_cfg->pfc))) {
  4313. need_reconfig = true;
  4314. dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
  4315. }
  4316. /* Check if APP Table has changed */
  4317. if (memcmp(&new_cfg->app,
  4318. &old_cfg->app,
  4319. sizeof(new_cfg->app))) {
  4320. need_reconfig = true;
  4321. dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
  4322. }
  4323. return need_reconfig;
  4324. }
  4325. /**
  4326. * i40e_handle_lldp_event - Handle LLDP Change MIB event
  4327. * @pf: board private structure
  4328. * @e: event info posted on ARQ
  4329. **/
  4330. static int i40e_handle_lldp_event(struct i40e_pf *pf,
  4331. struct i40e_arq_event_info *e)
  4332. {
  4333. struct i40e_aqc_lldp_get_mib *mib =
  4334. (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
  4335. struct i40e_hw *hw = &pf->hw;
  4336. struct i40e_dcbx_config *dcbx_cfg = &hw->local_dcbx_config;
  4337. struct i40e_dcbx_config tmp_dcbx_cfg;
  4338. bool need_reconfig = false;
  4339. int ret = 0;
  4340. u8 type;
  4341. /* Not DCB capable or capability disabled */
  4342. if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
  4343. return ret;
  4344. /* Ignore if event is not for Nearest Bridge */
  4345. type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
  4346. & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
  4347. if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
  4348. return ret;
  4349. /* Check MIB Type and return if event for Remote MIB update */
  4350. type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
  4351. if (type == I40E_AQ_LLDP_MIB_REMOTE) {
  4352. /* Update the remote cached instance and return */
  4353. ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
  4354. I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
  4355. &hw->remote_dcbx_config);
  4356. goto exit;
  4357. }
  4358. /* Convert/store the DCBX data from LLDPDU temporarily */
  4359. memset(&tmp_dcbx_cfg, 0, sizeof(tmp_dcbx_cfg));
  4360. ret = i40e_lldp_to_dcb_config(e->msg_buf, &tmp_dcbx_cfg);
  4361. if (ret) {
  4362. /* Error in LLDPDU parsing return */
  4363. dev_info(&pf->pdev->dev, "Failed parsing LLDPDU from event buffer\n");
  4364. goto exit;
  4365. }
  4366. /* No change detected in DCBX configs */
  4367. if (!memcmp(&tmp_dcbx_cfg, dcbx_cfg, sizeof(tmp_dcbx_cfg))) {
  4368. dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
  4369. goto exit;
  4370. }
  4371. need_reconfig = i40e_dcb_need_reconfig(pf, dcbx_cfg, &tmp_dcbx_cfg);
  4372. i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg);
  4373. /* Overwrite the new configuration */
  4374. *dcbx_cfg = tmp_dcbx_cfg;
  4375. if (!need_reconfig)
  4376. goto exit;
  4377. /* Enable DCB tagging only when more than one TC */
  4378. if (i40e_dcb_get_num_tc(dcbx_cfg) > 1)
  4379. pf->flags |= I40E_FLAG_DCB_ENABLED;
  4380. else
  4381. pf->flags &= ~I40E_FLAG_DCB_ENABLED;
  4382. /* Reconfiguration needed quiesce all VSIs */
  4383. i40e_pf_quiesce_all_vsi(pf);
  4384. /* Changes in configuration update VEB/VSI */
  4385. i40e_dcb_reconfigure(pf);
  4386. i40e_pf_unquiesce_all_vsi(pf);
  4387. exit:
  4388. return ret;
  4389. }
  4390. #endif /* CONFIG_I40E_DCB */
  4391. /**
  4392. * i40e_do_reset_safe - Protected reset path for userland calls.
  4393. * @pf: board private structure
  4394. * @reset_flags: which reset is requested
  4395. *
  4396. **/
  4397. void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
  4398. {
  4399. rtnl_lock();
  4400. i40e_do_reset(pf, reset_flags);
  4401. rtnl_unlock();
  4402. }
  4403. /**
  4404. * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
  4405. * @pf: board private structure
  4406. * @e: event info posted on ARQ
  4407. *
  4408. * Handler for LAN Queue Overflow Event generated by the firmware for PF
  4409. * and VF queues
  4410. **/
  4411. static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
  4412. struct i40e_arq_event_info *e)
  4413. {
  4414. struct i40e_aqc_lan_overflow *data =
  4415. (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
  4416. u32 queue = le32_to_cpu(data->prtdcb_rupto);
  4417. u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
  4418. struct i40e_hw *hw = &pf->hw;
  4419. struct i40e_vf *vf;
  4420. u16 vf_id;
  4421. dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
  4422. queue, qtx_ctl);
  4423. /* Queue belongs to VF, find the VF and issue VF reset */
  4424. if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
  4425. >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
  4426. vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
  4427. >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
  4428. vf_id -= hw->func_caps.vf_base_id;
  4429. vf = &pf->vf[vf_id];
  4430. i40e_vc_notify_vf_reset(vf);
  4431. /* Allow VF to process pending reset notification */
  4432. msleep(20);
  4433. i40e_reset_vf(vf, false);
  4434. }
  4435. }
  4436. /**
  4437. * i40e_service_event_complete - Finish up the service event
  4438. * @pf: board private structure
  4439. **/
  4440. static void i40e_service_event_complete(struct i40e_pf *pf)
  4441. {
  4442. BUG_ON(!test_bit(__I40E_SERVICE_SCHED, &pf->state));
  4443. /* flush memory to make sure state is correct before next watchog */
  4444. smp_mb__before_atomic();
  4445. clear_bit(__I40E_SERVICE_SCHED, &pf->state);
  4446. }
  4447. /**
  4448. * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
  4449. * @pf: board private structure
  4450. **/
  4451. int i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
  4452. {
  4453. int val, fcnt_prog;
  4454. val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
  4455. fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
  4456. return fcnt_prog;
  4457. }
  4458. /**
  4459. * i40e_get_current_fd_count - Get the count of total FD filters programmed
  4460. * @pf: board private structure
  4461. **/
  4462. int i40e_get_current_fd_count(struct i40e_pf *pf)
  4463. {
  4464. int val, fcnt_prog;
  4465. val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
  4466. fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
  4467. ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
  4468. I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
  4469. return fcnt_prog;
  4470. }
  4471. /**
  4472. * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
  4473. * @pf: board private structure
  4474. **/
  4475. void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
  4476. {
  4477. u32 fcnt_prog, fcnt_avail;
  4478. /* Check if, FD SB or ATR was auto disabled and if there is enough room
  4479. * to re-enable
  4480. */
  4481. if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
  4482. (pf->flags & I40E_FLAG_FD_SB_ENABLED))
  4483. return;
  4484. fcnt_prog = i40e_get_cur_guaranteed_fd_count(pf);
  4485. fcnt_avail = pf->fdir_pf_filter_count;
  4486. if (fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) {
  4487. if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
  4488. (pf->auto_disable_flags & I40E_FLAG_FD_SB_ENABLED)) {
  4489. pf->auto_disable_flags &= ~I40E_FLAG_FD_SB_ENABLED;
  4490. dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
  4491. }
  4492. }
  4493. /* Wait for some more space to be available to turn on ATR */
  4494. if (fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM * 2)) {
  4495. if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
  4496. (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED)) {
  4497. pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
  4498. dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table now\n");
  4499. }
  4500. }
  4501. }
  4502. /**
  4503. * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
  4504. * @pf: board private structure
  4505. **/
  4506. static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
  4507. {
  4508. if (!(pf->flags & I40E_FLAG_FDIR_REQUIRES_REINIT))
  4509. return;
  4510. /* if interface is down do nothing */
  4511. if (test_bit(__I40E_DOWN, &pf->state))
  4512. return;
  4513. i40e_fdir_check_and_reenable(pf);
  4514. if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
  4515. (pf->flags & I40E_FLAG_FD_SB_ENABLED))
  4516. pf->flags &= ~I40E_FLAG_FDIR_REQUIRES_REINIT;
  4517. }
  4518. /**
  4519. * i40e_vsi_link_event - notify VSI of a link event
  4520. * @vsi: vsi to be notified
  4521. * @link_up: link up or down
  4522. **/
  4523. static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
  4524. {
  4525. if (!vsi)
  4526. return;
  4527. switch (vsi->type) {
  4528. case I40E_VSI_MAIN:
  4529. #ifdef I40E_FCOE
  4530. case I40E_VSI_FCOE:
  4531. #endif
  4532. if (!vsi->netdev || !vsi->netdev_registered)
  4533. break;
  4534. if (link_up) {
  4535. netif_carrier_on(vsi->netdev);
  4536. netif_tx_wake_all_queues(vsi->netdev);
  4537. } else {
  4538. netif_carrier_off(vsi->netdev);
  4539. netif_tx_stop_all_queues(vsi->netdev);
  4540. }
  4541. break;
  4542. case I40E_VSI_SRIOV:
  4543. break;
  4544. case I40E_VSI_VMDQ2:
  4545. case I40E_VSI_CTRL:
  4546. case I40E_VSI_MIRROR:
  4547. default:
  4548. /* there is no notification for other VSIs */
  4549. break;
  4550. }
  4551. }
  4552. /**
  4553. * i40e_veb_link_event - notify elements on the veb of a link event
  4554. * @veb: veb to be notified
  4555. * @link_up: link up or down
  4556. **/
  4557. static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
  4558. {
  4559. struct i40e_pf *pf;
  4560. int i;
  4561. if (!veb || !veb->pf)
  4562. return;
  4563. pf = veb->pf;
  4564. /* depth first... */
  4565. for (i = 0; i < I40E_MAX_VEB; i++)
  4566. if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
  4567. i40e_veb_link_event(pf->veb[i], link_up);
  4568. /* ... now the local VSIs */
  4569. for (i = 0; i < pf->num_alloc_vsi; i++)
  4570. if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
  4571. i40e_vsi_link_event(pf->vsi[i], link_up);
  4572. }
  4573. /**
  4574. * i40e_link_event - Update netif_carrier status
  4575. * @pf: board private structure
  4576. **/
  4577. static void i40e_link_event(struct i40e_pf *pf)
  4578. {
  4579. bool new_link, old_link;
  4580. new_link = (pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP);
  4581. old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
  4582. if (new_link == old_link)
  4583. return;
  4584. if (!test_bit(__I40E_DOWN, &pf->vsi[pf->lan_vsi]->state))
  4585. i40e_print_link_message(pf->vsi[pf->lan_vsi], new_link);
  4586. /* Notify the base of the switch tree connected to
  4587. * the link. Floating VEBs are not notified.
  4588. */
  4589. if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
  4590. i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
  4591. else
  4592. i40e_vsi_link_event(pf->vsi[pf->lan_vsi], new_link);
  4593. if (pf->vf)
  4594. i40e_vc_notify_link_state(pf);
  4595. if (pf->flags & I40E_FLAG_PTP)
  4596. i40e_ptp_set_increment(pf);
  4597. }
  4598. /**
  4599. * i40e_check_hang_subtask - Check for hung queues and dropped interrupts
  4600. * @pf: board private structure
  4601. *
  4602. * Set the per-queue flags to request a check for stuck queues in the irq
  4603. * clean functions, then force interrupts to be sure the irq clean is called.
  4604. **/
  4605. static void i40e_check_hang_subtask(struct i40e_pf *pf)
  4606. {
  4607. int i, v;
  4608. /* If we're down or resetting, just bail */
  4609. if (test_bit(__I40E_CONFIG_BUSY, &pf->state))
  4610. return;
  4611. /* for each VSI/netdev
  4612. * for each Tx queue
  4613. * set the check flag
  4614. * for each q_vector
  4615. * force an interrupt
  4616. */
  4617. for (v = 0; v < pf->num_alloc_vsi; v++) {
  4618. struct i40e_vsi *vsi = pf->vsi[v];
  4619. int armed = 0;
  4620. if (!pf->vsi[v] ||
  4621. test_bit(__I40E_DOWN, &vsi->state) ||
  4622. (vsi->netdev && !netif_carrier_ok(vsi->netdev)))
  4623. continue;
  4624. for (i = 0; i < vsi->num_queue_pairs; i++) {
  4625. set_check_for_tx_hang(vsi->tx_rings[i]);
  4626. if (test_bit(__I40E_HANG_CHECK_ARMED,
  4627. &vsi->tx_rings[i]->state))
  4628. armed++;
  4629. }
  4630. if (armed) {
  4631. if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
  4632. wr32(&vsi->back->hw, I40E_PFINT_DYN_CTL0,
  4633. (I40E_PFINT_DYN_CTL0_INTENA_MASK |
  4634. I40E_PFINT_DYN_CTL0_SWINT_TRIG_MASK));
  4635. } else {
  4636. u16 vec = vsi->base_vector - 1;
  4637. u32 val = (I40E_PFINT_DYN_CTLN_INTENA_MASK |
  4638. I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK);
  4639. for (i = 0; i < vsi->num_q_vectors; i++, vec++)
  4640. wr32(&vsi->back->hw,
  4641. I40E_PFINT_DYN_CTLN(vec), val);
  4642. }
  4643. i40e_flush(&vsi->back->hw);
  4644. }
  4645. }
  4646. }
  4647. /**
  4648. * i40e_watchdog_subtask - Check and bring link up
  4649. * @pf: board private structure
  4650. **/
  4651. static void i40e_watchdog_subtask(struct i40e_pf *pf)
  4652. {
  4653. int i;
  4654. /* if interface is down do nothing */
  4655. if (test_bit(__I40E_DOWN, &pf->state) ||
  4656. test_bit(__I40E_CONFIG_BUSY, &pf->state))
  4657. return;
  4658. /* Update the stats for active netdevs so the network stack
  4659. * can look at updated numbers whenever it cares to
  4660. */
  4661. for (i = 0; i < pf->num_alloc_vsi; i++)
  4662. if (pf->vsi[i] && pf->vsi[i]->netdev)
  4663. i40e_update_stats(pf->vsi[i]);
  4664. /* Update the stats for the active switching components */
  4665. for (i = 0; i < I40E_MAX_VEB; i++)
  4666. if (pf->veb[i])
  4667. i40e_update_veb_stats(pf->veb[i]);
  4668. i40e_ptp_rx_hang(pf->vsi[pf->lan_vsi]);
  4669. }
  4670. /**
  4671. * i40e_reset_subtask - Set up for resetting the device and driver
  4672. * @pf: board private structure
  4673. **/
  4674. static void i40e_reset_subtask(struct i40e_pf *pf)
  4675. {
  4676. u32 reset_flags = 0;
  4677. rtnl_lock();
  4678. if (test_bit(__I40E_REINIT_REQUESTED, &pf->state)) {
  4679. reset_flags |= (1 << __I40E_REINIT_REQUESTED);
  4680. clear_bit(__I40E_REINIT_REQUESTED, &pf->state);
  4681. }
  4682. if (test_bit(__I40E_PF_RESET_REQUESTED, &pf->state)) {
  4683. reset_flags |= (1 << __I40E_PF_RESET_REQUESTED);
  4684. clear_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
  4685. }
  4686. if (test_bit(__I40E_CORE_RESET_REQUESTED, &pf->state)) {
  4687. reset_flags |= (1 << __I40E_CORE_RESET_REQUESTED);
  4688. clear_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
  4689. }
  4690. if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state)) {
  4691. reset_flags |= (1 << __I40E_GLOBAL_RESET_REQUESTED);
  4692. clear_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
  4693. }
  4694. if (test_bit(__I40E_DOWN_REQUESTED, &pf->state)) {
  4695. reset_flags |= (1 << __I40E_DOWN_REQUESTED);
  4696. clear_bit(__I40E_DOWN_REQUESTED, &pf->state);
  4697. }
  4698. /* If there's a recovery already waiting, it takes
  4699. * precedence before starting a new reset sequence.
  4700. */
  4701. if (test_bit(__I40E_RESET_INTR_RECEIVED, &pf->state)) {
  4702. i40e_handle_reset_warning(pf);
  4703. goto unlock;
  4704. }
  4705. /* If we're already down or resetting, just bail */
  4706. if (reset_flags &&
  4707. !test_bit(__I40E_DOWN, &pf->state) &&
  4708. !test_bit(__I40E_CONFIG_BUSY, &pf->state))
  4709. i40e_do_reset(pf, reset_flags);
  4710. unlock:
  4711. rtnl_unlock();
  4712. }
  4713. /**
  4714. * i40e_handle_link_event - Handle link event
  4715. * @pf: board private structure
  4716. * @e: event info posted on ARQ
  4717. **/
  4718. static void i40e_handle_link_event(struct i40e_pf *pf,
  4719. struct i40e_arq_event_info *e)
  4720. {
  4721. struct i40e_hw *hw = &pf->hw;
  4722. struct i40e_aqc_get_link_status *status =
  4723. (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
  4724. struct i40e_link_status *hw_link_info = &hw->phy.link_info;
  4725. /* save off old link status information */
  4726. memcpy(&pf->hw.phy.link_info_old, hw_link_info,
  4727. sizeof(pf->hw.phy.link_info_old));
  4728. /* update link status */
  4729. hw_link_info->phy_type = (enum i40e_aq_phy_type)status->phy_type;
  4730. hw_link_info->link_speed = (enum i40e_aq_link_speed)status->link_speed;
  4731. hw_link_info->link_info = status->link_info;
  4732. hw_link_info->an_info = status->an_info;
  4733. hw_link_info->ext_info = status->ext_info;
  4734. hw_link_info->lse_enable =
  4735. le16_to_cpu(status->command_flags) &
  4736. I40E_AQ_LSE_ENABLE;
  4737. /* process the event */
  4738. i40e_link_event(pf);
  4739. /* Do a new status request to re-enable LSE reporting
  4740. * and load new status information into the hw struct,
  4741. * then see if the status changed while processing the
  4742. * initial event.
  4743. */
  4744. i40e_update_link_info(&pf->hw, true);
  4745. i40e_link_event(pf);
  4746. }
  4747. /**
  4748. * i40e_clean_adminq_subtask - Clean the AdminQ rings
  4749. * @pf: board private structure
  4750. **/
  4751. static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
  4752. {
  4753. struct i40e_arq_event_info event;
  4754. struct i40e_hw *hw = &pf->hw;
  4755. u16 pending, i = 0;
  4756. i40e_status ret;
  4757. u16 opcode;
  4758. u32 oldval;
  4759. u32 val;
  4760. /* check for error indications */
  4761. val = rd32(&pf->hw, pf->hw.aq.arq.len);
  4762. oldval = val;
  4763. if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
  4764. dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
  4765. val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
  4766. }
  4767. if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
  4768. dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
  4769. val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
  4770. }
  4771. if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
  4772. dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
  4773. val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
  4774. }
  4775. if (oldval != val)
  4776. wr32(&pf->hw, pf->hw.aq.arq.len, val);
  4777. val = rd32(&pf->hw, pf->hw.aq.asq.len);
  4778. oldval = val;
  4779. if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
  4780. dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
  4781. val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
  4782. }
  4783. if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
  4784. dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
  4785. val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
  4786. }
  4787. if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
  4788. dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
  4789. val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
  4790. }
  4791. if (oldval != val)
  4792. wr32(&pf->hw, pf->hw.aq.asq.len, val);
  4793. event.msg_size = I40E_MAX_AQ_BUF_SIZE;
  4794. event.msg_buf = kzalloc(event.msg_size, GFP_KERNEL);
  4795. if (!event.msg_buf)
  4796. return;
  4797. do {
  4798. event.msg_size = I40E_MAX_AQ_BUF_SIZE; /* reinit each time */
  4799. ret = i40e_clean_arq_element(hw, &event, &pending);
  4800. if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
  4801. break;
  4802. else if (ret) {
  4803. dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
  4804. break;
  4805. }
  4806. opcode = le16_to_cpu(event.desc.opcode);
  4807. switch (opcode) {
  4808. case i40e_aqc_opc_get_link_status:
  4809. i40e_handle_link_event(pf, &event);
  4810. break;
  4811. case i40e_aqc_opc_send_msg_to_pf:
  4812. ret = i40e_vc_process_vf_msg(pf,
  4813. le16_to_cpu(event.desc.retval),
  4814. le32_to_cpu(event.desc.cookie_high),
  4815. le32_to_cpu(event.desc.cookie_low),
  4816. event.msg_buf,
  4817. event.msg_size);
  4818. break;
  4819. case i40e_aqc_opc_lldp_update_mib:
  4820. dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
  4821. #ifdef CONFIG_I40E_DCB
  4822. rtnl_lock();
  4823. ret = i40e_handle_lldp_event(pf, &event);
  4824. rtnl_unlock();
  4825. #endif /* CONFIG_I40E_DCB */
  4826. break;
  4827. case i40e_aqc_opc_event_lan_overflow:
  4828. dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
  4829. i40e_handle_lan_overflow_event(pf, &event);
  4830. break;
  4831. case i40e_aqc_opc_send_msg_to_peer:
  4832. dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
  4833. break;
  4834. default:
  4835. dev_info(&pf->pdev->dev,
  4836. "ARQ Error: Unknown event 0x%04x received\n",
  4837. opcode);
  4838. break;
  4839. }
  4840. } while (pending && (i++ < pf->adminq_work_limit));
  4841. clear_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
  4842. /* re-enable Admin queue interrupt cause */
  4843. val = rd32(hw, I40E_PFINT_ICR0_ENA);
  4844. val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
  4845. wr32(hw, I40E_PFINT_ICR0_ENA, val);
  4846. i40e_flush(hw);
  4847. kfree(event.msg_buf);
  4848. }
  4849. /**
  4850. * i40e_verify_eeprom - make sure eeprom is good to use
  4851. * @pf: board private structure
  4852. **/
  4853. static void i40e_verify_eeprom(struct i40e_pf *pf)
  4854. {
  4855. int err;
  4856. err = i40e_diag_eeprom_test(&pf->hw);
  4857. if (err) {
  4858. /* retry in case of garbage read */
  4859. err = i40e_diag_eeprom_test(&pf->hw);
  4860. if (err) {
  4861. dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
  4862. err);
  4863. set_bit(__I40E_BAD_EEPROM, &pf->state);
  4864. }
  4865. }
  4866. if (!err && test_bit(__I40E_BAD_EEPROM, &pf->state)) {
  4867. dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
  4868. clear_bit(__I40E_BAD_EEPROM, &pf->state);
  4869. }
  4870. }
  4871. /**
  4872. * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
  4873. * @veb: pointer to the VEB instance
  4874. *
  4875. * This is a recursive function that first builds the attached VSIs then
  4876. * recurses in to build the next layer of VEB. We track the connections
  4877. * through our own index numbers because the seid's from the HW could
  4878. * change across the reset.
  4879. **/
  4880. static int i40e_reconstitute_veb(struct i40e_veb *veb)
  4881. {
  4882. struct i40e_vsi *ctl_vsi = NULL;
  4883. struct i40e_pf *pf = veb->pf;
  4884. int v, veb_idx;
  4885. int ret;
  4886. /* build VSI that owns this VEB, temporarily attached to base VEB */
  4887. for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
  4888. if (pf->vsi[v] &&
  4889. pf->vsi[v]->veb_idx == veb->idx &&
  4890. pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
  4891. ctl_vsi = pf->vsi[v];
  4892. break;
  4893. }
  4894. }
  4895. if (!ctl_vsi) {
  4896. dev_info(&pf->pdev->dev,
  4897. "missing owner VSI for veb_idx %d\n", veb->idx);
  4898. ret = -ENOENT;
  4899. goto end_reconstitute;
  4900. }
  4901. if (ctl_vsi != pf->vsi[pf->lan_vsi])
  4902. ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
  4903. ret = i40e_add_vsi(ctl_vsi);
  4904. if (ret) {
  4905. dev_info(&pf->pdev->dev,
  4906. "rebuild of owner VSI failed: %d\n", ret);
  4907. goto end_reconstitute;
  4908. }
  4909. i40e_vsi_reset_stats(ctl_vsi);
  4910. /* create the VEB in the switch and move the VSI onto the VEB */
  4911. ret = i40e_add_veb(veb, ctl_vsi);
  4912. if (ret)
  4913. goto end_reconstitute;
  4914. /* create the remaining VSIs attached to this VEB */
  4915. for (v = 0; v < pf->num_alloc_vsi; v++) {
  4916. if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
  4917. continue;
  4918. if (pf->vsi[v]->veb_idx == veb->idx) {
  4919. struct i40e_vsi *vsi = pf->vsi[v];
  4920. vsi->uplink_seid = veb->seid;
  4921. ret = i40e_add_vsi(vsi);
  4922. if (ret) {
  4923. dev_info(&pf->pdev->dev,
  4924. "rebuild of vsi_idx %d failed: %d\n",
  4925. v, ret);
  4926. goto end_reconstitute;
  4927. }
  4928. i40e_vsi_reset_stats(vsi);
  4929. }
  4930. }
  4931. /* create any VEBs attached to this VEB - RECURSION */
  4932. for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
  4933. if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
  4934. pf->veb[veb_idx]->uplink_seid = veb->seid;
  4935. ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
  4936. if (ret)
  4937. break;
  4938. }
  4939. }
  4940. end_reconstitute:
  4941. return ret;
  4942. }
  4943. /**
  4944. * i40e_get_capabilities - get info about the HW
  4945. * @pf: the PF struct
  4946. **/
  4947. static int i40e_get_capabilities(struct i40e_pf *pf)
  4948. {
  4949. struct i40e_aqc_list_capabilities_element_resp *cap_buf;
  4950. u16 data_size;
  4951. int buf_len;
  4952. int err;
  4953. buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
  4954. do {
  4955. cap_buf = kzalloc(buf_len, GFP_KERNEL);
  4956. if (!cap_buf)
  4957. return -ENOMEM;
  4958. /* this loads the data into the hw struct for us */
  4959. err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
  4960. &data_size,
  4961. i40e_aqc_opc_list_func_capabilities,
  4962. NULL);
  4963. /* data loaded, buffer no longer needed */
  4964. kfree(cap_buf);
  4965. if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
  4966. /* retry with a larger buffer */
  4967. buf_len = data_size;
  4968. } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
  4969. dev_info(&pf->pdev->dev,
  4970. "capability discovery failed: aq=%d\n",
  4971. pf->hw.aq.asq_last_status);
  4972. return -ENODEV;
  4973. }
  4974. } while (err);
  4975. if (((pf->hw.aq.fw_maj_ver == 2) && (pf->hw.aq.fw_min_ver < 22)) ||
  4976. (pf->hw.aq.fw_maj_ver < 2)) {
  4977. pf->hw.func_caps.num_msix_vectors++;
  4978. pf->hw.func_caps.num_msix_vectors_vf++;
  4979. }
  4980. if (pf->hw.debug_mask & I40E_DEBUG_USER)
  4981. dev_info(&pf->pdev->dev,
  4982. "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
  4983. pf->hw.pf_id, pf->hw.func_caps.num_vfs,
  4984. pf->hw.func_caps.num_msix_vectors,
  4985. pf->hw.func_caps.num_msix_vectors_vf,
  4986. pf->hw.func_caps.fd_filters_guaranteed,
  4987. pf->hw.func_caps.fd_filters_best_effort,
  4988. pf->hw.func_caps.num_tx_qp,
  4989. pf->hw.func_caps.num_vsis);
  4990. #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
  4991. + pf->hw.func_caps.num_vfs)
  4992. if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) {
  4993. dev_info(&pf->pdev->dev,
  4994. "got num_vsis %d, setting num_vsis to %d\n",
  4995. pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
  4996. pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
  4997. }
  4998. return 0;
  4999. }
  5000. static int i40e_vsi_clear(struct i40e_vsi *vsi);
  5001. /**
  5002. * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
  5003. * @pf: board private structure
  5004. **/
  5005. static void i40e_fdir_sb_setup(struct i40e_pf *pf)
  5006. {
  5007. struct i40e_vsi *vsi;
  5008. int i;
  5009. /* quick workaround for an NVM issue that leaves a critical register
  5010. * uninitialized
  5011. */
  5012. if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
  5013. static const u32 hkey[] = {
  5014. 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
  5015. 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
  5016. 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
  5017. 0x95b3a76d};
  5018. for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
  5019. wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
  5020. }
  5021. if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
  5022. return;
  5023. /* find existing VSI and see if it needs configuring */
  5024. vsi = NULL;
  5025. for (i = 0; i < pf->num_alloc_vsi; i++) {
  5026. if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
  5027. vsi = pf->vsi[i];
  5028. break;
  5029. }
  5030. }
  5031. /* create a new VSI if none exists */
  5032. if (!vsi) {
  5033. vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
  5034. pf->vsi[pf->lan_vsi]->seid, 0);
  5035. if (!vsi) {
  5036. dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
  5037. pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
  5038. return;
  5039. }
  5040. }
  5041. i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
  5042. }
  5043. /**
  5044. * i40e_fdir_teardown - release the Flow Director resources
  5045. * @pf: board private structure
  5046. **/
  5047. static void i40e_fdir_teardown(struct i40e_pf *pf)
  5048. {
  5049. int i;
  5050. i40e_fdir_filter_exit(pf);
  5051. for (i = 0; i < pf->num_alloc_vsi; i++) {
  5052. if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
  5053. i40e_vsi_release(pf->vsi[i]);
  5054. break;
  5055. }
  5056. }
  5057. }
  5058. /**
  5059. * i40e_prep_for_reset - prep for the core to reset
  5060. * @pf: board private structure
  5061. *
  5062. * Close up the VFs and other things in prep for pf Reset.
  5063. **/
  5064. static void i40e_prep_for_reset(struct i40e_pf *pf)
  5065. {
  5066. struct i40e_hw *hw = &pf->hw;
  5067. i40e_status ret = 0;
  5068. u32 v;
  5069. clear_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
  5070. if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
  5071. return;
  5072. dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
  5073. /* quiesce the VSIs and their queues that are not already DOWN */
  5074. i40e_pf_quiesce_all_vsi(pf);
  5075. for (v = 0; v < pf->num_alloc_vsi; v++) {
  5076. if (pf->vsi[v])
  5077. pf->vsi[v]->seid = 0;
  5078. }
  5079. i40e_shutdown_adminq(&pf->hw);
  5080. /* call shutdown HMC */
  5081. if (hw->hmc.hmc_obj) {
  5082. ret = i40e_shutdown_lan_hmc(hw);
  5083. if (ret)
  5084. dev_warn(&pf->pdev->dev,
  5085. "shutdown_lan_hmc failed: %d\n", ret);
  5086. }
  5087. }
  5088. /**
  5089. * i40e_send_version - update firmware with driver version
  5090. * @pf: PF struct
  5091. */
  5092. static void i40e_send_version(struct i40e_pf *pf)
  5093. {
  5094. struct i40e_driver_version dv;
  5095. dv.major_version = DRV_VERSION_MAJOR;
  5096. dv.minor_version = DRV_VERSION_MINOR;
  5097. dv.build_version = DRV_VERSION_BUILD;
  5098. dv.subbuild_version = 0;
  5099. strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string));
  5100. i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
  5101. }
  5102. /**
  5103. * i40e_reset_and_rebuild - reset and rebuild using a saved config
  5104. * @pf: board private structure
  5105. * @reinit: if the Main VSI needs to re-initialized.
  5106. **/
  5107. static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit)
  5108. {
  5109. struct i40e_hw *hw = &pf->hw;
  5110. i40e_status ret;
  5111. u32 v;
  5112. /* Now we wait for GRST to settle out.
  5113. * We don't have to delete the VEBs or VSIs from the hw switch
  5114. * because the reset will make them disappear.
  5115. */
  5116. ret = i40e_pf_reset(hw);
  5117. if (ret) {
  5118. dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
  5119. goto end_core_reset;
  5120. }
  5121. pf->pfr_count++;
  5122. if (test_bit(__I40E_DOWN, &pf->state))
  5123. goto end_core_reset;
  5124. dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
  5125. /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
  5126. ret = i40e_init_adminq(&pf->hw);
  5127. if (ret) {
  5128. dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, %d\n", ret);
  5129. goto end_core_reset;
  5130. }
  5131. /* re-verify the eeprom if we just had an EMP reset */
  5132. if (test_bit(__I40E_EMP_RESET_REQUESTED, &pf->state)) {
  5133. clear_bit(__I40E_EMP_RESET_REQUESTED, &pf->state);
  5134. i40e_verify_eeprom(pf);
  5135. }
  5136. i40e_clear_pxe_mode(hw);
  5137. ret = i40e_get_capabilities(pf);
  5138. if (ret) {
  5139. dev_info(&pf->pdev->dev, "i40e_get_capabilities failed, %d\n",
  5140. ret);
  5141. goto end_core_reset;
  5142. }
  5143. ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
  5144. hw->func_caps.num_rx_qp,
  5145. pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num);
  5146. if (ret) {
  5147. dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
  5148. goto end_core_reset;
  5149. }
  5150. ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
  5151. if (ret) {
  5152. dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
  5153. goto end_core_reset;
  5154. }
  5155. #ifdef CONFIG_I40E_DCB
  5156. ret = i40e_init_pf_dcb(pf);
  5157. if (ret) {
  5158. dev_info(&pf->pdev->dev, "init_pf_dcb failed: %d\n", ret);
  5159. goto end_core_reset;
  5160. }
  5161. #endif /* CONFIG_I40E_DCB */
  5162. #ifdef I40E_FCOE
  5163. ret = i40e_init_pf_fcoe(pf);
  5164. if (ret)
  5165. dev_info(&pf->pdev->dev, "init_pf_fcoe failed: %d\n", ret);
  5166. #endif
  5167. /* do basic switch setup */
  5168. ret = i40e_setup_pf_switch(pf, reinit);
  5169. if (ret)
  5170. goto end_core_reset;
  5171. /* Rebuild the VSIs and VEBs that existed before reset.
  5172. * They are still in our local switch element arrays, so only
  5173. * need to rebuild the switch model in the HW.
  5174. *
  5175. * If there were VEBs but the reconstitution failed, we'll try
  5176. * try to recover minimal use by getting the basic PF VSI working.
  5177. */
  5178. if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) {
  5179. dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
  5180. /* find the one VEB connected to the MAC, and find orphans */
  5181. for (v = 0; v < I40E_MAX_VEB; v++) {
  5182. if (!pf->veb[v])
  5183. continue;
  5184. if (pf->veb[v]->uplink_seid == pf->mac_seid ||
  5185. pf->veb[v]->uplink_seid == 0) {
  5186. ret = i40e_reconstitute_veb(pf->veb[v]);
  5187. if (!ret)
  5188. continue;
  5189. /* If Main VEB failed, we're in deep doodoo,
  5190. * so give up rebuilding the switch and set up
  5191. * for minimal rebuild of PF VSI.
  5192. * If orphan failed, we'll report the error
  5193. * but try to keep going.
  5194. */
  5195. if (pf->veb[v]->uplink_seid == pf->mac_seid) {
  5196. dev_info(&pf->pdev->dev,
  5197. "rebuild of switch failed: %d, will try to set up simple PF connection\n",
  5198. ret);
  5199. pf->vsi[pf->lan_vsi]->uplink_seid
  5200. = pf->mac_seid;
  5201. break;
  5202. } else if (pf->veb[v]->uplink_seid == 0) {
  5203. dev_info(&pf->pdev->dev,
  5204. "rebuild of orphan VEB failed: %d\n",
  5205. ret);
  5206. }
  5207. }
  5208. }
  5209. }
  5210. if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) {
  5211. dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
  5212. /* no VEB, so rebuild only the Main VSI */
  5213. ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]);
  5214. if (ret) {
  5215. dev_info(&pf->pdev->dev,
  5216. "rebuild of Main VSI failed: %d\n", ret);
  5217. goto end_core_reset;
  5218. }
  5219. }
  5220. /* reinit the misc interrupt */
  5221. if (pf->flags & I40E_FLAG_MSIX_ENABLED)
  5222. ret = i40e_setup_misc_vector(pf);
  5223. /* restart the VSIs that were rebuilt and running before the reset */
  5224. i40e_pf_unquiesce_all_vsi(pf);
  5225. if (pf->num_alloc_vfs) {
  5226. for (v = 0; v < pf->num_alloc_vfs; v++)
  5227. i40e_reset_vf(&pf->vf[v], true);
  5228. }
  5229. /* tell the firmware that we're starting */
  5230. i40e_send_version(pf);
  5231. end_core_reset:
  5232. clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state);
  5233. }
  5234. /**
  5235. * i40e_handle_reset_warning - prep for the pf to reset, reset and rebuild
  5236. * @pf: board private structure
  5237. *
  5238. * Close up the VFs and other things in prep for a Core Reset,
  5239. * then get ready to rebuild the world.
  5240. **/
  5241. static void i40e_handle_reset_warning(struct i40e_pf *pf)
  5242. {
  5243. i40e_prep_for_reset(pf);
  5244. i40e_reset_and_rebuild(pf, false);
  5245. }
  5246. /**
  5247. * i40e_handle_mdd_event
  5248. * @pf: pointer to the pf structure
  5249. *
  5250. * Called from the MDD irq handler to identify possibly malicious vfs
  5251. **/
  5252. static void i40e_handle_mdd_event(struct i40e_pf *pf)
  5253. {
  5254. struct i40e_hw *hw = &pf->hw;
  5255. bool mdd_detected = false;
  5256. bool pf_mdd_detected = false;
  5257. struct i40e_vf *vf;
  5258. u32 reg;
  5259. int i;
  5260. if (!test_bit(__I40E_MDD_EVENT_PENDING, &pf->state))
  5261. return;
  5262. /* find what triggered the MDD event */
  5263. reg = rd32(hw, I40E_GL_MDET_TX);
  5264. if (reg & I40E_GL_MDET_TX_VALID_MASK) {
  5265. u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
  5266. I40E_GL_MDET_TX_PF_NUM_SHIFT;
  5267. u8 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
  5268. I40E_GL_MDET_TX_VF_NUM_SHIFT;
  5269. u8 event = (reg & I40E_GL_MDET_TX_EVENT_SHIFT) >>
  5270. I40E_GL_MDET_TX_EVENT_SHIFT;
  5271. u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
  5272. I40E_GL_MDET_TX_QUEUE_SHIFT;
  5273. dev_info(&pf->pdev->dev,
  5274. "Malicious Driver Detection event 0x%02x on TX queue %d pf number 0x%02x vf number 0x%02x\n",
  5275. event, queue, pf_num, vf_num);
  5276. wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
  5277. mdd_detected = true;
  5278. }
  5279. reg = rd32(hw, I40E_GL_MDET_RX);
  5280. if (reg & I40E_GL_MDET_RX_VALID_MASK) {
  5281. u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
  5282. I40E_GL_MDET_RX_FUNCTION_SHIFT;
  5283. u8 event = (reg & I40E_GL_MDET_RX_EVENT_SHIFT) >>
  5284. I40E_GL_MDET_RX_EVENT_SHIFT;
  5285. u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
  5286. I40E_GL_MDET_RX_QUEUE_SHIFT;
  5287. dev_info(&pf->pdev->dev,
  5288. "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
  5289. event, queue, func);
  5290. wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
  5291. mdd_detected = true;
  5292. }
  5293. if (mdd_detected) {
  5294. reg = rd32(hw, I40E_PF_MDET_TX);
  5295. if (reg & I40E_PF_MDET_TX_VALID_MASK) {
  5296. wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
  5297. dev_info(&pf->pdev->dev,
  5298. "MDD TX event is for this function 0x%08x, requesting PF reset.\n",
  5299. reg);
  5300. pf_mdd_detected = true;
  5301. }
  5302. reg = rd32(hw, I40E_PF_MDET_RX);
  5303. if (reg & I40E_PF_MDET_RX_VALID_MASK) {
  5304. wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
  5305. dev_info(&pf->pdev->dev,
  5306. "MDD RX event is for this function 0x%08x, requesting PF reset.\n",
  5307. reg);
  5308. pf_mdd_detected = true;
  5309. }
  5310. /* Queue belongs to the PF, initiate a reset */
  5311. if (pf_mdd_detected) {
  5312. set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
  5313. i40e_service_event_schedule(pf);
  5314. }
  5315. }
  5316. /* see if one of the VFs needs its hand slapped */
  5317. for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
  5318. vf = &(pf->vf[i]);
  5319. reg = rd32(hw, I40E_VP_MDET_TX(i));
  5320. if (reg & I40E_VP_MDET_TX_VALID_MASK) {
  5321. wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
  5322. vf->num_mdd_events++;
  5323. dev_info(&pf->pdev->dev, "MDD TX event on VF %d\n", i);
  5324. }
  5325. reg = rd32(hw, I40E_VP_MDET_RX(i));
  5326. if (reg & I40E_VP_MDET_RX_VALID_MASK) {
  5327. wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
  5328. vf->num_mdd_events++;
  5329. dev_info(&pf->pdev->dev, "MDD RX event on VF %d\n", i);
  5330. }
  5331. if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) {
  5332. dev_info(&pf->pdev->dev,
  5333. "Too many MDD events on VF %d, disabled\n", i);
  5334. dev_info(&pf->pdev->dev,
  5335. "Use PF Control I/F to re-enable the VF\n");
  5336. set_bit(I40E_VF_STAT_DISABLED, &vf->vf_states);
  5337. }
  5338. }
  5339. /* re-enable mdd interrupt cause */
  5340. clear_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
  5341. reg = rd32(hw, I40E_PFINT_ICR0_ENA);
  5342. reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
  5343. wr32(hw, I40E_PFINT_ICR0_ENA, reg);
  5344. i40e_flush(hw);
  5345. }
  5346. #ifdef CONFIG_I40E_VXLAN
  5347. /**
  5348. * i40e_sync_vxlan_filters_subtask - Sync the VSI filter list with HW
  5349. * @pf: board private structure
  5350. **/
  5351. static void i40e_sync_vxlan_filters_subtask(struct i40e_pf *pf)
  5352. {
  5353. struct i40e_hw *hw = &pf->hw;
  5354. i40e_status ret;
  5355. u8 filter_index;
  5356. __be16 port;
  5357. int i;
  5358. if (!(pf->flags & I40E_FLAG_VXLAN_FILTER_SYNC))
  5359. return;
  5360. pf->flags &= ~I40E_FLAG_VXLAN_FILTER_SYNC;
  5361. for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
  5362. if (pf->pending_vxlan_bitmap & (1 << i)) {
  5363. pf->pending_vxlan_bitmap &= ~(1 << i);
  5364. port = pf->vxlan_ports[i];
  5365. ret = port ?
  5366. i40e_aq_add_udp_tunnel(hw, ntohs(port),
  5367. I40E_AQC_TUNNEL_TYPE_VXLAN,
  5368. &filter_index, NULL)
  5369. : i40e_aq_del_udp_tunnel(hw, i, NULL);
  5370. if (ret) {
  5371. dev_info(&pf->pdev->dev, "Failed to execute AQ command for %s port %d with index %d\n",
  5372. port ? "adding" : "deleting",
  5373. ntohs(port), port ? i : i);
  5374. pf->vxlan_ports[i] = 0;
  5375. } else {
  5376. dev_info(&pf->pdev->dev, "%s port %d with AQ command with index %d\n",
  5377. port ? "Added" : "Deleted",
  5378. ntohs(port), port ? i : filter_index);
  5379. }
  5380. }
  5381. }
  5382. }
  5383. #endif
  5384. /**
  5385. * i40e_service_task - Run the driver's async subtasks
  5386. * @work: pointer to work_struct containing our data
  5387. **/
  5388. static void i40e_service_task(struct work_struct *work)
  5389. {
  5390. struct i40e_pf *pf = container_of(work,
  5391. struct i40e_pf,
  5392. service_task);
  5393. unsigned long start_time = jiffies;
  5394. /* don't bother with service tasks if a reset is in progress */
  5395. if (test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) {
  5396. i40e_service_event_complete(pf);
  5397. return;
  5398. }
  5399. i40e_reset_subtask(pf);
  5400. i40e_handle_mdd_event(pf);
  5401. i40e_vc_process_vflr_event(pf);
  5402. i40e_watchdog_subtask(pf);
  5403. i40e_fdir_reinit_subtask(pf);
  5404. i40e_check_hang_subtask(pf);
  5405. i40e_sync_filters_subtask(pf);
  5406. #ifdef CONFIG_I40E_VXLAN
  5407. i40e_sync_vxlan_filters_subtask(pf);
  5408. #endif
  5409. i40e_clean_adminq_subtask(pf);
  5410. i40e_service_event_complete(pf);
  5411. /* If the tasks have taken longer than one timer cycle or there
  5412. * is more work to be done, reschedule the service task now
  5413. * rather than wait for the timer to tick again.
  5414. */
  5415. if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
  5416. test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state) ||
  5417. test_bit(__I40E_MDD_EVENT_PENDING, &pf->state) ||
  5418. test_bit(__I40E_VFLR_EVENT_PENDING, &pf->state))
  5419. i40e_service_event_schedule(pf);
  5420. }
  5421. /**
  5422. * i40e_service_timer - timer callback
  5423. * @data: pointer to PF struct
  5424. **/
  5425. static void i40e_service_timer(unsigned long data)
  5426. {
  5427. struct i40e_pf *pf = (struct i40e_pf *)data;
  5428. mod_timer(&pf->service_timer,
  5429. round_jiffies(jiffies + pf->service_timer_period));
  5430. i40e_service_event_schedule(pf);
  5431. }
  5432. /**
  5433. * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
  5434. * @vsi: the VSI being configured
  5435. **/
  5436. static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
  5437. {
  5438. struct i40e_pf *pf = vsi->back;
  5439. switch (vsi->type) {
  5440. case I40E_VSI_MAIN:
  5441. vsi->alloc_queue_pairs = pf->num_lan_qps;
  5442. vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
  5443. I40E_REQ_DESCRIPTOR_MULTIPLE);
  5444. if (pf->flags & I40E_FLAG_MSIX_ENABLED)
  5445. vsi->num_q_vectors = pf->num_lan_msix;
  5446. else
  5447. vsi->num_q_vectors = 1;
  5448. break;
  5449. case I40E_VSI_FDIR:
  5450. vsi->alloc_queue_pairs = 1;
  5451. vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT,
  5452. I40E_REQ_DESCRIPTOR_MULTIPLE);
  5453. vsi->num_q_vectors = 1;
  5454. break;
  5455. case I40E_VSI_VMDQ2:
  5456. vsi->alloc_queue_pairs = pf->num_vmdq_qps;
  5457. vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
  5458. I40E_REQ_DESCRIPTOR_MULTIPLE);
  5459. vsi->num_q_vectors = pf->num_vmdq_msix;
  5460. break;
  5461. case I40E_VSI_SRIOV:
  5462. vsi->alloc_queue_pairs = pf->num_vf_qps;
  5463. vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
  5464. I40E_REQ_DESCRIPTOR_MULTIPLE);
  5465. break;
  5466. #ifdef I40E_FCOE
  5467. case I40E_VSI_FCOE:
  5468. vsi->alloc_queue_pairs = pf->num_fcoe_qps;
  5469. vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
  5470. I40E_REQ_DESCRIPTOR_MULTIPLE);
  5471. vsi->num_q_vectors = pf->num_fcoe_msix;
  5472. break;
  5473. #endif /* I40E_FCOE */
  5474. default:
  5475. WARN_ON(1);
  5476. return -ENODATA;
  5477. }
  5478. return 0;
  5479. }
  5480. /**
  5481. * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
  5482. * @type: VSI pointer
  5483. * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
  5484. *
  5485. * On error: returns error code (negative)
  5486. * On success: returns 0
  5487. **/
  5488. static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
  5489. {
  5490. int size;
  5491. int ret = 0;
  5492. /* allocate memory for both Tx and Rx ring pointers */
  5493. size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 2;
  5494. vsi->tx_rings = kzalloc(size, GFP_KERNEL);
  5495. if (!vsi->tx_rings)
  5496. return -ENOMEM;
  5497. vsi->rx_rings = &vsi->tx_rings[vsi->alloc_queue_pairs];
  5498. if (alloc_qvectors) {
  5499. /* allocate memory for q_vector pointers */
  5500. size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
  5501. vsi->q_vectors = kzalloc(size, GFP_KERNEL);
  5502. if (!vsi->q_vectors) {
  5503. ret = -ENOMEM;
  5504. goto err_vectors;
  5505. }
  5506. }
  5507. return ret;
  5508. err_vectors:
  5509. kfree(vsi->tx_rings);
  5510. return ret;
  5511. }
  5512. /**
  5513. * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
  5514. * @pf: board private structure
  5515. * @type: type of VSI
  5516. *
  5517. * On error: returns error code (negative)
  5518. * On success: returns vsi index in PF (positive)
  5519. **/
  5520. static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
  5521. {
  5522. int ret = -ENODEV;
  5523. struct i40e_vsi *vsi;
  5524. int vsi_idx;
  5525. int i;
  5526. /* Need to protect the allocation of the VSIs at the PF level */
  5527. mutex_lock(&pf->switch_mutex);
  5528. /* VSI list may be fragmented if VSI creation/destruction has
  5529. * been happening. We can afford to do a quick scan to look
  5530. * for any free VSIs in the list.
  5531. *
  5532. * find next empty vsi slot, looping back around if necessary
  5533. */
  5534. i = pf->next_vsi;
  5535. while (i < pf->num_alloc_vsi && pf->vsi[i])
  5536. i++;
  5537. if (i >= pf->num_alloc_vsi) {
  5538. i = 0;
  5539. while (i < pf->next_vsi && pf->vsi[i])
  5540. i++;
  5541. }
  5542. if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
  5543. vsi_idx = i; /* Found one! */
  5544. } else {
  5545. ret = -ENODEV;
  5546. goto unlock_pf; /* out of VSI slots! */
  5547. }
  5548. pf->next_vsi = ++i;
  5549. vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
  5550. if (!vsi) {
  5551. ret = -ENOMEM;
  5552. goto unlock_pf;
  5553. }
  5554. vsi->type = type;
  5555. vsi->back = pf;
  5556. set_bit(__I40E_DOWN, &vsi->state);
  5557. vsi->flags = 0;
  5558. vsi->idx = vsi_idx;
  5559. vsi->rx_itr_setting = pf->rx_itr_default;
  5560. vsi->tx_itr_setting = pf->tx_itr_default;
  5561. vsi->netdev_registered = false;
  5562. vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
  5563. INIT_LIST_HEAD(&vsi->mac_filter_list);
  5564. vsi->irqs_ready = false;
  5565. ret = i40e_set_num_rings_in_vsi(vsi);
  5566. if (ret)
  5567. goto err_rings;
  5568. ret = i40e_vsi_alloc_arrays(vsi, true);
  5569. if (ret)
  5570. goto err_rings;
  5571. /* Setup default MSIX irq handler for VSI */
  5572. i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
  5573. pf->vsi[vsi_idx] = vsi;
  5574. ret = vsi_idx;
  5575. goto unlock_pf;
  5576. err_rings:
  5577. pf->next_vsi = i - 1;
  5578. kfree(vsi);
  5579. unlock_pf:
  5580. mutex_unlock(&pf->switch_mutex);
  5581. return ret;
  5582. }
  5583. /**
  5584. * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
  5585. * @type: VSI pointer
  5586. * @free_qvectors: a bool to specify if q_vectors need to be freed.
  5587. *
  5588. * On error: returns error code (negative)
  5589. * On success: returns 0
  5590. **/
  5591. static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
  5592. {
  5593. /* free the ring and vector containers */
  5594. if (free_qvectors) {
  5595. kfree(vsi->q_vectors);
  5596. vsi->q_vectors = NULL;
  5597. }
  5598. kfree(vsi->tx_rings);
  5599. vsi->tx_rings = NULL;
  5600. vsi->rx_rings = NULL;
  5601. }
  5602. /**
  5603. * i40e_vsi_clear - Deallocate the VSI provided
  5604. * @vsi: the VSI being un-configured
  5605. **/
  5606. static int i40e_vsi_clear(struct i40e_vsi *vsi)
  5607. {
  5608. struct i40e_pf *pf;
  5609. if (!vsi)
  5610. return 0;
  5611. if (!vsi->back)
  5612. goto free_vsi;
  5613. pf = vsi->back;
  5614. mutex_lock(&pf->switch_mutex);
  5615. if (!pf->vsi[vsi->idx]) {
  5616. dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n",
  5617. vsi->idx, vsi->idx, vsi, vsi->type);
  5618. goto unlock_vsi;
  5619. }
  5620. if (pf->vsi[vsi->idx] != vsi) {
  5621. dev_err(&pf->pdev->dev,
  5622. "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n",
  5623. pf->vsi[vsi->idx]->idx,
  5624. pf->vsi[vsi->idx],
  5625. pf->vsi[vsi->idx]->type,
  5626. vsi->idx, vsi, vsi->type);
  5627. goto unlock_vsi;
  5628. }
  5629. /* updates the pf for this cleared vsi */
  5630. i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
  5631. i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
  5632. i40e_vsi_free_arrays(vsi, true);
  5633. pf->vsi[vsi->idx] = NULL;
  5634. if (vsi->idx < pf->next_vsi)
  5635. pf->next_vsi = vsi->idx;
  5636. unlock_vsi:
  5637. mutex_unlock(&pf->switch_mutex);
  5638. free_vsi:
  5639. kfree(vsi);
  5640. return 0;
  5641. }
  5642. /**
  5643. * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
  5644. * @vsi: the VSI being cleaned
  5645. **/
  5646. static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
  5647. {
  5648. int i;
  5649. if (vsi->tx_rings && vsi->tx_rings[0]) {
  5650. for (i = 0; i < vsi->alloc_queue_pairs; i++) {
  5651. kfree_rcu(vsi->tx_rings[i], rcu);
  5652. vsi->tx_rings[i] = NULL;
  5653. vsi->rx_rings[i] = NULL;
  5654. }
  5655. }
  5656. }
  5657. /**
  5658. * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
  5659. * @vsi: the VSI being configured
  5660. **/
  5661. static int i40e_alloc_rings(struct i40e_vsi *vsi)
  5662. {
  5663. struct i40e_ring *tx_ring, *rx_ring;
  5664. struct i40e_pf *pf = vsi->back;
  5665. int i;
  5666. /* Set basic values in the rings to be used later during open() */
  5667. for (i = 0; i < vsi->alloc_queue_pairs; i++) {
  5668. /* allocate space for both Tx and Rx in one shot */
  5669. tx_ring = kzalloc(sizeof(struct i40e_ring) * 2, GFP_KERNEL);
  5670. if (!tx_ring)
  5671. goto err_out;
  5672. tx_ring->queue_index = i;
  5673. tx_ring->reg_idx = vsi->base_queue + i;
  5674. tx_ring->ring_active = false;
  5675. tx_ring->vsi = vsi;
  5676. tx_ring->netdev = vsi->netdev;
  5677. tx_ring->dev = &pf->pdev->dev;
  5678. tx_ring->count = vsi->num_desc;
  5679. tx_ring->size = 0;
  5680. tx_ring->dcb_tc = 0;
  5681. vsi->tx_rings[i] = tx_ring;
  5682. rx_ring = &tx_ring[1];
  5683. rx_ring->queue_index = i;
  5684. rx_ring->reg_idx = vsi->base_queue + i;
  5685. rx_ring->ring_active = false;
  5686. rx_ring->vsi = vsi;
  5687. rx_ring->netdev = vsi->netdev;
  5688. rx_ring->dev = &pf->pdev->dev;
  5689. rx_ring->count = vsi->num_desc;
  5690. rx_ring->size = 0;
  5691. rx_ring->dcb_tc = 0;
  5692. if (pf->flags & I40E_FLAG_16BYTE_RX_DESC_ENABLED)
  5693. set_ring_16byte_desc_enabled(rx_ring);
  5694. else
  5695. clear_ring_16byte_desc_enabled(rx_ring);
  5696. vsi->rx_rings[i] = rx_ring;
  5697. }
  5698. return 0;
  5699. err_out:
  5700. i40e_vsi_clear_rings(vsi);
  5701. return -ENOMEM;
  5702. }
  5703. /**
  5704. * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
  5705. * @pf: board private structure
  5706. * @vectors: the number of MSI-X vectors to request
  5707. *
  5708. * Returns the number of vectors reserved, or error
  5709. **/
  5710. static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
  5711. {
  5712. vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
  5713. I40E_MIN_MSIX, vectors);
  5714. if (vectors < 0) {
  5715. dev_info(&pf->pdev->dev,
  5716. "MSI-X vector reservation failed: %d\n", vectors);
  5717. vectors = 0;
  5718. }
  5719. return vectors;
  5720. }
  5721. /**
  5722. * i40e_init_msix - Setup the MSIX capability
  5723. * @pf: board private structure
  5724. *
  5725. * Work with the OS to set up the MSIX vectors needed.
  5726. *
  5727. * Returns 0 on success, negative on failure
  5728. **/
  5729. static int i40e_init_msix(struct i40e_pf *pf)
  5730. {
  5731. i40e_status err = 0;
  5732. struct i40e_hw *hw = &pf->hw;
  5733. int v_budget, i;
  5734. int vec;
  5735. if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
  5736. return -ENODEV;
  5737. /* The number of vectors we'll request will be comprised of:
  5738. * - Add 1 for "other" cause for Admin Queue events, etc.
  5739. * - The number of LAN queue pairs
  5740. * - Queues being used for RSS.
  5741. * We don't need as many as max_rss_size vectors.
  5742. * use rss_size instead in the calculation since that
  5743. * is governed by number of cpus in the system.
  5744. * - assumes symmetric Tx/Rx pairing
  5745. * - The number of VMDq pairs
  5746. #ifdef I40E_FCOE
  5747. * - The number of FCOE qps.
  5748. #endif
  5749. * Once we count this up, try the request.
  5750. *
  5751. * If we can't get what we want, we'll simplify to nearly nothing
  5752. * and try again. If that still fails, we punt.
  5753. */
  5754. pf->num_lan_msix = pf->num_lan_qps - (pf->rss_size_max - pf->rss_size);
  5755. pf->num_vmdq_msix = pf->num_vmdq_qps;
  5756. v_budget = 1 + pf->num_lan_msix;
  5757. v_budget += (pf->num_vmdq_vsis * pf->num_vmdq_msix);
  5758. if (pf->flags & I40E_FLAG_FD_SB_ENABLED)
  5759. v_budget++;
  5760. #ifdef I40E_FCOE
  5761. if (pf->flags & I40E_FLAG_FCOE_ENABLED) {
  5762. pf->num_fcoe_msix = pf->num_fcoe_qps;
  5763. v_budget += pf->num_fcoe_msix;
  5764. }
  5765. #endif
  5766. /* Scale down if necessary, and the rings will share vectors */
  5767. v_budget = min_t(int, v_budget, hw->func_caps.num_msix_vectors);
  5768. pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
  5769. GFP_KERNEL);
  5770. if (!pf->msix_entries)
  5771. return -ENOMEM;
  5772. for (i = 0; i < v_budget; i++)
  5773. pf->msix_entries[i].entry = i;
  5774. vec = i40e_reserve_msix_vectors(pf, v_budget);
  5775. if (vec != v_budget) {
  5776. /* If we have limited resources, we will start with no vectors
  5777. * for the special features and then allocate vectors to some
  5778. * of these features based on the policy and at the end disable
  5779. * the features that did not get any vectors.
  5780. */
  5781. #ifdef I40E_FCOE
  5782. pf->num_fcoe_qps = 0;
  5783. pf->num_fcoe_msix = 0;
  5784. #endif
  5785. pf->num_vmdq_msix = 0;
  5786. }
  5787. if (vec < I40E_MIN_MSIX) {
  5788. pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
  5789. kfree(pf->msix_entries);
  5790. pf->msix_entries = NULL;
  5791. return -ENODEV;
  5792. } else if (vec == I40E_MIN_MSIX) {
  5793. /* Adjust for minimal MSIX use */
  5794. pf->num_vmdq_vsis = 0;
  5795. pf->num_vmdq_qps = 0;
  5796. pf->num_lan_qps = 1;
  5797. pf->num_lan_msix = 1;
  5798. } else if (vec != v_budget) {
  5799. /* reserve the misc vector */
  5800. vec--;
  5801. /* Scale vector usage down */
  5802. pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */
  5803. pf->num_vmdq_vsis = 1;
  5804. /* partition out the remaining vectors */
  5805. switch (vec) {
  5806. case 2:
  5807. pf->num_lan_msix = 1;
  5808. break;
  5809. case 3:
  5810. #ifdef I40E_FCOE
  5811. /* give one vector to FCoE */
  5812. if (pf->flags & I40E_FLAG_FCOE_ENABLED) {
  5813. pf->num_lan_msix = 1;
  5814. pf->num_fcoe_msix = 1;
  5815. }
  5816. #else
  5817. pf->num_lan_msix = 2;
  5818. #endif
  5819. break;
  5820. default:
  5821. #ifdef I40E_FCOE
  5822. /* give one vector to FCoE */
  5823. if (pf->flags & I40E_FLAG_FCOE_ENABLED) {
  5824. pf->num_fcoe_msix = 1;
  5825. vec--;
  5826. }
  5827. #endif
  5828. pf->num_lan_msix = min_t(int, (vec / 2),
  5829. pf->num_lan_qps);
  5830. pf->num_vmdq_vsis = min_t(int, (vec - pf->num_lan_msix),
  5831. I40E_DEFAULT_NUM_VMDQ_VSI);
  5832. break;
  5833. }
  5834. }
  5835. if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
  5836. (pf->num_vmdq_msix == 0)) {
  5837. dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
  5838. pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
  5839. }
  5840. #ifdef I40E_FCOE
  5841. if ((pf->flags & I40E_FLAG_FCOE_ENABLED) && (pf->num_fcoe_msix == 0)) {
  5842. dev_info(&pf->pdev->dev, "FCOE disabled, not enough MSI-X vectors\n");
  5843. pf->flags &= ~I40E_FLAG_FCOE_ENABLED;
  5844. }
  5845. #endif
  5846. return err;
  5847. }
  5848. /**
  5849. * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
  5850. * @vsi: the VSI being configured
  5851. * @v_idx: index of the vector in the vsi struct
  5852. *
  5853. * We allocate one q_vector. If allocation fails we return -ENOMEM.
  5854. **/
  5855. static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
  5856. {
  5857. struct i40e_q_vector *q_vector;
  5858. /* allocate q_vector */
  5859. q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
  5860. if (!q_vector)
  5861. return -ENOMEM;
  5862. q_vector->vsi = vsi;
  5863. q_vector->v_idx = v_idx;
  5864. cpumask_set_cpu(v_idx, &q_vector->affinity_mask);
  5865. if (vsi->netdev)
  5866. netif_napi_add(vsi->netdev, &q_vector->napi,
  5867. i40e_napi_poll, NAPI_POLL_WEIGHT);
  5868. q_vector->rx.latency_range = I40E_LOW_LATENCY;
  5869. q_vector->tx.latency_range = I40E_LOW_LATENCY;
  5870. /* tie q_vector and vsi together */
  5871. vsi->q_vectors[v_idx] = q_vector;
  5872. return 0;
  5873. }
  5874. /**
  5875. * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
  5876. * @vsi: the VSI being configured
  5877. *
  5878. * We allocate one q_vector per queue interrupt. If allocation fails we
  5879. * return -ENOMEM.
  5880. **/
  5881. static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
  5882. {
  5883. struct i40e_pf *pf = vsi->back;
  5884. int v_idx, num_q_vectors;
  5885. int err;
  5886. /* if not MSIX, give the one vector only to the LAN VSI */
  5887. if (pf->flags & I40E_FLAG_MSIX_ENABLED)
  5888. num_q_vectors = vsi->num_q_vectors;
  5889. else if (vsi == pf->vsi[pf->lan_vsi])
  5890. num_q_vectors = 1;
  5891. else
  5892. return -EINVAL;
  5893. for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
  5894. err = i40e_vsi_alloc_q_vector(vsi, v_idx);
  5895. if (err)
  5896. goto err_out;
  5897. }
  5898. return 0;
  5899. err_out:
  5900. while (v_idx--)
  5901. i40e_free_q_vector(vsi, v_idx);
  5902. return err;
  5903. }
  5904. /**
  5905. * i40e_init_interrupt_scheme - Determine proper interrupt scheme
  5906. * @pf: board private structure to initialize
  5907. **/
  5908. static void i40e_init_interrupt_scheme(struct i40e_pf *pf)
  5909. {
  5910. int err = 0;
  5911. if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
  5912. err = i40e_init_msix(pf);
  5913. if (err) {
  5914. pf->flags &= ~(I40E_FLAG_MSIX_ENABLED |
  5915. #ifdef I40E_FCOE
  5916. I40E_FLAG_FCOE_ENABLED |
  5917. #endif
  5918. I40E_FLAG_RSS_ENABLED |
  5919. I40E_FLAG_DCB_CAPABLE |
  5920. I40E_FLAG_SRIOV_ENABLED |
  5921. I40E_FLAG_FD_SB_ENABLED |
  5922. I40E_FLAG_FD_ATR_ENABLED |
  5923. I40E_FLAG_VMDQ_ENABLED);
  5924. /* rework the queue expectations without MSIX */
  5925. i40e_determine_queue_usage(pf);
  5926. }
  5927. }
  5928. if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
  5929. (pf->flags & I40E_FLAG_MSI_ENABLED)) {
  5930. dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
  5931. err = pci_enable_msi(pf->pdev);
  5932. if (err) {
  5933. dev_info(&pf->pdev->dev, "MSI init failed - %d\n", err);
  5934. pf->flags &= ~I40E_FLAG_MSI_ENABLED;
  5935. }
  5936. }
  5937. if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
  5938. dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
  5939. /* track first vector for misc interrupts */
  5940. err = i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT-1);
  5941. }
  5942. /**
  5943. * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
  5944. * @pf: board private structure
  5945. *
  5946. * This sets up the handler for MSIX 0, which is used to manage the
  5947. * non-queue interrupts, e.g. AdminQ and errors. This is not used
  5948. * when in MSI or Legacy interrupt mode.
  5949. **/
  5950. static int i40e_setup_misc_vector(struct i40e_pf *pf)
  5951. {
  5952. struct i40e_hw *hw = &pf->hw;
  5953. int err = 0;
  5954. /* Only request the irq if this is the first time through, and
  5955. * not when we're rebuilding after a Reset
  5956. */
  5957. if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) {
  5958. err = request_irq(pf->msix_entries[0].vector,
  5959. i40e_intr, 0, pf->misc_int_name, pf);
  5960. if (err) {
  5961. dev_info(&pf->pdev->dev,
  5962. "request_irq for %s failed: %d\n",
  5963. pf->misc_int_name, err);
  5964. return -EFAULT;
  5965. }
  5966. }
  5967. i40e_enable_misc_int_causes(hw);
  5968. /* associate no queues to the misc vector */
  5969. wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
  5970. wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K);
  5971. i40e_flush(hw);
  5972. i40e_irq_dynamic_enable_icr0(pf);
  5973. return err;
  5974. }
  5975. /**
  5976. * i40e_config_rss - Prepare for RSS if used
  5977. * @pf: board private structure
  5978. **/
  5979. static int i40e_config_rss(struct i40e_pf *pf)
  5980. {
  5981. /* Set of random keys generated using kernel random number generator */
  5982. static const u32 seed[I40E_PFQF_HKEY_MAX_INDEX + 1] = {0x41b01687,
  5983. 0x183cfd8c, 0xce880440, 0x580cbc3c, 0x35897377,
  5984. 0x328b25e1, 0x4fa98922, 0xb7d90c14, 0xd5bad70d,
  5985. 0xcd15a2c1, 0xe8580225, 0x4a1e9d11, 0xfe5731be};
  5986. struct i40e_hw *hw = &pf->hw;
  5987. u32 lut = 0;
  5988. int i, j;
  5989. u64 hena;
  5990. u32 reg_val;
  5991. /* Fill out hash function seed */
  5992. for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
  5993. wr32(hw, I40E_PFQF_HKEY(i), seed[i]);
  5994. /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
  5995. hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) |
  5996. ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32);
  5997. hena |= I40E_DEFAULT_RSS_HENA;
  5998. wr32(hw, I40E_PFQF_HENA(0), (u32)hena);
  5999. wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
  6000. /* Check capability and Set table size and register per hw expectation*/
  6001. reg_val = rd32(hw, I40E_PFQF_CTL_0);
  6002. if (hw->func_caps.rss_table_size == 512) {
  6003. reg_val |= I40E_PFQF_CTL_0_HASHLUTSIZE_512;
  6004. pf->rss_table_size = 512;
  6005. } else {
  6006. pf->rss_table_size = 128;
  6007. reg_val &= ~I40E_PFQF_CTL_0_HASHLUTSIZE_512;
  6008. }
  6009. wr32(hw, I40E_PFQF_CTL_0, reg_val);
  6010. /* Populate the LUT with max no. of queues in round robin fashion */
  6011. for (i = 0, j = 0; i < pf->rss_table_size; i++, j++) {
  6012. /* The assumption is that lan qp count will be the highest
  6013. * qp count for any PF VSI that needs RSS.
  6014. * If multiple VSIs need RSS support, all the qp counts
  6015. * for those VSIs should be a power of 2 for RSS to work.
  6016. * If LAN VSI is the only consumer for RSS then this requirement
  6017. * is not necessary.
  6018. */
  6019. if (j == pf->rss_size)
  6020. j = 0;
  6021. /* lut = 4-byte sliding window of 4 lut entries */
  6022. lut = (lut << 8) | (j &
  6023. ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1));
  6024. /* On i = 3, we have 4 entries in lut; write to the register */
  6025. if ((i & 3) == 3)
  6026. wr32(hw, I40E_PFQF_HLUT(i >> 2), lut);
  6027. }
  6028. i40e_flush(hw);
  6029. return 0;
  6030. }
  6031. /**
  6032. * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
  6033. * @pf: board private structure
  6034. * @queue_count: the requested queue count for rss.
  6035. *
  6036. * returns 0 if rss is not enabled, if enabled returns the final rss queue
  6037. * count which may be different from the requested queue count.
  6038. **/
  6039. int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
  6040. {
  6041. if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
  6042. return 0;
  6043. queue_count = min_t(int, queue_count, pf->rss_size_max);
  6044. if (queue_count != pf->rss_size) {
  6045. i40e_prep_for_reset(pf);
  6046. pf->rss_size = queue_count;
  6047. i40e_reset_and_rebuild(pf, true);
  6048. i40e_config_rss(pf);
  6049. }
  6050. dev_info(&pf->pdev->dev, "RSS count: %d\n", pf->rss_size);
  6051. return pf->rss_size;
  6052. }
  6053. /**
  6054. * i40e_sw_init - Initialize general software structures (struct i40e_pf)
  6055. * @pf: board private structure to initialize
  6056. *
  6057. * i40e_sw_init initializes the Adapter private data structure.
  6058. * Fields are initialized based on PCI device information and
  6059. * OS network device settings (MTU size).
  6060. **/
  6061. static int i40e_sw_init(struct i40e_pf *pf)
  6062. {
  6063. int err = 0;
  6064. int size;
  6065. pf->msg_enable = netif_msg_init(I40E_DEFAULT_MSG_ENABLE,
  6066. (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK));
  6067. pf->hw.debug_mask = pf->msg_enable | I40E_DEBUG_DIAG;
  6068. if (debug != -1 && debug != I40E_DEFAULT_MSG_ENABLE) {
  6069. if (I40E_DEBUG_USER & debug)
  6070. pf->hw.debug_mask = debug;
  6071. pf->msg_enable = netif_msg_init((debug & ~I40E_DEBUG_USER),
  6072. I40E_DEFAULT_MSG_ENABLE);
  6073. }
  6074. /* Set default capability flags */
  6075. pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
  6076. I40E_FLAG_MSI_ENABLED |
  6077. I40E_FLAG_MSIX_ENABLED |
  6078. I40E_FLAG_RX_1BUF_ENABLED;
  6079. /* Set default ITR */
  6080. pf->rx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_RX_DEF;
  6081. pf->tx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF;
  6082. /* Depending on PF configurations, it is possible that the RSS
  6083. * maximum might end up larger than the available queues
  6084. */
  6085. pf->rss_size_max = 0x1 << pf->hw.func_caps.rss_table_entry_width;
  6086. pf->rss_size = 1;
  6087. pf->rss_size_max = min_t(int, pf->rss_size_max,
  6088. pf->hw.func_caps.num_tx_qp);
  6089. if (pf->hw.func_caps.rss) {
  6090. pf->flags |= I40E_FLAG_RSS_ENABLED;
  6091. pf->rss_size = min_t(int, pf->rss_size_max, num_online_cpus());
  6092. }
  6093. /* MFP mode enabled */
  6094. if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.mfp_mode_1) {
  6095. pf->flags |= I40E_FLAG_MFP_ENABLED;
  6096. dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
  6097. }
  6098. /* FW/NVM is not yet fixed in this regard */
  6099. if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
  6100. (pf->hw.func_caps.fd_filters_best_effort > 0)) {
  6101. pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
  6102. pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
  6103. /* Setup a counter for fd_atr per pf */
  6104. pf->fd_atr_cnt_idx = I40E_FD_ATR_STAT_IDX(pf->hw.pf_id);
  6105. if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) {
  6106. pf->flags |= I40E_FLAG_FD_SB_ENABLED;
  6107. /* Setup a counter for fd_sb per pf */
  6108. pf->fd_sb_cnt_idx = I40E_FD_SB_STAT_IDX(pf->hw.pf_id);
  6109. } else {
  6110. dev_info(&pf->pdev->dev,
  6111. "Flow Director Sideband mode Disabled in MFP mode\n");
  6112. }
  6113. pf->fdir_pf_filter_count =
  6114. pf->hw.func_caps.fd_filters_guaranteed;
  6115. pf->hw.fdir_shared_filter_count =
  6116. pf->hw.func_caps.fd_filters_best_effort;
  6117. }
  6118. if (pf->hw.func_caps.vmdq) {
  6119. pf->flags |= I40E_FLAG_VMDQ_ENABLED;
  6120. pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
  6121. pf->num_vmdq_qps = I40E_DEFAULT_QUEUES_PER_VMDQ;
  6122. }
  6123. #ifdef I40E_FCOE
  6124. err = i40e_init_pf_fcoe(pf);
  6125. if (err)
  6126. dev_info(&pf->pdev->dev, "init_pf_fcoe failed: %d\n", err);
  6127. #endif /* I40E_FCOE */
  6128. #ifdef CONFIG_PCI_IOV
  6129. if (pf->hw.func_caps.num_vfs) {
  6130. pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
  6131. pf->flags |= I40E_FLAG_SRIOV_ENABLED;
  6132. pf->num_req_vfs = min_t(int,
  6133. pf->hw.func_caps.num_vfs,
  6134. I40E_MAX_VF_COUNT);
  6135. }
  6136. #endif /* CONFIG_PCI_IOV */
  6137. pf->eeprom_version = 0xDEAD;
  6138. pf->lan_veb = I40E_NO_VEB;
  6139. pf->lan_vsi = I40E_NO_VSI;
  6140. /* set up queue assignment tracking */
  6141. size = sizeof(struct i40e_lump_tracking)
  6142. + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
  6143. pf->qp_pile = kzalloc(size, GFP_KERNEL);
  6144. if (!pf->qp_pile) {
  6145. err = -ENOMEM;
  6146. goto sw_init_done;
  6147. }
  6148. pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
  6149. pf->qp_pile->search_hint = 0;
  6150. /* set up vector assignment tracking */
  6151. size = sizeof(struct i40e_lump_tracking)
  6152. + (sizeof(u16) * pf->hw.func_caps.num_msix_vectors);
  6153. pf->irq_pile = kzalloc(size, GFP_KERNEL);
  6154. if (!pf->irq_pile) {
  6155. kfree(pf->qp_pile);
  6156. err = -ENOMEM;
  6157. goto sw_init_done;
  6158. }
  6159. pf->irq_pile->num_entries = pf->hw.func_caps.num_msix_vectors;
  6160. pf->irq_pile->search_hint = 0;
  6161. pf->tx_timeout_recovery_level = 1;
  6162. mutex_init(&pf->switch_mutex);
  6163. sw_init_done:
  6164. return err;
  6165. }
  6166. /**
  6167. * i40e_set_ntuple - set the ntuple feature flag and take action
  6168. * @pf: board private structure to initialize
  6169. * @features: the feature set that the stack is suggesting
  6170. *
  6171. * returns a bool to indicate if reset needs to happen
  6172. **/
  6173. bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
  6174. {
  6175. bool need_reset = false;
  6176. /* Check if Flow Director n-tuple support was enabled or disabled. If
  6177. * the state changed, we need to reset.
  6178. */
  6179. if (features & NETIF_F_NTUPLE) {
  6180. /* Enable filters and mark for reset */
  6181. if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
  6182. need_reset = true;
  6183. pf->flags |= I40E_FLAG_FD_SB_ENABLED;
  6184. } else {
  6185. /* turn off filters, mark for reset and clear SW filter list */
  6186. if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
  6187. need_reset = true;
  6188. i40e_fdir_filter_exit(pf);
  6189. }
  6190. pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
  6191. pf->auto_disable_flags &= ~I40E_FLAG_FD_SB_ENABLED;
  6192. /* if ATR was auto disabled it can be re-enabled. */
  6193. if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
  6194. (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED))
  6195. pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
  6196. }
  6197. return need_reset;
  6198. }
  6199. /**
  6200. * i40e_set_features - set the netdev feature flags
  6201. * @netdev: ptr to the netdev being adjusted
  6202. * @features: the feature set that the stack is suggesting
  6203. **/
  6204. static int i40e_set_features(struct net_device *netdev,
  6205. netdev_features_t features)
  6206. {
  6207. struct i40e_netdev_priv *np = netdev_priv(netdev);
  6208. struct i40e_vsi *vsi = np->vsi;
  6209. struct i40e_pf *pf = vsi->back;
  6210. bool need_reset;
  6211. if (features & NETIF_F_HW_VLAN_CTAG_RX)
  6212. i40e_vlan_stripping_enable(vsi);
  6213. else
  6214. i40e_vlan_stripping_disable(vsi);
  6215. need_reset = i40e_set_ntuple(pf, features);
  6216. if (need_reset)
  6217. i40e_do_reset(pf, (1 << __I40E_PF_RESET_REQUESTED));
  6218. return 0;
  6219. }
  6220. #ifdef CONFIG_I40E_VXLAN
  6221. /**
  6222. * i40e_get_vxlan_port_idx - Lookup a possibly offloaded for Rx UDP port
  6223. * @pf: board private structure
  6224. * @port: The UDP port to look up
  6225. *
  6226. * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found
  6227. **/
  6228. static u8 i40e_get_vxlan_port_idx(struct i40e_pf *pf, __be16 port)
  6229. {
  6230. u8 i;
  6231. for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
  6232. if (pf->vxlan_ports[i] == port)
  6233. return i;
  6234. }
  6235. return i;
  6236. }
  6237. /**
  6238. * i40e_add_vxlan_port - Get notifications about VXLAN ports that come up
  6239. * @netdev: This physical port's netdev
  6240. * @sa_family: Socket Family that VXLAN is notifying us about
  6241. * @port: New UDP port number that VXLAN started listening to
  6242. **/
  6243. static void i40e_add_vxlan_port(struct net_device *netdev,
  6244. sa_family_t sa_family, __be16 port)
  6245. {
  6246. struct i40e_netdev_priv *np = netdev_priv(netdev);
  6247. struct i40e_vsi *vsi = np->vsi;
  6248. struct i40e_pf *pf = vsi->back;
  6249. u8 next_idx;
  6250. u8 idx;
  6251. if (sa_family == AF_INET6)
  6252. return;
  6253. idx = i40e_get_vxlan_port_idx(pf, port);
  6254. /* Check if port already exists */
  6255. if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
  6256. netdev_info(netdev, "Port %d already offloaded\n", ntohs(port));
  6257. return;
  6258. }
  6259. /* Now check if there is space to add the new port */
  6260. next_idx = i40e_get_vxlan_port_idx(pf, 0);
  6261. if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
  6262. netdev_info(netdev, "Maximum number of UDP ports reached, not adding port %d\n",
  6263. ntohs(port));
  6264. return;
  6265. }
  6266. /* New port: add it and mark its index in the bitmap */
  6267. pf->vxlan_ports[next_idx] = port;
  6268. pf->pending_vxlan_bitmap |= (1 << next_idx);
  6269. pf->flags |= I40E_FLAG_VXLAN_FILTER_SYNC;
  6270. }
  6271. /**
  6272. * i40e_del_vxlan_port - Get notifications about VXLAN ports that go away
  6273. * @netdev: This physical port's netdev
  6274. * @sa_family: Socket Family that VXLAN is notifying us about
  6275. * @port: UDP port number that VXLAN stopped listening to
  6276. **/
  6277. static void i40e_del_vxlan_port(struct net_device *netdev,
  6278. sa_family_t sa_family, __be16 port)
  6279. {
  6280. struct i40e_netdev_priv *np = netdev_priv(netdev);
  6281. struct i40e_vsi *vsi = np->vsi;
  6282. struct i40e_pf *pf = vsi->back;
  6283. u8 idx;
  6284. if (sa_family == AF_INET6)
  6285. return;
  6286. idx = i40e_get_vxlan_port_idx(pf, port);
  6287. /* Check if port already exists */
  6288. if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
  6289. /* if port exists, set it to 0 (mark for deletion)
  6290. * and make it pending
  6291. */
  6292. pf->vxlan_ports[idx] = 0;
  6293. pf->pending_vxlan_bitmap |= (1 << idx);
  6294. pf->flags |= I40E_FLAG_VXLAN_FILTER_SYNC;
  6295. } else {
  6296. netdev_warn(netdev, "Port %d was not found, not deleting\n",
  6297. ntohs(port));
  6298. }
  6299. }
  6300. #endif
  6301. static int i40e_get_phys_port_id(struct net_device *netdev,
  6302. struct netdev_phys_port_id *ppid)
  6303. {
  6304. struct i40e_netdev_priv *np = netdev_priv(netdev);
  6305. struct i40e_pf *pf = np->vsi->back;
  6306. struct i40e_hw *hw = &pf->hw;
  6307. if (!(pf->flags & I40E_FLAG_PORT_ID_VALID))
  6308. return -EOPNOTSUPP;
  6309. ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
  6310. memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
  6311. return 0;
  6312. }
  6313. #ifdef HAVE_FDB_OPS
  6314. #ifdef USE_CONST_DEV_UC_CHAR
  6315. static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
  6316. struct net_device *dev,
  6317. const unsigned char *addr,
  6318. u16 flags)
  6319. #else
  6320. static int i40e_ndo_fdb_add(struct ndmsg *ndm,
  6321. struct net_device *dev,
  6322. unsigned char *addr,
  6323. u16 flags)
  6324. #endif
  6325. {
  6326. struct i40e_netdev_priv *np = netdev_priv(dev);
  6327. struct i40e_pf *pf = np->vsi->back;
  6328. int err = 0;
  6329. if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
  6330. return -EOPNOTSUPP;
  6331. /* Hardware does not support aging addresses so if a
  6332. * ndm_state is given only allow permanent addresses
  6333. */
  6334. if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
  6335. netdev_info(dev, "FDB only supports static addresses\n");
  6336. return -EINVAL;
  6337. }
  6338. if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
  6339. err = dev_uc_add_excl(dev, addr);
  6340. else if (is_multicast_ether_addr(addr))
  6341. err = dev_mc_add_excl(dev, addr);
  6342. else
  6343. err = -EINVAL;
  6344. /* Only return duplicate errors if NLM_F_EXCL is set */
  6345. if (err == -EEXIST && !(flags & NLM_F_EXCL))
  6346. err = 0;
  6347. return err;
  6348. }
  6349. #ifndef USE_DEFAULT_FDB_DEL_DUMP
  6350. #ifdef USE_CONST_DEV_UC_CHAR
  6351. static int i40e_ndo_fdb_del(struct ndmsg *ndm,
  6352. struct net_device *dev,
  6353. const unsigned char *addr)
  6354. #else
  6355. static int i40e_ndo_fdb_del(struct ndmsg *ndm,
  6356. struct net_device *dev,
  6357. unsigned char *addr)
  6358. #endif
  6359. {
  6360. struct i40e_netdev_priv *np = netdev_priv(dev);
  6361. struct i40e_pf *pf = np->vsi->back;
  6362. int err = -EOPNOTSUPP;
  6363. if (ndm->ndm_state & NUD_PERMANENT) {
  6364. netdev_info(dev, "FDB only supports static addresses\n");
  6365. return -EINVAL;
  6366. }
  6367. if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
  6368. if (is_unicast_ether_addr(addr))
  6369. err = dev_uc_del(dev, addr);
  6370. else if (is_multicast_ether_addr(addr))
  6371. err = dev_mc_del(dev, addr);
  6372. else
  6373. err = -EINVAL;
  6374. }
  6375. return err;
  6376. }
  6377. static int i40e_ndo_fdb_dump(struct sk_buff *skb,
  6378. struct netlink_callback *cb,
  6379. struct net_device *dev,
  6380. struct net_device *filter_dev,
  6381. int idx)
  6382. {
  6383. struct i40e_netdev_priv *np = netdev_priv(dev);
  6384. struct i40e_pf *pf = np->vsi->back;
  6385. if (pf->flags & I40E_FLAG_SRIOV_ENABLED)
  6386. idx = ndo_dflt_fdb_dump(skb, cb, dev, filter_dev, idx);
  6387. return idx;
  6388. }
  6389. #endif /* USE_DEFAULT_FDB_DEL_DUMP */
  6390. #endif /* HAVE_FDB_OPS */
  6391. static const struct net_device_ops i40e_netdev_ops = {
  6392. .ndo_open = i40e_open,
  6393. .ndo_stop = i40e_close,
  6394. .ndo_start_xmit = i40e_lan_xmit_frame,
  6395. .ndo_get_stats64 = i40e_get_netdev_stats_struct,
  6396. .ndo_set_rx_mode = i40e_set_rx_mode,
  6397. .ndo_validate_addr = eth_validate_addr,
  6398. .ndo_set_mac_address = i40e_set_mac,
  6399. .ndo_change_mtu = i40e_change_mtu,
  6400. .ndo_do_ioctl = i40e_ioctl,
  6401. .ndo_tx_timeout = i40e_tx_timeout,
  6402. .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid,
  6403. .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid,
  6404. #ifdef CONFIG_NET_POLL_CONTROLLER
  6405. .ndo_poll_controller = i40e_netpoll,
  6406. #endif
  6407. .ndo_setup_tc = i40e_setup_tc,
  6408. #ifdef I40E_FCOE
  6409. .ndo_fcoe_enable = i40e_fcoe_enable,
  6410. .ndo_fcoe_disable = i40e_fcoe_disable,
  6411. #endif
  6412. .ndo_set_features = i40e_set_features,
  6413. .ndo_set_vf_mac = i40e_ndo_set_vf_mac,
  6414. .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan,
  6415. .ndo_set_vf_rate = i40e_ndo_set_vf_bw,
  6416. .ndo_get_vf_config = i40e_ndo_get_vf_config,
  6417. .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state,
  6418. .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofck,
  6419. #ifdef CONFIG_I40E_VXLAN
  6420. .ndo_add_vxlan_port = i40e_add_vxlan_port,
  6421. .ndo_del_vxlan_port = i40e_del_vxlan_port,
  6422. #endif
  6423. .ndo_get_phys_port_id = i40e_get_phys_port_id,
  6424. #ifdef HAVE_FDB_OPS
  6425. .ndo_fdb_add = i40e_ndo_fdb_add,
  6426. #ifndef USE_DEFAULT_FDB_DEL_DUMP
  6427. .ndo_fdb_del = i40e_ndo_fdb_del,
  6428. .ndo_fdb_dump = i40e_ndo_fdb_dump,
  6429. #endif
  6430. #endif
  6431. };
  6432. /**
  6433. * i40e_config_netdev - Setup the netdev flags
  6434. * @vsi: the VSI being configured
  6435. *
  6436. * Returns 0 on success, negative value on failure
  6437. **/
  6438. static int i40e_config_netdev(struct i40e_vsi *vsi)
  6439. {
  6440. u8 brdcast[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
  6441. struct i40e_pf *pf = vsi->back;
  6442. struct i40e_hw *hw = &pf->hw;
  6443. struct i40e_netdev_priv *np;
  6444. struct net_device *netdev;
  6445. u8 mac_addr[ETH_ALEN];
  6446. int etherdev_size;
  6447. etherdev_size = sizeof(struct i40e_netdev_priv);
  6448. netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
  6449. if (!netdev)
  6450. return -ENOMEM;
  6451. vsi->netdev = netdev;
  6452. np = netdev_priv(netdev);
  6453. np->vsi = vsi;
  6454. netdev->hw_enc_features |= NETIF_F_IP_CSUM |
  6455. NETIF_F_GSO_UDP_TUNNEL |
  6456. NETIF_F_TSO;
  6457. netdev->features = NETIF_F_SG |
  6458. NETIF_F_IP_CSUM |
  6459. NETIF_F_SCTP_CSUM |
  6460. NETIF_F_HIGHDMA |
  6461. NETIF_F_GSO_UDP_TUNNEL |
  6462. NETIF_F_HW_VLAN_CTAG_TX |
  6463. NETIF_F_HW_VLAN_CTAG_RX |
  6464. NETIF_F_HW_VLAN_CTAG_FILTER |
  6465. NETIF_F_IPV6_CSUM |
  6466. NETIF_F_TSO |
  6467. NETIF_F_TSO_ECN |
  6468. NETIF_F_TSO6 |
  6469. NETIF_F_RXCSUM |
  6470. NETIF_F_RXHASH |
  6471. 0;
  6472. if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
  6473. netdev->features |= NETIF_F_NTUPLE;
  6474. /* copy netdev features into list of user selectable features */
  6475. netdev->hw_features |= netdev->features;
  6476. if (vsi->type == I40E_VSI_MAIN) {
  6477. SET_NETDEV_DEV(netdev, &pf->pdev->dev);
  6478. ether_addr_copy(mac_addr, hw->mac.perm_addr);
  6479. /* The following two steps are necessary to prevent reception
  6480. * of tagged packets - by default the NVM loads a MAC-VLAN
  6481. * filter that will accept any tagged packet. This is to
  6482. * prevent that during normal operations until a specific
  6483. * VLAN tag filter has been set.
  6484. */
  6485. i40e_rm_default_mac_filter(vsi, mac_addr);
  6486. i40e_add_filter(vsi, mac_addr, I40E_VLAN_ANY, false, true);
  6487. } else {
  6488. /* relate the VSI_VMDQ name to the VSI_MAIN name */
  6489. snprintf(netdev->name, IFNAMSIZ, "%sv%%d",
  6490. pf->vsi[pf->lan_vsi]->netdev->name);
  6491. random_ether_addr(mac_addr);
  6492. i40e_add_filter(vsi, mac_addr, I40E_VLAN_ANY, false, false);
  6493. }
  6494. i40e_add_filter(vsi, brdcast, I40E_VLAN_ANY, false, false);
  6495. ether_addr_copy(netdev->dev_addr, mac_addr);
  6496. ether_addr_copy(netdev->perm_addr, mac_addr);
  6497. /* vlan gets same features (except vlan offload)
  6498. * after any tweaks for specific VSI types
  6499. */
  6500. netdev->vlan_features = netdev->features & ~(NETIF_F_HW_VLAN_CTAG_TX |
  6501. NETIF_F_HW_VLAN_CTAG_RX |
  6502. NETIF_F_HW_VLAN_CTAG_FILTER);
  6503. netdev->priv_flags |= IFF_UNICAST_FLT;
  6504. netdev->priv_flags |= IFF_SUPP_NOFCS;
  6505. /* Setup netdev TC information */
  6506. i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
  6507. netdev->netdev_ops = &i40e_netdev_ops;
  6508. netdev->watchdog_timeo = 5 * HZ;
  6509. i40e_set_ethtool_ops(netdev);
  6510. #ifdef I40E_FCOE
  6511. i40e_fcoe_config_netdev(netdev, vsi);
  6512. #endif
  6513. return 0;
  6514. }
  6515. /**
  6516. * i40e_vsi_delete - Delete a VSI from the switch
  6517. * @vsi: the VSI being removed
  6518. *
  6519. * Returns 0 on success, negative value on failure
  6520. **/
  6521. static void i40e_vsi_delete(struct i40e_vsi *vsi)
  6522. {
  6523. /* remove default VSI is not allowed */
  6524. if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
  6525. return;
  6526. i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
  6527. }
  6528. /**
  6529. * i40e_add_vsi - Add a VSI to the switch
  6530. * @vsi: the VSI being configured
  6531. *
  6532. * This initializes a VSI context depending on the VSI type to be added and
  6533. * passes it down to the add_vsi aq command.
  6534. **/
  6535. static int i40e_add_vsi(struct i40e_vsi *vsi)
  6536. {
  6537. int ret = -ENODEV;
  6538. struct i40e_mac_filter *f, *ftmp;
  6539. struct i40e_pf *pf = vsi->back;
  6540. struct i40e_hw *hw = &pf->hw;
  6541. struct i40e_vsi_context ctxt;
  6542. u8 enabled_tc = 0x1; /* TC0 enabled */
  6543. int f_count = 0;
  6544. memset(&ctxt, 0, sizeof(ctxt));
  6545. switch (vsi->type) {
  6546. case I40E_VSI_MAIN:
  6547. /* The PF's main VSI is already setup as part of the
  6548. * device initialization, so we'll not bother with
  6549. * the add_vsi call, but we will retrieve the current
  6550. * VSI context.
  6551. */
  6552. ctxt.seid = pf->main_vsi_seid;
  6553. ctxt.pf_num = pf->hw.pf_id;
  6554. ctxt.vf_num = 0;
  6555. ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
  6556. ctxt.flags = I40E_AQ_VSI_TYPE_PF;
  6557. if (ret) {
  6558. dev_info(&pf->pdev->dev,
  6559. "couldn't get pf vsi config, err %d, aq_err %d\n",
  6560. ret, pf->hw.aq.asq_last_status);
  6561. return -ENOENT;
  6562. }
  6563. memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
  6564. vsi->info.valid_sections = 0;
  6565. vsi->seid = ctxt.seid;
  6566. vsi->id = ctxt.vsi_number;
  6567. enabled_tc = i40e_pf_get_tc_map(pf);
  6568. /* MFP mode setup queue map and update VSI */
  6569. if (pf->flags & I40E_FLAG_MFP_ENABLED) {
  6570. memset(&ctxt, 0, sizeof(ctxt));
  6571. ctxt.seid = pf->main_vsi_seid;
  6572. ctxt.pf_num = pf->hw.pf_id;
  6573. ctxt.vf_num = 0;
  6574. i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
  6575. ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
  6576. if (ret) {
  6577. dev_info(&pf->pdev->dev,
  6578. "update vsi failed, aq_err=%d\n",
  6579. pf->hw.aq.asq_last_status);
  6580. ret = -ENOENT;
  6581. goto err;
  6582. }
  6583. /* update the local VSI info queue map */
  6584. i40e_vsi_update_queue_map(vsi, &ctxt);
  6585. vsi->info.valid_sections = 0;
  6586. } else {
  6587. /* Default/Main VSI is only enabled for TC0
  6588. * reconfigure it to enable all TCs that are
  6589. * available on the port in SFP mode.
  6590. */
  6591. ret = i40e_vsi_config_tc(vsi, enabled_tc);
  6592. if (ret) {
  6593. dev_info(&pf->pdev->dev,
  6594. "failed to configure TCs for main VSI tc_map 0x%08x, err %d, aq_err %d\n",
  6595. enabled_tc, ret,
  6596. pf->hw.aq.asq_last_status);
  6597. ret = -ENOENT;
  6598. }
  6599. }
  6600. break;
  6601. case I40E_VSI_FDIR:
  6602. ctxt.pf_num = hw->pf_id;
  6603. ctxt.vf_num = 0;
  6604. ctxt.uplink_seid = vsi->uplink_seid;
  6605. ctxt.connection_type = 0x1; /* regular data port */
  6606. ctxt.flags = I40E_AQ_VSI_TYPE_PF;
  6607. i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
  6608. break;
  6609. case I40E_VSI_VMDQ2:
  6610. ctxt.pf_num = hw->pf_id;
  6611. ctxt.vf_num = 0;
  6612. ctxt.uplink_seid = vsi->uplink_seid;
  6613. ctxt.connection_type = 0x1; /* regular data port */
  6614. ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
  6615. ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
  6616. /* This VSI is connected to VEB so the switch_id
  6617. * should be set to zero by default.
  6618. */
  6619. ctxt.info.switch_id = 0;
  6620. ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
  6621. /* Setup the VSI tx/rx queue map for TC0 only for now */
  6622. i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
  6623. break;
  6624. case I40E_VSI_SRIOV:
  6625. ctxt.pf_num = hw->pf_id;
  6626. ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
  6627. ctxt.uplink_seid = vsi->uplink_seid;
  6628. ctxt.connection_type = 0x1; /* regular data port */
  6629. ctxt.flags = I40E_AQ_VSI_TYPE_VF;
  6630. ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
  6631. /* This VSI is connected to VEB so the switch_id
  6632. * should be set to zero by default.
  6633. */
  6634. ctxt.info.switch_id = cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
  6635. ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
  6636. ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
  6637. if (pf->vf[vsi->vf_id].spoofchk) {
  6638. ctxt.info.valid_sections |=
  6639. cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
  6640. ctxt.info.sec_flags |=
  6641. (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
  6642. I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
  6643. }
  6644. /* Setup the VSI tx/rx queue map for TC0 only for now */
  6645. i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
  6646. break;
  6647. #ifdef I40E_FCOE
  6648. case I40E_VSI_FCOE:
  6649. ret = i40e_fcoe_vsi_init(vsi, &ctxt);
  6650. if (ret) {
  6651. dev_info(&pf->pdev->dev, "failed to initialize FCoE VSI\n");
  6652. return ret;
  6653. }
  6654. break;
  6655. #endif /* I40E_FCOE */
  6656. default:
  6657. return -ENODEV;
  6658. }
  6659. if (vsi->type != I40E_VSI_MAIN) {
  6660. ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
  6661. if (ret) {
  6662. dev_info(&vsi->back->pdev->dev,
  6663. "add vsi failed, aq_err=%d\n",
  6664. vsi->back->hw.aq.asq_last_status);
  6665. ret = -ENOENT;
  6666. goto err;
  6667. }
  6668. memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
  6669. vsi->info.valid_sections = 0;
  6670. vsi->seid = ctxt.seid;
  6671. vsi->id = ctxt.vsi_number;
  6672. }
  6673. /* If macvlan filters already exist, force them to get loaded */
  6674. list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
  6675. f->changed = true;
  6676. f_count++;
  6677. if (f->is_laa && vsi->type == I40E_VSI_MAIN) {
  6678. i40e_aq_mac_address_write(&vsi->back->hw,
  6679. I40E_AQC_WRITE_TYPE_LAA_WOL,
  6680. f->macaddr, NULL);
  6681. }
  6682. }
  6683. if (f_count) {
  6684. vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
  6685. pf->flags |= I40E_FLAG_FILTER_SYNC;
  6686. }
  6687. /* Update VSI BW information */
  6688. ret = i40e_vsi_get_bw_info(vsi);
  6689. if (ret) {
  6690. dev_info(&pf->pdev->dev,
  6691. "couldn't get vsi bw info, err %d, aq_err %d\n",
  6692. ret, pf->hw.aq.asq_last_status);
  6693. /* VSI is already added so not tearing that up */
  6694. ret = 0;
  6695. }
  6696. err:
  6697. return ret;
  6698. }
  6699. /**
  6700. * i40e_vsi_release - Delete a VSI and free its resources
  6701. * @vsi: the VSI being removed
  6702. *
  6703. * Returns 0 on success or < 0 on error
  6704. **/
  6705. int i40e_vsi_release(struct i40e_vsi *vsi)
  6706. {
  6707. struct i40e_mac_filter *f, *ftmp;
  6708. struct i40e_veb *veb = NULL;
  6709. struct i40e_pf *pf;
  6710. u16 uplink_seid;
  6711. int i, n;
  6712. pf = vsi->back;
  6713. /* release of a VEB-owner or last VSI is not allowed */
  6714. if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
  6715. dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
  6716. vsi->seid, vsi->uplink_seid);
  6717. return -ENODEV;
  6718. }
  6719. if (vsi == pf->vsi[pf->lan_vsi] &&
  6720. !test_bit(__I40E_DOWN, &pf->state)) {
  6721. dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
  6722. return -ENODEV;
  6723. }
  6724. uplink_seid = vsi->uplink_seid;
  6725. if (vsi->type != I40E_VSI_SRIOV) {
  6726. if (vsi->netdev_registered) {
  6727. vsi->netdev_registered = false;
  6728. if (vsi->netdev) {
  6729. /* results in a call to i40e_close() */
  6730. unregister_netdev(vsi->netdev);
  6731. }
  6732. } else {
  6733. i40e_vsi_close(vsi);
  6734. }
  6735. i40e_vsi_disable_irq(vsi);
  6736. }
  6737. list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list)
  6738. i40e_del_filter(vsi, f->macaddr, f->vlan,
  6739. f->is_vf, f->is_netdev);
  6740. i40e_sync_vsi_filters(vsi);
  6741. i40e_vsi_delete(vsi);
  6742. i40e_vsi_free_q_vectors(vsi);
  6743. if (vsi->netdev) {
  6744. free_netdev(vsi->netdev);
  6745. vsi->netdev = NULL;
  6746. }
  6747. i40e_vsi_clear_rings(vsi);
  6748. i40e_vsi_clear(vsi);
  6749. /* If this was the last thing on the VEB, except for the
  6750. * controlling VSI, remove the VEB, which puts the controlling
  6751. * VSI onto the next level down in the switch.
  6752. *
  6753. * Well, okay, there's one more exception here: don't remove
  6754. * the orphan VEBs yet. We'll wait for an explicit remove request
  6755. * from up the network stack.
  6756. */
  6757. for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
  6758. if (pf->vsi[i] &&
  6759. pf->vsi[i]->uplink_seid == uplink_seid &&
  6760. (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
  6761. n++; /* count the VSIs */
  6762. }
  6763. }
  6764. for (i = 0; i < I40E_MAX_VEB; i++) {
  6765. if (!pf->veb[i])
  6766. continue;
  6767. if (pf->veb[i]->uplink_seid == uplink_seid)
  6768. n++; /* count the VEBs */
  6769. if (pf->veb[i]->seid == uplink_seid)
  6770. veb = pf->veb[i];
  6771. }
  6772. if (n == 0 && veb && veb->uplink_seid != 0)
  6773. i40e_veb_release(veb);
  6774. return 0;
  6775. }
  6776. /**
  6777. * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
  6778. * @vsi: ptr to the VSI
  6779. *
  6780. * This should only be called after i40e_vsi_mem_alloc() which allocates the
  6781. * corresponding SW VSI structure and initializes num_queue_pairs for the
  6782. * newly allocated VSI.
  6783. *
  6784. * Returns 0 on success or negative on failure
  6785. **/
  6786. static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
  6787. {
  6788. int ret = -ENOENT;
  6789. struct i40e_pf *pf = vsi->back;
  6790. if (vsi->q_vectors[0]) {
  6791. dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
  6792. vsi->seid);
  6793. return -EEXIST;
  6794. }
  6795. if (vsi->base_vector) {
  6796. dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
  6797. vsi->seid, vsi->base_vector);
  6798. return -EEXIST;
  6799. }
  6800. ret = i40e_vsi_alloc_q_vectors(vsi);
  6801. if (ret) {
  6802. dev_info(&pf->pdev->dev,
  6803. "failed to allocate %d q_vector for VSI %d, ret=%d\n",
  6804. vsi->num_q_vectors, vsi->seid, ret);
  6805. vsi->num_q_vectors = 0;
  6806. goto vector_setup_out;
  6807. }
  6808. if (vsi->num_q_vectors)
  6809. vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
  6810. vsi->num_q_vectors, vsi->idx);
  6811. if (vsi->base_vector < 0) {
  6812. dev_info(&pf->pdev->dev,
  6813. "failed to get queue tracking for VSI %d, err=%d\n",
  6814. vsi->seid, vsi->base_vector);
  6815. i40e_vsi_free_q_vectors(vsi);
  6816. ret = -ENOENT;
  6817. goto vector_setup_out;
  6818. }
  6819. vector_setup_out:
  6820. return ret;
  6821. }
  6822. /**
  6823. * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
  6824. * @vsi: pointer to the vsi.
  6825. *
  6826. * This re-allocates a vsi's queue resources.
  6827. *
  6828. * Returns pointer to the successfully allocated and configured VSI sw struct
  6829. * on success, otherwise returns NULL on failure.
  6830. **/
  6831. static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
  6832. {
  6833. struct i40e_pf *pf = vsi->back;
  6834. u8 enabled_tc;
  6835. int ret;
  6836. i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
  6837. i40e_vsi_clear_rings(vsi);
  6838. i40e_vsi_free_arrays(vsi, false);
  6839. i40e_set_num_rings_in_vsi(vsi);
  6840. ret = i40e_vsi_alloc_arrays(vsi, false);
  6841. if (ret)
  6842. goto err_vsi;
  6843. ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx);
  6844. if (ret < 0) {
  6845. dev_info(&pf->pdev->dev, "VSI %d get_lump failed %d\n",
  6846. vsi->seid, ret);
  6847. goto err_vsi;
  6848. }
  6849. vsi->base_queue = ret;
  6850. /* Update the FW view of the VSI. Force a reset of TC and queue
  6851. * layout configurations.
  6852. */
  6853. enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
  6854. pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
  6855. pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
  6856. i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
  6857. /* assign it some queues */
  6858. ret = i40e_alloc_rings(vsi);
  6859. if (ret)
  6860. goto err_rings;
  6861. /* map all of the rings to the q_vectors */
  6862. i40e_vsi_map_rings_to_vectors(vsi);
  6863. return vsi;
  6864. err_rings:
  6865. i40e_vsi_free_q_vectors(vsi);
  6866. if (vsi->netdev_registered) {
  6867. vsi->netdev_registered = false;
  6868. unregister_netdev(vsi->netdev);
  6869. free_netdev(vsi->netdev);
  6870. vsi->netdev = NULL;
  6871. }
  6872. i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
  6873. err_vsi:
  6874. i40e_vsi_clear(vsi);
  6875. return NULL;
  6876. }
  6877. /**
  6878. * i40e_vsi_setup - Set up a VSI by a given type
  6879. * @pf: board private structure
  6880. * @type: VSI type
  6881. * @uplink_seid: the switch element to link to
  6882. * @param1: usage depends upon VSI type. For VF types, indicates VF id
  6883. *
  6884. * This allocates the sw VSI structure and its queue resources, then add a VSI
  6885. * to the identified VEB.
  6886. *
  6887. * Returns pointer to the successfully allocated and configure VSI sw struct on
  6888. * success, otherwise returns NULL on failure.
  6889. **/
  6890. struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
  6891. u16 uplink_seid, u32 param1)
  6892. {
  6893. struct i40e_vsi *vsi = NULL;
  6894. struct i40e_veb *veb = NULL;
  6895. int ret, i;
  6896. int v_idx;
  6897. /* The requested uplink_seid must be either
  6898. * - the PF's port seid
  6899. * no VEB is needed because this is the PF
  6900. * or this is a Flow Director special case VSI
  6901. * - seid of an existing VEB
  6902. * - seid of a VSI that owns an existing VEB
  6903. * - seid of a VSI that doesn't own a VEB
  6904. * a new VEB is created and the VSI becomes the owner
  6905. * - seid of the PF VSI, which is what creates the first VEB
  6906. * this is a special case of the previous
  6907. *
  6908. * Find which uplink_seid we were given and create a new VEB if needed
  6909. */
  6910. for (i = 0; i < I40E_MAX_VEB; i++) {
  6911. if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
  6912. veb = pf->veb[i];
  6913. break;
  6914. }
  6915. }
  6916. if (!veb && uplink_seid != pf->mac_seid) {
  6917. for (i = 0; i < pf->num_alloc_vsi; i++) {
  6918. if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
  6919. vsi = pf->vsi[i];
  6920. break;
  6921. }
  6922. }
  6923. if (!vsi) {
  6924. dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
  6925. uplink_seid);
  6926. return NULL;
  6927. }
  6928. if (vsi->uplink_seid == pf->mac_seid)
  6929. veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
  6930. vsi->tc_config.enabled_tc);
  6931. else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
  6932. veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
  6933. vsi->tc_config.enabled_tc);
  6934. for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
  6935. if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
  6936. veb = pf->veb[i];
  6937. }
  6938. if (!veb) {
  6939. dev_info(&pf->pdev->dev, "couldn't add VEB\n");
  6940. return NULL;
  6941. }
  6942. vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
  6943. uplink_seid = veb->seid;
  6944. }
  6945. /* get vsi sw struct */
  6946. v_idx = i40e_vsi_mem_alloc(pf, type);
  6947. if (v_idx < 0)
  6948. goto err_alloc;
  6949. vsi = pf->vsi[v_idx];
  6950. if (!vsi)
  6951. goto err_alloc;
  6952. vsi->type = type;
  6953. vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
  6954. if (type == I40E_VSI_MAIN)
  6955. pf->lan_vsi = v_idx;
  6956. else if (type == I40E_VSI_SRIOV)
  6957. vsi->vf_id = param1;
  6958. /* assign it some queues */
  6959. ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs,
  6960. vsi->idx);
  6961. if (ret < 0) {
  6962. dev_info(&pf->pdev->dev, "VSI %d get_lump failed %d\n",
  6963. vsi->seid, ret);
  6964. goto err_vsi;
  6965. }
  6966. vsi->base_queue = ret;
  6967. /* get a VSI from the hardware */
  6968. vsi->uplink_seid = uplink_seid;
  6969. ret = i40e_add_vsi(vsi);
  6970. if (ret)
  6971. goto err_vsi;
  6972. switch (vsi->type) {
  6973. /* setup the netdev if needed */
  6974. case I40E_VSI_MAIN:
  6975. case I40E_VSI_VMDQ2:
  6976. case I40E_VSI_FCOE:
  6977. ret = i40e_config_netdev(vsi);
  6978. if (ret)
  6979. goto err_netdev;
  6980. ret = register_netdev(vsi->netdev);
  6981. if (ret)
  6982. goto err_netdev;
  6983. vsi->netdev_registered = true;
  6984. netif_carrier_off(vsi->netdev);
  6985. #ifdef CONFIG_I40E_DCB
  6986. /* Setup DCB netlink interface */
  6987. i40e_dcbnl_setup(vsi);
  6988. #endif /* CONFIG_I40E_DCB */
  6989. /* fall through */
  6990. case I40E_VSI_FDIR:
  6991. /* set up vectors and rings if needed */
  6992. ret = i40e_vsi_setup_vectors(vsi);
  6993. if (ret)
  6994. goto err_msix;
  6995. ret = i40e_alloc_rings(vsi);
  6996. if (ret)
  6997. goto err_rings;
  6998. /* map all of the rings to the q_vectors */
  6999. i40e_vsi_map_rings_to_vectors(vsi);
  7000. i40e_vsi_reset_stats(vsi);
  7001. break;
  7002. default:
  7003. /* no netdev or rings for the other VSI types */
  7004. break;
  7005. }
  7006. return vsi;
  7007. err_rings:
  7008. i40e_vsi_free_q_vectors(vsi);
  7009. err_msix:
  7010. if (vsi->netdev_registered) {
  7011. vsi->netdev_registered = false;
  7012. unregister_netdev(vsi->netdev);
  7013. free_netdev(vsi->netdev);
  7014. vsi->netdev = NULL;
  7015. }
  7016. err_netdev:
  7017. i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
  7018. err_vsi:
  7019. i40e_vsi_clear(vsi);
  7020. err_alloc:
  7021. return NULL;
  7022. }
  7023. /**
  7024. * i40e_veb_get_bw_info - Query VEB BW information
  7025. * @veb: the veb to query
  7026. *
  7027. * Query the Tx scheduler BW configuration data for given VEB
  7028. **/
  7029. static int i40e_veb_get_bw_info(struct i40e_veb *veb)
  7030. {
  7031. struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
  7032. struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
  7033. struct i40e_pf *pf = veb->pf;
  7034. struct i40e_hw *hw = &pf->hw;
  7035. u32 tc_bw_max;
  7036. int ret = 0;
  7037. int i;
  7038. ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
  7039. &bw_data, NULL);
  7040. if (ret) {
  7041. dev_info(&pf->pdev->dev,
  7042. "query veb bw config failed, aq_err=%d\n",
  7043. hw->aq.asq_last_status);
  7044. goto out;
  7045. }
  7046. ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
  7047. &ets_data, NULL);
  7048. if (ret) {
  7049. dev_info(&pf->pdev->dev,
  7050. "query veb bw ets config failed, aq_err=%d\n",
  7051. hw->aq.asq_last_status);
  7052. goto out;
  7053. }
  7054. veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
  7055. veb->bw_max_quanta = ets_data.tc_bw_max;
  7056. veb->is_abs_credits = bw_data.absolute_credits_enable;
  7057. tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
  7058. (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
  7059. for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
  7060. veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
  7061. veb->bw_tc_limit_credits[i] =
  7062. le16_to_cpu(bw_data.tc_bw_limits[i]);
  7063. veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
  7064. }
  7065. out:
  7066. return ret;
  7067. }
  7068. /**
  7069. * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
  7070. * @pf: board private structure
  7071. *
  7072. * On error: returns error code (negative)
  7073. * On success: returns vsi index in PF (positive)
  7074. **/
  7075. static int i40e_veb_mem_alloc(struct i40e_pf *pf)
  7076. {
  7077. int ret = -ENOENT;
  7078. struct i40e_veb *veb;
  7079. int i;
  7080. /* Need to protect the allocation of switch elements at the PF level */
  7081. mutex_lock(&pf->switch_mutex);
  7082. /* VEB list may be fragmented if VEB creation/destruction has
  7083. * been happening. We can afford to do a quick scan to look
  7084. * for any free slots in the list.
  7085. *
  7086. * find next empty veb slot, looping back around if necessary
  7087. */
  7088. i = 0;
  7089. while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
  7090. i++;
  7091. if (i >= I40E_MAX_VEB) {
  7092. ret = -ENOMEM;
  7093. goto err_alloc_veb; /* out of VEB slots! */
  7094. }
  7095. veb = kzalloc(sizeof(*veb), GFP_KERNEL);
  7096. if (!veb) {
  7097. ret = -ENOMEM;
  7098. goto err_alloc_veb;
  7099. }
  7100. veb->pf = pf;
  7101. veb->idx = i;
  7102. veb->enabled_tc = 1;
  7103. pf->veb[i] = veb;
  7104. ret = i;
  7105. err_alloc_veb:
  7106. mutex_unlock(&pf->switch_mutex);
  7107. return ret;
  7108. }
  7109. /**
  7110. * i40e_switch_branch_release - Delete a branch of the switch tree
  7111. * @branch: where to start deleting
  7112. *
  7113. * This uses recursion to find the tips of the branch to be
  7114. * removed, deleting until we get back to and can delete this VEB.
  7115. **/
  7116. static void i40e_switch_branch_release(struct i40e_veb *branch)
  7117. {
  7118. struct i40e_pf *pf = branch->pf;
  7119. u16 branch_seid = branch->seid;
  7120. u16 veb_idx = branch->idx;
  7121. int i;
  7122. /* release any VEBs on this VEB - RECURSION */
  7123. for (i = 0; i < I40E_MAX_VEB; i++) {
  7124. if (!pf->veb[i])
  7125. continue;
  7126. if (pf->veb[i]->uplink_seid == branch->seid)
  7127. i40e_switch_branch_release(pf->veb[i]);
  7128. }
  7129. /* Release the VSIs on this VEB, but not the owner VSI.
  7130. *
  7131. * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
  7132. * the VEB itself, so don't use (*branch) after this loop.
  7133. */
  7134. for (i = 0; i < pf->num_alloc_vsi; i++) {
  7135. if (!pf->vsi[i])
  7136. continue;
  7137. if (pf->vsi[i]->uplink_seid == branch_seid &&
  7138. (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
  7139. i40e_vsi_release(pf->vsi[i]);
  7140. }
  7141. }
  7142. /* There's one corner case where the VEB might not have been
  7143. * removed, so double check it here and remove it if needed.
  7144. * This case happens if the veb was created from the debugfs
  7145. * commands and no VSIs were added to it.
  7146. */
  7147. if (pf->veb[veb_idx])
  7148. i40e_veb_release(pf->veb[veb_idx]);
  7149. }
  7150. /**
  7151. * i40e_veb_clear - remove veb struct
  7152. * @veb: the veb to remove
  7153. **/
  7154. static void i40e_veb_clear(struct i40e_veb *veb)
  7155. {
  7156. if (!veb)
  7157. return;
  7158. if (veb->pf) {
  7159. struct i40e_pf *pf = veb->pf;
  7160. mutex_lock(&pf->switch_mutex);
  7161. if (pf->veb[veb->idx] == veb)
  7162. pf->veb[veb->idx] = NULL;
  7163. mutex_unlock(&pf->switch_mutex);
  7164. }
  7165. kfree(veb);
  7166. }
  7167. /**
  7168. * i40e_veb_release - Delete a VEB and free its resources
  7169. * @veb: the VEB being removed
  7170. **/
  7171. void i40e_veb_release(struct i40e_veb *veb)
  7172. {
  7173. struct i40e_vsi *vsi = NULL;
  7174. struct i40e_pf *pf;
  7175. int i, n = 0;
  7176. pf = veb->pf;
  7177. /* find the remaining VSI and check for extras */
  7178. for (i = 0; i < pf->num_alloc_vsi; i++) {
  7179. if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
  7180. n++;
  7181. vsi = pf->vsi[i];
  7182. }
  7183. }
  7184. if (n != 1) {
  7185. dev_info(&pf->pdev->dev,
  7186. "can't remove VEB %d with %d VSIs left\n",
  7187. veb->seid, n);
  7188. return;
  7189. }
  7190. /* move the remaining VSI to uplink veb */
  7191. vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
  7192. if (veb->uplink_seid) {
  7193. vsi->uplink_seid = veb->uplink_seid;
  7194. if (veb->uplink_seid == pf->mac_seid)
  7195. vsi->veb_idx = I40E_NO_VEB;
  7196. else
  7197. vsi->veb_idx = veb->veb_idx;
  7198. } else {
  7199. /* floating VEB */
  7200. vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
  7201. vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
  7202. }
  7203. i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
  7204. i40e_veb_clear(veb);
  7205. }
  7206. /**
  7207. * i40e_add_veb - create the VEB in the switch
  7208. * @veb: the VEB to be instantiated
  7209. * @vsi: the controlling VSI
  7210. **/
  7211. static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
  7212. {
  7213. bool is_default = false;
  7214. bool is_cloud = false;
  7215. int ret;
  7216. /* get a VEB from the hardware */
  7217. ret = i40e_aq_add_veb(&veb->pf->hw, veb->uplink_seid, vsi->seid,
  7218. veb->enabled_tc, is_default,
  7219. is_cloud, &veb->seid, NULL);
  7220. if (ret) {
  7221. dev_info(&veb->pf->pdev->dev,
  7222. "couldn't add VEB, err %d, aq_err %d\n",
  7223. ret, veb->pf->hw.aq.asq_last_status);
  7224. return -EPERM;
  7225. }
  7226. /* get statistics counter */
  7227. ret = i40e_aq_get_veb_parameters(&veb->pf->hw, veb->seid, NULL, NULL,
  7228. &veb->stats_idx, NULL, NULL, NULL);
  7229. if (ret) {
  7230. dev_info(&veb->pf->pdev->dev,
  7231. "couldn't get VEB statistics idx, err %d, aq_err %d\n",
  7232. ret, veb->pf->hw.aq.asq_last_status);
  7233. return -EPERM;
  7234. }
  7235. ret = i40e_veb_get_bw_info(veb);
  7236. if (ret) {
  7237. dev_info(&veb->pf->pdev->dev,
  7238. "couldn't get VEB bw info, err %d, aq_err %d\n",
  7239. ret, veb->pf->hw.aq.asq_last_status);
  7240. i40e_aq_delete_element(&veb->pf->hw, veb->seid, NULL);
  7241. return -ENOENT;
  7242. }
  7243. vsi->uplink_seid = veb->seid;
  7244. vsi->veb_idx = veb->idx;
  7245. vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
  7246. return 0;
  7247. }
  7248. /**
  7249. * i40e_veb_setup - Set up a VEB
  7250. * @pf: board private structure
  7251. * @flags: VEB setup flags
  7252. * @uplink_seid: the switch element to link to
  7253. * @vsi_seid: the initial VSI seid
  7254. * @enabled_tc: Enabled TC bit-map
  7255. *
  7256. * This allocates the sw VEB structure and links it into the switch
  7257. * It is possible and legal for this to be a duplicate of an already
  7258. * existing VEB. It is also possible for both uplink and vsi seids
  7259. * to be zero, in order to create a floating VEB.
  7260. *
  7261. * Returns pointer to the successfully allocated VEB sw struct on
  7262. * success, otherwise returns NULL on failure.
  7263. **/
  7264. struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
  7265. u16 uplink_seid, u16 vsi_seid,
  7266. u8 enabled_tc)
  7267. {
  7268. struct i40e_veb *veb, *uplink_veb = NULL;
  7269. int vsi_idx, veb_idx;
  7270. int ret;
  7271. /* if one seid is 0, the other must be 0 to create a floating relay */
  7272. if ((uplink_seid == 0 || vsi_seid == 0) &&
  7273. (uplink_seid + vsi_seid != 0)) {
  7274. dev_info(&pf->pdev->dev,
  7275. "one, not both seid's are 0: uplink=%d vsi=%d\n",
  7276. uplink_seid, vsi_seid);
  7277. return NULL;
  7278. }
  7279. /* make sure there is such a vsi and uplink */
  7280. for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
  7281. if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
  7282. break;
  7283. if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) {
  7284. dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
  7285. vsi_seid);
  7286. return NULL;
  7287. }
  7288. if (uplink_seid && uplink_seid != pf->mac_seid) {
  7289. for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
  7290. if (pf->veb[veb_idx] &&
  7291. pf->veb[veb_idx]->seid == uplink_seid) {
  7292. uplink_veb = pf->veb[veb_idx];
  7293. break;
  7294. }
  7295. }
  7296. if (!uplink_veb) {
  7297. dev_info(&pf->pdev->dev,
  7298. "uplink seid %d not found\n", uplink_seid);
  7299. return NULL;
  7300. }
  7301. }
  7302. /* get veb sw struct */
  7303. veb_idx = i40e_veb_mem_alloc(pf);
  7304. if (veb_idx < 0)
  7305. goto err_alloc;
  7306. veb = pf->veb[veb_idx];
  7307. veb->flags = flags;
  7308. veb->uplink_seid = uplink_seid;
  7309. veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
  7310. veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
  7311. /* create the VEB in the switch */
  7312. ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
  7313. if (ret)
  7314. goto err_veb;
  7315. if (vsi_idx == pf->lan_vsi)
  7316. pf->lan_veb = veb->idx;
  7317. return veb;
  7318. err_veb:
  7319. i40e_veb_clear(veb);
  7320. err_alloc:
  7321. return NULL;
  7322. }
  7323. /**
  7324. * i40e_setup_pf_switch_element - set pf vars based on switch type
  7325. * @pf: board private structure
  7326. * @ele: element we are building info from
  7327. * @num_reported: total number of elements
  7328. * @printconfig: should we print the contents
  7329. *
  7330. * helper function to assist in extracting a few useful SEID values.
  7331. **/
  7332. static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
  7333. struct i40e_aqc_switch_config_element_resp *ele,
  7334. u16 num_reported, bool printconfig)
  7335. {
  7336. u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
  7337. u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
  7338. u8 element_type = ele->element_type;
  7339. u16 seid = le16_to_cpu(ele->seid);
  7340. if (printconfig)
  7341. dev_info(&pf->pdev->dev,
  7342. "type=%d seid=%d uplink=%d downlink=%d\n",
  7343. element_type, seid, uplink_seid, downlink_seid);
  7344. switch (element_type) {
  7345. case I40E_SWITCH_ELEMENT_TYPE_MAC:
  7346. pf->mac_seid = seid;
  7347. break;
  7348. case I40E_SWITCH_ELEMENT_TYPE_VEB:
  7349. /* Main VEB? */
  7350. if (uplink_seid != pf->mac_seid)
  7351. break;
  7352. if (pf->lan_veb == I40E_NO_VEB) {
  7353. int v;
  7354. /* find existing or else empty VEB */
  7355. for (v = 0; v < I40E_MAX_VEB; v++) {
  7356. if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
  7357. pf->lan_veb = v;
  7358. break;
  7359. }
  7360. }
  7361. if (pf->lan_veb == I40E_NO_VEB) {
  7362. v = i40e_veb_mem_alloc(pf);
  7363. if (v < 0)
  7364. break;
  7365. pf->lan_veb = v;
  7366. }
  7367. }
  7368. pf->veb[pf->lan_veb]->seid = seid;
  7369. pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
  7370. pf->veb[pf->lan_veb]->pf = pf;
  7371. pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
  7372. break;
  7373. case I40E_SWITCH_ELEMENT_TYPE_VSI:
  7374. if (num_reported != 1)
  7375. break;
  7376. /* This is immediately after a reset so we can assume this is
  7377. * the PF's VSI
  7378. */
  7379. pf->mac_seid = uplink_seid;
  7380. pf->pf_seid = downlink_seid;
  7381. pf->main_vsi_seid = seid;
  7382. if (printconfig)
  7383. dev_info(&pf->pdev->dev,
  7384. "pf_seid=%d main_vsi_seid=%d\n",
  7385. pf->pf_seid, pf->main_vsi_seid);
  7386. break;
  7387. case I40E_SWITCH_ELEMENT_TYPE_PF:
  7388. case I40E_SWITCH_ELEMENT_TYPE_VF:
  7389. case I40E_SWITCH_ELEMENT_TYPE_EMP:
  7390. case I40E_SWITCH_ELEMENT_TYPE_BMC:
  7391. case I40E_SWITCH_ELEMENT_TYPE_PE:
  7392. case I40E_SWITCH_ELEMENT_TYPE_PA:
  7393. /* ignore these for now */
  7394. break;
  7395. default:
  7396. dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
  7397. element_type, seid);
  7398. break;
  7399. }
  7400. }
  7401. /**
  7402. * i40e_fetch_switch_configuration - Get switch config from firmware
  7403. * @pf: board private structure
  7404. * @printconfig: should we print the contents
  7405. *
  7406. * Get the current switch configuration from the device and
  7407. * extract a few useful SEID values.
  7408. **/
  7409. int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
  7410. {
  7411. struct i40e_aqc_get_switch_config_resp *sw_config;
  7412. u16 next_seid = 0;
  7413. int ret = 0;
  7414. u8 *aq_buf;
  7415. int i;
  7416. aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
  7417. if (!aq_buf)
  7418. return -ENOMEM;
  7419. sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
  7420. do {
  7421. u16 num_reported, num_total;
  7422. ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
  7423. I40E_AQ_LARGE_BUF,
  7424. &next_seid, NULL);
  7425. if (ret) {
  7426. dev_info(&pf->pdev->dev,
  7427. "get switch config failed %d aq_err=%x\n",
  7428. ret, pf->hw.aq.asq_last_status);
  7429. kfree(aq_buf);
  7430. return -ENOENT;
  7431. }
  7432. num_reported = le16_to_cpu(sw_config->header.num_reported);
  7433. num_total = le16_to_cpu(sw_config->header.num_total);
  7434. if (printconfig)
  7435. dev_info(&pf->pdev->dev,
  7436. "header: %d reported %d total\n",
  7437. num_reported, num_total);
  7438. for (i = 0; i < num_reported; i++) {
  7439. struct i40e_aqc_switch_config_element_resp *ele =
  7440. &sw_config->element[i];
  7441. i40e_setup_pf_switch_element(pf, ele, num_reported,
  7442. printconfig);
  7443. }
  7444. } while (next_seid != 0);
  7445. kfree(aq_buf);
  7446. return ret;
  7447. }
  7448. /**
  7449. * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
  7450. * @pf: board private structure
  7451. * @reinit: if the Main VSI needs to re-initialized.
  7452. *
  7453. * Returns 0 on success, negative value on failure
  7454. **/
  7455. static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
  7456. {
  7457. int ret;
  7458. /* find out what's out there already */
  7459. ret = i40e_fetch_switch_configuration(pf, false);
  7460. if (ret) {
  7461. dev_info(&pf->pdev->dev,
  7462. "couldn't fetch switch config, err %d, aq_err %d\n",
  7463. ret, pf->hw.aq.asq_last_status);
  7464. return ret;
  7465. }
  7466. i40e_pf_reset_stats(pf);
  7467. /* first time setup */
  7468. if (pf->lan_vsi == I40E_NO_VSI || reinit) {
  7469. struct i40e_vsi *vsi = NULL;
  7470. u16 uplink_seid;
  7471. /* Set up the PF VSI associated with the PF's main VSI
  7472. * that is already in the HW switch
  7473. */
  7474. if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
  7475. uplink_seid = pf->veb[pf->lan_veb]->seid;
  7476. else
  7477. uplink_seid = pf->mac_seid;
  7478. if (pf->lan_vsi == I40E_NO_VSI)
  7479. vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
  7480. else if (reinit)
  7481. vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
  7482. if (!vsi) {
  7483. dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
  7484. i40e_fdir_teardown(pf);
  7485. return -EAGAIN;
  7486. }
  7487. } else {
  7488. /* force a reset of TC and queue layout configurations */
  7489. u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
  7490. pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
  7491. pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
  7492. i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
  7493. }
  7494. i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
  7495. i40e_fdir_sb_setup(pf);
  7496. /* Setup static PF queue filter control settings */
  7497. ret = i40e_setup_pf_filter_control(pf);
  7498. if (ret) {
  7499. dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
  7500. ret);
  7501. /* Failure here should not stop continuing other steps */
  7502. }
  7503. /* enable RSS in the HW, even for only one queue, as the stack can use
  7504. * the hash
  7505. */
  7506. if ((pf->flags & I40E_FLAG_RSS_ENABLED))
  7507. i40e_config_rss(pf);
  7508. /* fill in link information and enable LSE reporting */
  7509. i40e_update_link_info(&pf->hw, true);
  7510. i40e_link_event(pf);
  7511. /* Initialize user-specific link properties */
  7512. pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
  7513. I40E_AQ_AN_COMPLETED) ? true : false);
  7514. i40e_ptp_init(pf);
  7515. return ret;
  7516. }
  7517. /**
  7518. * i40e_determine_queue_usage - Work out queue distribution
  7519. * @pf: board private structure
  7520. **/
  7521. static void i40e_determine_queue_usage(struct i40e_pf *pf)
  7522. {
  7523. int queues_left;
  7524. pf->num_lan_qps = 0;
  7525. #ifdef I40E_FCOE
  7526. pf->num_fcoe_qps = 0;
  7527. #endif
  7528. /* Find the max queues to be put into basic use. We'll always be
  7529. * using TC0, whether or not DCB is running, and TC0 will get the
  7530. * big RSS set.
  7531. */
  7532. queues_left = pf->hw.func_caps.num_tx_qp;
  7533. if ((queues_left == 1) ||
  7534. !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
  7535. /* one qp for PF, no queues for anything else */
  7536. queues_left = 0;
  7537. pf->rss_size = pf->num_lan_qps = 1;
  7538. /* make sure all the fancies are disabled */
  7539. pf->flags &= ~(I40E_FLAG_RSS_ENABLED |
  7540. #ifdef I40E_FCOE
  7541. I40E_FLAG_FCOE_ENABLED |
  7542. #endif
  7543. I40E_FLAG_FD_SB_ENABLED |
  7544. I40E_FLAG_FD_ATR_ENABLED |
  7545. I40E_FLAG_DCB_CAPABLE |
  7546. I40E_FLAG_SRIOV_ENABLED |
  7547. I40E_FLAG_VMDQ_ENABLED);
  7548. } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
  7549. I40E_FLAG_FD_SB_ENABLED |
  7550. I40E_FLAG_FD_ATR_ENABLED |
  7551. I40E_FLAG_DCB_CAPABLE))) {
  7552. /* one qp for PF */
  7553. pf->rss_size = pf->num_lan_qps = 1;
  7554. queues_left -= pf->num_lan_qps;
  7555. pf->flags &= ~(I40E_FLAG_RSS_ENABLED |
  7556. #ifdef I40E_FCOE
  7557. I40E_FLAG_FCOE_ENABLED |
  7558. #endif
  7559. I40E_FLAG_FD_SB_ENABLED |
  7560. I40E_FLAG_FD_ATR_ENABLED |
  7561. I40E_FLAG_DCB_ENABLED |
  7562. I40E_FLAG_VMDQ_ENABLED);
  7563. } else {
  7564. /* Not enough queues for all TCs */
  7565. if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
  7566. (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
  7567. pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
  7568. dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
  7569. }
  7570. pf->num_lan_qps = pf->rss_size_max;
  7571. queues_left -= pf->num_lan_qps;
  7572. }
  7573. #ifdef I40E_FCOE
  7574. if (pf->flags & I40E_FLAG_FCOE_ENABLED) {
  7575. if (I40E_DEFAULT_FCOE <= queues_left) {
  7576. pf->num_fcoe_qps = I40E_DEFAULT_FCOE;
  7577. } else if (I40E_MINIMUM_FCOE <= queues_left) {
  7578. pf->num_fcoe_qps = I40E_MINIMUM_FCOE;
  7579. } else {
  7580. pf->num_fcoe_qps = 0;
  7581. pf->flags &= ~I40E_FLAG_FCOE_ENABLED;
  7582. dev_info(&pf->pdev->dev, "not enough queues for FCoE. FCoE feature will be disabled\n");
  7583. }
  7584. queues_left -= pf->num_fcoe_qps;
  7585. }
  7586. #endif
  7587. if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
  7588. if (queues_left > 1) {
  7589. queues_left -= 1; /* save 1 queue for FD */
  7590. } else {
  7591. pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
  7592. dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
  7593. }
  7594. }
  7595. if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
  7596. pf->num_vf_qps && pf->num_req_vfs && queues_left) {
  7597. pf->num_req_vfs = min_t(int, pf->num_req_vfs,
  7598. (queues_left / pf->num_vf_qps));
  7599. queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
  7600. }
  7601. if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
  7602. pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
  7603. pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
  7604. (queues_left / pf->num_vmdq_qps));
  7605. queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
  7606. }
  7607. pf->queues_left = queues_left;
  7608. #ifdef I40E_FCOE
  7609. dev_info(&pf->pdev->dev, "fcoe queues = %d\n", pf->num_fcoe_qps);
  7610. #endif
  7611. }
  7612. /**
  7613. * i40e_setup_pf_filter_control - Setup PF static filter control
  7614. * @pf: PF to be setup
  7615. *
  7616. * i40e_setup_pf_filter_control sets up a pf's initial filter control
  7617. * settings. If PE/FCoE are enabled then it will also set the per PF
  7618. * based filter sizes required for them. It also enables Flow director,
  7619. * ethertype and macvlan type filter settings for the pf.
  7620. *
  7621. * Returns 0 on success, negative on failure
  7622. **/
  7623. static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
  7624. {
  7625. struct i40e_filter_control_settings *settings = &pf->filter_settings;
  7626. settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
  7627. /* Flow Director is enabled */
  7628. if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
  7629. settings->enable_fdir = true;
  7630. /* Ethtype and MACVLAN filters enabled for PF */
  7631. settings->enable_ethtype = true;
  7632. settings->enable_macvlan = true;
  7633. if (i40e_set_filter_control(&pf->hw, settings))
  7634. return -ENOENT;
  7635. return 0;
  7636. }
  7637. #define INFO_STRING_LEN 255
  7638. static void i40e_print_features(struct i40e_pf *pf)
  7639. {
  7640. struct i40e_hw *hw = &pf->hw;
  7641. char *buf, *string;
  7642. string = kzalloc(INFO_STRING_LEN, GFP_KERNEL);
  7643. if (!string) {
  7644. dev_err(&pf->pdev->dev, "Features string allocation failed\n");
  7645. return;
  7646. }
  7647. buf = string;
  7648. buf += sprintf(string, "Features: PF-id[%d] ", hw->pf_id);
  7649. #ifdef CONFIG_PCI_IOV
  7650. buf += sprintf(buf, "VFs: %d ", pf->num_req_vfs);
  7651. #endif
  7652. buf += sprintf(buf, "VSIs: %d QP: %d ", pf->hw.func_caps.num_vsis,
  7653. pf->vsi[pf->lan_vsi]->num_queue_pairs);
  7654. if (pf->flags & I40E_FLAG_RSS_ENABLED)
  7655. buf += sprintf(buf, "RSS ");
  7656. if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
  7657. buf += sprintf(buf, "FD_ATR ");
  7658. if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
  7659. buf += sprintf(buf, "FD_SB ");
  7660. buf += sprintf(buf, "NTUPLE ");
  7661. }
  7662. if (pf->flags & I40E_FLAG_DCB_CAPABLE)
  7663. buf += sprintf(buf, "DCB ");
  7664. if (pf->flags & I40E_FLAG_PTP)
  7665. buf += sprintf(buf, "PTP ");
  7666. #ifdef I40E_FCOE
  7667. if (pf->flags & I40E_FLAG_FCOE_ENABLED)
  7668. buf += sprintf(buf, "FCOE ");
  7669. #endif
  7670. BUG_ON(buf > (string + INFO_STRING_LEN));
  7671. dev_info(&pf->pdev->dev, "%s\n", string);
  7672. kfree(string);
  7673. }
  7674. /**
  7675. * i40e_probe - Device initialization routine
  7676. * @pdev: PCI device information struct
  7677. * @ent: entry in i40e_pci_tbl
  7678. *
  7679. * i40e_probe initializes a pf identified by a pci_dev structure.
  7680. * The OS initialization, configuring of the pf private structure,
  7681. * and a hardware reset occur.
  7682. *
  7683. * Returns 0 on success, negative on failure
  7684. **/
  7685. static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
  7686. {
  7687. struct i40e_pf *pf;
  7688. struct i40e_hw *hw;
  7689. static u16 pfs_found;
  7690. u16 link_status;
  7691. int err = 0;
  7692. u32 len;
  7693. u32 i;
  7694. err = pci_enable_device_mem(pdev);
  7695. if (err)
  7696. return err;
  7697. /* set up for high or low dma */
  7698. err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
  7699. if (err) {
  7700. err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
  7701. if (err) {
  7702. dev_err(&pdev->dev,
  7703. "DMA configuration failed: 0x%x\n", err);
  7704. goto err_dma;
  7705. }
  7706. }
  7707. /* set up pci connections */
  7708. err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
  7709. IORESOURCE_MEM), i40e_driver_name);
  7710. if (err) {
  7711. dev_info(&pdev->dev,
  7712. "pci_request_selected_regions failed %d\n", err);
  7713. goto err_pci_reg;
  7714. }
  7715. pci_enable_pcie_error_reporting(pdev);
  7716. pci_set_master(pdev);
  7717. /* Now that we have a PCI connection, we need to do the
  7718. * low level device setup. This is primarily setting up
  7719. * the Admin Queue structures and then querying for the
  7720. * device's current profile information.
  7721. */
  7722. pf = kzalloc(sizeof(*pf), GFP_KERNEL);
  7723. if (!pf) {
  7724. err = -ENOMEM;
  7725. goto err_pf_alloc;
  7726. }
  7727. pf->next_vsi = 0;
  7728. pf->pdev = pdev;
  7729. set_bit(__I40E_DOWN, &pf->state);
  7730. hw = &pf->hw;
  7731. hw->back = pf;
  7732. hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
  7733. pci_resource_len(pdev, 0));
  7734. if (!hw->hw_addr) {
  7735. err = -EIO;
  7736. dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
  7737. (unsigned int)pci_resource_start(pdev, 0),
  7738. (unsigned int)pci_resource_len(pdev, 0), err);
  7739. goto err_ioremap;
  7740. }
  7741. hw->vendor_id = pdev->vendor;
  7742. hw->device_id = pdev->device;
  7743. pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
  7744. hw->subsystem_vendor_id = pdev->subsystem_vendor;
  7745. hw->subsystem_device_id = pdev->subsystem_device;
  7746. hw->bus.device = PCI_SLOT(pdev->devfn);
  7747. hw->bus.func = PCI_FUNC(pdev->devfn);
  7748. pf->instance = pfs_found;
  7749. /* do a special CORER for clearing PXE mode once at init */
  7750. if (hw->revision_id == 0 &&
  7751. (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
  7752. wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
  7753. i40e_flush(hw);
  7754. msleep(200);
  7755. pf->corer_count++;
  7756. i40e_clear_pxe_mode(hw);
  7757. }
  7758. /* Reset here to make sure all is clean and to define PF 'n' */
  7759. i40e_clear_hw(hw);
  7760. err = i40e_pf_reset(hw);
  7761. if (err) {
  7762. dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err);
  7763. goto err_pf_reset;
  7764. }
  7765. pf->pfr_count++;
  7766. hw->aq.num_arq_entries = I40E_AQ_LEN;
  7767. hw->aq.num_asq_entries = I40E_AQ_LEN;
  7768. hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
  7769. hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
  7770. pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
  7771. snprintf(pf->misc_int_name, sizeof(pf->misc_int_name) - 1,
  7772. "%s-pf%d:misc",
  7773. dev_driver_string(&pf->pdev->dev), pf->hw.pf_id);
  7774. err = i40e_init_shared_code(hw);
  7775. if (err) {
  7776. dev_info(&pdev->dev, "init_shared_code failed: %d\n", err);
  7777. goto err_pf_reset;
  7778. }
  7779. /* set up a default setting for link flow control */
  7780. pf->hw.fc.requested_mode = I40E_FC_NONE;
  7781. err = i40e_init_adminq(hw);
  7782. dev_info(&pdev->dev, "%s\n", i40e_fw_version_str(hw));
  7783. if (err) {
  7784. dev_info(&pdev->dev,
  7785. "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
  7786. goto err_pf_reset;
  7787. }
  7788. if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
  7789. hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
  7790. dev_info(&pdev->dev,
  7791. "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
  7792. else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
  7793. hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
  7794. dev_info(&pdev->dev,
  7795. "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
  7796. i40e_verify_eeprom(pf);
  7797. /* Rev 0 hardware was never productized */
  7798. if (hw->revision_id < 1)
  7799. dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
  7800. i40e_clear_pxe_mode(hw);
  7801. err = i40e_get_capabilities(pf);
  7802. if (err)
  7803. goto err_adminq_setup;
  7804. err = i40e_sw_init(pf);
  7805. if (err) {
  7806. dev_info(&pdev->dev, "sw_init failed: %d\n", err);
  7807. goto err_sw_init;
  7808. }
  7809. err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
  7810. hw->func_caps.num_rx_qp,
  7811. pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num);
  7812. if (err) {
  7813. dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
  7814. goto err_init_lan_hmc;
  7815. }
  7816. err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
  7817. if (err) {
  7818. dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
  7819. err = -ENOENT;
  7820. goto err_configure_lan_hmc;
  7821. }
  7822. i40e_get_mac_addr(hw, hw->mac.addr);
  7823. if (!is_valid_ether_addr(hw->mac.addr)) {
  7824. dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
  7825. err = -EIO;
  7826. goto err_mac_addr;
  7827. }
  7828. dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
  7829. ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
  7830. i40e_get_port_mac_addr(hw, hw->mac.port_addr);
  7831. if (is_valid_ether_addr(hw->mac.port_addr))
  7832. pf->flags |= I40E_FLAG_PORT_ID_VALID;
  7833. #ifdef I40E_FCOE
  7834. err = i40e_get_san_mac_addr(hw, hw->mac.san_addr);
  7835. if (err)
  7836. dev_info(&pdev->dev,
  7837. "(non-fatal) SAN MAC retrieval failed: %d\n", err);
  7838. if (!is_valid_ether_addr(hw->mac.san_addr)) {
  7839. dev_warn(&pdev->dev, "invalid SAN MAC address %pM, falling back to LAN MAC\n",
  7840. hw->mac.san_addr);
  7841. ether_addr_copy(hw->mac.san_addr, hw->mac.addr);
  7842. }
  7843. dev_info(&pf->pdev->dev, "SAN MAC: %pM\n", hw->mac.san_addr);
  7844. #endif /* I40E_FCOE */
  7845. pci_set_drvdata(pdev, pf);
  7846. pci_save_state(pdev);
  7847. #ifdef CONFIG_I40E_DCB
  7848. err = i40e_init_pf_dcb(pf);
  7849. if (err) {
  7850. dev_info(&pdev->dev, "init_pf_dcb failed: %d\n", err);
  7851. pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
  7852. /* Continue without DCB enabled */
  7853. }
  7854. #endif /* CONFIG_I40E_DCB */
  7855. /* set up periodic task facility */
  7856. setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf);
  7857. pf->service_timer_period = HZ;
  7858. INIT_WORK(&pf->service_task, i40e_service_task);
  7859. clear_bit(__I40E_SERVICE_SCHED, &pf->state);
  7860. pf->flags |= I40E_FLAG_NEED_LINK_UPDATE;
  7861. pf->link_check_timeout = jiffies;
  7862. /* WoL defaults to disabled */
  7863. pf->wol_en = false;
  7864. device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
  7865. /* set up the main switch operations */
  7866. i40e_determine_queue_usage(pf);
  7867. i40e_init_interrupt_scheme(pf);
  7868. /* The number of VSIs reported by the FW is the minimum guaranteed
  7869. * to us; HW supports far more and we share the remaining pool with
  7870. * the other PFs. We allocate space for more than the guarantee with
  7871. * the understanding that we might not get them all later.
  7872. */
  7873. if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
  7874. pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
  7875. else
  7876. pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
  7877. /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
  7878. len = sizeof(struct i40e_vsi *) * pf->num_alloc_vsi;
  7879. pf->vsi = kzalloc(len, GFP_KERNEL);
  7880. if (!pf->vsi) {
  7881. err = -ENOMEM;
  7882. goto err_switch_setup;
  7883. }
  7884. err = i40e_setup_pf_switch(pf, false);
  7885. if (err) {
  7886. dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
  7887. goto err_vsis;
  7888. }
  7889. /* if FDIR VSI was set up, start it now */
  7890. for (i = 0; i < pf->num_alloc_vsi; i++) {
  7891. if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
  7892. i40e_vsi_open(pf->vsi[i]);
  7893. break;
  7894. }
  7895. }
  7896. /* The main driver is (mostly) up and happy. We need to set this state
  7897. * before setting up the misc vector or we get a race and the vector
  7898. * ends up disabled forever.
  7899. */
  7900. clear_bit(__I40E_DOWN, &pf->state);
  7901. /* In case of MSIX we are going to setup the misc vector right here
  7902. * to handle admin queue events etc. In case of legacy and MSI
  7903. * the misc functionality and queue processing is combined in
  7904. * the same vector and that gets setup at open.
  7905. */
  7906. if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
  7907. err = i40e_setup_misc_vector(pf);
  7908. if (err) {
  7909. dev_info(&pdev->dev,
  7910. "setup of misc vector failed: %d\n", err);
  7911. goto err_vsis;
  7912. }
  7913. }
  7914. #ifdef CONFIG_PCI_IOV
  7915. /* prep for VF support */
  7916. if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
  7917. (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
  7918. !test_bit(__I40E_BAD_EEPROM, &pf->state)) {
  7919. u32 val;
  7920. /* disable link interrupts for VFs */
  7921. val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
  7922. val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
  7923. wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
  7924. i40e_flush(hw);
  7925. if (pci_num_vf(pdev)) {
  7926. dev_info(&pdev->dev,
  7927. "Active VFs found, allocating resources.\n");
  7928. err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
  7929. if (err)
  7930. dev_info(&pdev->dev,
  7931. "Error %d allocating resources for existing VFs\n",
  7932. err);
  7933. }
  7934. }
  7935. #endif /* CONFIG_PCI_IOV */
  7936. pfs_found++;
  7937. i40e_dbg_pf_init(pf);
  7938. /* tell the firmware that we're starting */
  7939. i40e_send_version(pf);
  7940. /* since everything's happy, start the service_task timer */
  7941. mod_timer(&pf->service_timer,
  7942. round_jiffies(jiffies + pf->service_timer_period));
  7943. #ifdef I40E_FCOE
  7944. /* create FCoE interface */
  7945. i40e_fcoe_vsi_setup(pf);
  7946. #endif
  7947. /* Get the negotiated link width and speed from PCI config space */
  7948. pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, &link_status);
  7949. i40e_set_pci_config_data(hw, link_status);
  7950. dev_info(&pdev->dev, "PCI-Express: %s %s\n",
  7951. (hw->bus.speed == i40e_bus_speed_8000 ? "Speed 8.0GT/s" :
  7952. hw->bus.speed == i40e_bus_speed_5000 ? "Speed 5.0GT/s" :
  7953. hw->bus.speed == i40e_bus_speed_2500 ? "Speed 2.5GT/s" :
  7954. "Unknown"),
  7955. (hw->bus.width == i40e_bus_width_pcie_x8 ? "Width x8" :
  7956. hw->bus.width == i40e_bus_width_pcie_x4 ? "Width x4" :
  7957. hw->bus.width == i40e_bus_width_pcie_x2 ? "Width x2" :
  7958. hw->bus.width == i40e_bus_width_pcie_x1 ? "Width x1" :
  7959. "Unknown"));
  7960. if (hw->bus.width < i40e_bus_width_pcie_x8 ||
  7961. hw->bus.speed < i40e_bus_speed_8000) {
  7962. dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
  7963. dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
  7964. }
  7965. /* print a string summarizing features */
  7966. i40e_print_features(pf);
  7967. return 0;
  7968. /* Unwind what we've done if something failed in the setup */
  7969. err_vsis:
  7970. set_bit(__I40E_DOWN, &pf->state);
  7971. i40e_clear_interrupt_scheme(pf);
  7972. kfree(pf->vsi);
  7973. err_switch_setup:
  7974. i40e_reset_interrupt_capability(pf);
  7975. del_timer_sync(&pf->service_timer);
  7976. err_mac_addr:
  7977. err_configure_lan_hmc:
  7978. (void)i40e_shutdown_lan_hmc(hw);
  7979. err_init_lan_hmc:
  7980. kfree(pf->qp_pile);
  7981. kfree(pf->irq_pile);
  7982. err_sw_init:
  7983. err_adminq_setup:
  7984. (void)i40e_shutdown_adminq(hw);
  7985. err_pf_reset:
  7986. iounmap(hw->hw_addr);
  7987. err_ioremap:
  7988. kfree(pf);
  7989. err_pf_alloc:
  7990. pci_disable_pcie_error_reporting(pdev);
  7991. pci_release_selected_regions(pdev,
  7992. pci_select_bars(pdev, IORESOURCE_MEM));
  7993. err_pci_reg:
  7994. err_dma:
  7995. pci_disable_device(pdev);
  7996. return err;
  7997. }
  7998. /**
  7999. * i40e_remove - Device removal routine
  8000. * @pdev: PCI device information struct
  8001. *
  8002. * i40e_remove is called by the PCI subsystem to alert the driver
  8003. * that is should release a PCI device. This could be caused by a
  8004. * Hot-Plug event, or because the driver is going to be removed from
  8005. * memory.
  8006. **/
  8007. static void i40e_remove(struct pci_dev *pdev)
  8008. {
  8009. struct i40e_pf *pf = pci_get_drvdata(pdev);
  8010. i40e_status ret_code;
  8011. int i;
  8012. i40e_dbg_pf_exit(pf);
  8013. i40e_ptp_stop(pf);
  8014. /* no more scheduling of any task */
  8015. set_bit(__I40E_DOWN, &pf->state);
  8016. del_timer_sync(&pf->service_timer);
  8017. cancel_work_sync(&pf->service_task);
  8018. if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
  8019. i40e_free_vfs(pf);
  8020. pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
  8021. }
  8022. i40e_fdir_teardown(pf);
  8023. /* If there is a switch structure or any orphans, remove them.
  8024. * This will leave only the PF's VSI remaining.
  8025. */
  8026. for (i = 0; i < I40E_MAX_VEB; i++) {
  8027. if (!pf->veb[i])
  8028. continue;
  8029. if (pf->veb[i]->uplink_seid == pf->mac_seid ||
  8030. pf->veb[i]->uplink_seid == 0)
  8031. i40e_switch_branch_release(pf->veb[i]);
  8032. }
  8033. /* Now we can shutdown the PF's VSI, just before we kill
  8034. * adminq and hmc.
  8035. */
  8036. if (pf->vsi[pf->lan_vsi])
  8037. i40e_vsi_release(pf->vsi[pf->lan_vsi]);
  8038. i40e_stop_misc_vector(pf);
  8039. if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
  8040. synchronize_irq(pf->msix_entries[0].vector);
  8041. free_irq(pf->msix_entries[0].vector, pf);
  8042. }
  8043. /* shutdown and destroy the HMC */
  8044. if (pf->hw.hmc.hmc_obj) {
  8045. ret_code = i40e_shutdown_lan_hmc(&pf->hw);
  8046. if (ret_code)
  8047. dev_warn(&pdev->dev,
  8048. "Failed to destroy the HMC resources: %d\n",
  8049. ret_code);
  8050. }
  8051. /* shutdown the adminq */
  8052. ret_code = i40e_shutdown_adminq(&pf->hw);
  8053. if (ret_code)
  8054. dev_warn(&pdev->dev,
  8055. "Failed to destroy the Admin Queue resources: %d\n",
  8056. ret_code);
  8057. /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
  8058. i40e_clear_interrupt_scheme(pf);
  8059. for (i = 0; i < pf->num_alloc_vsi; i++) {
  8060. if (pf->vsi[i]) {
  8061. i40e_vsi_clear_rings(pf->vsi[i]);
  8062. i40e_vsi_clear(pf->vsi[i]);
  8063. pf->vsi[i] = NULL;
  8064. }
  8065. }
  8066. for (i = 0; i < I40E_MAX_VEB; i++) {
  8067. kfree(pf->veb[i]);
  8068. pf->veb[i] = NULL;
  8069. }
  8070. kfree(pf->qp_pile);
  8071. kfree(pf->irq_pile);
  8072. kfree(pf->vsi);
  8073. iounmap(pf->hw.hw_addr);
  8074. kfree(pf);
  8075. pci_release_selected_regions(pdev,
  8076. pci_select_bars(pdev, IORESOURCE_MEM));
  8077. pci_disable_pcie_error_reporting(pdev);
  8078. pci_disable_device(pdev);
  8079. }
  8080. /**
  8081. * i40e_pci_error_detected - warning that something funky happened in PCI land
  8082. * @pdev: PCI device information struct
  8083. *
  8084. * Called to warn that something happened and the error handling steps
  8085. * are in progress. Allows the driver to quiesce things, be ready for
  8086. * remediation.
  8087. **/
  8088. static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
  8089. enum pci_channel_state error)
  8090. {
  8091. struct i40e_pf *pf = pci_get_drvdata(pdev);
  8092. dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
  8093. /* shutdown all operations */
  8094. if (!test_bit(__I40E_SUSPENDED, &pf->state)) {
  8095. rtnl_lock();
  8096. i40e_prep_for_reset(pf);
  8097. rtnl_unlock();
  8098. }
  8099. /* Request a slot reset */
  8100. return PCI_ERS_RESULT_NEED_RESET;
  8101. }
  8102. /**
  8103. * i40e_pci_error_slot_reset - a PCI slot reset just happened
  8104. * @pdev: PCI device information struct
  8105. *
  8106. * Called to find if the driver can work with the device now that
  8107. * the pci slot has been reset. If a basic connection seems good
  8108. * (registers are readable and have sane content) then return a
  8109. * happy little PCI_ERS_RESULT_xxx.
  8110. **/
  8111. static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
  8112. {
  8113. struct i40e_pf *pf = pci_get_drvdata(pdev);
  8114. pci_ers_result_t result;
  8115. int err;
  8116. u32 reg;
  8117. dev_info(&pdev->dev, "%s\n", __func__);
  8118. if (pci_enable_device_mem(pdev)) {
  8119. dev_info(&pdev->dev,
  8120. "Cannot re-enable PCI device after reset.\n");
  8121. result = PCI_ERS_RESULT_DISCONNECT;
  8122. } else {
  8123. pci_set_master(pdev);
  8124. pci_restore_state(pdev);
  8125. pci_save_state(pdev);
  8126. pci_wake_from_d3(pdev, false);
  8127. reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
  8128. if (reg == 0)
  8129. result = PCI_ERS_RESULT_RECOVERED;
  8130. else
  8131. result = PCI_ERS_RESULT_DISCONNECT;
  8132. }
  8133. err = pci_cleanup_aer_uncorrect_error_status(pdev);
  8134. if (err) {
  8135. dev_info(&pdev->dev,
  8136. "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
  8137. err);
  8138. /* non-fatal, continue */
  8139. }
  8140. return result;
  8141. }
  8142. /**
  8143. * i40e_pci_error_resume - restart operations after PCI error recovery
  8144. * @pdev: PCI device information struct
  8145. *
  8146. * Called to allow the driver to bring things back up after PCI error
  8147. * and/or reset recovery has finished.
  8148. **/
  8149. static void i40e_pci_error_resume(struct pci_dev *pdev)
  8150. {
  8151. struct i40e_pf *pf = pci_get_drvdata(pdev);
  8152. dev_info(&pdev->dev, "%s\n", __func__);
  8153. if (test_bit(__I40E_SUSPENDED, &pf->state))
  8154. return;
  8155. rtnl_lock();
  8156. i40e_handle_reset_warning(pf);
  8157. rtnl_lock();
  8158. }
  8159. /**
  8160. * i40e_shutdown - PCI callback for shutting down
  8161. * @pdev: PCI device information struct
  8162. **/
  8163. static void i40e_shutdown(struct pci_dev *pdev)
  8164. {
  8165. struct i40e_pf *pf = pci_get_drvdata(pdev);
  8166. struct i40e_hw *hw = &pf->hw;
  8167. set_bit(__I40E_SUSPENDED, &pf->state);
  8168. set_bit(__I40E_DOWN, &pf->state);
  8169. rtnl_lock();
  8170. i40e_prep_for_reset(pf);
  8171. rtnl_unlock();
  8172. wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
  8173. wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
  8174. if (system_state == SYSTEM_POWER_OFF) {
  8175. pci_wake_from_d3(pdev, pf->wol_en);
  8176. pci_set_power_state(pdev, PCI_D3hot);
  8177. }
  8178. }
  8179. #ifdef CONFIG_PM
  8180. /**
  8181. * i40e_suspend - PCI callback for moving to D3
  8182. * @pdev: PCI device information struct
  8183. **/
  8184. static int i40e_suspend(struct pci_dev *pdev, pm_message_t state)
  8185. {
  8186. struct i40e_pf *pf = pci_get_drvdata(pdev);
  8187. struct i40e_hw *hw = &pf->hw;
  8188. set_bit(__I40E_SUSPENDED, &pf->state);
  8189. set_bit(__I40E_DOWN, &pf->state);
  8190. rtnl_lock();
  8191. i40e_prep_for_reset(pf);
  8192. rtnl_unlock();
  8193. wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
  8194. wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
  8195. pci_wake_from_d3(pdev, pf->wol_en);
  8196. pci_set_power_state(pdev, PCI_D3hot);
  8197. return 0;
  8198. }
  8199. /**
  8200. * i40e_resume - PCI callback for waking up from D3
  8201. * @pdev: PCI device information struct
  8202. **/
  8203. static int i40e_resume(struct pci_dev *pdev)
  8204. {
  8205. struct i40e_pf *pf = pci_get_drvdata(pdev);
  8206. u32 err;
  8207. pci_set_power_state(pdev, PCI_D0);
  8208. pci_restore_state(pdev);
  8209. /* pci_restore_state() clears dev->state_saves, so
  8210. * call pci_save_state() again to restore it.
  8211. */
  8212. pci_save_state(pdev);
  8213. err = pci_enable_device_mem(pdev);
  8214. if (err) {
  8215. dev_err(&pdev->dev,
  8216. "%s: Cannot enable PCI device from suspend\n",
  8217. __func__);
  8218. return err;
  8219. }
  8220. pci_set_master(pdev);
  8221. /* no wakeup events while running */
  8222. pci_wake_from_d3(pdev, false);
  8223. /* handling the reset will rebuild the device state */
  8224. if (test_and_clear_bit(__I40E_SUSPENDED, &pf->state)) {
  8225. clear_bit(__I40E_DOWN, &pf->state);
  8226. rtnl_lock();
  8227. i40e_reset_and_rebuild(pf, false);
  8228. rtnl_unlock();
  8229. }
  8230. return 0;
  8231. }
  8232. #endif
  8233. static const struct pci_error_handlers i40e_err_handler = {
  8234. .error_detected = i40e_pci_error_detected,
  8235. .slot_reset = i40e_pci_error_slot_reset,
  8236. .resume = i40e_pci_error_resume,
  8237. };
  8238. static struct pci_driver i40e_driver = {
  8239. .name = i40e_driver_name,
  8240. .id_table = i40e_pci_tbl,
  8241. .probe = i40e_probe,
  8242. .remove = i40e_remove,
  8243. #ifdef CONFIG_PM
  8244. .suspend = i40e_suspend,
  8245. .resume = i40e_resume,
  8246. #endif
  8247. .shutdown = i40e_shutdown,
  8248. .err_handler = &i40e_err_handler,
  8249. .sriov_configure = i40e_pci_sriov_configure,
  8250. };
  8251. /**
  8252. * i40e_init_module - Driver registration routine
  8253. *
  8254. * i40e_init_module is the first routine called when the driver is
  8255. * loaded. All it does is register with the PCI subsystem.
  8256. **/
  8257. static int __init i40e_init_module(void)
  8258. {
  8259. pr_info("%s: %s - version %s\n", i40e_driver_name,
  8260. i40e_driver_string, i40e_driver_version_str);
  8261. pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
  8262. i40e_dbg_init();
  8263. return pci_register_driver(&i40e_driver);
  8264. }
  8265. module_init(i40e_init_module);
  8266. /**
  8267. * i40e_exit_module - Driver exit cleanup routine
  8268. *
  8269. * i40e_exit_module is called just before the driver is removed
  8270. * from memory.
  8271. **/
  8272. static void __exit i40e_exit_module(void)
  8273. {
  8274. pci_unregister_driver(&i40e_driver);
  8275. i40e_dbg_exit();
  8276. }
  8277. module_exit(i40e_exit_module);