ena_netdev.c 84 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299
  1. /*
  2. * Copyright 2015 Amazon.com, Inc. or its affiliates.
  3. *
  4. * This software is available to you under a choice of one of two
  5. * licenses. You may choose to be licensed under the terms of the GNU
  6. * General Public License (GPL) Version 2, available from the file
  7. * COPYING in the main directory of this source tree, or the
  8. * BSD license below:
  9. *
  10. * Redistribution and use in source and binary forms, with or
  11. * without modification, are permitted provided that the following
  12. * conditions are met:
  13. *
  14. * - Redistributions of source code must retain the above
  15. * copyright notice, this list of conditions and the following
  16. * disclaimer.
  17. *
  18. * - Redistributions in binary form must reproduce the above
  19. * copyright notice, this list of conditions and the following
  20. * disclaimer in the documentation and/or other materials
  21. * provided with the distribution.
  22. *
  23. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30. * SOFTWARE.
  31. */
  32. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  33. #ifdef CONFIG_RFS_ACCEL
  34. #include <linux/cpu_rmap.h>
  35. #endif /* CONFIG_RFS_ACCEL */
  36. #include <linux/ethtool.h>
  37. #include <linux/if_vlan.h>
  38. #include <linux/kernel.h>
  39. #include <linux/module.h>
  40. #include <linux/moduleparam.h>
  41. #include <linux/numa.h>
  42. #include <linux/pci.h>
  43. #include <linux/utsname.h>
  44. #include <linux/version.h>
  45. #include <linux/vmalloc.h>
  46. #include <net/ip.h>
  47. #include "ena_netdev.h"
  48. #include "ena_pci_id_tbl.h"
  49. static char version[] = DEVICE_NAME " v" DRV_MODULE_VERSION "\n";
  50. MODULE_AUTHOR("Amazon.com, Inc. or its affiliates");
  51. MODULE_DESCRIPTION(DEVICE_NAME);
  52. MODULE_LICENSE("GPL");
  53. MODULE_VERSION(DRV_MODULE_VERSION);
  54. /* Time in jiffies before concluding the transmitter is hung. */
  55. #define TX_TIMEOUT (5 * HZ)
  56. #define ENA_NAPI_BUDGET 64
  57. #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_IFUP | \
  58. NETIF_MSG_TX_DONE | NETIF_MSG_TX_ERR | NETIF_MSG_RX_ERR)
  59. static int debug = -1;
  60. module_param(debug, int, 0);
  61. MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  62. static struct ena_aenq_handlers aenq_handlers;
  63. static struct workqueue_struct *ena_wq;
  64. MODULE_DEVICE_TABLE(pci, ena_pci_tbl);
  65. static int ena_rss_init_default(struct ena_adapter *adapter);
  66. static void ena_tx_timeout(struct net_device *dev)
  67. {
  68. struct ena_adapter *adapter = netdev_priv(dev);
  69. /* Change the state of the device to trigger reset
  70. * Check that we are not in the middle or a trigger already
  71. */
  72. if (test_and_set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
  73. return;
  74. u64_stats_update_begin(&adapter->syncp);
  75. adapter->dev_stats.tx_timeout++;
  76. u64_stats_update_end(&adapter->syncp);
  77. netif_err(adapter, tx_err, dev, "Transmit time out\n");
  78. }
  79. static void update_rx_ring_mtu(struct ena_adapter *adapter, int mtu)
  80. {
  81. int i;
  82. for (i = 0; i < adapter->num_queues; i++)
  83. adapter->rx_ring[i].mtu = mtu;
  84. }
  85. static int ena_change_mtu(struct net_device *dev, int new_mtu)
  86. {
  87. struct ena_adapter *adapter = netdev_priv(dev);
  88. int ret;
  89. ret = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu);
  90. if (!ret) {
  91. netif_dbg(adapter, drv, dev, "set MTU to %d\n", new_mtu);
  92. update_rx_ring_mtu(adapter, new_mtu);
  93. dev->mtu = new_mtu;
  94. } else {
  95. netif_err(adapter, drv, dev, "Failed to set MTU to %d\n",
  96. new_mtu);
  97. }
  98. return ret;
  99. }
  100. static int ena_init_rx_cpu_rmap(struct ena_adapter *adapter)
  101. {
  102. #ifdef CONFIG_RFS_ACCEL
  103. u32 i;
  104. int rc;
  105. adapter->netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(adapter->num_queues);
  106. if (!adapter->netdev->rx_cpu_rmap)
  107. return -ENOMEM;
  108. for (i = 0; i < adapter->num_queues; i++) {
  109. int irq_idx = ENA_IO_IRQ_IDX(i);
  110. rc = irq_cpu_rmap_add(adapter->netdev->rx_cpu_rmap,
  111. pci_irq_vector(adapter->pdev, irq_idx));
  112. if (rc) {
  113. free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap);
  114. adapter->netdev->rx_cpu_rmap = NULL;
  115. return rc;
  116. }
  117. }
  118. #endif /* CONFIG_RFS_ACCEL */
  119. return 0;
  120. }
  121. static void ena_init_io_rings_common(struct ena_adapter *adapter,
  122. struct ena_ring *ring, u16 qid)
  123. {
  124. ring->qid = qid;
  125. ring->pdev = adapter->pdev;
  126. ring->dev = &adapter->pdev->dev;
  127. ring->netdev = adapter->netdev;
  128. ring->napi = &adapter->ena_napi[qid].napi;
  129. ring->adapter = adapter;
  130. ring->ena_dev = adapter->ena_dev;
  131. ring->per_napi_packets = 0;
  132. ring->per_napi_bytes = 0;
  133. ring->cpu = 0;
  134. u64_stats_init(&ring->syncp);
  135. }
  136. static void ena_init_io_rings(struct ena_adapter *adapter)
  137. {
  138. struct ena_com_dev *ena_dev;
  139. struct ena_ring *txr, *rxr;
  140. int i;
  141. ena_dev = adapter->ena_dev;
  142. for (i = 0; i < adapter->num_queues; i++) {
  143. txr = &adapter->tx_ring[i];
  144. rxr = &adapter->rx_ring[i];
  145. /* TX/RX common ring state */
  146. ena_init_io_rings_common(adapter, txr, i);
  147. ena_init_io_rings_common(adapter, rxr, i);
  148. /* TX specific ring state */
  149. txr->ring_size = adapter->tx_ring_size;
  150. txr->tx_max_header_size = ena_dev->tx_max_header_size;
  151. txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
  152. txr->sgl_size = adapter->max_tx_sgl_size;
  153. txr->smoothed_interval =
  154. ena_com_get_nonadaptive_moderation_interval_tx(ena_dev);
  155. /* RX specific ring state */
  156. rxr->ring_size = adapter->rx_ring_size;
  157. rxr->rx_copybreak = adapter->rx_copybreak;
  158. rxr->sgl_size = adapter->max_rx_sgl_size;
  159. rxr->smoothed_interval =
  160. ena_com_get_nonadaptive_moderation_interval_rx(ena_dev);
  161. }
  162. }
  163. /* ena_setup_tx_resources - allocate I/O Tx resources (Descriptors)
  164. * @adapter: network interface device structure
  165. * @qid: queue index
  166. *
  167. * Return 0 on success, negative on failure
  168. */
  169. static int ena_setup_tx_resources(struct ena_adapter *adapter, int qid)
  170. {
  171. struct ena_ring *tx_ring = &adapter->tx_ring[qid];
  172. struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];
  173. int size, i, node;
  174. if (tx_ring->tx_buffer_info) {
  175. netif_err(adapter, ifup,
  176. adapter->netdev, "tx_buffer_info info is not NULL");
  177. return -EEXIST;
  178. }
  179. size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
  180. node = cpu_to_node(ena_irq->cpu);
  181. tx_ring->tx_buffer_info = vzalloc_node(size, node);
  182. if (!tx_ring->tx_buffer_info) {
  183. tx_ring->tx_buffer_info = vzalloc(size);
  184. if (!tx_ring->tx_buffer_info)
  185. return -ENOMEM;
  186. }
  187. size = sizeof(u16) * tx_ring->ring_size;
  188. tx_ring->free_tx_ids = vzalloc_node(size, node);
  189. if (!tx_ring->free_tx_ids) {
  190. tx_ring->free_tx_ids = vzalloc(size);
  191. if (!tx_ring->free_tx_ids) {
  192. vfree(tx_ring->tx_buffer_info);
  193. return -ENOMEM;
  194. }
  195. }
  196. /* Req id ring for TX out of order completions */
  197. for (i = 0; i < tx_ring->ring_size; i++)
  198. tx_ring->free_tx_ids[i] = i;
  199. /* Reset tx statistics */
  200. memset(&tx_ring->tx_stats, 0x0, sizeof(tx_ring->tx_stats));
  201. tx_ring->next_to_use = 0;
  202. tx_ring->next_to_clean = 0;
  203. tx_ring->cpu = ena_irq->cpu;
  204. return 0;
  205. }
  206. /* ena_free_tx_resources - Free I/O Tx Resources per Queue
  207. * @adapter: network interface device structure
  208. * @qid: queue index
  209. *
  210. * Free all transmit software resources
  211. */
  212. static void ena_free_tx_resources(struct ena_adapter *adapter, int qid)
  213. {
  214. struct ena_ring *tx_ring = &adapter->tx_ring[qid];
  215. vfree(tx_ring->tx_buffer_info);
  216. tx_ring->tx_buffer_info = NULL;
  217. vfree(tx_ring->free_tx_ids);
  218. tx_ring->free_tx_ids = NULL;
  219. }
  220. /* ena_setup_all_tx_resources - allocate I/O Tx queues resources for All queues
  221. * @adapter: private structure
  222. *
  223. * Return 0 on success, negative on failure
  224. */
  225. static int ena_setup_all_tx_resources(struct ena_adapter *adapter)
  226. {
  227. int i, rc = 0;
  228. for (i = 0; i < adapter->num_queues; i++) {
  229. rc = ena_setup_tx_resources(adapter, i);
  230. if (rc)
  231. goto err_setup_tx;
  232. }
  233. return 0;
  234. err_setup_tx:
  235. netif_err(adapter, ifup, adapter->netdev,
  236. "Tx queue %d: allocation failed\n", i);
  237. /* rewind the index freeing the rings as we go */
  238. while (i--)
  239. ena_free_tx_resources(adapter, i);
  240. return rc;
  241. }
  242. /* ena_free_all_io_tx_resources - Free I/O Tx Resources for All Queues
  243. * @adapter: board private structure
  244. *
  245. * Free all transmit software resources
  246. */
  247. static void ena_free_all_io_tx_resources(struct ena_adapter *adapter)
  248. {
  249. int i;
  250. for (i = 0; i < adapter->num_queues; i++)
  251. ena_free_tx_resources(adapter, i);
  252. }
  253. /* ena_setup_rx_resources - allocate I/O Rx resources (Descriptors)
  254. * @adapter: network interface device structure
  255. * @qid: queue index
  256. *
  257. * Returns 0 on success, negative on failure
  258. */
  259. static int ena_setup_rx_resources(struct ena_adapter *adapter,
  260. u32 qid)
  261. {
  262. struct ena_ring *rx_ring = &adapter->rx_ring[qid];
  263. struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];
  264. int size, node;
  265. if (rx_ring->rx_buffer_info) {
  266. netif_err(adapter, ifup, adapter->netdev,
  267. "rx_buffer_info is not NULL");
  268. return -EEXIST;
  269. }
  270. /* alloc extra element so in rx path
  271. * we can always prefetch rx_info + 1
  272. */
  273. size = sizeof(struct ena_rx_buffer) * (rx_ring->ring_size + 1);
  274. node = cpu_to_node(ena_irq->cpu);
  275. rx_ring->rx_buffer_info = vzalloc_node(size, node);
  276. if (!rx_ring->rx_buffer_info) {
  277. rx_ring->rx_buffer_info = vzalloc(size);
  278. if (!rx_ring->rx_buffer_info)
  279. return -ENOMEM;
  280. }
  281. /* Reset rx statistics */
  282. memset(&rx_ring->rx_stats, 0x0, sizeof(rx_ring->rx_stats));
  283. rx_ring->next_to_clean = 0;
  284. rx_ring->next_to_use = 0;
  285. rx_ring->cpu = ena_irq->cpu;
  286. return 0;
  287. }
  288. /* ena_free_rx_resources - Free I/O Rx Resources
  289. * @adapter: network interface device structure
  290. * @qid: queue index
  291. *
  292. * Free all receive software resources
  293. */
  294. static void ena_free_rx_resources(struct ena_adapter *adapter,
  295. u32 qid)
  296. {
  297. struct ena_ring *rx_ring = &adapter->rx_ring[qid];
  298. vfree(rx_ring->rx_buffer_info);
  299. rx_ring->rx_buffer_info = NULL;
  300. }
  301. /* ena_setup_all_rx_resources - allocate I/O Rx queues resources for all queues
  302. * @adapter: board private structure
  303. *
  304. * Return 0 on success, negative on failure
  305. */
  306. static int ena_setup_all_rx_resources(struct ena_adapter *adapter)
  307. {
  308. int i, rc = 0;
  309. for (i = 0; i < adapter->num_queues; i++) {
  310. rc = ena_setup_rx_resources(adapter, i);
  311. if (rc)
  312. goto err_setup_rx;
  313. }
  314. return 0;
  315. err_setup_rx:
  316. netif_err(adapter, ifup, adapter->netdev,
  317. "Rx queue %d: allocation failed\n", i);
  318. /* rewind the index freeing the rings as we go */
  319. while (i--)
  320. ena_free_rx_resources(adapter, i);
  321. return rc;
  322. }
  323. /* ena_free_all_io_rx_resources - Free I/O Rx Resources for All Queues
  324. * @adapter: board private structure
  325. *
  326. * Free all receive software resources
  327. */
  328. static void ena_free_all_io_rx_resources(struct ena_adapter *adapter)
  329. {
  330. int i;
  331. for (i = 0; i < adapter->num_queues; i++)
  332. ena_free_rx_resources(adapter, i);
  333. }
  334. static inline int ena_alloc_rx_page(struct ena_ring *rx_ring,
  335. struct ena_rx_buffer *rx_info, gfp_t gfp)
  336. {
  337. struct ena_com_buf *ena_buf;
  338. struct page *page;
  339. dma_addr_t dma;
  340. /* if previous allocated page is not used */
  341. if (unlikely(rx_info->page))
  342. return 0;
  343. page = alloc_page(gfp);
  344. if (unlikely(!page)) {
  345. u64_stats_update_begin(&rx_ring->syncp);
  346. rx_ring->rx_stats.page_alloc_fail++;
  347. u64_stats_update_end(&rx_ring->syncp);
  348. return -ENOMEM;
  349. }
  350. dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE,
  351. DMA_FROM_DEVICE);
  352. if (unlikely(dma_mapping_error(rx_ring->dev, dma))) {
  353. u64_stats_update_begin(&rx_ring->syncp);
  354. rx_ring->rx_stats.dma_mapping_err++;
  355. u64_stats_update_end(&rx_ring->syncp);
  356. __free_page(page);
  357. return -EIO;
  358. }
  359. netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
  360. "alloc page %p, rx_info %p\n", page, rx_info);
  361. rx_info->page = page;
  362. rx_info->page_offset = 0;
  363. ena_buf = &rx_info->ena_buf;
  364. ena_buf->paddr = dma;
  365. ena_buf->len = PAGE_SIZE;
  366. return 0;
  367. }
  368. static void ena_free_rx_page(struct ena_ring *rx_ring,
  369. struct ena_rx_buffer *rx_info)
  370. {
  371. struct page *page = rx_info->page;
  372. struct ena_com_buf *ena_buf = &rx_info->ena_buf;
  373. if (unlikely(!page)) {
  374. netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
  375. "Trying to free unallocated buffer\n");
  376. return;
  377. }
  378. dma_unmap_page(rx_ring->dev, ena_buf->paddr, PAGE_SIZE,
  379. DMA_FROM_DEVICE);
  380. __free_page(page);
  381. rx_info->page = NULL;
  382. }
  383. static int ena_refill_rx_bufs(struct ena_ring *rx_ring, u32 num)
  384. {
  385. u16 next_to_use;
  386. u32 i;
  387. int rc;
  388. next_to_use = rx_ring->next_to_use;
  389. for (i = 0; i < num; i++) {
  390. struct ena_rx_buffer *rx_info =
  391. &rx_ring->rx_buffer_info[next_to_use];
  392. rc = ena_alloc_rx_page(rx_ring, rx_info,
  393. __GFP_COLD | GFP_ATOMIC | __GFP_COMP);
  394. if (unlikely(rc < 0)) {
  395. netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
  396. "failed to alloc buffer for rx queue %d\n",
  397. rx_ring->qid);
  398. break;
  399. }
  400. rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,
  401. &rx_info->ena_buf,
  402. next_to_use);
  403. if (unlikely(rc)) {
  404. netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
  405. "failed to add buffer for rx queue %d\n",
  406. rx_ring->qid);
  407. break;
  408. }
  409. next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use,
  410. rx_ring->ring_size);
  411. }
  412. if (unlikely(i < num)) {
  413. u64_stats_update_begin(&rx_ring->syncp);
  414. rx_ring->rx_stats.refil_partial++;
  415. u64_stats_update_end(&rx_ring->syncp);
  416. netdev_warn(rx_ring->netdev,
  417. "refilled rx qid %d with only %d buffers (from %d)\n",
  418. rx_ring->qid, i, num);
  419. }
  420. if (likely(i)) {
  421. /* Add memory barrier to make sure the desc were written before
  422. * issue a doorbell
  423. */
  424. wmb();
  425. ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq);
  426. }
  427. rx_ring->next_to_use = next_to_use;
  428. return i;
  429. }
  430. static void ena_free_rx_bufs(struct ena_adapter *adapter,
  431. u32 qid)
  432. {
  433. struct ena_ring *rx_ring = &adapter->rx_ring[qid];
  434. u32 i;
  435. for (i = 0; i < rx_ring->ring_size; i++) {
  436. struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i];
  437. if (rx_info->page)
  438. ena_free_rx_page(rx_ring, rx_info);
  439. }
  440. }
  441. /* ena_refill_all_rx_bufs - allocate all queues Rx buffers
  442. * @adapter: board private structure
  443. *
  444. */
  445. static void ena_refill_all_rx_bufs(struct ena_adapter *adapter)
  446. {
  447. struct ena_ring *rx_ring;
  448. int i, rc, bufs_num;
  449. for (i = 0; i < adapter->num_queues; i++) {
  450. rx_ring = &adapter->rx_ring[i];
  451. bufs_num = rx_ring->ring_size - 1;
  452. rc = ena_refill_rx_bufs(rx_ring, bufs_num);
  453. if (unlikely(rc != bufs_num))
  454. netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
  455. "refilling Queue %d failed. allocated %d buffers from: %d\n",
  456. i, rc, bufs_num);
  457. }
  458. }
  459. static void ena_free_all_rx_bufs(struct ena_adapter *adapter)
  460. {
  461. int i;
  462. for (i = 0; i < adapter->num_queues; i++)
  463. ena_free_rx_bufs(adapter, i);
  464. }
  465. /* ena_free_tx_bufs - Free Tx Buffers per Queue
  466. * @tx_ring: TX ring for which buffers be freed
  467. */
  468. static void ena_free_tx_bufs(struct ena_ring *tx_ring)
  469. {
  470. bool print_once = true;
  471. u32 i;
  472. for (i = 0; i < tx_ring->ring_size; i++) {
  473. struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];
  474. struct ena_com_buf *ena_buf;
  475. int nr_frags;
  476. int j;
  477. if (!tx_info->skb)
  478. continue;
  479. if (print_once) {
  480. netdev_notice(tx_ring->netdev,
  481. "free uncompleted tx skb qid %d idx 0x%x\n",
  482. tx_ring->qid, i);
  483. print_once = false;
  484. } else {
  485. netdev_dbg(tx_ring->netdev,
  486. "free uncompleted tx skb qid %d idx 0x%x\n",
  487. tx_ring->qid, i);
  488. }
  489. ena_buf = tx_info->bufs;
  490. dma_unmap_single(tx_ring->dev,
  491. ena_buf->paddr,
  492. ena_buf->len,
  493. DMA_TO_DEVICE);
  494. /* unmap remaining mapped pages */
  495. nr_frags = tx_info->num_of_bufs - 1;
  496. for (j = 0; j < nr_frags; j++) {
  497. ena_buf++;
  498. dma_unmap_page(tx_ring->dev,
  499. ena_buf->paddr,
  500. ena_buf->len,
  501. DMA_TO_DEVICE);
  502. }
  503. dev_kfree_skb_any(tx_info->skb);
  504. }
  505. netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
  506. tx_ring->qid));
  507. }
  508. static void ena_free_all_tx_bufs(struct ena_adapter *adapter)
  509. {
  510. struct ena_ring *tx_ring;
  511. int i;
  512. for (i = 0; i < adapter->num_queues; i++) {
  513. tx_ring = &adapter->tx_ring[i];
  514. ena_free_tx_bufs(tx_ring);
  515. }
  516. }
  517. static void ena_destroy_all_tx_queues(struct ena_adapter *adapter)
  518. {
  519. u16 ena_qid;
  520. int i;
  521. for (i = 0; i < adapter->num_queues; i++) {
  522. ena_qid = ENA_IO_TXQ_IDX(i);
  523. ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
  524. }
  525. }
  526. static void ena_destroy_all_rx_queues(struct ena_adapter *adapter)
  527. {
  528. u16 ena_qid;
  529. int i;
  530. for (i = 0; i < adapter->num_queues; i++) {
  531. ena_qid = ENA_IO_RXQ_IDX(i);
  532. ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
  533. }
  534. }
  535. static void ena_destroy_all_io_queues(struct ena_adapter *adapter)
  536. {
  537. ena_destroy_all_tx_queues(adapter);
  538. ena_destroy_all_rx_queues(adapter);
  539. }
  540. static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)
  541. {
  542. struct ena_tx_buffer *tx_info = NULL;
  543. if (likely(req_id < tx_ring->ring_size)) {
  544. tx_info = &tx_ring->tx_buffer_info[req_id];
  545. if (likely(tx_info->skb))
  546. return 0;
  547. }
  548. if (tx_info)
  549. netif_err(tx_ring->adapter, tx_done, tx_ring->netdev,
  550. "tx_info doesn't have valid skb\n");
  551. else
  552. netif_err(tx_ring->adapter, tx_done, tx_ring->netdev,
  553. "Invalid req_id: %hu\n", req_id);
  554. u64_stats_update_begin(&tx_ring->syncp);
  555. tx_ring->tx_stats.bad_req_id++;
  556. u64_stats_update_end(&tx_ring->syncp);
  557. /* Trigger device reset */
  558. set_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags);
  559. return -EFAULT;
  560. }
  561. static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget)
  562. {
  563. struct netdev_queue *txq;
  564. bool above_thresh;
  565. u32 tx_bytes = 0;
  566. u32 total_done = 0;
  567. u16 next_to_clean;
  568. u16 req_id;
  569. int tx_pkts = 0;
  570. int rc;
  571. next_to_clean = tx_ring->next_to_clean;
  572. txq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->qid);
  573. while (tx_pkts < budget) {
  574. struct ena_tx_buffer *tx_info;
  575. struct sk_buff *skb;
  576. struct ena_com_buf *ena_buf;
  577. int i, nr_frags;
  578. rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,
  579. &req_id);
  580. if (rc)
  581. break;
  582. rc = validate_tx_req_id(tx_ring, req_id);
  583. if (rc)
  584. break;
  585. tx_info = &tx_ring->tx_buffer_info[req_id];
  586. skb = tx_info->skb;
  587. /* prefetch skb_end_pointer() to speedup skb_shinfo(skb) */
  588. prefetch(&skb->end);
  589. tx_info->skb = NULL;
  590. tx_info->last_jiffies = 0;
  591. if (likely(tx_info->num_of_bufs != 0)) {
  592. ena_buf = tx_info->bufs;
  593. dma_unmap_single(tx_ring->dev,
  594. dma_unmap_addr(ena_buf, paddr),
  595. dma_unmap_len(ena_buf, len),
  596. DMA_TO_DEVICE);
  597. /* unmap remaining mapped pages */
  598. nr_frags = tx_info->num_of_bufs - 1;
  599. for (i = 0; i < nr_frags; i++) {
  600. ena_buf++;
  601. dma_unmap_page(tx_ring->dev,
  602. dma_unmap_addr(ena_buf, paddr),
  603. dma_unmap_len(ena_buf, len),
  604. DMA_TO_DEVICE);
  605. }
  606. }
  607. netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
  608. "tx_poll: q %d skb %p completed\n", tx_ring->qid,
  609. skb);
  610. tx_bytes += skb->len;
  611. dev_kfree_skb(skb);
  612. tx_pkts++;
  613. total_done += tx_info->tx_descs;
  614. tx_ring->free_tx_ids[next_to_clean] = req_id;
  615. next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
  616. tx_ring->ring_size);
  617. }
  618. tx_ring->next_to_clean = next_to_clean;
  619. ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done);
  620. ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq);
  621. netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
  622. netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
  623. "tx_poll: q %d done. total pkts: %d\n",
  624. tx_ring->qid, tx_pkts);
  625. /* need to make the rings circular update visible to
  626. * ena_start_xmit() before checking for netif_queue_stopped().
  627. */
  628. smp_mb();
  629. above_thresh = ena_com_sq_empty_space(tx_ring->ena_com_io_sq) >
  630. ENA_TX_WAKEUP_THRESH;
  631. if (unlikely(netif_tx_queue_stopped(txq) && above_thresh)) {
  632. __netif_tx_lock(txq, smp_processor_id());
  633. above_thresh = ena_com_sq_empty_space(tx_ring->ena_com_io_sq) >
  634. ENA_TX_WAKEUP_THRESH;
  635. if (netif_tx_queue_stopped(txq) && above_thresh) {
  636. netif_tx_wake_queue(txq);
  637. u64_stats_update_begin(&tx_ring->syncp);
  638. tx_ring->tx_stats.queue_wakeup++;
  639. u64_stats_update_end(&tx_ring->syncp);
  640. }
  641. __netif_tx_unlock(txq);
  642. }
  643. tx_ring->per_napi_bytes += tx_bytes;
  644. tx_ring->per_napi_packets += tx_pkts;
  645. return tx_pkts;
  646. }
  647. static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring,
  648. struct ena_com_rx_buf_info *ena_bufs,
  649. u32 descs,
  650. u16 *next_to_clean)
  651. {
  652. struct sk_buff *skb;
  653. struct ena_rx_buffer *rx_info =
  654. &rx_ring->rx_buffer_info[*next_to_clean];
  655. u32 len;
  656. u32 buf = 0;
  657. void *va;
  658. len = ena_bufs[0].len;
  659. if (unlikely(!rx_info->page)) {
  660. netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
  661. "Page is NULL\n");
  662. return NULL;
  663. }
  664. netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
  665. "rx_info %p page %p\n",
  666. rx_info, rx_info->page);
  667. /* save virt address of first buffer */
  668. va = page_address(rx_info->page) + rx_info->page_offset;
  669. prefetch(va + NET_IP_ALIGN);
  670. if (len <= rx_ring->rx_copybreak) {
  671. skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
  672. rx_ring->rx_copybreak);
  673. if (unlikely(!skb)) {
  674. u64_stats_update_begin(&rx_ring->syncp);
  675. rx_ring->rx_stats.skb_alloc_fail++;
  676. u64_stats_update_end(&rx_ring->syncp);
  677. netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
  678. "Failed to allocate skb\n");
  679. return NULL;
  680. }
  681. netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
  682. "rx allocated small packet. len %d. data_len %d\n",
  683. skb->len, skb->data_len);
  684. /* sync this buffer for CPU use */
  685. dma_sync_single_for_cpu(rx_ring->dev,
  686. dma_unmap_addr(&rx_info->ena_buf, paddr),
  687. len,
  688. DMA_FROM_DEVICE);
  689. skb_copy_to_linear_data(skb, va, len);
  690. dma_sync_single_for_device(rx_ring->dev,
  691. dma_unmap_addr(&rx_info->ena_buf, paddr),
  692. len,
  693. DMA_FROM_DEVICE);
  694. skb_put(skb, len);
  695. skb->protocol = eth_type_trans(skb, rx_ring->netdev);
  696. *next_to_clean = ENA_RX_RING_IDX_ADD(*next_to_clean, descs,
  697. rx_ring->ring_size);
  698. return skb;
  699. }
  700. skb = napi_get_frags(rx_ring->napi);
  701. if (unlikely(!skb)) {
  702. netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
  703. "Failed allocating skb\n");
  704. u64_stats_update_begin(&rx_ring->syncp);
  705. rx_ring->rx_stats.skb_alloc_fail++;
  706. u64_stats_update_end(&rx_ring->syncp);
  707. return NULL;
  708. }
  709. do {
  710. dma_unmap_page(rx_ring->dev,
  711. dma_unmap_addr(&rx_info->ena_buf, paddr),
  712. PAGE_SIZE, DMA_FROM_DEVICE);
  713. skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_info->page,
  714. rx_info->page_offset, len, PAGE_SIZE);
  715. netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
  716. "rx skb updated. len %d. data_len %d\n",
  717. skb->len, skb->data_len);
  718. rx_info->page = NULL;
  719. *next_to_clean =
  720. ENA_RX_RING_IDX_NEXT(*next_to_clean,
  721. rx_ring->ring_size);
  722. if (likely(--descs == 0))
  723. break;
  724. rx_info = &rx_ring->rx_buffer_info[*next_to_clean];
  725. len = ena_bufs[++buf].len;
  726. } while (1);
  727. return skb;
  728. }
  729. /* ena_rx_checksum - indicate in skb if hw indicated a good cksum
  730. * @adapter: structure containing adapter specific data
  731. * @ena_rx_ctx: received packet context/metadata
  732. * @skb: skb currently being received and modified
  733. */
  734. static inline void ena_rx_checksum(struct ena_ring *rx_ring,
  735. struct ena_com_rx_ctx *ena_rx_ctx,
  736. struct sk_buff *skb)
  737. {
  738. /* Rx csum disabled */
  739. if (unlikely(!(rx_ring->netdev->features & NETIF_F_RXCSUM))) {
  740. skb->ip_summed = CHECKSUM_NONE;
  741. return;
  742. }
  743. /* For fragmented packets the checksum isn't valid */
  744. if (ena_rx_ctx->frag) {
  745. skb->ip_summed = CHECKSUM_NONE;
  746. return;
  747. }
  748. /* if IP and error */
  749. if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) &&
  750. (ena_rx_ctx->l3_csum_err))) {
  751. /* ipv4 checksum error */
  752. skb->ip_summed = CHECKSUM_NONE;
  753. u64_stats_update_begin(&rx_ring->syncp);
  754. rx_ring->rx_stats.bad_csum++;
  755. u64_stats_update_end(&rx_ring->syncp);
  756. netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
  757. "RX IPv4 header checksum error\n");
  758. return;
  759. }
  760. /* if TCP/UDP */
  761. if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
  762. (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP))) {
  763. if (unlikely(ena_rx_ctx->l4_csum_err)) {
  764. /* TCP/UDP checksum error */
  765. u64_stats_update_begin(&rx_ring->syncp);
  766. rx_ring->rx_stats.bad_csum++;
  767. u64_stats_update_end(&rx_ring->syncp);
  768. netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
  769. "RX L4 checksum error\n");
  770. skb->ip_summed = CHECKSUM_NONE;
  771. return;
  772. }
  773. skb->ip_summed = CHECKSUM_UNNECESSARY;
  774. }
  775. }
  776. static void ena_set_rx_hash(struct ena_ring *rx_ring,
  777. struct ena_com_rx_ctx *ena_rx_ctx,
  778. struct sk_buff *skb)
  779. {
  780. enum pkt_hash_types hash_type;
  781. if (likely(rx_ring->netdev->features & NETIF_F_RXHASH)) {
  782. if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
  783. (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)))
  784. hash_type = PKT_HASH_TYPE_L4;
  785. else
  786. hash_type = PKT_HASH_TYPE_NONE;
  787. /* Override hash type if the packet is fragmented */
  788. if (ena_rx_ctx->frag)
  789. hash_type = PKT_HASH_TYPE_NONE;
  790. skb_set_hash(skb, ena_rx_ctx->hash, hash_type);
  791. }
  792. }
  793. /* ena_clean_rx_irq - Cleanup RX irq
  794. * @rx_ring: RX ring to clean
  795. * @napi: napi handler
  796. * @budget: how many packets driver is allowed to clean
  797. *
  798. * Returns the number of cleaned buffers.
  799. */
  800. static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi,
  801. u32 budget)
  802. {
  803. u16 next_to_clean = rx_ring->next_to_clean;
  804. u32 res_budget, work_done;
  805. struct ena_com_rx_ctx ena_rx_ctx;
  806. struct ena_adapter *adapter;
  807. struct sk_buff *skb;
  808. int refill_required;
  809. int refill_threshold;
  810. int rc = 0;
  811. int total_len = 0;
  812. int rx_copybreak_pkt = 0;
  813. netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
  814. "%s qid %d\n", __func__, rx_ring->qid);
  815. res_budget = budget;
  816. do {
  817. ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
  818. ena_rx_ctx.max_bufs = rx_ring->sgl_size;
  819. ena_rx_ctx.descs = 0;
  820. rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq,
  821. rx_ring->ena_com_io_sq,
  822. &ena_rx_ctx);
  823. if (unlikely(rc))
  824. goto error;
  825. if (unlikely(ena_rx_ctx.descs == 0))
  826. break;
  827. netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
  828. "rx_poll: q %d got packet from ena. descs #: %d l3 proto %d l4 proto %d hash: %x\n",
  829. rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,
  830. ena_rx_ctx.l4_proto, ena_rx_ctx.hash);
  831. /* allocate skb and fill it */
  832. skb = ena_rx_skb(rx_ring, rx_ring->ena_bufs, ena_rx_ctx.descs,
  833. &next_to_clean);
  834. /* exit if we failed to retrieve a buffer */
  835. if (unlikely(!skb)) {
  836. next_to_clean = ENA_RX_RING_IDX_ADD(next_to_clean,
  837. ena_rx_ctx.descs,
  838. rx_ring->ring_size);
  839. break;
  840. }
  841. ena_rx_checksum(rx_ring, &ena_rx_ctx, skb);
  842. ena_set_rx_hash(rx_ring, &ena_rx_ctx, skb);
  843. skb_record_rx_queue(skb, rx_ring->qid);
  844. if (rx_ring->ena_bufs[0].len <= rx_ring->rx_copybreak) {
  845. total_len += rx_ring->ena_bufs[0].len;
  846. rx_copybreak_pkt++;
  847. napi_gro_receive(napi, skb);
  848. } else {
  849. total_len += skb->len;
  850. napi_gro_frags(napi);
  851. }
  852. res_budget--;
  853. } while (likely(res_budget));
  854. work_done = budget - res_budget;
  855. rx_ring->per_napi_bytes += total_len;
  856. rx_ring->per_napi_packets += work_done;
  857. u64_stats_update_begin(&rx_ring->syncp);
  858. rx_ring->rx_stats.bytes += total_len;
  859. rx_ring->rx_stats.cnt += work_done;
  860. rx_ring->rx_stats.rx_copybreak_pkt += rx_copybreak_pkt;
  861. u64_stats_update_end(&rx_ring->syncp);
  862. rx_ring->next_to_clean = next_to_clean;
  863. refill_required = ena_com_sq_empty_space(rx_ring->ena_com_io_sq);
  864. refill_threshold = rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER;
  865. /* Optimization, try to batch new rx buffers */
  866. if (refill_required > refill_threshold) {
  867. ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq);
  868. ena_refill_rx_bufs(rx_ring, refill_required);
  869. }
  870. return work_done;
  871. error:
  872. adapter = netdev_priv(rx_ring->netdev);
  873. u64_stats_update_begin(&rx_ring->syncp);
  874. rx_ring->rx_stats.bad_desc_num++;
  875. u64_stats_update_end(&rx_ring->syncp);
  876. /* Too many desc from the device. Trigger reset */
  877. set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
  878. return 0;
  879. }
  880. inline void ena_adjust_intr_moderation(struct ena_ring *rx_ring,
  881. struct ena_ring *tx_ring)
  882. {
  883. /* We apply adaptive moderation on Rx path only.
  884. * Tx uses static interrupt moderation.
  885. */
  886. ena_com_calculate_interrupt_delay(rx_ring->ena_dev,
  887. rx_ring->per_napi_packets,
  888. rx_ring->per_napi_bytes,
  889. &rx_ring->smoothed_interval,
  890. &rx_ring->moder_tbl_idx);
  891. /* Reset per napi packets/bytes */
  892. tx_ring->per_napi_packets = 0;
  893. tx_ring->per_napi_bytes = 0;
  894. rx_ring->per_napi_packets = 0;
  895. rx_ring->per_napi_bytes = 0;
  896. }
  897. static inline void ena_update_ring_numa_node(struct ena_ring *tx_ring,
  898. struct ena_ring *rx_ring)
  899. {
  900. int cpu = get_cpu();
  901. int numa_node;
  902. /* Check only one ring since the 2 rings are running on the same cpu */
  903. if (likely(tx_ring->cpu == cpu))
  904. goto out;
  905. numa_node = cpu_to_node(cpu);
  906. put_cpu();
  907. if (numa_node != NUMA_NO_NODE) {
  908. ena_com_update_numa_node(tx_ring->ena_com_io_cq, numa_node);
  909. ena_com_update_numa_node(rx_ring->ena_com_io_cq, numa_node);
  910. }
  911. tx_ring->cpu = cpu;
  912. rx_ring->cpu = cpu;
  913. return;
  914. out:
  915. put_cpu();
  916. }
  917. static int ena_io_poll(struct napi_struct *napi, int budget)
  918. {
  919. struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);
  920. struct ena_ring *tx_ring, *rx_ring;
  921. struct ena_eth_io_intr_reg intr_reg;
  922. u32 tx_work_done;
  923. u32 rx_work_done;
  924. int tx_budget;
  925. int napi_comp_call = 0;
  926. int ret;
  927. tx_ring = ena_napi->tx_ring;
  928. rx_ring = ena_napi->rx_ring;
  929. tx_budget = tx_ring->ring_size / ENA_TX_POLL_BUDGET_DIVIDER;
  930. if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
  931. test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) {
  932. napi_complete_done(napi, 0);
  933. return 0;
  934. }
  935. tx_work_done = ena_clean_tx_irq(tx_ring, tx_budget);
  936. rx_work_done = ena_clean_rx_irq(rx_ring, napi, budget);
  937. /* If the device is about to reset or down, avoid unmask
  938. * the interrupt and return 0 so NAPI won't reschedule
  939. */
  940. if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
  941. test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags))) {
  942. napi_complete_done(napi, 0);
  943. ret = 0;
  944. } else if ((budget > rx_work_done) && (tx_budget > tx_work_done)) {
  945. napi_comp_call = 1;
  946. /* Update numa and unmask the interrupt only when schedule
  947. * from the interrupt context (vs from sk_busy_loop)
  948. */
  949. if (napi_complete_done(napi, rx_work_done)) {
  950. /* Tx and Rx share the same interrupt vector */
  951. if (ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev))
  952. ena_adjust_intr_moderation(rx_ring, tx_ring);
  953. /* Update intr register: rx intr delay,
  954. * tx intr delay and interrupt unmask
  955. */
  956. ena_com_update_intr_reg(&intr_reg,
  957. rx_ring->smoothed_interval,
  958. tx_ring->smoothed_interval,
  959. true);
  960. /* It is a shared MSI-X.
  961. * Tx and Rx CQ have pointer to it.
  962. * So we use one of them to reach the intr reg
  963. */
  964. ena_com_unmask_intr(rx_ring->ena_com_io_cq, &intr_reg);
  965. }
  966. ena_update_ring_numa_node(tx_ring, rx_ring);
  967. ret = rx_work_done;
  968. } else {
  969. ret = budget;
  970. }
  971. u64_stats_update_begin(&tx_ring->syncp);
  972. tx_ring->tx_stats.napi_comp += napi_comp_call;
  973. tx_ring->tx_stats.tx_poll++;
  974. u64_stats_update_end(&tx_ring->syncp);
  975. return ret;
  976. }
  977. static irqreturn_t ena_intr_msix_mgmnt(int irq, void *data)
  978. {
  979. struct ena_adapter *adapter = (struct ena_adapter *)data;
  980. ena_com_admin_q_comp_intr_handler(adapter->ena_dev);
  981. /* Don't call the aenq handler before probe is done */
  982. if (likely(test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags)))
  983. ena_com_aenq_intr_handler(adapter->ena_dev, data);
  984. return IRQ_HANDLED;
  985. }
  986. /* ena_intr_msix_io - MSI-X Interrupt Handler for Tx/Rx
  987. * @irq: interrupt number
  988. * @data: pointer to a network interface private napi device structure
  989. */
  990. static irqreturn_t ena_intr_msix_io(int irq, void *data)
  991. {
  992. struct ena_napi *ena_napi = data;
  993. napi_schedule(&ena_napi->napi);
  994. return IRQ_HANDLED;
  995. }
  996. static int ena_enable_msix(struct ena_adapter *adapter, int num_queues)
  997. {
  998. int msix_vecs, rc;
  999. /* Reserved the max msix vectors we might need */
  1000. msix_vecs = ENA_MAX_MSIX_VEC(num_queues);
  1001. netif_dbg(adapter, probe, adapter->netdev,
  1002. "trying to enable MSI-X, vectors %d\n", msix_vecs);
  1003. rc = pci_alloc_irq_vectors(adapter->pdev, msix_vecs, msix_vecs,
  1004. PCI_IRQ_MSIX);
  1005. if (rc < 0) {
  1006. netif_err(adapter, probe, adapter->netdev,
  1007. "Failed to enable MSI-X, vectors %d rc %d\n",
  1008. msix_vecs, rc);
  1009. return -ENOSPC;
  1010. }
  1011. netif_dbg(adapter, probe, adapter->netdev, "enable MSI-X, vectors %d\n",
  1012. msix_vecs);
  1013. if (msix_vecs >= 1) {
  1014. if (ena_init_rx_cpu_rmap(adapter))
  1015. netif_warn(adapter, probe, adapter->netdev,
  1016. "Failed to map IRQs to CPUs\n");
  1017. }
  1018. adapter->msix_vecs = msix_vecs;
  1019. return 0;
  1020. }
  1021. static void ena_setup_mgmnt_intr(struct ena_adapter *adapter)
  1022. {
  1023. u32 cpu;
  1024. snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name,
  1025. ENA_IRQNAME_SIZE, "ena-mgmnt@pci:%s",
  1026. pci_name(adapter->pdev));
  1027. adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler =
  1028. ena_intr_msix_mgmnt;
  1029. adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter;
  1030. adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector =
  1031. pci_irq_vector(adapter->pdev, ENA_MGMNT_IRQ_IDX);
  1032. cpu = cpumask_first(cpu_online_mask);
  1033. adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].cpu = cpu;
  1034. cpumask_set_cpu(cpu,
  1035. &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].affinity_hint_mask);
  1036. }
  1037. static void ena_setup_io_intr(struct ena_adapter *adapter)
  1038. {
  1039. struct net_device *netdev;
  1040. int irq_idx, i, cpu;
  1041. netdev = adapter->netdev;
  1042. for (i = 0; i < adapter->num_queues; i++) {
  1043. irq_idx = ENA_IO_IRQ_IDX(i);
  1044. cpu = i % num_online_cpus();
  1045. snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE,
  1046. "%s-Tx-Rx-%d", netdev->name, i);
  1047. adapter->irq_tbl[irq_idx].handler = ena_intr_msix_io;
  1048. adapter->irq_tbl[irq_idx].data = &adapter->ena_napi[i];
  1049. adapter->irq_tbl[irq_idx].vector =
  1050. pci_irq_vector(adapter->pdev, irq_idx);
  1051. adapter->irq_tbl[irq_idx].cpu = cpu;
  1052. cpumask_set_cpu(cpu,
  1053. &adapter->irq_tbl[irq_idx].affinity_hint_mask);
  1054. }
  1055. }
  1056. static int ena_request_mgmnt_irq(struct ena_adapter *adapter)
  1057. {
  1058. unsigned long flags = 0;
  1059. struct ena_irq *irq;
  1060. int rc;
  1061. irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
  1062. rc = request_irq(irq->vector, irq->handler, flags, irq->name,
  1063. irq->data);
  1064. if (rc) {
  1065. netif_err(adapter, probe, adapter->netdev,
  1066. "failed to request admin irq\n");
  1067. return rc;
  1068. }
  1069. netif_dbg(adapter, probe, adapter->netdev,
  1070. "set affinity hint of mgmnt irq.to 0x%lx (irq vector: %d)\n",
  1071. irq->affinity_hint_mask.bits[0], irq->vector);
  1072. irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
  1073. return rc;
  1074. }
  1075. static int ena_request_io_irq(struct ena_adapter *adapter)
  1076. {
  1077. unsigned long flags = 0;
  1078. struct ena_irq *irq;
  1079. int rc = 0, i, k;
  1080. for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) {
  1081. irq = &adapter->irq_tbl[i];
  1082. rc = request_irq(irq->vector, irq->handler, flags, irq->name,
  1083. irq->data);
  1084. if (rc) {
  1085. netif_err(adapter, ifup, adapter->netdev,
  1086. "Failed to request I/O IRQ. index %d rc %d\n",
  1087. i, rc);
  1088. goto err;
  1089. }
  1090. netif_dbg(adapter, ifup, adapter->netdev,
  1091. "set affinity hint of irq. index %d to 0x%lx (irq vector: %d)\n",
  1092. i, irq->affinity_hint_mask.bits[0], irq->vector);
  1093. irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
  1094. }
  1095. return rc;
  1096. err:
  1097. for (k = ENA_IO_IRQ_FIRST_IDX; k < i; k++) {
  1098. irq = &adapter->irq_tbl[k];
  1099. free_irq(irq->vector, irq->data);
  1100. }
  1101. return rc;
  1102. }
  1103. static void ena_free_mgmnt_irq(struct ena_adapter *adapter)
  1104. {
  1105. struct ena_irq *irq;
  1106. irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
  1107. synchronize_irq(irq->vector);
  1108. irq_set_affinity_hint(irq->vector, NULL);
  1109. free_irq(irq->vector, irq->data);
  1110. }
  1111. static void ena_free_io_irq(struct ena_adapter *adapter)
  1112. {
  1113. struct ena_irq *irq;
  1114. int i;
  1115. #ifdef CONFIG_RFS_ACCEL
  1116. if (adapter->msix_vecs >= 1) {
  1117. free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap);
  1118. adapter->netdev->rx_cpu_rmap = NULL;
  1119. }
  1120. #endif /* CONFIG_RFS_ACCEL */
  1121. for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) {
  1122. irq = &adapter->irq_tbl[i];
  1123. irq_set_affinity_hint(irq->vector, NULL);
  1124. free_irq(irq->vector, irq->data);
  1125. }
  1126. }
  1127. static void ena_disable_io_intr_sync(struct ena_adapter *adapter)
  1128. {
  1129. int i;
  1130. if (!netif_running(adapter->netdev))
  1131. return;
  1132. for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++)
  1133. synchronize_irq(adapter->irq_tbl[i].vector);
  1134. }
  1135. static void ena_del_napi(struct ena_adapter *adapter)
  1136. {
  1137. int i;
  1138. for (i = 0; i < adapter->num_queues; i++)
  1139. netif_napi_del(&adapter->ena_napi[i].napi);
  1140. }
  1141. static void ena_init_napi(struct ena_adapter *adapter)
  1142. {
  1143. struct ena_napi *napi;
  1144. int i;
  1145. for (i = 0; i < adapter->num_queues; i++) {
  1146. napi = &adapter->ena_napi[i];
  1147. netif_napi_add(adapter->netdev,
  1148. &adapter->ena_napi[i].napi,
  1149. ena_io_poll,
  1150. ENA_NAPI_BUDGET);
  1151. napi->rx_ring = &adapter->rx_ring[i];
  1152. napi->tx_ring = &adapter->tx_ring[i];
  1153. napi->qid = i;
  1154. }
  1155. }
  1156. static void ena_napi_disable_all(struct ena_adapter *adapter)
  1157. {
  1158. int i;
  1159. for (i = 0; i < adapter->num_queues; i++)
  1160. napi_disable(&adapter->ena_napi[i].napi);
  1161. }
  1162. static void ena_napi_enable_all(struct ena_adapter *adapter)
  1163. {
  1164. int i;
  1165. for (i = 0; i < adapter->num_queues; i++)
  1166. napi_enable(&adapter->ena_napi[i].napi);
  1167. }
  1168. static void ena_restore_ethtool_params(struct ena_adapter *adapter)
  1169. {
  1170. adapter->tx_usecs = 0;
  1171. adapter->rx_usecs = 0;
  1172. adapter->tx_frames = 1;
  1173. adapter->rx_frames = 1;
  1174. }
  1175. /* Configure the Rx forwarding */
  1176. static int ena_rss_configure(struct ena_adapter *adapter)
  1177. {
  1178. struct ena_com_dev *ena_dev = adapter->ena_dev;
  1179. int rc;
  1180. /* In case the RSS table wasn't initialized by probe */
  1181. if (!ena_dev->rss.tbl_log_size) {
  1182. rc = ena_rss_init_default(adapter);
  1183. if (rc && (rc != -EPERM)) {
  1184. netif_err(adapter, ifup, adapter->netdev,
  1185. "Failed to init RSS rc: %d\n", rc);
  1186. return rc;
  1187. }
  1188. }
  1189. /* Set indirect table */
  1190. rc = ena_com_indirect_table_set(ena_dev);
  1191. if (unlikely(rc && rc != -EPERM))
  1192. return rc;
  1193. /* Configure hash function (if supported) */
  1194. rc = ena_com_set_hash_function(ena_dev);
  1195. if (unlikely(rc && (rc != -EPERM)))
  1196. return rc;
  1197. /* Configure hash inputs (if supported) */
  1198. rc = ena_com_set_hash_ctrl(ena_dev);
  1199. if (unlikely(rc && (rc != -EPERM)))
  1200. return rc;
  1201. return 0;
  1202. }
  1203. static int ena_up_complete(struct ena_adapter *adapter)
  1204. {
  1205. int rc, i;
  1206. rc = ena_rss_configure(adapter);
  1207. if (rc)
  1208. return rc;
  1209. ena_init_napi(adapter);
  1210. ena_change_mtu(adapter->netdev, adapter->netdev->mtu);
  1211. ena_refill_all_rx_bufs(adapter);
  1212. /* enable transmits */
  1213. netif_tx_start_all_queues(adapter->netdev);
  1214. ena_restore_ethtool_params(adapter);
  1215. ena_napi_enable_all(adapter);
  1216. /* schedule napi in case we had pending packets
  1217. * from the last time we disable napi
  1218. */
  1219. for (i = 0; i < adapter->num_queues; i++)
  1220. napi_schedule(&adapter->ena_napi[i].napi);
  1221. return 0;
  1222. }
  1223. static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid)
  1224. {
  1225. struct ena_com_create_io_ctx ctx = { 0 };
  1226. struct ena_com_dev *ena_dev;
  1227. struct ena_ring *tx_ring;
  1228. u32 msix_vector;
  1229. u16 ena_qid;
  1230. int rc;
  1231. ena_dev = adapter->ena_dev;
  1232. tx_ring = &adapter->tx_ring[qid];
  1233. msix_vector = ENA_IO_IRQ_IDX(qid);
  1234. ena_qid = ENA_IO_TXQ_IDX(qid);
  1235. ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
  1236. ctx.qid = ena_qid;
  1237. ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
  1238. ctx.msix_vector = msix_vector;
  1239. ctx.queue_size = adapter->tx_ring_size;
  1240. ctx.numa_node = cpu_to_node(tx_ring->cpu);
  1241. rc = ena_com_create_io_queue(ena_dev, &ctx);
  1242. if (rc) {
  1243. netif_err(adapter, ifup, adapter->netdev,
  1244. "Failed to create I/O TX queue num %d rc: %d\n",
  1245. qid, rc);
  1246. return rc;
  1247. }
  1248. rc = ena_com_get_io_handlers(ena_dev, ena_qid,
  1249. &tx_ring->ena_com_io_sq,
  1250. &tx_ring->ena_com_io_cq);
  1251. if (rc) {
  1252. netif_err(adapter, ifup, adapter->netdev,
  1253. "Failed to get TX queue handlers. TX queue num %d rc: %d\n",
  1254. qid, rc);
  1255. ena_com_destroy_io_queue(ena_dev, ena_qid);
  1256. }
  1257. ena_com_update_numa_node(tx_ring->ena_com_io_cq, ctx.numa_node);
  1258. return rc;
  1259. }
  1260. static int ena_create_all_io_tx_queues(struct ena_adapter *adapter)
  1261. {
  1262. struct ena_com_dev *ena_dev = adapter->ena_dev;
  1263. int rc, i;
  1264. for (i = 0; i < adapter->num_queues; i++) {
  1265. rc = ena_create_io_tx_queue(adapter, i);
  1266. if (rc)
  1267. goto create_err;
  1268. }
  1269. return 0;
  1270. create_err:
  1271. while (i--)
  1272. ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i));
  1273. return rc;
  1274. }
  1275. static int ena_create_io_rx_queue(struct ena_adapter *adapter, int qid)
  1276. {
  1277. struct ena_com_dev *ena_dev;
  1278. struct ena_com_create_io_ctx ctx = { 0 };
  1279. struct ena_ring *rx_ring;
  1280. u32 msix_vector;
  1281. u16 ena_qid;
  1282. int rc;
  1283. ena_dev = adapter->ena_dev;
  1284. rx_ring = &adapter->rx_ring[qid];
  1285. msix_vector = ENA_IO_IRQ_IDX(qid);
  1286. ena_qid = ENA_IO_RXQ_IDX(qid);
  1287. ctx.qid = ena_qid;
  1288. ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
  1289. ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
  1290. ctx.msix_vector = msix_vector;
  1291. ctx.queue_size = adapter->rx_ring_size;
  1292. ctx.numa_node = cpu_to_node(rx_ring->cpu);
  1293. rc = ena_com_create_io_queue(ena_dev, &ctx);
  1294. if (rc) {
  1295. netif_err(adapter, ifup, adapter->netdev,
  1296. "Failed to create I/O RX queue num %d rc: %d\n",
  1297. qid, rc);
  1298. return rc;
  1299. }
  1300. rc = ena_com_get_io_handlers(ena_dev, ena_qid,
  1301. &rx_ring->ena_com_io_sq,
  1302. &rx_ring->ena_com_io_cq);
  1303. if (rc) {
  1304. netif_err(adapter, ifup, adapter->netdev,
  1305. "Failed to get RX queue handlers. RX queue num %d rc: %d\n",
  1306. qid, rc);
  1307. ena_com_destroy_io_queue(ena_dev, ena_qid);
  1308. }
  1309. ena_com_update_numa_node(rx_ring->ena_com_io_cq, ctx.numa_node);
  1310. return rc;
  1311. }
  1312. static int ena_create_all_io_rx_queues(struct ena_adapter *adapter)
  1313. {
  1314. struct ena_com_dev *ena_dev = adapter->ena_dev;
  1315. int rc, i;
  1316. for (i = 0; i < adapter->num_queues; i++) {
  1317. rc = ena_create_io_rx_queue(adapter, i);
  1318. if (rc)
  1319. goto create_err;
  1320. }
  1321. return 0;
  1322. create_err:
  1323. while (i--)
  1324. ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i));
  1325. return rc;
  1326. }
  1327. static int ena_up(struct ena_adapter *adapter)
  1328. {
  1329. int rc;
  1330. netdev_dbg(adapter->netdev, "%s\n", __func__);
  1331. ena_setup_io_intr(adapter);
  1332. rc = ena_request_io_irq(adapter);
  1333. if (rc)
  1334. goto err_req_irq;
  1335. /* allocate transmit descriptors */
  1336. rc = ena_setup_all_tx_resources(adapter);
  1337. if (rc)
  1338. goto err_setup_tx;
  1339. /* allocate receive descriptors */
  1340. rc = ena_setup_all_rx_resources(adapter);
  1341. if (rc)
  1342. goto err_setup_rx;
  1343. /* Create TX queues */
  1344. rc = ena_create_all_io_tx_queues(adapter);
  1345. if (rc)
  1346. goto err_create_tx_queues;
  1347. /* Create RX queues */
  1348. rc = ena_create_all_io_rx_queues(adapter);
  1349. if (rc)
  1350. goto err_create_rx_queues;
  1351. rc = ena_up_complete(adapter);
  1352. if (rc)
  1353. goto err_up;
  1354. if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags))
  1355. netif_carrier_on(adapter->netdev);
  1356. u64_stats_update_begin(&adapter->syncp);
  1357. adapter->dev_stats.interface_up++;
  1358. u64_stats_update_end(&adapter->syncp);
  1359. set_bit(ENA_FLAG_DEV_UP, &adapter->flags);
  1360. return rc;
  1361. err_up:
  1362. ena_destroy_all_rx_queues(adapter);
  1363. err_create_rx_queues:
  1364. ena_destroy_all_tx_queues(adapter);
  1365. err_create_tx_queues:
  1366. ena_free_all_io_rx_resources(adapter);
  1367. err_setup_rx:
  1368. ena_free_all_io_tx_resources(adapter);
  1369. err_setup_tx:
  1370. ena_free_io_irq(adapter);
  1371. err_req_irq:
  1372. return rc;
  1373. }
  1374. static void ena_down(struct ena_adapter *adapter)
  1375. {
  1376. netif_info(adapter, ifdown, adapter->netdev, "%s\n", __func__);
  1377. clear_bit(ENA_FLAG_DEV_UP, &adapter->flags);
  1378. u64_stats_update_begin(&adapter->syncp);
  1379. adapter->dev_stats.interface_down++;
  1380. u64_stats_update_end(&adapter->syncp);
  1381. netif_carrier_off(adapter->netdev);
  1382. netif_tx_disable(adapter->netdev);
  1383. /* After this point the napi handler won't enable the tx queue */
  1384. ena_napi_disable_all(adapter);
  1385. /* After destroy the queue there won't be any new interrupts */
  1386. if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) {
  1387. int rc;
  1388. rc = ena_com_dev_reset(adapter->ena_dev);
  1389. if (rc)
  1390. dev_err(&adapter->pdev->dev, "Device reset failed\n");
  1391. }
  1392. ena_destroy_all_io_queues(adapter);
  1393. ena_disable_io_intr_sync(adapter);
  1394. ena_free_io_irq(adapter);
  1395. ena_del_napi(adapter);
  1396. ena_free_all_tx_bufs(adapter);
  1397. ena_free_all_rx_bufs(adapter);
  1398. ena_free_all_io_tx_resources(adapter);
  1399. ena_free_all_io_rx_resources(adapter);
  1400. }
  1401. /* ena_open - Called when a network interface is made active
  1402. * @netdev: network interface device structure
  1403. *
  1404. * Returns 0 on success, negative value on failure
  1405. *
  1406. * The open entry point is called when a network interface is made
  1407. * active by the system (IFF_UP). At this point all resources needed
  1408. * for transmit and receive operations are allocated, the interrupt
  1409. * handler is registered with the OS, the watchdog timer is started,
  1410. * and the stack is notified that the interface is ready.
  1411. */
  1412. static int ena_open(struct net_device *netdev)
  1413. {
  1414. struct ena_adapter *adapter = netdev_priv(netdev);
  1415. int rc;
  1416. /* Notify the stack of the actual queue counts. */
  1417. rc = netif_set_real_num_tx_queues(netdev, adapter->num_queues);
  1418. if (rc) {
  1419. netif_err(adapter, ifup, netdev, "Can't set num tx queues\n");
  1420. return rc;
  1421. }
  1422. rc = netif_set_real_num_rx_queues(netdev, adapter->num_queues);
  1423. if (rc) {
  1424. netif_err(adapter, ifup, netdev, "Can't set num rx queues\n");
  1425. return rc;
  1426. }
  1427. rc = ena_up(adapter);
  1428. if (rc)
  1429. return rc;
  1430. return rc;
  1431. }
  1432. /* ena_close - Disables a network interface
  1433. * @netdev: network interface device structure
  1434. *
  1435. * Returns 0, this is not allowed to fail
  1436. *
  1437. * The close entry point is called when an interface is de-activated
  1438. * by the OS. The hardware is still under the drivers control, but
  1439. * needs to be disabled. A global MAC reset is issued to stop the
  1440. * hardware, and all transmit and receive resources are freed.
  1441. */
  1442. static int ena_close(struct net_device *netdev)
  1443. {
  1444. struct ena_adapter *adapter = netdev_priv(netdev);
  1445. netif_dbg(adapter, ifdown, netdev, "%s\n", __func__);
  1446. if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
  1447. ena_down(adapter);
  1448. return 0;
  1449. }
  1450. static void ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct sk_buff *skb)
  1451. {
  1452. u32 mss = skb_shinfo(skb)->gso_size;
  1453. struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
  1454. u8 l4_protocol = 0;
  1455. if ((skb->ip_summed == CHECKSUM_PARTIAL) || mss) {
  1456. ena_tx_ctx->l4_csum_enable = 1;
  1457. if (mss) {
  1458. ena_tx_ctx->tso_enable = 1;
  1459. ena_meta->l4_hdr_len = tcp_hdr(skb)->doff;
  1460. ena_tx_ctx->l4_csum_partial = 0;
  1461. } else {
  1462. ena_tx_ctx->tso_enable = 0;
  1463. ena_meta->l4_hdr_len = 0;
  1464. ena_tx_ctx->l4_csum_partial = 1;
  1465. }
  1466. switch (ip_hdr(skb)->version) {
  1467. case IPVERSION:
  1468. ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4;
  1469. if (ip_hdr(skb)->frag_off & htons(IP_DF))
  1470. ena_tx_ctx->df = 1;
  1471. if (mss)
  1472. ena_tx_ctx->l3_csum_enable = 1;
  1473. l4_protocol = ip_hdr(skb)->protocol;
  1474. break;
  1475. case 6:
  1476. ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6;
  1477. l4_protocol = ipv6_hdr(skb)->nexthdr;
  1478. break;
  1479. default:
  1480. break;
  1481. }
  1482. if (l4_protocol == IPPROTO_TCP)
  1483. ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP;
  1484. else
  1485. ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP;
  1486. ena_meta->mss = mss;
  1487. ena_meta->l3_hdr_len = skb_network_header_len(skb);
  1488. ena_meta->l3_hdr_offset = skb_network_offset(skb);
  1489. ena_tx_ctx->meta_valid = 1;
  1490. } else {
  1491. ena_tx_ctx->meta_valid = 0;
  1492. }
  1493. }
  1494. static int ena_check_and_linearize_skb(struct ena_ring *tx_ring,
  1495. struct sk_buff *skb)
  1496. {
  1497. int num_frags, header_len, rc;
  1498. num_frags = skb_shinfo(skb)->nr_frags;
  1499. header_len = skb_headlen(skb);
  1500. if (num_frags < tx_ring->sgl_size)
  1501. return 0;
  1502. if ((num_frags == tx_ring->sgl_size) &&
  1503. (header_len < tx_ring->tx_max_header_size))
  1504. return 0;
  1505. u64_stats_update_begin(&tx_ring->syncp);
  1506. tx_ring->tx_stats.linearize++;
  1507. u64_stats_update_end(&tx_ring->syncp);
  1508. rc = skb_linearize(skb);
  1509. if (unlikely(rc)) {
  1510. u64_stats_update_begin(&tx_ring->syncp);
  1511. tx_ring->tx_stats.linearize_failed++;
  1512. u64_stats_update_end(&tx_ring->syncp);
  1513. }
  1514. return rc;
  1515. }
  1516. /* Called with netif_tx_lock. */
  1517. static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
  1518. {
  1519. struct ena_adapter *adapter = netdev_priv(dev);
  1520. struct ena_tx_buffer *tx_info;
  1521. struct ena_com_tx_ctx ena_tx_ctx;
  1522. struct ena_ring *tx_ring;
  1523. struct netdev_queue *txq;
  1524. struct ena_com_buf *ena_buf;
  1525. void *push_hdr;
  1526. u32 len, last_frag;
  1527. u16 next_to_use;
  1528. u16 req_id;
  1529. u16 push_len;
  1530. u16 header_len;
  1531. dma_addr_t dma;
  1532. int qid, rc, nb_hw_desc;
  1533. int i = -1;
  1534. netif_dbg(adapter, tx_queued, dev, "%s skb %p\n", __func__, skb);
  1535. /* Determine which tx ring we will be placed on */
  1536. qid = skb_get_queue_mapping(skb);
  1537. tx_ring = &adapter->tx_ring[qid];
  1538. txq = netdev_get_tx_queue(dev, qid);
  1539. rc = ena_check_and_linearize_skb(tx_ring, skb);
  1540. if (unlikely(rc))
  1541. goto error_drop_packet;
  1542. skb_tx_timestamp(skb);
  1543. len = skb_headlen(skb);
  1544. next_to_use = tx_ring->next_to_use;
  1545. req_id = tx_ring->free_tx_ids[next_to_use];
  1546. tx_info = &tx_ring->tx_buffer_info[req_id];
  1547. tx_info->num_of_bufs = 0;
  1548. WARN(tx_info->skb, "SKB isn't NULL req_id %d\n", req_id);
  1549. ena_buf = tx_info->bufs;
  1550. tx_info->skb = skb;
  1551. if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
  1552. /* prepared the push buffer */
  1553. push_len = min_t(u32, len, tx_ring->tx_max_header_size);
  1554. header_len = push_len;
  1555. push_hdr = skb->data;
  1556. } else {
  1557. push_len = 0;
  1558. header_len = min_t(u32, len, tx_ring->tx_max_header_size);
  1559. push_hdr = NULL;
  1560. }
  1561. netif_dbg(adapter, tx_queued, dev,
  1562. "skb: %p header_buf->vaddr: %p push_len: %d\n", skb,
  1563. push_hdr, push_len);
  1564. if (len > push_len) {
  1565. dma = dma_map_single(tx_ring->dev, skb->data + push_len,
  1566. len - push_len, DMA_TO_DEVICE);
  1567. if (dma_mapping_error(tx_ring->dev, dma))
  1568. goto error_report_dma_error;
  1569. ena_buf->paddr = dma;
  1570. ena_buf->len = len - push_len;
  1571. ena_buf++;
  1572. tx_info->num_of_bufs++;
  1573. }
  1574. last_frag = skb_shinfo(skb)->nr_frags;
  1575. for (i = 0; i < last_frag; i++) {
  1576. const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  1577. len = skb_frag_size(frag);
  1578. dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
  1579. DMA_TO_DEVICE);
  1580. if (dma_mapping_error(tx_ring->dev, dma))
  1581. goto error_report_dma_error;
  1582. ena_buf->paddr = dma;
  1583. ena_buf->len = len;
  1584. ena_buf++;
  1585. }
  1586. tx_info->num_of_bufs += last_frag;
  1587. memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));
  1588. ena_tx_ctx.ena_bufs = tx_info->bufs;
  1589. ena_tx_ctx.push_header = push_hdr;
  1590. ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
  1591. ena_tx_ctx.req_id = req_id;
  1592. ena_tx_ctx.header_len = header_len;
  1593. /* set flags and meta data */
  1594. ena_tx_csum(&ena_tx_ctx, skb);
  1595. /* prepare the packet's descriptors to dma engine */
  1596. rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, &ena_tx_ctx,
  1597. &nb_hw_desc);
  1598. if (unlikely(rc)) {
  1599. netif_err(adapter, tx_queued, dev,
  1600. "failed to prepare tx bufs\n");
  1601. u64_stats_update_begin(&tx_ring->syncp);
  1602. tx_ring->tx_stats.queue_stop++;
  1603. tx_ring->tx_stats.prepare_ctx_err++;
  1604. u64_stats_update_end(&tx_ring->syncp);
  1605. netif_tx_stop_queue(txq);
  1606. goto error_unmap_dma;
  1607. }
  1608. netdev_tx_sent_queue(txq, skb->len);
  1609. u64_stats_update_begin(&tx_ring->syncp);
  1610. tx_ring->tx_stats.cnt++;
  1611. tx_ring->tx_stats.bytes += skb->len;
  1612. u64_stats_update_end(&tx_ring->syncp);
  1613. tx_info->tx_descs = nb_hw_desc;
  1614. tx_info->last_jiffies = jiffies;
  1615. tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,
  1616. tx_ring->ring_size);
  1617. /* This WMB is aimed to:
  1618. * 1 - perform smp barrier before reading next_to_completion
  1619. * 2 - make sure the desc were written before trigger DB
  1620. */
  1621. wmb();
  1622. /* stop the queue when no more space available, the packet can have up
  1623. * to sgl_size + 2. one for the meta descriptor and one for header
  1624. * (if the header is larger than tx_max_header_size).
  1625. */
  1626. if (unlikely(ena_com_sq_empty_space(tx_ring->ena_com_io_sq) <
  1627. (tx_ring->sgl_size + 2))) {
  1628. netif_dbg(adapter, tx_queued, dev, "%s stop queue %d\n",
  1629. __func__, qid);
  1630. netif_tx_stop_queue(txq);
  1631. u64_stats_update_begin(&tx_ring->syncp);
  1632. tx_ring->tx_stats.queue_stop++;
  1633. u64_stats_update_end(&tx_ring->syncp);
  1634. /* There is a rare condition where this function decide to
  1635. * stop the queue but meanwhile clean_tx_irq updates
  1636. * next_to_completion and terminates.
  1637. * The queue will remain stopped forever.
  1638. * To solve this issue this function perform rmb, check
  1639. * the wakeup condition and wake up the queue if needed.
  1640. */
  1641. smp_rmb();
  1642. if (ena_com_sq_empty_space(tx_ring->ena_com_io_sq)
  1643. > ENA_TX_WAKEUP_THRESH) {
  1644. netif_tx_wake_queue(txq);
  1645. u64_stats_update_begin(&tx_ring->syncp);
  1646. tx_ring->tx_stats.queue_wakeup++;
  1647. u64_stats_update_end(&tx_ring->syncp);
  1648. }
  1649. }
  1650. if (netif_xmit_stopped(txq) || !skb->xmit_more) {
  1651. /* trigger the dma engine */
  1652. ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);
  1653. u64_stats_update_begin(&tx_ring->syncp);
  1654. tx_ring->tx_stats.doorbells++;
  1655. u64_stats_update_end(&tx_ring->syncp);
  1656. }
  1657. return NETDEV_TX_OK;
  1658. error_report_dma_error:
  1659. u64_stats_update_begin(&tx_ring->syncp);
  1660. tx_ring->tx_stats.dma_mapping_err++;
  1661. u64_stats_update_end(&tx_ring->syncp);
  1662. netdev_warn(adapter->netdev, "failed to map skb\n");
  1663. tx_info->skb = NULL;
  1664. error_unmap_dma:
  1665. if (i >= 0) {
  1666. /* save value of frag that failed */
  1667. last_frag = i;
  1668. /* start back at beginning and unmap skb */
  1669. tx_info->skb = NULL;
  1670. ena_buf = tx_info->bufs;
  1671. dma_unmap_single(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),
  1672. dma_unmap_len(ena_buf, len), DMA_TO_DEVICE);
  1673. /* unmap remaining mapped pages */
  1674. for (i = 0; i < last_frag; i++) {
  1675. ena_buf++;
  1676. dma_unmap_page(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),
  1677. dma_unmap_len(ena_buf, len), DMA_TO_DEVICE);
  1678. }
  1679. }
  1680. error_drop_packet:
  1681. dev_kfree_skb(skb);
  1682. return NETDEV_TX_OK;
  1683. }
  1684. #ifdef CONFIG_NET_POLL_CONTROLLER
  1685. static void ena_netpoll(struct net_device *netdev)
  1686. {
  1687. struct ena_adapter *adapter = netdev_priv(netdev);
  1688. int i;
  1689. /* Dont schedule NAPI if the driver is in the middle of reset
  1690. * or netdev is down.
  1691. */
  1692. if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags) ||
  1693. test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
  1694. return;
  1695. for (i = 0; i < adapter->num_queues; i++)
  1696. napi_schedule(&adapter->ena_napi[i].napi);
  1697. }
  1698. #endif /* CONFIG_NET_POLL_CONTROLLER */
  1699. static u16 ena_select_queue(struct net_device *dev, struct sk_buff *skb,
  1700. void *accel_priv, select_queue_fallback_t fallback)
  1701. {
  1702. u16 qid;
  1703. /* we suspect that this is good for in--kernel network services that
  1704. * want to loop incoming skb rx to tx in normal user generated traffic,
  1705. * most probably we will not get to this
  1706. */
  1707. if (skb_rx_queue_recorded(skb))
  1708. qid = skb_get_rx_queue(skb);
  1709. else
  1710. qid = fallback(dev, skb);
  1711. return qid;
  1712. }
  1713. static void ena_config_host_info(struct ena_com_dev *ena_dev)
  1714. {
  1715. struct ena_admin_host_info *host_info;
  1716. int rc;
  1717. /* Allocate only the host info */
  1718. rc = ena_com_allocate_host_info(ena_dev);
  1719. if (rc) {
  1720. pr_err("Cannot allocate host info\n");
  1721. return;
  1722. }
  1723. host_info = ena_dev->host_attr.host_info;
  1724. host_info->os_type = ENA_ADMIN_OS_LINUX;
  1725. host_info->kernel_ver = LINUX_VERSION_CODE;
  1726. strncpy(host_info->kernel_ver_str, utsname()->version,
  1727. sizeof(host_info->kernel_ver_str) - 1);
  1728. host_info->os_dist = 0;
  1729. strncpy(host_info->os_dist_str, utsname()->release,
  1730. sizeof(host_info->os_dist_str) - 1);
  1731. host_info->driver_version =
  1732. (DRV_MODULE_VER_MAJOR) |
  1733. (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) |
  1734. (DRV_MODULE_VER_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT);
  1735. rc = ena_com_set_host_attributes(ena_dev);
  1736. if (rc) {
  1737. if (rc == -EPERM)
  1738. pr_warn("Cannot set host attributes\n");
  1739. else
  1740. pr_err("Cannot set host attributes\n");
  1741. goto err;
  1742. }
  1743. return;
  1744. err:
  1745. ena_com_delete_host_info(ena_dev);
  1746. }
  1747. static void ena_config_debug_area(struct ena_adapter *adapter)
  1748. {
  1749. u32 debug_area_size;
  1750. int rc, ss_count;
  1751. ss_count = ena_get_sset_count(adapter->netdev, ETH_SS_STATS);
  1752. if (ss_count <= 0) {
  1753. netif_err(adapter, drv, adapter->netdev,
  1754. "SS count is negative\n");
  1755. return;
  1756. }
  1757. /* allocate 32 bytes for each string and 64bit for the value */
  1758. debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count;
  1759. rc = ena_com_allocate_debug_area(adapter->ena_dev, debug_area_size);
  1760. if (rc) {
  1761. pr_err("Cannot allocate debug area\n");
  1762. return;
  1763. }
  1764. rc = ena_com_set_host_attributes(adapter->ena_dev);
  1765. if (rc) {
  1766. if (rc == -EPERM)
  1767. netif_warn(adapter, drv, adapter->netdev,
  1768. "Cannot set host attributes\n");
  1769. else
  1770. netif_err(adapter, drv, adapter->netdev,
  1771. "Cannot set host attributes\n");
  1772. goto err;
  1773. }
  1774. return;
  1775. err:
  1776. ena_com_delete_debug_area(adapter->ena_dev);
  1777. }
  1778. static void ena_get_stats64(struct net_device *netdev,
  1779. struct rtnl_link_stats64 *stats)
  1780. {
  1781. struct ena_adapter *adapter = netdev_priv(netdev);
  1782. struct ena_ring *rx_ring, *tx_ring;
  1783. unsigned int start;
  1784. u64 rx_drops;
  1785. int i;
  1786. if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
  1787. return;
  1788. for (i = 0; i < adapter->num_queues; i++) {
  1789. u64 bytes, packets;
  1790. tx_ring = &adapter->tx_ring[i];
  1791. do {
  1792. start = u64_stats_fetch_begin_irq(&tx_ring->syncp);
  1793. packets = tx_ring->tx_stats.cnt;
  1794. bytes = tx_ring->tx_stats.bytes;
  1795. } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start));
  1796. stats->tx_packets += packets;
  1797. stats->tx_bytes += bytes;
  1798. rx_ring = &adapter->rx_ring[i];
  1799. do {
  1800. start = u64_stats_fetch_begin_irq(&rx_ring->syncp);
  1801. packets = rx_ring->rx_stats.cnt;
  1802. bytes = rx_ring->rx_stats.bytes;
  1803. } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start));
  1804. stats->rx_packets += packets;
  1805. stats->rx_bytes += bytes;
  1806. }
  1807. do {
  1808. start = u64_stats_fetch_begin_irq(&adapter->syncp);
  1809. rx_drops = adapter->dev_stats.rx_drops;
  1810. } while (u64_stats_fetch_retry_irq(&adapter->syncp, start));
  1811. stats->rx_dropped = rx_drops;
  1812. stats->multicast = 0;
  1813. stats->collisions = 0;
  1814. stats->rx_length_errors = 0;
  1815. stats->rx_crc_errors = 0;
  1816. stats->rx_frame_errors = 0;
  1817. stats->rx_fifo_errors = 0;
  1818. stats->rx_missed_errors = 0;
  1819. stats->tx_window_errors = 0;
  1820. stats->rx_errors = 0;
  1821. stats->tx_errors = 0;
  1822. }
  1823. static const struct net_device_ops ena_netdev_ops = {
  1824. .ndo_open = ena_open,
  1825. .ndo_stop = ena_close,
  1826. .ndo_start_xmit = ena_start_xmit,
  1827. .ndo_select_queue = ena_select_queue,
  1828. .ndo_get_stats64 = ena_get_stats64,
  1829. .ndo_tx_timeout = ena_tx_timeout,
  1830. .ndo_change_mtu = ena_change_mtu,
  1831. .ndo_set_mac_address = NULL,
  1832. .ndo_validate_addr = eth_validate_addr,
  1833. #ifdef CONFIG_NET_POLL_CONTROLLER
  1834. .ndo_poll_controller = ena_netpoll,
  1835. #endif /* CONFIG_NET_POLL_CONTROLLER */
  1836. };
  1837. static void ena_device_io_suspend(struct work_struct *work)
  1838. {
  1839. struct ena_adapter *adapter =
  1840. container_of(work, struct ena_adapter, suspend_io_task);
  1841. struct net_device *netdev = adapter->netdev;
  1842. /* ena_napi_disable_all disables only the IO handling.
  1843. * We are still subject to AENQ keep alive watchdog.
  1844. */
  1845. u64_stats_update_begin(&adapter->syncp);
  1846. adapter->dev_stats.io_suspend++;
  1847. u64_stats_update_begin(&adapter->syncp);
  1848. ena_napi_disable_all(adapter);
  1849. netif_tx_lock(netdev);
  1850. netif_device_detach(netdev);
  1851. netif_tx_unlock(netdev);
  1852. }
  1853. static void ena_device_io_resume(struct work_struct *work)
  1854. {
  1855. struct ena_adapter *adapter =
  1856. container_of(work, struct ena_adapter, resume_io_task);
  1857. struct net_device *netdev = adapter->netdev;
  1858. u64_stats_update_begin(&adapter->syncp);
  1859. adapter->dev_stats.io_resume++;
  1860. u64_stats_update_end(&adapter->syncp);
  1861. netif_device_attach(netdev);
  1862. ena_napi_enable_all(adapter);
  1863. }
  1864. static int ena_device_validate_params(struct ena_adapter *adapter,
  1865. struct ena_com_dev_get_features_ctx *get_feat_ctx)
  1866. {
  1867. struct net_device *netdev = adapter->netdev;
  1868. int rc;
  1869. rc = ether_addr_equal(get_feat_ctx->dev_attr.mac_addr,
  1870. adapter->mac_addr);
  1871. if (!rc) {
  1872. netif_err(adapter, drv, netdev,
  1873. "Error, mac address are different\n");
  1874. return -EINVAL;
  1875. }
  1876. if ((get_feat_ctx->max_queues.max_cq_num < adapter->num_queues) ||
  1877. (get_feat_ctx->max_queues.max_sq_num < adapter->num_queues)) {
  1878. netif_err(adapter, drv, netdev,
  1879. "Error, device doesn't support enough queues\n");
  1880. return -EINVAL;
  1881. }
  1882. if (get_feat_ctx->dev_attr.max_mtu < netdev->mtu) {
  1883. netif_err(adapter, drv, netdev,
  1884. "Error, device max mtu is smaller than netdev MTU\n");
  1885. return -EINVAL;
  1886. }
  1887. return 0;
  1888. }
  1889. static int ena_device_init(struct ena_com_dev *ena_dev, struct pci_dev *pdev,
  1890. struct ena_com_dev_get_features_ctx *get_feat_ctx,
  1891. bool *wd_state)
  1892. {
  1893. struct device *dev = &pdev->dev;
  1894. bool readless_supported;
  1895. u32 aenq_groups;
  1896. int dma_width;
  1897. int rc;
  1898. rc = ena_com_mmio_reg_read_request_init(ena_dev);
  1899. if (rc) {
  1900. dev_err(dev, "failed to init mmio read less\n");
  1901. return rc;
  1902. }
  1903. /* The PCIe configuration space revision id indicate if mmio reg
  1904. * read is disabled
  1905. */
  1906. readless_supported = !(pdev->revision & ENA_MMIO_DISABLE_REG_READ);
  1907. ena_com_set_mmio_read_mode(ena_dev, readless_supported);
  1908. rc = ena_com_dev_reset(ena_dev);
  1909. if (rc) {
  1910. dev_err(dev, "Can not reset device\n");
  1911. goto err_mmio_read_less;
  1912. }
  1913. rc = ena_com_validate_version(ena_dev);
  1914. if (rc) {
  1915. dev_err(dev, "device version is too low\n");
  1916. goto err_mmio_read_less;
  1917. }
  1918. dma_width = ena_com_get_dma_width(ena_dev);
  1919. if (dma_width < 0) {
  1920. dev_err(dev, "Invalid dma width value %d", dma_width);
  1921. rc = dma_width;
  1922. goto err_mmio_read_less;
  1923. }
  1924. rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(dma_width));
  1925. if (rc) {
  1926. dev_err(dev, "pci_set_dma_mask failed 0x%x\n", rc);
  1927. goto err_mmio_read_less;
  1928. }
  1929. rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(dma_width));
  1930. if (rc) {
  1931. dev_err(dev, "err_pci_set_consistent_dma_mask failed 0x%x\n",
  1932. rc);
  1933. goto err_mmio_read_less;
  1934. }
  1935. /* ENA admin level init */
  1936. rc = ena_com_admin_init(ena_dev, &aenq_handlers, true);
  1937. if (rc) {
  1938. dev_err(dev,
  1939. "Can not initialize ena admin queue with device\n");
  1940. goto err_mmio_read_less;
  1941. }
  1942. /* To enable the msix interrupts the driver needs to know the number
  1943. * of queues. So the driver uses polling mode to retrieve this
  1944. * information
  1945. */
  1946. ena_com_set_admin_polling_mode(ena_dev, true);
  1947. ena_config_host_info(ena_dev);
  1948. /* Get Device Attributes*/
  1949. rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);
  1950. if (rc) {
  1951. dev_err(dev, "Cannot get attribute for ena device rc=%d\n", rc);
  1952. goto err_admin_init;
  1953. }
  1954. /* Try to turn all the available aenq groups */
  1955. aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) |
  1956. BIT(ENA_ADMIN_FATAL_ERROR) |
  1957. BIT(ENA_ADMIN_WARNING) |
  1958. BIT(ENA_ADMIN_NOTIFICATION) |
  1959. BIT(ENA_ADMIN_KEEP_ALIVE);
  1960. aenq_groups &= get_feat_ctx->aenq.supported_groups;
  1961. rc = ena_com_set_aenq_config(ena_dev, aenq_groups);
  1962. if (rc) {
  1963. dev_err(dev, "Cannot configure aenq groups rc= %d\n", rc);
  1964. goto err_admin_init;
  1965. }
  1966. *wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE));
  1967. return 0;
  1968. err_admin_init:
  1969. ena_com_delete_host_info(ena_dev);
  1970. ena_com_admin_destroy(ena_dev);
  1971. err_mmio_read_less:
  1972. ena_com_mmio_reg_read_request_destroy(ena_dev);
  1973. return rc;
  1974. }
  1975. static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter,
  1976. int io_vectors)
  1977. {
  1978. struct ena_com_dev *ena_dev = adapter->ena_dev;
  1979. struct device *dev = &adapter->pdev->dev;
  1980. int rc;
  1981. rc = ena_enable_msix(adapter, io_vectors);
  1982. if (rc) {
  1983. dev_err(dev, "Can not reserve msix vectors\n");
  1984. return rc;
  1985. }
  1986. ena_setup_mgmnt_intr(adapter);
  1987. rc = ena_request_mgmnt_irq(adapter);
  1988. if (rc) {
  1989. dev_err(dev, "Can not setup management interrupts\n");
  1990. goto err_disable_msix;
  1991. }
  1992. ena_com_set_admin_polling_mode(ena_dev, false);
  1993. ena_com_admin_aenq_enable(ena_dev);
  1994. return 0;
  1995. err_disable_msix:
  1996. pci_free_irq_vectors(adapter->pdev);
  1997. return rc;
  1998. }
  1999. static void ena_fw_reset_device(struct work_struct *work)
  2000. {
  2001. struct ena_com_dev_get_features_ctx get_feat_ctx;
  2002. struct ena_adapter *adapter =
  2003. container_of(work, struct ena_adapter, reset_task);
  2004. struct net_device *netdev = adapter->netdev;
  2005. struct ena_com_dev *ena_dev = adapter->ena_dev;
  2006. struct pci_dev *pdev = adapter->pdev;
  2007. bool dev_up, wd_state;
  2008. int rc;
  2009. if (unlikely(!test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
  2010. dev_err(&pdev->dev,
  2011. "device reset schedule while reset bit is off\n");
  2012. return;
  2013. }
  2014. netif_carrier_off(netdev);
  2015. del_timer_sync(&adapter->timer_service);
  2016. rtnl_lock();
  2017. dev_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
  2018. ena_com_set_admin_running_state(ena_dev, false);
  2019. /* After calling ena_close the tx queues and the napi
  2020. * are disabled so no one can interfere or touch the
  2021. * data structures
  2022. */
  2023. ena_close(netdev);
  2024. ena_free_mgmnt_irq(adapter);
  2025. pci_free_irq_vectors(adapter->pdev);
  2026. ena_com_abort_admin_commands(ena_dev);
  2027. ena_com_wait_for_abort_completion(ena_dev);
  2028. ena_com_admin_destroy(ena_dev);
  2029. ena_com_mmio_reg_read_request_destroy(ena_dev);
  2030. clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
  2031. /* Finish with the destroy part. Start the init part */
  2032. rc = ena_device_init(ena_dev, adapter->pdev, &get_feat_ctx, &wd_state);
  2033. if (rc) {
  2034. dev_err(&pdev->dev, "Can not initialize device\n");
  2035. goto err;
  2036. }
  2037. adapter->wd_state = wd_state;
  2038. rc = ena_device_validate_params(adapter, &get_feat_ctx);
  2039. if (rc) {
  2040. dev_err(&pdev->dev, "Validation of device parameters failed\n");
  2041. goto err_device_destroy;
  2042. }
  2043. rc = ena_enable_msix_and_set_admin_interrupts(adapter,
  2044. adapter->num_queues);
  2045. if (rc) {
  2046. dev_err(&pdev->dev, "Enable MSI-X failed\n");
  2047. goto err_device_destroy;
  2048. }
  2049. /* If the interface was up before the reset bring it up */
  2050. if (dev_up) {
  2051. rc = ena_up(adapter);
  2052. if (rc) {
  2053. dev_err(&pdev->dev, "Failed to create I/O queues\n");
  2054. goto err_disable_msix;
  2055. }
  2056. }
  2057. mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
  2058. rtnl_unlock();
  2059. dev_err(&pdev->dev, "Device reset completed successfully\n");
  2060. return;
  2061. err_disable_msix:
  2062. ena_free_mgmnt_irq(adapter);
  2063. pci_free_irq_vectors(adapter->pdev);
  2064. err_device_destroy:
  2065. ena_com_admin_destroy(ena_dev);
  2066. err:
  2067. rtnl_unlock();
  2068. clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
  2069. dev_err(&pdev->dev,
  2070. "Reset attempt failed. Can not reset the device\n");
  2071. }
  2072. static void check_for_missing_tx_completions(struct ena_adapter *adapter)
  2073. {
  2074. struct ena_tx_buffer *tx_buf;
  2075. unsigned long last_jiffies;
  2076. struct ena_ring *tx_ring;
  2077. int i, j, budget;
  2078. u32 missed_tx;
  2079. /* Make sure the driver doesn't turn the device in other process */
  2080. smp_rmb();
  2081. if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
  2082. return;
  2083. if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
  2084. return;
  2085. budget = ENA_MONITORED_TX_QUEUES;
  2086. for (i = adapter->last_monitored_tx_qid; i < adapter->num_queues; i++) {
  2087. tx_ring = &adapter->tx_ring[i];
  2088. for (j = 0; j < tx_ring->ring_size; j++) {
  2089. tx_buf = &tx_ring->tx_buffer_info[j];
  2090. last_jiffies = tx_buf->last_jiffies;
  2091. if (unlikely(last_jiffies && time_is_before_jiffies(last_jiffies + TX_TIMEOUT))) {
  2092. netif_notice(adapter, tx_err, adapter->netdev,
  2093. "Found a Tx that wasn't completed on time, qid %d, index %d.\n",
  2094. tx_ring->qid, j);
  2095. u64_stats_update_begin(&tx_ring->syncp);
  2096. missed_tx = tx_ring->tx_stats.missing_tx_comp++;
  2097. u64_stats_update_end(&tx_ring->syncp);
  2098. /* Clear last jiffies so the lost buffer won't
  2099. * be counted twice.
  2100. */
  2101. tx_buf->last_jiffies = 0;
  2102. if (unlikely(missed_tx > MAX_NUM_OF_TIMEOUTED_PACKETS)) {
  2103. netif_err(adapter, tx_err, adapter->netdev,
  2104. "The number of lost tx completion is above the threshold (%d > %d). Reset the device\n",
  2105. missed_tx, MAX_NUM_OF_TIMEOUTED_PACKETS);
  2106. set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
  2107. }
  2108. }
  2109. }
  2110. budget--;
  2111. if (!budget)
  2112. break;
  2113. }
  2114. adapter->last_monitored_tx_qid = i % adapter->num_queues;
  2115. }
  2116. /* Check for keep alive expiration */
  2117. static void check_for_missing_keep_alive(struct ena_adapter *adapter)
  2118. {
  2119. unsigned long keep_alive_expired;
  2120. if (!adapter->wd_state)
  2121. return;
  2122. keep_alive_expired = round_jiffies(adapter->last_keep_alive_jiffies
  2123. + ENA_DEVICE_KALIVE_TIMEOUT);
  2124. if (unlikely(time_is_before_jiffies(keep_alive_expired))) {
  2125. netif_err(adapter, drv, adapter->netdev,
  2126. "Keep alive watchdog timeout.\n");
  2127. u64_stats_update_begin(&adapter->syncp);
  2128. adapter->dev_stats.wd_expired++;
  2129. u64_stats_update_end(&adapter->syncp);
  2130. set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
  2131. }
  2132. }
  2133. static void check_for_admin_com_state(struct ena_adapter *adapter)
  2134. {
  2135. if (unlikely(!ena_com_get_admin_running_state(adapter->ena_dev))) {
  2136. netif_err(adapter, drv, adapter->netdev,
  2137. "ENA admin queue is not in running state!\n");
  2138. u64_stats_update_begin(&adapter->syncp);
  2139. adapter->dev_stats.admin_q_pause++;
  2140. u64_stats_update_end(&adapter->syncp);
  2141. set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
  2142. }
  2143. }
  2144. static void ena_update_host_info(struct ena_admin_host_info *host_info,
  2145. struct net_device *netdev)
  2146. {
  2147. host_info->supported_network_features[0] =
  2148. netdev->features & GENMASK_ULL(31, 0);
  2149. host_info->supported_network_features[1] =
  2150. (netdev->features & GENMASK_ULL(63, 32)) >> 32;
  2151. }
  2152. static void ena_timer_service(unsigned long data)
  2153. {
  2154. struct ena_adapter *adapter = (struct ena_adapter *)data;
  2155. u8 *debug_area = adapter->ena_dev->host_attr.debug_area_virt_addr;
  2156. struct ena_admin_host_info *host_info =
  2157. adapter->ena_dev->host_attr.host_info;
  2158. check_for_missing_keep_alive(adapter);
  2159. check_for_admin_com_state(adapter);
  2160. check_for_missing_tx_completions(adapter);
  2161. if (debug_area)
  2162. ena_dump_stats_to_buf(adapter, debug_area);
  2163. if (host_info)
  2164. ena_update_host_info(host_info, adapter->netdev);
  2165. if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
  2166. netif_err(adapter, drv, adapter->netdev,
  2167. "Trigger reset is on\n");
  2168. ena_dump_stats_to_dmesg(adapter);
  2169. queue_work(ena_wq, &adapter->reset_task);
  2170. return;
  2171. }
  2172. /* Reset the timer */
  2173. mod_timer(&adapter->timer_service, jiffies + HZ);
  2174. }
  2175. static int ena_calc_io_queue_num(struct pci_dev *pdev,
  2176. struct ena_com_dev *ena_dev,
  2177. struct ena_com_dev_get_features_ctx *get_feat_ctx)
  2178. {
  2179. int io_sq_num, io_queue_num;
  2180. /* In case of LLQ use the llq number in the get feature cmd */
  2181. if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
  2182. io_sq_num = get_feat_ctx->max_queues.max_llq_num;
  2183. if (io_sq_num == 0) {
  2184. dev_err(&pdev->dev,
  2185. "Trying to use LLQ but llq_num is 0. Fall back into regular queues\n");
  2186. ena_dev->tx_mem_queue_type =
  2187. ENA_ADMIN_PLACEMENT_POLICY_HOST;
  2188. io_sq_num = get_feat_ctx->max_queues.max_sq_num;
  2189. }
  2190. } else {
  2191. io_sq_num = get_feat_ctx->max_queues.max_sq_num;
  2192. }
  2193. io_queue_num = min_t(int, num_online_cpus(), ENA_MAX_NUM_IO_QUEUES);
  2194. io_queue_num = min_t(int, io_queue_num, io_sq_num);
  2195. io_queue_num = min_t(int, io_queue_num,
  2196. get_feat_ctx->max_queues.max_cq_num);
  2197. /* 1 IRQ for for mgmnt and 1 IRQs for each IO direction */
  2198. io_queue_num = min_t(int, io_queue_num, pci_msix_vec_count(pdev) - 1);
  2199. if (unlikely(!io_queue_num)) {
  2200. dev_err(&pdev->dev, "The device doesn't have io queues\n");
  2201. return -EFAULT;
  2202. }
  2203. return io_queue_num;
  2204. }
  2205. static void ena_set_push_mode(struct pci_dev *pdev, struct ena_com_dev *ena_dev,
  2206. struct ena_com_dev_get_features_ctx *get_feat_ctx)
  2207. {
  2208. bool has_mem_bar;
  2209. has_mem_bar = pci_select_bars(pdev, IORESOURCE_MEM) & BIT(ENA_MEM_BAR);
  2210. /* Enable push mode if device supports LLQ */
  2211. if (has_mem_bar && (get_feat_ctx->max_queues.max_llq_num > 0))
  2212. ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV;
  2213. else
  2214. ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
  2215. }
  2216. static void ena_set_dev_offloads(struct ena_com_dev_get_features_ctx *feat,
  2217. struct net_device *netdev)
  2218. {
  2219. netdev_features_t dev_features = 0;
  2220. /* Set offload features */
  2221. if (feat->offload.tx &
  2222. ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)
  2223. dev_features |= NETIF_F_IP_CSUM;
  2224. if (feat->offload.tx &
  2225. ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)
  2226. dev_features |= NETIF_F_IPV6_CSUM;
  2227. if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK)
  2228. dev_features |= NETIF_F_TSO;
  2229. if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK)
  2230. dev_features |= NETIF_F_TSO6;
  2231. if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK)
  2232. dev_features |= NETIF_F_TSO_ECN;
  2233. if (feat->offload.rx_supported &
  2234. ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK)
  2235. dev_features |= NETIF_F_RXCSUM;
  2236. if (feat->offload.rx_supported &
  2237. ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK)
  2238. dev_features |= NETIF_F_RXCSUM;
  2239. netdev->features =
  2240. dev_features |
  2241. NETIF_F_SG |
  2242. NETIF_F_RXHASH |
  2243. NETIF_F_HIGHDMA;
  2244. netdev->hw_features |= netdev->features;
  2245. netdev->vlan_features |= netdev->features;
  2246. }
  2247. static void ena_set_conf_feat_params(struct ena_adapter *adapter,
  2248. struct ena_com_dev_get_features_ctx *feat)
  2249. {
  2250. struct net_device *netdev = adapter->netdev;
  2251. /* Copy mac address */
  2252. if (!is_valid_ether_addr(feat->dev_attr.mac_addr)) {
  2253. eth_hw_addr_random(netdev);
  2254. ether_addr_copy(adapter->mac_addr, netdev->dev_addr);
  2255. } else {
  2256. ether_addr_copy(adapter->mac_addr, feat->dev_attr.mac_addr);
  2257. ether_addr_copy(netdev->dev_addr, adapter->mac_addr);
  2258. }
  2259. /* Set offload features */
  2260. ena_set_dev_offloads(feat, netdev);
  2261. adapter->max_mtu = feat->dev_attr.max_mtu;
  2262. netdev->max_mtu = adapter->max_mtu;
  2263. netdev->min_mtu = ENA_MIN_MTU;
  2264. }
  2265. static int ena_rss_init_default(struct ena_adapter *adapter)
  2266. {
  2267. struct ena_com_dev *ena_dev = adapter->ena_dev;
  2268. struct device *dev = &adapter->pdev->dev;
  2269. int rc, i;
  2270. u32 val;
  2271. rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE);
  2272. if (unlikely(rc)) {
  2273. dev_err(dev, "Cannot init indirect table\n");
  2274. goto err_rss_init;
  2275. }
  2276. for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
  2277. val = ethtool_rxfh_indir_default(i, adapter->num_queues);
  2278. rc = ena_com_indirect_table_fill_entry(ena_dev, i,
  2279. ENA_IO_RXQ_IDX(val));
  2280. if (unlikely(rc && (rc != -EPERM))) {
  2281. dev_err(dev, "Cannot fill indirect table\n");
  2282. goto err_fill_indir;
  2283. }
  2284. }
  2285. rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL,
  2286. ENA_HASH_KEY_SIZE, 0xFFFFFFFF);
  2287. if (unlikely(rc && (rc != -EPERM))) {
  2288. dev_err(dev, "Cannot fill hash function\n");
  2289. goto err_fill_indir;
  2290. }
  2291. rc = ena_com_set_default_hash_ctrl(ena_dev);
  2292. if (unlikely(rc && (rc != -EPERM))) {
  2293. dev_err(dev, "Cannot fill hash control\n");
  2294. goto err_fill_indir;
  2295. }
  2296. return 0;
  2297. err_fill_indir:
  2298. ena_com_rss_destroy(ena_dev);
  2299. err_rss_init:
  2300. return rc;
  2301. }
  2302. static void ena_release_bars(struct ena_com_dev *ena_dev, struct pci_dev *pdev)
  2303. {
  2304. int release_bars;
  2305. release_bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;
  2306. pci_release_selected_regions(pdev, release_bars);
  2307. }
  2308. static int ena_calc_queue_size(struct pci_dev *pdev,
  2309. struct ena_com_dev *ena_dev,
  2310. u16 *max_tx_sgl_size,
  2311. u16 *max_rx_sgl_size,
  2312. struct ena_com_dev_get_features_ctx *get_feat_ctx)
  2313. {
  2314. u32 queue_size = ENA_DEFAULT_RING_SIZE;
  2315. queue_size = min_t(u32, queue_size,
  2316. get_feat_ctx->max_queues.max_cq_depth);
  2317. queue_size = min_t(u32, queue_size,
  2318. get_feat_ctx->max_queues.max_sq_depth);
  2319. if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
  2320. queue_size = min_t(u32, queue_size,
  2321. get_feat_ctx->max_queues.max_llq_depth);
  2322. queue_size = rounddown_pow_of_two(queue_size);
  2323. if (unlikely(!queue_size)) {
  2324. dev_err(&pdev->dev, "Invalid queue size\n");
  2325. return -EFAULT;
  2326. }
  2327. *max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
  2328. get_feat_ctx->max_queues.max_packet_tx_descs);
  2329. *max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
  2330. get_feat_ctx->max_queues.max_packet_rx_descs);
  2331. return queue_size;
  2332. }
  2333. /* ena_probe - Device Initialization Routine
  2334. * @pdev: PCI device information struct
  2335. * @ent: entry in ena_pci_tbl
  2336. *
  2337. * Returns 0 on success, negative on failure
  2338. *
  2339. * ena_probe initializes an adapter identified by a pci_dev structure.
  2340. * The OS initialization, configuring of the adapter private structure,
  2341. * and a hardware reset occur.
  2342. */
  2343. static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
  2344. {
  2345. struct ena_com_dev_get_features_ctx get_feat_ctx;
  2346. static int version_printed;
  2347. struct net_device *netdev;
  2348. struct ena_adapter *adapter;
  2349. struct ena_com_dev *ena_dev = NULL;
  2350. static int adapters_found;
  2351. int io_queue_num, bars, rc;
  2352. int queue_size;
  2353. u16 tx_sgl_size = 0;
  2354. u16 rx_sgl_size = 0;
  2355. bool wd_state;
  2356. dev_dbg(&pdev->dev, "%s\n", __func__);
  2357. if (version_printed++ == 0)
  2358. dev_info(&pdev->dev, "%s", version);
  2359. rc = pci_enable_device_mem(pdev);
  2360. if (rc) {
  2361. dev_err(&pdev->dev, "pci_enable_device_mem() failed!\n");
  2362. return rc;
  2363. }
  2364. pci_set_master(pdev);
  2365. ena_dev = vzalloc(sizeof(*ena_dev));
  2366. if (!ena_dev) {
  2367. rc = -ENOMEM;
  2368. goto err_disable_device;
  2369. }
  2370. bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;
  2371. rc = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
  2372. if (rc) {
  2373. dev_err(&pdev->dev, "pci_request_selected_regions failed %d\n",
  2374. rc);
  2375. goto err_free_ena_dev;
  2376. }
  2377. ena_dev->reg_bar = ioremap(pci_resource_start(pdev, ENA_REG_BAR),
  2378. pci_resource_len(pdev, ENA_REG_BAR));
  2379. if (!ena_dev->reg_bar) {
  2380. dev_err(&pdev->dev, "failed to remap regs bar\n");
  2381. rc = -EFAULT;
  2382. goto err_free_region;
  2383. }
  2384. ena_dev->dmadev = &pdev->dev;
  2385. rc = ena_device_init(ena_dev, pdev, &get_feat_ctx, &wd_state);
  2386. if (rc) {
  2387. dev_err(&pdev->dev, "ena device init failed\n");
  2388. if (rc == -ETIME)
  2389. rc = -EPROBE_DEFER;
  2390. goto err_free_region;
  2391. }
  2392. ena_set_push_mode(pdev, ena_dev, &get_feat_ctx);
  2393. if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
  2394. ena_dev->mem_bar = ioremap_wc(pci_resource_start(pdev, ENA_MEM_BAR),
  2395. pci_resource_len(pdev, ENA_MEM_BAR));
  2396. if (!ena_dev->mem_bar) {
  2397. rc = -EFAULT;
  2398. goto err_device_destroy;
  2399. }
  2400. }
  2401. /* initial Tx interrupt delay, Assumes 1 usec granularity.
  2402. * Updated during device initialization with the real granularity
  2403. */
  2404. ena_dev->intr_moder_tx_interval = ENA_INTR_INITIAL_TX_INTERVAL_USECS;
  2405. io_queue_num = ena_calc_io_queue_num(pdev, ena_dev, &get_feat_ctx);
  2406. queue_size = ena_calc_queue_size(pdev, ena_dev, &tx_sgl_size,
  2407. &rx_sgl_size, &get_feat_ctx);
  2408. if ((queue_size <= 0) || (io_queue_num <= 0)) {
  2409. rc = -EFAULT;
  2410. goto err_device_destroy;
  2411. }
  2412. dev_info(&pdev->dev, "creating %d io queues. queue size: %d\n",
  2413. io_queue_num, queue_size);
  2414. /* dev zeroed in init_etherdev */
  2415. netdev = alloc_etherdev_mq(sizeof(struct ena_adapter), io_queue_num);
  2416. if (!netdev) {
  2417. dev_err(&pdev->dev, "alloc_etherdev_mq failed\n");
  2418. rc = -ENOMEM;
  2419. goto err_device_destroy;
  2420. }
  2421. SET_NETDEV_DEV(netdev, &pdev->dev);
  2422. adapter = netdev_priv(netdev);
  2423. pci_set_drvdata(pdev, adapter);
  2424. adapter->ena_dev = ena_dev;
  2425. adapter->netdev = netdev;
  2426. adapter->pdev = pdev;
  2427. ena_set_conf_feat_params(adapter, &get_feat_ctx);
  2428. adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
  2429. adapter->tx_ring_size = queue_size;
  2430. adapter->rx_ring_size = queue_size;
  2431. adapter->max_tx_sgl_size = tx_sgl_size;
  2432. adapter->max_rx_sgl_size = rx_sgl_size;
  2433. adapter->num_queues = io_queue_num;
  2434. adapter->last_monitored_tx_qid = 0;
  2435. adapter->rx_copybreak = ENA_DEFAULT_RX_COPYBREAK;
  2436. adapter->wd_state = wd_state;
  2437. snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", adapters_found);
  2438. rc = ena_com_init_interrupt_moderation(adapter->ena_dev);
  2439. if (rc) {
  2440. dev_err(&pdev->dev,
  2441. "Failed to query interrupt moderation feature\n");
  2442. goto err_netdev_destroy;
  2443. }
  2444. ena_init_io_rings(adapter);
  2445. netdev->netdev_ops = &ena_netdev_ops;
  2446. netdev->watchdog_timeo = TX_TIMEOUT;
  2447. ena_set_ethtool_ops(netdev);
  2448. netdev->priv_flags |= IFF_UNICAST_FLT;
  2449. u64_stats_init(&adapter->syncp);
  2450. rc = ena_enable_msix_and_set_admin_interrupts(adapter, io_queue_num);
  2451. if (rc) {
  2452. dev_err(&pdev->dev,
  2453. "Failed to enable and set the admin interrupts\n");
  2454. goto err_worker_destroy;
  2455. }
  2456. rc = ena_rss_init_default(adapter);
  2457. if (rc && (rc != -EPERM)) {
  2458. dev_err(&pdev->dev, "Cannot init RSS rc: %d\n", rc);
  2459. goto err_free_msix;
  2460. }
  2461. ena_config_debug_area(adapter);
  2462. memcpy(adapter->netdev->perm_addr, adapter->mac_addr, netdev->addr_len);
  2463. netif_carrier_off(netdev);
  2464. rc = register_netdev(netdev);
  2465. if (rc) {
  2466. dev_err(&pdev->dev, "Cannot register net device\n");
  2467. goto err_rss;
  2468. }
  2469. INIT_WORK(&adapter->suspend_io_task, ena_device_io_suspend);
  2470. INIT_WORK(&adapter->resume_io_task, ena_device_io_resume);
  2471. INIT_WORK(&adapter->reset_task, ena_fw_reset_device);
  2472. adapter->last_keep_alive_jiffies = jiffies;
  2473. setup_timer(&adapter->timer_service, ena_timer_service,
  2474. (unsigned long)adapter);
  2475. mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
  2476. dev_info(&pdev->dev, "%s found at mem %lx, mac addr %pM Queues %d\n",
  2477. DEVICE_NAME, (long)pci_resource_start(pdev, 0),
  2478. netdev->dev_addr, io_queue_num);
  2479. set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
  2480. adapters_found++;
  2481. return 0;
  2482. err_rss:
  2483. ena_com_delete_debug_area(ena_dev);
  2484. ena_com_rss_destroy(ena_dev);
  2485. err_free_msix:
  2486. ena_com_dev_reset(ena_dev);
  2487. ena_free_mgmnt_irq(adapter);
  2488. pci_free_irq_vectors(adapter->pdev);
  2489. err_worker_destroy:
  2490. ena_com_destroy_interrupt_moderation(ena_dev);
  2491. del_timer(&adapter->timer_service);
  2492. cancel_work_sync(&adapter->suspend_io_task);
  2493. cancel_work_sync(&adapter->resume_io_task);
  2494. err_netdev_destroy:
  2495. free_netdev(netdev);
  2496. err_device_destroy:
  2497. ena_com_delete_host_info(ena_dev);
  2498. ena_com_admin_destroy(ena_dev);
  2499. err_free_region:
  2500. ena_release_bars(ena_dev, pdev);
  2501. err_free_ena_dev:
  2502. vfree(ena_dev);
  2503. err_disable_device:
  2504. pci_disable_device(pdev);
  2505. return rc;
  2506. }
  2507. /*****************************************************************************/
  2508. static int ena_sriov_configure(struct pci_dev *dev, int numvfs)
  2509. {
  2510. int rc;
  2511. if (numvfs > 0) {
  2512. rc = pci_enable_sriov(dev, numvfs);
  2513. if (rc != 0) {
  2514. dev_err(&dev->dev,
  2515. "pci_enable_sriov failed to enable: %d vfs with the error: %d\n",
  2516. numvfs, rc);
  2517. return rc;
  2518. }
  2519. return numvfs;
  2520. }
  2521. if (numvfs == 0) {
  2522. pci_disable_sriov(dev);
  2523. return 0;
  2524. }
  2525. return -EINVAL;
  2526. }
  2527. /*****************************************************************************/
  2528. /*****************************************************************************/
  2529. /* ena_remove - Device Removal Routine
  2530. * @pdev: PCI device information struct
  2531. *
  2532. * ena_remove is called by the PCI subsystem to alert the driver
  2533. * that it should release a PCI device.
  2534. */
  2535. static void ena_remove(struct pci_dev *pdev)
  2536. {
  2537. struct ena_adapter *adapter = pci_get_drvdata(pdev);
  2538. struct ena_com_dev *ena_dev;
  2539. struct net_device *netdev;
  2540. ena_dev = adapter->ena_dev;
  2541. netdev = adapter->netdev;
  2542. #ifdef CONFIG_RFS_ACCEL
  2543. if ((adapter->msix_vecs >= 1) && (netdev->rx_cpu_rmap)) {
  2544. free_irq_cpu_rmap(netdev->rx_cpu_rmap);
  2545. netdev->rx_cpu_rmap = NULL;
  2546. }
  2547. #endif /* CONFIG_RFS_ACCEL */
  2548. unregister_netdev(netdev);
  2549. del_timer_sync(&adapter->timer_service);
  2550. cancel_work_sync(&adapter->reset_task);
  2551. cancel_work_sync(&adapter->suspend_io_task);
  2552. cancel_work_sync(&adapter->resume_io_task);
  2553. /* Reset the device only if the device is running. */
  2554. if (test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))
  2555. ena_com_dev_reset(ena_dev);
  2556. ena_free_mgmnt_irq(adapter);
  2557. pci_free_irq_vectors(adapter->pdev);
  2558. free_netdev(netdev);
  2559. ena_com_mmio_reg_read_request_destroy(ena_dev);
  2560. ena_com_abort_admin_commands(ena_dev);
  2561. ena_com_wait_for_abort_completion(ena_dev);
  2562. ena_com_admin_destroy(ena_dev);
  2563. ena_com_rss_destroy(ena_dev);
  2564. ena_com_delete_debug_area(ena_dev);
  2565. ena_com_delete_host_info(ena_dev);
  2566. ena_release_bars(ena_dev, pdev);
  2567. pci_disable_device(pdev);
  2568. ena_com_destroy_interrupt_moderation(ena_dev);
  2569. vfree(ena_dev);
  2570. }
  2571. static struct pci_driver ena_pci_driver = {
  2572. .name = DRV_MODULE_NAME,
  2573. .id_table = ena_pci_tbl,
  2574. .probe = ena_probe,
  2575. .remove = ena_remove,
  2576. .sriov_configure = ena_sriov_configure,
  2577. };
  2578. static int __init ena_init(void)
  2579. {
  2580. pr_info("%s", version);
  2581. ena_wq = create_singlethread_workqueue(DRV_MODULE_NAME);
  2582. if (!ena_wq) {
  2583. pr_err("Failed to create workqueue\n");
  2584. return -ENOMEM;
  2585. }
  2586. return pci_register_driver(&ena_pci_driver);
  2587. }
  2588. static void __exit ena_cleanup(void)
  2589. {
  2590. pci_unregister_driver(&ena_pci_driver);
  2591. if (ena_wq) {
  2592. destroy_workqueue(ena_wq);
  2593. ena_wq = NULL;
  2594. }
  2595. }
  2596. /******************************************************************************
  2597. ******************************** AENQ Handlers *******************************
  2598. *****************************************************************************/
  2599. /* ena_update_on_link_change:
  2600. * Notify the network interface about the change in link status
  2601. */
  2602. static void ena_update_on_link_change(void *adapter_data,
  2603. struct ena_admin_aenq_entry *aenq_e)
  2604. {
  2605. struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
  2606. struct ena_admin_aenq_link_change_desc *aenq_desc =
  2607. (struct ena_admin_aenq_link_change_desc *)aenq_e;
  2608. int status = aenq_desc->flags &
  2609. ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK;
  2610. if (status) {
  2611. netdev_dbg(adapter->netdev, "%s\n", __func__);
  2612. set_bit(ENA_FLAG_LINK_UP, &adapter->flags);
  2613. netif_carrier_on(adapter->netdev);
  2614. } else {
  2615. clear_bit(ENA_FLAG_LINK_UP, &adapter->flags);
  2616. netif_carrier_off(adapter->netdev);
  2617. }
  2618. }
  2619. static void ena_keep_alive_wd(void *adapter_data,
  2620. struct ena_admin_aenq_entry *aenq_e)
  2621. {
  2622. struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
  2623. adapter->last_keep_alive_jiffies = jiffies;
  2624. }
  2625. static void ena_notification(void *adapter_data,
  2626. struct ena_admin_aenq_entry *aenq_e)
  2627. {
  2628. struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
  2629. WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION,
  2630. "Invalid group(%x) expected %x\n",
  2631. aenq_e->aenq_common_desc.group,
  2632. ENA_ADMIN_NOTIFICATION);
  2633. switch (aenq_e->aenq_common_desc.syndrom) {
  2634. case ENA_ADMIN_SUSPEND:
  2635. /* Suspend just the IO queues.
  2636. * We deliberately don't suspend admin so the timer and
  2637. * the keep_alive events should remain.
  2638. */
  2639. queue_work(ena_wq, &adapter->suspend_io_task);
  2640. break;
  2641. case ENA_ADMIN_RESUME:
  2642. queue_work(ena_wq, &adapter->resume_io_task);
  2643. break;
  2644. default:
  2645. netif_err(adapter, drv, adapter->netdev,
  2646. "Invalid aenq notification link state %d\n",
  2647. aenq_e->aenq_common_desc.syndrom);
  2648. }
  2649. }
  2650. /* This handler will called for unknown event group or unimplemented handlers*/
  2651. static void unimplemented_aenq_handler(void *data,
  2652. struct ena_admin_aenq_entry *aenq_e)
  2653. {
  2654. struct ena_adapter *adapter = (struct ena_adapter *)data;
  2655. netif_err(adapter, drv, adapter->netdev,
  2656. "Unknown event was received or event with unimplemented handler\n");
  2657. }
  2658. static struct ena_aenq_handlers aenq_handlers = {
  2659. .handlers = {
  2660. [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change,
  2661. [ENA_ADMIN_NOTIFICATION] = ena_notification,
  2662. [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd,
  2663. },
  2664. .unimplemented_handler = unimplemented_aenq_handler
  2665. };
  2666. module_init(ena_init);
  2667. module_exit(ena_cleanup);