dpaa_eth.c 77 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948
  1. /* Copyright 2008 - 2016 Freescale Semiconductor Inc.
  2. *
  3. * Redistribution and use in source and binary forms, with or without
  4. * modification, are permitted provided that the following conditions are met:
  5. * * Redistributions of source code must retain the above copyright
  6. * notice, this list of conditions and the following disclaimer.
  7. * * Redistributions in binary form must reproduce the above copyright
  8. * notice, this list of conditions and the following disclaimer in the
  9. * documentation and/or other materials provided with the distribution.
  10. * * Neither the name of Freescale Semiconductor nor the
  11. * names of its contributors may be used to endorse or promote products
  12. * derived from this software without specific prior written permission.
  13. *
  14. * ALTERNATIVELY, this software may be distributed under the terms of the
  15. * GNU General Public License ("GPL") as published by the Free Software
  16. * Foundation, either version 2 of that License or (at your option) any
  17. * later version.
  18. *
  19. * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
  20. * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  21. * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  22. * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
  23. * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  24. * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  25. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  26. * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  28. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29. */
  30. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  31. #include <linux/init.h>
  32. #include <linux/module.h>
  33. #include <linux/of_platform.h>
  34. #include <linux/of_mdio.h>
  35. #include <linux/of_net.h>
  36. #include <linux/io.h>
  37. #include <linux/if_arp.h>
  38. #include <linux/if_vlan.h>
  39. #include <linux/icmp.h>
  40. #include <linux/ip.h>
  41. #include <linux/ipv6.h>
  42. #include <linux/udp.h>
  43. #include <linux/tcp.h>
  44. #include <linux/net.h>
  45. #include <linux/skbuff.h>
  46. #include <linux/etherdevice.h>
  47. #include <linux/if_ether.h>
  48. #include <linux/highmem.h>
  49. #include <linux/percpu.h>
  50. #include <linux/dma-mapping.h>
  51. #include <linux/sort.h>
  52. #include <soc/fsl/bman.h>
  53. #include <soc/fsl/qman.h>
  54. #include "fman.h"
  55. #include "fman_port.h"
  56. #include "mac.h"
  57. #include "dpaa_eth.h"
  58. /* CREATE_TRACE_POINTS only needs to be defined once. Other dpaa files
  59. * using trace events only need to #include <trace/events/sched.h>
  60. */
  61. #define CREATE_TRACE_POINTS
  62. #include "dpaa_eth_trace.h"
  63. static int debug = -1;
  64. module_param(debug, int, 0444);
  65. MODULE_PARM_DESC(debug, "Module/Driver verbosity level (0=none,...,16=all)");
  66. static u16 tx_timeout = 1000;
  67. module_param(tx_timeout, ushort, 0444);
  68. MODULE_PARM_DESC(tx_timeout, "The Tx timeout in ms");
  69. #define FM_FD_STAT_RX_ERRORS \
  70. (FM_FD_ERR_DMA | FM_FD_ERR_PHYSICAL | \
  71. FM_FD_ERR_SIZE | FM_FD_ERR_CLS_DISCARD | \
  72. FM_FD_ERR_EXTRACTION | FM_FD_ERR_NO_SCHEME | \
  73. FM_FD_ERR_PRS_TIMEOUT | FM_FD_ERR_PRS_ILL_INSTRUCT | \
  74. FM_FD_ERR_PRS_HDR_ERR)
  75. #define FM_FD_STAT_TX_ERRORS \
  76. (FM_FD_ERR_UNSUPPORTED_FORMAT | \
  77. FM_FD_ERR_LENGTH | FM_FD_ERR_DMA)
  78. #define DPAA_MSG_DEFAULT (NETIF_MSG_DRV | NETIF_MSG_PROBE | \
  79. NETIF_MSG_LINK | NETIF_MSG_IFUP | \
  80. NETIF_MSG_IFDOWN)
  81. #define DPAA_INGRESS_CS_THRESHOLD 0x10000000
  82. /* Ingress congestion threshold on FMan ports
  83. * The size in bytes of the ingress tail-drop threshold on FMan ports.
  84. * Traffic piling up above this value will be rejected by QMan and discarded
  85. * by FMan.
  86. */
  87. /* Size in bytes of the FQ taildrop threshold */
  88. #define DPAA_FQ_TD 0x200000
  89. #define DPAA_CS_THRESHOLD_1G 0x06000000
  90. /* Egress congestion threshold on 1G ports, range 0x1000 .. 0x10000000
  91. * The size in bytes of the egress Congestion State notification threshold on
  92. * 1G ports. The 1G dTSECs can quite easily be flooded by cores doing Tx in a
  93. * tight loop (e.g. by sending UDP datagrams at "while(1) speed"),
  94. * and the larger the frame size, the more acute the problem.
  95. * So we have to find a balance between these factors:
  96. * - avoiding the device staying congested for a prolonged time (risking
  97. * the netdev watchdog to fire - see also the tx_timeout module param);
  98. * - affecting performance of protocols such as TCP, which otherwise
  99. * behave well under the congestion notification mechanism;
  100. * - preventing the Tx cores from tightly-looping (as if the congestion
  101. * threshold was too low to be effective);
  102. * - running out of memory if the CS threshold is set too high.
  103. */
  104. #define DPAA_CS_THRESHOLD_10G 0x10000000
  105. /* The size in bytes of the egress Congestion State notification threshold on
  106. * 10G ports, range 0x1000 .. 0x10000000
  107. */
  108. /* Largest value that the FQD's OAL field can hold */
  109. #define FSL_QMAN_MAX_OAL 127
  110. /* Default alignment for start of data in an Rx FD */
  111. #define DPAA_FD_DATA_ALIGNMENT 16
  112. /* Values for the L3R field of the FM Parse Results
  113. */
  114. /* L3 Type field: First IP Present IPv4 */
  115. #define FM_L3_PARSE_RESULT_IPV4 0x8000
  116. /* L3 Type field: First IP Present IPv6 */
  117. #define FM_L3_PARSE_RESULT_IPV6 0x4000
  118. /* Values for the L4R field of the FM Parse Results */
  119. /* L4 Type field: UDP */
  120. #define FM_L4_PARSE_RESULT_UDP 0x40
  121. /* L4 Type field: TCP */
  122. #define FM_L4_PARSE_RESULT_TCP 0x20
  123. /* FD status field indicating whether the FM Parser has attempted to validate
  124. * the L4 csum of the frame.
  125. * Note that having this bit set doesn't necessarily imply that the checksum
  126. * is valid. One would have to check the parse results to find that out.
  127. */
  128. #define FM_FD_STAT_L4CV 0x00000004
  129. #define DPAA_SGT_MAX_ENTRIES 16 /* maximum number of entries in SG Table */
  130. #define DPAA_BUFF_RELEASE_MAX 8 /* maximum number of buffers released at once */
  131. #define FSL_DPAA_BPID_INV 0xff
  132. #define FSL_DPAA_ETH_MAX_BUF_COUNT 128
  133. #define FSL_DPAA_ETH_REFILL_THRESHOLD 80
  134. #define DPAA_TX_PRIV_DATA_SIZE 16
  135. #define DPAA_PARSE_RESULTS_SIZE sizeof(struct fman_prs_result)
  136. #define DPAA_TIME_STAMP_SIZE 8
  137. #define DPAA_HASH_RESULTS_SIZE 8
  138. #define DPAA_RX_PRIV_DATA_SIZE (u16)(DPAA_TX_PRIV_DATA_SIZE + \
  139. dpaa_rx_extra_headroom)
  140. #define DPAA_ETH_PCD_RXQ_NUM 128
  141. #define DPAA_ENQUEUE_RETRIES 100000
  142. enum port_type {RX, TX};
  143. struct fm_port_fqs {
  144. struct dpaa_fq *tx_defq;
  145. struct dpaa_fq *tx_errq;
  146. struct dpaa_fq *rx_defq;
  147. struct dpaa_fq *rx_errq;
  148. struct dpaa_fq *rx_pcdq;
  149. };
  150. /* All the dpa bps in use at any moment */
  151. static struct dpaa_bp *dpaa_bp_array[BM_MAX_NUM_OF_POOLS];
  152. /* The raw buffer size must be cacheline aligned */
  153. #define DPAA_BP_RAW_SIZE 4096
  154. /* When using more than one buffer pool, the raw sizes are as follows:
  155. * 1 bp: 4KB
  156. * 2 bp: 2KB, 4KB
  157. * 3 bp: 1KB, 2KB, 4KB
  158. * 4 bp: 1KB, 2KB, 4KB, 8KB
  159. */
  160. static inline size_t bpool_buffer_raw_size(u8 index, u8 cnt)
  161. {
  162. size_t res = DPAA_BP_RAW_SIZE / 4;
  163. u8 i;
  164. for (i = (cnt < 3) ? cnt : 3; i < 3 + index; i++)
  165. res *= 2;
  166. return res;
  167. }
  168. /* FMan-DMA requires 16-byte alignment for Rx buffers, but SKB_DATA_ALIGN is
  169. * even stronger (SMP_CACHE_BYTES-aligned), so we just get away with that,
  170. * via SKB_WITH_OVERHEAD(). We can't rely on netdev_alloc_frag() giving us
  171. * half-page-aligned buffers, so we reserve some more space for start-of-buffer
  172. * alignment.
  173. */
  174. #define dpaa_bp_size(raw_size) SKB_WITH_OVERHEAD((raw_size) - SMP_CACHE_BYTES)
  175. static int dpaa_max_frm;
  176. static int dpaa_rx_extra_headroom;
  177. #define dpaa_get_max_mtu() \
  178. (dpaa_max_frm - (VLAN_ETH_HLEN + ETH_FCS_LEN))
  179. static int dpaa_netdev_init(struct net_device *net_dev,
  180. const struct net_device_ops *dpaa_ops,
  181. u16 tx_timeout)
  182. {
  183. struct dpaa_priv *priv = netdev_priv(net_dev);
  184. struct device *dev = net_dev->dev.parent;
  185. struct dpaa_percpu_priv *percpu_priv;
  186. const u8 *mac_addr;
  187. int i, err;
  188. /* Although we access another CPU's private data here
  189. * we do it at initialization so it is safe
  190. */
  191. for_each_possible_cpu(i) {
  192. percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
  193. percpu_priv->net_dev = net_dev;
  194. }
  195. net_dev->netdev_ops = dpaa_ops;
  196. mac_addr = priv->mac_dev->addr;
  197. net_dev->mem_start = priv->mac_dev->res->start;
  198. net_dev->mem_end = priv->mac_dev->res->end;
  199. net_dev->min_mtu = ETH_MIN_MTU;
  200. net_dev->max_mtu = dpaa_get_max_mtu();
  201. net_dev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
  202. NETIF_F_LLTX | NETIF_F_RXHASH);
  203. net_dev->hw_features |= NETIF_F_SG | NETIF_F_HIGHDMA;
  204. /* The kernels enables GSO automatically, if we declare NETIF_F_SG.
  205. * For conformity, we'll still declare GSO explicitly.
  206. */
  207. net_dev->features |= NETIF_F_GSO;
  208. net_dev->features |= NETIF_F_RXCSUM;
  209. net_dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
  210. /* we do not want shared skbs on TX */
  211. net_dev->priv_flags &= ~IFF_TX_SKB_SHARING;
  212. net_dev->features |= net_dev->hw_features;
  213. net_dev->vlan_features = net_dev->features;
  214. memcpy(net_dev->perm_addr, mac_addr, net_dev->addr_len);
  215. memcpy(net_dev->dev_addr, mac_addr, net_dev->addr_len);
  216. net_dev->ethtool_ops = &dpaa_ethtool_ops;
  217. net_dev->needed_headroom = priv->tx_headroom;
  218. net_dev->watchdog_timeo = msecs_to_jiffies(tx_timeout);
  219. /* start without the RUNNING flag, phylib controls it later */
  220. netif_carrier_off(net_dev);
  221. err = register_netdev(net_dev);
  222. if (err < 0) {
  223. dev_err(dev, "register_netdev() = %d\n", err);
  224. return err;
  225. }
  226. return 0;
  227. }
  228. static int dpaa_stop(struct net_device *net_dev)
  229. {
  230. struct mac_device *mac_dev;
  231. struct dpaa_priv *priv;
  232. int i, err, error;
  233. priv = netdev_priv(net_dev);
  234. mac_dev = priv->mac_dev;
  235. netif_tx_stop_all_queues(net_dev);
  236. /* Allow the Fman (Tx) port to process in-flight frames before we
  237. * try switching it off.
  238. */
  239. usleep_range(5000, 10000);
  240. err = mac_dev->stop(mac_dev);
  241. if (err < 0)
  242. netif_err(priv, ifdown, net_dev, "mac_dev->stop() = %d\n",
  243. err);
  244. for (i = 0; i < ARRAY_SIZE(mac_dev->port); i++) {
  245. error = fman_port_disable(mac_dev->port[i]);
  246. if (error)
  247. err = error;
  248. }
  249. if (net_dev->phydev)
  250. phy_disconnect(net_dev->phydev);
  251. net_dev->phydev = NULL;
  252. return err;
  253. }
  254. static void dpaa_tx_timeout(struct net_device *net_dev)
  255. {
  256. struct dpaa_percpu_priv *percpu_priv;
  257. const struct dpaa_priv *priv;
  258. priv = netdev_priv(net_dev);
  259. percpu_priv = this_cpu_ptr(priv->percpu_priv);
  260. netif_crit(priv, timer, net_dev, "Transmit timeout latency: %u ms\n",
  261. jiffies_to_msecs(jiffies - dev_trans_start(net_dev)));
  262. percpu_priv->stats.tx_errors++;
  263. }
  264. /* Calculates the statistics for the given device by adding the statistics
  265. * collected by each CPU.
  266. */
  267. static void dpaa_get_stats64(struct net_device *net_dev,
  268. struct rtnl_link_stats64 *s)
  269. {
  270. int numstats = sizeof(struct rtnl_link_stats64) / sizeof(u64);
  271. struct dpaa_priv *priv = netdev_priv(net_dev);
  272. struct dpaa_percpu_priv *percpu_priv;
  273. u64 *netstats = (u64 *)s;
  274. u64 *cpustats;
  275. int i, j;
  276. for_each_possible_cpu(i) {
  277. percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
  278. cpustats = (u64 *)&percpu_priv->stats;
  279. /* add stats from all CPUs */
  280. for (j = 0; j < numstats; j++)
  281. netstats[j] += cpustats[j];
  282. }
  283. }
  284. static int dpaa_setup_tc(struct net_device *net_dev, enum tc_setup_type type,
  285. void *type_data)
  286. {
  287. struct dpaa_priv *priv = netdev_priv(net_dev);
  288. struct tc_mqprio_qopt *mqprio = type_data;
  289. u8 num_tc;
  290. int i;
  291. if (type != TC_SETUP_QDISC_MQPRIO)
  292. return -EOPNOTSUPP;
  293. mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
  294. num_tc = mqprio->num_tc;
  295. if (num_tc == priv->num_tc)
  296. return 0;
  297. if (!num_tc) {
  298. netdev_reset_tc(net_dev);
  299. goto out;
  300. }
  301. if (num_tc > DPAA_TC_NUM) {
  302. netdev_err(net_dev, "Too many traffic classes: max %d supported.\n",
  303. DPAA_TC_NUM);
  304. return -EINVAL;
  305. }
  306. netdev_set_num_tc(net_dev, num_tc);
  307. for (i = 0; i < num_tc; i++)
  308. netdev_set_tc_queue(net_dev, i, DPAA_TC_TXQ_NUM,
  309. i * DPAA_TC_TXQ_NUM);
  310. out:
  311. priv->num_tc = num_tc ? : 1;
  312. netif_set_real_num_tx_queues(net_dev, priv->num_tc * DPAA_TC_TXQ_NUM);
  313. return 0;
  314. }
  315. static struct mac_device *dpaa_mac_dev_get(struct platform_device *pdev)
  316. {
  317. struct dpaa_eth_data *eth_data;
  318. struct device *dpaa_dev;
  319. struct mac_device *mac_dev;
  320. dpaa_dev = &pdev->dev;
  321. eth_data = dpaa_dev->platform_data;
  322. if (!eth_data) {
  323. dev_err(dpaa_dev, "eth_data missing\n");
  324. return ERR_PTR(-ENODEV);
  325. }
  326. mac_dev = eth_data->mac_dev;
  327. if (!mac_dev) {
  328. dev_err(dpaa_dev, "mac_dev missing\n");
  329. return ERR_PTR(-EINVAL);
  330. }
  331. return mac_dev;
  332. }
  333. static int dpaa_set_mac_address(struct net_device *net_dev, void *addr)
  334. {
  335. const struct dpaa_priv *priv;
  336. struct mac_device *mac_dev;
  337. struct sockaddr old_addr;
  338. int err;
  339. priv = netdev_priv(net_dev);
  340. memcpy(old_addr.sa_data, net_dev->dev_addr, ETH_ALEN);
  341. err = eth_mac_addr(net_dev, addr);
  342. if (err < 0) {
  343. netif_err(priv, drv, net_dev, "eth_mac_addr() = %d\n", err);
  344. return err;
  345. }
  346. mac_dev = priv->mac_dev;
  347. err = mac_dev->change_addr(mac_dev->fman_mac,
  348. (enet_addr_t *)net_dev->dev_addr);
  349. if (err < 0) {
  350. netif_err(priv, drv, net_dev, "mac_dev->change_addr() = %d\n",
  351. err);
  352. /* reverting to previous address */
  353. eth_mac_addr(net_dev, &old_addr);
  354. return err;
  355. }
  356. return 0;
  357. }
  358. static void dpaa_set_rx_mode(struct net_device *net_dev)
  359. {
  360. const struct dpaa_priv *priv;
  361. int err;
  362. priv = netdev_priv(net_dev);
  363. if (!!(net_dev->flags & IFF_PROMISC) != priv->mac_dev->promisc) {
  364. priv->mac_dev->promisc = !priv->mac_dev->promisc;
  365. err = priv->mac_dev->set_promisc(priv->mac_dev->fman_mac,
  366. priv->mac_dev->promisc);
  367. if (err < 0)
  368. netif_err(priv, drv, net_dev,
  369. "mac_dev->set_promisc() = %d\n",
  370. err);
  371. }
  372. if (!!(net_dev->flags & IFF_ALLMULTI) != priv->mac_dev->allmulti) {
  373. priv->mac_dev->allmulti = !priv->mac_dev->allmulti;
  374. err = priv->mac_dev->set_allmulti(priv->mac_dev->fman_mac,
  375. priv->mac_dev->allmulti);
  376. if (err < 0)
  377. netif_err(priv, drv, net_dev,
  378. "mac_dev->set_allmulti() = %d\n",
  379. err);
  380. }
  381. err = priv->mac_dev->set_multi(net_dev, priv->mac_dev);
  382. if (err < 0)
  383. netif_err(priv, drv, net_dev, "mac_dev->set_multi() = %d\n",
  384. err);
  385. }
  386. static struct dpaa_bp *dpaa_bpid2pool(int bpid)
  387. {
  388. if (WARN_ON(bpid < 0 || bpid >= BM_MAX_NUM_OF_POOLS))
  389. return NULL;
  390. return dpaa_bp_array[bpid];
  391. }
  392. /* checks if this bpool is already allocated */
  393. static bool dpaa_bpid2pool_use(int bpid)
  394. {
  395. if (dpaa_bpid2pool(bpid)) {
  396. atomic_inc(&dpaa_bp_array[bpid]->refs);
  397. return true;
  398. }
  399. return false;
  400. }
  401. /* called only once per bpid by dpaa_bp_alloc_pool() */
  402. static void dpaa_bpid2pool_map(int bpid, struct dpaa_bp *dpaa_bp)
  403. {
  404. dpaa_bp_array[bpid] = dpaa_bp;
  405. atomic_set(&dpaa_bp->refs, 1);
  406. }
  407. static int dpaa_bp_alloc_pool(struct dpaa_bp *dpaa_bp)
  408. {
  409. int err;
  410. if (dpaa_bp->size == 0 || dpaa_bp->config_count == 0) {
  411. pr_err("%s: Buffer pool is not properly initialized! Missing size or initial number of buffers\n",
  412. __func__);
  413. return -EINVAL;
  414. }
  415. /* If the pool is already specified, we only create one per bpid */
  416. if (dpaa_bp->bpid != FSL_DPAA_BPID_INV &&
  417. dpaa_bpid2pool_use(dpaa_bp->bpid))
  418. return 0;
  419. if (dpaa_bp->bpid == FSL_DPAA_BPID_INV) {
  420. dpaa_bp->pool = bman_new_pool();
  421. if (!dpaa_bp->pool) {
  422. pr_err("%s: bman_new_pool() failed\n",
  423. __func__);
  424. return -ENODEV;
  425. }
  426. dpaa_bp->bpid = (u8)bman_get_bpid(dpaa_bp->pool);
  427. }
  428. if (dpaa_bp->seed_cb) {
  429. err = dpaa_bp->seed_cb(dpaa_bp);
  430. if (err)
  431. goto pool_seed_failed;
  432. }
  433. dpaa_bpid2pool_map(dpaa_bp->bpid, dpaa_bp);
  434. return 0;
  435. pool_seed_failed:
  436. pr_err("%s: pool seeding failed\n", __func__);
  437. bman_free_pool(dpaa_bp->pool);
  438. return err;
  439. }
  440. /* remove and free all the buffers from the given buffer pool */
  441. static void dpaa_bp_drain(struct dpaa_bp *bp)
  442. {
  443. u8 num = 8;
  444. int ret;
  445. do {
  446. struct bm_buffer bmb[8];
  447. int i;
  448. ret = bman_acquire(bp->pool, bmb, num);
  449. if (ret < 0) {
  450. if (num == 8) {
  451. /* we have less than 8 buffers left;
  452. * drain them one by one
  453. */
  454. num = 1;
  455. ret = 1;
  456. continue;
  457. } else {
  458. /* Pool is fully drained */
  459. break;
  460. }
  461. }
  462. if (bp->free_buf_cb)
  463. for (i = 0; i < num; i++)
  464. bp->free_buf_cb(bp, &bmb[i]);
  465. } while (ret > 0);
  466. }
  467. static void dpaa_bp_free(struct dpaa_bp *dpaa_bp)
  468. {
  469. struct dpaa_bp *bp = dpaa_bpid2pool(dpaa_bp->bpid);
  470. /* the mapping between bpid and dpaa_bp is done very late in the
  471. * allocation procedure; if something failed before the mapping, the bp
  472. * was not configured, therefore we don't need the below instructions
  473. */
  474. if (!bp)
  475. return;
  476. if (!atomic_dec_and_test(&bp->refs))
  477. return;
  478. if (bp->free_buf_cb)
  479. dpaa_bp_drain(bp);
  480. dpaa_bp_array[bp->bpid] = NULL;
  481. bman_free_pool(bp->pool);
  482. }
  483. static void dpaa_bps_free(struct dpaa_priv *priv)
  484. {
  485. int i;
  486. for (i = 0; i < DPAA_BPS_NUM; i++)
  487. dpaa_bp_free(priv->dpaa_bps[i]);
  488. }
  489. /* Use multiple WQs for FQ assignment:
  490. * - Tx Confirmation queues go to WQ1.
  491. * - Rx Error and Tx Error queues go to WQ5 (giving them a better chance
  492. * to be scheduled, in case there are many more FQs in WQ6).
  493. * - Rx Default goes to WQ6.
  494. * - Tx queues go to different WQs depending on their priority. Equal
  495. * chunks of NR_CPUS queues go to WQ6 (lowest priority), WQ2, WQ1 and
  496. * WQ0 (highest priority).
  497. * This ensures that Tx-confirmed buffers are timely released. In particular,
  498. * it avoids congestion on the Tx Confirm FQs, which can pile up PFDRs if they
  499. * are greatly outnumbered by other FQs in the system, while
  500. * dequeue scheduling is round-robin.
  501. */
  502. static inline void dpaa_assign_wq(struct dpaa_fq *fq, int idx)
  503. {
  504. switch (fq->fq_type) {
  505. case FQ_TYPE_TX_CONFIRM:
  506. case FQ_TYPE_TX_CONF_MQ:
  507. fq->wq = 1;
  508. break;
  509. case FQ_TYPE_RX_ERROR:
  510. case FQ_TYPE_TX_ERROR:
  511. fq->wq = 5;
  512. break;
  513. case FQ_TYPE_RX_DEFAULT:
  514. case FQ_TYPE_RX_PCD:
  515. fq->wq = 6;
  516. break;
  517. case FQ_TYPE_TX:
  518. switch (idx / DPAA_TC_TXQ_NUM) {
  519. case 0:
  520. /* Low priority (best effort) */
  521. fq->wq = 6;
  522. break;
  523. case 1:
  524. /* Medium priority */
  525. fq->wq = 2;
  526. break;
  527. case 2:
  528. /* High priority */
  529. fq->wq = 1;
  530. break;
  531. case 3:
  532. /* Very high priority */
  533. fq->wq = 0;
  534. break;
  535. default:
  536. WARN(1, "Too many TX FQs: more than %d!\n",
  537. DPAA_ETH_TXQ_NUM);
  538. }
  539. break;
  540. default:
  541. WARN(1, "Invalid FQ type %d for FQID %d!\n",
  542. fq->fq_type, fq->fqid);
  543. }
  544. }
  545. static struct dpaa_fq *dpaa_fq_alloc(struct device *dev,
  546. u32 start, u32 count,
  547. struct list_head *list,
  548. enum dpaa_fq_type fq_type)
  549. {
  550. struct dpaa_fq *dpaa_fq;
  551. int i;
  552. dpaa_fq = devm_kzalloc(dev, sizeof(*dpaa_fq) * count,
  553. GFP_KERNEL);
  554. if (!dpaa_fq)
  555. return NULL;
  556. for (i = 0; i < count; i++) {
  557. dpaa_fq[i].fq_type = fq_type;
  558. dpaa_fq[i].fqid = start ? start + i : 0;
  559. list_add_tail(&dpaa_fq[i].list, list);
  560. }
  561. for (i = 0; i < count; i++)
  562. dpaa_assign_wq(dpaa_fq + i, i);
  563. return dpaa_fq;
  564. }
  565. static int dpaa_alloc_all_fqs(struct device *dev, struct list_head *list,
  566. struct fm_port_fqs *port_fqs)
  567. {
  568. struct dpaa_fq *dpaa_fq;
  569. u32 fq_base, fq_base_aligned, i;
  570. dpaa_fq = dpaa_fq_alloc(dev, 0, 1, list, FQ_TYPE_RX_ERROR);
  571. if (!dpaa_fq)
  572. goto fq_alloc_failed;
  573. port_fqs->rx_errq = &dpaa_fq[0];
  574. dpaa_fq = dpaa_fq_alloc(dev, 0, 1, list, FQ_TYPE_RX_DEFAULT);
  575. if (!dpaa_fq)
  576. goto fq_alloc_failed;
  577. port_fqs->rx_defq = &dpaa_fq[0];
  578. /* the PCD FQIDs range needs to be aligned for correct operation */
  579. if (qman_alloc_fqid_range(&fq_base, 2 * DPAA_ETH_PCD_RXQ_NUM))
  580. goto fq_alloc_failed;
  581. fq_base_aligned = ALIGN(fq_base, DPAA_ETH_PCD_RXQ_NUM);
  582. for (i = fq_base; i < fq_base_aligned; i++)
  583. qman_release_fqid(i);
  584. for (i = fq_base_aligned + DPAA_ETH_PCD_RXQ_NUM;
  585. i < (fq_base + 2 * DPAA_ETH_PCD_RXQ_NUM); i++)
  586. qman_release_fqid(i);
  587. dpaa_fq = dpaa_fq_alloc(dev, fq_base_aligned, DPAA_ETH_PCD_RXQ_NUM,
  588. list, FQ_TYPE_RX_PCD);
  589. if (!dpaa_fq)
  590. goto fq_alloc_failed;
  591. port_fqs->rx_pcdq = &dpaa_fq[0];
  592. if (!dpaa_fq_alloc(dev, 0, DPAA_ETH_TXQ_NUM, list, FQ_TYPE_TX_CONF_MQ))
  593. goto fq_alloc_failed;
  594. dpaa_fq = dpaa_fq_alloc(dev, 0, 1, list, FQ_TYPE_TX_ERROR);
  595. if (!dpaa_fq)
  596. goto fq_alloc_failed;
  597. port_fqs->tx_errq = &dpaa_fq[0];
  598. dpaa_fq = dpaa_fq_alloc(dev, 0, 1, list, FQ_TYPE_TX_CONFIRM);
  599. if (!dpaa_fq)
  600. goto fq_alloc_failed;
  601. port_fqs->tx_defq = &dpaa_fq[0];
  602. if (!dpaa_fq_alloc(dev, 0, DPAA_ETH_TXQ_NUM, list, FQ_TYPE_TX))
  603. goto fq_alloc_failed;
  604. return 0;
  605. fq_alloc_failed:
  606. dev_err(dev, "dpaa_fq_alloc() failed\n");
  607. return -ENOMEM;
  608. }
  609. static u32 rx_pool_channel;
  610. static DEFINE_SPINLOCK(rx_pool_channel_init);
  611. static int dpaa_get_channel(void)
  612. {
  613. spin_lock(&rx_pool_channel_init);
  614. if (!rx_pool_channel) {
  615. u32 pool;
  616. int ret;
  617. ret = qman_alloc_pool(&pool);
  618. if (!ret)
  619. rx_pool_channel = pool;
  620. }
  621. spin_unlock(&rx_pool_channel_init);
  622. if (!rx_pool_channel)
  623. return -ENOMEM;
  624. return rx_pool_channel;
  625. }
  626. static void dpaa_release_channel(void)
  627. {
  628. qman_release_pool(rx_pool_channel);
  629. }
  630. static void dpaa_eth_add_channel(u16 channel)
  631. {
  632. u32 pool = QM_SDQCR_CHANNELS_POOL_CONV(channel);
  633. const cpumask_t *cpus = qman_affine_cpus();
  634. struct qman_portal *portal;
  635. int cpu;
  636. for_each_cpu(cpu, cpus) {
  637. portal = qman_get_affine_portal(cpu);
  638. qman_p_static_dequeue_add(portal, pool);
  639. }
  640. }
  641. /* Congestion group state change notification callback.
  642. * Stops the device's egress queues while they are congested and
  643. * wakes them upon exiting congested state.
  644. * Also updates some CGR-related stats.
  645. */
  646. static void dpaa_eth_cgscn(struct qman_portal *qm, struct qman_cgr *cgr,
  647. int congested)
  648. {
  649. struct dpaa_priv *priv = (struct dpaa_priv *)container_of(cgr,
  650. struct dpaa_priv, cgr_data.cgr);
  651. if (congested) {
  652. priv->cgr_data.congestion_start_jiffies = jiffies;
  653. netif_tx_stop_all_queues(priv->net_dev);
  654. priv->cgr_data.cgr_congested_count++;
  655. } else {
  656. priv->cgr_data.congested_jiffies +=
  657. (jiffies - priv->cgr_data.congestion_start_jiffies);
  658. netif_tx_wake_all_queues(priv->net_dev);
  659. }
  660. }
  661. static int dpaa_eth_cgr_init(struct dpaa_priv *priv)
  662. {
  663. struct qm_mcc_initcgr initcgr;
  664. u32 cs_th;
  665. int err;
  666. err = qman_alloc_cgrid(&priv->cgr_data.cgr.cgrid);
  667. if (err < 0) {
  668. if (netif_msg_drv(priv))
  669. pr_err("%s: Error %d allocating CGR ID\n",
  670. __func__, err);
  671. goto out_error;
  672. }
  673. priv->cgr_data.cgr.cb = dpaa_eth_cgscn;
  674. /* Enable Congestion State Change Notifications and CS taildrop */
  675. memset(&initcgr, 0, sizeof(initcgr));
  676. initcgr.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES);
  677. initcgr.cgr.cscn_en = QM_CGR_EN;
  678. /* Set different thresholds based on the MAC speed.
  679. * This may turn suboptimal if the MAC is reconfigured at a speed
  680. * lower than its max, e.g. if a dTSEC later negotiates a 100Mbps link.
  681. * In such cases, we ought to reconfigure the threshold, too.
  682. */
  683. if (priv->mac_dev->if_support & SUPPORTED_10000baseT_Full)
  684. cs_th = DPAA_CS_THRESHOLD_10G;
  685. else
  686. cs_th = DPAA_CS_THRESHOLD_1G;
  687. qm_cgr_cs_thres_set64(&initcgr.cgr.cs_thres, cs_th, 1);
  688. initcgr.we_mask |= cpu_to_be16(QM_CGR_WE_CSTD_EN);
  689. initcgr.cgr.cstd_en = QM_CGR_EN;
  690. err = qman_create_cgr(&priv->cgr_data.cgr, QMAN_CGR_FLAG_USE_INIT,
  691. &initcgr);
  692. if (err < 0) {
  693. if (netif_msg_drv(priv))
  694. pr_err("%s: Error %d creating CGR with ID %d\n",
  695. __func__, err, priv->cgr_data.cgr.cgrid);
  696. qman_release_cgrid(priv->cgr_data.cgr.cgrid);
  697. goto out_error;
  698. }
  699. if (netif_msg_drv(priv))
  700. pr_debug("Created CGR %d for netdev with hwaddr %pM on QMan channel %d\n",
  701. priv->cgr_data.cgr.cgrid, priv->mac_dev->addr,
  702. priv->cgr_data.cgr.chan);
  703. out_error:
  704. return err;
  705. }
  706. static inline void dpaa_setup_ingress(const struct dpaa_priv *priv,
  707. struct dpaa_fq *fq,
  708. const struct qman_fq *template)
  709. {
  710. fq->fq_base = *template;
  711. fq->net_dev = priv->net_dev;
  712. fq->flags = QMAN_FQ_FLAG_NO_ENQUEUE;
  713. fq->channel = priv->channel;
  714. }
  715. static inline void dpaa_setup_egress(const struct dpaa_priv *priv,
  716. struct dpaa_fq *fq,
  717. struct fman_port *port,
  718. const struct qman_fq *template)
  719. {
  720. fq->fq_base = *template;
  721. fq->net_dev = priv->net_dev;
  722. if (port) {
  723. fq->flags = QMAN_FQ_FLAG_TO_DCPORTAL;
  724. fq->channel = (u16)fman_port_get_qman_channel_id(port);
  725. } else {
  726. fq->flags = QMAN_FQ_FLAG_NO_MODIFY;
  727. }
  728. }
  729. static void dpaa_fq_setup(struct dpaa_priv *priv,
  730. const struct dpaa_fq_cbs *fq_cbs,
  731. struct fman_port *tx_port)
  732. {
  733. int egress_cnt = 0, conf_cnt = 0, num_portals = 0, portal_cnt = 0, cpu;
  734. const cpumask_t *affine_cpus = qman_affine_cpus();
  735. u16 channels[NR_CPUS];
  736. struct dpaa_fq *fq;
  737. for_each_cpu(cpu, affine_cpus)
  738. channels[num_portals++] = qman_affine_channel(cpu);
  739. if (num_portals == 0)
  740. dev_err(priv->net_dev->dev.parent,
  741. "No Qman software (affine) channels found");
  742. /* Initialize each FQ in the list */
  743. list_for_each_entry(fq, &priv->dpaa_fq_list, list) {
  744. switch (fq->fq_type) {
  745. case FQ_TYPE_RX_DEFAULT:
  746. dpaa_setup_ingress(priv, fq, &fq_cbs->rx_defq);
  747. break;
  748. case FQ_TYPE_RX_ERROR:
  749. dpaa_setup_ingress(priv, fq, &fq_cbs->rx_errq);
  750. break;
  751. case FQ_TYPE_RX_PCD:
  752. if (!num_portals)
  753. continue;
  754. dpaa_setup_ingress(priv, fq, &fq_cbs->rx_defq);
  755. fq->channel = channels[portal_cnt++ % num_portals];
  756. break;
  757. case FQ_TYPE_TX:
  758. dpaa_setup_egress(priv, fq, tx_port,
  759. &fq_cbs->egress_ern);
  760. /* If we have more Tx queues than the number of cores,
  761. * just ignore the extra ones.
  762. */
  763. if (egress_cnt < DPAA_ETH_TXQ_NUM)
  764. priv->egress_fqs[egress_cnt++] = &fq->fq_base;
  765. break;
  766. case FQ_TYPE_TX_CONF_MQ:
  767. priv->conf_fqs[conf_cnt++] = &fq->fq_base;
  768. /* fall through */
  769. case FQ_TYPE_TX_CONFIRM:
  770. dpaa_setup_ingress(priv, fq, &fq_cbs->tx_defq);
  771. break;
  772. case FQ_TYPE_TX_ERROR:
  773. dpaa_setup_ingress(priv, fq, &fq_cbs->tx_errq);
  774. break;
  775. default:
  776. dev_warn(priv->net_dev->dev.parent,
  777. "Unknown FQ type detected!\n");
  778. break;
  779. }
  780. }
  781. /* Make sure all CPUs receive a corresponding Tx queue. */
  782. while (egress_cnt < DPAA_ETH_TXQ_NUM) {
  783. list_for_each_entry(fq, &priv->dpaa_fq_list, list) {
  784. if (fq->fq_type != FQ_TYPE_TX)
  785. continue;
  786. priv->egress_fqs[egress_cnt++] = &fq->fq_base;
  787. if (egress_cnt == DPAA_ETH_TXQ_NUM)
  788. break;
  789. }
  790. }
  791. }
  792. static inline int dpaa_tx_fq_to_id(const struct dpaa_priv *priv,
  793. struct qman_fq *tx_fq)
  794. {
  795. int i;
  796. for (i = 0; i < DPAA_ETH_TXQ_NUM; i++)
  797. if (priv->egress_fqs[i] == tx_fq)
  798. return i;
  799. return -EINVAL;
  800. }
  801. static int dpaa_fq_init(struct dpaa_fq *dpaa_fq, bool td_enable)
  802. {
  803. const struct dpaa_priv *priv;
  804. struct qman_fq *confq = NULL;
  805. struct qm_mcc_initfq initfq;
  806. struct device *dev;
  807. struct qman_fq *fq;
  808. int queue_id;
  809. int err;
  810. priv = netdev_priv(dpaa_fq->net_dev);
  811. dev = dpaa_fq->net_dev->dev.parent;
  812. if (dpaa_fq->fqid == 0)
  813. dpaa_fq->flags |= QMAN_FQ_FLAG_DYNAMIC_FQID;
  814. dpaa_fq->init = !(dpaa_fq->flags & QMAN_FQ_FLAG_NO_MODIFY);
  815. err = qman_create_fq(dpaa_fq->fqid, dpaa_fq->flags, &dpaa_fq->fq_base);
  816. if (err) {
  817. dev_err(dev, "qman_create_fq() failed\n");
  818. return err;
  819. }
  820. fq = &dpaa_fq->fq_base;
  821. if (dpaa_fq->init) {
  822. memset(&initfq, 0, sizeof(initfq));
  823. initfq.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL);
  824. /* Note: we may get to keep an empty FQ in cache */
  825. initfq.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_PREFERINCACHE);
  826. /* Try to reduce the number of portal interrupts for
  827. * Tx Confirmation FQs.
  828. */
  829. if (dpaa_fq->fq_type == FQ_TYPE_TX_CONFIRM)
  830. initfq.fqd.fq_ctrl |= cpu_to_be16(QM_FQCTRL_AVOIDBLOCK);
  831. /* FQ placement */
  832. initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_DESTWQ);
  833. qm_fqd_set_destwq(&initfq.fqd, dpaa_fq->channel, dpaa_fq->wq);
  834. /* Put all egress queues in a congestion group of their own.
  835. * Sensu stricto, the Tx confirmation queues are Rx FQs,
  836. * rather than Tx - but they nonetheless account for the
  837. * memory footprint on behalf of egress traffic. We therefore
  838. * place them in the netdev's CGR, along with the Tx FQs.
  839. */
  840. if (dpaa_fq->fq_type == FQ_TYPE_TX ||
  841. dpaa_fq->fq_type == FQ_TYPE_TX_CONFIRM ||
  842. dpaa_fq->fq_type == FQ_TYPE_TX_CONF_MQ) {
  843. initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_CGID);
  844. initfq.fqd.fq_ctrl |= cpu_to_be16(QM_FQCTRL_CGE);
  845. initfq.fqd.cgid = (u8)priv->cgr_data.cgr.cgrid;
  846. /* Set a fixed overhead accounting, in an attempt to
  847. * reduce the impact of fixed-size skb shells and the
  848. * driver's needed headroom on system memory. This is
  849. * especially the case when the egress traffic is
  850. * composed of small datagrams.
  851. * Unfortunately, QMan's OAL value is capped to an
  852. * insufficient value, but even that is better than
  853. * no overhead accounting at all.
  854. */
  855. initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_OAC);
  856. qm_fqd_set_oac(&initfq.fqd, QM_OAC_CG);
  857. qm_fqd_set_oal(&initfq.fqd,
  858. min(sizeof(struct sk_buff) +
  859. priv->tx_headroom,
  860. (size_t)FSL_QMAN_MAX_OAL));
  861. }
  862. if (td_enable) {
  863. initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_TDTHRESH);
  864. qm_fqd_set_taildrop(&initfq.fqd, DPAA_FQ_TD, 1);
  865. initfq.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_TDE);
  866. }
  867. if (dpaa_fq->fq_type == FQ_TYPE_TX) {
  868. queue_id = dpaa_tx_fq_to_id(priv, &dpaa_fq->fq_base);
  869. if (queue_id >= 0)
  870. confq = priv->conf_fqs[queue_id];
  871. if (confq) {
  872. initfq.we_mask |=
  873. cpu_to_be16(QM_INITFQ_WE_CONTEXTA);
  874. /* ContextA: OVOM=1(use contextA2 bits instead of ICAD)
  875. * A2V=1 (contextA A2 field is valid)
  876. * A0V=1 (contextA A0 field is valid)
  877. * B0V=1 (contextB field is valid)
  878. * ContextA A2: EBD=1 (deallocate buffers inside FMan)
  879. * ContextB B0(ASPID): 0 (absolute Virtual Storage ID)
  880. */
  881. qm_fqd_context_a_set64(&initfq.fqd,
  882. 0x1e00000080000000ULL);
  883. }
  884. }
  885. /* Put all the ingress queues in our "ingress CGR". */
  886. if (priv->use_ingress_cgr &&
  887. (dpaa_fq->fq_type == FQ_TYPE_RX_DEFAULT ||
  888. dpaa_fq->fq_type == FQ_TYPE_RX_ERROR ||
  889. dpaa_fq->fq_type == FQ_TYPE_RX_PCD)) {
  890. initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_CGID);
  891. initfq.fqd.fq_ctrl |= cpu_to_be16(QM_FQCTRL_CGE);
  892. initfq.fqd.cgid = (u8)priv->ingress_cgr.cgrid;
  893. /* Set a fixed overhead accounting, just like for the
  894. * egress CGR.
  895. */
  896. initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_OAC);
  897. qm_fqd_set_oac(&initfq.fqd, QM_OAC_CG);
  898. qm_fqd_set_oal(&initfq.fqd,
  899. min(sizeof(struct sk_buff) +
  900. priv->tx_headroom,
  901. (size_t)FSL_QMAN_MAX_OAL));
  902. }
  903. /* Initialization common to all ingress queues */
  904. if (dpaa_fq->flags & QMAN_FQ_FLAG_NO_ENQUEUE) {
  905. initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_CONTEXTA);
  906. initfq.fqd.fq_ctrl |= cpu_to_be16(QM_FQCTRL_HOLDACTIVE |
  907. QM_FQCTRL_CTXASTASHING);
  908. initfq.fqd.context_a.stashing.exclusive =
  909. QM_STASHING_EXCL_DATA | QM_STASHING_EXCL_CTX |
  910. QM_STASHING_EXCL_ANNOTATION;
  911. qm_fqd_set_stashing(&initfq.fqd, 1, 2,
  912. DIV_ROUND_UP(sizeof(struct qman_fq),
  913. 64));
  914. }
  915. err = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &initfq);
  916. if (err < 0) {
  917. dev_err(dev, "qman_init_fq(%u) = %d\n",
  918. qman_fq_fqid(fq), err);
  919. qman_destroy_fq(fq);
  920. return err;
  921. }
  922. }
  923. dpaa_fq->fqid = qman_fq_fqid(fq);
  924. return 0;
  925. }
  926. static int dpaa_fq_free_entry(struct device *dev, struct qman_fq *fq)
  927. {
  928. const struct dpaa_priv *priv;
  929. struct dpaa_fq *dpaa_fq;
  930. int err, error;
  931. err = 0;
  932. dpaa_fq = container_of(fq, struct dpaa_fq, fq_base);
  933. priv = netdev_priv(dpaa_fq->net_dev);
  934. if (dpaa_fq->init) {
  935. err = qman_retire_fq(fq, NULL);
  936. if (err < 0 && netif_msg_drv(priv))
  937. dev_err(dev, "qman_retire_fq(%u) = %d\n",
  938. qman_fq_fqid(fq), err);
  939. error = qman_oos_fq(fq);
  940. if (error < 0 && netif_msg_drv(priv)) {
  941. dev_err(dev, "qman_oos_fq(%u) = %d\n",
  942. qman_fq_fqid(fq), error);
  943. if (err >= 0)
  944. err = error;
  945. }
  946. }
  947. qman_destroy_fq(fq);
  948. list_del(&dpaa_fq->list);
  949. return err;
  950. }
  951. static int dpaa_fq_free(struct device *dev, struct list_head *list)
  952. {
  953. struct dpaa_fq *dpaa_fq, *tmp;
  954. int err, error;
  955. err = 0;
  956. list_for_each_entry_safe(dpaa_fq, tmp, list, list) {
  957. error = dpaa_fq_free_entry(dev, (struct qman_fq *)dpaa_fq);
  958. if (error < 0 && err >= 0)
  959. err = error;
  960. }
  961. return err;
  962. }
  963. static int dpaa_eth_init_tx_port(struct fman_port *port, struct dpaa_fq *errq,
  964. struct dpaa_fq *defq,
  965. struct dpaa_buffer_layout *buf_layout)
  966. {
  967. struct fman_buffer_prefix_content buf_prefix_content;
  968. struct fman_port_params params;
  969. int err;
  970. memset(&params, 0, sizeof(params));
  971. memset(&buf_prefix_content, 0, sizeof(buf_prefix_content));
  972. buf_prefix_content.priv_data_size = buf_layout->priv_data_size;
  973. buf_prefix_content.pass_prs_result = true;
  974. buf_prefix_content.pass_hash_result = true;
  975. buf_prefix_content.pass_time_stamp = false;
  976. buf_prefix_content.data_align = DPAA_FD_DATA_ALIGNMENT;
  977. params.specific_params.non_rx_params.err_fqid = errq->fqid;
  978. params.specific_params.non_rx_params.dflt_fqid = defq->fqid;
  979. err = fman_port_config(port, &params);
  980. if (err) {
  981. pr_err("%s: fman_port_config failed\n", __func__);
  982. return err;
  983. }
  984. err = fman_port_cfg_buf_prefix_content(port, &buf_prefix_content);
  985. if (err) {
  986. pr_err("%s: fman_port_cfg_buf_prefix_content failed\n",
  987. __func__);
  988. return err;
  989. }
  990. err = fman_port_init(port);
  991. if (err)
  992. pr_err("%s: fm_port_init failed\n", __func__);
  993. return err;
  994. }
  995. static int dpaa_eth_init_rx_port(struct fman_port *port, struct dpaa_bp **bps,
  996. size_t count, struct dpaa_fq *errq,
  997. struct dpaa_fq *defq, struct dpaa_fq *pcdq,
  998. struct dpaa_buffer_layout *buf_layout)
  999. {
  1000. struct fman_buffer_prefix_content buf_prefix_content;
  1001. struct fman_port_rx_params *rx_p;
  1002. struct fman_port_params params;
  1003. int i, err;
  1004. memset(&params, 0, sizeof(params));
  1005. memset(&buf_prefix_content, 0, sizeof(buf_prefix_content));
  1006. buf_prefix_content.priv_data_size = buf_layout->priv_data_size;
  1007. buf_prefix_content.pass_prs_result = true;
  1008. buf_prefix_content.pass_hash_result = true;
  1009. buf_prefix_content.pass_time_stamp = false;
  1010. buf_prefix_content.data_align = DPAA_FD_DATA_ALIGNMENT;
  1011. rx_p = &params.specific_params.rx_params;
  1012. rx_p->err_fqid = errq->fqid;
  1013. rx_p->dflt_fqid = defq->fqid;
  1014. if (pcdq) {
  1015. rx_p->pcd_base_fqid = pcdq->fqid;
  1016. rx_p->pcd_fqs_count = DPAA_ETH_PCD_RXQ_NUM;
  1017. }
  1018. count = min(ARRAY_SIZE(rx_p->ext_buf_pools.ext_buf_pool), count);
  1019. rx_p->ext_buf_pools.num_of_pools_used = (u8)count;
  1020. for (i = 0; i < count; i++) {
  1021. rx_p->ext_buf_pools.ext_buf_pool[i].id = bps[i]->bpid;
  1022. rx_p->ext_buf_pools.ext_buf_pool[i].size = (u16)bps[i]->size;
  1023. }
  1024. err = fman_port_config(port, &params);
  1025. if (err) {
  1026. pr_err("%s: fman_port_config failed\n", __func__);
  1027. return err;
  1028. }
  1029. err = fman_port_cfg_buf_prefix_content(port, &buf_prefix_content);
  1030. if (err) {
  1031. pr_err("%s: fman_port_cfg_buf_prefix_content failed\n",
  1032. __func__);
  1033. return err;
  1034. }
  1035. err = fman_port_init(port);
  1036. if (err)
  1037. pr_err("%s: fm_port_init failed\n", __func__);
  1038. return err;
  1039. }
  1040. static int dpaa_eth_init_ports(struct mac_device *mac_dev,
  1041. struct dpaa_bp **bps, size_t count,
  1042. struct fm_port_fqs *port_fqs,
  1043. struct dpaa_buffer_layout *buf_layout,
  1044. struct device *dev)
  1045. {
  1046. struct fman_port *rxport = mac_dev->port[RX];
  1047. struct fman_port *txport = mac_dev->port[TX];
  1048. int err;
  1049. err = dpaa_eth_init_tx_port(txport, port_fqs->tx_errq,
  1050. port_fqs->tx_defq, &buf_layout[TX]);
  1051. if (err)
  1052. return err;
  1053. err = dpaa_eth_init_rx_port(rxport, bps, count, port_fqs->rx_errq,
  1054. port_fqs->rx_defq, port_fqs->rx_pcdq,
  1055. &buf_layout[RX]);
  1056. return err;
  1057. }
  1058. static int dpaa_bman_release(const struct dpaa_bp *dpaa_bp,
  1059. struct bm_buffer *bmb, int cnt)
  1060. {
  1061. int err;
  1062. err = bman_release(dpaa_bp->pool, bmb, cnt);
  1063. /* Should never occur, address anyway to avoid leaking the buffers */
  1064. if (unlikely(WARN_ON(err)) && dpaa_bp->free_buf_cb)
  1065. while (cnt-- > 0)
  1066. dpaa_bp->free_buf_cb(dpaa_bp, &bmb[cnt]);
  1067. return cnt;
  1068. }
  1069. static void dpaa_release_sgt_members(struct qm_sg_entry *sgt)
  1070. {
  1071. struct bm_buffer bmb[DPAA_BUFF_RELEASE_MAX];
  1072. struct dpaa_bp *dpaa_bp;
  1073. int i = 0, j;
  1074. memset(bmb, 0, sizeof(bmb));
  1075. do {
  1076. dpaa_bp = dpaa_bpid2pool(sgt[i].bpid);
  1077. if (!dpaa_bp)
  1078. return;
  1079. j = 0;
  1080. do {
  1081. WARN_ON(qm_sg_entry_is_ext(&sgt[i]));
  1082. bm_buffer_set64(&bmb[j], qm_sg_entry_get64(&sgt[i]));
  1083. j++; i++;
  1084. } while (j < ARRAY_SIZE(bmb) &&
  1085. !qm_sg_entry_is_final(&sgt[i - 1]) &&
  1086. sgt[i - 1].bpid == sgt[i].bpid);
  1087. dpaa_bman_release(dpaa_bp, bmb, j);
  1088. } while (!qm_sg_entry_is_final(&sgt[i - 1]));
  1089. }
  1090. static void dpaa_fd_release(const struct net_device *net_dev,
  1091. const struct qm_fd *fd)
  1092. {
  1093. struct qm_sg_entry *sgt;
  1094. struct dpaa_bp *dpaa_bp;
  1095. struct bm_buffer bmb;
  1096. dma_addr_t addr;
  1097. void *vaddr;
  1098. bmb.data = 0;
  1099. bm_buffer_set64(&bmb, qm_fd_addr(fd));
  1100. dpaa_bp = dpaa_bpid2pool(fd->bpid);
  1101. if (!dpaa_bp)
  1102. return;
  1103. if (qm_fd_get_format(fd) == qm_fd_sg) {
  1104. vaddr = phys_to_virt(qm_fd_addr(fd));
  1105. sgt = vaddr + qm_fd_get_offset(fd);
  1106. dma_unmap_single(dpaa_bp->dev, qm_fd_addr(fd), dpaa_bp->size,
  1107. DMA_FROM_DEVICE);
  1108. dpaa_release_sgt_members(sgt);
  1109. addr = dma_map_single(dpaa_bp->dev, vaddr, dpaa_bp->size,
  1110. DMA_FROM_DEVICE);
  1111. if (dma_mapping_error(dpaa_bp->dev, addr)) {
  1112. dev_err(dpaa_bp->dev, "DMA mapping failed");
  1113. return;
  1114. }
  1115. bm_buffer_set64(&bmb, addr);
  1116. }
  1117. dpaa_bman_release(dpaa_bp, &bmb, 1);
  1118. }
  1119. static void count_ern(struct dpaa_percpu_priv *percpu_priv,
  1120. const union qm_mr_entry *msg)
  1121. {
  1122. switch (msg->ern.rc & QM_MR_RC_MASK) {
  1123. case QM_MR_RC_CGR_TAILDROP:
  1124. percpu_priv->ern_cnt.cg_tdrop++;
  1125. break;
  1126. case QM_MR_RC_WRED:
  1127. percpu_priv->ern_cnt.wred++;
  1128. break;
  1129. case QM_MR_RC_ERROR:
  1130. percpu_priv->ern_cnt.err_cond++;
  1131. break;
  1132. case QM_MR_RC_ORPWINDOW_EARLY:
  1133. percpu_priv->ern_cnt.early_window++;
  1134. break;
  1135. case QM_MR_RC_ORPWINDOW_LATE:
  1136. percpu_priv->ern_cnt.late_window++;
  1137. break;
  1138. case QM_MR_RC_FQ_TAILDROP:
  1139. percpu_priv->ern_cnt.fq_tdrop++;
  1140. break;
  1141. case QM_MR_RC_ORPWINDOW_RETIRED:
  1142. percpu_priv->ern_cnt.fq_retired++;
  1143. break;
  1144. case QM_MR_RC_ORP_ZERO:
  1145. percpu_priv->ern_cnt.orp_zero++;
  1146. break;
  1147. }
  1148. }
  1149. /* Turn on HW checksum computation for this outgoing frame.
  1150. * If the current protocol is not something we support in this regard
  1151. * (or if the stack has already computed the SW checksum), we do nothing.
  1152. *
  1153. * Returns 0 if all goes well (or HW csum doesn't apply), and a negative value
  1154. * otherwise.
  1155. *
  1156. * Note that this function may modify the fd->cmd field and the skb data buffer
  1157. * (the Parse Results area).
  1158. */
  1159. static int dpaa_enable_tx_csum(struct dpaa_priv *priv,
  1160. struct sk_buff *skb,
  1161. struct qm_fd *fd,
  1162. char *parse_results)
  1163. {
  1164. struct fman_prs_result *parse_result;
  1165. u16 ethertype = ntohs(skb->protocol);
  1166. struct ipv6hdr *ipv6h = NULL;
  1167. struct iphdr *iph;
  1168. int retval = 0;
  1169. u8 l4_proto;
  1170. if (skb->ip_summed != CHECKSUM_PARTIAL)
  1171. return 0;
  1172. /* Note: L3 csum seems to be already computed in sw, but we can't choose
  1173. * L4 alone from the FM configuration anyway.
  1174. */
  1175. /* Fill in some fields of the Parse Results array, so the FMan
  1176. * can find them as if they came from the FMan Parser.
  1177. */
  1178. parse_result = (struct fman_prs_result *)parse_results;
  1179. /* If we're dealing with VLAN, get the real Ethernet type */
  1180. if (ethertype == ETH_P_8021Q) {
  1181. /* We can't always assume the MAC header is set correctly
  1182. * by the stack, so reset to beginning of skb->data
  1183. */
  1184. skb_reset_mac_header(skb);
  1185. ethertype = ntohs(vlan_eth_hdr(skb)->h_vlan_encapsulated_proto);
  1186. }
  1187. /* Fill in the relevant L3 parse result fields
  1188. * and read the L4 protocol type
  1189. */
  1190. switch (ethertype) {
  1191. case ETH_P_IP:
  1192. parse_result->l3r = cpu_to_be16(FM_L3_PARSE_RESULT_IPV4);
  1193. iph = ip_hdr(skb);
  1194. WARN_ON(!iph);
  1195. l4_proto = iph->protocol;
  1196. break;
  1197. case ETH_P_IPV6:
  1198. parse_result->l3r = cpu_to_be16(FM_L3_PARSE_RESULT_IPV6);
  1199. ipv6h = ipv6_hdr(skb);
  1200. WARN_ON(!ipv6h);
  1201. l4_proto = ipv6h->nexthdr;
  1202. break;
  1203. default:
  1204. /* We shouldn't even be here */
  1205. if (net_ratelimit())
  1206. netif_alert(priv, tx_err, priv->net_dev,
  1207. "Can't compute HW csum for L3 proto 0x%x\n",
  1208. ntohs(skb->protocol));
  1209. retval = -EIO;
  1210. goto return_error;
  1211. }
  1212. /* Fill in the relevant L4 parse result fields */
  1213. switch (l4_proto) {
  1214. case IPPROTO_UDP:
  1215. parse_result->l4r = FM_L4_PARSE_RESULT_UDP;
  1216. break;
  1217. case IPPROTO_TCP:
  1218. parse_result->l4r = FM_L4_PARSE_RESULT_TCP;
  1219. break;
  1220. default:
  1221. if (net_ratelimit())
  1222. netif_alert(priv, tx_err, priv->net_dev,
  1223. "Can't compute HW csum for L4 proto 0x%x\n",
  1224. l4_proto);
  1225. retval = -EIO;
  1226. goto return_error;
  1227. }
  1228. /* At index 0 is IPOffset_1 as defined in the Parse Results */
  1229. parse_result->ip_off[0] = (u8)skb_network_offset(skb);
  1230. parse_result->l4_off = (u8)skb_transport_offset(skb);
  1231. /* Enable L3 (and L4, if TCP or UDP) HW checksum. */
  1232. fd->cmd |= cpu_to_be32(FM_FD_CMD_RPD | FM_FD_CMD_DTC);
  1233. /* On P1023 and similar platforms fd->cmd interpretation could
  1234. * be disabled by setting CONTEXT_A bit ICMD; currently this bit
  1235. * is not set so we do not need to check; in the future, if/when
  1236. * using context_a we need to check this bit
  1237. */
  1238. return_error:
  1239. return retval;
  1240. }
  1241. static int dpaa_bp_add_8_bufs(const struct dpaa_bp *dpaa_bp)
  1242. {
  1243. struct device *dev = dpaa_bp->dev;
  1244. struct bm_buffer bmb[8];
  1245. dma_addr_t addr;
  1246. void *new_buf;
  1247. u8 i;
  1248. for (i = 0; i < 8; i++) {
  1249. new_buf = netdev_alloc_frag(dpaa_bp->raw_size);
  1250. if (unlikely(!new_buf)) {
  1251. dev_err(dev, "netdev_alloc_frag() failed, size %zu\n",
  1252. dpaa_bp->raw_size);
  1253. goto release_previous_buffs;
  1254. }
  1255. new_buf = PTR_ALIGN(new_buf, SMP_CACHE_BYTES);
  1256. addr = dma_map_single(dev, new_buf,
  1257. dpaa_bp->size, DMA_FROM_DEVICE);
  1258. if (unlikely(dma_mapping_error(dev, addr))) {
  1259. dev_err(dpaa_bp->dev, "DMA map failed");
  1260. goto release_previous_buffs;
  1261. }
  1262. bmb[i].data = 0;
  1263. bm_buffer_set64(&bmb[i], addr);
  1264. }
  1265. release_bufs:
  1266. return dpaa_bman_release(dpaa_bp, bmb, i);
  1267. release_previous_buffs:
  1268. WARN_ONCE(1, "dpaa_eth: failed to add buffers on Rx\n");
  1269. bm_buffer_set64(&bmb[i], 0);
  1270. /* Avoid releasing a completely null buffer; bman_release() requires
  1271. * at least one buffer.
  1272. */
  1273. if (likely(i))
  1274. goto release_bufs;
  1275. return 0;
  1276. }
  1277. static int dpaa_bp_seed(struct dpaa_bp *dpaa_bp)
  1278. {
  1279. int i;
  1280. /* Give each CPU an allotment of "config_count" buffers */
  1281. for_each_possible_cpu(i) {
  1282. int *count_ptr = per_cpu_ptr(dpaa_bp->percpu_count, i);
  1283. int j;
  1284. /* Although we access another CPU's counters here
  1285. * we do it at boot time so it is safe
  1286. */
  1287. for (j = 0; j < dpaa_bp->config_count; j += 8)
  1288. *count_ptr += dpaa_bp_add_8_bufs(dpaa_bp);
  1289. }
  1290. return 0;
  1291. }
  1292. /* Add buffers/(pages) for Rx processing whenever bpool count falls below
  1293. * REFILL_THRESHOLD.
  1294. */
  1295. static int dpaa_eth_refill_bpool(struct dpaa_bp *dpaa_bp, int *countptr)
  1296. {
  1297. int count = *countptr;
  1298. int new_bufs;
  1299. if (unlikely(count < FSL_DPAA_ETH_REFILL_THRESHOLD)) {
  1300. do {
  1301. new_bufs = dpaa_bp_add_8_bufs(dpaa_bp);
  1302. if (unlikely(!new_bufs)) {
  1303. /* Avoid looping forever if we've temporarily
  1304. * run out of memory. We'll try again at the
  1305. * next NAPI cycle.
  1306. */
  1307. break;
  1308. }
  1309. count += new_bufs;
  1310. } while (count < FSL_DPAA_ETH_MAX_BUF_COUNT);
  1311. *countptr = count;
  1312. if (unlikely(count < FSL_DPAA_ETH_MAX_BUF_COUNT))
  1313. return -ENOMEM;
  1314. }
  1315. return 0;
  1316. }
  1317. static int dpaa_eth_refill_bpools(struct dpaa_priv *priv)
  1318. {
  1319. struct dpaa_bp *dpaa_bp;
  1320. int *countptr;
  1321. int res, i;
  1322. for (i = 0; i < DPAA_BPS_NUM; i++) {
  1323. dpaa_bp = priv->dpaa_bps[i];
  1324. if (!dpaa_bp)
  1325. return -EINVAL;
  1326. countptr = this_cpu_ptr(dpaa_bp->percpu_count);
  1327. res = dpaa_eth_refill_bpool(dpaa_bp, countptr);
  1328. if (res)
  1329. return res;
  1330. }
  1331. return 0;
  1332. }
  1333. /* Cleanup function for outgoing frame descriptors that were built on Tx path,
  1334. * either contiguous frames or scatter/gather ones.
  1335. * Skb freeing is not handled here.
  1336. *
  1337. * This function may be called on error paths in the Tx function, so guard
  1338. * against cases when not all fd relevant fields were filled in.
  1339. *
  1340. * Return the skb backpointer, since for S/G frames the buffer containing it
  1341. * gets freed here.
  1342. */
  1343. static struct sk_buff *dpaa_cleanup_tx_fd(const struct dpaa_priv *priv,
  1344. const struct qm_fd *fd)
  1345. {
  1346. const enum dma_data_direction dma_dir = DMA_TO_DEVICE;
  1347. struct device *dev = priv->net_dev->dev.parent;
  1348. dma_addr_t addr = qm_fd_addr(fd);
  1349. const struct qm_sg_entry *sgt;
  1350. struct sk_buff **skbh, *skb;
  1351. int nr_frags, i;
  1352. skbh = (struct sk_buff **)phys_to_virt(addr);
  1353. skb = *skbh;
  1354. if (unlikely(qm_fd_get_format(fd) == qm_fd_sg)) {
  1355. nr_frags = skb_shinfo(skb)->nr_frags;
  1356. dma_unmap_single(dev, addr, qm_fd_get_offset(fd) +
  1357. sizeof(struct qm_sg_entry) * (1 + nr_frags),
  1358. dma_dir);
  1359. /* The sgt buffer has been allocated with netdev_alloc_frag(),
  1360. * it's from lowmem.
  1361. */
  1362. sgt = phys_to_virt(addr + qm_fd_get_offset(fd));
  1363. /* sgt[0] is from lowmem, was dma_map_single()-ed */
  1364. dma_unmap_single(dev, qm_sg_addr(&sgt[0]),
  1365. qm_sg_entry_get_len(&sgt[0]), dma_dir);
  1366. /* remaining pages were mapped with skb_frag_dma_map() */
  1367. for (i = 1; i < nr_frags; i++) {
  1368. WARN_ON(qm_sg_entry_is_ext(&sgt[i]));
  1369. dma_unmap_page(dev, qm_sg_addr(&sgt[i]),
  1370. qm_sg_entry_get_len(&sgt[i]), dma_dir);
  1371. }
  1372. /* Free the page frag that we allocated on Tx */
  1373. skb_free_frag(phys_to_virt(addr));
  1374. } else {
  1375. dma_unmap_single(dev, addr,
  1376. skb_tail_pointer(skb) - (u8 *)skbh, dma_dir);
  1377. }
  1378. return skb;
  1379. }
  1380. static u8 rx_csum_offload(const struct dpaa_priv *priv, const struct qm_fd *fd)
  1381. {
  1382. /* The parser has run and performed L4 checksum validation.
  1383. * We know there were no parser errors (and implicitly no
  1384. * L4 csum error), otherwise we wouldn't be here.
  1385. */
  1386. if ((priv->net_dev->features & NETIF_F_RXCSUM) &&
  1387. (be32_to_cpu(fd->status) & FM_FD_STAT_L4CV))
  1388. return CHECKSUM_UNNECESSARY;
  1389. /* We're here because either the parser didn't run or the L4 checksum
  1390. * was not verified. This may include the case of a UDP frame with
  1391. * checksum zero or an L4 proto other than TCP/UDP
  1392. */
  1393. return CHECKSUM_NONE;
  1394. }
  1395. /* Build a linear skb around the received buffer.
  1396. * We are guaranteed there is enough room at the end of the data buffer to
  1397. * accommodate the shared info area of the skb.
  1398. */
  1399. static struct sk_buff *contig_fd_to_skb(const struct dpaa_priv *priv,
  1400. const struct qm_fd *fd)
  1401. {
  1402. ssize_t fd_off = qm_fd_get_offset(fd);
  1403. dma_addr_t addr = qm_fd_addr(fd);
  1404. struct dpaa_bp *dpaa_bp;
  1405. struct sk_buff *skb;
  1406. void *vaddr;
  1407. vaddr = phys_to_virt(addr);
  1408. WARN_ON(!IS_ALIGNED((unsigned long)vaddr, SMP_CACHE_BYTES));
  1409. dpaa_bp = dpaa_bpid2pool(fd->bpid);
  1410. if (!dpaa_bp)
  1411. goto free_buffer;
  1412. skb = build_skb(vaddr, dpaa_bp->size +
  1413. SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
  1414. if (unlikely(!skb)) {
  1415. WARN_ONCE(1, "Build skb failure on Rx\n");
  1416. goto free_buffer;
  1417. }
  1418. WARN_ON(fd_off != priv->rx_headroom);
  1419. skb_reserve(skb, fd_off);
  1420. skb_put(skb, qm_fd_get_length(fd));
  1421. skb->ip_summed = rx_csum_offload(priv, fd);
  1422. return skb;
  1423. free_buffer:
  1424. skb_free_frag(vaddr);
  1425. return NULL;
  1426. }
  1427. /* Build an skb with the data of the first S/G entry in the linear portion and
  1428. * the rest of the frame as skb fragments.
  1429. *
  1430. * The page fragment holding the S/G Table is recycled here.
  1431. */
  1432. static struct sk_buff *sg_fd_to_skb(const struct dpaa_priv *priv,
  1433. const struct qm_fd *fd)
  1434. {
  1435. ssize_t fd_off = qm_fd_get_offset(fd);
  1436. dma_addr_t addr = qm_fd_addr(fd);
  1437. const struct qm_sg_entry *sgt;
  1438. struct page *page, *head_page;
  1439. struct dpaa_bp *dpaa_bp;
  1440. void *vaddr, *sg_vaddr;
  1441. int frag_off, frag_len;
  1442. struct sk_buff *skb;
  1443. dma_addr_t sg_addr;
  1444. int page_offset;
  1445. unsigned int sz;
  1446. int *count_ptr;
  1447. int i;
  1448. vaddr = phys_to_virt(addr);
  1449. WARN_ON(!IS_ALIGNED((unsigned long)vaddr, SMP_CACHE_BYTES));
  1450. /* Iterate through the SGT entries and add data buffers to the skb */
  1451. sgt = vaddr + fd_off;
  1452. skb = NULL;
  1453. for (i = 0; i < DPAA_SGT_MAX_ENTRIES; i++) {
  1454. /* Extension bit is not supported */
  1455. WARN_ON(qm_sg_entry_is_ext(&sgt[i]));
  1456. sg_addr = qm_sg_addr(&sgt[i]);
  1457. sg_vaddr = phys_to_virt(sg_addr);
  1458. WARN_ON(!IS_ALIGNED((unsigned long)sg_vaddr,
  1459. SMP_CACHE_BYTES));
  1460. /* We may use multiple Rx pools */
  1461. dpaa_bp = dpaa_bpid2pool(sgt[i].bpid);
  1462. if (!dpaa_bp)
  1463. goto free_buffers;
  1464. count_ptr = this_cpu_ptr(dpaa_bp->percpu_count);
  1465. dma_unmap_single(dpaa_bp->dev, sg_addr, dpaa_bp->size,
  1466. DMA_FROM_DEVICE);
  1467. if (!skb) {
  1468. sz = dpaa_bp->size +
  1469. SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
  1470. skb = build_skb(sg_vaddr, sz);
  1471. if (WARN_ON(unlikely(!skb)))
  1472. goto free_buffers;
  1473. skb->ip_summed = rx_csum_offload(priv, fd);
  1474. /* Make sure forwarded skbs will have enough space
  1475. * on Tx, if extra headers are added.
  1476. */
  1477. WARN_ON(fd_off != priv->rx_headroom);
  1478. skb_reserve(skb, fd_off);
  1479. skb_put(skb, qm_sg_entry_get_len(&sgt[i]));
  1480. } else {
  1481. /* Not the first S/G entry; all data from buffer will
  1482. * be added in an skb fragment; fragment index is offset
  1483. * by one since first S/G entry was incorporated in the
  1484. * linear part of the skb.
  1485. *
  1486. * Caution: 'page' may be a tail page.
  1487. */
  1488. page = virt_to_page(sg_vaddr);
  1489. head_page = virt_to_head_page(sg_vaddr);
  1490. /* Compute offset in (possibly tail) page */
  1491. page_offset = ((unsigned long)sg_vaddr &
  1492. (PAGE_SIZE - 1)) +
  1493. (page_address(page) - page_address(head_page));
  1494. /* page_offset only refers to the beginning of sgt[i];
  1495. * but the buffer itself may have an internal offset.
  1496. */
  1497. frag_off = qm_sg_entry_get_off(&sgt[i]) + page_offset;
  1498. frag_len = qm_sg_entry_get_len(&sgt[i]);
  1499. /* skb_add_rx_frag() does no checking on the page; if
  1500. * we pass it a tail page, we'll end up with
  1501. * bad page accounting and eventually with segafults.
  1502. */
  1503. skb_add_rx_frag(skb, i - 1, head_page, frag_off,
  1504. frag_len, dpaa_bp->size);
  1505. }
  1506. /* Update the pool count for the current {cpu x bpool} */
  1507. (*count_ptr)--;
  1508. if (qm_sg_entry_is_final(&sgt[i]))
  1509. break;
  1510. }
  1511. WARN_ONCE(i == DPAA_SGT_MAX_ENTRIES, "No final bit on SGT\n");
  1512. /* free the SG table buffer */
  1513. skb_free_frag(vaddr);
  1514. return skb;
  1515. free_buffers:
  1516. /* compensate sw bpool counter changes */
  1517. for (i--; i >= 0; i--) {
  1518. dpaa_bp = dpaa_bpid2pool(sgt[i].bpid);
  1519. if (dpaa_bp) {
  1520. count_ptr = this_cpu_ptr(dpaa_bp->percpu_count);
  1521. (*count_ptr)++;
  1522. }
  1523. }
  1524. /* free all the SG entries */
  1525. for (i = 0; i < DPAA_SGT_MAX_ENTRIES ; i++) {
  1526. sg_addr = qm_sg_addr(&sgt[i]);
  1527. sg_vaddr = phys_to_virt(sg_addr);
  1528. skb_free_frag(sg_vaddr);
  1529. dpaa_bp = dpaa_bpid2pool(sgt[i].bpid);
  1530. if (dpaa_bp) {
  1531. count_ptr = this_cpu_ptr(dpaa_bp->percpu_count);
  1532. (*count_ptr)--;
  1533. }
  1534. if (qm_sg_entry_is_final(&sgt[i]))
  1535. break;
  1536. }
  1537. /* free the SGT fragment */
  1538. skb_free_frag(vaddr);
  1539. return NULL;
  1540. }
  1541. static int skb_to_contig_fd(struct dpaa_priv *priv,
  1542. struct sk_buff *skb, struct qm_fd *fd,
  1543. int *offset)
  1544. {
  1545. struct net_device *net_dev = priv->net_dev;
  1546. struct device *dev = net_dev->dev.parent;
  1547. enum dma_data_direction dma_dir;
  1548. unsigned char *buffer_start;
  1549. struct sk_buff **skbh;
  1550. dma_addr_t addr;
  1551. int err;
  1552. /* We are guaranteed to have at least tx_headroom bytes
  1553. * available, so just use that for offset.
  1554. */
  1555. fd->bpid = FSL_DPAA_BPID_INV;
  1556. buffer_start = skb->data - priv->tx_headroom;
  1557. dma_dir = DMA_TO_DEVICE;
  1558. skbh = (struct sk_buff **)buffer_start;
  1559. *skbh = skb;
  1560. /* Enable L3/L4 hardware checksum computation.
  1561. *
  1562. * We must do this before dma_map_single(DMA_TO_DEVICE), because we may
  1563. * need to write into the skb.
  1564. */
  1565. err = dpaa_enable_tx_csum(priv, skb, fd,
  1566. ((char *)skbh) + DPAA_TX_PRIV_DATA_SIZE);
  1567. if (unlikely(err < 0)) {
  1568. if (net_ratelimit())
  1569. netif_err(priv, tx_err, net_dev, "HW csum error: %d\n",
  1570. err);
  1571. return err;
  1572. }
  1573. /* Fill in the rest of the FD fields */
  1574. qm_fd_set_contig(fd, priv->tx_headroom, skb->len);
  1575. fd->cmd |= cpu_to_be32(FM_FD_CMD_FCO);
  1576. /* Map the entire buffer size that may be seen by FMan, but no more */
  1577. addr = dma_map_single(dev, skbh,
  1578. skb_tail_pointer(skb) - buffer_start, dma_dir);
  1579. if (unlikely(dma_mapping_error(dev, addr))) {
  1580. if (net_ratelimit())
  1581. netif_err(priv, tx_err, net_dev, "dma_map_single() failed\n");
  1582. return -EINVAL;
  1583. }
  1584. qm_fd_addr_set64(fd, addr);
  1585. return 0;
  1586. }
  1587. static int skb_to_sg_fd(struct dpaa_priv *priv,
  1588. struct sk_buff *skb, struct qm_fd *fd)
  1589. {
  1590. const enum dma_data_direction dma_dir = DMA_TO_DEVICE;
  1591. const int nr_frags = skb_shinfo(skb)->nr_frags;
  1592. struct net_device *net_dev = priv->net_dev;
  1593. struct device *dev = net_dev->dev.parent;
  1594. struct qm_sg_entry *sgt;
  1595. struct sk_buff **skbh;
  1596. int i, j, err, sz;
  1597. void *buffer_start;
  1598. skb_frag_t *frag;
  1599. dma_addr_t addr;
  1600. size_t frag_len;
  1601. void *sgt_buf;
  1602. /* get a page frag to store the SGTable */
  1603. sz = SKB_DATA_ALIGN(priv->tx_headroom +
  1604. sizeof(struct qm_sg_entry) * (1 + nr_frags));
  1605. sgt_buf = netdev_alloc_frag(sz);
  1606. if (unlikely(!sgt_buf)) {
  1607. netdev_err(net_dev, "netdev_alloc_frag() failed for size %d\n",
  1608. sz);
  1609. return -ENOMEM;
  1610. }
  1611. /* Enable L3/L4 hardware checksum computation.
  1612. *
  1613. * We must do this before dma_map_single(DMA_TO_DEVICE), because we may
  1614. * need to write into the skb.
  1615. */
  1616. err = dpaa_enable_tx_csum(priv, skb, fd,
  1617. sgt_buf + DPAA_TX_PRIV_DATA_SIZE);
  1618. if (unlikely(err < 0)) {
  1619. if (net_ratelimit())
  1620. netif_err(priv, tx_err, net_dev, "HW csum error: %d\n",
  1621. err);
  1622. goto csum_failed;
  1623. }
  1624. /* SGT[0] is used by the linear part */
  1625. sgt = (struct qm_sg_entry *)(sgt_buf + priv->tx_headroom);
  1626. frag_len = skb_headlen(skb);
  1627. qm_sg_entry_set_len(&sgt[0], frag_len);
  1628. sgt[0].bpid = FSL_DPAA_BPID_INV;
  1629. sgt[0].offset = 0;
  1630. addr = dma_map_single(dev, skb->data,
  1631. skb_headlen(skb), dma_dir);
  1632. if (unlikely(dma_mapping_error(dev, addr))) {
  1633. dev_err(dev, "DMA mapping failed");
  1634. err = -EINVAL;
  1635. goto sg0_map_failed;
  1636. }
  1637. qm_sg_entry_set64(&sgt[0], addr);
  1638. /* populate the rest of SGT entries */
  1639. for (i = 0; i < nr_frags; i++) {
  1640. frag = &skb_shinfo(skb)->frags[i];
  1641. frag_len = frag->size;
  1642. WARN_ON(!skb_frag_page(frag));
  1643. addr = skb_frag_dma_map(dev, frag, 0,
  1644. frag_len, dma_dir);
  1645. if (unlikely(dma_mapping_error(dev, addr))) {
  1646. dev_err(dev, "DMA mapping failed");
  1647. err = -EINVAL;
  1648. goto sg_map_failed;
  1649. }
  1650. qm_sg_entry_set_len(&sgt[i + 1], frag_len);
  1651. sgt[i + 1].bpid = FSL_DPAA_BPID_INV;
  1652. sgt[i + 1].offset = 0;
  1653. /* keep the offset in the address */
  1654. qm_sg_entry_set64(&sgt[i + 1], addr);
  1655. }
  1656. /* Set the final bit in the last used entry of the SGT */
  1657. qm_sg_entry_set_f(&sgt[nr_frags], frag_len);
  1658. qm_fd_set_sg(fd, priv->tx_headroom, skb->len);
  1659. /* DMA map the SGT page */
  1660. buffer_start = (void *)sgt - priv->tx_headroom;
  1661. skbh = (struct sk_buff **)buffer_start;
  1662. *skbh = skb;
  1663. addr = dma_map_single(dev, buffer_start, priv->tx_headroom +
  1664. sizeof(struct qm_sg_entry) * (1 + nr_frags),
  1665. dma_dir);
  1666. if (unlikely(dma_mapping_error(dev, addr))) {
  1667. dev_err(dev, "DMA mapping failed");
  1668. err = -EINVAL;
  1669. goto sgt_map_failed;
  1670. }
  1671. fd->bpid = FSL_DPAA_BPID_INV;
  1672. fd->cmd |= cpu_to_be32(FM_FD_CMD_FCO);
  1673. qm_fd_addr_set64(fd, addr);
  1674. return 0;
  1675. sgt_map_failed:
  1676. sg_map_failed:
  1677. for (j = 0; j < i; j++)
  1678. dma_unmap_page(dev, qm_sg_addr(&sgt[j]),
  1679. qm_sg_entry_get_len(&sgt[j]), dma_dir);
  1680. sg0_map_failed:
  1681. csum_failed:
  1682. skb_free_frag(sgt_buf);
  1683. return err;
  1684. }
  1685. static inline int dpaa_xmit(struct dpaa_priv *priv,
  1686. struct rtnl_link_stats64 *percpu_stats,
  1687. int queue,
  1688. struct qm_fd *fd)
  1689. {
  1690. struct qman_fq *egress_fq;
  1691. int err, i;
  1692. egress_fq = priv->egress_fqs[queue];
  1693. if (fd->bpid == FSL_DPAA_BPID_INV)
  1694. fd->cmd |= cpu_to_be32(qman_fq_fqid(priv->conf_fqs[queue]));
  1695. /* Trace this Tx fd */
  1696. trace_dpaa_tx_fd(priv->net_dev, egress_fq, fd);
  1697. for (i = 0; i < DPAA_ENQUEUE_RETRIES; i++) {
  1698. err = qman_enqueue(egress_fq, fd);
  1699. if (err != -EBUSY)
  1700. break;
  1701. }
  1702. if (unlikely(err < 0)) {
  1703. percpu_stats->tx_fifo_errors++;
  1704. return err;
  1705. }
  1706. percpu_stats->tx_packets++;
  1707. percpu_stats->tx_bytes += qm_fd_get_length(fd);
  1708. return 0;
  1709. }
  1710. static int dpaa_start_xmit(struct sk_buff *skb, struct net_device *net_dev)
  1711. {
  1712. const int queue_mapping = skb_get_queue_mapping(skb);
  1713. bool nonlinear = skb_is_nonlinear(skb);
  1714. struct rtnl_link_stats64 *percpu_stats;
  1715. struct dpaa_percpu_priv *percpu_priv;
  1716. struct dpaa_priv *priv;
  1717. struct qm_fd fd;
  1718. int offset = 0;
  1719. int err = 0;
  1720. priv = netdev_priv(net_dev);
  1721. percpu_priv = this_cpu_ptr(priv->percpu_priv);
  1722. percpu_stats = &percpu_priv->stats;
  1723. qm_fd_clear_fd(&fd);
  1724. if (!nonlinear) {
  1725. /* We're going to store the skb backpointer at the beginning
  1726. * of the data buffer, so we need a privately owned skb
  1727. *
  1728. * We've made sure skb is not shared in dev->priv_flags,
  1729. * we need to verify the skb head is not cloned
  1730. */
  1731. if (skb_cow_head(skb, priv->tx_headroom))
  1732. goto enomem;
  1733. WARN_ON(skb_is_nonlinear(skb));
  1734. }
  1735. /* MAX_SKB_FRAGS is equal or larger than our dpaa_SGT_MAX_ENTRIES;
  1736. * make sure we don't feed FMan with more fragments than it supports.
  1737. */
  1738. if (unlikely(nonlinear &&
  1739. (skb_shinfo(skb)->nr_frags >= DPAA_SGT_MAX_ENTRIES))) {
  1740. /* If the egress skb contains more fragments than we support
  1741. * we have no choice but to linearize it ourselves.
  1742. */
  1743. if (__skb_linearize(skb))
  1744. goto enomem;
  1745. nonlinear = skb_is_nonlinear(skb);
  1746. }
  1747. if (nonlinear) {
  1748. /* Just create a S/G fd based on the skb */
  1749. err = skb_to_sg_fd(priv, skb, &fd);
  1750. percpu_priv->tx_frag_skbuffs++;
  1751. } else {
  1752. /* Create a contig FD from this skb */
  1753. err = skb_to_contig_fd(priv, skb, &fd, &offset);
  1754. }
  1755. if (unlikely(err < 0))
  1756. goto skb_to_fd_failed;
  1757. if (likely(dpaa_xmit(priv, percpu_stats, queue_mapping, &fd) == 0))
  1758. return NETDEV_TX_OK;
  1759. dpaa_cleanup_tx_fd(priv, &fd);
  1760. skb_to_fd_failed:
  1761. enomem:
  1762. percpu_stats->tx_errors++;
  1763. dev_kfree_skb(skb);
  1764. return NETDEV_TX_OK;
  1765. }
  1766. static void dpaa_rx_error(struct net_device *net_dev,
  1767. const struct dpaa_priv *priv,
  1768. struct dpaa_percpu_priv *percpu_priv,
  1769. const struct qm_fd *fd,
  1770. u32 fqid)
  1771. {
  1772. if (net_ratelimit())
  1773. netif_err(priv, hw, net_dev, "Err FD status = 0x%08x\n",
  1774. be32_to_cpu(fd->status) & FM_FD_STAT_RX_ERRORS);
  1775. percpu_priv->stats.rx_errors++;
  1776. if (be32_to_cpu(fd->status) & FM_FD_ERR_DMA)
  1777. percpu_priv->rx_errors.dme++;
  1778. if (be32_to_cpu(fd->status) & FM_FD_ERR_PHYSICAL)
  1779. percpu_priv->rx_errors.fpe++;
  1780. if (be32_to_cpu(fd->status) & FM_FD_ERR_SIZE)
  1781. percpu_priv->rx_errors.fse++;
  1782. if (be32_to_cpu(fd->status) & FM_FD_ERR_PRS_HDR_ERR)
  1783. percpu_priv->rx_errors.phe++;
  1784. dpaa_fd_release(net_dev, fd);
  1785. }
  1786. static void dpaa_tx_error(struct net_device *net_dev,
  1787. const struct dpaa_priv *priv,
  1788. struct dpaa_percpu_priv *percpu_priv,
  1789. const struct qm_fd *fd,
  1790. u32 fqid)
  1791. {
  1792. struct sk_buff *skb;
  1793. if (net_ratelimit())
  1794. netif_warn(priv, hw, net_dev, "FD status = 0x%08x\n",
  1795. be32_to_cpu(fd->status) & FM_FD_STAT_TX_ERRORS);
  1796. percpu_priv->stats.tx_errors++;
  1797. skb = dpaa_cleanup_tx_fd(priv, fd);
  1798. dev_kfree_skb(skb);
  1799. }
  1800. static int dpaa_eth_poll(struct napi_struct *napi, int budget)
  1801. {
  1802. struct dpaa_napi_portal *np =
  1803. container_of(napi, struct dpaa_napi_portal, napi);
  1804. int cleaned = qman_p_poll_dqrr(np->p, budget);
  1805. if (cleaned < budget) {
  1806. napi_complete_done(napi, cleaned);
  1807. qman_p_irqsource_add(np->p, QM_PIRQ_DQRI);
  1808. } else if (np->down) {
  1809. qman_p_irqsource_add(np->p, QM_PIRQ_DQRI);
  1810. }
  1811. return cleaned;
  1812. }
  1813. static void dpaa_tx_conf(struct net_device *net_dev,
  1814. const struct dpaa_priv *priv,
  1815. struct dpaa_percpu_priv *percpu_priv,
  1816. const struct qm_fd *fd,
  1817. u32 fqid)
  1818. {
  1819. struct sk_buff *skb;
  1820. if (unlikely(be32_to_cpu(fd->status) & FM_FD_STAT_TX_ERRORS)) {
  1821. if (net_ratelimit())
  1822. netif_warn(priv, hw, net_dev, "FD status = 0x%08x\n",
  1823. be32_to_cpu(fd->status) &
  1824. FM_FD_STAT_TX_ERRORS);
  1825. percpu_priv->stats.tx_errors++;
  1826. }
  1827. percpu_priv->tx_confirm++;
  1828. skb = dpaa_cleanup_tx_fd(priv, fd);
  1829. consume_skb(skb);
  1830. }
  1831. static inline int dpaa_eth_napi_schedule(struct dpaa_percpu_priv *percpu_priv,
  1832. struct qman_portal *portal)
  1833. {
  1834. if (unlikely(in_irq() || !in_serving_softirq())) {
  1835. /* Disable QMan IRQ and invoke NAPI */
  1836. qman_p_irqsource_remove(portal, QM_PIRQ_DQRI);
  1837. percpu_priv->np.p = portal;
  1838. napi_schedule(&percpu_priv->np.napi);
  1839. percpu_priv->in_interrupt++;
  1840. return 1;
  1841. }
  1842. return 0;
  1843. }
  1844. static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal,
  1845. struct qman_fq *fq,
  1846. const struct qm_dqrr_entry *dq)
  1847. {
  1848. struct dpaa_fq *dpaa_fq = container_of(fq, struct dpaa_fq, fq_base);
  1849. struct dpaa_percpu_priv *percpu_priv;
  1850. struct net_device *net_dev;
  1851. struct dpaa_bp *dpaa_bp;
  1852. struct dpaa_priv *priv;
  1853. net_dev = dpaa_fq->net_dev;
  1854. priv = netdev_priv(net_dev);
  1855. dpaa_bp = dpaa_bpid2pool(dq->fd.bpid);
  1856. if (!dpaa_bp)
  1857. return qman_cb_dqrr_consume;
  1858. percpu_priv = this_cpu_ptr(priv->percpu_priv);
  1859. if (dpaa_eth_napi_schedule(percpu_priv, portal))
  1860. return qman_cb_dqrr_stop;
  1861. dpaa_eth_refill_bpools(priv);
  1862. dpaa_rx_error(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);
  1863. return qman_cb_dqrr_consume;
  1864. }
  1865. static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
  1866. struct qman_fq *fq,
  1867. const struct qm_dqrr_entry *dq)
  1868. {
  1869. struct rtnl_link_stats64 *percpu_stats;
  1870. struct dpaa_percpu_priv *percpu_priv;
  1871. const struct qm_fd *fd = &dq->fd;
  1872. dma_addr_t addr = qm_fd_addr(fd);
  1873. enum qm_fd_format fd_format;
  1874. struct net_device *net_dev;
  1875. u32 fd_status, hash_offset;
  1876. struct dpaa_bp *dpaa_bp;
  1877. struct dpaa_priv *priv;
  1878. unsigned int skb_len;
  1879. struct sk_buff *skb;
  1880. int *count_ptr;
  1881. void *vaddr;
  1882. fd_status = be32_to_cpu(fd->status);
  1883. fd_format = qm_fd_get_format(fd);
  1884. net_dev = ((struct dpaa_fq *)fq)->net_dev;
  1885. priv = netdev_priv(net_dev);
  1886. dpaa_bp = dpaa_bpid2pool(dq->fd.bpid);
  1887. if (!dpaa_bp)
  1888. return qman_cb_dqrr_consume;
  1889. /* Trace the Rx fd */
  1890. trace_dpaa_rx_fd(net_dev, fq, &dq->fd);
  1891. percpu_priv = this_cpu_ptr(priv->percpu_priv);
  1892. percpu_stats = &percpu_priv->stats;
  1893. if (unlikely(dpaa_eth_napi_schedule(percpu_priv, portal)))
  1894. return qman_cb_dqrr_stop;
  1895. /* Make sure we didn't run out of buffers */
  1896. if (unlikely(dpaa_eth_refill_bpools(priv))) {
  1897. /* Unable to refill the buffer pool due to insufficient
  1898. * system memory. Just release the frame back into the pool,
  1899. * otherwise we'll soon end up with an empty buffer pool.
  1900. */
  1901. dpaa_fd_release(net_dev, &dq->fd);
  1902. return qman_cb_dqrr_consume;
  1903. }
  1904. if (unlikely(fd_status & FM_FD_STAT_RX_ERRORS) != 0) {
  1905. if (net_ratelimit())
  1906. netif_warn(priv, hw, net_dev, "FD status = 0x%08x\n",
  1907. fd_status & FM_FD_STAT_RX_ERRORS);
  1908. percpu_stats->rx_errors++;
  1909. dpaa_fd_release(net_dev, fd);
  1910. return qman_cb_dqrr_consume;
  1911. }
  1912. dpaa_bp = dpaa_bpid2pool(fd->bpid);
  1913. if (!dpaa_bp)
  1914. return qman_cb_dqrr_consume;
  1915. dma_unmap_single(dpaa_bp->dev, addr, dpaa_bp->size, DMA_FROM_DEVICE);
  1916. /* prefetch the first 64 bytes of the frame or the SGT start */
  1917. vaddr = phys_to_virt(addr);
  1918. prefetch(vaddr + qm_fd_get_offset(fd));
  1919. /* The only FD types that we may receive are contig and S/G */
  1920. WARN_ON((fd_format != qm_fd_contig) && (fd_format != qm_fd_sg));
  1921. /* Account for either the contig buffer or the SGT buffer (depending on
  1922. * which case we were in) having been removed from the pool.
  1923. */
  1924. count_ptr = this_cpu_ptr(dpaa_bp->percpu_count);
  1925. (*count_ptr)--;
  1926. if (likely(fd_format == qm_fd_contig))
  1927. skb = contig_fd_to_skb(priv, fd);
  1928. else
  1929. skb = sg_fd_to_skb(priv, fd);
  1930. if (!skb)
  1931. return qman_cb_dqrr_consume;
  1932. skb->protocol = eth_type_trans(skb, net_dev);
  1933. if (net_dev->features & NETIF_F_RXHASH && priv->keygen_in_use &&
  1934. !fman_port_get_hash_result_offset(priv->mac_dev->port[RX],
  1935. &hash_offset)) {
  1936. enum pkt_hash_types type;
  1937. /* if L4 exists, it was used in the hash generation */
  1938. type = be32_to_cpu(fd->status) & FM_FD_STAT_L4CV ?
  1939. PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
  1940. skb_set_hash(skb, be32_to_cpu(*(u32 *)(vaddr + hash_offset)),
  1941. type);
  1942. }
  1943. skb_len = skb->len;
  1944. if (unlikely(netif_receive_skb(skb) == NET_RX_DROP)) {
  1945. percpu_stats->rx_dropped++;
  1946. return qman_cb_dqrr_consume;
  1947. }
  1948. percpu_stats->rx_packets++;
  1949. percpu_stats->rx_bytes += skb_len;
  1950. return qman_cb_dqrr_consume;
  1951. }
  1952. static enum qman_cb_dqrr_result conf_error_dqrr(struct qman_portal *portal,
  1953. struct qman_fq *fq,
  1954. const struct qm_dqrr_entry *dq)
  1955. {
  1956. struct dpaa_percpu_priv *percpu_priv;
  1957. struct net_device *net_dev;
  1958. struct dpaa_priv *priv;
  1959. net_dev = ((struct dpaa_fq *)fq)->net_dev;
  1960. priv = netdev_priv(net_dev);
  1961. percpu_priv = this_cpu_ptr(priv->percpu_priv);
  1962. if (dpaa_eth_napi_schedule(percpu_priv, portal))
  1963. return qman_cb_dqrr_stop;
  1964. dpaa_tx_error(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);
  1965. return qman_cb_dqrr_consume;
  1966. }
  1967. static enum qman_cb_dqrr_result conf_dflt_dqrr(struct qman_portal *portal,
  1968. struct qman_fq *fq,
  1969. const struct qm_dqrr_entry *dq)
  1970. {
  1971. struct dpaa_percpu_priv *percpu_priv;
  1972. struct net_device *net_dev;
  1973. struct dpaa_priv *priv;
  1974. net_dev = ((struct dpaa_fq *)fq)->net_dev;
  1975. priv = netdev_priv(net_dev);
  1976. /* Trace the fd */
  1977. trace_dpaa_tx_conf_fd(net_dev, fq, &dq->fd);
  1978. percpu_priv = this_cpu_ptr(priv->percpu_priv);
  1979. if (dpaa_eth_napi_schedule(percpu_priv, portal))
  1980. return qman_cb_dqrr_stop;
  1981. dpaa_tx_conf(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);
  1982. return qman_cb_dqrr_consume;
  1983. }
  1984. static void egress_ern(struct qman_portal *portal,
  1985. struct qman_fq *fq,
  1986. const union qm_mr_entry *msg)
  1987. {
  1988. const struct qm_fd *fd = &msg->ern.fd;
  1989. struct dpaa_percpu_priv *percpu_priv;
  1990. const struct dpaa_priv *priv;
  1991. struct net_device *net_dev;
  1992. struct sk_buff *skb;
  1993. net_dev = ((struct dpaa_fq *)fq)->net_dev;
  1994. priv = netdev_priv(net_dev);
  1995. percpu_priv = this_cpu_ptr(priv->percpu_priv);
  1996. percpu_priv->stats.tx_dropped++;
  1997. percpu_priv->stats.tx_fifo_errors++;
  1998. count_ern(percpu_priv, msg);
  1999. skb = dpaa_cleanup_tx_fd(priv, fd);
  2000. dev_kfree_skb_any(skb);
  2001. }
  2002. static const struct dpaa_fq_cbs dpaa_fq_cbs = {
  2003. .rx_defq = { .cb = { .dqrr = rx_default_dqrr } },
  2004. .tx_defq = { .cb = { .dqrr = conf_dflt_dqrr } },
  2005. .rx_errq = { .cb = { .dqrr = rx_error_dqrr } },
  2006. .tx_errq = { .cb = { .dqrr = conf_error_dqrr } },
  2007. .egress_ern = { .cb = { .ern = egress_ern } }
  2008. };
  2009. static void dpaa_eth_napi_enable(struct dpaa_priv *priv)
  2010. {
  2011. struct dpaa_percpu_priv *percpu_priv;
  2012. int i;
  2013. for_each_possible_cpu(i) {
  2014. percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
  2015. percpu_priv->np.down = 0;
  2016. napi_enable(&percpu_priv->np.napi);
  2017. }
  2018. }
  2019. static void dpaa_eth_napi_disable(struct dpaa_priv *priv)
  2020. {
  2021. struct dpaa_percpu_priv *percpu_priv;
  2022. int i;
  2023. for_each_possible_cpu(i) {
  2024. percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
  2025. percpu_priv->np.down = 1;
  2026. napi_disable(&percpu_priv->np.napi);
  2027. }
  2028. }
  2029. static void dpaa_adjust_link(struct net_device *net_dev)
  2030. {
  2031. struct mac_device *mac_dev;
  2032. struct dpaa_priv *priv;
  2033. priv = netdev_priv(net_dev);
  2034. mac_dev = priv->mac_dev;
  2035. mac_dev->adjust_link(mac_dev);
  2036. }
  2037. static int dpaa_phy_init(struct net_device *net_dev)
  2038. {
  2039. struct mac_device *mac_dev;
  2040. struct phy_device *phy_dev;
  2041. struct dpaa_priv *priv;
  2042. priv = netdev_priv(net_dev);
  2043. mac_dev = priv->mac_dev;
  2044. phy_dev = of_phy_connect(net_dev, mac_dev->phy_node,
  2045. &dpaa_adjust_link, 0,
  2046. mac_dev->phy_if);
  2047. if (!phy_dev) {
  2048. netif_err(priv, ifup, net_dev, "init_phy() failed\n");
  2049. return -ENODEV;
  2050. }
  2051. /* Remove any features not supported by the controller */
  2052. phy_dev->supported &= mac_dev->if_support;
  2053. phy_dev->supported |= (SUPPORTED_Pause | SUPPORTED_Asym_Pause);
  2054. phy_dev->advertising = phy_dev->supported;
  2055. mac_dev->phy_dev = phy_dev;
  2056. net_dev->phydev = phy_dev;
  2057. return 0;
  2058. }
  2059. static int dpaa_open(struct net_device *net_dev)
  2060. {
  2061. struct mac_device *mac_dev;
  2062. struct dpaa_priv *priv;
  2063. int err, i;
  2064. priv = netdev_priv(net_dev);
  2065. mac_dev = priv->mac_dev;
  2066. dpaa_eth_napi_enable(priv);
  2067. err = dpaa_phy_init(net_dev);
  2068. if (err)
  2069. goto phy_init_failed;
  2070. for (i = 0; i < ARRAY_SIZE(mac_dev->port); i++) {
  2071. err = fman_port_enable(mac_dev->port[i]);
  2072. if (err)
  2073. goto mac_start_failed;
  2074. }
  2075. err = priv->mac_dev->start(mac_dev);
  2076. if (err < 0) {
  2077. netif_err(priv, ifup, net_dev, "mac_dev->start() = %d\n", err);
  2078. goto mac_start_failed;
  2079. }
  2080. netif_tx_start_all_queues(net_dev);
  2081. return 0;
  2082. mac_start_failed:
  2083. for (i = 0; i < ARRAY_SIZE(mac_dev->port); i++)
  2084. fman_port_disable(mac_dev->port[i]);
  2085. phy_init_failed:
  2086. dpaa_eth_napi_disable(priv);
  2087. return err;
  2088. }
  2089. static int dpaa_eth_stop(struct net_device *net_dev)
  2090. {
  2091. struct dpaa_priv *priv;
  2092. int err;
  2093. err = dpaa_stop(net_dev);
  2094. priv = netdev_priv(net_dev);
  2095. dpaa_eth_napi_disable(priv);
  2096. return err;
  2097. }
  2098. static int dpaa_ioctl(struct net_device *net_dev, struct ifreq *rq, int cmd)
  2099. {
  2100. if (!net_dev->phydev)
  2101. return -EINVAL;
  2102. return phy_mii_ioctl(net_dev->phydev, rq, cmd);
  2103. }
  2104. static const struct net_device_ops dpaa_ops = {
  2105. .ndo_open = dpaa_open,
  2106. .ndo_start_xmit = dpaa_start_xmit,
  2107. .ndo_stop = dpaa_eth_stop,
  2108. .ndo_tx_timeout = dpaa_tx_timeout,
  2109. .ndo_get_stats64 = dpaa_get_stats64,
  2110. .ndo_set_mac_address = dpaa_set_mac_address,
  2111. .ndo_validate_addr = eth_validate_addr,
  2112. .ndo_set_rx_mode = dpaa_set_rx_mode,
  2113. .ndo_do_ioctl = dpaa_ioctl,
  2114. .ndo_setup_tc = dpaa_setup_tc,
  2115. };
  2116. static int dpaa_napi_add(struct net_device *net_dev)
  2117. {
  2118. struct dpaa_priv *priv = netdev_priv(net_dev);
  2119. struct dpaa_percpu_priv *percpu_priv;
  2120. int cpu;
  2121. for_each_possible_cpu(cpu) {
  2122. percpu_priv = per_cpu_ptr(priv->percpu_priv, cpu);
  2123. netif_napi_add(net_dev, &percpu_priv->np.napi,
  2124. dpaa_eth_poll, NAPI_POLL_WEIGHT);
  2125. }
  2126. return 0;
  2127. }
  2128. static void dpaa_napi_del(struct net_device *net_dev)
  2129. {
  2130. struct dpaa_priv *priv = netdev_priv(net_dev);
  2131. struct dpaa_percpu_priv *percpu_priv;
  2132. int cpu;
  2133. for_each_possible_cpu(cpu) {
  2134. percpu_priv = per_cpu_ptr(priv->percpu_priv, cpu);
  2135. netif_napi_del(&percpu_priv->np.napi);
  2136. }
  2137. }
  2138. static inline void dpaa_bp_free_pf(const struct dpaa_bp *bp,
  2139. struct bm_buffer *bmb)
  2140. {
  2141. dma_addr_t addr = bm_buf_addr(bmb);
  2142. dma_unmap_single(bp->dev, addr, bp->size, DMA_FROM_DEVICE);
  2143. skb_free_frag(phys_to_virt(addr));
  2144. }
  2145. /* Alloc the dpaa_bp struct and configure default values */
  2146. static struct dpaa_bp *dpaa_bp_alloc(struct device *dev)
  2147. {
  2148. struct dpaa_bp *dpaa_bp;
  2149. dpaa_bp = devm_kzalloc(dev, sizeof(*dpaa_bp), GFP_KERNEL);
  2150. if (!dpaa_bp)
  2151. return ERR_PTR(-ENOMEM);
  2152. dpaa_bp->bpid = FSL_DPAA_BPID_INV;
  2153. dpaa_bp->percpu_count = devm_alloc_percpu(dev, *dpaa_bp->percpu_count);
  2154. if (!dpaa_bp->percpu_count)
  2155. return ERR_PTR(-ENOMEM);
  2156. dpaa_bp->config_count = FSL_DPAA_ETH_MAX_BUF_COUNT;
  2157. dpaa_bp->seed_cb = dpaa_bp_seed;
  2158. dpaa_bp->free_buf_cb = dpaa_bp_free_pf;
  2159. return dpaa_bp;
  2160. }
  2161. /* Place all ingress FQs (Rx Default, Rx Error) in a dedicated CGR.
  2162. * We won't be sending congestion notifications to FMan; for now, we just use
  2163. * this CGR to generate enqueue rejections to FMan in order to drop the frames
  2164. * before they reach our ingress queues and eat up memory.
  2165. */
  2166. static int dpaa_ingress_cgr_init(struct dpaa_priv *priv)
  2167. {
  2168. struct qm_mcc_initcgr initcgr;
  2169. u32 cs_th;
  2170. int err;
  2171. err = qman_alloc_cgrid(&priv->ingress_cgr.cgrid);
  2172. if (err < 0) {
  2173. if (netif_msg_drv(priv))
  2174. pr_err("Error %d allocating CGR ID\n", err);
  2175. goto out_error;
  2176. }
  2177. /* Enable CS TD, but disable Congestion State Change Notifications. */
  2178. memset(&initcgr, 0, sizeof(initcgr));
  2179. initcgr.we_mask = cpu_to_be16(QM_CGR_WE_CS_THRES);
  2180. initcgr.cgr.cscn_en = QM_CGR_EN;
  2181. cs_th = DPAA_INGRESS_CS_THRESHOLD;
  2182. qm_cgr_cs_thres_set64(&initcgr.cgr.cs_thres, cs_th, 1);
  2183. initcgr.we_mask |= cpu_to_be16(QM_CGR_WE_CSTD_EN);
  2184. initcgr.cgr.cstd_en = QM_CGR_EN;
  2185. /* This CGR will be associated with the SWP affined to the current CPU.
  2186. * However, we'll place all our ingress FQs in it.
  2187. */
  2188. err = qman_create_cgr(&priv->ingress_cgr, QMAN_CGR_FLAG_USE_INIT,
  2189. &initcgr);
  2190. if (err < 0) {
  2191. if (netif_msg_drv(priv))
  2192. pr_err("Error %d creating ingress CGR with ID %d\n",
  2193. err, priv->ingress_cgr.cgrid);
  2194. qman_release_cgrid(priv->ingress_cgr.cgrid);
  2195. goto out_error;
  2196. }
  2197. if (netif_msg_drv(priv))
  2198. pr_debug("Created ingress CGR %d for netdev with hwaddr %pM\n",
  2199. priv->ingress_cgr.cgrid, priv->mac_dev->addr);
  2200. priv->use_ingress_cgr = true;
  2201. out_error:
  2202. return err;
  2203. }
  2204. static const struct of_device_id dpaa_match[];
  2205. static inline u16 dpaa_get_headroom(struct dpaa_buffer_layout *bl)
  2206. {
  2207. u16 headroom;
  2208. /* The frame headroom must accommodate:
  2209. * - the driver private data area
  2210. * - parse results, hash results, timestamp if selected
  2211. * If either hash results or time stamp are selected, both will
  2212. * be copied to/from the frame headroom, as TS is located between PR and
  2213. * HR in the IC and IC copy size has a granularity of 16bytes
  2214. * (see description of FMBM_RICP and FMBM_TICP registers in DPAARM)
  2215. *
  2216. * Also make sure the headroom is a multiple of data_align bytes
  2217. */
  2218. headroom = (u16)(bl->priv_data_size + DPAA_PARSE_RESULTS_SIZE +
  2219. DPAA_TIME_STAMP_SIZE + DPAA_HASH_RESULTS_SIZE);
  2220. return DPAA_FD_DATA_ALIGNMENT ? ALIGN(headroom,
  2221. DPAA_FD_DATA_ALIGNMENT) :
  2222. headroom;
  2223. }
  2224. static int dpaa_eth_probe(struct platform_device *pdev)
  2225. {
  2226. struct dpaa_bp *dpaa_bps[DPAA_BPS_NUM] = {NULL};
  2227. struct net_device *net_dev = NULL;
  2228. struct dpaa_fq *dpaa_fq, *tmp;
  2229. struct dpaa_priv *priv = NULL;
  2230. struct fm_port_fqs port_fqs;
  2231. struct mac_device *mac_dev;
  2232. int err = 0, i, channel;
  2233. struct device *dev;
  2234. /* device used for DMA mapping */
  2235. dev = pdev->dev.parent;
  2236. err = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(40));
  2237. if (err) {
  2238. dev_err(dev, "dma_coerce_mask_and_coherent() failed\n");
  2239. return err;
  2240. }
  2241. /* Allocate this early, so we can store relevant information in
  2242. * the private area
  2243. */
  2244. net_dev = alloc_etherdev_mq(sizeof(*priv), DPAA_ETH_TXQ_NUM);
  2245. if (!net_dev) {
  2246. dev_err(dev, "alloc_etherdev_mq() failed\n");
  2247. return -ENOMEM;
  2248. }
  2249. /* Do this here, so we can be verbose early */
  2250. SET_NETDEV_DEV(net_dev, dev);
  2251. dev_set_drvdata(dev, net_dev);
  2252. priv = netdev_priv(net_dev);
  2253. priv->net_dev = net_dev;
  2254. priv->msg_enable = netif_msg_init(debug, DPAA_MSG_DEFAULT);
  2255. mac_dev = dpaa_mac_dev_get(pdev);
  2256. if (IS_ERR(mac_dev)) {
  2257. dev_err(dev, "dpaa_mac_dev_get() failed\n");
  2258. err = PTR_ERR(mac_dev);
  2259. goto free_netdev;
  2260. }
  2261. /* If fsl_fm_max_frm is set to a higher value than the all-common 1500,
  2262. * we choose conservatively and let the user explicitly set a higher
  2263. * MTU via ifconfig. Otherwise, the user may end up with different MTUs
  2264. * in the same LAN.
  2265. * If on the other hand fsl_fm_max_frm has been chosen below 1500,
  2266. * start with the maximum allowed.
  2267. */
  2268. net_dev->mtu = min(dpaa_get_max_mtu(), ETH_DATA_LEN);
  2269. netdev_dbg(net_dev, "Setting initial MTU on net device: %d\n",
  2270. net_dev->mtu);
  2271. priv->buf_layout[RX].priv_data_size = DPAA_RX_PRIV_DATA_SIZE; /* Rx */
  2272. priv->buf_layout[TX].priv_data_size = DPAA_TX_PRIV_DATA_SIZE; /* Tx */
  2273. /* bp init */
  2274. for (i = 0; i < DPAA_BPS_NUM; i++) {
  2275. dpaa_bps[i] = dpaa_bp_alloc(dev);
  2276. if (IS_ERR(dpaa_bps[i])) {
  2277. err = PTR_ERR(dpaa_bps[i]);
  2278. goto free_dpaa_bps;
  2279. }
  2280. /* the raw size of the buffers used for reception */
  2281. dpaa_bps[i]->raw_size = bpool_buffer_raw_size(i, DPAA_BPS_NUM);
  2282. /* avoid runtime computations by keeping the usable size here */
  2283. dpaa_bps[i]->size = dpaa_bp_size(dpaa_bps[i]->raw_size);
  2284. dpaa_bps[i]->dev = dev;
  2285. err = dpaa_bp_alloc_pool(dpaa_bps[i]);
  2286. if (err < 0)
  2287. goto free_dpaa_bps;
  2288. priv->dpaa_bps[i] = dpaa_bps[i];
  2289. }
  2290. INIT_LIST_HEAD(&priv->dpaa_fq_list);
  2291. memset(&port_fqs, 0, sizeof(port_fqs));
  2292. err = dpaa_alloc_all_fqs(dev, &priv->dpaa_fq_list, &port_fqs);
  2293. if (err < 0) {
  2294. dev_err(dev, "dpaa_alloc_all_fqs() failed\n");
  2295. goto free_dpaa_bps;
  2296. }
  2297. priv->mac_dev = mac_dev;
  2298. channel = dpaa_get_channel();
  2299. if (channel < 0) {
  2300. dev_err(dev, "dpaa_get_channel() failed\n");
  2301. err = channel;
  2302. goto free_dpaa_bps;
  2303. }
  2304. priv->channel = (u16)channel;
  2305. /* Walk the CPUs with affine portals
  2306. * and add this pool channel to each's dequeue mask.
  2307. */
  2308. dpaa_eth_add_channel(priv->channel);
  2309. dpaa_fq_setup(priv, &dpaa_fq_cbs, priv->mac_dev->port[TX]);
  2310. /* Create a congestion group for this netdev, with
  2311. * dynamically-allocated CGR ID.
  2312. * Must be executed after probing the MAC, but before
  2313. * assigning the egress FQs to the CGRs.
  2314. */
  2315. err = dpaa_eth_cgr_init(priv);
  2316. if (err < 0) {
  2317. dev_err(dev, "Error initializing CGR\n");
  2318. goto free_dpaa_bps;
  2319. }
  2320. err = dpaa_ingress_cgr_init(priv);
  2321. if (err < 0) {
  2322. dev_err(dev, "Error initializing ingress CGR\n");
  2323. goto delete_egress_cgr;
  2324. }
  2325. /* Add the FQs to the interface, and make them active */
  2326. list_for_each_entry_safe(dpaa_fq, tmp, &priv->dpaa_fq_list, list) {
  2327. err = dpaa_fq_init(dpaa_fq, false);
  2328. if (err < 0)
  2329. goto free_dpaa_fqs;
  2330. }
  2331. priv->tx_headroom = dpaa_get_headroom(&priv->buf_layout[TX]);
  2332. priv->rx_headroom = dpaa_get_headroom(&priv->buf_layout[RX]);
  2333. /* All real interfaces need their ports initialized */
  2334. err = dpaa_eth_init_ports(mac_dev, dpaa_bps, DPAA_BPS_NUM, &port_fqs,
  2335. &priv->buf_layout[0], dev);
  2336. if (err)
  2337. goto free_dpaa_fqs;
  2338. /* Rx traffic distribution based on keygen hashing defaults to on */
  2339. priv->keygen_in_use = true;
  2340. priv->percpu_priv = devm_alloc_percpu(dev, *priv->percpu_priv);
  2341. if (!priv->percpu_priv) {
  2342. dev_err(dev, "devm_alloc_percpu() failed\n");
  2343. err = -ENOMEM;
  2344. goto free_dpaa_fqs;
  2345. }
  2346. priv->num_tc = 1;
  2347. netif_set_real_num_tx_queues(net_dev, priv->num_tc * DPAA_TC_TXQ_NUM);
  2348. /* Initialize NAPI */
  2349. err = dpaa_napi_add(net_dev);
  2350. if (err < 0)
  2351. goto delete_dpaa_napi;
  2352. err = dpaa_netdev_init(net_dev, &dpaa_ops, tx_timeout);
  2353. if (err < 0)
  2354. goto delete_dpaa_napi;
  2355. dpaa_eth_sysfs_init(&net_dev->dev);
  2356. netif_info(priv, probe, net_dev, "Probed interface %s\n",
  2357. net_dev->name);
  2358. return 0;
  2359. delete_dpaa_napi:
  2360. dpaa_napi_del(net_dev);
  2361. free_dpaa_fqs:
  2362. dpaa_fq_free(dev, &priv->dpaa_fq_list);
  2363. qman_delete_cgr_safe(&priv->ingress_cgr);
  2364. qman_release_cgrid(priv->ingress_cgr.cgrid);
  2365. delete_egress_cgr:
  2366. qman_delete_cgr_safe(&priv->cgr_data.cgr);
  2367. qman_release_cgrid(priv->cgr_data.cgr.cgrid);
  2368. free_dpaa_bps:
  2369. dpaa_bps_free(priv);
  2370. free_netdev:
  2371. dev_set_drvdata(dev, NULL);
  2372. free_netdev(net_dev);
  2373. return err;
  2374. }
  2375. static int dpaa_remove(struct platform_device *pdev)
  2376. {
  2377. struct net_device *net_dev;
  2378. struct dpaa_priv *priv;
  2379. struct device *dev;
  2380. int err;
  2381. dev = pdev->dev.parent;
  2382. net_dev = dev_get_drvdata(dev);
  2383. priv = netdev_priv(net_dev);
  2384. dpaa_eth_sysfs_remove(dev);
  2385. dev_set_drvdata(dev, NULL);
  2386. unregister_netdev(net_dev);
  2387. err = dpaa_fq_free(dev, &priv->dpaa_fq_list);
  2388. qman_delete_cgr_safe(&priv->ingress_cgr);
  2389. qman_release_cgrid(priv->ingress_cgr.cgrid);
  2390. qman_delete_cgr_safe(&priv->cgr_data.cgr);
  2391. qman_release_cgrid(priv->cgr_data.cgr.cgrid);
  2392. dpaa_napi_del(net_dev);
  2393. dpaa_bps_free(priv);
  2394. free_netdev(net_dev);
  2395. return err;
  2396. }
  2397. static const struct platform_device_id dpaa_devtype[] = {
  2398. {
  2399. .name = "dpaa-ethernet",
  2400. .driver_data = 0,
  2401. }, {
  2402. }
  2403. };
  2404. MODULE_DEVICE_TABLE(platform, dpaa_devtype);
  2405. static struct platform_driver dpaa_driver = {
  2406. .driver = {
  2407. .name = KBUILD_MODNAME,
  2408. },
  2409. .id_table = dpaa_devtype,
  2410. .probe = dpaa_eth_probe,
  2411. .remove = dpaa_remove
  2412. };
  2413. static int __init dpaa_load(void)
  2414. {
  2415. int err;
  2416. pr_debug("FSL DPAA Ethernet driver\n");
  2417. /* initialize dpaa_eth mirror values */
  2418. dpaa_rx_extra_headroom = fman_get_rx_extra_headroom();
  2419. dpaa_max_frm = fman_get_max_frm();
  2420. err = platform_driver_register(&dpaa_driver);
  2421. if (err < 0)
  2422. pr_err("Error, platform_driver_register() = %d\n", err);
  2423. return err;
  2424. }
  2425. module_init(dpaa_load);
  2426. static void __exit dpaa_unload(void)
  2427. {
  2428. platform_driver_unregister(&dpaa_driver);
  2429. /* Only one channel is used and needs to be released after all
  2430. * interfaces are removed
  2431. */
  2432. dpaa_release_channel();
  2433. }
  2434. module_exit(dpaa_unload);
  2435. MODULE_LICENSE("Dual BSD/GPL");
  2436. MODULE_DESCRIPTION("FSL DPAA Ethernet driver");