virtio_net.c 51 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028
  1. /* A network driver using virtio.
  2. *
  3. * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
  4. *
  5. * This program is free software; you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License as published by
  7. * the Free Software Foundation; either version 2 of the License, or
  8. * (at your option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program; if not, see <http://www.gnu.org/licenses/>.
  17. */
  18. //#define DEBUG
  19. #include <linux/netdevice.h>
  20. #include <linux/etherdevice.h>
  21. #include <linux/ethtool.h>
  22. #include <linux/module.h>
  23. #include <linux/virtio.h>
  24. #include <linux/virtio_net.h>
  25. #include <linux/scatterlist.h>
  26. #include <linux/if_vlan.h>
  27. #include <linux/slab.h>
  28. #include <linux/cpu.h>
  29. #include <linux/average.h>
  30. #include <net/busy_poll.h>
  31. static int napi_weight = NAPI_POLL_WEIGHT;
  32. module_param(napi_weight, int, 0444);
  33. static bool csum = true, gso = true;
  34. module_param(csum, bool, 0444);
  35. module_param(gso, bool, 0444);
  36. /* FIXME: MTU in config. */
  37. #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
  38. #define GOOD_COPY_LEN 128
  39. /* Weight used for the RX packet size EWMA. The average packet size is used to
  40. * determine the packet buffer size when refilling RX rings. As the entire RX
  41. * ring may be refilled at once, the weight is chosen so that the EWMA will be
  42. * insensitive to short-term, transient changes in packet size.
  43. */
  44. #define RECEIVE_AVG_WEIGHT 64
  45. /* Minimum alignment for mergeable packet buffers. */
  46. #define MERGEABLE_BUFFER_ALIGN max(L1_CACHE_BYTES, 256)
  47. #define VIRTNET_DRIVER_VERSION "1.0.0"
  48. struct virtnet_stats {
  49. struct u64_stats_sync tx_syncp;
  50. struct u64_stats_sync rx_syncp;
  51. u64 tx_bytes;
  52. u64 tx_packets;
  53. u64 rx_bytes;
  54. u64 rx_packets;
  55. };
  56. /* Internal representation of a send virtqueue */
  57. struct send_queue {
  58. /* Virtqueue associated with this send _queue */
  59. struct virtqueue *vq;
  60. /* TX: fragments + linear part + virtio header */
  61. struct scatterlist sg[MAX_SKB_FRAGS + 2];
  62. /* Name of the send queue: output.$index */
  63. char name[40];
  64. };
  65. /* Internal representation of a receive virtqueue */
  66. struct receive_queue {
  67. /* Virtqueue associated with this receive_queue */
  68. struct virtqueue *vq;
  69. struct napi_struct napi;
  70. /* Chain pages by the private ptr. */
  71. struct page *pages;
  72. /* Average packet length for mergeable receive buffers. */
  73. struct ewma mrg_avg_pkt_len;
  74. /* Page frag for packet buffer allocation. */
  75. struct page_frag alloc_frag;
  76. /* RX: fragments + linear part + virtio header */
  77. struct scatterlist sg[MAX_SKB_FRAGS + 2];
  78. /* Name of this receive queue: input.$index */
  79. char name[40];
  80. };
  81. struct virtnet_info {
  82. struct virtio_device *vdev;
  83. struct virtqueue *cvq;
  84. struct net_device *dev;
  85. struct send_queue *sq;
  86. struct receive_queue *rq;
  87. unsigned int status;
  88. /* Max # of queue pairs supported by the device */
  89. u16 max_queue_pairs;
  90. /* # of queue pairs currently used by the driver */
  91. u16 curr_queue_pairs;
  92. /* I like... big packets and I cannot lie! */
  93. bool big_packets;
  94. /* Host will merge rx buffers for big packets (shake it! shake it!) */
  95. bool mergeable_rx_bufs;
  96. /* Has control virtqueue */
  97. bool has_cvq;
  98. /* Host can handle any s/g split between our header and packet data */
  99. bool any_header_sg;
  100. /* Packet virtio header size */
  101. u8 hdr_len;
  102. /* Active statistics */
  103. struct virtnet_stats __percpu *stats;
  104. /* Work struct for refilling if we run low on memory. */
  105. struct delayed_work refill;
  106. /* Work struct for config space updates */
  107. struct work_struct config_work;
  108. /* Does the affinity hint is set for virtqueues? */
  109. bool affinity_hint_set;
  110. /* CPU hot plug notifier */
  111. struct notifier_block nb;
  112. };
  113. struct padded_vnet_hdr {
  114. struct virtio_net_hdr_mrg_rxbuf hdr;
  115. /*
  116. * hdr is in a separate sg buffer, and data sg buffer shares same page
  117. * with this header sg. This padding makes next sg 16 byte aligned
  118. * after the header.
  119. */
  120. char padding[4];
  121. };
  122. /* Converting between virtqueue no. and kernel tx/rx queue no.
  123. * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
  124. */
  125. static int vq2txq(struct virtqueue *vq)
  126. {
  127. return (vq->index - 1) / 2;
  128. }
  129. static int txq2vq(int txq)
  130. {
  131. return txq * 2 + 1;
  132. }
  133. static int vq2rxq(struct virtqueue *vq)
  134. {
  135. return vq->index / 2;
  136. }
  137. static int rxq2vq(int rxq)
  138. {
  139. return rxq * 2;
  140. }
  141. static inline struct virtio_net_hdr_mrg_rxbuf *skb_vnet_hdr(struct sk_buff *skb)
  142. {
  143. return (struct virtio_net_hdr_mrg_rxbuf *)skb->cb;
  144. }
  145. /*
  146. * private is used to chain pages for big packets, put the whole
  147. * most recent used list in the beginning for reuse
  148. */
  149. static void give_pages(struct receive_queue *rq, struct page *page)
  150. {
  151. struct page *end;
  152. /* Find end of list, sew whole thing into vi->rq.pages. */
  153. for (end = page; end->private; end = (struct page *)end->private);
  154. end->private = (unsigned long)rq->pages;
  155. rq->pages = page;
  156. }
  157. static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
  158. {
  159. struct page *p = rq->pages;
  160. if (p) {
  161. rq->pages = (struct page *)p->private;
  162. /* clear private here, it is used to chain pages */
  163. p->private = 0;
  164. } else
  165. p = alloc_page(gfp_mask);
  166. return p;
  167. }
  168. static void skb_xmit_done(struct virtqueue *vq)
  169. {
  170. struct virtnet_info *vi = vq->vdev->priv;
  171. /* Suppress further interrupts. */
  172. virtqueue_disable_cb(vq);
  173. /* We were probably waiting for more output buffers. */
  174. netif_wake_subqueue(vi->dev, vq2txq(vq));
  175. }
  176. static unsigned int mergeable_ctx_to_buf_truesize(unsigned long mrg_ctx)
  177. {
  178. unsigned int truesize = mrg_ctx & (MERGEABLE_BUFFER_ALIGN - 1);
  179. return (truesize + 1) * MERGEABLE_BUFFER_ALIGN;
  180. }
  181. static void *mergeable_ctx_to_buf_address(unsigned long mrg_ctx)
  182. {
  183. return (void *)(mrg_ctx & -MERGEABLE_BUFFER_ALIGN);
  184. }
  185. static unsigned long mergeable_buf_to_ctx(void *buf, unsigned int truesize)
  186. {
  187. unsigned int size = truesize / MERGEABLE_BUFFER_ALIGN;
  188. return (unsigned long)buf | (size - 1);
  189. }
  190. /* Called from bottom half context */
  191. static struct sk_buff *page_to_skb(struct virtnet_info *vi,
  192. struct receive_queue *rq,
  193. struct page *page, unsigned int offset,
  194. unsigned int len, unsigned int truesize)
  195. {
  196. struct sk_buff *skb;
  197. struct virtio_net_hdr_mrg_rxbuf *hdr;
  198. unsigned int copy, hdr_len, hdr_padded_len;
  199. char *p;
  200. p = page_address(page) + offset;
  201. /* copy small packet so we can reuse these pages for small data */
  202. skb = netdev_alloc_skb_ip_align(vi->dev, GOOD_COPY_LEN);
  203. if (unlikely(!skb))
  204. return NULL;
  205. hdr = skb_vnet_hdr(skb);
  206. hdr_len = vi->hdr_len;
  207. if (vi->mergeable_rx_bufs)
  208. hdr_padded_len = sizeof *hdr;
  209. else
  210. hdr_padded_len = sizeof(struct padded_vnet_hdr);
  211. memcpy(hdr, p, hdr_len);
  212. len -= hdr_len;
  213. offset += hdr_padded_len;
  214. p += hdr_padded_len;
  215. copy = len;
  216. if (copy > skb_tailroom(skb))
  217. copy = skb_tailroom(skb);
  218. memcpy(skb_put(skb, copy), p, copy);
  219. len -= copy;
  220. offset += copy;
  221. if (vi->mergeable_rx_bufs) {
  222. if (len)
  223. skb_add_rx_frag(skb, 0, page, offset, len, truesize);
  224. else
  225. put_page(page);
  226. return skb;
  227. }
  228. /*
  229. * Verify that we can indeed put this data into a skb.
  230. * This is here to handle cases when the device erroneously
  231. * tries to receive more than is possible. This is usually
  232. * the case of a broken device.
  233. */
  234. if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
  235. net_dbg_ratelimited("%s: too much data\n", skb->dev->name);
  236. dev_kfree_skb(skb);
  237. return NULL;
  238. }
  239. BUG_ON(offset >= PAGE_SIZE);
  240. while (len) {
  241. unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len);
  242. skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset,
  243. frag_size, truesize);
  244. len -= frag_size;
  245. page = (struct page *)page->private;
  246. offset = 0;
  247. }
  248. if (page)
  249. give_pages(rq, page);
  250. return skb;
  251. }
  252. static struct sk_buff *receive_small(struct virtnet_info *vi, void *buf, unsigned int len)
  253. {
  254. struct sk_buff * skb = buf;
  255. len -= vi->hdr_len;
  256. skb_trim(skb, len);
  257. return skb;
  258. }
  259. static struct sk_buff *receive_big(struct net_device *dev,
  260. struct virtnet_info *vi,
  261. struct receive_queue *rq,
  262. void *buf,
  263. unsigned int len)
  264. {
  265. struct page *page = buf;
  266. struct sk_buff *skb = page_to_skb(vi, rq, page, 0, len, PAGE_SIZE);
  267. if (unlikely(!skb))
  268. goto err;
  269. return skb;
  270. err:
  271. dev->stats.rx_dropped++;
  272. give_pages(rq, page);
  273. return NULL;
  274. }
  275. static struct sk_buff *receive_mergeable(struct net_device *dev,
  276. struct virtnet_info *vi,
  277. struct receive_queue *rq,
  278. unsigned long ctx,
  279. unsigned int len)
  280. {
  281. void *buf = mergeable_ctx_to_buf_address(ctx);
  282. struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
  283. u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
  284. struct page *page = virt_to_head_page(buf);
  285. int offset = buf - page_address(page);
  286. unsigned int truesize = max(len, mergeable_ctx_to_buf_truesize(ctx));
  287. struct sk_buff *head_skb = page_to_skb(vi, rq, page, offset, len,
  288. truesize);
  289. struct sk_buff *curr_skb = head_skb;
  290. if (unlikely(!curr_skb))
  291. goto err_skb;
  292. while (--num_buf) {
  293. int num_skb_frags;
  294. ctx = (unsigned long)virtqueue_get_buf(rq->vq, &len);
  295. if (unlikely(!ctx)) {
  296. pr_debug("%s: rx error: %d buffers out of %d missing\n",
  297. dev->name, num_buf,
  298. virtio16_to_cpu(vi->vdev,
  299. hdr->num_buffers));
  300. dev->stats.rx_length_errors++;
  301. goto err_buf;
  302. }
  303. buf = mergeable_ctx_to_buf_address(ctx);
  304. page = virt_to_head_page(buf);
  305. num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
  306. if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
  307. struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC);
  308. if (unlikely(!nskb))
  309. goto err_skb;
  310. if (curr_skb == head_skb)
  311. skb_shinfo(curr_skb)->frag_list = nskb;
  312. else
  313. curr_skb->next = nskb;
  314. curr_skb = nskb;
  315. head_skb->truesize += nskb->truesize;
  316. num_skb_frags = 0;
  317. }
  318. truesize = max(len, mergeable_ctx_to_buf_truesize(ctx));
  319. if (curr_skb != head_skb) {
  320. head_skb->data_len += len;
  321. head_skb->len += len;
  322. head_skb->truesize += truesize;
  323. }
  324. offset = buf - page_address(page);
  325. if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
  326. put_page(page);
  327. skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
  328. len, truesize);
  329. } else {
  330. skb_add_rx_frag(curr_skb, num_skb_frags, page,
  331. offset, len, truesize);
  332. }
  333. }
  334. ewma_add(&rq->mrg_avg_pkt_len, head_skb->len);
  335. return head_skb;
  336. err_skb:
  337. put_page(page);
  338. while (--num_buf) {
  339. ctx = (unsigned long)virtqueue_get_buf(rq->vq, &len);
  340. if (unlikely(!ctx)) {
  341. pr_debug("%s: rx error: %d buffers missing\n",
  342. dev->name, num_buf);
  343. dev->stats.rx_length_errors++;
  344. break;
  345. }
  346. page = virt_to_head_page(mergeable_ctx_to_buf_address(ctx));
  347. put_page(page);
  348. }
  349. err_buf:
  350. dev->stats.rx_dropped++;
  351. dev_kfree_skb(head_skb);
  352. return NULL;
  353. }
  354. static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
  355. void *buf, unsigned int len)
  356. {
  357. struct net_device *dev = vi->dev;
  358. struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
  359. struct sk_buff *skb;
  360. struct virtio_net_hdr_mrg_rxbuf *hdr;
  361. if (unlikely(len < vi->hdr_len + ETH_HLEN)) {
  362. pr_debug("%s: short packet %i\n", dev->name, len);
  363. dev->stats.rx_length_errors++;
  364. if (vi->mergeable_rx_bufs) {
  365. unsigned long ctx = (unsigned long)buf;
  366. void *base = mergeable_ctx_to_buf_address(ctx);
  367. put_page(virt_to_head_page(base));
  368. } else if (vi->big_packets) {
  369. give_pages(rq, buf);
  370. } else {
  371. dev_kfree_skb(buf);
  372. }
  373. return;
  374. }
  375. if (vi->mergeable_rx_bufs)
  376. skb = receive_mergeable(dev, vi, rq, (unsigned long)buf, len);
  377. else if (vi->big_packets)
  378. skb = receive_big(dev, vi, rq, buf, len);
  379. else
  380. skb = receive_small(vi, buf, len);
  381. if (unlikely(!skb))
  382. return;
  383. hdr = skb_vnet_hdr(skb);
  384. u64_stats_update_begin(&stats->rx_syncp);
  385. stats->rx_bytes += skb->len;
  386. stats->rx_packets++;
  387. u64_stats_update_end(&stats->rx_syncp);
  388. if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
  389. pr_debug("Needs csum!\n");
  390. if (!skb_partial_csum_set(skb,
  391. virtio16_to_cpu(vi->vdev, hdr->hdr.csum_start),
  392. virtio16_to_cpu(vi->vdev, hdr->hdr.csum_offset)))
  393. goto frame_err;
  394. } else if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) {
  395. skb->ip_summed = CHECKSUM_UNNECESSARY;
  396. }
  397. skb->protocol = eth_type_trans(skb, dev);
  398. pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
  399. ntohs(skb->protocol), skb->len, skb->pkt_type);
  400. if (hdr->hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
  401. pr_debug("GSO!\n");
  402. switch (hdr->hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
  403. case VIRTIO_NET_HDR_GSO_TCPV4:
  404. skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
  405. break;
  406. case VIRTIO_NET_HDR_GSO_UDP:
  407. skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
  408. break;
  409. case VIRTIO_NET_HDR_GSO_TCPV6:
  410. skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
  411. break;
  412. default:
  413. net_warn_ratelimited("%s: bad gso type %u.\n",
  414. dev->name, hdr->hdr.gso_type);
  415. goto frame_err;
  416. }
  417. if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
  418. skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
  419. skb_shinfo(skb)->gso_size = virtio16_to_cpu(vi->vdev,
  420. hdr->hdr.gso_size);
  421. if (skb_shinfo(skb)->gso_size == 0) {
  422. net_warn_ratelimited("%s: zero gso size.\n", dev->name);
  423. goto frame_err;
  424. }
  425. /* Header must be checked, and gso_segs computed. */
  426. skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
  427. skb_shinfo(skb)->gso_segs = 0;
  428. }
  429. skb_mark_napi_id(skb, &rq->napi);
  430. netif_receive_skb(skb);
  431. return;
  432. frame_err:
  433. dev->stats.rx_frame_errors++;
  434. dev_kfree_skb(skb);
  435. }
  436. static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
  437. gfp_t gfp)
  438. {
  439. struct sk_buff *skb;
  440. struct virtio_net_hdr_mrg_rxbuf *hdr;
  441. int err;
  442. skb = __netdev_alloc_skb_ip_align(vi->dev, GOOD_PACKET_LEN, gfp);
  443. if (unlikely(!skb))
  444. return -ENOMEM;
  445. skb_put(skb, GOOD_PACKET_LEN);
  446. hdr = skb_vnet_hdr(skb);
  447. sg_init_table(rq->sg, MAX_SKB_FRAGS + 2);
  448. sg_set_buf(rq->sg, hdr, vi->hdr_len);
  449. skb_to_sgvec(skb, rq->sg + 1, 0, skb->len);
  450. err = virtqueue_add_inbuf(rq->vq, rq->sg, 2, skb, gfp);
  451. if (err < 0)
  452. dev_kfree_skb(skb);
  453. return err;
  454. }
  455. static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
  456. gfp_t gfp)
  457. {
  458. struct page *first, *list = NULL;
  459. char *p;
  460. int i, err, offset;
  461. sg_init_table(rq->sg, MAX_SKB_FRAGS + 2);
  462. /* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */
  463. for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
  464. first = get_a_page(rq, gfp);
  465. if (!first) {
  466. if (list)
  467. give_pages(rq, list);
  468. return -ENOMEM;
  469. }
  470. sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE);
  471. /* chain new page in list head to match sg */
  472. first->private = (unsigned long)list;
  473. list = first;
  474. }
  475. first = get_a_page(rq, gfp);
  476. if (!first) {
  477. give_pages(rq, list);
  478. return -ENOMEM;
  479. }
  480. p = page_address(first);
  481. /* rq->sg[0], rq->sg[1] share the same page */
  482. /* a separated rq->sg[0] for header - required in case !any_header_sg */
  483. sg_set_buf(&rq->sg[0], p, vi->hdr_len);
  484. /* rq->sg[1] for data packet, from offset */
  485. offset = sizeof(struct padded_vnet_hdr);
  486. sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset);
  487. /* chain first in list head */
  488. first->private = (unsigned long)list;
  489. err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2,
  490. first, gfp);
  491. if (err < 0)
  492. give_pages(rq, first);
  493. return err;
  494. }
  495. static unsigned int get_mergeable_buf_len(struct ewma *avg_pkt_len)
  496. {
  497. const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
  498. unsigned int len;
  499. len = hdr_len + clamp_t(unsigned int, ewma_read(avg_pkt_len),
  500. GOOD_PACKET_LEN, PAGE_SIZE - hdr_len);
  501. return ALIGN(len, MERGEABLE_BUFFER_ALIGN);
  502. }
  503. static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
  504. {
  505. struct page_frag *alloc_frag = &rq->alloc_frag;
  506. char *buf;
  507. unsigned long ctx;
  508. int err;
  509. unsigned int len, hole;
  510. len = get_mergeable_buf_len(&rq->mrg_avg_pkt_len);
  511. if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp)))
  512. return -ENOMEM;
  513. buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
  514. ctx = mergeable_buf_to_ctx(buf, len);
  515. get_page(alloc_frag->page);
  516. alloc_frag->offset += len;
  517. hole = alloc_frag->size - alloc_frag->offset;
  518. if (hole < len) {
  519. /* To avoid internal fragmentation, if there is very likely not
  520. * enough space for another buffer, add the remaining space to
  521. * the current buffer. This extra space is not included in
  522. * the truesize stored in ctx.
  523. */
  524. len += hole;
  525. alloc_frag->offset += hole;
  526. }
  527. sg_init_one(rq->sg, buf, len);
  528. err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, (void *)ctx, gfp);
  529. if (err < 0)
  530. put_page(virt_to_head_page(buf));
  531. return err;
  532. }
  533. /*
  534. * Returns false if we couldn't fill entirely (OOM).
  535. *
  536. * Normally run in the receive path, but can also be run from ndo_open
  537. * before we're receiving packets, or from refill_work which is
  538. * careful to disable receiving (using napi_disable).
  539. */
  540. static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
  541. gfp_t gfp)
  542. {
  543. int err;
  544. bool oom;
  545. gfp |= __GFP_COLD;
  546. do {
  547. if (vi->mergeable_rx_bufs)
  548. err = add_recvbuf_mergeable(rq, gfp);
  549. else if (vi->big_packets)
  550. err = add_recvbuf_big(vi, rq, gfp);
  551. else
  552. err = add_recvbuf_small(vi, rq, gfp);
  553. oom = err == -ENOMEM;
  554. if (err)
  555. break;
  556. } while (rq->vq->num_free);
  557. virtqueue_kick(rq->vq);
  558. return !oom;
  559. }
  560. static void skb_recv_done(struct virtqueue *rvq)
  561. {
  562. struct virtnet_info *vi = rvq->vdev->priv;
  563. struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
  564. /* Schedule NAPI, Suppress further interrupts if successful. */
  565. if (napi_schedule_prep(&rq->napi)) {
  566. virtqueue_disable_cb(rvq);
  567. __napi_schedule(&rq->napi);
  568. }
  569. }
  570. static void virtnet_napi_enable(struct receive_queue *rq)
  571. {
  572. napi_enable(&rq->napi);
  573. /* If all buffers were filled by other side before we napi_enabled, we
  574. * won't get another interrupt, so process any outstanding packets
  575. * now. virtnet_poll wants re-enable the queue, so we disable here.
  576. * We synchronize against interrupts via NAPI_STATE_SCHED */
  577. if (napi_schedule_prep(&rq->napi)) {
  578. virtqueue_disable_cb(rq->vq);
  579. local_bh_disable();
  580. __napi_schedule(&rq->napi);
  581. local_bh_enable();
  582. }
  583. }
  584. static void refill_work(struct work_struct *work)
  585. {
  586. struct virtnet_info *vi =
  587. container_of(work, struct virtnet_info, refill.work);
  588. bool still_empty;
  589. int i;
  590. for (i = 0; i < vi->curr_queue_pairs; i++) {
  591. struct receive_queue *rq = &vi->rq[i];
  592. napi_disable(&rq->napi);
  593. still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);
  594. virtnet_napi_enable(rq);
  595. /* In theory, this can happen: if we don't get any buffers in
  596. * we will *never* try to fill again.
  597. */
  598. if (still_empty)
  599. schedule_delayed_work(&vi->refill, HZ/2);
  600. }
  601. }
  602. static int virtnet_receive(struct receive_queue *rq, int budget)
  603. {
  604. struct virtnet_info *vi = rq->vq->vdev->priv;
  605. unsigned int len, received = 0;
  606. void *buf;
  607. while (received < budget &&
  608. (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
  609. receive_buf(vi, rq, buf, len);
  610. received++;
  611. }
  612. if (rq->vq->num_free > virtqueue_get_vring_size(rq->vq) / 2) {
  613. if (!try_fill_recv(vi, rq, GFP_ATOMIC))
  614. schedule_delayed_work(&vi->refill, 0);
  615. }
  616. return received;
  617. }
  618. static int virtnet_poll(struct napi_struct *napi, int budget)
  619. {
  620. struct receive_queue *rq =
  621. container_of(napi, struct receive_queue, napi);
  622. unsigned int r, received = 0;
  623. received += virtnet_receive(rq, budget - received);
  624. /* Out of packets? */
  625. if (received < budget) {
  626. r = virtqueue_enable_cb_prepare(rq->vq);
  627. napi_complete(napi);
  628. if (unlikely(virtqueue_poll(rq->vq, r)) &&
  629. napi_schedule_prep(napi)) {
  630. virtqueue_disable_cb(rq->vq);
  631. __napi_schedule(napi);
  632. }
  633. }
  634. return received;
  635. }
  636. #ifdef CONFIG_NET_RX_BUSY_POLL
  637. /* must be called with local_bh_disable()d */
  638. static int virtnet_busy_poll(struct napi_struct *napi)
  639. {
  640. struct receive_queue *rq =
  641. container_of(napi, struct receive_queue, napi);
  642. struct virtnet_info *vi = rq->vq->vdev->priv;
  643. int r, received = 0, budget = 4;
  644. if (!(vi->status & VIRTIO_NET_S_LINK_UP))
  645. return LL_FLUSH_FAILED;
  646. if (!napi_schedule_prep(napi))
  647. return LL_FLUSH_BUSY;
  648. virtqueue_disable_cb(rq->vq);
  649. again:
  650. received += virtnet_receive(rq, budget);
  651. r = virtqueue_enable_cb_prepare(rq->vq);
  652. clear_bit(NAPI_STATE_SCHED, &napi->state);
  653. if (unlikely(virtqueue_poll(rq->vq, r)) &&
  654. napi_schedule_prep(napi)) {
  655. virtqueue_disable_cb(rq->vq);
  656. if (received < budget) {
  657. budget -= received;
  658. goto again;
  659. } else {
  660. __napi_schedule(napi);
  661. }
  662. }
  663. return received;
  664. }
  665. #endif /* CONFIG_NET_RX_BUSY_POLL */
  666. static int virtnet_open(struct net_device *dev)
  667. {
  668. struct virtnet_info *vi = netdev_priv(dev);
  669. int i;
  670. for (i = 0; i < vi->max_queue_pairs; i++) {
  671. if (i < vi->curr_queue_pairs)
  672. /* Make sure we have some buffers: if oom use wq. */
  673. if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
  674. schedule_delayed_work(&vi->refill, 0);
  675. virtnet_napi_enable(&vi->rq[i]);
  676. }
  677. return 0;
  678. }
  679. static void free_old_xmit_skbs(struct send_queue *sq)
  680. {
  681. struct sk_buff *skb;
  682. unsigned int len;
  683. struct virtnet_info *vi = sq->vq->vdev->priv;
  684. struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
  685. while ((skb = virtqueue_get_buf(sq->vq, &len)) != NULL) {
  686. pr_debug("Sent skb %p\n", skb);
  687. u64_stats_update_begin(&stats->tx_syncp);
  688. stats->tx_bytes += skb->len;
  689. stats->tx_packets++;
  690. u64_stats_update_end(&stats->tx_syncp);
  691. dev_kfree_skb_any(skb);
  692. }
  693. }
  694. static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
  695. {
  696. struct virtio_net_hdr_mrg_rxbuf *hdr;
  697. const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
  698. struct virtnet_info *vi = sq->vq->vdev->priv;
  699. unsigned num_sg;
  700. unsigned hdr_len = vi->hdr_len;
  701. bool can_push;
  702. pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
  703. can_push = vi->any_header_sg &&
  704. !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) &&
  705. !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len;
  706. /* Even if we can, don't push here yet as this would skew
  707. * csum_start offset below. */
  708. if (can_push)
  709. hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len);
  710. else
  711. hdr = skb_vnet_hdr(skb);
  712. if (skb->ip_summed == CHECKSUM_PARTIAL) {
  713. hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
  714. hdr->hdr.csum_start = cpu_to_virtio16(vi->vdev,
  715. skb_checksum_start_offset(skb));
  716. hdr->hdr.csum_offset = cpu_to_virtio16(vi->vdev,
  717. skb->csum_offset);
  718. } else {
  719. hdr->hdr.flags = 0;
  720. hdr->hdr.csum_offset = hdr->hdr.csum_start = 0;
  721. }
  722. if (skb_is_gso(skb)) {
  723. hdr->hdr.hdr_len = cpu_to_virtio16(vi->vdev, skb_headlen(skb));
  724. hdr->hdr.gso_size = cpu_to_virtio16(vi->vdev,
  725. skb_shinfo(skb)->gso_size);
  726. if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
  727. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
  728. else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
  729. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
  730. else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
  731. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
  732. else
  733. BUG();
  734. if (skb_shinfo(skb)->gso_type & SKB_GSO_TCP_ECN)
  735. hdr->hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
  736. } else {
  737. hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
  738. hdr->hdr.gso_size = hdr->hdr.hdr_len = 0;
  739. }
  740. if (vi->mergeable_rx_bufs)
  741. hdr->num_buffers = 0;
  742. sg_init_table(sq->sg, MAX_SKB_FRAGS + 2);
  743. if (can_push) {
  744. __skb_push(skb, hdr_len);
  745. num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len);
  746. /* Pull header back to avoid skew in tx bytes calculations. */
  747. __skb_pull(skb, hdr_len);
  748. } else {
  749. sg_set_buf(sq->sg, hdr, hdr_len);
  750. num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len) + 1;
  751. }
  752. return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
  753. }
  754. static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
  755. {
  756. struct virtnet_info *vi = netdev_priv(dev);
  757. int qnum = skb_get_queue_mapping(skb);
  758. struct send_queue *sq = &vi->sq[qnum];
  759. int err;
  760. struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);
  761. bool kick = !skb->xmit_more;
  762. /* Free up any pending old buffers before queueing new ones. */
  763. free_old_xmit_skbs(sq);
  764. /* timestamp packet in software */
  765. skb_tx_timestamp(skb);
  766. /* Try to transmit */
  767. err = xmit_skb(sq, skb);
  768. /* This should not happen! */
  769. if (unlikely(err)) {
  770. dev->stats.tx_fifo_errors++;
  771. if (net_ratelimit())
  772. dev_warn(&dev->dev,
  773. "Unexpected TXQ (%d) queue failure: %d\n", qnum, err);
  774. dev->stats.tx_dropped++;
  775. dev_kfree_skb_any(skb);
  776. return NETDEV_TX_OK;
  777. }
  778. /* Don't wait up for transmitted skbs to be freed. */
  779. skb_orphan(skb);
  780. nf_reset(skb);
  781. /* Apparently nice girls don't return TX_BUSY; stop the queue
  782. * before it gets out of hand. Naturally, this wastes entries. */
  783. if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
  784. netif_stop_subqueue(dev, qnum);
  785. if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
  786. /* More just got used, free them then recheck. */
  787. free_old_xmit_skbs(sq);
  788. if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
  789. netif_start_subqueue(dev, qnum);
  790. virtqueue_disable_cb(sq->vq);
  791. }
  792. }
  793. }
  794. if (kick || netif_xmit_stopped(txq))
  795. virtqueue_kick(sq->vq);
  796. return NETDEV_TX_OK;
  797. }
  798. /*
  799. * Send command via the control virtqueue and check status. Commands
  800. * supported by the hypervisor, as indicated by feature bits, should
  801. * never fail unless improperly formatted.
  802. */
  803. static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
  804. struct scatterlist *out)
  805. {
  806. struct scatterlist *sgs[4], hdr, stat;
  807. struct virtio_net_ctrl_hdr ctrl;
  808. virtio_net_ctrl_ack status = ~0;
  809. unsigned out_num = 0, tmp;
  810. /* Caller should know better */
  811. BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));
  812. ctrl.class = class;
  813. ctrl.cmd = cmd;
  814. /* Add header */
  815. sg_init_one(&hdr, &ctrl, sizeof(ctrl));
  816. sgs[out_num++] = &hdr;
  817. if (out)
  818. sgs[out_num++] = out;
  819. /* Add return status. */
  820. sg_init_one(&stat, &status, sizeof(status));
  821. sgs[out_num] = &stat;
  822. BUG_ON(out_num + 1 > ARRAY_SIZE(sgs));
  823. virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC);
  824. if (unlikely(!virtqueue_kick(vi->cvq)))
  825. return status == VIRTIO_NET_OK;
  826. /* Spin for a response, the kick causes an ioport write, trapping
  827. * into the hypervisor, so the request should be handled immediately.
  828. */
  829. while (!virtqueue_get_buf(vi->cvq, &tmp) &&
  830. !virtqueue_is_broken(vi->cvq))
  831. cpu_relax();
  832. return status == VIRTIO_NET_OK;
  833. }
  834. static int virtnet_set_mac_address(struct net_device *dev, void *p)
  835. {
  836. struct virtnet_info *vi = netdev_priv(dev);
  837. struct virtio_device *vdev = vi->vdev;
  838. int ret;
  839. struct sockaddr *addr = p;
  840. struct scatterlist sg;
  841. ret = eth_prepare_mac_addr_change(dev, p);
  842. if (ret)
  843. return ret;
  844. if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
  845. sg_init_one(&sg, addr->sa_data, dev->addr_len);
  846. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
  847. VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) {
  848. dev_warn(&vdev->dev,
  849. "Failed to set mac address by vq command.\n");
  850. return -EINVAL;
  851. }
  852. } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) &&
  853. !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
  854. unsigned int i;
  855. /* Naturally, this has an atomicity problem. */
  856. for (i = 0; i < dev->addr_len; i++)
  857. virtio_cwrite8(vdev,
  858. offsetof(struct virtio_net_config, mac) +
  859. i, addr->sa_data[i]);
  860. }
  861. eth_commit_mac_addr_change(dev, p);
  862. return 0;
  863. }
  864. static struct rtnl_link_stats64 *virtnet_stats(struct net_device *dev,
  865. struct rtnl_link_stats64 *tot)
  866. {
  867. struct virtnet_info *vi = netdev_priv(dev);
  868. int cpu;
  869. unsigned int start;
  870. for_each_possible_cpu(cpu) {
  871. struct virtnet_stats *stats = per_cpu_ptr(vi->stats, cpu);
  872. u64 tpackets, tbytes, rpackets, rbytes;
  873. do {
  874. start = u64_stats_fetch_begin_irq(&stats->tx_syncp);
  875. tpackets = stats->tx_packets;
  876. tbytes = stats->tx_bytes;
  877. } while (u64_stats_fetch_retry_irq(&stats->tx_syncp, start));
  878. do {
  879. start = u64_stats_fetch_begin_irq(&stats->rx_syncp);
  880. rpackets = stats->rx_packets;
  881. rbytes = stats->rx_bytes;
  882. } while (u64_stats_fetch_retry_irq(&stats->rx_syncp, start));
  883. tot->rx_packets += rpackets;
  884. tot->tx_packets += tpackets;
  885. tot->rx_bytes += rbytes;
  886. tot->tx_bytes += tbytes;
  887. }
  888. tot->tx_dropped = dev->stats.tx_dropped;
  889. tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
  890. tot->rx_dropped = dev->stats.rx_dropped;
  891. tot->rx_length_errors = dev->stats.rx_length_errors;
  892. tot->rx_frame_errors = dev->stats.rx_frame_errors;
  893. return tot;
  894. }
  895. #ifdef CONFIG_NET_POLL_CONTROLLER
  896. static void virtnet_netpoll(struct net_device *dev)
  897. {
  898. struct virtnet_info *vi = netdev_priv(dev);
  899. int i;
  900. for (i = 0; i < vi->curr_queue_pairs; i++)
  901. napi_schedule(&vi->rq[i].napi);
  902. }
  903. #endif
  904. static void virtnet_ack_link_announce(struct virtnet_info *vi)
  905. {
  906. rtnl_lock();
  907. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
  908. VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL))
  909. dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
  910. rtnl_unlock();
  911. }
  912. static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
  913. {
  914. struct scatterlist sg;
  915. struct virtio_net_ctrl_mq s;
  916. struct net_device *dev = vi->dev;
  917. if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
  918. return 0;
  919. s.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs);
  920. sg_init_one(&sg, &s, sizeof(s));
  921. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
  922. VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) {
  923. dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n",
  924. queue_pairs);
  925. return -EINVAL;
  926. } else {
  927. vi->curr_queue_pairs = queue_pairs;
  928. /* virtnet_open() will refill when device is going to up. */
  929. if (dev->flags & IFF_UP)
  930. schedule_delayed_work(&vi->refill, 0);
  931. }
  932. return 0;
  933. }
  934. static int virtnet_close(struct net_device *dev)
  935. {
  936. struct virtnet_info *vi = netdev_priv(dev);
  937. int i;
  938. /* Make sure refill_work doesn't re-enable napi! */
  939. cancel_delayed_work_sync(&vi->refill);
  940. for (i = 0; i < vi->max_queue_pairs; i++)
  941. napi_disable(&vi->rq[i].napi);
  942. return 0;
  943. }
  944. static void virtnet_set_rx_mode(struct net_device *dev)
  945. {
  946. struct virtnet_info *vi = netdev_priv(dev);
  947. struct scatterlist sg[2];
  948. u8 promisc, allmulti;
  949. struct virtio_net_ctrl_mac *mac_data;
  950. struct netdev_hw_addr *ha;
  951. int uc_count;
  952. int mc_count;
  953. void *buf;
  954. int i;
  955. /* We can't dynamically set ndo_set_rx_mode, so return gracefully */
  956. if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
  957. return;
  958. promisc = ((dev->flags & IFF_PROMISC) != 0);
  959. allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
  960. sg_init_one(sg, &promisc, sizeof(promisc));
  961. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
  962. VIRTIO_NET_CTRL_RX_PROMISC, sg))
  963. dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
  964. promisc ? "en" : "dis");
  965. sg_init_one(sg, &allmulti, sizeof(allmulti));
  966. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
  967. VIRTIO_NET_CTRL_RX_ALLMULTI, sg))
  968. dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
  969. allmulti ? "en" : "dis");
  970. uc_count = netdev_uc_count(dev);
  971. mc_count = netdev_mc_count(dev);
  972. /* MAC filter - use one buffer for both lists */
  973. buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
  974. (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
  975. mac_data = buf;
  976. if (!buf)
  977. return;
  978. sg_init_table(sg, 2);
  979. /* Store the unicast list and count in the front of the buffer */
  980. mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count);
  981. i = 0;
  982. netdev_for_each_uc_addr(ha, dev)
  983. memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
  984. sg_set_buf(&sg[0], mac_data,
  985. sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
  986. /* multicast list and count fill the end */
  987. mac_data = (void *)&mac_data->macs[uc_count][0];
  988. mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count);
  989. i = 0;
  990. netdev_for_each_mc_addr(ha, dev)
  991. memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
  992. sg_set_buf(&sg[1], mac_data,
  993. sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
  994. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
  995. VIRTIO_NET_CTRL_MAC_TABLE_SET, sg))
  996. dev_warn(&dev->dev, "Failed to set MAC filter table.\n");
  997. kfree(buf);
  998. }
  999. static int virtnet_vlan_rx_add_vid(struct net_device *dev,
  1000. __be16 proto, u16 vid)
  1001. {
  1002. struct virtnet_info *vi = netdev_priv(dev);
  1003. struct scatterlist sg;
  1004. sg_init_one(&sg, &vid, sizeof(vid));
  1005. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
  1006. VIRTIO_NET_CTRL_VLAN_ADD, &sg))
  1007. dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
  1008. return 0;
  1009. }
  1010. static int virtnet_vlan_rx_kill_vid(struct net_device *dev,
  1011. __be16 proto, u16 vid)
  1012. {
  1013. struct virtnet_info *vi = netdev_priv(dev);
  1014. struct scatterlist sg;
  1015. sg_init_one(&sg, &vid, sizeof(vid));
  1016. if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
  1017. VIRTIO_NET_CTRL_VLAN_DEL, &sg))
  1018. dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
  1019. return 0;
  1020. }
  1021. static void virtnet_clean_affinity(struct virtnet_info *vi, long hcpu)
  1022. {
  1023. int i;
  1024. if (vi->affinity_hint_set) {
  1025. for (i = 0; i < vi->max_queue_pairs; i++) {
  1026. virtqueue_set_affinity(vi->rq[i].vq, -1);
  1027. virtqueue_set_affinity(vi->sq[i].vq, -1);
  1028. }
  1029. vi->affinity_hint_set = false;
  1030. }
  1031. }
  1032. static void virtnet_set_affinity(struct virtnet_info *vi)
  1033. {
  1034. int i;
  1035. int cpu;
  1036. /* In multiqueue mode, when the number of cpu is equal to the number of
  1037. * queue pairs, we let the queue pairs to be private to one cpu by
  1038. * setting the affinity hint to eliminate the contention.
  1039. */
  1040. if (vi->curr_queue_pairs == 1 ||
  1041. vi->max_queue_pairs != num_online_cpus()) {
  1042. virtnet_clean_affinity(vi, -1);
  1043. return;
  1044. }
  1045. i = 0;
  1046. for_each_online_cpu(cpu) {
  1047. virtqueue_set_affinity(vi->rq[i].vq, cpu);
  1048. virtqueue_set_affinity(vi->sq[i].vq, cpu);
  1049. netif_set_xps_queue(vi->dev, cpumask_of(cpu), i);
  1050. i++;
  1051. }
  1052. vi->affinity_hint_set = true;
  1053. }
  1054. static int virtnet_cpu_callback(struct notifier_block *nfb,
  1055. unsigned long action, void *hcpu)
  1056. {
  1057. struct virtnet_info *vi = container_of(nfb, struct virtnet_info, nb);
  1058. switch(action & ~CPU_TASKS_FROZEN) {
  1059. case CPU_ONLINE:
  1060. case CPU_DOWN_FAILED:
  1061. case CPU_DEAD:
  1062. virtnet_set_affinity(vi);
  1063. break;
  1064. case CPU_DOWN_PREPARE:
  1065. virtnet_clean_affinity(vi, (long)hcpu);
  1066. break;
  1067. default:
  1068. break;
  1069. }
  1070. return NOTIFY_OK;
  1071. }
  1072. static void virtnet_get_ringparam(struct net_device *dev,
  1073. struct ethtool_ringparam *ring)
  1074. {
  1075. struct virtnet_info *vi = netdev_priv(dev);
  1076. ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);
  1077. ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);
  1078. ring->rx_pending = ring->rx_max_pending;
  1079. ring->tx_pending = ring->tx_max_pending;
  1080. }
  1081. static void virtnet_get_drvinfo(struct net_device *dev,
  1082. struct ethtool_drvinfo *info)
  1083. {
  1084. struct virtnet_info *vi = netdev_priv(dev);
  1085. struct virtio_device *vdev = vi->vdev;
  1086. strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
  1087. strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
  1088. strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
  1089. }
  1090. /* TODO: Eliminate OOO packets during switching */
  1091. static int virtnet_set_channels(struct net_device *dev,
  1092. struct ethtool_channels *channels)
  1093. {
  1094. struct virtnet_info *vi = netdev_priv(dev);
  1095. u16 queue_pairs = channels->combined_count;
  1096. int err;
  1097. /* We don't support separate rx/tx channels.
  1098. * We don't allow setting 'other' channels.
  1099. */
  1100. if (channels->rx_count || channels->tx_count || channels->other_count)
  1101. return -EINVAL;
  1102. if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0)
  1103. return -EINVAL;
  1104. get_online_cpus();
  1105. err = virtnet_set_queues(vi, queue_pairs);
  1106. if (!err) {
  1107. netif_set_real_num_tx_queues(dev, queue_pairs);
  1108. netif_set_real_num_rx_queues(dev, queue_pairs);
  1109. virtnet_set_affinity(vi);
  1110. }
  1111. put_online_cpus();
  1112. return err;
  1113. }
  1114. static void virtnet_get_channels(struct net_device *dev,
  1115. struct ethtool_channels *channels)
  1116. {
  1117. struct virtnet_info *vi = netdev_priv(dev);
  1118. channels->combined_count = vi->curr_queue_pairs;
  1119. channels->max_combined = vi->max_queue_pairs;
  1120. channels->max_other = 0;
  1121. channels->rx_count = 0;
  1122. channels->tx_count = 0;
  1123. channels->other_count = 0;
  1124. }
  1125. static const struct ethtool_ops virtnet_ethtool_ops = {
  1126. .get_drvinfo = virtnet_get_drvinfo,
  1127. .get_link = ethtool_op_get_link,
  1128. .get_ringparam = virtnet_get_ringparam,
  1129. .set_channels = virtnet_set_channels,
  1130. .get_channels = virtnet_get_channels,
  1131. .get_ts_info = ethtool_op_get_ts_info,
  1132. };
  1133. #define MIN_MTU 68
  1134. #define MAX_MTU 65535
  1135. static int virtnet_change_mtu(struct net_device *dev, int new_mtu)
  1136. {
  1137. if (new_mtu < MIN_MTU || new_mtu > MAX_MTU)
  1138. return -EINVAL;
  1139. dev->mtu = new_mtu;
  1140. return 0;
  1141. }
  1142. static const struct net_device_ops virtnet_netdev = {
  1143. .ndo_open = virtnet_open,
  1144. .ndo_stop = virtnet_close,
  1145. .ndo_start_xmit = start_xmit,
  1146. .ndo_validate_addr = eth_validate_addr,
  1147. .ndo_set_mac_address = virtnet_set_mac_address,
  1148. .ndo_set_rx_mode = virtnet_set_rx_mode,
  1149. .ndo_change_mtu = virtnet_change_mtu,
  1150. .ndo_get_stats64 = virtnet_stats,
  1151. .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
  1152. .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
  1153. #ifdef CONFIG_NET_POLL_CONTROLLER
  1154. .ndo_poll_controller = virtnet_netpoll,
  1155. #endif
  1156. #ifdef CONFIG_NET_RX_BUSY_POLL
  1157. .ndo_busy_poll = virtnet_busy_poll,
  1158. #endif
  1159. };
  1160. static void virtnet_config_changed_work(struct work_struct *work)
  1161. {
  1162. struct virtnet_info *vi =
  1163. container_of(work, struct virtnet_info, config_work);
  1164. u16 v;
  1165. if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS,
  1166. struct virtio_net_config, status, &v) < 0)
  1167. return;
  1168. if (v & VIRTIO_NET_S_ANNOUNCE) {
  1169. netdev_notify_peers(vi->dev);
  1170. virtnet_ack_link_announce(vi);
  1171. }
  1172. /* Ignore unknown (future) status bits */
  1173. v &= VIRTIO_NET_S_LINK_UP;
  1174. if (vi->status == v)
  1175. return;
  1176. vi->status = v;
  1177. if (vi->status & VIRTIO_NET_S_LINK_UP) {
  1178. netif_carrier_on(vi->dev);
  1179. netif_tx_wake_all_queues(vi->dev);
  1180. } else {
  1181. netif_carrier_off(vi->dev);
  1182. netif_tx_stop_all_queues(vi->dev);
  1183. }
  1184. }
  1185. static void virtnet_config_changed(struct virtio_device *vdev)
  1186. {
  1187. struct virtnet_info *vi = vdev->priv;
  1188. schedule_work(&vi->config_work);
  1189. }
  1190. static void virtnet_free_queues(struct virtnet_info *vi)
  1191. {
  1192. int i;
  1193. for (i = 0; i < vi->max_queue_pairs; i++)
  1194. netif_napi_del(&vi->rq[i].napi);
  1195. kfree(vi->rq);
  1196. kfree(vi->sq);
  1197. }
  1198. static void free_receive_bufs(struct virtnet_info *vi)
  1199. {
  1200. int i;
  1201. for (i = 0; i < vi->max_queue_pairs; i++) {
  1202. while (vi->rq[i].pages)
  1203. __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);
  1204. }
  1205. }
  1206. static void free_receive_page_frags(struct virtnet_info *vi)
  1207. {
  1208. int i;
  1209. for (i = 0; i < vi->max_queue_pairs; i++)
  1210. if (vi->rq[i].alloc_frag.page)
  1211. put_page(vi->rq[i].alloc_frag.page);
  1212. }
  1213. static void free_unused_bufs(struct virtnet_info *vi)
  1214. {
  1215. void *buf;
  1216. int i;
  1217. for (i = 0; i < vi->max_queue_pairs; i++) {
  1218. struct virtqueue *vq = vi->sq[i].vq;
  1219. while ((buf = virtqueue_detach_unused_buf(vq)) != NULL)
  1220. dev_kfree_skb(buf);
  1221. }
  1222. for (i = 0; i < vi->max_queue_pairs; i++) {
  1223. struct virtqueue *vq = vi->rq[i].vq;
  1224. while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
  1225. if (vi->mergeable_rx_bufs) {
  1226. unsigned long ctx = (unsigned long)buf;
  1227. void *base = mergeable_ctx_to_buf_address(ctx);
  1228. put_page(virt_to_head_page(base));
  1229. } else if (vi->big_packets) {
  1230. give_pages(&vi->rq[i], buf);
  1231. } else {
  1232. dev_kfree_skb(buf);
  1233. }
  1234. }
  1235. }
  1236. }
  1237. static void virtnet_del_vqs(struct virtnet_info *vi)
  1238. {
  1239. struct virtio_device *vdev = vi->vdev;
  1240. virtnet_clean_affinity(vi, -1);
  1241. vdev->config->del_vqs(vdev);
  1242. virtnet_free_queues(vi);
  1243. }
  1244. static int virtnet_find_vqs(struct virtnet_info *vi)
  1245. {
  1246. vq_callback_t **callbacks;
  1247. struct virtqueue **vqs;
  1248. int ret = -ENOMEM;
  1249. int i, total_vqs;
  1250. const char **names;
  1251. /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
  1252. * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by
  1253. * possible control vq.
  1254. */
  1255. total_vqs = vi->max_queue_pairs * 2 +
  1256. virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);
  1257. /* Allocate space for find_vqs parameters */
  1258. vqs = kzalloc(total_vqs * sizeof(*vqs), GFP_KERNEL);
  1259. if (!vqs)
  1260. goto err_vq;
  1261. callbacks = kmalloc(total_vqs * sizeof(*callbacks), GFP_KERNEL);
  1262. if (!callbacks)
  1263. goto err_callback;
  1264. names = kmalloc(total_vqs * sizeof(*names), GFP_KERNEL);
  1265. if (!names)
  1266. goto err_names;
  1267. /* Parameters for control virtqueue, if any */
  1268. if (vi->has_cvq) {
  1269. callbacks[total_vqs - 1] = NULL;
  1270. names[total_vqs - 1] = "control";
  1271. }
  1272. /* Allocate/initialize parameters for send/receive virtqueues */
  1273. for (i = 0; i < vi->max_queue_pairs; i++) {
  1274. callbacks[rxq2vq(i)] = skb_recv_done;
  1275. callbacks[txq2vq(i)] = skb_xmit_done;
  1276. sprintf(vi->rq[i].name, "input.%d", i);
  1277. sprintf(vi->sq[i].name, "output.%d", i);
  1278. names[rxq2vq(i)] = vi->rq[i].name;
  1279. names[txq2vq(i)] = vi->sq[i].name;
  1280. }
  1281. ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,
  1282. names);
  1283. if (ret)
  1284. goto err_find;
  1285. if (vi->has_cvq) {
  1286. vi->cvq = vqs[total_vqs - 1];
  1287. if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
  1288. vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
  1289. }
  1290. for (i = 0; i < vi->max_queue_pairs; i++) {
  1291. vi->rq[i].vq = vqs[rxq2vq(i)];
  1292. vi->sq[i].vq = vqs[txq2vq(i)];
  1293. }
  1294. kfree(names);
  1295. kfree(callbacks);
  1296. kfree(vqs);
  1297. return 0;
  1298. err_find:
  1299. kfree(names);
  1300. err_names:
  1301. kfree(callbacks);
  1302. err_callback:
  1303. kfree(vqs);
  1304. err_vq:
  1305. return ret;
  1306. }
  1307. static int virtnet_alloc_queues(struct virtnet_info *vi)
  1308. {
  1309. int i;
  1310. vi->sq = kzalloc(sizeof(*vi->sq) * vi->max_queue_pairs, GFP_KERNEL);
  1311. if (!vi->sq)
  1312. goto err_sq;
  1313. vi->rq = kzalloc(sizeof(*vi->rq) * vi->max_queue_pairs, GFP_KERNEL);
  1314. if (!vi->rq)
  1315. goto err_rq;
  1316. INIT_DELAYED_WORK(&vi->refill, refill_work);
  1317. for (i = 0; i < vi->max_queue_pairs; i++) {
  1318. vi->rq[i].pages = NULL;
  1319. netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,
  1320. napi_weight);
  1321. napi_hash_add(&vi->rq[i].napi);
  1322. sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
  1323. ewma_init(&vi->rq[i].mrg_avg_pkt_len, 1, RECEIVE_AVG_WEIGHT);
  1324. sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
  1325. }
  1326. return 0;
  1327. err_rq:
  1328. kfree(vi->sq);
  1329. err_sq:
  1330. return -ENOMEM;
  1331. }
  1332. static int init_vqs(struct virtnet_info *vi)
  1333. {
  1334. int ret;
  1335. /* Allocate send & receive queues */
  1336. ret = virtnet_alloc_queues(vi);
  1337. if (ret)
  1338. goto err;
  1339. ret = virtnet_find_vqs(vi);
  1340. if (ret)
  1341. goto err_free;
  1342. get_online_cpus();
  1343. virtnet_set_affinity(vi);
  1344. put_online_cpus();
  1345. return 0;
  1346. err_free:
  1347. virtnet_free_queues(vi);
  1348. err:
  1349. return ret;
  1350. }
  1351. #ifdef CONFIG_SYSFS
  1352. static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue,
  1353. struct rx_queue_attribute *attribute, char *buf)
  1354. {
  1355. struct virtnet_info *vi = netdev_priv(queue->dev);
  1356. unsigned int queue_index = get_netdev_rx_queue_index(queue);
  1357. struct ewma *avg;
  1358. BUG_ON(queue_index >= vi->max_queue_pairs);
  1359. avg = &vi->rq[queue_index].mrg_avg_pkt_len;
  1360. return sprintf(buf, "%u\n", get_mergeable_buf_len(avg));
  1361. }
  1362. static struct rx_queue_attribute mergeable_rx_buffer_size_attribute =
  1363. __ATTR_RO(mergeable_rx_buffer_size);
  1364. static struct attribute *virtio_net_mrg_rx_attrs[] = {
  1365. &mergeable_rx_buffer_size_attribute.attr,
  1366. NULL
  1367. };
  1368. static const struct attribute_group virtio_net_mrg_rx_group = {
  1369. .name = "virtio_net",
  1370. .attrs = virtio_net_mrg_rx_attrs
  1371. };
  1372. #endif
  1373. static bool virtnet_fail_on_feature(struct virtio_device *vdev,
  1374. unsigned int fbit,
  1375. const char *fname, const char *dname)
  1376. {
  1377. if (!virtio_has_feature(vdev, fbit))
  1378. return false;
  1379. dev_err(&vdev->dev, "device advertises feature %s but not %s",
  1380. fname, dname);
  1381. return true;
  1382. }
  1383. #define VIRTNET_FAIL_ON(vdev, fbit, dbit) \
  1384. virtnet_fail_on_feature(vdev, fbit, #fbit, dbit)
  1385. static bool virtnet_validate_features(struct virtio_device *vdev)
  1386. {
  1387. if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) &&
  1388. (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX,
  1389. "VIRTIO_NET_F_CTRL_VQ") ||
  1390. VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN,
  1391. "VIRTIO_NET_F_CTRL_VQ") ||
  1392. VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE,
  1393. "VIRTIO_NET_F_CTRL_VQ") ||
  1394. VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") ||
  1395. VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR,
  1396. "VIRTIO_NET_F_CTRL_VQ"))) {
  1397. return false;
  1398. }
  1399. return true;
  1400. }
  1401. static int virtnet_probe(struct virtio_device *vdev)
  1402. {
  1403. int i, err;
  1404. struct net_device *dev;
  1405. struct virtnet_info *vi;
  1406. u16 max_queue_pairs;
  1407. if (!virtnet_validate_features(vdev))
  1408. return -EINVAL;
  1409. /* Find if host supports multiqueue virtio_net device */
  1410. err = virtio_cread_feature(vdev, VIRTIO_NET_F_MQ,
  1411. struct virtio_net_config,
  1412. max_virtqueue_pairs, &max_queue_pairs);
  1413. /* We need at least 2 queue's */
  1414. if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
  1415. max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
  1416. !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
  1417. max_queue_pairs = 1;
  1418. /* Allocate ourselves a network device with room for our info */
  1419. dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs);
  1420. if (!dev)
  1421. return -ENOMEM;
  1422. /* Set up network device as normal. */
  1423. dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE;
  1424. dev->netdev_ops = &virtnet_netdev;
  1425. dev->features = NETIF_F_HIGHDMA;
  1426. dev->ethtool_ops = &virtnet_ethtool_ops;
  1427. SET_NETDEV_DEV(dev, &vdev->dev);
  1428. /* Do we support "hardware" checksums? */
  1429. if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
  1430. /* This opens up the world of extra features. */
  1431. dev->hw_features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
  1432. if (csum)
  1433. dev->features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
  1434. if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
  1435. dev->hw_features |= NETIF_F_TSO | NETIF_F_UFO
  1436. | NETIF_F_TSO_ECN | NETIF_F_TSO6;
  1437. }
  1438. /* Individual feature bits: what can host handle? */
  1439. if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
  1440. dev->hw_features |= NETIF_F_TSO;
  1441. if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
  1442. dev->hw_features |= NETIF_F_TSO6;
  1443. if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
  1444. dev->hw_features |= NETIF_F_TSO_ECN;
  1445. if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
  1446. dev->hw_features |= NETIF_F_UFO;
  1447. dev->features |= NETIF_F_GSO_ROBUST;
  1448. if (gso)
  1449. dev->features |= dev->hw_features & (NETIF_F_ALL_TSO|NETIF_F_UFO);
  1450. /* (!csum && gso) case will be fixed by register_netdev() */
  1451. }
  1452. if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM))
  1453. dev->features |= NETIF_F_RXCSUM;
  1454. dev->vlan_features = dev->features;
  1455. /* Configuration may specify what MAC to use. Otherwise random. */
  1456. if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
  1457. virtio_cread_bytes(vdev,
  1458. offsetof(struct virtio_net_config, mac),
  1459. dev->dev_addr, dev->addr_len);
  1460. else
  1461. eth_hw_addr_random(dev);
  1462. /* Set up our device-specific information */
  1463. vi = netdev_priv(dev);
  1464. vi->dev = dev;
  1465. vi->vdev = vdev;
  1466. vdev->priv = vi;
  1467. vi->stats = alloc_percpu(struct virtnet_stats);
  1468. err = -ENOMEM;
  1469. if (vi->stats == NULL)
  1470. goto free;
  1471. for_each_possible_cpu(i) {
  1472. struct virtnet_stats *virtnet_stats;
  1473. virtnet_stats = per_cpu_ptr(vi->stats, i);
  1474. u64_stats_init(&virtnet_stats->tx_syncp);
  1475. u64_stats_init(&virtnet_stats->rx_syncp);
  1476. }
  1477. INIT_WORK(&vi->config_work, virtnet_config_changed_work);
  1478. /* If we can receive ANY GSO packets, we must allocate large ones. */
  1479. if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
  1480. virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
  1481. virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN) ||
  1482. virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UFO))
  1483. vi->big_packets = true;
  1484. if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
  1485. vi->mergeable_rx_bufs = true;
  1486. if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) ||
  1487. virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
  1488. vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
  1489. else
  1490. vi->hdr_len = sizeof(struct virtio_net_hdr);
  1491. if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT))
  1492. vi->any_header_sg = true;
  1493. if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
  1494. vi->has_cvq = true;
  1495. if (vi->any_header_sg)
  1496. dev->needed_headroom = vi->hdr_len;
  1497. /* Use single tx/rx queue pair as default */
  1498. vi->curr_queue_pairs = 1;
  1499. vi->max_queue_pairs = max_queue_pairs;
  1500. /* Allocate/initialize the rx/tx queues, and invoke find_vqs */
  1501. err = init_vqs(vi);
  1502. if (err)
  1503. goto free_stats;
  1504. #ifdef CONFIG_SYSFS
  1505. if (vi->mergeable_rx_bufs)
  1506. dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group;
  1507. #endif
  1508. netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);
  1509. netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);
  1510. err = register_netdev(dev);
  1511. if (err) {
  1512. pr_debug("virtio_net: registering device failed\n");
  1513. goto free_vqs;
  1514. }
  1515. virtio_device_ready(vdev);
  1516. /* Last of all, set up some receive buffers. */
  1517. for (i = 0; i < vi->curr_queue_pairs; i++) {
  1518. try_fill_recv(vi, &vi->rq[i], GFP_KERNEL);
  1519. /* If we didn't even get one input buffer, we're useless. */
  1520. if (vi->rq[i].vq->num_free ==
  1521. virtqueue_get_vring_size(vi->rq[i].vq)) {
  1522. free_unused_bufs(vi);
  1523. err = -ENOMEM;
  1524. goto free_recv_bufs;
  1525. }
  1526. }
  1527. vi->nb.notifier_call = &virtnet_cpu_callback;
  1528. err = register_hotcpu_notifier(&vi->nb);
  1529. if (err) {
  1530. pr_debug("virtio_net: registering cpu notifier failed\n");
  1531. goto free_recv_bufs;
  1532. }
  1533. /* Assume link up if device can't report link status,
  1534. otherwise get link status from config. */
  1535. if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
  1536. netif_carrier_off(dev);
  1537. schedule_work(&vi->config_work);
  1538. } else {
  1539. vi->status = VIRTIO_NET_S_LINK_UP;
  1540. netif_carrier_on(dev);
  1541. }
  1542. pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
  1543. dev->name, max_queue_pairs);
  1544. return 0;
  1545. free_recv_bufs:
  1546. vi->vdev->config->reset(vdev);
  1547. free_receive_bufs(vi);
  1548. unregister_netdev(dev);
  1549. free_vqs:
  1550. cancel_delayed_work_sync(&vi->refill);
  1551. free_receive_page_frags(vi);
  1552. virtnet_del_vqs(vi);
  1553. free_stats:
  1554. free_percpu(vi->stats);
  1555. free:
  1556. free_netdev(dev);
  1557. return err;
  1558. }
  1559. static void remove_vq_common(struct virtnet_info *vi)
  1560. {
  1561. vi->vdev->config->reset(vi->vdev);
  1562. /* Free unused buffers in both send and recv, if any. */
  1563. free_unused_bufs(vi);
  1564. free_receive_bufs(vi);
  1565. free_receive_page_frags(vi);
  1566. virtnet_del_vqs(vi);
  1567. }
  1568. static void virtnet_remove(struct virtio_device *vdev)
  1569. {
  1570. struct virtnet_info *vi = vdev->priv;
  1571. unregister_hotcpu_notifier(&vi->nb);
  1572. /* Make sure no work handler is accessing the device. */
  1573. flush_work(&vi->config_work);
  1574. unregister_netdev(vi->dev);
  1575. remove_vq_common(vi);
  1576. free_percpu(vi->stats);
  1577. free_netdev(vi->dev);
  1578. }
  1579. #ifdef CONFIG_PM_SLEEP
  1580. static int virtnet_freeze(struct virtio_device *vdev)
  1581. {
  1582. struct virtnet_info *vi = vdev->priv;
  1583. int i;
  1584. unregister_hotcpu_notifier(&vi->nb);
  1585. /* Make sure no work handler is accessing the device */
  1586. flush_work(&vi->config_work);
  1587. netif_device_detach(vi->dev);
  1588. cancel_delayed_work_sync(&vi->refill);
  1589. if (netif_running(vi->dev)) {
  1590. for (i = 0; i < vi->max_queue_pairs; i++) {
  1591. napi_disable(&vi->rq[i].napi);
  1592. napi_hash_del(&vi->rq[i].napi);
  1593. netif_napi_del(&vi->rq[i].napi);
  1594. }
  1595. }
  1596. remove_vq_common(vi);
  1597. return 0;
  1598. }
  1599. static int virtnet_restore(struct virtio_device *vdev)
  1600. {
  1601. struct virtnet_info *vi = vdev->priv;
  1602. int err, i;
  1603. err = init_vqs(vi);
  1604. if (err)
  1605. return err;
  1606. virtio_device_ready(vdev);
  1607. if (netif_running(vi->dev)) {
  1608. for (i = 0; i < vi->curr_queue_pairs; i++)
  1609. if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
  1610. schedule_delayed_work(&vi->refill, 0);
  1611. for (i = 0; i < vi->max_queue_pairs; i++)
  1612. virtnet_napi_enable(&vi->rq[i]);
  1613. }
  1614. netif_device_attach(vi->dev);
  1615. rtnl_lock();
  1616. virtnet_set_queues(vi, vi->curr_queue_pairs);
  1617. rtnl_unlock();
  1618. err = register_hotcpu_notifier(&vi->nb);
  1619. if (err)
  1620. return err;
  1621. return 0;
  1622. }
  1623. #endif
  1624. static struct virtio_device_id id_table[] = {
  1625. { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
  1626. { 0 },
  1627. };
  1628. static unsigned int features[] = {
  1629. VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM,
  1630. VIRTIO_NET_F_GSO, VIRTIO_NET_F_MAC,
  1631. VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6,
  1632. VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6,
  1633. VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO,
  1634. VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ,
  1635. VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN,
  1636. VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ,
  1637. VIRTIO_NET_F_CTRL_MAC_ADDR,
  1638. VIRTIO_F_ANY_LAYOUT,
  1639. };
  1640. static struct virtio_driver virtio_net_driver = {
  1641. .feature_table = features,
  1642. .feature_table_size = ARRAY_SIZE(features),
  1643. .driver.name = KBUILD_MODNAME,
  1644. .driver.owner = THIS_MODULE,
  1645. .id_table = id_table,
  1646. .probe = virtnet_probe,
  1647. .remove = virtnet_remove,
  1648. .config_changed = virtnet_config_changed,
  1649. #ifdef CONFIG_PM_SLEEP
  1650. .freeze = virtnet_freeze,
  1651. .restore = virtnet_restore,
  1652. #endif
  1653. };
  1654. module_virtio_driver(virtio_net_driver);
  1655. MODULE_DEVICE_TABLE(virtio, id_table);
  1656. MODULE_DESCRIPTION("Virtio network driver");
  1657. MODULE_LICENSE("GPL");