netvsc.c 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236
  1. /*
  2. * Copyright (c) 2009, Microsoft Corporation.
  3. *
  4. * This program is free software; you can redistribute it and/or modify it
  5. * under the terms and conditions of the GNU General Public License,
  6. * version 2, as published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope it will be useful, but WITHOUT
  9. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  11. * more details.
  12. *
  13. * You should have received a copy of the GNU General Public License along with
  14. * this program; if not, see <http://www.gnu.org/licenses/>.
  15. *
  16. * Authors:
  17. * Haiyang Zhang <haiyangz@microsoft.com>
  18. * Hank Janssen <hjanssen@microsoft.com>
  19. */
  20. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  21. #include <linux/kernel.h>
  22. #include <linux/sched.h>
  23. #include <linux/wait.h>
  24. #include <linux/mm.h>
  25. #include <linux/delay.h>
  26. #include <linux/io.h>
  27. #include <linux/slab.h>
  28. #include <linux/netdevice.h>
  29. #include <linux/if_ether.h>
  30. #include <asm/sync_bitops.h>
  31. #include "hyperv_net.h"
  32. static struct netvsc_device *alloc_net_device(struct hv_device *device)
  33. {
  34. struct netvsc_device *net_device;
  35. struct net_device *ndev = hv_get_drvdata(device);
  36. int i;
  37. net_device = kzalloc(sizeof(struct netvsc_device), GFP_KERNEL);
  38. if (!net_device)
  39. return NULL;
  40. net_device->cb_buffer = kzalloc(NETVSC_PACKET_SIZE, GFP_KERNEL);
  41. if (!net_device->cb_buffer) {
  42. kfree(net_device);
  43. return NULL;
  44. }
  45. init_waitqueue_head(&net_device->wait_drain);
  46. net_device->start_remove = false;
  47. net_device->destroy = false;
  48. net_device->dev = device;
  49. net_device->ndev = ndev;
  50. net_device->max_pkt = RNDIS_MAX_PKT_DEFAULT;
  51. net_device->pkt_align = RNDIS_PKT_ALIGN_DEFAULT;
  52. for (i = 0; i < num_online_cpus(); i++)
  53. spin_lock_init(&net_device->msd[i].lock);
  54. hv_set_drvdata(device, net_device);
  55. return net_device;
  56. }
  57. static void free_netvsc_device(struct netvsc_device *nvdev)
  58. {
  59. kfree(nvdev->cb_buffer);
  60. kfree(nvdev);
  61. }
  62. static struct netvsc_device *get_outbound_net_device(struct hv_device *device)
  63. {
  64. struct netvsc_device *net_device;
  65. net_device = hv_get_drvdata(device);
  66. if (net_device && net_device->destroy)
  67. net_device = NULL;
  68. return net_device;
  69. }
  70. static struct netvsc_device *get_inbound_net_device(struct hv_device *device)
  71. {
  72. struct netvsc_device *net_device;
  73. net_device = hv_get_drvdata(device);
  74. if (!net_device)
  75. goto get_in_err;
  76. if (net_device->destroy &&
  77. atomic_read(&net_device->num_outstanding_sends) == 0)
  78. net_device = NULL;
  79. get_in_err:
  80. return net_device;
  81. }
  82. static int netvsc_destroy_buf(struct netvsc_device *net_device)
  83. {
  84. struct nvsp_message *revoke_packet;
  85. int ret = 0;
  86. struct net_device *ndev = net_device->ndev;
  87. /*
  88. * If we got a section count, it means we received a
  89. * SendReceiveBufferComplete msg (ie sent
  90. * NvspMessage1TypeSendReceiveBuffer msg) therefore, we need
  91. * to send a revoke msg here
  92. */
  93. if (net_device->recv_section_cnt) {
  94. /* Send the revoke receive buffer */
  95. revoke_packet = &net_device->revoke_packet;
  96. memset(revoke_packet, 0, sizeof(struct nvsp_message));
  97. revoke_packet->hdr.msg_type =
  98. NVSP_MSG1_TYPE_REVOKE_RECV_BUF;
  99. revoke_packet->msg.v1_msg.
  100. revoke_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID;
  101. ret = vmbus_sendpacket(net_device->dev->channel,
  102. revoke_packet,
  103. sizeof(struct nvsp_message),
  104. (unsigned long)revoke_packet,
  105. VM_PKT_DATA_INBAND, 0);
  106. /*
  107. * If we failed here, we might as well return and
  108. * have a leak rather than continue and a bugchk
  109. */
  110. if (ret != 0) {
  111. netdev_err(ndev, "unable to send "
  112. "revoke receive buffer to netvsp\n");
  113. return ret;
  114. }
  115. }
  116. /* Teardown the gpadl on the vsp end */
  117. if (net_device->recv_buf_gpadl_handle) {
  118. ret = vmbus_teardown_gpadl(net_device->dev->channel,
  119. net_device->recv_buf_gpadl_handle);
  120. /* If we failed here, we might as well return and have a leak
  121. * rather than continue and a bugchk
  122. */
  123. if (ret != 0) {
  124. netdev_err(ndev,
  125. "unable to teardown receive buffer's gpadl\n");
  126. return ret;
  127. }
  128. net_device->recv_buf_gpadl_handle = 0;
  129. }
  130. if (net_device->recv_buf) {
  131. /* Free up the receive buffer */
  132. vfree(net_device->recv_buf);
  133. net_device->recv_buf = NULL;
  134. }
  135. if (net_device->recv_section) {
  136. net_device->recv_section_cnt = 0;
  137. kfree(net_device->recv_section);
  138. net_device->recv_section = NULL;
  139. }
  140. /* Deal with the send buffer we may have setup.
  141. * If we got a send section size, it means we received a
  142. * NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE msg (ie sent
  143. * NVSP_MSG1_TYPE_SEND_SEND_BUF msg) therefore, we need
  144. * to send a revoke msg here
  145. */
  146. if (net_device->send_section_size) {
  147. /* Send the revoke receive buffer */
  148. revoke_packet = &net_device->revoke_packet;
  149. memset(revoke_packet, 0, sizeof(struct nvsp_message));
  150. revoke_packet->hdr.msg_type =
  151. NVSP_MSG1_TYPE_REVOKE_SEND_BUF;
  152. revoke_packet->msg.v1_msg.revoke_send_buf.id =
  153. NETVSC_SEND_BUFFER_ID;
  154. ret = vmbus_sendpacket(net_device->dev->channel,
  155. revoke_packet,
  156. sizeof(struct nvsp_message),
  157. (unsigned long)revoke_packet,
  158. VM_PKT_DATA_INBAND, 0);
  159. /* If we failed here, we might as well return and
  160. * have a leak rather than continue and a bugchk
  161. */
  162. if (ret != 0) {
  163. netdev_err(ndev, "unable to send "
  164. "revoke send buffer to netvsp\n");
  165. return ret;
  166. }
  167. }
  168. /* Teardown the gpadl on the vsp end */
  169. if (net_device->send_buf_gpadl_handle) {
  170. ret = vmbus_teardown_gpadl(net_device->dev->channel,
  171. net_device->send_buf_gpadl_handle);
  172. /* If we failed here, we might as well return and have a leak
  173. * rather than continue and a bugchk
  174. */
  175. if (ret != 0) {
  176. netdev_err(ndev,
  177. "unable to teardown send buffer's gpadl\n");
  178. return ret;
  179. }
  180. net_device->send_buf_gpadl_handle = 0;
  181. }
  182. if (net_device->send_buf) {
  183. /* Free up the send buffer */
  184. vfree(net_device->send_buf);
  185. net_device->send_buf = NULL;
  186. }
  187. kfree(net_device->send_section_map);
  188. return ret;
  189. }
  190. static int netvsc_init_buf(struct hv_device *device)
  191. {
  192. int ret = 0;
  193. unsigned long t;
  194. struct netvsc_device *net_device;
  195. struct nvsp_message *init_packet;
  196. struct net_device *ndev;
  197. net_device = get_outbound_net_device(device);
  198. if (!net_device)
  199. return -ENODEV;
  200. ndev = net_device->ndev;
  201. net_device->recv_buf = vzalloc(net_device->recv_buf_size);
  202. if (!net_device->recv_buf) {
  203. netdev_err(ndev, "unable to allocate receive "
  204. "buffer of size %d\n", net_device->recv_buf_size);
  205. ret = -ENOMEM;
  206. goto cleanup;
  207. }
  208. /*
  209. * Establish the gpadl handle for this buffer on this
  210. * channel. Note: This call uses the vmbus connection rather
  211. * than the channel to establish the gpadl handle.
  212. */
  213. ret = vmbus_establish_gpadl(device->channel, net_device->recv_buf,
  214. net_device->recv_buf_size,
  215. &net_device->recv_buf_gpadl_handle);
  216. if (ret != 0) {
  217. netdev_err(ndev,
  218. "unable to establish receive buffer's gpadl\n");
  219. goto cleanup;
  220. }
  221. /* Notify the NetVsp of the gpadl handle */
  222. init_packet = &net_device->channel_init_pkt;
  223. memset(init_packet, 0, sizeof(struct nvsp_message));
  224. init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_RECV_BUF;
  225. init_packet->msg.v1_msg.send_recv_buf.
  226. gpadl_handle = net_device->recv_buf_gpadl_handle;
  227. init_packet->msg.v1_msg.
  228. send_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID;
  229. /* Send the gpadl notification request */
  230. ret = vmbus_sendpacket(device->channel, init_packet,
  231. sizeof(struct nvsp_message),
  232. (unsigned long)init_packet,
  233. VM_PKT_DATA_INBAND,
  234. VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
  235. if (ret != 0) {
  236. netdev_err(ndev,
  237. "unable to send receive buffer's gpadl to netvsp\n");
  238. goto cleanup;
  239. }
  240. t = wait_for_completion_timeout(&net_device->channel_init_wait, 5*HZ);
  241. BUG_ON(t == 0);
  242. /* Check the response */
  243. if (init_packet->msg.v1_msg.
  244. send_recv_buf_complete.status != NVSP_STAT_SUCCESS) {
  245. netdev_err(ndev, "Unable to complete receive buffer "
  246. "initialization with NetVsp - status %d\n",
  247. init_packet->msg.v1_msg.
  248. send_recv_buf_complete.status);
  249. ret = -EINVAL;
  250. goto cleanup;
  251. }
  252. /* Parse the response */
  253. net_device->recv_section_cnt = init_packet->msg.
  254. v1_msg.send_recv_buf_complete.num_sections;
  255. net_device->recv_section = kmemdup(
  256. init_packet->msg.v1_msg.send_recv_buf_complete.sections,
  257. net_device->recv_section_cnt *
  258. sizeof(struct nvsp_1_receive_buffer_section),
  259. GFP_KERNEL);
  260. if (net_device->recv_section == NULL) {
  261. ret = -EINVAL;
  262. goto cleanup;
  263. }
  264. /*
  265. * For 1st release, there should only be 1 section that represents the
  266. * entire receive buffer
  267. */
  268. if (net_device->recv_section_cnt != 1 ||
  269. net_device->recv_section->offset != 0) {
  270. ret = -EINVAL;
  271. goto cleanup;
  272. }
  273. /* Now setup the send buffer.
  274. */
  275. net_device->send_buf = vzalloc(net_device->send_buf_size);
  276. if (!net_device->send_buf) {
  277. netdev_err(ndev, "unable to allocate send "
  278. "buffer of size %d\n", net_device->send_buf_size);
  279. ret = -ENOMEM;
  280. goto cleanup;
  281. }
  282. /* Establish the gpadl handle for this buffer on this
  283. * channel. Note: This call uses the vmbus connection rather
  284. * than the channel to establish the gpadl handle.
  285. */
  286. ret = vmbus_establish_gpadl(device->channel, net_device->send_buf,
  287. net_device->send_buf_size,
  288. &net_device->send_buf_gpadl_handle);
  289. if (ret != 0) {
  290. netdev_err(ndev,
  291. "unable to establish send buffer's gpadl\n");
  292. goto cleanup;
  293. }
  294. /* Notify the NetVsp of the gpadl handle */
  295. init_packet = &net_device->channel_init_pkt;
  296. memset(init_packet, 0, sizeof(struct nvsp_message));
  297. init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_SEND_BUF;
  298. init_packet->msg.v1_msg.send_send_buf.gpadl_handle =
  299. net_device->send_buf_gpadl_handle;
  300. init_packet->msg.v1_msg.send_send_buf.id = NETVSC_SEND_BUFFER_ID;
  301. /* Send the gpadl notification request */
  302. ret = vmbus_sendpacket(device->channel, init_packet,
  303. sizeof(struct nvsp_message),
  304. (unsigned long)init_packet,
  305. VM_PKT_DATA_INBAND,
  306. VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
  307. if (ret != 0) {
  308. netdev_err(ndev,
  309. "unable to send send buffer's gpadl to netvsp\n");
  310. goto cleanup;
  311. }
  312. t = wait_for_completion_timeout(&net_device->channel_init_wait, 5*HZ);
  313. BUG_ON(t == 0);
  314. /* Check the response */
  315. if (init_packet->msg.v1_msg.
  316. send_send_buf_complete.status != NVSP_STAT_SUCCESS) {
  317. netdev_err(ndev, "Unable to complete send buffer "
  318. "initialization with NetVsp - status %d\n",
  319. init_packet->msg.v1_msg.
  320. send_send_buf_complete.status);
  321. ret = -EINVAL;
  322. goto cleanup;
  323. }
  324. /* Parse the response */
  325. net_device->send_section_size = init_packet->msg.
  326. v1_msg.send_send_buf_complete.section_size;
  327. /* Section count is simply the size divided by the section size.
  328. */
  329. net_device->send_section_cnt =
  330. net_device->send_buf_size/net_device->send_section_size;
  331. dev_info(&device->device, "Send section size: %d, Section count:%d\n",
  332. net_device->send_section_size, net_device->send_section_cnt);
  333. /* Setup state for managing the send buffer. */
  334. net_device->map_words = DIV_ROUND_UP(net_device->send_section_cnt,
  335. BITS_PER_LONG);
  336. net_device->send_section_map =
  337. kzalloc(net_device->map_words * sizeof(ulong), GFP_KERNEL);
  338. if (net_device->send_section_map == NULL) {
  339. ret = -ENOMEM;
  340. goto cleanup;
  341. }
  342. goto exit;
  343. cleanup:
  344. netvsc_destroy_buf(net_device);
  345. exit:
  346. return ret;
  347. }
  348. /* Negotiate NVSP protocol version */
  349. static int negotiate_nvsp_ver(struct hv_device *device,
  350. struct netvsc_device *net_device,
  351. struct nvsp_message *init_packet,
  352. u32 nvsp_ver)
  353. {
  354. int ret;
  355. unsigned long t;
  356. memset(init_packet, 0, sizeof(struct nvsp_message));
  357. init_packet->hdr.msg_type = NVSP_MSG_TYPE_INIT;
  358. init_packet->msg.init_msg.init.min_protocol_ver = nvsp_ver;
  359. init_packet->msg.init_msg.init.max_protocol_ver = nvsp_ver;
  360. /* Send the init request */
  361. ret = vmbus_sendpacket(device->channel, init_packet,
  362. sizeof(struct nvsp_message),
  363. (unsigned long)init_packet,
  364. VM_PKT_DATA_INBAND,
  365. VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
  366. if (ret != 0)
  367. return ret;
  368. t = wait_for_completion_timeout(&net_device->channel_init_wait, 5*HZ);
  369. if (t == 0)
  370. return -ETIMEDOUT;
  371. if (init_packet->msg.init_msg.init_complete.status !=
  372. NVSP_STAT_SUCCESS)
  373. return -EINVAL;
  374. if (nvsp_ver == NVSP_PROTOCOL_VERSION_1)
  375. return 0;
  376. /* NVSPv2 only: Send NDIS config */
  377. memset(init_packet, 0, sizeof(struct nvsp_message));
  378. init_packet->hdr.msg_type = NVSP_MSG2_TYPE_SEND_NDIS_CONFIG;
  379. init_packet->msg.v2_msg.send_ndis_config.mtu = net_device->ndev->mtu +
  380. ETH_HLEN;
  381. init_packet->msg.v2_msg.send_ndis_config.capability.ieee8021q = 1;
  382. ret = vmbus_sendpacket(device->channel, init_packet,
  383. sizeof(struct nvsp_message),
  384. (unsigned long)init_packet,
  385. VM_PKT_DATA_INBAND, 0);
  386. return ret;
  387. }
  388. static int netvsc_connect_vsp(struct hv_device *device)
  389. {
  390. int ret;
  391. struct netvsc_device *net_device;
  392. struct nvsp_message *init_packet;
  393. int ndis_version;
  394. struct net_device *ndev;
  395. u32 ver_list[] = { NVSP_PROTOCOL_VERSION_1, NVSP_PROTOCOL_VERSION_2,
  396. NVSP_PROTOCOL_VERSION_4, NVSP_PROTOCOL_VERSION_5 };
  397. int i, num_ver = 4; /* number of different NVSP versions */
  398. net_device = get_outbound_net_device(device);
  399. if (!net_device)
  400. return -ENODEV;
  401. ndev = net_device->ndev;
  402. init_packet = &net_device->channel_init_pkt;
  403. /* Negotiate the latest NVSP protocol supported */
  404. for (i = num_ver - 1; i >= 0; i--)
  405. if (negotiate_nvsp_ver(device, net_device, init_packet,
  406. ver_list[i]) == 0) {
  407. net_device->nvsp_version = ver_list[i];
  408. break;
  409. }
  410. if (i < 0) {
  411. ret = -EPROTO;
  412. goto cleanup;
  413. }
  414. pr_debug("Negotiated NVSP version:%x\n", net_device->nvsp_version);
  415. /* Send the ndis version */
  416. memset(init_packet, 0, sizeof(struct nvsp_message));
  417. if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_4)
  418. ndis_version = 0x00060001;
  419. else
  420. ndis_version = 0x0006001e;
  421. init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_NDIS_VER;
  422. init_packet->msg.v1_msg.
  423. send_ndis_ver.ndis_major_ver =
  424. (ndis_version & 0xFFFF0000) >> 16;
  425. init_packet->msg.v1_msg.
  426. send_ndis_ver.ndis_minor_ver =
  427. ndis_version & 0xFFFF;
  428. /* Send the init request */
  429. ret = vmbus_sendpacket(device->channel, init_packet,
  430. sizeof(struct nvsp_message),
  431. (unsigned long)init_packet,
  432. VM_PKT_DATA_INBAND, 0);
  433. if (ret != 0)
  434. goto cleanup;
  435. /* Post the big receive buffer to NetVSP */
  436. if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_2)
  437. net_device->recv_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY;
  438. else
  439. net_device->recv_buf_size = NETVSC_RECEIVE_BUFFER_SIZE;
  440. net_device->send_buf_size = NETVSC_SEND_BUFFER_SIZE;
  441. ret = netvsc_init_buf(device);
  442. cleanup:
  443. return ret;
  444. }
  445. static void netvsc_disconnect_vsp(struct netvsc_device *net_device)
  446. {
  447. netvsc_destroy_buf(net_device);
  448. }
  449. /*
  450. * netvsc_device_remove - Callback when the root bus device is removed
  451. */
  452. int netvsc_device_remove(struct hv_device *device)
  453. {
  454. struct netvsc_device *net_device;
  455. unsigned long flags;
  456. net_device = hv_get_drvdata(device);
  457. netvsc_disconnect_vsp(net_device);
  458. /*
  459. * Since we have already drained, we don't need to busy wait
  460. * as was done in final_release_stor_device()
  461. * Note that we cannot set the ext pointer to NULL until
  462. * we have drained - to drain the outgoing packets, we need to
  463. * allow incoming packets.
  464. */
  465. spin_lock_irqsave(&device->channel->inbound_lock, flags);
  466. hv_set_drvdata(device, NULL);
  467. spin_unlock_irqrestore(&device->channel->inbound_lock, flags);
  468. /*
  469. * At this point, no one should be accessing net_device
  470. * except in here
  471. */
  472. dev_notice(&device->device, "net device safe to remove\n");
  473. /* Now, we can close the channel safely */
  474. vmbus_close(device->channel);
  475. /* Release all resources */
  476. vfree(net_device->sub_cb_buf);
  477. free_netvsc_device(net_device);
  478. return 0;
  479. }
  480. #define RING_AVAIL_PERCENT_HIWATER 20
  481. #define RING_AVAIL_PERCENT_LOWATER 10
  482. /*
  483. * Get the percentage of available bytes to write in the ring.
  484. * The return value is in range from 0 to 100.
  485. */
  486. static inline u32 hv_ringbuf_avail_percent(
  487. struct hv_ring_buffer_info *ring_info)
  488. {
  489. u32 avail_read, avail_write;
  490. hv_get_ringbuffer_availbytes(ring_info, &avail_read, &avail_write);
  491. return avail_write * 100 / ring_info->ring_datasize;
  492. }
  493. static inline void netvsc_free_send_slot(struct netvsc_device *net_device,
  494. u32 index)
  495. {
  496. sync_change_bit(index, net_device->send_section_map);
  497. }
  498. static void netvsc_send_completion(struct netvsc_device *net_device,
  499. struct hv_device *device,
  500. struct vmpacket_descriptor *packet)
  501. {
  502. struct nvsp_message *nvsp_packet;
  503. struct hv_netvsc_packet *nvsc_packet;
  504. struct net_device *ndev;
  505. u32 send_index;
  506. ndev = net_device->ndev;
  507. nvsp_packet = (struct nvsp_message *)((unsigned long)packet +
  508. (packet->offset8 << 3));
  509. if ((nvsp_packet->hdr.msg_type == NVSP_MSG_TYPE_INIT_COMPLETE) ||
  510. (nvsp_packet->hdr.msg_type ==
  511. NVSP_MSG1_TYPE_SEND_RECV_BUF_COMPLETE) ||
  512. (nvsp_packet->hdr.msg_type ==
  513. NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE) ||
  514. (nvsp_packet->hdr.msg_type ==
  515. NVSP_MSG5_TYPE_SUBCHANNEL)) {
  516. /* Copy the response back */
  517. memcpy(&net_device->channel_init_pkt, nvsp_packet,
  518. sizeof(struct nvsp_message));
  519. complete(&net_device->channel_init_wait);
  520. } else if (nvsp_packet->hdr.msg_type ==
  521. NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE) {
  522. int num_outstanding_sends;
  523. u16 q_idx = 0;
  524. struct vmbus_channel *channel = device->channel;
  525. int queue_sends;
  526. /* Get the send context */
  527. nvsc_packet = (struct hv_netvsc_packet *)(unsigned long)
  528. packet->trans_id;
  529. /* Notify the layer above us */
  530. if (nvsc_packet) {
  531. send_index = nvsc_packet->send_buf_index;
  532. if (send_index != NETVSC_INVALID_INDEX)
  533. netvsc_free_send_slot(net_device, send_index);
  534. q_idx = nvsc_packet->q_idx;
  535. channel = nvsc_packet->channel;
  536. nvsc_packet->send_completion(nvsc_packet->
  537. send_completion_ctx);
  538. }
  539. num_outstanding_sends =
  540. atomic_dec_return(&net_device->num_outstanding_sends);
  541. queue_sends = atomic_dec_return(&net_device->
  542. queue_sends[q_idx]);
  543. if (net_device->destroy && num_outstanding_sends == 0)
  544. wake_up(&net_device->wait_drain);
  545. if (netif_tx_queue_stopped(netdev_get_tx_queue(ndev, q_idx)) &&
  546. !net_device->start_remove &&
  547. (hv_ringbuf_avail_percent(&channel->outbound) >
  548. RING_AVAIL_PERCENT_HIWATER || queue_sends < 1))
  549. netif_tx_wake_queue(netdev_get_tx_queue(
  550. ndev, q_idx));
  551. } else {
  552. netdev_err(ndev, "Unknown send completion packet type- "
  553. "%d received!!\n", nvsp_packet->hdr.msg_type);
  554. }
  555. }
  556. static u32 netvsc_get_next_send_section(struct netvsc_device *net_device)
  557. {
  558. unsigned long index;
  559. u32 max_words = net_device->map_words;
  560. unsigned long *map_addr = (unsigned long *)net_device->send_section_map;
  561. u32 section_cnt = net_device->send_section_cnt;
  562. int ret_val = NETVSC_INVALID_INDEX;
  563. int i;
  564. int prev_val;
  565. for (i = 0; i < max_words; i++) {
  566. if (!~(map_addr[i]))
  567. continue;
  568. index = ffz(map_addr[i]);
  569. prev_val = sync_test_and_set_bit(index, &map_addr[i]);
  570. if (prev_val)
  571. continue;
  572. if ((index + (i * BITS_PER_LONG)) >= section_cnt)
  573. break;
  574. ret_val = (index + (i * BITS_PER_LONG));
  575. break;
  576. }
  577. return ret_val;
  578. }
  579. static u32 netvsc_copy_to_send_buf(struct netvsc_device *net_device,
  580. unsigned int section_index,
  581. u32 pend_size,
  582. struct hv_netvsc_packet *packet)
  583. {
  584. char *start = net_device->send_buf;
  585. char *dest = start + (section_index * net_device->send_section_size)
  586. + pend_size;
  587. int i;
  588. u32 msg_size = 0;
  589. u32 padding = 0;
  590. u32 remain = packet->total_data_buflen % net_device->pkt_align;
  591. u32 page_count = packet->cp_partial ? packet->rmsg_pgcnt :
  592. packet->page_buf_cnt;
  593. /* Add padding */
  594. if (packet->is_data_pkt && packet->xmit_more && remain &&
  595. !packet->cp_partial) {
  596. padding = net_device->pkt_align - remain;
  597. packet->rndis_msg->msg_len += padding;
  598. packet->total_data_buflen += padding;
  599. }
  600. for (i = 0; i < page_count; i++) {
  601. char *src = phys_to_virt(packet->page_buf[i].pfn << PAGE_SHIFT);
  602. u32 offset = packet->page_buf[i].offset;
  603. u32 len = packet->page_buf[i].len;
  604. memcpy(dest, (src + offset), len);
  605. msg_size += len;
  606. dest += len;
  607. }
  608. if (padding) {
  609. memset(dest, 0, padding);
  610. msg_size += padding;
  611. }
  612. return msg_size;
  613. }
  614. static inline int netvsc_send_pkt(
  615. struct hv_netvsc_packet *packet,
  616. struct netvsc_device *net_device)
  617. {
  618. struct nvsp_message nvmsg;
  619. struct vmbus_channel *out_channel = packet->channel;
  620. u16 q_idx = packet->q_idx;
  621. struct net_device *ndev = net_device->ndev;
  622. u64 req_id;
  623. int ret;
  624. struct hv_page_buffer *pgbuf;
  625. nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT;
  626. if (packet->is_data_pkt) {
  627. /* 0 is RMC_DATA; */
  628. nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 0;
  629. } else {
  630. /* 1 is RMC_CONTROL; */
  631. nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 1;
  632. }
  633. nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_index =
  634. packet->send_buf_index;
  635. if (packet->send_buf_index == NETVSC_INVALID_INDEX)
  636. nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size = 0;
  637. else
  638. nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size =
  639. packet->total_data_buflen;
  640. if (packet->send_completion)
  641. req_id = (ulong)packet;
  642. else
  643. req_id = 0;
  644. if (out_channel->rescind)
  645. return -ENODEV;
  646. if (packet->page_buf_cnt) {
  647. pgbuf = packet->cp_partial ? packet->page_buf +
  648. packet->rmsg_pgcnt : packet->page_buf;
  649. ret = vmbus_sendpacket_pagebuffer(out_channel,
  650. pgbuf,
  651. packet->page_buf_cnt,
  652. &nvmsg,
  653. sizeof(struct nvsp_message),
  654. req_id);
  655. } else {
  656. ret = vmbus_sendpacket(
  657. out_channel, &nvmsg,
  658. sizeof(struct nvsp_message),
  659. req_id,
  660. VM_PKT_DATA_INBAND,
  661. VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
  662. }
  663. if (ret == 0) {
  664. atomic_inc(&net_device->num_outstanding_sends);
  665. atomic_inc(&net_device->queue_sends[q_idx]);
  666. if (hv_ringbuf_avail_percent(&out_channel->outbound) <
  667. RING_AVAIL_PERCENT_LOWATER) {
  668. netif_tx_stop_queue(netdev_get_tx_queue(
  669. ndev, q_idx));
  670. if (atomic_read(&net_device->
  671. queue_sends[q_idx]) < 1)
  672. netif_tx_wake_queue(netdev_get_tx_queue(
  673. ndev, q_idx));
  674. }
  675. } else if (ret == -EAGAIN) {
  676. netif_tx_stop_queue(netdev_get_tx_queue(
  677. ndev, q_idx));
  678. if (atomic_read(&net_device->queue_sends[q_idx]) < 1) {
  679. netif_tx_wake_queue(netdev_get_tx_queue(
  680. ndev, q_idx));
  681. ret = -ENOSPC;
  682. }
  683. } else {
  684. netdev_err(ndev, "Unable to send packet %p ret %d\n",
  685. packet, ret);
  686. }
  687. return ret;
  688. }
  689. int netvsc_send(struct hv_device *device,
  690. struct hv_netvsc_packet *packet)
  691. {
  692. struct netvsc_device *net_device;
  693. int ret = 0, m_ret = 0;
  694. struct vmbus_channel *out_channel;
  695. u16 q_idx = packet->q_idx;
  696. u32 pktlen = packet->total_data_buflen, msd_len = 0;
  697. unsigned int section_index = NETVSC_INVALID_INDEX;
  698. struct sk_buff *skb = NULL;
  699. unsigned long flag;
  700. struct multi_send_data *msdp;
  701. struct hv_netvsc_packet *msd_send = NULL, *cur_send = NULL;
  702. bool try_batch;
  703. net_device = get_outbound_net_device(device);
  704. if (!net_device)
  705. return -ENODEV;
  706. out_channel = net_device->chn_table[q_idx];
  707. if (!out_channel) {
  708. out_channel = device->channel;
  709. q_idx = 0;
  710. packet->q_idx = 0;
  711. }
  712. packet->channel = out_channel;
  713. packet->send_buf_index = NETVSC_INVALID_INDEX;
  714. packet->cp_partial = false;
  715. msdp = &net_device->msd[q_idx];
  716. /* batch packets in send buffer if possible */
  717. spin_lock_irqsave(&msdp->lock, flag);
  718. if (msdp->pkt)
  719. msd_len = msdp->pkt->total_data_buflen;
  720. try_batch = packet->is_data_pkt && msd_len > 0 && msdp->count <
  721. net_device->max_pkt;
  722. if (try_batch && msd_len + pktlen + net_device->pkt_align <
  723. net_device->send_section_size) {
  724. section_index = msdp->pkt->send_buf_index;
  725. } else if (try_batch && msd_len + packet->rmsg_size <
  726. net_device->send_section_size) {
  727. section_index = msdp->pkt->send_buf_index;
  728. packet->cp_partial = true;
  729. } else if (packet->is_data_pkt && pktlen + net_device->pkt_align <
  730. net_device->send_section_size) {
  731. section_index = netvsc_get_next_send_section(net_device);
  732. if (section_index != NETVSC_INVALID_INDEX) {
  733. msd_send = msdp->pkt;
  734. msdp->pkt = NULL;
  735. msdp->count = 0;
  736. msd_len = 0;
  737. }
  738. }
  739. if (section_index != NETVSC_INVALID_INDEX) {
  740. netvsc_copy_to_send_buf(net_device,
  741. section_index, msd_len,
  742. packet);
  743. packet->send_buf_index = section_index;
  744. if (packet->cp_partial) {
  745. packet->page_buf_cnt -= packet->rmsg_pgcnt;
  746. packet->total_data_buflen = msd_len + packet->rmsg_size;
  747. } else {
  748. packet->page_buf_cnt = 0;
  749. packet->total_data_buflen += msd_len;
  750. }
  751. if (msdp->pkt)
  752. netvsc_xmit_completion(msdp->pkt);
  753. if (packet->xmit_more && !packet->cp_partial) {
  754. msdp->pkt = packet;
  755. msdp->count++;
  756. } else {
  757. cur_send = packet;
  758. msdp->pkt = NULL;
  759. msdp->count = 0;
  760. }
  761. } else {
  762. msd_send = msdp->pkt;
  763. msdp->pkt = NULL;
  764. msdp->count = 0;
  765. cur_send = packet;
  766. }
  767. spin_unlock_irqrestore(&msdp->lock, flag);
  768. if (msd_send) {
  769. m_ret = netvsc_send_pkt(msd_send, net_device);
  770. if (m_ret != 0) {
  771. netvsc_free_send_slot(net_device,
  772. msd_send->send_buf_index);
  773. netvsc_xmit_completion(msd_send);
  774. }
  775. }
  776. if (cur_send)
  777. ret = netvsc_send_pkt(cur_send, net_device);
  778. if (ret != 0) {
  779. if (section_index != NETVSC_INVALID_INDEX)
  780. netvsc_free_send_slot(net_device, section_index);
  781. } else if (skb) {
  782. dev_kfree_skb_any(skb);
  783. }
  784. return ret;
  785. }
  786. static void netvsc_send_recv_completion(struct hv_device *device,
  787. struct vmbus_channel *channel,
  788. struct netvsc_device *net_device,
  789. u64 transaction_id, u32 status)
  790. {
  791. struct nvsp_message recvcompMessage;
  792. int retries = 0;
  793. int ret;
  794. struct net_device *ndev;
  795. ndev = net_device->ndev;
  796. recvcompMessage.hdr.msg_type =
  797. NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE;
  798. recvcompMessage.msg.v1_msg.send_rndis_pkt_complete.status = status;
  799. retry_send_cmplt:
  800. /* Send the completion */
  801. ret = vmbus_sendpacket(channel, &recvcompMessage,
  802. sizeof(struct nvsp_message), transaction_id,
  803. VM_PKT_COMP, 0);
  804. if (ret == 0) {
  805. /* success */
  806. /* no-op */
  807. } else if (ret == -EAGAIN) {
  808. /* no more room...wait a bit and attempt to retry 3 times */
  809. retries++;
  810. netdev_err(ndev, "unable to send receive completion pkt"
  811. " (tid %llx)...retrying %d\n", transaction_id, retries);
  812. if (retries < 4) {
  813. udelay(100);
  814. goto retry_send_cmplt;
  815. } else {
  816. netdev_err(ndev, "unable to send receive "
  817. "completion pkt (tid %llx)...give up retrying\n",
  818. transaction_id);
  819. }
  820. } else {
  821. netdev_err(ndev, "unable to send receive "
  822. "completion pkt - %llx\n", transaction_id);
  823. }
  824. }
  825. static void netvsc_receive(struct netvsc_device *net_device,
  826. struct vmbus_channel *channel,
  827. struct hv_device *device,
  828. struct vmpacket_descriptor *packet)
  829. {
  830. struct vmtransfer_page_packet_header *vmxferpage_packet;
  831. struct nvsp_message *nvsp_packet;
  832. struct hv_netvsc_packet nv_pkt;
  833. struct hv_netvsc_packet *netvsc_packet = &nv_pkt;
  834. u32 status = NVSP_STAT_SUCCESS;
  835. int i;
  836. int count = 0;
  837. struct net_device *ndev;
  838. ndev = net_device->ndev;
  839. /*
  840. * All inbound packets other than send completion should be xfer page
  841. * packet
  842. */
  843. if (packet->type != VM_PKT_DATA_USING_XFER_PAGES) {
  844. netdev_err(ndev, "Unknown packet type received - %d\n",
  845. packet->type);
  846. return;
  847. }
  848. nvsp_packet = (struct nvsp_message *)((unsigned long)packet +
  849. (packet->offset8 << 3));
  850. /* Make sure this is a valid nvsp packet */
  851. if (nvsp_packet->hdr.msg_type !=
  852. NVSP_MSG1_TYPE_SEND_RNDIS_PKT) {
  853. netdev_err(ndev, "Unknown nvsp packet type received-"
  854. " %d\n", nvsp_packet->hdr.msg_type);
  855. return;
  856. }
  857. vmxferpage_packet = (struct vmtransfer_page_packet_header *)packet;
  858. if (vmxferpage_packet->xfer_pageset_id != NETVSC_RECEIVE_BUFFER_ID) {
  859. netdev_err(ndev, "Invalid xfer page set id - "
  860. "expecting %x got %x\n", NETVSC_RECEIVE_BUFFER_ID,
  861. vmxferpage_packet->xfer_pageset_id);
  862. return;
  863. }
  864. count = vmxferpage_packet->range_cnt;
  865. netvsc_packet->channel = channel;
  866. /* Each range represents 1 RNDIS pkt that contains 1 ethernet frame */
  867. for (i = 0; i < count; i++) {
  868. /* Initialize the netvsc packet */
  869. netvsc_packet->status = NVSP_STAT_SUCCESS;
  870. netvsc_packet->data = (void *)((unsigned long)net_device->
  871. recv_buf + vmxferpage_packet->ranges[i].byte_offset);
  872. netvsc_packet->total_data_buflen =
  873. vmxferpage_packet->ranges[i].byte_count;
  874. /* Pass it to the upper layer */
  875. rndis_filter_receive(device, netvsc_packet);
  876. if (netvsc_packet->status != NVSP_STAT_SUCCESS)
  877. status = NVSP_STAT_FAIL;
  878. }
  879. netvsc_send_recv_completion(device, channel, net_device,
  880. vmxferpage_packet->d.trans_id, status);
  881. }
  882. static void netvsc_send_table(struct hv_device *hdev,
  883. struct vmpacket_descriptor *vmpkt)
  884. {
  885. struct netvsc_device *nvscdev;
  886. struct net_device *ndev;
  887. struct nvsp_message *nvmsg;
  888. int i;
  889. u32 count, *tab;
  890. nvscdev = get_outbound_net_device(hdev);
  891. if (!nvscdev)
  892. return;
  893. ndev = nvscdev->ndev;
  894. nvmsg = (struct nvsp_message *)((unsigned long)vmpkt +
  895. (vmpkt->offset8 << 3));
  896. if (nvmsg->hdr.msg_type != NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE)
  897. return;
  898. count = nvmsg->msg.v5_msg.send_table.count;
  899. if (count != VRSS_SEND_TAB_SIZE) {
  900. netdev_err(ndev, "Received wrong send-table size:%u\n", count);
  901. return;
  902. }
  903. tab = (u32 *)((unsigned long)&nvmsg->msg.v5_msg.send_table +
  904. nvmsg->msg.v5_msg.send_table.offset);
  905. for (i = 0; i < count; i++)
  906. nvscdev->send_table[i] = tab[i];
  907. }
  908. void netvsc_channel_cb(void *context)
  909. {
  910. int ret;
  911. struct vmbus_channel *channel = (struct vmbus_channel *)context;
  912. struct hv_device *device;
  913. struct netvsc_device *net_device;
  914. u32 bytes_recvd;
  915. u64 request_id;
  916. struct vmpacket_descriptor *desc;
  917. unsigned char *buffer;
  918. int bufferlen = NETVSC_PACKET_SIZE;
  919. struct net_device *ndev;
  920. if (channel->primary_channel != NULL)
  921. device = channel->primary_channel->device_obj;
  922. else
  923. device = channel->device_obj;
  924. net_device = get_inbound_net_device(device);
  925. if (!net_device)
  926. return;
  927. ndev = net_device->ndev;
  928. buffer = get_per_channel_state(channel);
  929. do {
  930. ret = vmbus_recvpacket_raw(channel, buffer, bufferlen,
  931. &bytes_recvd, &request_id);
  932. if (ret == 0) {
  933. if (bytes_recvd > 0) {
  934. desc = (struct vmpacket_descriptor *)buffer;
  935. switch (desc->type) {
  936. case VM_PKT_COMP:
  937. netvsc_send_completion(net_device,
  938. device, desc);
  939. break;
  940. case VM_PKT_DATA_USING_XFER_PAGES:
  941. netvsc_receive(net_device, channel,
  942. device, desc);
  943. break;
  944. case VM_PKT_DATA_INBAND:
  945. netvsc_send_table(device, desc);
  946. break;
  947. default:
  948. netdev_err(ndev,
  949. "unhandled packet type %d, "
  950. "tid %llx len %d\n",
  951. desc->type, request_id,
  952. bytes_recvd);
  953. break;
  954. }
  955. } else {
  956. /*
  957. * We are done for this pass.
  958. */
  959. break;
  960. }
  961. } else if (ret == -ENOBUFS) {
  962. if (bufferlen > NETVSC_PACKET_SIZE)
  963. kfree(buffer);
  964. /* Handle large packet */
  965. buffer = kmalloc(bytes_recvd, GFP_ATOMIC);
  966. if (buffer == NULL) {
  967. /* Try again next time around */
  968. netdev_err(ndev,
  969. "unable to allocate buffer of size "
  970. "(%d)!!\n", bytes_recvd);
  971. break;
  972. }
  973. bufferlen = bytes_recvd;
  974. }
  975. } while (1);
  976. if (bufferlen > NETVSC_PACKET_SIZE)
  977. kfree(buffer);
  978. return;
  979. }
  980. /*
  981. * netvsc_device_add - Callback when the device belonging to this
  982. * driver is added
  983. */
  984. int netvsc_device_add(struct hv_device *device, void *additional_info)
  985. {
  986. int ret = 0;
  987. int ring_size =
  988. ((struct netvsc_device_info *)additional_info)->ring_size;
  989. struct netvsc_device *net_device;
  990. struct net_device *ndev;
  991. net_device = alloc_net_device(device);
  992. if (!net_device)
  993. return -ENOMEM;
  994. net_device->ring_size = ring_size;
  995. /*
  996. * Coming into this function, struct net_device * is
  997. * registered as the driver private data.
  998. * In alloc_net_device(), we register struct netvsc_device *
  999. * as the driver private data and stash away struct net_device *
  1000. * in struct netvsc_device *.
  1001. */
  1002. ndev = net_device->ndev;
  1003. /* Add netvsc_device context to netvsc_device */
  1004. net_device->nd_ctx = netdev_priv(ndev);
  1005. /* Initialize the NetVSC channel extension */
  1006. init_completion(&net_device->channel_init_wait);
  1007. set_per_channel_state(device->channel, net_device->cb_buffer);
  1008. /* Open the channel */
  1009. ret = vmbus_open(device->channel, ring_size * PAGE_SIZE,
  1010. ring_size * PAGE_SIZE, NULL, 0,
  1011. netvsc_channel_cb, device->channel);
  1012. if (ret != 0) {
  1013. netdev_err(ndev, "unable to open channel: %d\n", ret);
  1014. goto cleanup;
  1015. }
  1016. /* Channel is opened */
  1017. pr_info("hv_netvsc channel opened successfully\n");
  1018. net_device->chn_table[0] = device->channel;
  1019. /* Connect with the NetVsp */
  1020. ret = netvsc_connect_vsp(device);
  1021. if (ret != 0) {
  1022. netdev_err(ndev,
  1023. "unable to connect to NetVSP - %d\n", ret);
  1024. goto close;
  1025. }
  1026. return ret;
  1027. close:
  1028. /* Now, we can close the channel safely */
  1029. vmbus_close(device->channel);
  1030. cleanup:
  1031. free_netvsc_device(net_device);
  1032. return ret;
  1033. }