main.c 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455
  1. /*
  2. * Copyright (c) 2013, Mellanox Technologies inc. All rights reserved.
  3. *
  4. * This software is available to you under a choice of one of two
  5. * licenses. You may choose to be licensed under the terms of the GNU
  6. * General Public License (GPL) Version 2, available from the file
  7. * COPYING in the main directory of this source tree, or the
  8. * OpenIB.org BSD license below:
  9. *
  10. * Redistribution and use in source and binary forms, with or
  11. * without modification, are permitted provided that the following
  12. * conditions are met:
  13. *
  14. * - Redistributions of source code must retain the above
  15. * copyright notice, this list of conditions and the following
  16. * disclaimer.
  17. *
  18. * - Redistributions in binary form must reproduce the above
  19. * copyright notice, this list of conditions and the following
  20. * disclaimer in the documentation and/or other materials
  21. * provided with the distribution.
  22. *
  23. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30. * SOFTWARE.
  31. */
  32. #include <asm-generic/kmap_types.h>
  33. #include <linux/module.h>
  34. #include <linux/init.h>
  35. #include <linux/errno.h>
  36. #include <linux/pci.h>
  37. #include <linux/dma-mapping.h>
  38. #include <linux/slab.h>
  39. #include <linux/io-mapping.h>
  40. #include <linux/sched.h>
  41. #include <rdma/ib_user_verbs.h>
  42. #include <rdma/ib_smi.h>
  43. #include <rdma/ib_umem.h>
  44. #include "user.h"
  45. #include "mlx5_ib.h"
  46. #define DRIVER_NAME "mlx5_ib"
  47. #define DRIVER_VERSION "2.2-1"
  48. #define DRIVER_RELDATE "Feb 2014"
  49. MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
  50. MODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver");
  51. MODULE_LICENSE("Dual BSD/GPL");
  52. MODULE_VERSION(DRIVER_VERSION);
  53. static int deprecated_prof_sel = 2;
  54. module_param_named(prof_sel, deprecated_prof_sel, int, 0444);
  55. MODULE_PARM_DESC(prof_sel, "profile selector. Deprecated here. Moved to module mlx5_core");
  56. static char mlx5_version[] =
  57. DRIVER_NAME ": Mellanox Connect-IB Infiniband driver v"
  58. DRIVER_VERSION " (" DRIVER_RELDATE ")\n";
  59. int mlx5_vector2eqn(struct mlx5_ib_dev *dev, int vector, int *eqn, int *irqn)
  60. {
  61. struct mlx5_eq_table *table = &dev->mdev->priv.eq_table;
  62. struct mlx5_eq *eq, *n;
  63. int err = -ENOENT;
  64. spin_lock(&table->lock);
  65. list_for_each_entry_safe(eq, n, &dev->eqs_list, list) {
  66. if (eq->index == vector) {
  67. *eqn = eq->eqn;
  68. *irqn = eq->irqn;
  69. err = 0;
  70. break;
  71. }
  72. }
  73. spin_unlock(&table->lock);
  74. return err;
  75. }
  76. static int alloc_comp_eqs(struct mlx5_ib_dev *dev)
  77. {
  78. struct mlx5_eq_table *table = &dev->mdev->priv.eq_table;
  79. char name[MLX5_MAX_EQ_NAME];
  80. struct mlx5_eq *eq, *n;
  81. int ncomp_vec;
  82. int nent;
  83. int err;
  84. int i;
  85. INIT_LIST_HEAD(&dev->eqs_list);
  86. ncomp_vec = table->num_comp_vectors;
  87. nent = MLX5_COMP_EQ_SIZE;
  88. for (i = 0; i < ncomp_vec; i++) {
  89. eq = kzalloc(sizeof(*eq), GFP_KERNEL);
  90. if (!eq) {
  91. err = -ENOMEM;
  92. goto clean;
  93. }
  94. snprintf(name, MLX5_MAX_EQ_NAME, "mlx5_comp%d", i);
  95. err = mlx5_create_map_eq(dev->mdev, eq,
  96. i + MLX5_EQ_VEC_COMP_BASE, nent, 0,
  97. name, &dev->mdev->priv.uuari.uars[0]);
  98. if (err) {
  99. kfree(eq);
  100. goto clean;
  101. }
  102. mlx5_ib_dbg(dev, "allocated completion EQN %d\n", eq->eqn);
  103. eq->index = i;
  104. spin_lock(&table->lock);
  105. list_add_tail(&eq->list, &dev->eqs_list);
  106. spin_unlock(&table->lock);
  107. }
  108. dev->num_comp_vectors = ncomp_vec;
  109. return 0;
  110. clean:
  111. spin_lock(&table->lock);
  112. list_for_each_entry_safe(eq, n, &dev->eqs_list, list) {
  113. list_del(&eq->list);
  114. spin_unlock(&table->lock);
  115. if (mlx5_destroy_unmap_eq(dev->mdev, eq))
  116. mlx5_ib_warn(dev, "failed to destroy EQ 0x%x\n", eq->eqn);
  117. kfree(eq);
  118. spin_lock(&table->lock);
  119. }
  120. spin_unlock(&table->lock);
  121. return err;
  122. }
  123. static void free_comp_eqs(struct mlx5_ib_dev *dev)
  124. {
  125. struct mlx5_eq_table *table = &dev->mdev->priv.eq_table;
  126. struct mlx5_eq *eq, *n;
  127. spin_lock(&table->lock);
  128. list_for_each_entry_safe(eq, n, &dev->eqs_list, list) {
  129. list_del(&eq->list);
  130. spin_unlock(&table->lock);
  131. if (mlx5_destroy_unmap_eq(dev->mdev, eq))
  132. mlx5_ib_warn(dev, "failed to destroy EQ 0x%x\n", eq->eqn);
  133. kfree(eq);
  134. spin_lock(&table->lock);
  135. }
  136. spin_unlock(&table->lock);
  137. }
  138. static int mlx5_ib_query_device(struct ib_device *ibdev,
  139. struct ib_device_attr *props)
  140. {
  141. struct mlx5_ib_dev *dev = to_mdev(ibdev);
  142. struct ib_smp *in_mad = NULL;
  143. struct ib_smp *out_mad = NULL;
  144. struct mlx5_general_caps *gen;
  145. int err = -ENOMEM;
  146. int max_rq_sg;
  147. int max_sq_sg;
  148. u64 flags;
  149. gen = &dev->mdev->caps.gen;
  150. in_mad = kzalloc(sizeof(*in_mad), GFP_KERNEL);
  151. out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL);
  152. if (!in_mad || !out_mad)
  153. goto out;
  154. init_query_mad(in_mad);
  155. in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
  156. err = mlx5_MAD_IFC(to_mdev(ibdev), 1, 1, 1, NULL, NULL, in_mad, out_mad);
  157. if (err)
  158. goto out;
  159. memset(props, 0, sizeof(*props));
  160. props->fw_ver = ((u64)fw_rev_maj(dev->mdev) << 32) |
  161. (fw_rev_min(dev->mdev) << 16) |
  162. fw_rev_sub(dev->mdev);
  163. props->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT |
  164. IB_DEVICE_PORT_ACTIVE_EVENT |
  165. IB_DEVICE_SYS_IMAGE_GUID |
  166. IB_DEVICE_RC_RNR_NAK_GEN;
  167. flags = gen->flags;
  168. if (flags & MLX5_DEV_CAP_FLAG_BAD_PKEY_CNTR)
  169. props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
  170. if (flags & MLX5_DEV_CAP_FLAG_BAD_QKEY_CNTR)
  171. props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
  172. if (flags & MLX5_DEV_CAP_FLAG_APM)
  173. props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
  174. props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY;
  175. if (flags & MLX5_DEV_CAP_FLAG_XRC)
  176. props->device_cap_flags |= IB_DEVICE_XRC;
  177. props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
  178. if (flags & MLX5_DEV_CAP_FLAG_SIG_HAND_OVER) {
  179. props->device_cap_flags |= IB_DEVICE_SIGNATURE_HANDOVER;
  180. /* At this stage no support for signature handover */
  181. props->sig_prot_cap = IB_PROT_T10DIF_TYPE_1 |
  182. IB_PROT_T10DIF_TYPE_2 |
  183. IB_PROT_T10DIF_TYPE_3;
  184. props->sig_guard_cap = IB_GUARD_T10DIF_CRC |
  185. IB_GUARD_T10DIF_CSUM;
  186. }
  187. if (flags & MLX5_DEV_CAP_FLAG_BLOCK_MCAST)
  188. props->device_cap_flags |= IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;
  189. props->vendor_id = be32_to_cpup((__be32 *)(out_mad->data + 36)) &
  190. 0xffffff;
  191. props->vendor_part_id = be16_to_cpup((__be16 *)(out_mad->data + 30));
  192. props->hw_ver = be32_to_cpup((__be32 *)(out_mad->data + 32));
  193. memcpy(&props->sys_image_guid, out_mad->data + 4, 8);
  194. props->max_mr_size = ~0ull;
  195. props->page_size_cap = gen->min_page_sz;
  196. props->max_qp = 1 << gen->log_max_qp;
  197. props->max_qp_wr = gen->max_wqes;
  198. max_rq_sg = gen->max_rq_desc_sz / sizeof(struct mlx5_wqe_data_seg);
  199. max_sq_sg = (gen->max_sq_desc_sz - sizeof(struct mlx5_wqe_ctrl_seg)) /
  200. sizeof(struct mlx5_wqe_data_seg);
  201. props->max_sge = min(max_rq_sg, max_sq_sg);
  202. props->max_cq = 1 << gen->log_max_cq;
  203. props->max_cqe = gen->max_cqes - 1;
  204. props->max_mr = 1 << gen->log_max_mkey;
  205. props->max_pd = 1 << gen->log_max_pd;
  206. props->max_qp_rd_atom = 1 << gen->log_max_ra_req_qp;
  207. props->max_qp_init_rd_atom = 1 << gen->log_max_ra_res_qp;
  208. props->max_srq = 1 << gen->log_max_srq;
  209. props->max_srq_wr = gen->max_srq_wqes - 1;
  210. props->local_ca_ack_delay = gen->local_ca_ack_delay;
  211. props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp;
  212. props->max_srq_sge = max_rq_sg - 1;
  213. props->max_fast_reg_page_list_len = (unsigned int)-1;
  214. props->local_ca_ack_delay = gen->local_ca_ack_delay;
  215. props->atomic_cap = IB_ATOMIC_NONE;
  216. props->masked_atomic_cap = IB_ATOMIC_NONE;
  217. props->max_pkeys = be16_to_cpup((__be16 *)(out_mad->data + 28));
  218. props->max_mcast_grp = 1 << gen->log_max_mcg;
  219. props->max_mcast_qp_attach = gen->max_qp_mcg;
  220. props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
  221. props->max_mcast_grp;
  222. props->max_map_per_fmr = INT_MAX; /* no limit in ConnectIB */
  223. out:
  224. kfree(in_mad);
  225. kfree(out_mad);
  226. return err;
  227. }
  228. int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
  229. struct ib_port_attr *props)
  230. {
  231. struct mlx5_ib_dev *dev = to_mdev(ibdev);
  232. struct ib_smp *in_mad = NULL;
  233. struct ib_smp *out_mad = NULL;
  234. struct mlx5_general_caps *gen;
  235. int ext_active_speed;
  236. int err = -ENOMEM;
  237. gen = &dev->mdev->caps.gen;
  238. if (port < 1 || port > gen->num_ports) {
  239. mlx5_ib_warn(dev, "invalid port number %d\n", port);
  240. return -EINVAL;
  241. }
  242. in_mad = kzalloc(sizeof(*in_mad), GFP_KERNEL);
  243. out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL);
  244. if (!in_mad || !out_mad)
  245. goto out;
  246. memset(props, 0, sizeof(*props));
  247. init_query_mad(in_mad);
  248. in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;
  249. in_mad->attr_mod = cpu_to_be32(port);
  250. err = mlx5_MAD_IFC(dev, 1, 1, port, NULL, NULL, in_mad, out_mad);
  251. if (err) {
  252. mlx5_ib_warn(dev, "err %d\n", err);
  253. goto out;
  254. }
  255. props->lid = be16_to_cpup((__be16 *)(out_mad->data + 16));
  256. props->lmc = out_mad->data[34] & 0x7;
  257. props->sm_lid = be16_to_cpup((__be16 *)(out_mad->data + 18));
  258. props->sm_sl = out_mad->data[36] & 0xf;
  259. props->state = out_mad->data[32] & 0xf;
  260. props->phys_state = out_mad->data[33] >> 4;
  261. props->port_cap_flags = be32_to_cpup((__be32 *)(out_mad->data + 20));
  262. props->gid_tbl_len = out_mad->data[50];
  263. props->max_msg_sz = 1 << gen->log_max_msg;
  264. props->pkey_tbl_len = gen->port[port - 1].pkey_table_len;
  265. props->bad_pkey_cntr = be16_to_cpup((__be16 *)(out_mad->data + 46));
  266. props->qkey_viol_cntr = be16_to_cpup((__be16 *)(out_mad->data + 48));
  267. props->active_width = out_mad->data[31] & 0xf;
  268. props->active_speed = out_mad->data[35] >> 4;
  269. props->max_mtu = out_mad->data[41] & 0xf;
  270. props->active_mtu = out_mad->data[36] >> 4;
  271. props->subnet_timeout = out_mad->data[51] & 0x1f;
  272. props->max_vl_num = out_mad->data[37] >> 4;
  273. props->init_type_reply = out_mad->data[41] >> 4;
  274. /* Check if extended speeds (EDR/FDR/...) are supported */
  275. if (props->port_cap_flags & IB_PORT_EXTENDED_SPEEDS_SUP) {
  276. ext_active_speed = out_mad->data[62] >> 4;
  277. switch (ext_active_speed) {
  278. case 1:
  279. props->active_speed = 16; /* FDR */
  280. break;
  281. case 2:
  282. props->active_speed = 32; /* EDR */
  283. break;
  284. }
  285. }
  286. /* If reported active speed is QDR, check if is FDR-10 */
  287. if (props->active_speed == 4) {
  288. if (gen->ext_port_cap[port - 1] &
  289. MLX_EXT_PORT_CAP_FLAG_EXTENDED_PORT_INFO) {
  290. init_query_mad(in_mad);
  291. in_mad->attr_id = MLX5_ATTR_EXTENDED_PORT_INFO;
  292. in_mad->attr_mod = cpu_to_be32(port);
  293. err = mlx5_MAD_IFC(dev, 1, 1, port,
  294. NULL, NULL, in_mad, out_mad);
  295. if (err)
  296. goto out;
  297. /* Checking LinkSpeedActive for FDR-10 */
  298. if (out_mad->data[15] & 0x1)
  299. props->active_speed = 8;
  300. }
  301. }
  302. out:
  303. kfree(in_mad);
  304. kfree(out_mad);
  305. return err;
  306. }
  307. static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
  308. union ib_gid *gid)
  309. {
  310. struct ib_smp *in_mad = NULL;
  311. struct ib_smp *out_mad = NULL;
  312. int err = -ENOMEM;
  313. in_mad = kzalloc(sizeof(*in_mad), GFP_KERNEL);
  314. out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL);
  315. if (!in_mad || !out_mad)
  316. goto out;
  317. init_query_mad(in_mad);
  318. in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;
  319. in_mad->attr_mod = cpu_to_be32(port);
  320. err = mlx5_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad);
  321. if (err)
  322. goto out;
  323. memcpy(gid->raw, out_mad->data + 8, 8);
  324. init_query_mad(in_mad);
  325. in_mad->attr_id = IB_SMP_ATTR_GUID_INFO;
  326. in_mad->attr_mod = cpu_to_be32(index / 8);
  327. err = mlx5_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad);
  328. if (err)
  329. goto out;
  330. memcpy(gid->raw + 8, out_mad->data + (index % 8) * 8, 8);
  331. out:
  332. kfree(in_mad);
  333. kfree(out_mad);
  334. return err;
  335. }
  336. static int mlx5_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
  337. u16 *pkey)
  338. {
  339. struct ib_smp *in_mad = NULL;
  340. struct ib_smp *out_mad = NULL;
  341. int err = -ENOMEM;
  342. in_mad = kzalloc(sizeof(*in_mad), GFP_KERNEL);
  343. out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL);
  344. if (!in_mad || !out_mad)
  345. goto out;
  346. init_query_mad(in_mad);
  347. in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE;
  348. in_mad->attr_mod = cpu_to_be32(index / 32);
  349. err = mlx5_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad);
  350. if (err)
  351. goto out;
  352. *pkey = be16_to_cpu(((__be16 *)out_mad->data)[index % 32]);
  353. out:
  354. kfree(in_mad);
  355. kfree(out_mad);
  356. return err;
  357. }
  358. struct mlx5_reg_node_desc {
  359. u8 desc[64];
  360. };
  361. static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,
  362. struct ib_device_modify *props)
  363. {
  364. struct mlx5_ib_dev *dev = to_mdev(ibdev);
  365. struct mlx5_reg_node_desc in;
  366. struct mlx5_reg_node_desc out;
  367. int err;
  368. if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
  369. return -EOPNOTSUPP;
  370. if (!(mask & IB_DEVICE_MODIFY_NODE_DESC))
  371. return 0;
  372. /*
  373. * If possible, pass node desc to FW, so it can generate
  374. * a 144 trap. If cmd fails, just ignore.
  375. */
  376. memcpy(&in, props->node_desc, 64);
  377. err = mlx5_core_access_reg(dev->mdev, &in, sizeof(in), &out,
  378. sizeof(out), MLX5_REG_NODE_DESC, 0, 1);
  379. if (err)
  380. return err;
  381. memcpy(ibdev->node_desc, props->node_desc, 64);
  382. return err;
  383. }
  384. static int mlx5_ib_modify_port(struct ib_device *ibdev, u8 port, int mask,
  385. struct ib_port_modify *props)
  386. {
  387. struct mlx5_ib_dev *dev = to_mdev(ibdev);
  388. struct ib_port_attr attr;
  389. u32 tmp;
  390. int err;
  391. mutex_lock(&dev->cap_mask_mutex);
  392. err = mlx5_ib_query_port(ibdev, port, &attr);
  393. if (err)
  394. goto out;
  395. tmp = (attr.port_cap_flags | props->set_port_cap_mask) &
  396. ~props->clr_port_cap_mask;
  397. err = mlx5_set_port_caps(dev->mdev, port, tmp);
  398. out:
  399. mutex_unlock(&dev->cap_mask_mutex);
  400. return err;
  401. }
  402. static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
  403. struct ib_udata *udata)
  404. {
  405. struct mlx5_ib_dev *dev = to_mdev(ibdev);
  406. struct mlx5_ib_alloc_ucontext_req_v2 req;
  407. struct mlx5_ib_alloc_ucontext_resp resp;
  408. struct mlx5_ib_ucontext *context;
  409. struct mlx5_general_caps *gen;
  410. struct mlx5_uuar_info *uuari;
  411. struct mlx5_uar *uars;
  412. int gross_uuars;
  413. int num_uars;
  414. int ver;
  415. int uuarn;
  416. int err;
  417. int i;
  418. size_t reqlen;
  419. gen = &dev->mdev->caps.gen;
  420. if (!dev->ib_active)
  421. return ERR_PTR(-EAGAIN);
  422. memset(&req, 0, sizeof(req));
  423. reqlen = udata->inlen - sizeof(struct ib_uverbs_cmd_hdr);
  424. if (reqlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
  425. ver = 0;
  426. else if (reqlen == sizeof(struct mlx5_ib_alloc_ucontext_req_v2))
  427. ver = 2;
  428. else
  429. return ERR_PTR(-EINVAL);
  430. err = ib_copy_from_udata(&req, udata, reqlen);
  431. if (err)
  432. return ERR_PTR(err);
  433. if (req.flags || req.reserved)
  434. return ERR_PTR(-EINVAL);
  435. if (req.total_num_uuars > MLX5_MAX_UUARS)
  436. return ERR_PTR(-ENOMEM);
  437. if (req.total_num_uuars == 0)
  438. return ERR_PTR(-EINVAL);
  439. req.total_num_uuars = ALIGN(req.total_num_uuars,
  440. MLX5_NON_FP_BF_REGS_PER_PAGE);
  441. if (req.num_low_latency_uuars > req.total_num_uuars - 1)
  442. return ERR_PTR(-EINVAL);
  443. num_uars = req.total_num_uuars / MLX5_NON_FP_BF_REGS_PER_PAGE;
  444. gross_uuars = num_uars * MLX5_BF_REGS_PER_PAGE;
  445. resp.qp_tab_size = 1 << gen->log_max_qp;
  446. resp.bf_reg_size = gen->bf_reg_size;
  447. resp.cache_line_size = L1_CACHE_BYTES;
  448. resp.max_sq_desc_sz = gen->max_sq_desc_sz;
  449. resp.max_rq_desc_sz = gen->max_rq_desc_sz;
  450. resp.max_send_wqebb = gen->max_wqes;
  451. resp.max_recv_wr = gen->max_wqes;
  452. resp.max_srq_recv_wr = gen->max_srq_wqes;
  453. context = kzalloc(sizeof(*context), GFP_KERNEL);
  454. if (!context)
  455. return ERR_PTR(-ENOMEM);
  456. uuari = &context->uuari;
  457. mutex_init(&uuari->lock);
  458. uars = kcalloc(num_uars, sizeof(*uars), GFP_KERNEL);
  459. if (!uars) {
  460. err = -ENOMEM;
  461. goto out_ctx;
  462. }
  463. uuari->bitmap = kcalloc(BITS_TO_LONGS(gross_uuars),
  464. sizeof(*uuari->bitmap),
  465. GFP_KERNEL);
  466. if (!uuari->bitmap) {
  467. err = -ENOMEM;
  468. goto out_uar_ctx;
  469. }
  470. /*
  471. * clear all fast path uuars
  472. */
  473. for (i = 0; i < gross_uuars; i++) {
  474. uuarn = i & 3;
  475. if (uuarn == 2 || uuarn == 3)
  476. set_bit(i, uuari->bitmap);
  477. }
  478. uuari->count = kcalloc(gross_uuars, sizeof(*uuari->count), GFP_KERNEL);
  479. if (!uuari->count) {
  480. err = -ENOMEM;
  481. goto out_bitmap;
  482. }
  483. for (i = 0; i < num_uars; i++) {
  484. err = mlx5_cmd_alloc_uar(dev->mdev, &uars[i].index);
  485. if (err)
  486. goto out_count;
  487. }
  488. INIT_LIST_HEAD(&context->db_page_list);
  489. mutex_init(&context->db_page_mutex);
  490. resp.tot_uuars = req.total_num_uuars;
  491. resp.num_ports = gen->num_ports;
  492. err = ib_copy_to_udata(udata, &resp,
  493. sizeof(resp) - sizeof(resp.reserved));
  494. if (err)
  495. goto out_uars;
  496. uuari->ver = ver;
  497. uuari->num_low_latency_uuars = req.num_low_latency_uuars;
  498. uuari->uars = uars;
  499. uuari->num_uars = num_uars;
  500. return &context->ibucontext;
  501. out_uars:
  502. for (i--; i >= 0; i--)
  503. mlx5_cmd_free_uar(dev->mdev, uars[i].index);
  504. out_count:
  505. kfree(uuari->count);
  506. out_bitmap:
  507. kfree(uuari->bitmap);
  508. out_uar_ctx:
  509. kfree(uars);
  510. out_ctx:
  511. kfree(context);
  512. return ERR_PTR(err);
  513. }
  514. static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
  515. {
  516. struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
  517. struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
  518. struct mlx5_uuar_info *uuari = &context->uuari;
  519. int i;
  520. for (i = 0; i < uuari->num_uars; i++) {
  521. if (mlx5_cmd_free_uar(dev->mdev, uuari->uars[i].index))
  522. mlx5_ib_warn(dev, "failed to free UAR 0x%x\n", uuari->uars[i].index);
  523. }
  524. kfree(uuari->count);
  525. kfree(uuari->bitmap);
  526. kfree(uuari->uars);
  527. kfree(context);
  528. return 0;
  529. }
  530. static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev, int index)
  531. {
  532. return (pci_resource_start(dev->mdev->pdev, 0) >> PAGE_SHIFT) + index;
  533. }
  534. static int get_command(unsigned long offset)
  535. {
  536. return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
  537. }
  538. static int get_arg(unsigned long offset)
  539. {
  540. return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1);
  541. }
  542. static int get_index(unsigned long offset)
  543. {
  544. return get_arg(offset);
  545. }
  546. static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
  547. {
  548. struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
  549. struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
  550. struct mlx5_uuar_info *uuari = &context->uuari;
  551. unsigned long command;
  552. unsigned long idx;
  553. phys_addr_t pfn;
  554. command = get_command(vma->vm_pgoff);
  555. switch (command) {
  556. case MLX5_IB_MMAP_REGULAR_PAGE:
  557. if (vma->vm_end - vma->vm_start != PAGE_SIZE)
  558. return -EINVAL;
  559. idx = get_index(vma->vm_pgoff);
  560. if (idx >= uuari->num_uars)
  561. return -EINVAL;
  562. pfn = uar_index2pfn(dev, uuari->uars[idx].index);
  563. mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn 0x%llx\n", idx,
  564. (unsigned long long)pfn);
  565. vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
  566. if (io_remap_pfn_range(vma, vma->vm_start, pfn,
  567. PAGE_SIZE, vma->vm_page_prot))
  568. return -EAGAIN;
  569. mlx5_ib_dbg(dev, "mapped WC at 0x%lx, PA 0x%llx\n",
  570. vma->vm_start,
  571. (unsigned long long)pfn << PAGE_SHIFT);
  572. break;
  573. case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
  574. return -ENOSYS;
  575. default:
  576. return -EINVAL;
  577. }
  578. return 0;
  579. }
  580. static int alloc_pa_mkey(struct mlx5_ib_dev *dev, u32 *key, u32 pdn)
  581. {
  582. struct mlx5_create_mkey_mbox_in *in;
  583. struct mlx5_mkey_seg *seg;
  584. struct mlx5_core_mr mr;
  585. int err;
  586. in = kzalloc(sizeof(*in), GFP_KERNEL);
  587. if (!in)
  588. return -ENOMEM;
  589. seg = &in->seg;
  590. seg->flags = MLX5_PERM_LOCAL_READ | MLX5_ACCESS_MODE_PA;
  591. seg->flags_pd = cpu_to_be32(pdn | MLX5_MKEY_LEN64);
  592. seg->qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
  593. seg->start_addr = 0;
  594. err = mlx5_core_create_mkey(dev->mdev, &mr, in, sizeof(*in),
  595. NULL, NULL, NULL);
  596. if (err) {
  597. mlx5_ib_warn(dev, "failed to create mkey, %d\n", err);
  598. goto err_in;
  599. }
  600. kfree(in);
  601. *key = mr.key;
  602. return 0;
  603. err_in:
  604. kfree(in);
  605. return err;
  606. }
  607. static void free_pa_mkey(struct mlx5_ib_dev *dev, u32 key)
  608. {
  609. struct mlx5_core_mr mr;
  610. int err;
  611. memset(&mr, 0, sizeof(mr));
  612. mr.key = key;
  613. err = mlx5_core_destroy_mkey(dev->mdev, &mr);
  614. if (err)
  615. mlx5_ib_warn(dev, "failed to destroy mkey 0x%x\n", key);
  616. }
  617. static struct ib_pd *mlx5_ib_alloc_pd(struct ib_device *ibdev,
  618. struct ib_ucontext *context,
  619. struct ib_udata *udata)
  620. {
  621. struct mlx5_ib_alloc_pd_resp resp;
  622. struct mlx5_ib_pd *pd;
  623. int err;
  624. pd = kmalloc(sizeof(*pd), GFP_KERNEL);
  625. if (!pd)
  626. return ERR_PTR(-ENOMEM);
  627. err = mlx5_core_alloc_pd(to_mdev(ibdev)->mdev, &pd->pdn);
  628. if (err) {
  629. kfree(pd);
  630. return ERR_PTR(err);
  631. }
  632. if (context) {
  633. resp.pdn = pd->pdn;
  634. if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
  635. mlx5_core_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn);
  636. kfree(pd);
  637. return ERR_PTR(-EFAULT);
  638. }
  639. } else {
  640. err = alloc_pa_mkey(to_mdev(ibdev), &pd->pa_lkey, pd->pdn);
  641. if (err) {
  642. mlx5_core_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn);
  643. kfree(pd);
  644. return ERR_PTR(err);
  645. }
  646. }
  647. return &pd->ibpd;
  648. }
  649. static int mlx5_ib_dealloc_pd(struct ib_pd *pd)
  650. {
  651. struct mlx5_ib_dev *mdev = to_mdev(pd->device);
  652. struct mlx5_ib_pd *mpd = to_mpd(pd);
  653. if (!pd->uobject)
  654. free_pa_mkey(mdev, mpd->pa_lkey);
  655. mlx5_core_dealloc_pd(mdev->mdev, mpd->pdn);
  656. kfree(mpd);
  657. return 0;
  658. }
  659. static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
  660. {
  661. struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
  662. int err;
  663. err = mlx5_core_attach_mcg(dev->mdev, gid, ibqp->qp_num);
  664. if (err)
  665. mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n",
  666. ibqp->qp_num, gid->raw);
  667. return err;
  668. }
  669. static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
  670. {
  671. struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
  672. int err;
  673. err = mlx5_core_detach_mcg(dev->mdev, gid, ibqp->qp_num);
  674. if (err)
  675. mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n",
  676. ibqp->qp_num, gid->raw);
  677. return err;
  678. }
  679. static int init_node_data(struct mlx5_ib_dev *dev)
  680. {
  681. struct ib_smp *in_mad = NULL;
  682. struct ib_smp *out_mad = NULL;
  683. int err = -ENOMEM;
  684. in_mad = kzalloc(sizeof(*in_mad), GFP_KERNEL);
  685. out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL);
  686. if (!in_mad || !out_mad)
  687. goto out;
  688. init_query_mad(in_mad);
  689. in_mad->attr_id = IB_SMP_ATTR_NODE_DESC;
  690. err = mlx5_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad);
  691. if (err)
  692. goto out;
  693. memcpy(dev->ib_dev.node_desc, out_mad->data, 64);
  694. in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
  695. err = mlx5_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad);
  696. if (err)
  697. goto out;
  698. dev->mdev->rev_id = be32_to_cpup((__be32 *)(out_mad->data + 32));
  699. memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8);
  700. out:
  701. kfree(in_mad);
  702. kfree(out_mad);
  703. return err;
  704. }
  705. static ssize_t show_fw_pages(struct device *device, struct device_attribute *attr,
  706. char *buf)
  707. {
  708. struct mlx5_ib_dev *dev =
  709. container_of(device, struct mlx5_ib_dev, ib_dev.dev);
  710. return sprintf(buf, "%d\n", dev->mdev->priv.fw_pages);
  711. }
  712. static ssize_t show_reg_pages(struct device *device,
  713. struct device_attribute *attr, char *buf)
  714. {
  715. struct mlx5_ib_dev *dev =
  716. container_of(device, struct mlx5_ib_dev, ib_dev.dev);
  717. return sprintf(buf, "%d\n", dev->mdev->priv.reg_pages);
  718. }
  719. static ssize_t show_hca(struct device *device, struct device_attribute *attr,
  720. char *buf)
  721. {
  722. struct mlx5_ib_dev *dev =
  723. container_of(device, struct mlx5_ib_dev, ib_dev.dev);
  724. return sprintf(buf, "MT%d\n", dev->mdev->pdev->device);
  725. }
  726. static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr,
  727. char *buf)
  728. {
  729. struct mlx5_ib_dev *dev =
  730. container_of(device, struct mlx5_ib_dev, ib_dev.dev);
  731. return sprintf(buf, "%d.%d.%d\n", fw_rev_maj(dev->mdev),
  732. fw_rev_min(dev->mdev), fw_rev_sub(dev->mdev));
  733. }
  734. static ssize_t show_rev(struct device *device, struct device_attribute *attr,
  735. char *buf)
  736. {
  737. struct mlx5_ib_dev *dev =
  738. container_of(device, struct mlx5_ib_dev, ib_dev.dev);
  739. return sprintf(buf, "%x\n", dev->mdev->rev_id);
  740. }
  741. static ssize_t show_board(struct device *device, struct device_attribute *attr,
  742. char *buf)
  743. {
  744. struct mlx5_ib_dev *dev =
  745. container_of(device, struct mlx5_ib_dev, ib_dev.dev);
  746. return sprintf(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
  747. dev->mdev->board_id);
  748. }
  749. static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
  750. static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL);
  751. static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL);
  752. static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL);
  753. static DEVICE_ATTR(fw_pages, S_IRUGO, show_fw_pages, NULL);
  754. static DEVICE_ATTR(reg_pages, S_IRUGO, show_reg_pages, NULL);
  755. static struct device_attribute *mlx5_class_attributes[] = {
  756. &dev_attr_hw_rev,
  757. &dev_attr_fw_ver,
  758. &dev_attr_hca_type,
  759. &dev_attr_board_id,
  760. &dev_attr_fw_pages,
  761. &dev_attr_reg_pages,
  762. };
  763. static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
  764. enum mlx5_dev_event event, unsigned long param)
  765. {
  766. struct mlx5_ib_dev *ibdev = (struct mlx5_ib_dev *)context;
  767. struct ib_event ibev;
  768. u8 port = 0;
  769. switch (event) {
  770. case MLX5_DEV_EVENT_SYS_ERROR:
  771. ibdev->ib_active = false;
  772. ibev.event = IB_EVENT_DEVICE_FATAL;
  773. break;
  774. case MLX5_DEV_EVENT_PORT_UP:
  775. ibev.event = IB_EVENT_PORT_ACTIVE;
  776. port = (u8)param;
  777. break;
  778. case MLX5_DEV_EVENT_PORT_DOWN:
  779. ibev.event = IB_EVENT_PORT_ERR;
  780. port = (u8)param;
  781. break;
  782. case MLX5_DEV_EVENT_PORT_INITIALIZED:
  783. /* not used by ULPs */
  784. return;
  785. case MLX5_DEV_EVENT_LID_CHANGE:
  786. ibev.event = IB_EVENT_LID_CHANGE;
  787. port = (u8)param;
  788. break;
  789. case MLX5_DEV_EVENT_PKEY_CHANGE:
  790. ibev.event = IB_EVENT_PKEY_CHANGE;
  791. port = (u8)param;
  792. break;
  793. case MLX5_DEV_EVENT_GUID_CHANGE:
  794. ibev.event = IB_EVENT_GID_CHANGE;
  795. port = (u8)param;
  796. break;
  797. case MLX5_DEV_EVENT_CLIENT_REREG:
  798. ibev.event = IB_EVENT_CLIENT_REREGISTER;
  799. port = (u8)param;
  800. break;
  801. }
  802. ibev.device = &ibdev->ib_dev;
  803. ibev.element.port_num = port;
  804. if (port < 1 || port > ibdev->num_ports) {
  805. mlx5_ib_warn(ibdev, "warning: event on port %d\n", port);
  806. return;
  807. }
  808. if (ibdev->ib_active)
  809. ib_dispatch_event(&ibev);
  810. }
  811. static void get_ext_port_caps(struct mlx5_ib_dev *dev)
  812. {
  813. struct mlx5_general_caps *gen;
  814. int port;
  815. gen = &dev->mdev->caps.gen;
  816. for (port = 1; port <= gen->num_ports; port++)
  817. mlx5_query_ext_port_caps(dev, port);
  818. }
  819. static int get_port_caps(struct mlx5_ib_dev *dev)
  820. {
  821. struct ib_device_attr *dprops = NULL;
  822. struct ib_port_attr *pprops = NULL;
  823. struct mlx5_general_caps *gen;
  824. int err = 0;
  825. int port;
  826. gen = &dev->mdev->caps.gen;
  827. pprops = kmalloc(sizeof(*pprops), GFP_KERNEL);
  828. if (!pprops)
  829. goto out;
  830. dprops = kmalloc(sizeof(*dprops), GFP_KERNEL);
  831. if (!dprops)
  832. goto out;
  833. err = mlx5_ib_query_device(&dev->ib_dev, dprops);
  834. if (err) {
  835. mlx5_ib_warn(dev, "query_device failed %d\n", err);
  836. goto out;
  837. }
  838. for (port = 1; port <= gen->num_ports; port++) {
  839. err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
  840. if (err) {
  841. mlx5_ib_warn(dev, "query_port %d failed %d\n", port, err);
  842. break;
  843. }
  844. gen->port[port - 1].pkey_table_len = dprops->max_pkeys;
  845. gen->port[port - 1].gid_table_len = pprops->gid_tbl_len;
  846. mlx5_ib_dbg(dev, "pkey_table_len %d, gid_table_len %d\n",
  847. dprops->max_pkeys, pprops->gid_tbl_len);
  848. }
  849. out:
  850. kfree(pprops);
  851. kfree(dprops);
  852. return err;
  853. }
  854. static void destroy_umrc_res(struct mlx5_ib_dev *dev)
  855. {
  856. int err;
  857. err = mlx5_mr_cache_cleanup(dev);
  858. if (err)
  859. mlx5_ib_warn(dev, "mr cache cleanup failed\n");
  860. mlx5_ib_destroy_qp(dev->umrc.qp);
  861. ib_destroy_cq(dev->umrc.cq);
  862. ib_dereg_mr(dev->umrc.mr);
  863. ib_dealloc_pd(dev->umrc.pd);
  864. }
  865. enum {
  866. MAX_UMR_WR = 128,
  867. };
  868. static int create_umr_res(struct mlx5_ib_dev *dev)
  869. {
  870. struct ib_qp_init_attr *init_attr = NULL;
  871. struct ib_qp_attr *attr = NULL;
  872. struct ib_pd *pd;
  873. struct ib_cq *cq;
  874. struct ib_qp *qp;
  875. struct ib_mr *mr;
  876. int ret;
  877. attr = kzalloc(sizeof(*attr), GFP_KERNEL);
  878. init_attr = kzalloc(sizeof(*init_attr), GFP_KERNEL);
  879. if (!attr || !init_attr) {
  880. ret = -ENOMEM;
  881. goto error_0;
  882. }
  883. pd = ib_alloc_pd(&dev->ib_dev);
  884. if (IS_ERR(pd)) {
  885. mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n");
  886. ret = PTR_ERR(pd);
  887. goto error_0;
  888. }
  889. mr = ib_get_dma_mr(pd, IB_ACCESS_LOCAL_WRITE);
  890. if (IS_ERR(mr)) {
  891. mlx5_ib_dbg(dev, "Couldn't create DMA MR for sync UMR QP\n");
  892. ret = PTR_ERR(mr);
  893. goto error_1;
  894. }
  895. cq = ib_create_cq(&dev->ib_dev, mlx5_umr_cq_handler, NULL, NULL, 128,
  896. 0);
  897. if (IS_ERR(cq)) {
  898. mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n");
  899. ret = PTR_ERR(cq);
  900. goto error_2;
  901. }
  902. ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
  903. init_attr->send_cq = cq;
  904. init_attr->recv_cq = cq;
  905. init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
  906. init_attr->cap.max_send_wr = MAX_UMR_WR;
  907. init_attr->cap.max_send_sge = 1;
  908. init_attr->qp_type = MLX5_IB_QPT_REG_UMR;
  909. init_attr->port_num = 1;
  910. qp = mlx5_ib_create_qp(pd, init_attr, NULL);
  911. if (IS_ERR(qp)) {
  912. mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n");
  913. ret = PTR_ERR(qp);
  914. goto error_3;
  915. }
  916. qp->device = &dev->ib_dev;
  917. qp->real_qp = qp;
  918. qp->uobject = NULL;
  919. qp->qp_type = MLX5_IB_QPT_REG_UMR;
  920. attr->qp_state = IB_QPS_INIT;
  921. attr->port_num = 1;
  922. ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_PKEY_INDEX |
  923. IB_QP_PORT, NULL);
  924. if (ret) {
  925. mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n");
  926. goto error_4;
  927. }
  928. memset(attr, 0, sizeof(*attr));
  929. attr->qp_state = IB_QPS_RTR;
  930. attr->path_mtu = IB_MTU_256;
  931. ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
  932. if (ret) {
  933. mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n");
  934. goto error_4;
  935. }
  936. memset(attr, 0, sizeof(*attr));
  937. attr->qp_state = IB_QPS_RTS;
  938. ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
  939. if (ret) {
  940. mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n");
  941. goto error_4;
  942. }
  943. dev->umrc.qp = qp;
  944. dev->umrc.cq = cq;
  945. dev->umrc.mr = mr;
  946. dev->umrc.pd = pd;
  947. sema_init(&dev->umrc.sem, MAX_UMR_WR);
  948. ret = mlx5_mr_cache_init(dev);
  949. if (ret) {
  950. mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
  951. goto error_4;
  952. }
  953. kfree(attr);
  954. kfree(init_attr);
  955. return 0;
  956. error_4:
  957. mlx5_ib_destroy_qp(qp);
  958. error_3:
  959. ib_destroy_cq(cq);
  960. error_2:
  961. ib_dereg_mr(mr);
  962. error_1:
  963. ib_dealloc_pd(pd);
  964. error_0:
  965. kfree(attr);
  966. kfree(init_attr);
  967. return ret;
  968. }
  969. static int create_dev_resources(struct mlx5_ib_resources *devr)
  970. {
  971. struct ib_srq_init_attr attr;
  972. struct mlx5_ib_dev *dev;
  973. int ret = 0;
  974. dev = container_of(devr, struct mlx5_ib_dev, devr);
  975. devr->p0 = mlx5_ib_alloc_pd(&dev->ib_dev, NULL, NULL);
  976. if (IS_ERR(devr->p0)) {
  977. ret = PTR_ERR(devr->p0);
  978. goto error0;
  979. }
  980. devr->p0->device = &dev->ib_dev;
  981. devr->p0->uobject = NULL;
  982. atomic_set(&devr->p0->usecnt, 0);
  983. devr->c0 = mlx5_ib_create_cq(&dev->ib_dev, 1, 0, NULL, NULL);
  984. if (IS_ERR(devr->c0)) {
  985. ret = PTR_ERR(devr->c0);
  986. goto error1;
  987. }
  988. devr->c0->device = &dev->ib_dev;
  989. devr->c0->uobject = NULL;
  990. devr->c0->comp_handler = NULL;
  991. devr->c0->event_handler = NULL;
  992. devr->c0->cq_context = NULL;
  993. atomic_set(&devr->c0->usecnt, 0);
  994. devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL);
  995. if (IS_ERR(devr->x0)) {
  996. ret = PTR_ERR(devr->x0);
  997. goto error2;
  998. }
  999. devr->x0->device = &dev->ib_dev;
  1000. devr->x0->inode = NULL;
  1001. atomic_set(&devr->x0->usecnt, 0);
  1002. mutex_init(&devr->x0->tgt_qp_mutex);
  1003. INIT_LIST_HEAD(&devr->x0->tgt_qp_list);
  1004. devr->x1 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL);
  1005. if (IS_ERR(devr->x1)) {
  1006. ret = PTR_ERR(devr->x1);
  1007. goto error3;
  1008. }
  1009. devr->x1->device = &dev->ib_dev;
  1010. devr->x1->inode = NULL;
  1011. atomic_set(&devr->x1->usecnt, 0);
  1012. mutex_init(&devr->x1->tgt_qp_mutex);
  1013. INIT_LIST_HEAD(&devr->x1->tgt_qp_list);
  1014. memset(&attr, 0, sizeof(attr));
  1015. attr.attr.max_sge = 1;
  1016. attr.attr.max_wr = 1;
  1017. attr.srq_type = IB_SRQT_XRC;
  1018. attr.ext.xrc.cq = devr->c0;
  1019. attr.ext.xrc.xrcd = devr->x0;
  1020. devr->s0 = mlx5_ib_create_srq(devr->p0, &attr, NULL);
  1021. if (IS_ERR(devr->s0)) {
  1022. ret = PTR_ERR(devr->s0);
  1023. goto error4;
  1024. }
  1025. devr->s0->device = &dev->ib_dev;
  1026. devr->s0->pd = devr->p0;
  1027. devr->s0->uobject = NULL;
  1028. devr->s0->event_handler = NULL;
  1029. devr->s0->srq_context = NULL;
  1030. devr->s0->srq_type = IB_SRQT_XRC;
  1031. devr->s0->ext.xrc.xrcd = devr->x0;
  1032. devr->s0->ext.xrc.cq = devr->c0;
  1033. atomic_inc(&devr->s0->ext.xrc.xrcd->usecnt);
  1034. atomic_inc(&devr->s0->ext.xrc.cq->usecnt);
  1035. atomic_inc(&devr->p0->usecnt);
  1036. atomic_set(&devr->s0->usecnt, 0);
  1037. return 0;
  1038. error4:
  1039. mlx5_ib_dealloc_xrcd(devr->x1);
  1040. error3:
  1041. mlx5_ib_dealloc_xrcd(devr->x0);
  1042. error2:
  1043. mlx5_ib_destroy_cq(devr->c0);
  1044. error1:
  1045. mlx5_ib_dealloc_pd(devr->p0);
  1046. error0:
  1047. return ret;
  1048. }
  1049. static void destroy_dev_resources(struct mlx5_ib_resources *devr)
  1050. {
  1051. mlx5_ib_destroy_srq(devr->s0);
  1052. mlx5_ib_dealloc_xrcd(devr->x0);
  1053. mlx5_ib_dealloc_xrcd(devr->x1);
  1054. mlx5_ib_destroy_cq(devr->c0);
  1055. mlx5_ib_dealloc_pd(devr->p0);
  1056. }
  1057. static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
  1058. {
  1059. struct mlx5_ib_dev *dev;
  1060. int err;
  1061. int i;
  1062. printk_once(KERN_INFO "%s", mlx5_version);
  1063. dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev));
  1064. if (!dev)
  1065. return NULL;
  1066. dev->mdev = mdev;
  1067. err = get_port_caps(dev);
  1068. if (err)
  1069. goto err_dealloc;
  1070. get_ext_port_caps(dev);
  1071. err = alloc_comp_eqs(dev);
  1072. if (err)
  1073. goto err_dealloc;
  1074. MLX5_INIT_DOORBELL_LOCK(&dev->uar_lock);
  1075. strlcpy(dev->ib_dev.name, "mlx5_%d", IB_DEVICE_NAME_MAX);
  1076. dev->ib_dev.owner = THIS_MODULE;
  1077. dev->ib_dev.node_type = RDMA_NODE_IB_CA;
  1078. dev->ib_dev.local_dma_lkey = mdev->caps.gen.reserved_lkey;
  1079. dev->num_ports = mdev->caps.gen.num_ports;
  1080. dev->ib_dev.phys_port_cnt = dev->num_ports;
  1081. dev->ib_dev.num_comp_vectors = dev->num_comp_vectors;
  1082. dev->ib_dev.dma_device = &mdev->pdev->dev;
  1083. dev->ib_dev.uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION;
  1084. dev->ib_dev.uverbs_cmd_mask =
  1085. (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
  1086. (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
  1087. (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
  1088. (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
  1089. (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
  1090. (1ull << IB_USER_VERBS_CMD_REG_MR) |
  1091. (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
  1092. (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
  1093. (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
  1094. (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) |
  1095. (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
  1096. (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
  1097. (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
  1098. (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
  1099. (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
  1100. (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) |
  1101. (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) |
  1102. (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
  1103. (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
  1104. (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
  1105. (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) |
  1106. (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ) |
  1107. (1ull << IB_USER_VERBS_CMD_OPEN_QP);
  1108. dev->ib_dev.query_device = mlx5_ib_query_device;
  1109. dev->ib_dev.query_port = mlx5_ib_query_port;
  1110. dev->ib_dev.query_gid = mlx5_ib_query_gid;
  1111. dev->ib_dev.query_pkey = mlx5_ib_query_pkey;
  1112. dev->ib_dev.modify_device = mlx5_ib_modify_device;
  1113. dev->ib_dev.modify_port = mlx5_ib_modify_port;
  1114. dev->ib_dev.alloc_ucontext = mlx5_ib_alloc_ucontext;
  1115. dev->ib_dev.dealloc_ucontext = mlx5_ib_dealloc_ucontext;
  1116. dev->ib_dev.mmap = mlx5_ib_mmap;
  1117. dev->ib_dev.alloc_pd = mlx5_ib_alloc_pd;
  1118. dev->ib_dev.dealloc_pd = mlx5_ib_dealloc_pd;
  1119. dev->ib_dev.create_ah = mlx5_ib_create_ah;
  1120. dev->ib_dev.query_ah = mlx5_ib_query_ah;
  1121. dev->ib_dev.destroy_ah = mlx5_ib_destroy_ah;
  1122. dev->ib_dev.create_srq = mlx5_ib_create_srq;
  1123. dev->ib_dev.modify_srq = mlx5_ib_modify_srq;
  1124. dev->ib_dev.query_srq = mlx5_ib_query_srq;
  1125. dev->ib_dev.destroy_srq = mlx5_ib_destroy_srq;
  1126. dev->ib_dev.post_srq_recv = mlx5_ib_post_srq_recv;
  1127. dev->ib_dev.create_qp = mlx5_ib_create_qp;
  1128. dev->ib_dev.modify_qp = mlx5_ib_modify_qp;
  1129. dev->ib_dev.query_qp = mlx5_ib_query_qp;
  1130. dev->ib_dev.destroy_qp = mlx5_ib_destroy_qp;
  1131. dev->ib_dev.post_send = mlx5_ib_post_send;
  1132. dev->ib_dev.post_recv = mlx5_ib_post_recv;
  1133. dev->ib_dev.create_cq = mlx5_ib_create_cq;
  1134. dev->ib_dev.modify_cq = mlx5_ib_modify_cq;
  1135. dev->ib_dev.resize_cq = mlx5_ib_resize_cq;
  1136. dev->ib_dev.destroy_cq = mlx5_ib_destroy_cq;
  1137. dev->ib_dev.poll_cq = mlx5_ib_poll_cq;
  1138. dev->ib_dev.req_notify_cq = mlx5_ib_arm_cq;
  1139. dev->ib_dev.get_dma_mr = mlx5_ib_get_dma_mr;
  1140. dev->ib_dev.reg_user_mr = mlx5_ib_reg_user_mr;
  1141. dev->ib_dev.dereg_mr = mlx5_ib_dereg_mr;
  1142. dev->ib_dev.destroy_mr = mlx5_ib_destroy_mr;
  1143. dev->ib_dev.attach_mcast = mlx5_ib_mcg_attach;
  1144. dev->ib_dev.detach_mcast = mlx5_ib_mcg_detach;
  1145. dev->ib_dev.process_mad = mlx5_ib_process_mad;
  1146. dev->ib_dev.create_mr = mlx5_ib_create_mr;
  1147. dev->ib_dev.alloc_fast_reg_mr = mlx5_ib_alloc_fast_reg_mr;
  1148. dev->ib_dev.alloc_fast_reg_page_list = mlx5_ib_alloc_fast_reg_page_list;
  1149. dev->ib_dev.free_fast_reg_page_list = mlx5_ib_free_fast_reg_page_list;
  1150. dev->ib_dev.check_mr_status = mlx5_ib_check_mr_status;
  1151. if (mdev->caps.gen.flags & MLX5_DEV_CAP_FLAG_XRC) {
  1152. dev->ib_dev.alloc_xrcd = mlx5_ib_alloc_xrcd;
  1153. dev->ib_dev.dealloc_xrcd = mlx5_ib_dealloc_xrcd;
  1154. dev->ib_dev.uverbs_cmd_mask |=
  1155. (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) |
  1156. (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
  1157. }
  1158. err = init_node_data(dev);
  1159. if (err)
  1160. goto err_eqs;
  1161. mutex_init(&dev->cap_mask_mutex);
  1162. spin_lock_init(&dev->mr_lock);
  1163. err = create_dev_resources(&dev->devr);
  1164. if (err)
  1165. goto err_eqs;
  1166. err = ib_register_device(&dev->ib_dev, NULL);
  1167. if (err)
  1168. goto err_rsrc;
  1169. err = create_umr_res(dev);
  1170. if (err)
  1171. goto err_dev;
  1172. for (i = 0; i < ARRAY_SIZE(mlx5_class_attributes); i++) {
  1173. err = device_create_file(&dev->ib_dev.dev,
  1174. mlx5_class_attributes[i]);
  1175. if (err)
  1176. goto err_umrc;
  1177. }
  1178. dev->ib_active = true;
  1179. return dev;
  1180. err_umrc:
  1181. destroy_umrc_res(dev);
  1182. err_dev:
  1183. ib_unregister_device(&dev->ib_dev);
  1184. err_rsrc:
  1185. destroy_dev_resources(&dev->devr);
  1186. err_eqs:
  1187. free_comp_eqs(dev);
  1188. err_dealloc:
  1189. ib_dealloc_device((struct ib_device *)dev);
  1190. return NULL;
  1191. }
  1192. static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
  1193. {
  1194. struct mlx5_ib_dev *dev = context;
  1195. ib_unregister_device(&dev->ib_dev);
  1196. destroy_umrc_res(dev);
  1197. destroy_dev_resources(&dev->devr);
  1198. free_comp_eqs(dev);
  1199. ib_dealloc_device(&dev->ib_dev);
  1200. }
  1201. static struct mlx5_interface mlx5_ib_interface = {
  1202. .add = mlx5_ib_add,
  1203. .remove = mlx5_ib_remove,
  1204. .event = mlx5_ib_event,
  1205. };
  1206. static int __init mlx5_ib_init(void)
  1207. {
  1208. if (deprecated_prof_sel != 2)
  1209. pr_warn("prof_sel is deprecated for mlx5_ib, set it for mlx5_core\n");
  1210. return mlx5_register_interface(&mlx5_ib_interface);
  1211. }
  1212. static void __exit mlx5_ib_cleanup(void)
  1213. {
  1214. mlx5_unregister_interface(&mlx5_ib_interface);
  1215. }
  1216. module_init(mlx5_ib_init);
  1217. module_exit(mlx5_ib_cleanup);