p2pdma.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * PCI Peer 2 Peer DMA support.
  4. *
  5. * Copyright (c) 2016-2018, Logan Gunthorpe
  6. * Copyright (c) 2016-2017, Microsemi Corporation
  7. * Copyright (c) 2017, Christoph Hellwig
  8. * Copyright (c) 2018, Eideticom Inc.
  9. */
  10. #define pr_fmt(fmt) "pci-p2pdma: " fmt
  11. #include <linux/ctype.h>
  12. #include <linux/pci-p2pdma.h>
  13. #include <linux/module.h>
  14. #include <linux/slab.h>
  15. #include <linux/genalloc.h>
  16. #include <linux/memremap.h>
  17. #include <linux/percpu-refcount.h>
  18. #include <linux/random.h>
  19. #include <linux/seq_buf.h>
  20. struct pci_p2pdma {
  21. struct percpu_ref devmap_ref;
  22. struct completion devmap_ref_done;
  23. struct gen_pool *pool;
  24. bool p2pmem_published;
  25. };
  26. static ssize_t size_show(struct device *dev, struct device_attribute *attr,
  27. char *buf)
  28. {
  29. struct pci_dev *pdev = to_pci_dev(dev);
  30. size_t size = 0;
  31. if (pdev->p2pdma->pool)
  32. size = gen_pool_size(pdev->p2pdma->pool);
  33. return snprintf(buf, PAGE_SIZE, "%zd\n", size);
  34. }
  35. static DEVICE_ATTR_RO(size);
  36. static ssize_t available_show(struct device *dev, struct device_attribute *attr,
  37. char *buf)
  38. {
  39. struct pci_dev *pdev = to_pci_dev(dev);
  40. size_t avail = 0;
  41. if (pdev->p2pdma->pool)
  42. avail = gen_pool_avail(pdev->p2pdma->pool);
  43. return snprintf(buf, PAGE_SIZE, "%zd\n", avail);
  44. }
  45. static DEVICE_ATTR_RO(available);
  46. static ssize_t published_show(struct device *dev, struct device_attribute *attr,
  47. char *buf)
  48. {
  49. struct pci_dev *pdev = to_pci_dev(dev);
  50. return snprintf(buf, PAGE_SIZE, "%d\n",
  51. pdev->p2pdma->p2pmem_published);
  52. }
  53. static DEVICE_ATTR_RO(published);
  54. static struct attribute *p2pmem_attrs[] = {
  55. &dev_attr_size.attr,
  56. &dev_attr_available.attr,
  57. &dev_attr_published.attr,
  58. NULL,
  59. };
  60. static const struct attribute_group p2pmem_group = {
  61. .attrs = p2pmem_attrs,
  62. .name = "p2pmem",
  63. };
  64. static void pci_p2pdma_percpu_release(struct percpu_ref *ref)
  65. {
  66. struct pci_p2pdma *p2p =
  67. container_of(ref, struct pci_p2pdma, devmap_ref);
  68. complete_all(&p2p->devmap_ref_done);
  69. }
  70. static void pci_p2pdma_percpu_kill(void *data)
  71. {
  72. struct percpu_ref *ref = data;
  73. /*
  74. * pci_p2pdma_add_resource() may be called multiple times
  75. * by a driver and may register the percpu_kill devm action multiple
  76. * times. We only want the first action to actually kill the
  77. * percpu_ref.
  78. */
  79. if (percpu_ref_is_dying(ref))
  80. return;
  81. percpu_ref_kill(ref);
  82. }
  83. static void pci_p2pdma_release(void *data)
  84. {
  85. struct pci_dev *pdev = data;
  86. if (!pdev->p2pdma)
  87. return;
  88. wait_for_completion(&pdev->p2pdma->devmap_ref_done);
  89. percpu_ref_exit(&pdev->p2pdma->devmap_ref);
  90. gen_pool_destroy(pdev->p2pdma->pool);
  91. sysfs_remove_group(&pdev->dev.kobj, &p2pmem_group);
  92. pdev->p2pdma = NULL;
  93. }
  94. static int pci_p2pdma_setup(struct pci_dev *pdev)
  95. {
  96. int error = -ENOMEM;
  97. struct pci_p2pdma *p2p;
  98. p2p = devm_kzalloc(&pdev->dev, sizeof(*p2p), GFP_KERNEL);
  99. if (!p2p)
  100. return -ENOMEM;
  101. p2p->pool = gen_pool_create(PAGE_SHIFT, dev_to_node(&pdev->dev));
  102. if (!p2p->pool)
  103. goto out;
  104. init_completion(&p2p->devmap_ref_done);
  105. error = percpu_ref_init(&p2p->devmap_ref,
  106. pci_p2pdma_percpu_release, 0, GFP_KERNEL);
  107. if (error)
  108. goto out_pool_destroy;
  109. error = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_release, pdev);
  110. if (error)
  111. goto out_pool_destroy;
  112. pdev->p2pdma = p2p;
  113. error = sysfs_create_group(&pdev->dev.kobj, &p2pmem_group);
  114. if (error)
  115. goto out_pool_destroy;
  116. return 0;
  117. out_pool_destroy:
  118. pdev->p2pdma = NULL;
  119. gen_pool_destroy(p2p->pool);
  120. out:
  121. devm_kfree(&pdev->dev, p2p);
  122. return error;
  123. }
  124. /**
  125. * pci_p2pdma_add_resource - add memory for use as p2p memory
  126. * @pdev: the device to add the memory to
  127. * @bar: PCI BAR to add
  128. * @size: size of the memory to add, may be zero to use the whole BAR
  129. * @offset: offset into the PCI BAR
  130. *
  131. * The memory will be given ZONE_DEVICE struct pages so that it may
  132. * be used with any DMA request.
  133. */
  134. int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size,
  135. u64 offset)
  136. {
  137. struct dev_pagemap *pgmap;
  138. void *addr;
  139. int error;
  140. if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM))
  141. return -EINVAL;
  142. if (offset >= pci_resource_len(pdev, bar))
  143. return -EINVAL;
  144. if (!size)
  145. size = pci_resource_len(pdev, bar) - offset;
  146. if (size + offset > pci_resource_len(pdev, bar))
  147. return -EINVAL;
  148. if (!pdev->p2pdma) {
  149. error = pci_p2pdma_setup(pdev);
  150. if (error)
  151. return error;
  152. }
  153. pgmap = devm_kzalloc(&pdev->dev, sizeof(*pgmap), GFP_KERNEL);
  154. if (!pgmap)
  155. return -ENOMEM;
  156. pgmap->res.start = pci_resource_start(pdev, bar) + offset;
  157. pgmap->res.end = pgmap->res.start + size - 1;
  158. pgmap->res.flags = pci_resource_flags(pdev, bar);
  159. pgmap->ref = &pdev->p2pdma->devmap_ref;
  160. pgmap->type = MEMORY_DEVICE_PCI_P2PDMA;
  161. pgmap->pci_p2pdma_bus_offset = pci_bus_address(pdev, bar) -
  162. pci_resource_start(pdev, bar);
  163. addr = devm_memremap_pages(&pdev->dev, pgmap);
  164. if (IS_ERR(addr)) {
  165. error = PTR_ERR(addr);
  166. goto pgmap_free;
  167. }
  168. error = gen_pool_add_virt(pdev->p2pdma->pool, (unsigned long)addr,
  169. pci_bus_address(pdev, bar) + offset,
  170. resource_size(&pgmap->res), dev_to_node(&pdev->dev));
  171. if (error)
  172. goto pgmap_free;
  173. error = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_percpu_kill,
  174. &pdev->p2pdma->devmap_ref);
  175. if (error)
  176. goto pgmap_free;
  177. pci_info(pdev, "added peer-to-peer DMA memory %pR\n",
  178. &pgmap->res);
  179. return 0;
  180. pgmap_free:
  181. devm_kfree(&pdev->dev, pgmap);
  182. return error;
  183. }
  184. EXPORT_SYMBOL_GPL(pci_p2pdma_add_resource);
  185. /*
  186. * Note this function returns the parent PCI device with a
  187. * reference taken. It is the caller's responsibily to drop
  188. * the reference.
  189. */
  190. static struct pci_dev *find_parent_pci_dev(struct device *dev)
  191. {
  192. struct device *parent;
  193. dev = get_device(dev);
  194. while (dev) {
  195. if (dev_is_pci(dev))
  196. return to_pci_dev(dev);
  197. parent = get_device(dev->parent);
  198. put_device(dev);
  199. dev = parent;
  200. }
  201. return NULL;
  202. }
  203. /*
  204. * Check if a PCI bridge has its ACS redirection bits set to redirect P2P
  205. * TLPs upstream via ACS. Returns 1 if the packets will be redirected
  206. * upstream, 0 otherwise.
  207. */
  208. static int pci_bridge_has_acs_redir(struct pci_dev *pdev)
  209. {
  210. int pos;
  211. u16 ctrl;
  212. pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ACS);
  213. if (!pos)
  214. return 0;
  215. pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &ctrl);
  216. if (ctrl & (PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC))
  217. return 1;
  218. return 0;
  219. }
  220. static void seq_buf_print_bus_devfn(struct seq_buf *buf, struct pci_dev *pdev)
  221. {
  222. if (!buf)
  223. return;
  224. seq_buf_printf(buf, "%s;", pci_name(pdev));
  225. }
  226. /*
  227. * Find the distance through the nearest common upstream bridge between
  228. * two PCI devices.
  229. *
  230. * If the two devices are the same device then 0 will be returned.
  231. *
  232. * If there are two virtual functions of the same device behind the same
  233. * bridge port then 2 will be returned (one step down to the PCIe switch,
  234. * then one step back to the same device).
  235. *
  236. * In the case where two devices are connected to the same PCIe switch, the
  237. * value 4 will be returned. This corresponds to the following PCI tree:
  238. *
  239. * -+ Root Port
  240. * \+ Switch Upstream Port
  241. * +-+ Switch Downstream Port
  242. * + \- Device A
  243. * \-+ Switch Downstream Port
  244. * \- Device B
  245. *
  246. * The distance is 4 because we traverse from Device A through the downstream
  247. * port of the switch, to the common upstream port, back up to the second
  248. * downstream port and then to Device B.
  249. *
  250. * Any two devices that don't have a common upstream bridge will return -1.
  251. * In this way devices on separate PCIe root ports will be rejected, which
  252. * is what we want for peer-to-peer seeing each PCIe root port defines a
  253. * separate hierarchy domain and there's no way to determine whether the root
  254. * complex supports forwarding between them.
  255. *
  256. * In the case where two devices are connected to different PCIe switches,
  257. * this function will still return a positive distance as long as both
  258. * switches eventually have a common upstream bridge. Note this covers
  259. * the case of using multiple PCIe switches to achieve a desired level of
  260. * fan-out from a root port. The exact distance will be a function of the
  261. * number of switches between Device A and Device B.
  262. *
  263. * If a bridge which has any ACS redirection bits set is in the path
  264. * then this functions will return -2. This is so we reject any
  265. * cases where the TLPs are forwarded up into the root complex.
  266. * In this case, a list of all infringing bridge addresses will be
  267. * populated in acs_list (assuming it's non-null) for printk purposes.
  268. */
  269. static int upstream_bridge_distance(struct pci_dev *a,
  270. struct pci_dev *b,
  271. struct seq_buf *acs_list)
  272. {
  273. int dist_a = 0;
  274. int dist_b = 0;
  275. struct pci_dev *bb = NULL;
  276. int acs_cnt = 0;
  277. /*
  278. * Note, we don't need to take references to devices returned by
  279. * pci_upstream_bridge() seeing we hold a reference to a child
  280. * device which will already hold a reference to the upstream bridge.
  281. */
  282. while (a) {
  283. dist_b = 0;
  284. if (pci_bridge_has_acs_redir(a)) {
  285. seq_buf_print_bus_devfn(acs_list, a);
  286. acs_cnt++;
  287. }
  288. bb = b;
  289. while (bb) {
  290. if (a == bb)
  291. goto check_b_path_acs;
  292. bb = pci_upstream_bridge(bb);
  293. dist_b++;
  294. }
  295. a = pci_upstream_bridge(a);
  296. dist_a++;
  297. }
  298. return -1;
  299. check_b_path_acs:
  300. bb = b;
  301. while (bb) {
  302. if (a == bb)
  303. break;
  304. if (pci_bridge_has_acs_redir(bb)) {
  305. seq_buf_print_bus_devfn(acs_list, bb);
  306. acs_cnt++;
  307. }
  308. bb = pci_upstream_bridge(bb);
  309. }
  310. if (acs_cnt)
  311. return -2;
  312. return dist_a + dist_b;
  313. }
  314. static int upstream_bridge_distance_warn(struct pci_dev *provider,
  315. struct pci_dev *client)
  316. {
  317. struct seq_buf acs_list;
  318. int ret;
  319. seq_buf_init(&acs_list, kmalloc(PAGE_SIZE, GFP_KERNEL), PAGE_SIZE);
  320. if (!acs_list.buffer)
  321. return -ENOMEM;
  322. ret = upstream_bridge_distance(provider, client, &acs_list);
  323. if (ret == -2) {
  324. pci_warn(client, "cannot be used for peer-to-peer DMA as ACS redirect is set between the client and provider (%s)\n",
  325. pci_name(provider));
  326. /* Drop final semicolon */
  327. acs_list.buffer[acs_list.len-1] = 0;
  328. pci_warn(client, "to disable ACS redirect for this path, add the kernel parameter: pci=disable_acs_redir=%s\n",
  329. acs_list.buffer);
  330. } else if (ret < 0) {
  331. pci_warn(client, "cannot be used for peer-to-peer DMA as the client and provider (%s) do not share an upstream bridge\n",
  332. pci_name(provider));
  333. }
  334. kfree(acs_list.buffer);
  335. return ret;
  336. }
  337. /**
  338. * pci_p2pdma_distance_many - Determive the cumulative distance between
  339. * a p2pdma provider and the clients in use.
  340. * @provider: p2pdma provider to check against the client list
  341. * @clients: array of devices to check (NULL-terminated)
  342. * @num_clients: number of clients in the array
  343. * @verbose: if true, print warnings for devices when we return -1
  344. *
  345. * Returns -1 if any of the clients are not compatible (behind the same
  346. * root port as the provider), otherwise returns a positive number where
  347. * a lower number is the preferrable choice. (If there's one client
  348. * that's the same as the provider it will return 0, which is best choice).
  349. *
  350. * For now, "compatible" means the provider and the clients are all behind
  351. * the same PCI root port. This cuts out cases that may work but is safest
  352. * for the user. Future work can expand this to white-list root complexes that
  353. * can safely forward between each ports.
  354. */
  355. int pci_p2pdma_distance_many(struct pci_dev *provider, struct device **clients,
  356. int num_clients, bool verbose)
  357. {
  358. bool not_supported = false;
  359. struct pci_dev *pci_client;
  360. int distance = 0;
  361. int i, ret;
  362. if (num_clients == 0)
  363. return -1;
  364. for (i = 0; i < num_clients; i++) {
  365. pci_client = find_parent_pci_dev(clients[i]);
  366. if (!pci_client) {
  367. if (verbose)
  368. dev_warn(clients[i],
  369. "cannot be used for peer-to-peer DMA as it is not a PCI device\n");
  370. return -1;
  371. }
  372. if (verbose)
  373. ret = upstream_bridge_distance_warn(provider,
  374. pci_client);
  375. else
  376. ret = upstream_bridge_distance(provider, pci_client,
  377. NULL);
  378. pci_dev_put(pci_client);
  379. if (ret < 0)
  380. not_supported = true;
  381. if (not_supported && !verbose)
  382. break;
  383. distance += ret;
  384. }
  385. if (not_supported)
  386. return -1;
  387. return distance;
  388. }
  389. EXPORT_SYMBOL_GPL(pci_p2pdma_distance_many);
  390. /**
  391. * pci_has_p2pmem - check if a given PCI device has published any p2pmem
  392. * @pdev: PCI device to check
  393. */
  394. bool pci_has_p2pmem(struct pci_dev *pdev)
  395. {
  396. return pdev->p2pdma && pdev->p2pdma->p2pmem_published;
  397. }
  398. EXPORT_SYMBOL_GPL(pci_has_p2pmem);
  399. /**
  400. * pci_p2pmem_find - find a peer-to-peer DMA memory device compatible with
  401. * the specified list of clients and shortest distance (as determined
  402. * by pci_p2pmem_dma())
  403. * @clients: array of devices to check (NULL-terminated)
  404. * @num_clients: number of client devices in the list
  405. *
  406. * If multiple devices are behind the same switch, the one "closest" to the
  407. * client devices in use will be chosen first. (So if one of the providers are
  408. * the same as one of the clients, that provider will be used ahead of any
  409. * other providers that are unrelated). If multiple providers are an equal
  410. * distance away, one will be chosen at random.
  411. *
  412. * Returns a pointer to the PCI device with a reference taken (use pci_dev_put
  413. * to return the reference) or NULL if no compatible device is found. The
  414. * found provider will also be assigned to the client list.
  415. */
  416. struct pci_dev *pci_p2pmem_find_many(struct device **clients, int num_clients)
  417. {
  418. struct pci_dev *pdev = NULL;
  419. int distance;
  420. int closest_distance = INT_MAX;
  421. struct pci_dev **closest_pdevs;
  422. int dev_cnt = 0;
  423. const int max_devs = PAGE_SIZE / sizeof(*closest_pdevs);
  424. int i;
  425. closest_pdevs = kmalloc(PAGE_SIZE, GFP_KERNEL);
  426. if (!closest_pdevs)
  427. return NULL;
  428. while ((pdev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, pdev))) {
  429. if (!pci_has_p2pmem(pdev))
  430. continue;
  431. distance = pci_p2pdma_distance_many(pdev, clients,
  432. num_clients, false);
  433. if (distance < 0 || distance > closest_distance)
  434. continue;
  435. if (distance == closest_distance && dev_cnt >= max_devs)
  436. continue;
  437. if (distance < closest_distance) {
  438. for (i = 0; i < dev_cnt; i++)
  439. pci_dev_put(closest_pdevs[i]);
  440. dev_cnt = 0;
  441. closest_distance = distance;
  442. }
  443. closest_pdevs[dev_cnt++] = pci_dev_get(pdev);
  444. }
  445. if (dev_cnt)
  446. pdev = pci_dev_get(closest_pdevs[prandom_u32_max(dev_cnt)]);
  447. for (i = 0; i < dev_cnt; i++)
  448. pci_dev_put(closest_pdevs[i]);
  449. kfree(closest_pdevs);
  450. return pdev;
  451. }
  452. EXPORT_SYMBOL_GPL(pci_p2pmem_find_many);
  453. /**
  454. * pci_alloc_p2p_mem - allocate peer-to-peer DMA memory
  455. * @pdev: the device to allocate memory from
  456. * @size: number of bytes to allocate
  457. *
  458. * Returns the allocated memory or NULL on error.
  459. */
  460. void *pci_alloc_p2pmem(struct pci_dev *pdev, size_t size)
  461. {
  462. void *ret;
  463. if (unlikely(!pdev->p2pdma))
  464. return NULL;
  465. if (unlikely(!percpu_ref_tryget_live(&pdev->p2pdma->devmap_ref)))
  466. return NULL;
  467. ret = (void *)gen_pool_alloc(pdev->p2pdma->pool, size);
  468. if (unlikely(!ret))
  469. percpu_ref_put(&pdev->p2pdma->devmap_ref);
  470. return ret;
  471. }
  472. EXPORT_SYMBOL_GPL(pci_alloc_p2pmem);
  473. /**
  474. * pci_free_p2pmem - free peer-to-peer DMA memory
  475. * @pdev: the device the memory was allocated from
  476. * @addr: address of the memory that was allocated
  477. * @size: number of bytes that was allocated
  478. */
  479. void pci_free_p2pmem(struct pci_dev *pdev, void *addr, size_t size)
  480. {
  481. gen_pool_free(pdev->p2pdma->pool, (uintptr_t)addr, size);
  482. percpu_ref_put(&pdev->p2pdma->devmap_ref);
  483. }
  484. EXPORT_SYMBOL_GPL(pci_free_p2pmem);
  485. /**
  486. * pci_virt_to_bus - return the PCI bus address for a given virtual
  487. * address obtained with pci_alloc_p2pmem()
  488. * @pdev: the device the memory was allocated from
  489. * @addr: address of the memory that was allocated
  490. */
  491. pci_bus_addr_t pci_p2pmem_virt_to_bus(struct pci_dev *pdev, void *addr)
  492. {
  493. if (!addr)
  494. return 0;
  495. if (!pdev->p2pdma)
  496. return 0;
  497. /*
  498. * Note: when we added the memory to the pool we used the PCI
  499. * bus address as the physical address. So gen_pool_virt_to_phys()
  500. * actually returns the bus address despite the misleading name.
  501. */
  502. return gen_pool_virt_to_phys(pdev->p2pdma->pool, (unsigned long)addr);
  503. }
  504. EXPORT_SYMBOL_GPL(pci_p2pmem_virt_to_bus);
  505. /**
  506. * pci_p2pmem_alloc_sgl - allocate peer-to-peer DMA memory in a scatterlist
  507. * @pdev: the device to allocate memory from
  508. * @nents: the number of SG entries in the list
  509. * @length: number of bytes to allocate
  510. *
  511. * Returns 0 on success
  512. */
  513. struct scatterlist *pci_p2pmem_alloc_sgl(struct pci_dev *pdev,
  514. unsigned int *nents, u32 length)
  515. {
  516. struct scatterlist *sg;
  517. void *addr;
  518. sg = kzalloc(sizeof(*sg), GFP_KERNEL);
  519. if (!sg)
  520. return NULL;
  521. sg_init_table(sg, 1);
  522. addr = pci_alloc_p2pmem(pdev, length);
  523. if (!addr)
  524. goto out_free_sg;
  525. sg_set_buf(sg, addr, length);
  526. *nents = 1;
  527. return sg;
  528. out_free_sg:
  529. kfree(sg);
  530. return NULL;
  531. }
  532. EXPORT_SYMBOL_GPL(pci_p2pmem_alloc_sgl);
  533. /**
  534. * pci_p2pmem_free_sgl - free a scatterlist allocated by pci_p2pmem_alloc_sgl()
  535. * @pdev: the device to allocate memory from
  536. * @sgl: the allocated scatterlist
  537. */
  538. void pci_p2pmem_free_sgl(struct pci_dev *pdev, struct scatterlist *sgl)
  539. {
  540. struct scatterlist *sg;
  541. int count;
  542. for_each_sg(sgl, sg, INT_MAX, count) {
  543. if (!sg)
  544. break;
  545. pci_free_p2pmem(pdev, sg_virt(sg), sg->length);
  546. }
  547. kfree(sgl);
  548. }
  549. EXPORT_SYMBOL_GPL(pci_p2pmem_free_sgl);
  550. /**
  551. * pci_p2pmem_publish - publish the peer-to-peer DMA memory for use by
  552. * other devices with pci_p2pmem_find()
  553. * @pdev: the device with peer-to-peer DMA memory to publish
  554. * @publish: set to true to publish the memory, false to unpublish it
  555. *
  556. * Published memory can be used by other PCI device drivers for
  557. * peer-2-peer DMA operations. Non-published memory is reserved for
  558. * exlusive use of the device driver that registers the peer-to-peer
  559. * memory.
  560. */
  561. void pci_p2pmem_publish(struct pci_dev *pdev, bool publish)
  562. {
  563. if (pdev->p2pdma)
  564. pdev->p2pdma->p2pmem_published = publish;
  565. }
  566. EXPORT_SYMBOL_GPL(pci_p2pmem_publish);
  567. /**
  568. * pci_p2pdma_map_sg - map a PCI peer-to-peer scatterlist for DMA
  569. * @dev: device doing the DMA request
  570. * @sg: scatter list to map
  571. * @nents: elements in the scatterlist
  572. * @dir: DMA direction
  573. *
  574. * Scatterlists mapped with this function should not be unmapped in any way.
  575. *
  576. * Returns the number of SG entries mapped or 0 on error.
  577. */
  578. int pci_p2pdma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
  579. enum dma_data_direction dir)
  580. {
  581. struct dev_pagemap *pgmap;
  582. struct scatterlist *s;
  583. phys_addr_t paddr;
  584. int i;
  585. /*
  586. * p2pdma mappings are not compatible with devices that use
  587. * dma_virt_ops. If the upper layers do the right thing
  588. * this should never happen because it will be prevented
  589. * by the check in pci_p2pdma_add_client()
  590. */
  591. if (WARN_ON_ONCE(IS_ENABLED(CONFIG_DMA_VIRT_OPS) &&
  592. dev->dma_ops == &dma_virt_ops))
  593. return 0;
  594. for_each_sg(sg, s, nents, i) {
  595. pgmap = sg_page(s)->pgmap;
  596. paddr = sg_phys(s);
  597. s->dma_address = paddr - pgmap->pci_p2pdma_bus_offset;
  598. sg_dma_len(s) = s->length;
  599. }
  600. return nents;
  601. }
  602. EXPORT_SYMBOL_GPL(pci_p2pdma_map_sg);
  603. /**
  604. * pci_p2pdma_enable_store - parse a configfs/sysfs attribute store
  605. * to enable p2pdma
  606. * @page: contents of the value to be stored
  607. * @p2p_dev: returns the PCI device that was selected to be used
  608. * (if one was specified in the stored value)
  609. * @use_p2pdma: returns whether to enable p2pdma or not
  610. *
  611. * Parses an attribute value to decide whether to enable p2pdma.
  612. * The value can select a PCI device (using it's full BDF device
  613. * name) or a boolean (in any format strtobool() accepts). A false
  614. * value disables p2pdma, a true value expects the caller
  615. * to automatically find a compatible device and specifying a PCI device
  616. * expects the caller to use the specific provider.
  617. *
  618. * pci_p2pdma_enable_show() should be used as the show operation for
  619. * the attribute.
  620. *
  621. * Returns 0 on success
  622. */
  623. int pci_p2pdma_enable_store(const char *page, struct pci_dev **p2p_dev,
  624. bool *use_p2pdma)
  625. {
  626. struct device *dev;
  627. dev = bus_find_device_by_name(&pci_bus_type, NULL, page);
  628. if (dev) {
  629. *use_p2pdma = true;
  630. *p2p_dev = to_pci_dev(dev);
  631. if (!pci_has_p2pmem(*p2p_dev)) {
  632. pci_err(*p2p_dev,
  633. "PCI device has no peer-to-peer memory: %s\n",
  634. page);
  635. pci_dev_put(*p2p_dev);
  636. return -ENODEV;
  637. }
  638. return 0;
  639. } else if ((page[0] == '0' || page[0] == '1') && !iscntrl(page[1])) {
  640. /*
  641. * If the user enters a PCI device that doesn't exist
  642. * like "0000:01:00.1", we don't want strtobool to think
  643. * it's a '0' when it's clearly not what the user wanted.
  644. * So we require 0's and 1's to be exactly one character.
  645. */
  646. } else if (!strtobool(page, use_p2pdma)) {
  647. return 0;
  648. }
  649. pr_err("No such PCI device: %.*s\n", (int)strcspn(page, "\n"), page);
  650. return -ENODEV;
  651. }
  652. EXPORT_SYMBOL_GPL(pci_p2pdma_enable_store);
  653. /**
  654. * pci_p2pdma_enable_show - show a configfs/sysfs attribute indicating
  655. * whether p2pdma is enabled
  656. * @page: contents of the stored value
  657. * @p2p_dev: the selected p2p device (NULL if no device is selected)
  658. * @use_p2pdma: whether p2pdme has been enabled
  659. *
  660. * Attributes that use pci_p2pdma_enable_store() should use this function
  661. * to show the value of the attribute.
  662. *
  663. * Returns 0 on success
  664. */
  665. ssize_t pci_p2pdma_enable_show(char *page, struct pci_dev *p2p_dev,
  666. bool use_p2pdma)
  667. {
  668. if (!use_p2pdma)
  669. return sprintf(page, "0\n");
  670. if (!p2p_dev)
  671. return sprintf(page, "1\n");
  672. return sprintf(page, "%s\n", pci_name(p2p_dev));
  673. }
  674. EXPORT_SYMBOL_GPL(pci_p2pdma_enable_show);