file_ops.c 44 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709
  1. /*
  2. * Copyright(c) 2015-2017 Intel Corporation.
  3. *
  4. * This file is provided under a dual BSD/GPLv2 license. When using or
  5. * redistributing this file, you may do so under either license.
  6. *
  7. * GPL LICENSE SUMMARY
  8. *
  9. * This program is free software; you can redistribute it and/or modify
  10. * it under the terms of version 2 of the GNU General Public License as
  11. * published by the Free Software Foundation.
  12. *
  13. * This program is distributed in the hope that it will be useful, but
  14. * WITHOUT ANY WARRANTY; without even the implied warranty of
  15. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16. * General Public License for more details.
  17. *
  18. * BSD LICENSE
  19. *
  20. * Redistribution and use in source and binary forms, with or without
  21. * modification, are permitted provided that the following conditions
  22. * are met:
  23. *
  24. * - Redistributions of source code must retain the above copyright
  25. * notice, this list of conditions and the following disclaimer.
  26. * - Redistributions in binary form must reproduce the above copyright
  27. * notice, this list of conditions and the following disclaimer in
  28. * the documentation and/or other materials provided with the
  29. * distribution.
  30. * - Neither the name of Intel Corporation nor the names of its
  31. * contributors may be used to endorse or promote products derived
  32. * from this software without specific prior written permission.
  33. *
  34. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  35. * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  36. * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  37. * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  38. * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  39. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  40. * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  41. * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  42. * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  43. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  44. * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  45. *
  46. */
  47. #include <linux/poll.h>
  48. #include <linux/cdev.h>
  49. #include <linux/vmalloc.h>
  50. #include <linux/io.h>
  51. #include <linux/sched/mm.h>
  52. #include <linux/bitmap.h>
  53. #include <rdma/ib.h>
  54. #include "hfi.h"
  55. #include "pio.h"
  56. #include "device.h"
  57. #include "common.h"
  58. #include "trace.h"
  59. #include "mmu_rb.h"
  60. #include "user_sdma.h"
  61. #include "user_exp_rcv.h"
  62. #include "aspm.h"
  63. #undef pr_fmt
  64. #define pr_fmt(fmt) DRIVER_NAME ": " fmt
  65. #define SEND_CTXT_HALT_TIMEOUT 1000 /* msecs */
  66. /*
  67. * File operation functions
  68. */
  69. static int hfi1_file_open(struct inode *inode, struct file *fp);
  70. static int hfi1_file_close(struct inode *inode, struct file *fp);
  71. static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from);
  72. static __poll_t hfi1_poll(struct file *fp, struct poll_table_struct *pt);
  73. static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma);
  74. static u64 kvirt_to_phys(void *addr);
  75. static int assign_ctxt(struct hfi1_filedata *fd, unsigned long arg, u32 len);
  76. static void init_subctxts(struct hfi1_ctxtdata *uctxt,
  77. const struct hfi1_user_info *uinfo);
  78. static int init_user_ctxt(struct hfi1_filedata *fd,
  79. struct hfi1_ctxtdata *uctxt);
  80. static void user_init(struct hfi1_ctxtdata *uctxt);
  81. static int get_ctxt_info(struct hfi1_filedata *fd, unsigned long arg, u32 len);
  82. static int get_base_info(struct hfi1_filedata *fd, unsigned long arg, u32 len);
  83. static int user_exp_rcv_setup(struct hfi1_filedata *fd, unsigned long arg,
  84. u32 len);
  85. static int user_exp_rcv_clear(struct hfi1_filedata *fd, unsigned long arg,
  86. u32 len);
  87. static int user_exp_rcv_invalid(struct hfi1_filedata *fd, unsigned long arg,
  88. u32 len);
  89. static int setup_base_ctxt(struct hfi1_filedata *fd,
  90. struct hfi1_ctxtdata *uctxt);
  91. static int setup_subctxt(struct hfi1_ctxtdata *uctxt);
  92. static int find_sub_ctxt(struct hfi1_filedata *fd,
  93. const struct hfi1_user_info *uinfo);
  94. static int allocate_ctxt(struct hfi1_filedata *fd, struct hfi1_devdata *dd,
  95. struct hfi1_user_info *uinfo,
  96. struct hfi1_ctxtdata **cd);
  97. static void deallocate_ctxt(struct hfi1_ctxtdata *uctxt);
  98. static __poll_t poll_urgent(struct file *fp, struct poll_table_struct *pt);
  99. static __poll_t poll_next(struct file *fp, struct poll_table_struct *pt);
  100. static int user_event_ack(struct hfi1_ctxtdata *uctxt, u16 subctxt,
  101. unsigned long arg);
  102. static int set_ctxt_pkey(struct hfi1_ctxtdata *uctxt, unsigned long arg);
  103. static int ctxt_reset(struct hfi1_ctxtdata *uctxt);
  104. static int manage_rcvq(struct hfi1_ctxtdata *uctxt, u16 subctxt,
  105. unsigned long arg);
  106. static vm_fault_t vma_fault(struct vm_fault *vmf);
  107. static long hfi1_file_ioctl(struct file *fp, unsigned int cmd,
  108. unsigned long arg);
  109. static const struct file_operations hfi1_file_ops = {
  110. .owner = THIS_MODULE,
  111. .write_iter = hfi1_write_iter,
  112. .open = hfi1_file_open,
  113. .release = hfi1_file_close,
  114. .unlocked_ioctl = hfi1_file_ioctl,
  115. .poll = hfi1_poll,
  116. .mmap = hfi1_file_mmap,
  117. .llseek = noop_llseek,
  118. };
  119. static const struct vm_operations_struct vm_ops = {
  120. .fault = vma_fault,
  121. };
  122. /*
  123. * Types of memories mapped into user processes' space
  124. */
  125. enum mmap_types {
  126. PIO_BUFS = 1,
  127. PIO_BUFS_SOP,
  128. PIO_CRED,
  129. RCV_HDRQ,
  130. RCV_EGRBUF,
  131. UREGS,
  132. EVENTS,
  133. STATUS,
  134. RTAIL,
  135. SUBCTXT_UREGS,
  136. SUBCTXT_RCV_HDRQ,
  137. SUBCTXT_EGRBUF,
  138. SDMA_COMP
  139. };
  140. /*
  141. * Masks and offsets defining the mmap tokens
  142. */
  143. #define HFI1_MMAP_OFFSET_MASK 0xfffULL
  144. #define HFI1_MMAP_OFFSET_SHIFT 0
  145. #define HFI1_MMAP_SUBCTXT_MASK 0xfULL
  146. #define HFI1_MMAP_SUBCTXT_SHIFT 12
  147. #define HFI1_MMAP_CTXT_MASK 0xffULL
  148. #define HFI1_MMAP_CTXT_SHIFT 16
  149. #define HFI1_MMAP_TYPE_MASK 0xfULL
  150. #define HFI1_MMAP_TYPE_SHIFT 24
  151. #define HFI1_MMAP_MAGIC_MASK 0xffffffffULL
  152. #define HFI1_MMAP_MAGIC_SHIFT 32
  153. #define HFI1_MMAP_MAGIC 0xdabbad00
  154. #define HFI1_MMAP_TOKEN_SET(field, val) \
  155. (((val) & HFI1_MMAP_##field##_MASK) << HFI1_MMAP_##field##_SHIFT)
  156. #define HFI1_MMAP_TOKEN_GET(field, token) \
  157. (((token) >> HFI1_MMAP_##field##_SHIFT) & HFI1_MMAP_##field##_MASK)
  158. #define HFI1_MMAP_TOKEN(type, ctxt, subctxt, addr) \
  159. (HFI1_MMAP_TOKEN_SET(MAGIC, HFI1_MMAP_MAGIC) | \
  160. HFI1_MMAP_TOKEN_SET(TYPE, type) | \
  161. HFI1_MMAP_TOKEN_SET(CTXT, ctxt) | \
  162. HFI1_MMAP_TOKEN_SET(SUBCTXT, subctxt) | \
  163. HFI1_MMAP_TOKEN_SET(OFFSET, (offset_in_page(addr))))
  164. #define dbg(fmt, ...) \
  165. pr_info(fmt, ##__VA_ARGS__)
  166. static inline int is_valid_mmap(u64 token)
  167. {
  168. return (HFI1_MMAP_TOKEN_GET(MAGIC, token) == HFI1_MMAP_MAGIC);
  169. }
  170. static int hfi1_file_open(struct inode *inode, struct file *fp)
  171. {
  172. struct hfi1_filedata *fd;
  173. struct hfi1_devdata *dd = container_of(inode->i_cdev,
  174. struct hfi1_devdata,
  175. user_cdev);
  176. if (!((dd->flags & HFI1_PRESENT) && dd->kregbase1))
  177. return -EINVAL;
  178. if (!atomic_inc_not_zero(&dd->user_refcount))
  179. return -ENXIO;
  180. /* The real work is performed later in assign_ctxt() */
  181. fd = kzalloc(sizeof(*fd), GFP_KERNEL);
  182. if (fd) {
  183. fd->rec_cpu_num = -1; /* no cpu affinity by default */
  184. fd->mm = current->mm;
  185. mmgrab(fd->mm);
  186. fd->dd = dd;
  187. kobject_get(&fd->dd->kobj);
  188. fp->private_data = fd;
  189. } else {
  190. fp->private_data = NULL;
  191. if (atomic_dec_and_test(&dd->user_refcount))
  192. complete(&dd->user_comp);
  193. return -ENOMEM;
  194. }
  195. return 0;
  196. }
  197. static long hfi1_file_ioctl(struct file *fp, unsigned int cmd,
  198. unsigned long arg)
  199. {
  200. struct hfi1_filedata *fd = fp->private_data;
  201. struct hfi1_ctxtdata *uctxt = fd->uctxt;
  202. int ret = 0;
  203. int uval = 0;
  204. hfi1_cdbg(IOCTL, "IOCTL recv: 0x%x", cmd);
  205. if (cmd != HFI1_IOCTL_ASSIGN_CTXT &&
  206. cmd != HFI1_IOCTL_GET_VERS &&
  207. !uctxt)
  208. return -EINVAL;
  209. switch (cmd) {
  210. case HFI1_IOCTL_ASSIGN_CTXT:
  211. ret = assign_ctxt(fd, arg, _IOC_SIZE(cmd));
  212. break;
  213. case HFI1_IOCTL_CTXT_INFO:
  214. ret = get_ctxt_info(fd, arg, _IOC_SIZE(cmd));
  215. break;
  216. case HFI1_IOCTL_USER_INFO:
  217. ret = get_base_info(fd, arg, _IOC_SIZE(cmd));
  218. break;
  219. case HFI1_IOCTL_CREDIT_UPD:
  220. if (uctxt)
  221. sc_return_credits(uctxt->sc);
  222. break;
  223. case HFI1_IOCTL_TID_UPDATE:
  224. ret = user_exp_rcv_setup(fd, arg, _IOC_SIZE(cmd));
  225. break;
  226. case HFI1_IOCTL_TID_FREE:
  227. ret = user_exp_rcv_clear(fd, arg, _IOC_SIZE(cmd));
  228. break;
  229. case HFI1_IOCTL_TID_INVAL_READ:
  230. ret = user_exp_rcv_invalid(fd, arg, _IOC_SIZE(cmd));
  231. break;
  232. case HFI1_IOCTL_RECV_CTRL:
  233. ret = manage_rcvq(uctxt, fd->subctxt, arg);
  234. break;
  235. case HFI1_IOCTL_POLL_TYPE:
  236. if (get_user(uval, (int __user *)arg))
  237. return -EFAULT;
  238. uctxt->poll_type = (typeof(uctxt->poll_type))uval;
  239. break;
  240. case HFI1_IOCTL_ACK_EVENT:
  241. ret = user_event_ack(uctxt, fd->subctxt, arg);
  242. break;
  243. case HFI1_IOCTL_SET_PKEY:
  244. ret = set_ctxt_pkey(uctxt, arg);
  245. break;
  246. case HFI1_IOCTL_CTXT_RESET:
  247. ret = ctxt_reset(uctxt);
  248. break;
  249. case HFI1_IOCTL_GET_VERS:
  250. uval = HFI1_USER_SWVERSION;
  251. if (put_user(uval, (int __user *)arg))
  252. return -EFAULT;
  253. break;
  254. default:
  255. return -EINVAL;
  256. }
  257. return ret;
  258. }
  259. static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from)
  260. {
  261. struct hfi1_filedata *fd = kiocb->ki_filp->private_data;
  262. struct hfi1_user_sdma_pkt_q *pq = fd->pq;
  263. struct hfi1_user_sdma_comp_q *cq = fd->cq;
  264. int done = 0, reqs = 0;
  265. unsigned long dim = from->nr_segs;
  266. if (!cq || !pq)
  267. return -EIO;
  268. if (!iter_is_iovec(from) || !dim)
  269. return -EINVAL;
  270. trace_hfi1_sdma_request(fd->dd, fd->uctxt->ctxt, fd->subctxt, dim);
  271. if (atomic_read(&pq->n_reqs) == pq->n_max_reqs)
  272. return -ENOSPC;
  273. while (dim) {
  274. int ret;
  275. unsigned long count = 0;
  276. ret = hfi1_user_sdma_process_request(
  277. fd, (struct iovec *)(from->iov + done),
  278. dim, &count);
  279. if (ret) {
  280. reqs = ret;
  281. break;
  282. }
  283. dim -= count;
  284. done += count;
  285. reqs++;
  286. }
  287. return reqs;
  288. }
  289. static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
  290. {
  291. struct hfi1_filedata *fd = fp->private_data;
  292. struct hfi1_ctxtdata *uctxt = fd->uctxt;
  293. struct hfi1_devdata *dd;
  294. unsigned long flags;
  295. u64 token = vma->vm_pgoff << PAGE_SHIFT,
  296. memaddr = 0;
  297. void *memvirt = NULL;
  298. u8 subctxt, mapio = 0, vmf = 0, type;
  299. ssize_t memlen = 0;
  300. int ret = 0;
  301. u16 ctxt;
  302. if (!is_valid_mmap(token) || !uctxt ||
  303. !(vma->vm_flags & VM_SHARED)) {
  304. ret = -EINVAL;
  305. goto done;
  306. }
  307. dd = uctxt->dd;
  308. ctxt = HFI1_MMAP_TOKEN_GET(CTXT, token);
  309. subctxt = HFI1_MMAP_TOKEN_GET(SUBCTXT, token);
  310. type = HFI1_MMAP_TOKEN_GET(TYPE, token);
  311. if (ctxt != uctxt->ctxt || subctxt != fd->subctxt) {
  312. ret = -EINVAL;
  313. goto done;
  314. }
  315. flags = vma->vm_flags;
  316. switch (type) {
  317. case PIO_BUFS:
  318. case PIO_BUFS_SOP:
  319. memaddr = ((dd->physaddr + TXE_PIO_SEND) +
  320. /* chip pio base */
  321. (uctxt->sc->hw_context * BIT(16))) +
  322. /* 64K PIO space / ctxt */
  323. (type == PIO_BUFS_SOP ?
  324. (TXE_PIO_SIZE / 2) : 0); /* sop? */
  325. /*
  326. * Map only the amount allocated to the context, not the
  327. * entire available context's PIO space.
  328. */
  329. memlen = PAGE_ALIGN(uctxt->sc->credits * PIO_BLOCK_SIZE);
  330. flags &= ~VM_MAYREAD;
  331. flags |= VM_DONTCOPY | VM_DONTEXPAND;
  332. vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
  333. mapio = 1;
  334. break;
  335. case PIO_CRED:
  336. if (flags & VM_WRITE) {
  337. ret = -EPERM;
  338. goto done;
  339. }
  340. /*
  341. * The credit return location for this context could be on the
  342. * second or third page allocated for credit returns (if number
  343. * of enabled contexts > 64 and 128 respectively).
  344. */
  345. memvirt = dd->cr_base[uctxt->numa_id].va;
  346. memaddr = virt_to_phys(memvirt) +
  347. (((u64)uctxt->sc->hw_free -
  348. (u64)dd->cr_base[uctxt->numa_id].va) & PAGE_MASK);
  349. memlen = PAGE_SIZE;
  350. flags &= ~VM_MAYWRITE;
  351. flags |= VM_DONTCOPY | VM_DONTEXPAND;
  352. /*
  353. * The driver has already allocated memory for credit
  354. * returns and programmed it into the chip. Has that
  355. * memory been flagged as non-cached?
  356. */
  357. /* vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); */
  358. mapio = 1;
  359. break;
  360. case RCV_HDRQ:
  361. memlen = rcvhdrq_size(uctxt);
  362. memvirt = uctxt->rcvhdrq;
  363. break;
  364. case RCV_EGRBUF: {
  365. unsigned long addr;
  366. int i;
  367. /*
  368. * The RcvEgr buffer need to be handled differently
  369. * as multiple non-contiguous pages need to be mapped
  370. * into the user process.
  371. */
  372. memlen = uctxt->egrbufs.size;
  373. if ((vma->vm_end - vma->vm_start) != memlen) {
  374. dd_dev_err(dd, "Eager buffer map size invalid (%lu != %lu)\n",
  375. (vma->vm_end - vma->vm_start), memlen);
  376. ret = -EINVAL;
  377. goto done;
  378. }
  379. if (vma->vm_flags & VM_WRITE) {
  380. ret = -EPERM;
  381. goto done;
  382. }
  383. vma->vm_flags &= ~VM_MAYWRITE;
  384. addr = vma->vm_start;
  385. for (i = 0 ; i < uctxt->egrbufs.numbufs; i++) {
  386. memlen = uctxt->egrbufs.buffers[i].len;
  387. memvirt = uctxt->egrbufs.buffers[i].addr;
  388. ret = remap_pfn_range(
  389. vma, addr,
  390. /*
  391. * virt_to_pfn() does the same, but
  392. * it's not available on x86_64
  393. * when CONFIG_MMU is enabled.
  394. */
  395. PFN_DOWN(__pa(memvirt)),
  396. memlen,
  397. vma->vm_page_prot);
  398. if (ret < 0)
  399. goto done;
  400. addr += memlen;
  401. }
  402. ret = 0;
  403. goto done;
  404. }
  405. case UREGS:
  406. /*
  407. * Map only the page that contains this context's user
  408. * registers.
  409. */
  410. memaddr = (unsigned long)
  411. (dd->physaddr + RXE_PER_CONTEXT_USER)
  412. + (uctxt->ctxt * RXE_PER_CONTEXT_SIZE);
  413. /*
  414. * TidFlow table is on the same page as the rest of the
  415. * user registers.
  416. */
  417. memlen = PAGE_SIZE;
  418. flags |= VM_DONTCOPY | VM_DONTEXPAND;
  419. vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
  420. mapio = 1;
  421. break;
  422. case EVENTS:
  423. /*
  424. * Use the page where this context's flags are. User level
  425. * knows where it's own bitmap is within the page.
  426. */
  427. memaddr = (unsigned long)
  428. (dd->events + uctxt_offset(uctxt)) & PAGE_MASK;
  429. memlen = PAGE_SIZE;
  430. /*
  431. * v3.7 removes VM_RESERVED but the effect is kept by
  432. * using VM_IO.
  433. */
  434. flags |= VM_IO | VM_DONTEXPAND;
  435. vmf = 1;
  436. break;
  437. case STATUS:
  438. if (flags & (unsigned long)(VM_WRITE | VM_EXEC)) {
  439. ret = -EPERM;
  440. goto done;
  441. }
  442. memaddr = kvirt_to_phys((void *)dd->status);
  443. memlen = PAGE_SIZE;
  444. flags |= VM_IO | VM_DONTEXPAND;
  445. break;
  446. case RTAIL:
  447. if (!HFI1_CAP_IS_USET(DMA_RTAIL)) {
  448. /*
  449. * If the memory allocation failed, the context alloc
  450. * also would have failed, so we would never get here
  451. */
  452. ret = -EINVAL;
  453. goto done;
  454. }
  455. if ((flags & VM_WRITE) || !uctxt->rcvhdrtail_kvaddr) {
  456. ret = -EPERM;
  457. goto done;
  458. }
  459. memlen = PAGE_SIZE;
  460. memvirt = (void *)uctxt->rcvhdrtail_kvaddr;
  461. flags &= ~VM_MAYWRITE;
  462. break;
  463. case SUBCTXT_UREGS:
  464. memaddr = (u64)uctxt->subctxt_uregbase;
  465. memlen = PAGE_SIZE;
  466. flags |= VM_IO | VM_DONTEXPAND;
  467. vmf = 1;
  468. break;
  469. case SUBCTXT_RCV_HDRQ:
  470. memaddr = (u64)uctxt->subctxt_rcvhdr_base;
  471. memlen = rcvhdrq_size(uctxt) * uctxt->subctxt_cnt;
  472. flags |= VM_IO | VM_DONTEXPAND;
  473. vmf = 1;
  474. break;
  475. case SUBCTXT_EGRBUF:
  476. memaddr = (u64)uctxt->subctxt_rcvegrbuf;
  477. memlen = uctxt->egrbufs.size * uctxt->subctxt_cnt;
  478. flags |= VM_IO | VM_DONTEXPAND;
  479. flags &= ~VM_MAYWRITE;
  480. vmf = 1;
  481. break;
  482. case SDMA_COMP: {
  483. struct hfi1_user_sdma_comp_q *cq = fd->cq;
  484. if (!cq) {
  485. ret = -EFAULT;
  486. goto done;
  487. }
  488. memaddr = (u64)cq->comps;
  489. memlen = PAGE_ALIGN(sizeof(*cq->comps) * cq->nentries);
  490. flags |= VM_IO | VM_DONTEXPAND;
  491. vmf = 1;
  492. break;
  493. }
  494. default:
  495. ret = -EINVAL;
  496. break;
  497. }
  498. if ((vma->vm_end - vma->vm_start) != memlen) {
  499. hfi1_cdbg(PROC, "%u:%u Memory size mismatch %lu:%lu",
  500. uctxt->ctxt, fd->subctxt,
  501. (vma->vm_end - vma->vm_start), memlen);
  502. ret = -EINVAL;
  503. goto done;
  504. }
  505. vma->vm_flags = flags;
  506. hfi1_cdbg(PROC,
  507. "%u:%u type:%u io/vf:%d/%d, addr:0x%llx, len:%lu(%lu), flags:0x%lx\n",
  508. ctxt, subctxt, type, mapio, vmf, memaddr, memlen,
  509. vma->vm_end - vma->vm_start, vma->vm_flags);
  510. if (vmf) {
  511. vma->vm_pgoff = PFN_DOWN(memaddr);
  512. vma->vm_ops = &vm_ops;
  513. ret = 0;
  514. } else if (mapio) {
  515. ret = io_remap_pfn_range(vma, vma->vm_start,
  516. PFN_DOWN(memaddr),
  517. memlen,
  518. vma->vm_page_prot);
  519. } else if (memvirt) {
  520. ret = remap_pfn_range(vma, vma->vm_start,
  521. PFN_DOWN(__pa(memvirt)),
  522. memlen,
  523. vma->vm_page_prot);
  524. } else {
  525. ret = remap_pfn_range(vma, vma->vm_start,
  526. PFN_DOWN(memaddr),
  527. memlen,
  528. vma->vm_page_prot);
  529. }
  530. done:
  531. return ret;
  532. }
  533. /*
  534. * Local (non-chip) user memory is not mapped right away but as it is
  535. * accessed by the user-level code.
  536. */
  537. static vm_fault_t vma_fault(struct vm_fault *vmf)
  538. {
  539. struct page *page;
  540. page = vmalloc_to_page((void *)(vmf->pgoff << PAGE_SHIFT));
  541. if (!page)
  542. return VM_FAULT_SIGBUS;
  543. get_page(page);
  544. vmf->page = page;
  545. return 0;
  546. }
  547. static __poll_t hfi1_poll(struct file *fp, struct poll_table_struct *pt)
  548. {
  549. struct hfi1_ctxtdata *uctxt;
  550. __poll_t pollflag;
  551. uctxt = ((struct hfi1_filedata *)fp->private_data)->uctxt;
  552. if (!uctxt)
  553. pollflag = EPOLLERR;
  554. else if (uctxt->poll_type == HFI1_POLL_TYPE_URGENT)
  555. pollflag = poll_urgent(fp, pt);
  556. else if (uctxt->poll_type == HFI1_POLL_TYPE_ANYRCV)
  557. pollflag = poll_next(fp, pt);
  558. else /* invalid */
  559. pollflag = EPOLLERR;
  560. return pollflag;
  561. }
  562. static int hfi1_file_close(struct inode *inode, struct file *fp)
  563. {
  564. struct hfi1_filedata *fdata = fp->private_data;
  565. struct hfi1_ctxtdata *uctxt = fdata->uctxt;
  566. struct hfi1_devdata *dd = container_of(inode->i_cdev,
  567. struct hfi1_devdata,
  568. user_cdev);
  569. unsigned long flags, *ev;
  570. fp->private_data = NULL;
  571. if (!uctxt)
  572. goto done;
  573. hfi1_cdbg(PROC, "closing ctxt %u:%u", uctxt->ctxt, fdata->subctxt);
  574. flush_wc();
  575. /* drain user sdma queue */
  576. hfi1_user_sdma_free_queues(fdata, uctxt);
  577. /* release the cpu */
  578. hfi1_put_proc_affinity(fdata->rec_cpu_num);
  579. /* clean up rcv side */
  580. hfi1_user_exp_rcv_free(fdata);
  581. /*
  582. * fdata->uctxt is used in the above cleanup. It is not ready to be
  583. * removed until here.
  584. */
  585. fdata->uctxt = NULL;
  586. hfi1_rcd_put(uctxt);
  587. /*
  588. * Clear any left over, unhandled events so the next process that
  589. * gets this context doesn't get confused.
  590. */
  591. ev = dd->events + uctxt_offset(uctxt) + fdata->subctxt;
  592. *ev = 0;
  593. spin_lock_irqsave(&dd->uctxt_lock, flags);
  594. __clear_bit(fdata->subctxt, uctxt->in_use_ctxts);
  595. if (!bitmap_empty(uctxt->in_use_ctxts, HFI1_MAX_SHARED_CTXTS)) {
  596. spin_unlock_irqrestore(&dd->uctxt_lock, flags);
  597. goto done;
  598. }
  599. spin_unlock_irqrestore(&dd->uctxt_lock, flags);
  600. /*
  601. * Disable receive context and interrupt available, reset all
  602. * RcvCtxtCtrl bits to default values.
  603. */
  604. hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS |
  605. HFI1_RCVCTRL_TIDFLOW_DIS |
  606. HFI1_RCVCTRL_INTRAVAIL_DIS |
  607. HFI1_RCVCTRL_TAILUPD_DIS |
  608. HFI1_RCVCTRL_ONE_PKT_EGR_DIS |
  609. HFI1_RCVCTRL_NO_RHQ_DROP_DIS |
  610. HFI1_RCVCTRL_NO_EGR_DROP_DIS |
  611. HFI1_RCVCTRL_URGENT_DIS, uctxt);
  612. /* Clear the context's J_KEY */
  613. hfi1_clear_ctxt_jkey(dd, uctxt);
  614. /*
  615. * If a send context is allocated, reset context integrity
  616. * checks to default and disable the send context.
  617. */
  618. if (uctxt->sc) {
  619. sc_disable(uctxt->sc);
  620. set_pio_integrity(uctxt->sc);
  621. }
  622. hfi1_free_ctxt_rcv_groups(uctxt);
  623. hfi1_clear_ctxt_pkey(dd, uctxt);
  624. uctxt->event_flags = 0;
  625. deallocate_ctxt(uctxt);
  626. done:
  627. mmdrop(fdata->mm);
  628. kobject_put(&dd->kobj);
  629. if (atomic_dec_and_test(&dd->user_refcount))
  630. complete(&dd->user_comp);
  631. kfree(fdata);
  632. return 0;
  633. }
  634. /*
  635. * Convert kernel *virtual* addresses to physical addresses.
  636. * This is used to vmalloc'ed addresses.
  637. */
  638. static u64 kvirt_to_phys(void *addr)
  639. {
  640. struct page *page;
  641. u64 paddr = 0;
  642. page = vmalloc_to_page(addr);
  643. if (page)
  644. paddr = page_to_pfn(page) << PAGE_SHIFT;
  645. return paddr;
  646. }
  647. /**
  648. * complete_subctxt
  649. * @fd: valid filedata pointer
  650. *
  651. * Sub-context info can only be set up after the base context
  652. * has been completed. This is indicated by the clearing of the
  653. * HFI1_CTXT_BASE_UINIT bit.
  654. *
  655. * Wait for the bit to be cleared, and then complete the subcontext
  656. * initialization.
  657. *
  658. */
  659. static int complete_subctxt(struct hfi1_filedata *fd)
  660. {
  661. int ret;
  662. unsigned long flags;
  663. /*
  664. * sub-context info can only be set up after the base context
  665. * has been completed.
  666. */
  667. ret = wait_event_interruptible(
  668. fd->uctxt->wait,
  669. !test_bit(HFI1_CTXT_BASE_UNINIT, &fd->uctxt->event_flags));
  670. if (test_bit(HFI1_CTXT_BASE_FAILED, &fd->uctxt->event_flags))
  671. ret = -ENOMEM;
  672. /* Finish the sub-context init */
  673. if (!ret) {
  674. fd->rec_cpu_num = hfi1_get_proc_affinity(fd->uctxt->numa_id);
  675. ret = init_user_ctxt(fd, fd->uctxt);
  676. }
  677. if (ret) {
  678. spin_lock_irqsave(&fd->dd->uctxt_lock, flags);
  679. __clear_bit(fd->subctxt, fd->uctxt->in_use_ctxts);
  680. spin_unlock_irqrestore(&fd->dd->uctxt_lock, flags);
  681. hfi1_rcd_put(fd->uctxt);
  682. fd->uctxt = NULL;
  683. }
  684. return ret;
  685. }
  686. static int assign_ctxt(struct hfi1_filedata *fd, unsigned long arg, u32 len)
  687. {
  688. int ret;
  689. unsigned int swmajor;
  690. struct hfi1_ctxtdata *uctxt = NULL;
  691. struct hfi1_user_info uinfo;
  692. if (fd->uctxt)
  693. return -EINVAL;
  694. if (sizeof(uinfo) != len)
  695. return -EINVAL;
  696. if (copy_from_user(&uinfo, (void __user *)arg, sizeof(uinfo)))
  697. return -EFAULT;
  698. swmajor = uinfo.userversion >> 16;
  699. if (swmajor != HFI1_USER_SWMAJOR)
  700. return -ENODEV;
  701. if (uinfo.subctxt_cnt > HFI1_MAX_SHARED_CTXTS)
  702. return -EINVAL;
  703. /*
  704. * Acquire the mutex to protect against multiple creations of what
  705. * could be a shared base context.
  706. */
  707. mutex_lock(&hfi1_mutex);
  708. /*
  709. * Get a sub context if available (fd->uctxt will be set).
  710. * ret < 0 error, 0 no context, 1 sub-context found
  711. */
  712. ret = find_sub_ctxt(fd, &uinfo);
  713. /*
  714. * Allocate a base context if context sharing is not required or a
  715. * sub context wasn't found.
  716. */
  717. if (!ret)
  718. ret = allocate_ctxt(fd, fd->dd, &uinfo, &uctxt);
  719. mutex_unlock(&hfi1_mutex);
  720. /* Depending on the context type, finish the appropriate init */
  721. switch (ret) {
  722. case 0:
  723. ret = setup_base_ctxt(fd, uctxt);
  724. if (ret)
  725. deallocate_ctxt(uctxt);
  726. break;
  727. case 1:
  728. ret = complete_subctxt(fd);
  729. break;
  730. default:
  731. break;
  732. }
  733. return ret;
  734. }
  735. /**
  736. * match_ctxt
  737. * @fd: valid filedata pointer
  738. * @uinfo: user info to compare base context with
  739. * @uctxt: context to compare uinfo to.
  740. *
  741. * Compare the given context with the given information to see if it
  742. * can be used for a sub context.
  743. */
  744. static int match_ctxt(struct hfi1_filedata *fd,
  745. const struct hfi1_user_info *uinfo,
  746. struct hfi1_ctxtdata *uctxt)
  747. {
  748. struct hfi1_devdata *dd = fd->dd;
  749. unsigned long flags;
  750. u16 subctxt;
  751. /* Skip dynamically allocated kernel contexts */
  752. if (uctxt->sc && (uctxt->sc->type == SC_KERNEL))
  753. return 0;
  754. /* Skip ctxt if it doesn't match the requested one */
  755. if (memcmp(uctxt->uuid, uinfo->uuid, sizeof(uctxt->uuid)) ||
  756. uctxt->jkey != generate_jkey(current_uid()) ||
  757. uctxt->subctxt_id != uinfo->subctxt_id ||
  758. uctxt->subctxt_cnt != uinfo->subctxt_cnt)
  759. return 0;
  760. /* Verify the sharing process matches the base */
  761. if (uctxt->userversion != uinfo->userversion)
  762. return -EINVAL;
  763. /* Find an unused sub context */
  764. spin_lock_irqsave(&dd->uctxt_lock, flags);
  765. if (bitmap_empty(uctxt->in_use_ctxts, HFI1_MAX_SHARED_CTXTS)) {
  766. /* context is being closed, do not use */
  767. spin_unlock_irqrestore(&dd->uctxt_lock, flags);
  768. return 0;
  769. }
  770. subctxt = find_first_zero_bit(uctxt->in_use_ctxts,
  771. HFI1_MAX_SHARED_CTXTS);
  772. if (subctxt >= uctxt->subctxt_cnt) {
  773. spin_unlock_irqrestore(&dd->uctxt_lock, flags);
  774. return -EBUSY;
  775. }
  776. fd->subctxt = subctxt;
  777. __set_bit(fd->subctxt, uctxt->in_use_ctxts);
  778. spin_unlock_irqrestore(&dd->uctxt_lock, flags);
  779. fd->uctxt = uctxt;
  780. hfi1_rcd_get(uctxt);
  781. return 1;
  782. }
  783. /**
  784. * find_sub_ctxt
  785. * @fd: valid filedata pointer
  786. * @uinfo: matching info to use to find a possible context to share.
  787. *
  788. * The hfi1_mutex must be held when this function is called. It is
  789. * necessary to ensure serialized creation of shared contexts.
  790. *
  791. * Return:
  792. * 0 No sub-context found
  793. * 1 Subcontext found and allocated
  794. * errno EINVAL (incorrect parameters)
  795. * EBUSY (all sub contexts in use)
  796. */
  797. static int find_sub_ctxt(struct hfi1_filedata *fd,
  798. const struct hfi1_user_info *uinfo)
  799. {
  800. struct hfi1_ctxtdata *uctxt;
  801. struct hfi1_devdata *dd = fd->dd;
  802. u16 i;
  803. int ret;
  804. if (!uinfo->subctxt_cnt)
  805. return 0;
  806. for (i = dd->first_dyn_alloc_ctxt; i < dd->num_rcv_contexts; i++) {
  807. uctxt = hfi1_rcd_get_by_index(dd, i);
  808. if (uctxt) {
  809. ret = match_ctxt(fd, uinfo, uctxt);
  810. hfi1_rcd_put(uctxt);
  811. /* value of != 0 will return */
  812. if (ret)
  813. return ret;
  814. }
  815. }
  816. return 0;
  817. }
  818. static int allocate_ctxt(struct hfi1_filedata *fd, struct hfi1_devdata *dd,
  819. struct hfi1_user_info *uinfo,
  820. struct hfi1_ctxtdata **rcd)
  821. {
  822. struct hfi1_ctxtdata *uctxt;
  823. int ret, numa;
  824. if (dd->flags & HFI1_FROZEN) {
  825. /*
  826. * Pick an error that is unique from all other errors
  827. * that are returned so the user process knows that
  828. * it tried to allocate while the SPC was frozen. It
  829. * it should be able to retry with success in a short
  830. * while.
  831. */
  832. return -EIO;
  833. }
  834. if (!dd->freectxts)
  835. return -EBUSY;
  836. /*
  837. * If we don't have a NUMA node requested, preference is towards
  838. * device NUMA node.
  839. */
  840. fd->rec_cpu_num = hfi1_get_proc_affinity(dd->node);
  841. if (fd->rec_cpu_num != -1)
  842. numa = cpu_to_node(fd->rec_cpu_num);
  843. else
  844. numa = numa_node_id();
  845. ret = hfi1_create_ctxtdata(dd->pport, numa, &uctxt);
  846. if (ret < 0) {
  847. dd_dev_err(dd, "user ctxtdata allocation failed\n");
  848. return ret;
  849. }
  850. hfi1_cdbg(PROC, "[%u:%u] pid %u assigned to CPU %d (NUMA %u)",
  851. uctxt->ctxt, fd->subctxt, current->pid, fd->rec_cpu_num,
  852. uctxt->numa_id);
  853. /*
  854. * Allocate and enable a PIO send context.
  855. */
  856. uctxt->sc = sc_alloc(dd, SC_USER, uctxt->rcvhdrqentsize, dd->node);
  857. if (!uctxt->sc) {
  858. ret = -ENOMEM;
  859. goto ctxdata_free;
  860. }
  861. hfi1_cdbg(PROC, "allocated send context %u(%u)\n", uctxt->sc->sw_index,
  862. uctxt->sc->hw_context);
  863. ret = sc_enable(uctxt->sc);
  864. if (ret)
  865. goto ctxdata_free;
  866. /*
  867. * Setup sub context information if the user-level has requested
  868. * sub contexts.
  869. * This has to be done here so the rest of the sub-contexts find the
  870. * proper base context.
  871. * NOTE: _set_bit() can be used here because the context creation is
  872. * protected by the mutex (rather than the spin_lock), and will be the
  873. * very first instance of this context.
  874. */
  875. __set_bit(0, uctxt->in_use_ctxts);
  876. if (uinfo->subctxt_cnt)
  877. init_subctxts(uctxt, uinfo);
  878. uctxt->userversion = uinfo->userversion;
  879. uctxt->flags = hfi1_cap_mask; /* save current flag state */
  880. init_waitqueue_head(&uctxt->wait);
  881. strlcpy(uctxt->comm, current->comm, sizeof(uctxt->comm));
  882. memcpy(uctxt->uuid, uinfo->uuid, sizeof(uctxt->uuid));
  883. uctxt->jkey = generate_jkey(current_uid());
  884. hfi1_stats.sps_ctxts++;
  885. /*
  886. * Disable ASPM when there are open user/PSM contexts to avoid
  887. * issues with ASPM L1 exit latency
  888. */
  889. if (dd->freectxts-- == dd->num_user_contexts)
  890. aspm_disable_all(dd);
  891. *rcd = uctxt;
  892. return 0;
  893. ctxdata_free:
  894. hfi1_free_ctxt(uctxt);
  895. return ret;
  896. }
  897. static void deallocate_ctxt(struct hfi1_ctxtdata *uctxt)
  898. {
  899. mutex_lock(&hfi1_mutex);
  900. hfi1_stats.sps_ctxts--;
  901. if (++uctxt->dd->freectxts == uctxt->dd->num_user_contexts)
  902. aspm_enable_all(uctxt->dd);
  903. mutex_unlock(&hfi1_mutex);
  904. hfi1_free_ctxt(uctxt);
  905. }
  906. static void init_subctxts(struct hfi1_ctxtdata *uctxt,
  907. const struct hfi1_user_info *uinfo)
  908. {
  909. uctxt->subctxt_cnt = uinfo->subctxt_cnt;
  910. uctxt->subctxt_id = uinfo->subctxt_id;
  911. set_bit(HFI1_CTXT_BASE_UNINIT, &uctxt->event_flags);
  912. }
  913. static int setup_subctxt(struct hfi1_ctxtdata *uctxt)
  914. {
  915. int ret = 0;
  916. u16 num_subctxts = uctxt->subctxt_cnt;
  917. uctxt->subctxt_uregbase = vmalloc_user(PAGE_SIZE);
  918. if (!uctxt->subctxt_uregbase)
  919. return -ENOMEM;
  920. /* We can take the size of the RcvHdr Queue from the master */
  921. uctxt->subctxt_rcvhdr_base = vmalloc_user(rcvhdrq_size(uctxt) *
  922. num_subctxts);
  923. if (!uctxt->subctxt_rcvhdr_base) {
  924. ret = -ENOMEM;
  925. goto bail_ureg;
  926. }
  927. uctxt->subctxt_rcvegrbuf = vmalloc_user(uctxt->egrbufs.size *
  928. num_subctxts);
  929. if (!uctxt->subctxt_rcvegrbuf) {
  930. ret = -ENOMEM;
  931. goto bail_rhdr;
  932. }
  933. return 0;
  934. bail_rhdr:
  935. vfree(uctxt->subctxt_rcvhdr_base);
  936. uctxt->subctxt_rcvhdr_base = NULL;
  937. bail_ureg:
  938. vfree(uctxt->subctxt_uregbase);
  939. uctxt->subctxt_uregbase = NULL;
  940. return ret;
  941. }
  942. static void user_init(struct hfi1_ctxtdata *uctxt)
  943. {
  944. unsigned int rcvctrl_ops = 0;
  945. /* initialize poll variables... */
  946. uctxt->urgent = 0;
  947. uctxt->urgent_poll = 0;
  948. /*
  949. * Now enable the ctxt for receive.
  950. * For chips that are set to DMA the tail register to memory
  951. * when they change (and when the update bit transitions from
  952. * 0 to 1. So for those chips, we turn it off and then back on.
  953. * This will (very briefly) affect any other open ctxts, but the
  954. * duration is very short, and therefore isn't an issue. We
  955. * explicitly set the in-memory tail copy to 0 beforehand, so we
  956. * don't have to wait to be sure the DMA update has happened
  957. * (chip resets head/tail to 0 on transition to enable).
  958. */
  959. if (uctxt->rcvhdrtail_kvaddr)
  960. clear_rcvhdrtail(uctxt);
  961. /* Setup J_KEY before enabling the context */
  962. hfi1_set_ctxt_jkey(uctxt->dd, uctxt, uctxt->jkey);
  963. rcvctrl_ops = HFI1_RCVCTRL_CTXT_ENB;
  964. rcvctrl_ops |= HFI1_RCVCTRL_URGENT_ENB;
  965. if (HFI1_CAP_UGET_MASK(uctxt->flags, HDRSUPP))
  966. rcvctrl_ops |= HFI1_RCVCTRL_TIDFLOW_ENB;
  967. /*
  968. * Ignore the bit in the flags for now until proper
  969. * support for multiple packet per rcv array entry is
  970. * added.
  971. */
  972. if (!HFI1_CAP_UGET_MASK(uctxt->flags, MULTI_PKT_EGR))
  973. rcvctrl_ops |= HFI1_RCVCTRL_ONE_PKT_EGR_ENB;
  974. if (HFI1_CAP_UGET_MASK(uctxt->flags, NODROP_EGR_FULL))
  975. rcvctrl_ops |= HFI1_RCVCTRL_NO_EGR_DROP_ENB;
  976. if (HFI1_CAP_UGET_MASK(uctxt->flags, NODROP_RHQ_FULL))
  977. rcvctrl_ops |= HFI1_RCVCTRL_NO_RHQ_DROP_ENB;
  978. /*
  979. * The RcvCtxtCtrl.TailUpd bit has to be explicitly written.
  980. * We can't rely on the correct value to be set from prior
  981. * uses of the chip or ctxt. Therefore, add the rcvctrl op
  982. * for both cases.
  983. */
  984. if (HFI1_CAP_UGET_MASK(uctxt->flags, DMA_RTAIL))
  985. rcvctrl_ops |= HFI1_RCVCTRL_TAILUPD_ENB;
  986. else
  987. rcvctrl_ops |= HFI1_RCVCTRL_TAILUPD_DIS;
  988. hfi1_rcvctrl(uctxt->dd, rcvctrl_ops, uctxt);
  989. }
  990. static int get_ctxt_info(struct hfi1_filedata *fd, unsigned long arg, u32 len)
  991. {
  992. struct hfi1_ctxt_info cinfo;
  993. struct hfi1_ctxtdata *uctxt = fd->uctxt;
  994. if (sizeof(cinfo) != len)
  995. return -EINVAL;
  996. memset(&cinfo, 0, sizeof(cinfo));
  997. cinfo.runtime_flags = (((uctxt->flags >> HFI1_CAP_MISC_SHIFT) &
  998. HFI1_CAP_MISC_MASK) << HFI1_CAP_USER_SHIFT) |
  999. HFI1_CAP_UGET_MASK(uctxt->flags, MASK) |
  1000. HFI1_CAP_KGET_MASK(uctxt->flags, K2U);
  1001. /* adjust flag if this fd is not able to cache */
  1002. if (!fd->handler)
  1003. cinfo.runtime_flags |= HFI1_CAP_TID_UNMAP; /* no caching */
  1004. cinfo.num_active = hfi1_count_active_units();
  1005. cinfo.unit = uctxt->dd->unit;
  1006. cinfo.ctxt = uctxt->ctxt;
  1007. cinfo.subctxt = fd->subctxt;
  1008. cinfo.rcvtids = roundup(uctxt->egrbufs.alloced,
  1009. uctxt->dd->rcv_entries.group_size) +
  1010. uctxt->expected_count;
  1011. cinfo.credits = uctxt->sc->credits;
  1012. cinfo.numa_node = uctxt->numa_id;
  1013. cinfo.rec_cpu = fd->rec_cpu_num;
  1014. cinfo.send_ctxt = uctxt->sc->hw_context;
  1015. cinfo.egrtids = uctxt->egrbufs.alloced;
  1016. cinfo.rcvhdrq_cnt = uctxt->rcvhdrq_cnt;
  1017. cinfo.rcvhdrq_entsize = uctxt->rcvhdrqentsize << 2;
  1018. cinfo.sdma_ring_size = fd->cq->nentries;
  1019. cinfo.rcvegr_size = uctxt->egrbufs.rcvtid_size;
  1020. trace_hfi1_ctxt_info(uctxt->dd, uctxt->ctxt, fd->subctxt, &cinfo);
  1021. if (copy_to_user((void __user *)arg, &cinfo, len))
  1022. return -EFAULT;
  1023. return 0;
  1024. }
  1025. static int init_user_ctxt(struct hfi1_filedata *fd,
  1026. struct hfi1_ctxtdata *uctxt)
  1027. {
  1028. int ret;
  1029. ret = hfi1_user_sdma_alloc_queues(uctxt, fd);
  1030. if (ret)
  1031. return ret;
  1032. ret = hfi1_user_exp_rcv_init(fd, uctxt);
  1033. if (ret)
  1034. hfi1_user_sdma_free_queues(fd, uctxt);
  1035. return ret;
  1036. }
  1037. static int setup_base_ctxt(struct hfi1_filedata *fd,
  1038. struct hfi1_ctxtdata *uctxt)
  1039. {
  1040. struct hfi1_devdata *dd = uctxt->dd;
  1041. int ret = 0;
  1042. hfi1_init_ctxt(uctxt->sc);
  1043. /* Now allocate the RcvHdr queue and eager buffers. */
  1044. ret = hfi1_create_rcvhdrq(dd, uctxt);
  1045. if (ret)
  1046. goto done;
  1047. ret = hfi1_setup_eagerbufs(uctxt);
  1048. if (ret)
  1049. goto done;
  1050. /* If sub-contexts are enabled, do the appropriate setup */
  1051. if (uctxt->subctxt_cnt)
  1052. ret = setup_subctxt(uctxt);
  1053. if (ret)
  1054. goto done;
  1055. ret = hfi1_alloc_ctxt_rcv_groups(uctxt);
  1056. if (ret)
  1057. goto done;
  1058. ret = init_user_ctxt(fd, uctxt);
  1059. if (ret)
  1060. goto done;
  1061. user_init(uctxt);
  1062. /* Now that the context is set up, the fd can get a reference. */
  1063. fd->uctxt = uctxt;
  1064. hfi1_rcd_get(uctxt);
  1065. done:
  1066. if (uctxt->subctxt_cnt) {
  1067. /*
  1068. * On error, set the failed bit so sub-contexts will clean up
  1069. * correctly.
  1070. */
  1071. if (ret)
  1072. set_bit(HFI1_CTXT_BASE_FAILED, &uctxt->event_flags);
  1073. /*
  1074. * Base context is done (successfully or not), notify anybody
  1075. * using a sub-context that is waiting for this completion.
  1076. */
  1077. clear_bit(HFI1_CTXT_BASE_UNINIT, &uctxt->event_flags);
  1078. wake_up(&uctxt->wait);
  1079. }
  1080. return ret;
  1081. }
  1082. static int get_base_info(struct hfi1_filedata *fd, unsigned long arg, u32 len)
  1083. {
  1084. struct hfi1_base_info binfo;
  1085. struct hfi1_ctxtdata *uctxt = fd->uctxt;
  1086. struct hfi1_devdata *dd = uctxt->dd;
  1087. unsigned offset;
  1088. trace_hfi1_uctxtdata(uctxt->dd, uctxt, fd->subctxt);
  1089. if (sizeof(binfo) != len)
  1090. return -EINVAL;
  1091. memset(&binfo, 0, sizeof(binfo));
  1092. binfo.hw_version = dd->revision;
  1093. binfo.sw_version = HFI1_KERN_SWVERSION;
  1094. binfo.bthqp = kdeth_qp;
  1095. binfo.jkey = uctxt->jkey;
  1096. /*
  1097. * If more than 64 contexts are enabled the allocated credit
  1098. * return will span two or three contiguous pages. Since we only
  1099. * map the page containing the context's credit return address,
  1100. * we need to calculate the offset in the proper page.
  1101. */
  1102. offset = ((u64)uctxt->sc->hw_free -
  1103. (u64)dd->cr_base[uctxt->numa_id].va) % PAGE_SIZE;
  1104. binfo.sc_credits_addr = HFI1_MMAP_TOKEN(PIO_CRED, uctxt->ctxt,
  1105. fd->subctxt, offset);
  1106. binfo.pio_bufbase = HFI1_MMAP_TOKEN(PIO_BUFS, uctxt->ctxt,
  1107. fd->subctxt,
  1108. uctxt->sc->base_addr);
  1109. binfo.pio_bufbase_sop = HFI1_MMAP_TOKEN(PIO_BUFS_SOP,
  1110. uctxt->ctxt,
  1111. fd->subctxt,
  1112. uctxt->sc->base_addr);
  1113. binfo.rcvhdr_bufbase = HFI1_MMAP_TOKEN(RCV_HDRQ, uctxt->ctxt,
  1114. fd->subctxt,
  1115. uctxt->rcvhdrq);
  1116. binfo.rcvegr_bufbase = HFI1_MMAP_TOKEN(RCV_EGRBUF, uctxt->ctxt,
  1117. fd->subctxt,
  1118. uctxt->egrbufs.rcvtids[0].dma);
  1119. binfo.sdma_comp_bufbase = HFI1_MMAP_TOKEN(SDMA_COMP, uctxt->ctxt,
  1120. fd->subctxt, 0);
  1121. /*
  1122. * user regs are at
  1123. * (RXE_PER_CONTEXT_USER + (ctxt * RXE_PER_CONTEXT_SIZE))
  1124. */
  1125. binfo.user_regbase = HFI1_MMAP_TOKEN(UREGS, uctxt->ctxt,
  1126. fd->subctxt, 0);
  1127. offset = offset_in_page((uctxt_offset(uctxt) + fd->subctxt) *
  1128. sizeof(*dd->events));
  1129. binfo.events_bufbase = HFI1_MMAP_TOKEN(EVENTS, uctxt->ctxt,
  1130. fd->subctxt,
  1131. offset);
  1132. binfo.status_bufbase = HFI1_MMAP_TOKEN(STATUS, uctxt->ctxt,
  1133. fd->subctxt,
  1134. dd->status);
  1135. if (HFI1_CAP_IS_USET(DMA_RTAIL))
  1136. binfo.rcvhdrtail_base = HFI1_MMAP_TOKEN(RTAIL, uctxt->ctxt,
  1137. fd->subctxt, 0);
  1138. if (uctxt->subctxt_cnt) {
  1139. binfo.subctxt_uregbase = HFI1_MMAP_TOKEN(SUBCTXT_UREGS,
  1140. uctxt->ctxt,
  1141. fd->subctxt, 0);
  1142. binfo.subctxt_rcvhdrbuf = HFI1_MMAP_TOKEN(SUBCTXT_RCV_HDRQ,
  1143. uctxt->ctxt,
  1144. fd->subctxt, 0);
  1145. binfo.subctxt_rcvegrbuf = HFI1_MMAP_TOKEN(SUBCTXT_EGRBUF,
  1146. uctxt->ctxt,
  1147. fd->subctxt, 0);
  1148. }
  1149. if (copy_to_user((void __user *)arg, &binfo, len))
  1150. return -EFAULT;
  1151. return 0;
  1152. }
  1153. /**
  1154. * user_exp_rcv_setup - Set up the given tid rcv list
  1155. * @fd: file data of the current driver instance
  1156. * @arg: ioctl argumnent for user space information
  1157. * @len: length of data structure associated with ioctl command
  1158. *
  1159. * Wrapper to validate ioctl information before doing _rcv_setup.
  1160. *
  1161. */
  1162. static int user_exp_rcv_setup(struct hfi1_filedata *fd, unsigned long arg,
  1163. u32 len)
  1164. {
  1165. int ret;
  1166. unsigned long addr;
  1167. struct hfi1_tid_info tinfo;
  1168. if (sizeof(tinfo) != len)
  1169. return -EINVAL;
  1170. if (copy_from_user(&tinfo, (void __user *)arg, (sizeof(tinfo))))
  1171. return -EFAULT;
  1172. ret = hfi1_user_exp_rcv_setup(fd, &tinfo);
  1173. if (!ret) {
  1174. /*
  1175. * Copy the number of tidlist entries we used
  1176. * and the length of the buffer we registered.
  1177. */
  1178. addr = arg + offsetof(struct hfi1_tid_info, tidcnt);
  1179. if (copy_to_user((void __user *)addr, &tinfo.tidcnt,
  1180. sizeof(tinfo.tidcnt)))
  1181. return -EFAULT;
  1182. addr = arg + offsetof(struct hfi1_tid_info, length);
  1183. if (copy_to_user((void __user *)addr, &tinfo.length,
  1184. sizeof(tinfo.length)))
  1185. ret = -EFAULT;
  1186. }
  1187. return ret;
  1188. }
  1189. /**
  1190. * user_exp_rcv_clear - Clear the given tid rcv list
  1191. * @fd: file data of the current driver instance
  1192. * @arg: ioctl argumnent for user space information
  1193. * @len: length of data structure associated with ioctl command
  1194. *
  1195. * The hfi1_user_exp_rcv_clear() can be called from the error path. Because
  1196. * of this, we need to use this wrapper to copy the user space information
  1197. * before doing the clear.
  1198. */
  1199. static int user_exp_rcv_clear(struct hfi1_filedata *fd, unsigned long arg,
  1200. u32 len)
  1201. {
  1202. int ret;
  1203. unsigned long addr;
  1204. struct hfi1_tid_info tinfo;
  1205. if (sizeof(tinfo) != len)
  1206. return -EINVAL;
  1207. if (copy_from_user(&tinfo, (void __user *)arg, (sizeof(tinfo))))
  1208. return -EFAULT;
  1209. ret = hfi1_user_exp_rcv_clear(fd, &tinfo);
  1210. if (!ret) {
  1211. addr = arg + offsetof(struct hfi1_tid_info, tidcnt);
  1212. if (copy_to_user((void __user *)addr, &tinfo.tidcnt,
  1213. sizeof(tinfo.tidcnt)))
  1214. return -EFAULT;
  1215. }
  1216. return ret;
  1217. }
  1218. /**
  1219. * user_exp_rcv_invalid - Invalidate the given tid rcv list
  1220. * @fd: file data of the current driver instance
  1221. * @arg: ioctl argumnent for user space information
  1222. * @len: length of data structure associated with ioctl command
  1223. *
  1224. * Wrapper to validate ioctl information before doing _rcv_invalid.
  1225. *
  1226. */
  1227. static int user_exp_rcv_invalid(struct hfi1_filedata *fd, unsigned long arg,
  1228. u32 len)
  1229. {
  1230. int ret;
  1231. unsigned long addr;
  1232. struct hfi1_tid_info tinfo;
  1233. if (sizeof(tinfo) != len)
  1234. return -EINVAL;
  1235. if (!fd->invalid_tids)
  1236. return -EINVAL;
  1237. if (copy_from_user(&tinfo, (void __user *)arg, (sizeof(tinfo))))
  1238. return -EFAULT;
  1239. ret = hfi1_user_exp_rcv_invalid(fd, &tinfo);
  1240. if (ret)
  1241. return ret;
  1242. addr = arg + offsetof(struct hfi1_tid_info, tidcnt);
  1243. if (copy_to_user((void __user *)addr, &tinfo.tidcnt,
  1244. sizeof(tinfo.tidcnt)))
  1245. ret = -EFAULT;
  1246. return ret;
  1247. }
  1248. static __poll_t poll_urgent(struct file *fp,
  1249. struct poll_table_struct *pt)
  1250. {
  1251. struct hfi1_filedata *fd = fp->private_data;
  1252. struct hfi1_ctxtdata *uctxt = fd->uctxt;
  1253. struct hfi1_devdata *dd = uctxt->dd;
  1254. __poll_t pollflag;
  1255. poll_wait(fp, &uctxt->wait, pt);
  1256. spin_lock_irq(&dd->uctxt_lock);
  1257. if (uctxt->urgent != uctxt->urgent_poll) {
  1258. pollflag = EPOLLIN | EPOLLRDNORM;
  1259. uctxt->urgent_poll = uctxt->urgent;
  1260. } else {
  1261. pollflag = 0;
  1262. set_bit(HFI1_CTXT_WAITING_URG, &uctxt->event_flags);
  1263. }
  1264. spin_unlock_irq(&dd->uctxt_lock);
  1265. return pollflag;
  1266. }
  1267. static __poll_t poll_next(struct file *fp,
  1268. struct poll_table_struct *pt)
  1269. {
  1270. struct hfi1_filedata *fd = fp->private_data;
  1271. struct hfi1_ctxtdata *uctxt = fd->uctxt;
  1272. struct hfi1_devdata *dd = uctxt->dd;
  1273. __poll_t pollflag;
  1274. poll_wait(fp, &uctxt->wait, pt);
  1275. spin_lock_irq(&dd->uctxt_lock);
  1276. if (hdrqempty(uctxt)) {
  1277. set_bit(HFI1_CTXT_WAITING_RCV, &uctxt->event_flags);
  1278. hfi1_rcvctrl(dd, HFI1_RCVCTRL_INTRAVAIL_ENB, uctxt);
  1279. pollflag = 0;
  1280. } else {
  1281. pollflag = EPOLLIN | EPOLLRDNORM;
  1282. }
  1283. spin_unlock_irq(&dd->uctxt_lock);
  1284. return pollflag;
  1285. }
  1286. /*
  1287. * Find all user contexts in use, and set the specified bit in their
  1288. * event mask.
  1289. * See also find_ctxt() for a similar use, that is specific to send buffers.
  1290. */
  1291. int hfi1_set_uevent_bits(struct hfi1_pportdata *ppd, const int evtbit)
  1292. {
  1293. struct hfi1_ctxtdata *uctxt;
  1294. struct hfi1_devdata *dd = ppd->dd;
  1295. u16 ctxt;
  1296. if (!dd->events)
  1297. return -EINVAL;
  1298. for (ctxt = dd->first_dyn_alloc_ctxt; ctxt < dd->num_rcv_contexts;
  1299. ctxt++) {
  1300. uctxt = hfi1_rcd_get_by_index(dd, ctxt);
  1301. if (uctxt) {
  1302. unsigned long *evs;
  1303. int i;
  1304. /*
  1305. * subctxt_cnt is 0 if not shared, so do base
  1306. * separately, first, then remaining subctxt, if any
  1307. */
  1308. evs = dd->events + uctxt_offset(uctxt);
  1309. set_bit(evtbit, evs);
  1310. for (i = 1; i < uctxt->subctxt_cnt; i++)
  1311. set_bit(evtbit, evs + i);
  1312. hfi1_rcd_put(uctxt);
  1313. }
  1314. }
  1315. return 0;
  1316. }
  1317. /**
  1318. * manage_rcvq - manage a context's receive queue
  1319. * @uctxt: the context
  1320. * @subctxt: the sub-context
  1321. * @start_stop: action to carry out
  1322. *
  1323. * start_stop == 0 disables receive on the context, for use in queue
  1324. * overflow conditions. start_stop==1 re-enables, to be used to
  1325. * re-init the software copy of the head register
  1326. */
  1327. static int manage_rcvq(struct hfi1_ctxtdata *uctxt, u16 subctxt,
  1328. unsigned long arg)
  1329. {
  1330. struct hfi1_devdata *dd = uctxt->dd;
  1331. unsigned int rcvctrl_op;
  1332. int start_stop;
  1333. if (subctxt)
  1334. return 0;
  1335. if (get_user(start_stop, (int __user *)arg))
  1336. return -EFAULT;
  1337. /* atomically clear receive enable ctxt. */
  1338. if (start_stop) {
  1339. /*
  1340. * On enable, force in-memory copy of the tail register to
  1341. * 0, so that protocol code doesn't have to worry about
  1342. * whether or not the chip has yet updated the in-memory
  1343. * copy or not on return from the system call. The chip
  1344. * always resets it's tail register back to 0 on a
  1345. * transition from disabled to enabled.
  1346. */
  1347. if (uctxt->rcvhdrtail_kvaddr)
  1348. clear_rcvhdrtail(uctxt);
  1349. rcvctrl_op = HFI1_RCVCTRL_CTXT_ENB;
  1350. } else {
  1351. rcvctrl_op = HFI1_RCVCTRL_CTXT_DIS;
  1352. }
  1353. hfi1_rcvctrl(dd, rcvctrl_op, uctxt);
  1354. /* always; new head should be equal to new tail; see above */
  1355. return 0;
  1356. }
  1357. /*
  1358. * clear the event notifier events for this context.
  1359. * User process then performs actions appropriate to bit having been
  1360. * set, if desired, and checks again in future.
  1361. */
  1362. static int user_event_ack(struct hfi1_ctxtdata *uctxt, u16 subctxt,
  1363. unsigned long arg)
  1364. {
  1365. int i;
  1366. struct hfi1_devdata *dd = uctxt->dd;
  1367. unsigned long *evs;
  1368. unsigned long events;
  1369. if (!dd->events)
  1370. return 0;
  1371. if (get_user(events, (unsigned long __user *)arg))
  1372. return -EFAULT;
  1373. evs = dd->events + uctxt_offset(uctxt) + subctxt;
  1374. for (i = 0; i <= _HFI1_MAX_EVENT_BIT; i++) {
  1375. if (!test_bit(i, &events))
  1376. continue;
  1377. clear_bit(i, evs);
  1378. }
  1379. return 0;
  1380. }
  1381. static int set_ctxt_pkey(struct hfi1_ctxtdata *uctxt, unsigned long arg)
  1382. {
  1383. int i;
  1384. struct hfi1_pportdata *ppd = uctxt->ppd;
  1385. struct hfi1_devdata *dd = uctxt->dd;
  1386. u16 pkey;
  1387. if (!HFI1_CAP_IS_USET(PKEY_CHECK))
  1388. return -EPERM;
  1389. if (get_user(pkey, (u16 __user *)arg))
  1390. return -EFAULT;
  1391. if (pkey == LIM_MGMT_P_KEY || pkey == FULL_MGMT_P_KEY)
  1392. return -EINVAL;
  1393. for (i = 0; i < ARRAY_SIZE(ppd->pkeys); i++)
  1394. if (pkey == ppd->pkeys[i])
  1395. return hfi1_set_ctxt_pkey(dd, uctxt, pkey);
  1396. return -ENOENT;
  1397. }
  1398. /**
  1399. * ctxt_reset - Reset the user context
  1400. * @uctxt: valid user context
  1401. */
  1402. static int ctxt_reset(struct hfi1_ctxtdata *uctxt)
  1403. {
  1404. struct send_context *sc;
  1405. struct hfi1_devdata *dd;
  1406. int ret = 0;
  1407. if (!uctxt || !uctxt->dd || !uctxt->sc)
  1408. return -EINVAL;
  1409. /*
  1410. * There is no protection here. User level has to guarantee that
  1411. * no one will be writing to the send context while it is being
  1412. * re-initialized. If user level breaks that guarantee, it will
  1413. * break it's own context and no one else's.
  1414. */
  1415. dd = uctxt->dd;
  1416. sc = uctxt->sc;
  1417. /*
  1418. * Wait until the interrupt handler has marked the context as
  1419. * halted or frozen. Report error if we time out.
  1420. */
  1421. wait_event_interruptible_timeout(
  1422. sc->halt_wait, (sc->flags & SCF_HALTED),
  1423. msecs_to_jiffies(SEND_CTXT_HALT_TIMEOUT));
  1424. if (!(sc->flags & SCF_HALTED))
  1425. return -ENOLCK;
  1426. /*
  1427. * If the send context was halted due to a Freeze, wait until the
  1428. * device has been "unfrozen" before resetting the context.
  1429. */
  1430. if (sc->flags & SCF_FROZEN) {
  1431. wait_event_interruptible_timeout(
  1432. dd->event_queue,
  1433. !(READ_ONCE(dd->flags) & HFI1_FROZEN),
  1434. msecs_to_jiffies(SEND_CTXT_HALT_TIMEOUT));
  1435. if (dd->flags & HFI1_FROZEN)
  1436. return -ENOLCK;
  1437. if (dd->flags & HFI1_FORCED_FREEZE)
  1438. /*
  1439. * Don't allow context reset if we are into
  1440. * forced freeze
  1441. */
  1442. return -ENODEV;
  1443. sc_disable(sc);
  1444. ret = sc_enable(sc);
  1445. hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_ENB, uctxt);
  1446. } else {
  1447. ret = sc_restart(sc);
  1448. }
  1449. if (!ret)
  1450. sc_return_credits(sc);
  1451. return ret;
  1452. }
  1453. static void user_remove(struct hfi1_devdata *dd)
  1454. {
  1455. hfi1_cdev_cleanup(&dd->user_cdev, &dd->user_device);
  1456. }
  1457. static int user_add(struct hfi1_devdata *dd)
  1458. {
  1459. char name[10];
  1460. int ret;
  1461. snprintf(name, sizeof(name), "%s_%d", class_name(), dd->unit);
  1462. ret = hfi1_cdev_init(dd->unit, name, &hfi1_file_ops,
  1463. &dd->user_cdev, &dd->user_device,
  1464. true, &dd->kobj);
  1465. if (ret)
  1466. user_remove(dd);
  1467. return ret;
  1468. }
  1469. /*
  1470. * Create per-unit files in /dev
  1471. */
  1472. int hfi1_device_create(struct hfi1_devdata *dd)
  1473. {
  1474. return user_add(dd);
  1475. }
  1476. /*
  1477. * Remove per-unit files in /dev
  1478. * void, core kernel returns no errors for this stuff
  1479. */
  1480. void hfi1_device_remove(struct hfi1_devdata *dd)
  1481. {
  1482. user_remove(dd);
  1483. }