affinity.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748
  1. /*
  2. * Copyright(c) 2015, 2016 Intel Corporation.
  3. *
  4. * This file is provided under a dual BSD/GPLv2 license. When using or
  5. * redistributing this file, you may do so under either license.
  6. *
  7. * GPL LICENSE SUMMARY
  8. *
  9. * This program is free software; you can redistribute it and/or modify
  10. * it under the terms of version 2 of the GNU General Public License as
  11. * published by the Free Software Foundation.
  12. *
  13. * This program is distributed in the hope that it will be useful, but
  14. * WITHOUT ANY WARRANTY; without even the implied warranty of
  15. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16. * General Public License for more details.
  17. *
  18. * BSD LICENSE
  19. *
  20. * Redistribution and use in source and binary forms, with or without
  21. * modification, are permitted provided that the following conditions
  22. * are met:
  23. *
  24. * - Redistributions of source code must retain the above copyright
  25. * notice, this list of conditions and the following disclaimer.
  26. * - Redistributions in binary form must reproduce the above copyright
  27. * notice, this list of conditions and the following disclaimer in
  28. * the documentation and/or other materials provided with the
  29. * distribution.
  30. * - Neither the name of Intel Corporation nor the names of its
  31. * contributors may be used to endorse or promote products derived
  32. * from this software without specific prior written permission.
  33. *
  34. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  35. * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  36. * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  37. * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  38. * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  39. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  40. * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  41. * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  42. * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  43. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  44. * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  45. *
  46. */
  47. #include <linux/topology.h>
  48. #include <linux/cpumask.h>
  49. #include <linux/module.h>
  50. #include "hfi.h"
  51. #include "affinity.h"
  52. #include "sdma.h"
  53. #include "trace.h"
  54. struct hfi1_affinity_node_list node_affinity = {
  55. .list = LIST_HEAD_INIT(node_affinity.list),
  56. .lock = __SPIN_LOCK_UNLOCKED(&node_affinity.lock),
  57. };
  58. /* Name of IRQ types, indexed by enum irq_type */
  59. static const char * const irq_type_names[] = {
  60. "SDMA",
  61. "RCVCTXT",
  62. "GENERAL",
  63. "OTHER",
  64. };
  65. /* Per NUMA node count of HFI devices */
  66. static unsigned int *hfi1_per_node_cntr;
  67. static inline void init_cpu_mask_set(struct cpu_mask_set *set)
  68. {
  69. cpumask_clear(&set->mask);
  70. cpumask_clear(&set->used);
  71. set->gen = 0;
  72. }
  73. /* Initialize non-HT cpu cores mask */
  74. void init_real_cpu_mask(void)
  75. {
  76. int possible, curr_cpu, i, ht;
  77. cpumask_clear(&node_affinity.real_cpu_mask);
  78. /* Start with cpu online mask as the real cpu mask */
  79. cpumask_copy(&node_affinity.real_cpu_mask, cpu_online_mask);
  80. /*
  81. * Remove HT cores from the real cpu mask. Do this in two steps below.
  82. */
  83. possible = cpumask_weight(&node_affinity.real_cpu_mask);
  84. ht = cpumask_weight(topology_sibling_cpumask(
  85. cpumask_first(&node_affinity.real_cpu_mask)));
  86. /*
  87. * Step 1. Skip over the first N HT siblings and use them as the
  88. * "real" cores. Assumes that HT cores are not enumerated in
  89. * succession (except in the single core case).
  90. */
  91. curr_cpu = cpumask_first(&node_affinity.real_cpu_mask);
  92. for (i = 0; i < possible / ht; i++)
  93. curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
  94. /*
  95. * Step 2. Remove the remaining HT siblings. Use cpumask_next() to
  96. * skip any gaps.
  97. */
  98. for (; i < possible; i++) {
  99. cpumask_clear_cpu(curr_cpu, &node_affinity.real_cpu_mask);
  100. curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
  101. }
  102. }
  103. int node_affinity_init(void)
  104. {
  105. int node;
  106. struct pci_dev *dev = NULL;
  107. const struct pci_device_id *ids = hfi1_pci_tbl;
  108. cpumask_clear(&node_affinity.proc.used);
  109. cpumask_copy(&node_affinity.proc.mask, cpu_online_mask);
  110. node_affinity.proc.gen = 0;
  111. node_affinity.num_core_siblings =
  112. cpumask_weight(topology_sibling_cpumask(
  113. cpumask_first(&node_affinity.proc.mask)
  114. ));
  115. node_affinity.num_online_nodes = num_online_nodes();
  116. node_affinity.num_online_cpus = num_online_cpus();
  117. /*
  118. * The real cpu mask is part of the affinity struct but it has to be
  119. * initialized early. It is needed to calculate the number of user
  120. * contexts in set_up_context_variables().
  121. */
  122. init_real_cpu_mask();
  123. hfi1_per_node_cntr = kcalloc(num_possible_nodes(),
  124. sizeof(*hfi1_per_node_cntr), GFP_KERNEL);
  125. if (!hfi1_per_node_cntr)
  126. return -ENOMEM;
  127. while (ids->vendor) {
  128. dev = NULL;
  129. while ((dev = pci_get_device(ids->vendor, ids->device, dev))) {
  130. node = pcibus_to_node(dev->bus);
  131. if (node < 0)
  132. node = numa_node_id();
  133. hfi1_per_node_cntr[node]++;
  134. }
  135. ids++;
  136. }
  137. return 0;
  138. }
  139. void node_affinity_destroy(void)
  140. {
  141. struct list_head *pos, *q;
  142. struct hfi1_affinity_node *entry;
  143. spin_lock(&node_affinity.lock);
  144. list_for_each_safe(pos, q, &node_affinity.list) {
  145. entry = list_entry(pos, struct hfi1_affinity_node,
  146. list);
  147. list_del(pos);
  148. kfree(entry);
  149. }
  150. spin_unlock(&node_affinity.lock);
  151. kfree(hfi1_per_node_cntr);
  152. }
  153. static struct hfi1_affinity_node *node_affinity_allocate(int node)
  154. {
  155. struct hfi1_affinity_node *entry;
  156. entry = kzalloc(sizeof(*entry), GFP_KERNEL);
  157. if (!entry)
  158. return NULL;
  159. entry->node = node;
  160. INIT_LIST_HEAD(&entry->list);
  161. return entry;
  162. }
  163. /*
  164. * It appends an entry to the list.
  165. * It *must* be called with node_affinity.lock held.
  166. */
  167. static void node_affinity_add_tail(struct hfi1_affinity_node *entry)
  168. {
  169. list_add_tail(&entry->list, &node_affinity.list);
  170. }
  171. /* It must be called with node_affinity.lock held */
  172. static struct hfi1_affinity_node *node_affinity_lookup(int node)
  173. {
  174. struct list_head *pos;
  175. struct hfi1_affinity_node *entry;
  176. list_for_each(pos, &node_affinity.list) {
  177. entry = list_entry(pos, struct hfi1_affinity_node, list);
  178. if (entry->node == node)
  179. return entry;
  180. }
  181. return NULL;
  182. }
  183. /*
  184. * Interrupt affinity.
  185. *
  186. * non-rcv avail gets a default mask that
  187. * starts as possible cpus with threads reset
  188. * and each rcv avail reset.
  189. *
  190. * rcv avail gets node relative 1 wrapping back
  191. * to the node relative 1 as necessary.
  192. *
  193. */
  194. int hfi1_dev_affinity_init(struct hfi1_devdata *dd)
  195. {
  196. int node = pcibus_to_node(dd->pcidev->bus);
  197. struct hfi1_affinity_node *entry;
  198. const struct cpumask *local_mask;
  199. int curr_cpu, possible, i;
  200. if (node < 0)
  201. node = numa_node_id();
  202. dd->node = node;
  203. local_mask = cpumask_of_node(dd->node);
  204. if (cpumask_first(local_mask) >= nr_cpu_ids)
  205. local_mask = topology_core_cpumask(0);
  206. spin_lock(&node_affinity.lock);
  207. entry = node_affinity_lookup(dd->node);
  208. spin_unlock(&node_affinity.lock);
  209. /*
  210. * If this is the first time this NUMA node's affinity is used,
  211. * create an entry in the global affinity structure and initialize it.
  212. */
  213. if (!entry) {
  214. entry = node_affinity_allocate(node);
  215. if (!entry) {
  216. dd_dev_err(dd,
  217. "Unable to allocate global affinity node\n");
  218. return -ENOMEM;
  219. }
  220. init_cpu_mask_set(&entry->def_intr);
  221. init_cpu_mask_set(&entry->rcv_intr);
  222. cpumask_clear(&entry->general_intr_mask);
  223. /* Use the "real" cpu mask of this node as the default */
  224. cpumask_and(&entry->def_intr.mask, &node_affinity.real_cpu_mask,
  225. local_mask);
  226. /* fill in the receive list */
  227. possible = cpumask_weight(&entry->def_intr.mask);
  228. curr_cpu = cpumask_first(&entry->def_intr.mask);
  229. if (possible == 1) {
  230. /* only one CPU, everyone will use it */
  231. cpumask_set_cpu(curr_cpu, &entry->rcv_intr.mask);
  232. cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
  233. } else {
  234. /*
  235. * The general/control context will be the first CPU in
  236. * the default list, so it is removed from the default
  237. * list and added to the general interrupt list.
  238. */
  239. cpumask_clear_cpu(curr_cpu, &entry->def_intr.mask);
  240. cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
  241. curr_cpu = cpumask_next(curr_cpu,
  242. &entry->def_intr.mask);
  243. /*
  244. * Remove the remaining kernel receive queues from
  245. * the default list and add them to the receive list.
  246. */
  247. for (i = 0;
  248. i < (dd->n_krcv_queues - 1) *
  249. hfi1_per_node_cntr[dd->node];
  250. i++) {
  251. cpumask_clear_cpu(curr_cpu,
  252. &entry->def_intr.mask);
  253. cpumask_set_cpu(curr_cpu,
  254. &entry->rcv_intr.mask);
  255. curr_cpu = cpumask_next(curr_cpu,
  256. &entry->def_intr.mask);
  257. if (curr_cpu >= nr_cpu_ids)
  258. break;
  259. }
  260. /*
  261. * If there ends up being 0 CPU cores leftover for SDMA
  262. * engines, use the same CPU cores as general/control
  263. * context.
  264. */
  265. if (cpumask_weight(&entry->def_intr.mask) == 0)
  266. cpumask_copy(&entry->def_intr.mask,
  267. &entry->general_intr_mask);
  268. }
  269. spin_lock(&node_affinity.lock);
  270. node_affinity_add_tail(entry);
  271. spin_unlock(&node_affinity.lock);
  272. }
  273. return 0;
  274. }
  275. int hfi1_get_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix)
  276. {
  277. int ret;
  278. cpumask_var_t diff;
  279. struct hfi1_affinity_node *entry;
  280. struct cpu_mask_set *set = NULL;
  281. struct sdma_engine *sde = NULL;
  282. struct hfi1_ctxtdata *rcd = NULL;
  283. char extra[64];
  284. int cpu = -1;
  285. extra[0] = '\0';
  286. cpumask_clear(&msix->mask);
  287. ret = zalloc_cpumask_var(&diff, GFP_KERNEL);
  288. if (!ret)
  289. return -ENOMEM;
  290. spin_lock(&node_affinity.lock);
  291. entry = node_affinity_lookup(dd->node);
  292. spin_unlock(&node_affinity.lock);
  293. switch (msix->type) {
  294. case IRQ_SDMA:
  295. sde = (struct sdma_engine *)msix->arg;
  296. scnprintf(extra, 64, "engine %u", sde->this_idx);
  297. set = &entry->def_intr;
  298. break;
  299. case IRQ_GENERAL:
  300. cpu = cpumask_first(&entry->general_intr_mask);
  301. break;
  302. case IRQ_RCVCTXT:
  303. rcd = (struct hfi1_ctxtdata *)msix->arg;
  304. if (rcd->ctxt == HFI1_CTRL_CTXT)
  305. cpu = cpumask_first(&entry->general_intr_mask);
  306. else
  307. set = &entry->rcv_intr;
  308. scnprintf(extra, 64, "ctxt %u", rcd->ctxt);
  309. break;
  310. default:
  311. dd_dev_err(dd, "Invalid IRQ type %d\n", msix->type);
  312. return -EINVAL;
  313. }
  314. /*
  315. * The general and control contexts are placed on a particular
  316. * CPU, which is set above. Skip accounting for it. Everything else
  317. * finds its CPU here.
  318. */
  319. if (cpu == -1 && set) {
  320. spin_lock(&node_affinity.lock);
  321. if (cpumask_equal(&set->mask, &set->used)) {
  322. /*
  323. * We've used up all the CPUs, bump up the generation
  324. * and reset the 'used' map
  325. */
  326. set->gen++;
  327. cpumask_clear(&set->used);
  328. }
  329. cpumask_andnot(diff, &set->mask, &set->used);
  330. cpu = cpumask_first(diff);
  331. cpumask_set_cpu(cpu, &set->used);
  332. spin_unlock(&node_affinity.lock);
  333. }
  334. switch (msix->type) {
  335. case IRQ_SDMA:
  336. sde->cpu = cpu;
  337. break;
  338. case IRQ_GENERAL:
  339. case IRQ_RCVCTXT:
  340. case IRQ_OTHER:
  341. break;
  342. }
  343. cpumask_set_cpu(cpu, &msix->mask);
  344. dd_dev_info(dd, "IRQ vector: %u, type %s %s -> cpu: %d\n",
  345. msix->msix.vector, irq_type_names[msix->type],
  346. extra, cpu);
  347. irq_set_affinity_hint(msix->msix.vector, &msix->mask);
  348. free_cpumask_var(diff);
  349. return 0;
  350. }
  351. void hfi1_put_irq_affinity(struct hfi1_devdata *dd,
  352. struct hfi1_msix_entry *msix)
  353. {
  354. struct cpu_mask_set *set = NULL;
  355. struct hfi1_ctxtdata *rcd;
  356. struct hfi1_affinity_node *entry;
  357. spin_lock(&node_affinity.lock);
  358. entry = node_affinity_lookup(dd->node);
  359. spin_unlock(&node_affinity.lock);
  360. switch (msix->type) {
  361. case IRQ_SDMA:
  362. set = &entry->def_intr;
  363. break;
  364. case IRQ_GENERAL:
  365. /* Don't do accounting for general contexts */
  366. break;
  367. case IRQ_RCVCTXT:
  368. rcd = (struct hfi1_ctxtdata *)msix->arg;
  369. /* Don't do accounting for control contexts */
  370. if (rcd->ctxt != HFI1_CTRL_CTXT)
  371. set = &entry->rcv_intr;
  372. break;
  373. default:
  374. return;
  375. }
  376. if (set) {
  377. spin_lock(&node_affinity.lock);
  378. cpumask_andnot(&set->used, &set->used, &msix->mask);
  379. if (cpumask_empty(&set->used) && set->gen) {
  380. set->gen--;
  381. cpumask_copy(&set->used, &set->mask);
  382. }
  383. spin_unlock(&node_affinity.lock);
  384. }
  385. irq_set_affinity_hint(msix->msix.vector, NULL);
  386. cpumask_clear(&msix->mask);
  387. }
  388. /* This should be called with node_affinity.lock held */
  389. static void find_hw_thread_mask(uint hw_thread_no, cpumask_var_t hw_thread_mask,
  390. struct hfi1_affinity_node_list *affinity)
  391. {
  392. int possible, curr_cpu, i;
  393. uint num_cores_per_socket = node_affinity.num_online_cpus /
  394. affinity->num_core_siblings /
  395. node_affinity.num_online_nodes;
  396. cpumask_copy(hw_thread_mask, &affinity->proc.mask);
  397. if (affinity->num_core_siblings > 0) {
  398. /* Removing other siblings not needed for now */
  399. possible = cpumask_weight(hw_thread_mask);
  400. curr_cpu = cpumask_first(hw_thread_mask);
  401. for (i = 0;
  402. i < num_cores_per_socket * node_affinity.num_online_nodes;
  403. i++)
  404. curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
  405. for (; i < possible; i++) {
  406. cpumask_clear_cpu(curr_cpu, hw_thread_mask);
  407. curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
  408. }
  409. /* Identifying correct HW threads within physical cores */
  410. cpumask_shift_left(hw_thread_mask, hw_thread_mask,
  411. num_cores_per_socket *
  412. node_affinity.num_online_nodes *
  413. hw_thread_no);
  414. }
  415. }
  416. int hfi1_get_proc_affinity(int node)
  417. {
  418. int cpu = -1, ret, i;
  419. struct hfi1_affinity_node *entry;
  420. cpumask_var_t diff, hw_thread_mask, available_mask, intrs_mask;
  421. const struct cpumask *node_mask,
  422. *proc_mask = tsk_cpus_allowed(current);
  423. struct hfi1_affinity_node_list *affinity = &node_affinity;
  424. struct cpu_mask_set *set = &affinity->proc;
  425. /*
  426. * check whether process/context affinity has already
  427. * been set
  428. */
  429. if (cpumask_weight(proc_mask) == 1) {
  430. hfi1_cdbg(PROC, "PID %u %s affinity set to CPU %*pbl",
  431. current->pid, current->comm,
  432. cpumask_pr_args(proc_mask));
  433. /*
  434. * Mark the pre-set CPU as used. This is atomic so we don't
  435. * need the lock
  436. */
  437. cpu = cpumask_first(proc_mask);
  438. cpumask_set_cpu(cpu, &set->used);
  439. goto done;
  440. } else if (cpumask_weight(proc_mask) < cpumask_weight(&set->mask)) {
  441. hfi1_cdbg(PROC, "PID %u %s affinity set to CPU set(s) %*pbl",
  442. current->pid, current->comm,
  443. cpumask_pr_args(proc_mask));
  444. goto done;
  445. }
  446. /*
  447. * The process does not have a preset CPU affinity so find one to
  448. * recommend using the following algorithm:
  449. *
  450. * For each user process that is opening a context on HFI Y:
  451. * a) If all cores are filled, reinitialize the bitmask
  452. * b) Fill real cores first, then HT cores (First set of HT
  453. * cores on all physical cores, then second set of HT core,
  454. * and, so on) in the following order:
  455. *
  456. * 1. Same NUMA node as HFI Y and not running an IRQ
  457. * handler
  458. * 2. Same NUMA node as HFI Y and running an IRQ handler
  459. * 3. Different NUMA node to HFI Y and not running an IRQ
  460. * handler
  461. * 4. Different NUMA node to HFI Y and running an IRQ
  462. * handler
  463. * c) Mark core as filled in the bitmask. As user processes are
  464. * done, clear cores from the bitmask.
  465. */
  466. ret = zalloc_cpumask_var(&diff, GFP_KERNEL);
  467. if (!ret)
  468. goto done;
  469. ret = zalloc_cpumask_var(&hw_thread_mask, GFP_KERNEL);
  470. if (!ret)
  471. goto free_diff;
  472. ret = zalloc_cpumask_var(&available_mask, GFP_KERNEL);
  473. if (!ret)
  474. goto free_hw_thread_mask;
  475. ret = zalloc_cpumask_var(&intrs_mask, GFP_KERNEL);
  476. if (!ret)
  477. goto free_available_mask;
  478. spin_lock(&affinity->lock);
  479. /*
  480. * If we've used all available HW threads, clear the mask and start
  481. * overloading.
  482. */
  483. if (cpumask_equal(&set->mask, &set->used)) {
  484. set->gen++;
  485. cpumask_clear(&set->used);
  486. }
  487. /*
  488. * If NUMA node has CPUs used by interrupt handlers, include them in the
  489. * interrupt handler mask.
  490. */
  491. entry = node_affinity_lookup(node);
  492. if (entry) {
  493. cpumask_copy(intrs_mask, (entry->def_intr.gen ?
  494. &entry->def_intr.mask :
  495. &entry->def_intr.used));
  496. cpumask_or(intrs_mask, intrs_mask, (entry->rcv_intr.gen ?
  497. &entry->rcv_intr.mask :
  498. &entry->rcv_intr.used));
  499. cpumask_or(intrs_mask, intrs_mask, &entry->general_intr_mask);
  500. }
  501. hfi1_cdbg(PROC, "CPUs used by interrupts: %*pbl",
  502. cpumask_pr_args(intrs_mask));
  503. cpumask_copy(hw_thread_mask, &set->mask);
  504. /*
  505. * If HT cores are enabled, identify which HW threads within the
  506. * physical cores should be used.
  507. */
  508. if (affinity->num_core_siblings > 0) {
  509. for (i = 0; i < affinity->num_core_siblings; i++) {
  510. find_hw_thread_mask(i, hw_thread_mask, affinity);
  511. /*
  512. * If there's at least one available core for this HW
  513. * thread number, stop looking for a core.
  514. *
  515. * diff will always be not empty at least once in this
  516. * loop as the used mask gets reset when
  517. * (set->mask == set->used) before this loop.
  518. */
  519. cpumask_andnot(diff, hw_thread_mask, &set->used);
  520. if (!cpumask_empty(diff))
  521. break;
  522. }
  523. }
  524. hfi1_cdbg(PROC, "Same available HW thread on all physical CPUs: %*pbl",
  525. cpumask_pr_args(hw_thread_mask));
  526. node_mask = cpumask_of_node(node);
  527. hfi1_cdbg(PROC, "Device on NUMA %u, CPUs %*pbl", node,
  528. cpumask_pr_args(node_mask));
  529. /* Get cpumask of available CPUs on preferred NUMA */
  530. cpumask_and(available_mask, hw_thread_mask, node_mask);
  531. cpumask_andnot(available_mask, available_mask, &set->used);
  532. hfi1_cdbg(PROC, "Available CPUs on NUMA %u: %*pbl", node,
  533. cpumask_pr_args(available_mask));
  534. /*
  535. * At first, we don't want to place processes on the same
  536. * CPUs as interrupt handlers. Then, CPUs running interrupt
  537. * handlers are used.
  538. *
  539. * 1) If diff is not empty, then there are CPUs not running
  540. * non-interrupt handlers available, so diff gets copied
  541. * over to available_mask.
  542. * 2) If diff is empty, then all CPUs not running interrupt
  543. * handlers are taken, so available_mask contains all
  544. * available CPUs running interrupt handlers.
  545. * 3) If available_mask is empty, then all CPUs on the
  546. * preferred NUMA node are taken, so other NUMA nodes are
  547. * used for process assignments using the same method as
  548. * the preferred NUMA node.
  549. */
  550. cpumask_andnot(diff, available_mask, intrs_mask);
  551. if (!cpumask_empty(diff))
  552. cpumask_copy(available_mask, diff);
  553. /* If we don't have CPUs on the preferred node, use other NUMA nodes */
  554. if (cpumask_empty(available_mask)) {
  555. cpumask_andnot(available_mask, hw_thread_mask, &set->used);
  556. /* Excluding preferred NUMA cores */
  557. cpumask_andnot(available_mask, available_mask, node_mask);
  558. hfi1_cdbg(PROC,
  559. "Preferred NUMA node cores are taken, cores available in other NUMA nodes: %*pbl",
  560. cpumask_pr_args(available_mask));
  561. /*
  562. * At first, we don't want to place processes on the same
  563. * CPUs as interrupt handlers.
  564. */
  565. cpumask_andnot(diff, available_mask, intrs_mask);
  566. if (!cpumask_empty(diff))
  567. cpumask_copy(available_mask, diff);
  568. }
  569. hfi1_cdbg(PROC, "Possible CPUs for process: %*pbl",
  570. cpumask_pr_args(available_mask));
  571. cpu = cpumask_first(available_mask);
  572. if (cpu >= nr_cpu_ids) /* empty */
  573. cpu = -1;
  574. else
  575. cpumask_set_cpu(cpu, &set->used);
  576. spin_unlock(&affinity->lock);
  577. hfi1_cdbg(PROC, "Process assigned to CPU %d", cpu);
  578. free_cpumask_var(intrs_mask);
  579. free_available_mask:
  580. free_cpumask_var(available_mask);
  581. free_hw_thread_mask:
  582. free_cpumask_var(hw_thread_mask);
  583. free_diff:
  584. free_cpumask_var(diff);
  585. done:
  586. return cpu;
  587. }
  588. void hfi1_put_proc_affinity(int cpu)
  589. {
  590. struct hfi1_affinity_node_list *affinity = &node_affinity;
  591. struct cpu_mask_set *set = &affinity->proc;
  592. if (cpu < 0)
  593. return;
  594. spin_lock(&affinity->lock);
  595. cpumask_clear_cpu(cpu, &set->used);
  596. hfi1_cdbg(PROC, "Returning CPU %d for future process assignment", cpu);
  597. if (cpumask_empty(&set->used) && set->gen) {
  598. set->gen--;
  599. cpumask_copy(&set->used, &set->mask);
  600. }
  601. spin_unlock(&affinity->lock);
  602. }
  603. /* Prevents concurrent reads and writes of the sdma_affinity attrib */
  604. static DEFINE_MUTEX(sdma_affinity_mutex);
  605. int hfi1_set_sdma_affinity(struct hfi1_devdata *dd, const char *buf,
  606. size_t count)
  607. {
  608. struct hfi1_affinity_node *entry;
  609. cpumask_var_t mask;
  610. int ret, i;
  611. spin_lock(&node_affinity.lock);
  612. entry = node_affinity_lookup(dd->node);
  613. spin_unlock(&node_affinity.lock);
  614. if (!entry)
  615. return -EINVAL;
  616. ret = zalloc_cpumask_var(&mask, GFP_KERNEL);
  617. if (!ret)
  618. return -ENOMEM;
  619. ret = cpulist_parse(buf, mask);
  620. if (ret)
  621. goto out;
  622. if (!cpumask_subset(mask, cpu_online_mask) || cpumask_empty(mask)) {
  623. dd_dev_warn(dd, "Invalid CPU mask\n");
  624. ret = -EINVAL;
  625. goto out;
  626. }
  627. mutex_lock(&sdma_affinity_mutex);
  628. /* reset the SDMA interrupt affinity details */
  629. init_cpu_mask_set(&entry->def_intr);
  630. cpumask_copy(&entry->def_intr.mask, mask);
  631. /*
  632. * Reassign the affinity for each SDMA interrupt.
  633. */
  634. for (i = 0; i < dd->num_msix_entries; i++) {
  635. struct hfi1_msix_entry *msix;
  636. msix = &dd->msix_entries[i];
  637. if (msix->type != IRQ_SDMA)
  638. continue;
  639. ret = hfi1_get_irq_affinity(dd, msix);
  640. if (ret)
  641. break;
  642. }
  643. mutex_unlock(&sdma_affinity_mutex);
  644. out:
  645. free_cpumask_var(mask);
  646. return ret ? ret : strnlen(buf, PAGE_SIZE);
  647. }
  648. int hfi1_get_sdma_affinity(struct hfi1_devdata *dd, char *buf)
  649. {
  650. struct hfi1_affinity_node *entry;
  651. spin_lock(&node_affinity.lock);
  652. entry = node_affinity_lookup(dd->node);
  653. spin_unlock(&node_affinity.lock);
  654. if (!entry)
  655. return -EINVAL;
  656. mutex_lock(&sdma_affinity_mutex);
  657. cpumap_print_to_pagebuf(true, buf, &entry->def_intr.mask);
  658. mutex_unlock(&sdma_affinity_mutex);
  659. return strnlen(buf, PAGE_SIZE);
  660. }