irq-mips-gic.c 28 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141
  1. /*
  2. * This file is subject to the terms and conditions of the GNU General Public
  3. * License. See the file "COPYING" in the main directory of this archive
  4. * for more details.
  5. *
  6. * Copyright (C) 2008 Ralf Baechle (ralf@linux-mips.org)
  7. * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved.
  8. */
  9. #include <linux/bitmap.h>
  10. #include <linux/clocksource.h>
  11. #include <linux/init.h>
  12. #include <linux/interrupt.h>
  13. #include <linux/irq.h>
  14. #include <linux/irqchip.h>
  15. #include <linux/irqchip/mips-gic.h>
  16. #include <linux/of_address.h>
  17. #include <linux/sched.h>
  18. #include <linux/smp.h>
  19. #include <asm/mips-cm.h>
  20. #include <asm/setup.h>
  21. #include <asm/traps.h>
  22. #include <dt-bindings/interrupt-controller/mips-gic.h>
  23. unsigned int gic_present;
  24. struct gic_pcpu_mask {
  25. DECLARE_BITMAP(pcpu_mask, GIC_MAX_INTRS);
  26. };
  27. struct gic_irq_spec {
  28. enum {
  29. GIC_DEVICE,
  30. GIC_IPI
  31. } type;
  32. union {
  33. struct cpumask *ipimask;
  34. unsigned int hwirq;
  35. };
  36. };
  37. static unsigned long __gic_base_addr;
  38. static void __iomem *gic_base;
  39. static struct gic_pcpu_mask pcpu_masks[NR_CPUS];
  40. static DEFINE_SPINLOCK(gic_lock);
  41. static struct irq_domain *gic_irq_domain;
  42. static struct irq_domain *gic_dev_domain;
  43. static struct irq_domain *gic_ipi_domain;
  44. static int gic_shared_intrs;
  45. static int gic_vpes;
  46. static unsigned int gic_cpu_pin;
  47. static unsigned int timer_cpu_pin;
  48. static struct irq_chip gic_level_irq_controller, gic_edge_irq_controller;
  49. DECLARE_BITMAP(ipi_resrv, GIC_MAX_INTRS);
  50. static void __gic_irq_dispatch(void);
  51. static inline u32 gic_read32(unsigned int reg)
  52. {
  53. return __raw_readl(gic_base + reg);
  54. }
  55. static inline u64 gic_read64(unsigned int reg)
  56. {
  57. return __raw_readq(gic_base + reg);
  58. }
  59. static inline unsigned long gic_read(unsigned int reg)
  60. {
  61. if (!mips_cm_is64)
  62. return gic_read32(reg);
  63. else
  64. return gic_read64(reg);
  65. }
  66. static inline void gic_write32(unsigned int reg, u32 val)
  67. {
  68. return __raw_writel(val, gic_base + reg);
  69. }
  70. static inline void gic_write64(unsigned int reg, u64 val)
  71. {
  72. return __raw_writeq(val, gic_base + reg);
  73. }
  74. static inline void gic_write(unsigned int reg, unsigned long val)
  75. {
  76. if (!mips_cm_is64)
  77. return gic_write32(reg, (u32)val);
  78. else
  79. return gic_write64(reg, (u64)val);
  80. }
  81. static inline void gic_update_bits(unsigned int reg, unsigned long mask,
  82. unsigned long val)
  83. {
  84. unsigned long regval;
  85. regval = gic_read(reg);
  86. regval &= ~mask;
  87. regval |= val;
  88. gic_write(reg, regval);
  89. }
  90. static inline void gic_reset_mask(unsigned int intr)
  91. {
  92. gic_write(GIC_REG(SHARED, GIC_SH_RMASK) + GIC_INTR_OFS(intr),
  93. 1ul << GIC_INTR_BIT(intr));
  94. }
  95. static inline void gic_set_mask(unsigned int intr)
  96. {
  97. gic_write(GIC_REG(SHARED, GIC_SH_SMASK) + GIC_INTR_OFS(intr),
  98. 1ul << GIC_INTR_BIT(intr));
  99. }
  100. static inline void gic_set_polarity(unsigned int intr, unsigned int pol)
  101. {
  102. gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_POLARITY) +
  103. GIC_INTR_OFS(intr), 1ul << GIC_INTR_BIT(intr),
  104. (unsigned long)pol << GIC_INTR_BIT(intr));
  105. }
  106. static inline void gic_set_trigger(unsigned int intr, unsigned int trig)
  107. {
  108. gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_TRIGGER) +
  109. GIC_INTR_OFS(intr), 1ul << GIC_INTR_BIT(intr),
  110. (unsigned long)trig << GIC_INTR_BIT(intr));
  111. }
  112. static inline void gic_set_dual_edge(unsigned int intr, unsigned int dual)
  113. {
  114. gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_DUAL) + GIC_INTR_OFS(intr),
  115. 1ul << GIC_INTR_BIT(intr),
  116. (unsigned long)dual << GIC_INTR_BIT(intr));
  117. }
  118. static inline void gic_map_to_pin(unsigned int intr, unsigned int pin)
  119. {
  120. gic_write32(GIC_REG(SHARED, GIC_SH_INTR_MAP_TO_PIN_BASE) +
  121. GIC_SH_MAP_TO_PIN(intr), GIC_MAP_TO_PIN_MSK | pin);
  122. }
  123. static inline void gic_map_to_vpe(unsigned int intr, unsigned int vpe)
  124. {
  125. gic_write(GIC_REG(SHARED, GIC_SH_INTR_MAP_TO_VPE_BASE) +
  126. GIC_SH_MAP_TO_VPE_REG_OFF(intr, vpe),
  127. GIC_SH_MAP_TO_VPE_REG_BIT(vpe));
  128. }
  129. #ifdef CONFIG_CLKSRC_MIPS_GIC
  130. cycle_t gic_read_count(void)
  131. {
  132. unsigned int hi, hi2, lo;
  133. if (mips_cm_is64)
  134. return (cycle_t)gic_read(GIC_REG(SHARED, GIC_SH_COUNTER));
  135. do {
  136. hi = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_63_32));
  137. lo = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_31_00));
  138. hi2 = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_63_32));
  139. } while (hi2 != hi);
  140. return (((cycle_t) hi) << 32) + lo;
  141. }
  142. unsigned int gic_get_count_width(void)
  143. {
  144. unsigned int bits, config;
  145. config = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
  146. bits = 32 + 4 * ((config & GIC_SH_CONFIG_COUNTBITS_MSK) >>
  147. GIC_SH_CONFIG_COUNTBITS_SHF);
  148. return bits;
  149. }
  150. void gic_write_compare(cycle_t cnt)
  151. {
  152. if (mips_cm_is64) {
  153. gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE), cnt);
  154. } else {
  155. gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_HI),
  156. (int)(cnt >> 32));
  157. gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_LO),
  158. (int)(cnt & 0xffffffff));
  159. }
  160. }
  161. void gic_write_cpu_compare(cycle_t cnt, int cpu)
  162. {
  163. unsigned long flags;
  164. local_irq_save(flags);
  165. gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR), mips_cm_vp_id(cpu));
  166. if (mips_cm_is64) {
  167. gic_write(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE), cnt);
  168. } else {
  169. gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_HI),
  170. (int)(cnt >> 32));
  171. gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_LO),
  172. (int)(cnt & 0xffffffff));
  173. }
  174. local_irq_restore(flags);
  175. }
  176. cycle_t gic_read_compare(void)
  177. {
  178. unsigned int hi, lo;
  179. if (mips_cm_is64)
  180. return (cycle_t)gic_read(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE));
  181. hi = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_HI));
  182. lo = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_LO));
  183. return (((cycle_t) hi) << 32) + lo;
  184. }
  185. void gic_start_count(void)
  186. {
  187. u32 gicconfig;
  188. /* Start the counter */
  189. gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
  190. gicconfig &= ~(1 << GIC_SH_CONFIG_COUNTSTOP_SHF);
  191. gic_write(GIC_REG(SHARED, GIC_SH_CONFIG), gicconfig);
  192. }
  193. void gic_stop_count(void)
  194. {
  195. u32 gicconfig;
  196. /* Stop the counter */
  197. gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
  198. gicconfig |= 1 << GIC_SH_CONFIG_COUNTSTOP_SHF;
  199. gic_write(GIC_REG(SHARED, GIC_SH_CONFIG), gicconfig);
  200. }
  201. #endif
  202. unsigned gic_read_local_vp_id(void)
  203. {
  204. unsigned long ident;
  205. ident = gic_read(GIC_REG(VPE_LOCAL, GIC_VP_IDENT));
  206. return ident & GIC_VP_IDENT_VCNUM_MSK;
  207. }
  208. static bool gic_local_irq_is_routable(int intr)
  209. {
  210. u32 vpe_ctl;
  211. /* All local interrupts are routable in EIC mode. */
  212. if (cpu_has_veic)
  213. return true;
  214. vpe_ctl = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_CTL));
  215. switch (intr) {
  216. case GIC_LOCAL_INT_TIMER:
  217. return vpe_ctl & GIC_VPE_CTL_TIMER_RTBL_MSK;
  218. case GIC_LOCAL_INT_PERFCTR:
  219. return vpe_ctl & GIC_VPE_CTL_PERFCNT_RTBL_MSK;
  220. case GIC_LOCAL_INT_FDC:
  221. return vpe_ctl & GIC_VPE_CTL_FDC_RTBL_MSK;
  222. case GIC_LOCAL_INT_SWINT0:
  223. case GIC_LOCAL_INT_SWINT1:
  224. return vpe_ctl & GIC_VPE_CTL_SWINT_RTBL_MSK;
  225. default:
  226. return true;
  227. }
  228. }
  229. static void gic_bind_eic_interrupt(int irq, int set)
  230. {
  231. /* Convert irq vector # to hw int # */
  232. irq -= GIC_PIN_TO_VEC_OFFSET;
  233. /* Set irq to use shadow set */
  234. gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_EIC_SHADOW_SET_BASE) +
  235. GIC_VPE_EIC_SS(irq), set);
  236. }
  237. static void gic_send_ipi(struct irq_data *d, unsigned int cpu)
  238. {
  239. irq_hw_number_t hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(d));
  240. gic_write(GIC_REG(SHARED, GIC_SH_WEDGE), GIC_SH_WEDGE_SET(hwirq));
  241. }
  242. int gic_get_c0_compare_int(void)
  243. {
  244. if (!gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER))
  245. return MIPS_CPU_IRQ_BASE + cp0_compare_irq;
  246. return irq_create_mapping(gic_irq_domain,
  247. GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_TIMER));
  248. }
  249. int gic_get_c0_perfcount_int(void)
  250. {
  251. if (!gic_local_irq_is_routable(GIC_LOCAL_INT_PERFCTR)) {
  252. /* Is the performance counter shared with the timer? */
  253. if (cp0_perfcount_irq < 0)
  254. return -1;
  255. return MIPS_CPU_IRQ_BASE + cp0_perfcount_irq;
  256. }
  257. return irq_create_mapping(gic_irq_domain,
  258. GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_PERFCTR));
  259. }
  260. int gic_get_c0_fdc_int(void)
  261. {
  262. if (!gic_local_irq_is_routable(GIC_LOCAL_INT_FDC)) {
  263. /* Is the FDC IRQ even present? */
  264. if (cp0_fdc_irq < 0)
  265. return -1;
  266. return MIPS_CPU_IRQ_BASE + cp0_fdc_irq;
  267. }
  268. return irq_create_mapping(gic_irq_domain,
  269. GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_FDC));
  270. }
  271. int gic_get_usm_range(struct resource *gic_usm_res)
  272. {
  273. if (!gic_present)
  274. return -1;
  275. gic_usm_res->start = __gic_base_addr + USM_VISIBLE_SECTION_OFS;
  276. gic_usm_res->end = gic_usm_res->start + (USM_VISIBLE_SECTION_SIZE - 1);
  277. return 0;
  278. }
  279. static void gic_handle_shared_int(bool chained)
  280. {
  281. unsigned int i, intr, virq, gic_reg_step = mips_cm_is64 ? 8 : 4;
  282. unsigned long *pcpu_mask;
  283. unsigned long pending_reg, intrmask_reg;
  284. DECLARE_BITMAP(pending, GIC_MAX_INTRS);
  285. DECLARE_BITMAP(intrmask, GIC_MAX_INTRS);
  286. /* Get per-cpu bitmaps */
  287. pcpu_mask = pcpu_masks[smp_processor_id()].pcpu_mask;
  288. pending_reg = GIC_REG(SHARED, GIC_SH_PEND);
  289. intrmask_reg = GIC_REG(SHARED, GIC_SH_MASK);
  290. for (i = 0; i < BITS_TO_LONGS(gic_shared_intrs); i++) {
  291. pending[i] = gic_read(pending_reg);
  292. intrmask[i] = gic_read(intrmask_reg);
  293. pending_reg += gic_reg_step;
  294. intrmask_reg += gic_reg_step;
  295. if (!IS_ENABLED(CONFIG_64BIT) || mips_cm_is64)
  296. continue;
  297. pending[i] |= (u64)gic_read(pending_reg) << 32;
  298. intrmask[i] |= (u64)gic_read(intrmask_reg) << 32;
  299. pending_reg += gic_reg_step;
  300. intrmask_reg += gic_reg_step;
  301. }
  302. bitmap_and(pending, pending, intrmask, gic_shared_intrs);
  303. bitmap_and(pending, pending, pcpu_mask, gic_shared_intrs);
  304. intr = find_first_bit(pending, gic_shared_intrs);
  305. while (intr != gic_shared_intrs) {
  306. virq = irq_linear_revmap(gic_irq_domain,
  307. GIC_SHARED_TO_HWIRQ(intr));
  308. if (chained)
  309. generic_handle_irq(virq);
  310. else
  311. do_IRQ(virq);
  312. /* go to next pending bit */
  313. bitmap_clear(pending, intr, 1);
  314. intr = find_first_bit(pending, gic_shared_intrs);
  315. }
  316. }
  317. static void gic_mask_irq(struct irq_data *d)
  318. {
  319. gic_reset_mask(GIC_HWIRQ_TO_SHARED(d->hwirq));
  320. }
  321. static void gic_unmask_irq(struct irq_data *d)
  322. {
  323. gic_set_mask(GIC_HWIRQ_TO_SHARED(d->hwirq));
  324. }
  325. static void gic_ack_irq(struct irq_data *d)
  326. {
  327. unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
  328. gic_write(GIC_REG(SHARED, GIC_SH_WEDGE), GIC_SH_WEDGE_CLR(irq));
  329. }
  330. static int gic_set_type(struct irq_data *d, unsigned int type)
  331. {
  332. unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
  333. unsigned long flags;
  334. bool is_edge;
  335. spin_lock_irqsave(&gic_lock, flags);
  336. switch (type & IRQ_TYPE_SENSE_MASK) {
  337. case IRQ_TYPE_EDGE_FALLING:
  338. gic_set_polarity(irq, GIC_POL_NEG);
  339. gic_set_trigger(irq, GIC_TRIG_EDGE);
  340. gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
  341. is_edge = true;
  342. break;
  343. case IRQ_TYPE_EDGE_RISING:
  344. gic_set_polarity(irq, GIC_POL_POS);
  345. gic_set_trigger(irq, GIC_TRIG_EDGE);
  346. gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
  347. is_edge = true;
  348. break;
  349. case IRQ_TYPE_EDGE_BOTH:
  350. /* polarity is irrelevant in this case */
  351. gic_set_trigger(irq, GIC_TRIG_EDGE);
  352. gic_set_dual_edge(irq, GIC_TRIG_DUAL_ENABLE);
  353. is_edge = true;
  354. break;
  355. case IRQ_TYPE_LEVEL_LOW:
  356. gic_set_polarity(irq, GIC_POL_NEG);
  357. gic_set_trigger(irq, GIC_TRIG_LEVEL);
  358. gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
  359. is_edge = false;
  360. break;
  361. case IRQ_TYPE_LEVEL_HIGH:
  362. default:
  363. gic_set_polarity(irq, GIC_POL_POS);
  364. gic_set_trigger(irq, GIC_TRIG_LEVEL);
  365. gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
  366. is_edge = false;
  367. break;
  368. }
  369. if (is_edge)
  370. irq_set_chip_handler_name_locked(d, &gic_edge_irq_controller,
  371. handle_edge_irq, NULL);
  372. else
  373. irq_set_chip_handler_name_locked(d, &gic_level_irq_controller,
  374. handle_level_irq, NULL);
  375. spin_unlock_irqrestore(&gic_lock, flags);
  376. return 0;
  377. }
  378. #ifdef CONFIG_SMP
  379. static int gic_set_affinity(struct irq_data *d, const struct cpumask *cpumask,
  380. bool force)
  381. {
  382. unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
  383. cpumask_t tmp = CPU_MASK_NONE;
  384. unsigned long flags;
  385. int i;
  386. cpumask_and(&tmp, cpumask, cpu_online_mask);
  387. if (cpumask_empty(&tmp))
  388. return -EINVAL;
  389. /* Assumption : cpumask refers to a single CPU */
  390. spin_lock_irqsave(&gic_lock, flags);
  391. /* Re-route this IRQ */
  392. gic_map_to_vpe(irq, mips_cm_vp_id(cpumask_first(&tmp)));
  393. /* Update the pcpu_masks */
  394. for (i = 0; i < min(gic_vpes, NR_CPUS); i++)
  395. clear_bit(irq, pcpu_masks[i].pcpu_mask);
  396. set_bit(irq, pcpu_masks[cpumask_first(&tmp)].pcpu_mask);
  397. cpumask_copy(irq_data_get_affinity_mask(d), cpumask);
  398. spin_unlock_irqrestore(&gic_lock, flags);
  399. return IRQ_SET_MASK_OK_NOCOPY;
  400. }
  401. #endif
  402. static struct irq_chip gic_level_irq_controller = {
  403. .name = "MIPS GIC",
  404. .irq_mask = gic_mask_irq,
  405. .irq_unmask = gic_unmask_irq,
  406. .irq_set_type = gic_set_type,
  407. #ifdef CONFIG_SMP
  408. .irq_set_affinity = gic_set_affinity,
  409. #endif
  410. };
  411. static struct irq_chip gic_edge_irq_controller = {
  412. .name = "MIPS GIC",
  413. .irq_ack = gic_ack_irq,
  414. .irq_mask = gic_mask_irq,
  415. .irq_unmask = gic_unmask_irq,
  416. .irq_set_type = gic_set_type,
  417. #ifdef CONFIG_SMP
  418. .irq_set_affinity = gic_set_affinity,
  419. #endif
  420. .ipi_send_single = gic_send_ipi,
  421. };
  422. static void gic_handle_local_int(bool chained)
  423. {
  424. unsigned long pending, masked;
  425. unsigned int intr, virq;
  426. pending = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_PEND));
  427. masked = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_MASK));
  428. bitmap_and(&pending, &pending, &masked, GIC_NUM_LOCAL_INTRS);
  429. intr = find_first_bit(&pending, GIC_NUM_LOCAL_INTRS);
  430. while (intr != GIC_NUM_LOCAL_INTRS) {
  431. virq = irq_linear_revmap(gic_irq_domain,
  432. GIC_LOCAL_TO_HWIRQ(intr));
  433. if (chained)
  434. generic_handle_irq(virq);
  435. else
  436. do_IRQ(virq);
  437. /* go to next pending bit */
  438. bitmap_clear(&pending, intr, 1);
  439. intr = find_first_bit(&pending, GIC_NUM_LOCAL_INTRS);
  440. }
  441. }
  442. static void gic_mask_local_irq(struct irq_data *d)
  443. {
  444. int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
  445. gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_RMASK), 1 << intr);
  446. }
  447. static void gic_unmask_local_irq(struct irq_data *d)
  448. {
  449. int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
  450. gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_SMASK), 1 << intr);
  451. }
  452. static struct irq_chip gic_local_irq_controller = {
  453. .name = "MIPS GIC Local",
  454. .irq_mask = gic_mask_local_irq,
  455. .irq_unmask = gic_unmask_local_irq,
  456. };
  457. static void gic_mask_local_irq_all_vpes(struct irq_data *d)
  458. {
  459. int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
  460. int i;
  461. unsigned long flags;
  462. spin_lock_irqsave(&gic_lock, flags);
  463. for (i = 0; i < gic_vpes; i++) {
  464. gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
  465. mips_cm_vp_id(i));
  466. gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_RMASK), 1 << intr);
  467. }
  468. spin_unlock_irqrestore(&gic_lock, flags);
  469. }
  470. static void gic_unmask_local_irq_all_vpes(struct irq_data *d)
  471. {
  472. int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
  473. int i;
  474. unsigned long flags;
  475. spin_lock_irqsave(&gic_lock, flags);
  476. for (i = 0; i < gic_vpes; i++) {
  477. gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
  478. mips_cm_vp_id(i));
  479. gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SMASK), 1 << intr);
  480. }
  481. spin_unlock_irqrestore(&gic_lock, flags);
  482. }
  483. static struct irq_chip gic_all_vpes_local_irq_controller = {
  484. .name = "MIPS GIC Local",
  485. .irq_mask = gic_mask_local_irq_all_vpes,
  486. .irq_unmask = gic_unmask_local_irq_all_vpes,
  487. };
  488. static void __gic_irq_dispatch(void)
  489. {
  490. gic_handle_local_int(false);
  491. gic_handle_shared_int(false);
  492. }
  493. static void gic_irq_dispatch(struct irq_desc *desc)
  494. {
  495. gic_handle_local_int(true);
  496. gic_handle_shared_int(true);
  497. }
  498. static void __init gic_basic_init(void)
  499. {
  500. unsigned int i;
  501. board_bind_eic_interrupt = &gic_bind_eic_interrupt;
  502. /* Setup defaults */
  503. for (i = 0; i < gic_shared_intrs; i++) {
  504. gic_set_polarity(i, GIC_POL_POS);
  505. gic_set_trigger(i, GIC_TRIG_LEVEL);
  506. gic_reset_mask(i);
  507. }
  508. for (i = 0; i < gic_vpes; i++) {
  509. unsigned int j;
  510. gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
  511. mips_cm_vp_id(i));
  512. for (j = 0; j < GIC_NUM_LOCAL_INTRS; j++) {
  513. if (!gic_local_irq_is_routable(j))
  514. continue;
  515. gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_RMASK), 1 << j);
  516. }
  517. }
  518. }
  519. static int gic_local_irq_domain_map(struct irq_domain *d, unsigned int virq,
  520. irq_hw_number_t hw)
  521. {
  522. int intr = GIC_HWIRQ_TO_LOCAL(hw);
  523. int ret = 0;
  524. int i;
  525. unsigned long flags;
  526. if (!gic_local_irq_is_routable(intr))
  527. return -EPERM;
  528. /*
  529. * HACK: These are all really percpu interrupts, but the rest
  530. * of the MIPS kernel code does not use the percpu IRQ API for
  531. * the CP0 timer and performance counter interrupts.
  532. */
  533. switch (intr) {
  534. case GIC_LOCAL_INT_TIMER:
  535. case GIC_LOCAL_INT_PERFCTR:
  536. case GIC_LOCAL_INT_FDC:
  537. irq_set_chip_and_handler(virq,
  538. &gic_all_vpes_local_irq_controller,
  539. handle_percpu_irq);
  540. break;
  541. default:
  542. irq_set_chip_and_handler(virq,
  543. &gic_local_irq_controller,
  544. handle_percpu_devid_irq);
  545. irq_set_percpu_devid(virq);
  546. break;
  547. }
  548. spin_lock_irqsave(&gic_lock, flags);
  549. for (i = 0; i < gic_vpes; i++) {
  550. u32 val = GIC_MAP_TO_PIN_MSK | gic_cpu_pin;
  551. gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
  552. mips_cm_vp_id(i));
  553. switch (intr) {
  554. case GIC_LOCAL_INT_WD:
  555. gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_WD_MAP), val);
  556. break;
  557. case GIC_LOCAL_INT_COMPARE:
  558. gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_MAP),
  559. val);
  560. break;
  561. case GIC_LOCAL_INT_TIMER:
  562. /* CONFIG_MIPS_CMP workaround (see __gic_init) */
  563. val = GIC_MAP_TO_PIN_MSK | timer_cpu_pin;
  564. gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_TIMER_MAP),
  565. val);
  566. break;
  567. case GIC_LOCAL_INT_PERFCTR:
  568. gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_PERFCTR_MAP),
  569. val);
  570. break;
  571. case GIC_LOCAL_INT_SWINT0:
  572. gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SWINT0_MAP),
  573. val);
  574. break;
  575. case GIC_LOCAL_INT_SWINT1:
  576. gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SWINT1_MAP),
  577. val);
  578. break;
  579. case GIC_LOCAL_INT_FDC:
  580. gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_FDC_MAP), val);
  581. break;
  582. default:
  583. pr_err("Invalid local IRQ %d\n", intr);
  584. ret = -EINVAL;
  585. break;
  586. }
  587. }
  588. spin_unlock_irqrestore(&gic_lock, flags);
  589. return ret;
  590. }
  591. static int gic_shared_irq_domain_map(struct irq_domain *d, unsigned int virq,
  592. irq_hw_number_t hw, unsigned int vpe)
  593. {
  594. int intr = GIC_HWIRQ_TO_SHARED(hw);
  595. unsigned long flags;
  596. int i;
  597. spin_lock_irqsave(&gic_lock, flags);
  598. gic_map_to_pin(intr, gic_cpu_pin);
  599. gic_map_to_vpe(intr, mips_cm_vp_id(vpe));
  600. for (i = 0; i < min(gic_vpes, NR_CPUS); i++)
  601. clear_bit(intr, pcpu_masks[i].pcpu_mask);
  602. set_bit(intr, pcpu_masks[vpe].pcpu_mask);
  603. spin_unlock_irqrestore(&gic_lock, flags);
  604. return 0;
  605. }
  606. static int gic_irq_domain_map(struct irq_domain *d, unsigned int virq,
  607. irq_hw_number_t hw)
  608. {
  609. if (GIC_HWIRQ_TO_LOCAL(hw) < GIC_NUM_LOCAL_INTRS)
  610. return gic_local_irq_domain_map(d, virq, hw);
  611. irq_set_chip_and_handler(virq, &gic_level_irq_controller,
  612. handle_level_irq);
  613. return gic_shared_irq_domain_map(d, virq, hw, 0);
  614. }
  615. static int gic_irq_domain_alloc(struct irq_domain *d, unsigned int virq,
  616. unsigned int nr_irqs, void *arg)
  617. {
  618. struct gic_irq_spec *spec = arg;
  619. irq_hw_number_t hwirq, base_hwirq;
  620. int cpu, ret, i;
  621. if (spec->type == GIC_DEVICE) {
  622. /* verify that it doesn't conflict with an IPI irq */
  623. if (test_bit(spec->hwirq, ipi_resrv))
  624. return -EBUSY;
  625. hwirq = GIC_SHARED_TO_HWIRQ(spec->hwirq);
  626. return irq_domain_set_hwirq_and_chip(d, virq, hwirq,
  627. &gic_level_irq_controller,
  628. NULL);
  629. } else {
  630. base_hwirq = find_first_bit(ipi_resrv, gic_shared_intrs);
  631. if (base_hwirq == gic_shared_intrs) {
  632. return -ENOMEM;
  633. }
  634. /* check that we have enough space */
  635. for (i = base_hwirq; i < nr_irqs; i++) {
  636. if (!test_bit(i, ipi_resrv))
  637. return -EBUSY;
  638. }
  639. bitmap_clear(ipi_resrv, base_hwirq, nr_irqs);
  640. /* map the hwirq for each cpu consecutively */
  641. i = 0;
  642. for_each_cpu(cpu, spec->ipimask) {
  643. hwirq = GIC_SHARED_TO_HWIRQ(base_hwirq + i);
  644. ret = irq_domain_set_hwirq_and_chip(d, virq + i, hwirq,
  645. &gic_level_irq_controller,
  646. NULL);
  647. if (ret)
  648. goto error;
  649. irq_set_handler(virq + i, handle_level_irq);
  650. ret = gic_shared_irq_domain_map(d, virq + i, hwirq, cpu);
  651. if (ret)
  652. goto error;
  653. i++;
  654. }
  655. /*
  656. * tell the parent about the base hwirq we allocated so it can
  657. * set its own domain data
  658. */
  659. spec->hwirq = base_hwirq;
  660. }
  661. return 0;
  662. error:
  663. bitmap_set(ipi_resrv, base_hwirq, nr_irqs);
  664. return ret;
  665. }
  666. void gic_irq_domain_free(struct irq_domain *d, unsigned int virq,
  667. unsigned int nr_irqs)
  668. {
  669. irq_hw_number_t base_hwirq;
  670. struct irq_data *data;
  671. data = irq_get_irq_data(virq);
  672. if (!data)
  673. return;
  674. base_hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(data));
  675. bitmap_set(ipi_resrv, base_hwirq, nr_irqs);
  676. }
  677. int gic_irq_domain_match(struct irq_domain *d, struct device_node *node,
  678. enum irq_domain_bus_token bus_token)
  679. {
  680. /* this domain should'nt be accessed directly */
  681. return 0;
  682. }
  683. static const struct irq_domain_ops gic_irq_domain_ops = {
  684. .map = gic_irq_domain_map,
  685. .alloc = gic_irq_domain_alloc,
  686. .free = gic_irq_domain_free,
  687. .match = gic_irq_domain_match,
  688. };
  689. static int gic_dev_domain_xlate(struct irq_domain *d, struct device_node *ctrlr,
  690. const u32 *intspec, unsigned int intsize,
  691. irq_hw_number_t *out_hwirq,
  692. unsigned int *out_type)
  693. {
  694. if (intsize != 3)
  695. return -EINVAL;
  696. if (intspec[0] == GIC_SHARED)
  697. *out_hwirq = GIC_SHARED_TO_HWIRQ(intspec[1]);
  698. else if (intspec[0] == GIC_LOCAL)
  699. *out_hwirq = GIC_LOCAL_TO_HWIRQ(intspec[1]);
  700. else
  701. return -EINVAL;
  702. *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK;
  703. return 0;
  704. }
  705. static int gic_dev_domain_alloc(struct irq_domain *d, unsigned int virq,
  706. unsigned int nr_irqs, void *arg)
  707. {
  708. struct irq_fwspec *fwspec = arg;
  709. struct gic_irq_spec spec = {
  710. .type = GIC_DEVICE,
  711. .hwirq = fwspec->param[1],
  712. };
  713. int i, ret;
  714. bool is_shared = fwspec->param[0] == GIC_SHARED;
  715. if (is_shared) {
  716. ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, &spec);
  717. if (ret)
  718. return ret;
  719. }
  720. for (i = 0; i < nr_irqs; i++) {
  721. irq_hw_number_t hwirq;
  722. if (is_shared)
  723. hwirq = GIC_SHARED_TO_HWIRQ(spec.hwirq + i);
  724. else
  725. hwirq = GIC_LOCAL_TO_HWIRQ(spec.hwirq + i);
  726. ret = irq_domain_set_hwirq_and_chip(d, virq + i,
  727. hwirq,
  728. &gic_level_irq_controller,
  729. NULL);
  730. if (ret)
  731. goto error;
  732. }
  733. return 0;
  734. error:
  735. irq_domain_free_irqs_parent(d, virq, nr_irqs);
  736. return ret;
  737. }
  738. void gic_dev_domain_free(struct irq_domain *d, unsigned int virq,
  739. unsigned int nr_irqs)
  740. {
  741. /* no real allocation is done for dev irqs, so no need to free anything */
  742. return;
  743. }
  744. static void gic_dev_domain_activate(struct irq_domain *domain,
  745. struct irq_data *d)
  746. {
  747. gic_shared_irq_domain_map(domain, d->irq, d->hwirq, 0);
  748. }
  749. static struct irq_domain_ops gic_dev_domain_ops = {
  750. .xlate = gic_dev_domain_xlate,
  751. .alloc = gic_dev_domain_alloc,
  752. .free = gic_dev_domain_free,
  753. .activate = gic_dev_domain_activate,
  754. };
  755. static int gic_ipi_domain_xlate(struct irq_domain *d, struct device_node *ctrlr,
  756. const u32 *intspec, unsigned int intsize,
  757. irq_hw_number_t *out_hwirq,
  758. unsigned int *out_type)
  759. {
  760. /*
  761. * There's nothing to translate here. hwirq is dynamically allocated and
  762. * the irq type is always edge triggered.
  763. * */
  764. *out_hwirq = 0;
  765. *out_type = IRQ_TYPE_EDGE_RISING;
  766. return 0;
  767. }
  768. static int gic_ipi_domain_alloc(struct irq_domain *d, unsigned int virq,
  769. unsigned int nr_irqs, void *arg)
  770. {
  771. struct cpumask *ipimask = arg;
  772. struct gic_irq_spec spec = {
  773. .type = GIC_IPI,
  774. .ipimask = ipimask
  775. };
  776. int ret, i;
  777. ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, &spec);
  778. if (ret)
  779. return ret;
  780. /* the parent should have set spec.hwirq to the base_hwirq it allocated */
  781. for (i = 0; i < nr_irqs; i++) {
  782. ret = irq_domain_set_hwirq_and_chip(d, virq + i,
  783. GIC_SHARED_TO_HWIRQ(spec.hwirq + i),
  784. &gic_edge_irq_controller,
  785. NULL);
  786. if (ret)
  787. goto error;
  788. ret = irq_set_irq_type(virq + i, IRQ_TYPE_EDGE_RISING);
  789. if (ret)
  790. goto error;
  791. }
  792. return 0;
  793. error:
  794. irq_domain_free_irqs_parent(d, virq, nr_irqs);
  795. return ret;
  796. }
  797. void gic_ipi_domain_free(struct irq_domain *d, unsigned int virq,
  798. unsigned int nr_irqs)
  799. {
  800. irq_domain_free_irqs_parent(d, virq, nr_irqs);
  801. }
  802. int gic_ipi_domain_match(struct irq_domain *d, struct device_node *node,
  803. enum irq_domain_bus_token bus_token)
  804. {
  805. bool is_ipi;
  806. switch (bus_token) {
  807. case DOMAIN_BUS_IPI:
  808. is_ipi = d->bus_token == bus_token;
  809. return (!node || to_of_node(d->fwnode) == node) && is_ipi;
  810. break;
  811. default:
  812. return 0;
  813. }
  814. }
  815. static struct irq_domain_ops gic_ipi_domain_ops = {
  816. .xlate = gic_ipi_domain_xlate,
  817. .alloc = gic_ipi_domain_alloc,
  818. .free = gic_ipi_domain_free,
  819. .match = gic_ipi_domain_match,
  820. };
  821. static void __init __gic_init(unsigned long gic_base_addr,
  822. unsigned long gic_addrspace_size,
  823. unsigned int cpu_vec, unsigned int irqbase,
  824. struct device_node *node)
  825. {
  826. unsigned int gicconfig, cpu;
  827. unsigned int v[2];
  828. __gic_base_addr = gic_base_addr;
  829. gic_base = ioremap_nocache(gic_base_addr, gic_addrspace_size);
  830. gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
  831. gic_shared_intrs = (gicconfig & GIC_SH_CONFIG_NUMINTRS_MSK) >>
  832. GIC_SH_CONFIG_NUMINTRS_SHF;
  833. gic_shared_intrs = ((gic_shared_intrs + 1) * 8);
  834. gic_vpes = (gicconfig & GIC_SH_CONFIG_NUMVPES_MSK) >>
  835. GIC_SH_CONFIG_NUMVPES_SHF;
  836. gic_vpes = gic_vpes + 1;
  837. if (cpu_has_veic) {
  838. /* Set EIC mode for all VPEs */
  839. for_each_present_cpu(cpu) {
  840. gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
  841. mips_cm_vp_id(cpu));
  842. gic_write(GIC_REG(VPE_OTHER, GIC_VPE_CTL),
  843. GIC_VPE_CTL_EIC_MODE_MSK);
  844. }
  845. /* Always use vector 1 in EIC mode */
  846. gic_cpu_pin = 0;
  847. timer_cpu_pin = gic_cpu_pin;
  848. set_vi_handler(gic_cpu_pin + GIC_PIN_TO_VEC_OFFSET,
  849. __gic_irq_dispatch);
  850. } else {
  851. gic_cpu_pin = cpu_vec - GIC_CPU_PIN_OFFSET;
  852. irq_set_chained_handler(MIPS_CPU_IRQ_BASE + cpu_vec,
  853. gic_irq_dispatch);
  854. /*
  855. * With the CMP implementation of SMP (deprecated), other CPUs
  856. * are started by the bootloader and put into a timer based
  857. * waiting poll loop. We must not re-route those CPU's local
  858. * timer interrupts as the wait instruction will never finish,
  859. * so just handle whatever CPU interrupt it is routed to by
  860. * default.
  861. *
  862. * This workaround should be removed when CMP support is
  863. * dropped.
  864. */
  865. if (IS_ENABLED(CONFIG_MIPS_CMP) &&
  866. gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER)) {
  867. timer_cpu_pin = gic_read32(GIC_REG(VPE_LOCAL,
  868. GIC_VPE_TIMER_MAP)) &
  869. GIC_MAP_MSK;
  870. irq_set_chained_handler(MIPS_CPU_IRQ_BASE +
  871. GIC_CPU_PIN_OFFSET +
  872. timer_cpu_pin,
  873. gic_irq_dispatch);
  874. } else {
  875. timer_cpu_pin = gic_cpu_pin;
  876. }
  877. }
  878. gic_irq_domain = irq_domain_add_simple(node, GIC_NUM_LOCAL_INTRS +
  879. gic_shared_intrs, irqbase,
  880. &gic_irq_domain_ops, NULL);
  881. if (!gic_irq_domain)
  882. panic("Failed to add GIC IRQ domain");
  883. gic_irq_domain->name = "mips-gic-irq";
  884. gic_dev_domain = irq_domain_add_hierarchy(gic_irq_domain, 0,
  885. GIC_NUM_LOCAL_INTRS + gic_shared_intrs,
  886. node, &gic_dev_domain_ops, NULL);
  887. if (!gic_dev_domain)
  888. panic("Failed to add GIC DEV domain");
  889. gic_dev_domain->name = "mips-gic-dev";
  890. gic_ipi_domain = irq_domain_add_hierarchy(gic_irq_domain,
  891. IRQ_DOMAIN_FLAG_IPI_PER_CPU,
  892. GIC_NUM_LOCAL_INTRS + gic_shared_intrs,
  893. node, &gic_ipi_domain_ops, NULL);
  894. if (!gic_ipi_domain)
  895. panic("Failed to add GIC IPI domain");
  896. gic_ipi_domain->name = "mips-gic-ipi";
  897. gic_ipi_domain->bus_token = DOMAIN_BUS_IPI;
  898. if (node &&
  899. !of_property_read_u32_array(node, "mti,reserved-ipi-vectors", v, 2)) {
  900. bitmap_set(ipi_resrv, v[0], v[1]);
  901. } else {
  902. /* Make the last 2 * gic_vpes available for IPIs */
  903. bitmap_set(ipi_resrv,
  904. gic_shared_intrs - 2 * gic_vpes,
  905. 2 * gic_vpes);
  906. }
  907. gic_basic_init();
  908. }
  909. void __init gic_init(unsigned long gic_base_addr,
  910. unsigned long gic_addrspace_size,
  911. unsigned int cpu_vec, unsigned int irqbase)
  912. {
  913. __gic_init(gic_base_addr, gic_addrspace_size, cpu_vec, irqbase, NULL);
  914. }
  915. static int __init gic_of_init(struct device_node *node,
  916. struct device_node *parent)
  917. {
  918. struct resource res;
  919. unsigned int cpu_vec, i = 0, reserved = 0;
  920. phys_addr_t gic_base;
  921. size_t gic_len;
  922. /* Find the first available CPU vector. */
  923. while (!of_property_read_u32_index(node, "mti,reserved-cpu-vectors",
  924. i++, &cpu_vec))
  925. reserved |= BIT(cpu_vec);
  926. for (cpu_vec = 2; cpu_vec < 8; cpu_vec++) {
  927. if (!(reserved & BIT(cpu_vec)))
  928. break;
  929. }
  930. if (cpu_vec == 8) {
  931. pr_err("No CPU vectors available for GIC\n");
  932. return -ENODEV;
  933. }
  934. if (of_address_to_resource(node, 0, &res)) {
  935. /*
  936. * Probe the CM for the GIC base address if not specified
  937. * in the device-tree.
  938. */
  939. if (mips_cm_present()) {
  940. gic_base = read_gcr_gic_base() &
  941. ~CM_GCR_GIC_BASE_GICEN_MSK;
  942. gic_len = 0x20000;
  943. } else {
  944. pr_err("Failed to get GIC memory range\n");
  945. return -ENODEV;
  946. }
  947. } else {
  948. gic_base = res.start;
  949. gic_len = resource_size(&res);
  950. }
  951. if (mips_cm_present())
  952. write_gcr_gic_base(gic_base | CM_GCR_GIC_BASE_GICEN_MSK);
  953. gic_present = true;
  954. __gic_init(gic_base, gic_len, cpu_vec, 0, node);
  955. return 0;
  956. }
  957. IRQCHIP_DECLARE(mips_gic, "mti,gic", gic_of_init);