irq-gic.c 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458
  1. /*
  2. * Copyright (C) 2002 ARM Limited, All Rights Reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License version 2 as
  6. * published by the Free Software Foundation.
  7. *
  8. * Interrupt architecture for the GIC:
  9. *
  10. * o There is one Interrupt Distributor, which receives interrupts
  11. * from system devices and sends them to the Interrupt Controllers.
  12. *
  13. * o There is one CPU Interface per CPU, which sends interrupts sent
  14. * by the Distributor, and interrupts generated locally, to the
  15. * associated CPU. The base address of the CPU interface is usually
  16. * aliased so that the same address points to different chips depending
  17. * on the CPU it is accessed from.
  18. *
  19. * Note that IRQs 0-31 are special - they are local to each CPU.
  20. * As such, the enable set/clear, pending set/clear and active bit
  21. * registers are banked per-cpu for these sources.
  22. */
  23. #include <linux/init.h>
  24. #include <linux/kernel.h>
  25. #include <linux/err.h>
  26. #include <linux/module.h>
  27. #include <linux/list.h>
  28. #include <linux/smp.h>
  29. #include <linux/cpu.h>
  30. #include <linux/cpu_pm.h>
  31. #include <linux/cpumask.h>
  32. #include <linux/io.h>
  33. #include <linux/of.h>
  34. #include <linux/of_address.h>
  35. #include <linux/of_irq.h>
  36. #include <linux/acpi.h>
  37. #include <linux/irqdomain.h>
  38. #include <linux/interrupt.h>
  39. #include <linux/percpu.h>
  40. #include <linux/slab.h>
  41. #include <linux/irqchip.h>
  42. #include <linux/irqchip/chained_irq.h>
  43. #include <linux/irqchip/arm-gic.h>
  44. #include <asm/cputype.h>
  45. #include <asm/irq.h>
  46. #include <asm/exception.h>
  47. #include <asm/smp_plat.h>
  48. #include <asm/virt.h>
  49. #include "irq-gic-common.h"
  50. #ifdef CONFIG_ARM64
  51. #include <asm/cpufeature.h>
  52. static void gic_check_cpu_features(void)
  53. {
  54. WARN_TAINT_ONCE(this_cpu_has_cap(ARM64_HAS_SYSREG_GIC_CPUIF),
  55. TAINT_CPU_OUT_OF_SPEC,
  56. "GICv3 system registers enabled, broken firmware!\n");
  57. }
  58. #else
  59. #define gic_check_cpu_features() do { } while(0)
  60. #endif
  61. union gic_base {
  62. void __iomem *common_base;
  63. void __percpu * __iomem *percpu_base;
  64. };
  65. struct gic_chip_data {
  66. struct irq_chip chip;
  67. union gic_base dist_base;
  68. union gic_base cpu_base;
  69. void __iomem *raw_dist_base;
  70. void __iomem *raw_cpu_base;
  71. u32 percpu_offset;
  72. #ifdef CONFIG_CPU_PM
  73. u32 saved_spi_enable[DIV_ROUND_UP(1020, 32)];
  74. u32 saved_spi_active[DIV_ROUND_UP(1020, 32)];
  75. u32 saved_spi_conf[DIV_ROUND_UP(1020, 16)];
  76. u32 saved_spi_target[DIV_ROUND_UP(1020, 4)];
  77. u32 __percpu *saved_ppi_enable;
  78. u32 __percpu *saved_ppi_active;
  79. u32 __percpu *saved_ppi_conf;
  80. #endif
  81. struct irq_domain *domain;
  82. unsigned int gic_irqs;
  83. #ifdef CONFIG_GIC_NON_BANKED
  84. void __iomem *(*get_base)(union gic_base *);
  85. #endif
  86. };
  87. static DEFINE_RAW_SPINLOCK(irq_controller_lock);
  88. /*
  89. * The GIC mapping of CPU interfaces does not necessarily match
  90. * the logical CPU numbering. Let's use a mapping as returned
  91. * by the GIC itself.
  92. */
  93. #define NR_GIC_CPU_IF 8
  94. static u8 gic_cpu_map[NR_GIC_CPU_IF] __read_mostly;
  95. static struct static_key supports_deactivate = STATIC_KEY_INIT_TRUE;
  96. static struct gic_chip_data gic_data[CONFIG_ARM_GIC_MAX_NR] __read_mostly;
  97. #ifdef CONFIG_GIC_NON_BANKED
  98. static void __iomem *gic_get_percpu_base(union gic_base *base)
  99. {
  100. return raw_cpu_read(*base->percpu_base);
  101. }
  102. static void __iomem *gic_get_common_base(union gic_base *base)
  103. {
  104. return base->common_base;
  105. }
  106. static inline void __iomem *gic_data_dist_base(struct gic_chip_data *data)
  107. {
  108. return data->get_base(&data->dist_base);
  109. }
  110. static inline void __iomem *gic_data_cpu_base(struct gic_chip_data *data)
  111. {
  112. return data->get_base(&data->cpu_base);
  113. }
  114. static inline void gic_set_base_accessor(struct gic_chip_data *data,
  115. void __iomem *(*f)(union gic_base *))
  116. {
  117. data->get_base = f;
  118. }
  119. #else
  120. #define gic_data_dist_base(d) ((d)->dist_base.common_base)
  121. #define gic_data_cpu_base(d) ((d)->cpu_base.common_base)
  122. #define gic_set_base_accessor(d, f)
  123. #endif
  124. static inline void __iomem *gic_dist_base(struct irq_data *d)
  125. {
  126. struct gic_chip_data *gic_data = irq_data_get_irq_chip_data(d);
  127. return gic_data_dist_base(gic_data);
  128. }
  129. static inline void __iomem *gic_cpu_base(struct irq_data *d)
  130. {
  131. struct gic_chip_data *gic_data = irq_data_get_irq_chip_data(d);
  132. return gic_data_cpu_base(gic_data);
  133. }
  134. static inline unsigned int gic_irq(struct irq_data *d)
  135. {
  136. return d->hwirq;
  137. }
  138. static inline bool cascading_gic_irq(struct irq_data *d)
  139. {
  140. void *data = irq_data_get_irq_handler_data(d);
  141. /*
  142. * If handler_data is set, this is a cascading interrupt, and
  143. * it cannot possibly be forwarded.
  144. */
  145. return data != NULL;
  146. }
  147. /*
  148. * Routines to acknowledge, disable and enable interrupts
  149. */
  150. static void gic_poke_irq(struct irq_data *d, u32 offset)
  151. {
  152. u32 mask = 1 << (gic_irq(d) % 32);
  153. writel_relaxed(mask, gic_dist_base(d) + offset + (gic_irq(d) / 32) * 4);
  154. }
  155. static int gic_peek_irq(struct irq_data *d, u32 offset)
  156. {
  157. u32 mask = 1 << (gic_irq(d) % 32);
  158. return !!(readl_relaxed(gic_dist_base(d) + offset + (gic_irq(d) / 32) * 4) & mask);
  159. }
  160. static void gic_mask_irq(struct irq_data *d)
  161. {
  162. gic_poke_irq(d, GIC_DIST_ENABLE_CLEAR);
  163. }
  164. static void gic_eoimode1_mask_irq(struct irq_data *d)
  165. {
  166. gic_mask_irq(d);
  167. /*
  168. * When masking a forwarded interrupt, make sure it is
  169. * deactivated as well.
  170. *
  171. * This ensures that an interrupt that is getting
  172. * disabled/masked will not get "stuck", because there is
  173. * noone to deactivate it (guest is being terminated).
  174. */
  175. if (irqd_is_forwarded_to_vcpu(d))
  176. gic_poke_irq(d, GIC_DIST_ACTIVE_CLEAR);
  177. }
  178. static void gic_unmask_irq(struct irq_data *d)
  179. {
  180. gic_poke_irq(d, GIC_DIST_ENABLE_SET);
  181. }
  182. static void gic_eoi_irq(struct irq_data *d)
  183. {
  184. writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_EOI);
  185. }
  186. static void gic_eoimode1_eoi_irq(struct irq_data *d)
  187. {
  188. /* Do not deactivate an IRQ forwarded to a vcpu. */
  189. if (irqd_is_forwarded_to_vcpu(d))
  190. return;
  191. writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_DEACTIVATE);
  192. }
  193. static int gic_irq_set_irqchip_state(struct irq_data *d,
  194. enum irqchip_irq_state which, bool val)
  195. {
  196. u32 reg;
  197. switch (which) {
  198. case IRQCHIP_STATE_PENDING:
  199. reg = val ? GIC_DIST_PENDING_SET : GIC_DIST_PENDING_CLEAR;
  200. break;
  201. case IRQCHIP_STATE_ACTIVE:
  202. reg = val ? GIC_DIST_ACTIVE_SET : GIC_DIST_ACTIVE_CLEAR;
  203. break;
  204. case IRQCHIP_STATE_MASKED:
  205. reg = val ? GIC_DIST_ENABLE_CLEAR : GIC_DIST_ENABLE_SET;
  206. break;
  207. default:
  208. return -EINVAL;
  209. }
  210. gic_poke_irq(d, reg);
  211. return 0;
  212. }
  213. static int gic_irq_get_irqchip_state(struct irq_data *d,
  214. enum irqchip_irq_state which, bool *val)
  215. {
  216. switch (which) {
  217. case IRQCHIP_STATE_PENDING:
  218. *val = gic_peek_irq(d, GIC_DIST_PENDING_SET);
  219. break;
  220. case IRQCHIP_STATE_ACTIVE:
  221. *val = gic_peek_irq(d, GIC_DIST_ACTIVE_SET);
  222. break;
  223. case IRQCHIP_STATE_MASKED:
  224. *val = !gic_peek_irq(d, GIC_DIST_ENABLE_SET);
  225. break;
  226. default:
  227. return -EINVAL;
  228. }
  229. return 0;
  230. }
  231. static int gic_set_type(struct irq_data *d, unsigned int type)
  232. {
  233. void __iomem *base = gic_dist_base(d);
  234. unsigned int gicirq = gic_irq(d);
  235. /* Interrupt configuration for SGIs can't be changed */
  236. if (gicirq < 16)
  237. return -EINVAL;
  238. /* SPIs have restrictions on the supported types */
  239. if (gicirq >= 32 && type != IRQ_TYPE_LEVEL_HIGH &&
  240. type != IRQ_TYPE_EDGE_RISING)
  241. return -EINVAL;
  242. return gic_configure_irq(gicirq, type, base, NULL);
  243. }
  244. static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
  245. {
  246. /* Only interrupts on the primary GIC can be forwarded to a vcpu. */
  247. if (cascading_gic_irq(d))
  248. return -EINVAL;
  249. if (vcpu)
  250. irqd_set_forwarded_to_vcpu(d);
  251. else
  252. irqd_clr_forwarded_to_vcpu(d);
  253. return 0;
  254. }
  255. #ifdef CONFIG_SMP
  256. static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
  257. bool force)
  258. {
  259. void __iomem *reg = gic_dist_base(d) + GIC_DIST_TARGET + (gic_irq(d) & ~3);
  260. unsigned int cpu, shift = (gic_irq(d) % 4) * 8;
  261. u32 val, mask, bit;
  262. unsigned long flags;
  263. if (!force)
  264. cpu = cpumask_any_and(mask_val, cpu_online_mask);
  265. else
  266. cpu = cpumask_first(mask_val);
  267. if (cpu >= NR_GIC_CPU_IF || cpu >= nr_cpu_ids)
  268. return -EINVAL;
  269. raw_spin_lock_irqsave(&irq_controller_lock, flags);
  270. mask = 0xff << shift;
  271. bit = gic_cpu_map[cpu] << shift;
  272. val = readl_relaxed(reg) & ~mask;
  273. writel_relaxed(val | bit, reg);
  274. raw_spin_unlock_irqrestore(&irq_controller_lock, flags);
  275. return IRQ_SET_MASK_OK_DONE;
  276. }
  277. #endif
  278. static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
  279. {
  280. u32 irqstat, irqnr;
  281. struct gic_chip_data *gic = &gic_data[0];
  282. void __iomem *cpu_base = gic_data_cpu_base(gic);
  283. do {
  284. irqstat = readl_relaxed(cpu_base + GIC_CPU_INTACK);
  285. irqnr = irqstat & GICC_IAR_INT_ID_MASK;
  286. if (likely(irqnr > 15 && irqnr < 1020)) {
  287. if (static_key_true(&supports_deactivate))
  288. writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI);
  289. handle_domain_irq(gic->domain, irqnr, regs);
  290. continue;
  291. }
  292. if (irqnr < 16) {
  293. writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI);
  294. if (static_key_true(&supports_deactivate))
  295. writel_relaxed(irqstat, cpu_base + GIC_CPU_DEACTIVATE);
  296. #ifdef CONFIG_SMP
  297. /*
  298. * Ensure any shared data written by the CPU sending
  299. * the IPI is read after we've read the ACK register
  300. * on the GIC.
  301. *
  302. * Pairs with the write barrier in gic_raise_softirq
  303. */
  304. smp_rmb();
  305. handle_IPI(irqnr, regs);
  306. #endif
  307. continue;
  308. }
  309. break;
  310. } while (1);
  311. }
  312. static void gic_handle_cascade_irq(struct irq_desc *desc)
  313. {
  314. struct gic_chip_data *chip_data = irq_desc_get_handler_data(desc);
  315. struct irq_chip *chip = irq_desc_get_chip(desc);
  316. unsigned int cascade_irq, gic_irq;
  317. unsigned long status;
  318. chained_irq_enter(chip, desc);
  319. raw_spin_lock(&irq_controller_lock);
  320. status = readl_relaxed(gic_data_cpu_base(chip_data) + GIC_CPU_INTACK);
  321. raw_spin_unlock(&irq_controller_lock);
  322. gic_irq = (status & GICC_IAR_INT_ID_MASK);
  323. if (gic_irq == GICC_INT_SPURIOUS)
  324. goto out;
  325. cascade_irq = irq_find_mapping(chip_data->domain, gic_irq);
  326. if (unlikely(gic_irq < 32 || gic_irq > 1020))
  327. handle_bad_irq(desc);
  328. else
  329. generic_handle_irq(cascade_irq);
  330. out:
  331. chained_irq_exit(chip, desc);
  332. }
  333. static struct irq_chip gic_chip = {
  334. .irq_mask = gic_mask_irq,
  335. .irq_unmask = gic_unmask_irq,
  336. .irq_eoi = gic_eoi_irq,
  337. .irq_set_type = gic_set_type,
  338. .irq_get_irqchip_state = gic_irq_get_irqchip_state,
  339. .irq_set_irqchip_state = gic_irq_set_irqchip_state,
  340. .flags = IRQCHIP_SET_TYPE_MASKED |
  341. IRQCHIP_SKIP_SET_WAKE |
  342. IRQCHIP_MASK_ON_SUSPEND,
  343. };
  344. void __init gic_cascade_irq(unsigned int gic_nr, unsigned int irq)
  345. {
  346. BUG_ON(gic_nr >= CONFIG_ARM_GIC_MAX_NR);
  347. irq_set_chained_handler_and_data(irq, gic_handle_cascade_irq,
  348. &gic_data[gic_nr]);
  349. }
  350. static u8 gic_get_cpumask(struct gic_chip_data *gic)
  351. {
  352. void __iomem *base = gic_data_dist_base(gic);
  353. u32 mask, i;
  354. for (i = mask = 0; i < 32; i += 4) {
  355. mask = readl_relaxed(base + GIC_DIST_TARGET + i);
  356. mask |= mask >> 16;
  357. mask |= mask >> 8;
  358. if (mask)
  359. break;
  360. }
  361. if (!mask && num_possible_cpus() > 1)
  362. pr_crit("GIC CPU mask not found - kernel will fail to boot.\n");
  363. return mask;
  364. }
  365. static void gic_cpu_if_up(struct gic_chip_data *gic)
  366. {
  367. void __iomem *cpu_base = gic_data_cpu_base(gic);
  368. u32 bypass = 0;
  369. u32 mode = 0;
  370. if (gic == &gic_data[0] && static_key_true(&supports_deactivate))
  371. mode = GIC_CPU_CTRL_EOImodeNS;
  372. /*
  373. * Preserve bypass disable bits to be written back later
  374. */
  375. bypass = readl(cpu_base + GIC_CPU_CTRL);
  376. bypass &= GICC_DIS_BYPASS_MASK;
  377. writel_relaxed(bypass | mode | GICC_ENABLE, cpu_base + GIC_CPU_CTRL);
  378. }
  379. static void __init gic_dist_init(struct gic_chip_data *gic)
  380. {
  381. unsigned int i;
  382. u32 cpumask;
  383. unsigned int gic_irqs = gic->gic_irqs;
  384. void __iomem *base = gic_data_dist_base(gic);
  385. writel_relaxed(GICD_DISABLE, base + GIC_DIST_CTRL);
  386. /*
  387. * Set all global interrupts to this CPU only.
  388. */
  389. cpumask = gic_get_cpumask(gic);
  390. cpumask |= cpumask << 8;
  391. cpumask |= cpumask << 16;
  392. for (i = 32; i < gic_irqs; i += 4)
  393. writel_relaxed(cpumask, base + GIC_DIST_TARGET + i * 4 / 4);
  394. gic_dist_config(base, gic_irqs, NULL);
  395. writel_relaxed(GICD_ENABLE, base + GIC_DIST_CTRL);
  396. }
  397. static int gic_cpu_init(struct gic_chip_data *gic)
  398. {
  399. void __iomem *dist_base = gic_data_dist_base(gic);
  400. void __iomem *base = gic_data_cpu_base(gic);
  401. unsigned int cpu_mask, cpu = smp_processor_id();
  402. int i;
  403. /*
  404. * Setting up the CPU map is only relevant for the primary GIC
  405. * because any nested/secondary GICs do not directly interface
  406. * with the CPU(s).
  407. */
  408. if (gic == &gic_data[0]) {
  409. /*
  410. * Get what the GIC says our CPU mask is.
  411. */
  412. if (WARN_ON(cpu >= NR_GIC_CPU_IF))
  413. return -EINVAL;
  414. gic_check_cpu_features();
  415. cpu_mask = gic_get_cpumask(gic);
  416. gic_cpu_map[cpu] = cpu_mask;
  417. /*
  418. * Clear our mask from the other map entries in case they're
  419. * still undefined.
  420. */
  421. for (i = 0; i < NR_GIC_CPU_IF; i++)
  422. if (i != cpu)
  423. gic_cpu_map[i] &= ~cpu_mask;
  424. }
  425. gic_cpu_config(dist_base, NULL);
  426. writel_relaxed(GICC_INT_PRI_THRESHOLD, base + GIC_CPU_PRIMASK);
  427. gic_cpu_if_up(gic);
  428. return 0;
  429. }
  430. int gic_cpu_if_down(unsigned int gic_nr)
  431. {
  432. void __iomem *cpu_base;
  433. u32 val = 0;
  434. if (gic_nr >= CONFIG_ARM_GIC_MAX_NR)
  435. return -EINVAL;
  436. cpu_base = gic_data_cpu_base(&gic_data[gic_nr]);
  437. val = readl(cpu_base + GIC_CPU_CTRL);
  438. val &= ~GICC_ENABLE;
  439. writel_relaxed(val, cpu_base + GIC_CPU_CTRL);
  440. return 0;
  441. }
  442. #ifdef CONFIG_CPU_PM
  443. /*
  444. * Saves the GIC distributor registers during suspend or idle. Must be called
  445. * with interrupts disabled but before powering down the GIC. After calling
  446. * this function, no interrupts will be delivered by the GIC, and another
  447. * platform-specific wakeup source must be enabled.
  448. */
  449. static void gic_dist_save(struct gic_chip_data *gic)
  450. {
  451. unsigned int gic_irqs;
  452. void __iomem *dist_base;
  453. int i;
  454. if (WARN_ON(!gic))
  455. return;
  456. gic_irqs = gic->gic_irqs;
  457. dist_base = gic_data_dist_base(gic);
  458. if (!dist_base)
  459. return;
  460. for (i = 0; i < DIV_ROUND_UP(gic_irqs, 16); i++)
  461. gic->saved_spi_conf[i] =
  462. readl_relaxed(dist_base + GIC_DIST_CONFIG + i * 4);
  463. for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++)
  464. gic->saved_spi_target[i] =
  465. readl_relaxed(dist_base + GIC_DIST_TARGET + i * 4);
  466. for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++)
  467. gic->saved_spi_enable[i] =
  468. readl_relaxed(dist_base + GIC_DIST_ENABLE_SET + i * 4);
  469. for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++)
  470. gic->saved_spi_active[i] =
  471. readl_relaxed(dist_base + GIC_DIST_ACTIVE_SET + i * 4);
  472. }
  473. /*
  474. * Restores the GIC distributor registers during resume or when coming out of
  475. * idle. Must be called before enabling interrupts. If a level interrupt
  476. * that occured while the GIC was suspended is still present, it will be
  477. * handled normally, but any edge interrupts that occured will not be seen by
  478. * the GIC and need to be handled by the platform-specific wakeup source.
  479. */
  480. static void gic_dist_restore(struct gic_chip_data *gic)
  481. {
  482. unsigned int gic_irqs;
  483. unsigned int i;
  484. void __iomem *dist_base;
  485. if (WARN_ON(!gic))
  486. return;
  487. gic_irqs = gic->gic_irqs;
  488. dist_base = gic_data_dist_base(gic);
  489. if (!dist_base)
  490. return;
  491. writel_relaxed(GICD_DISABLE, dist_base + GIC_DIST_CTRL);
  492. for (i = 0; i < DIV_ROUND_UP(gic_irqs, 16); i++)
  493. writel_relaxed(gic->saved_spi_conf[i],
  494. dist_base + GIC_DIST_CONFIG + i * 4);
  495. for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++)
  496. writel_relaxed(GICD_INT_DEF_PRI_X4,
  497. dist_base + GIC_DIST_PRI + i * 4);
  498. for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++)
  499. writel_relaxed(gic->saved_spi_target[i],
  500. dist_base + GIC_DIST_TARGET + i * 4);
  501. for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++) {
  502. writel_relaxed(GICD_INT_EN_CLR_X32,
  503. dist_base + GIC_DIST_ENABLE_CLEAR + i * 4);
  504. writel_relaxed(gic->saved_spi_enable[i],
  505. dist_base + GIC_DIST_ENABLE_SET + i * 4);
  506. }
  507. for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++) {
  508. writel_relaxed(GICD_INT_EN_CLR_X32,
  509. dist_base + GIC_DIST_ACTIVE_CLEAR + i * 4);
  510. writel_relaxed(gic->saved_spi_active[i],
  511. dist_base + GIC_DIST_ACTIVE_SET + i * 4);
  512. }
  513. writel_relaxed(GICD_ENABLE, dist_base + GIC_DIST_CTRL);
  514. }
  515. static void gic_cpu_save(struct gic_chip_data *gic)
  516. {
  517. int i;
  518. u32 *ptr;
  519. void __iomem *dist_base;
  520. void __iomem *cpu_base;
  521. if (WARN_ON(!gic))
  522. return;
  523. dist_base = gic_data_dist_base(gic);
  524. cpu_base = gic_data_cpu_base(gic);
  525. if (!dist_base || !cpu_base)
  526. return;
  527. ptr = raw_cpu_ptr(gic->saved_ppi_enable);
  528. for (i = 0; i < DIV_ROUND_UP(32, 32); i++)
  529. ptr[i] = readl_relaxed(dist_base + GIC_DIST_ENABLE_SET + i * 4);
  530. ptr = raw_cpu_ptr(gic->saved_ppi_active);
  531. for (i = 0; i < DIV_ROUND_UP(32, 32); i++)
  532. ptr[i] = readl_relaxed(dist_base + GIC_DIST_ACTIVE_SET + i * 4);
  533. ptr = raw_cpu_ptr(gic->saved_ppi_conf);
  534. for (i = 0; i < DIV_ROUND_UP(32, 16); i++)
  535. ptr[i] = readl_relaxed(dist_base + GIC_DIST_CONFIG + i * 4);
  536. }
  537. static void gic_cpu_restore(struct gic_chip_data *gic)
  538. {
  539. int i;
  540. u32 *ptr;
  541. void __iomem *dist_base;
  542. void __iomem *cpu_base;
  543. if (WARN_ON(!gic))
  544. return;
  545. dist_base = gic_data_dist_base(gic);
  546. cpu_base = gic_data_cpu_base(gic);
  547. if (!dist_base || !cpu_base)
  548. return;
  549. ptr = raw_cpu_ptr(gic->saved_ppi_enable);
  550. for (i = 0; i < DIV_ROUND_UP(32, 32); i++) {
  551. writel_relaxed(GICD_INT_EN_CLR_X32,
  552. dist_base + GIC_DIST_ENABLE_CLEAR + i * 4);
  553. writel_relaxed(ptr[i], dist_base + GIC_DIST_ENABLE_SET + i * 4);
  554. }
  555. ptr = raw_cpu_ptr(gic->saved_ppi_active);
  556. for (i = 0; i < DIV_ROUND_UP(32, 32); i++) {
  557. writel_relaxed(GICD_INT_EN_CLR_X32,
  558. dist_base + GIC_DIST_ACTIVE_CLEAR + i * 4);
  559. writel_relaxed(ptr[i], dist_base + GIC_DIST_ACTIVE_SET + i * 4);
  560. }
  561. ptr = raw_cpu_ptr(gic->saved_ppi_conf);
  562. for (i = 0; i < DIV_ROUND_UP(32, 16); i++)
  563. writel_relaxed(ptr[i], dist_base + GIC_DIST_CONFIG + i * 4);
  564. for (i = 0; i < DIV_ROUND_UP(32, 4); i++)
  565. writel_relaxed(GICD_INT_DEF_PRI_X4,
  566. dist_base + GIC_DIST_PRI + i * 4);
  567. writel_relaxed(GICC_INT_PRI_THRESHOLD, cpu_base + GIC_CPU_PRIMASK);
  568. gic_cpu_if_up(gic);
  569. }
  570. static int gic_notifier(struct notifier_block *self, unsigned long cmd, void *v)
  571. {
  572. int i;
  573. for (i = 0; i < CONFIG_ARM_GIC_MAX_NR; i++) {
  574. #ifdef CONFIG_GIC_NON_BANKED
  575. /* Skip over unused GICs */
  576. if (!gic_data[i].get_base)
  577. continue;
  578. #endif
  579. switch (cmd) {
  580. case CPU_PM_ENTER:
  581. gic_cpu_save(&gic_data[i]);
  582. break;
  583. case CPU_PM_ENTER_FAILED:
  584. case CPU_PM_EXIT:
  585. gic_cpu_restore(&gic_data[i]);
  586. break;
  587. case CPU_CLUSTER_PM_ENTER:
  588. gic_dist_save(&gic_data[i]);
  589. break;
  590. case CPU_CLUSTER_PM_ENTER_FAILED:
  591. case CPU_CLUSTER_PM_EXIT:
  592. gic_dist_restore(&gic_data[i]);
  593. break;
  594. }
  595. }
  596. return NOTIFY_OK;
  597. }
  598. static struct notifier_block gic_notifier_block = {
  599. .notifier_call = gic_notifier,
  600. };
  601. static int __init gic_pm_init(struct gic_chip_data *gic)
  602. {
  603. gic->saved_ppi_enable = __alloc_percpu(DIV_ROUND_UP(32, 32) * 4,
  604. sizeof(u32));
  605. if (WARN_ON(!gic->saved_ppi_enable))
  606. return -ENOMEM;
  607. gic->saved_ppi_active = __alloc_percpu(DIV_ROUND_UP(32, 32) * 4,
  608. sizeof(u32));
  609. if (WARN_ON(!gic->saved_ppi_active))
  610. goto free_ppi_enable;
  611. gic->saved_ppi_conf = __alloc_percpu(DIV_ROUND_UP(32, 16) * 4,
  612. sizeof(u32));
  613. if (WARN_ON(!gic->saved_ppi_conf))
  614. goto free_ppi_active;
  615. if (gic == &gic_data[0])
  616. cpu_pm_register_notifier(&gic_notifier_block);
  617. return 0;
  618. free_ppi_active:
  619. free_percpu(gic->saved_ppi_active);
  620. free_ppi_enable:
  621. free_percpu(gic->saved_ppi_enable);
  622. return -ENOMEM;
  623. }
  624. #else
  625. static int __init gic_pm_init(struct gic_chip_data *gic)
  626. {
  627. return 0;
  628. }
  629. #endif
  630. #ifdef CONFIG_SMP
  631. static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
  632. {
  633. int cpu;
  634. unsigned long flags, map = 0;
  635. raw_spin_lock_irqsave(&irq_controller_lock, flags);
  636. /* Convert our logical CPU mask into a physical one. */
  637. for_each_cpu(cpu, mask)
  638. map |= gic_cpu_map[cpu];
  639. /*
  640. * Ensure that stores to Normal memory are visible to the
  641. * other CPUs before they observe us issuing the IPI.
  642. */
  643. dmb(ishst);
  644. /* this always happens on GIC0 */
  645. writel_relaxed(map << 16 | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT);
  646. raw_spin_unlock_irqrestore(&irq_controller_lock, flags);
  647. }
  648. #endif
  649. #ifdef CONFIG_BL_SWITCHER
  650. /*
  651. * gic_send_sgi - send a SGI directly to given CPU interface number
  652. *
  653. * cpu_id: the ID for the destination CPU interface
  654. * irq: the IPI number to send a SGI for
  655. */
  656. void gic_send_sgi(unsigned int cpu_id, unsigned int irq)
  657. {
  658. BUG_ON(cpu_id >= NR_GIC_CPU_IF);
  659. cpu_id = 1 << cpu_id;
  660. /* this always happens on GIC0 */
  661. writel_relaxed((cpu_id << 16) | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT);
  662. }
  663. /*
  664. * gic_get_cpu_id - get the CPU interface ID for the specified CPU
  665. *
  666. * @cpu: the logical CPU number to get the GIC ID for.
  667. *
  668. * Return the CPU interface ID for the given logical CPU number,
  669. * or -1 if the CPU number is too large or the interface ID is
  670. * unknown (more than one bit set).
  671. */
  672. int gic_get_cpu_id(unsigned int cpu)
  673. {
  674. unsigned int cpu_bit;
  675. if (cpu >= NR_GIC_CPU_IF)
  676. return -1;
  677. cpu_bit = gic_cpu_map[cpu];
  678. if (cpu_bit & (cpu_bit - 1))
  679. return -1;
  680. return __ffs(cpu_bit);
  681. }
  682. /*
  683. * gic_migrate_target - migrate IRQs to another CPU interface
  684. *
  685. * @new_cpu_id: the CPU target ID to migrate IRQs to
  686. *
  687. * Migrate all peripheral interrupts with a target matching the current CPU
  688. * to the interface corresponding to @new_cpu_id. The CPU interface mapping
  689. * is also updated. Targets to other CPU interfaces are unchanged.
  690. * This must be called with IRQs locally disabled.
  691. */
  692. void gic_migrate_target(unsigned int new_cpu_id)
  693. {
  694. unsigned int cur_cpu_id, gic_irqs, gic_nr = 0;
  695. void __iomem *dist_base;
  696. int i, ror_val, cpu = smp_processor_id();
  697. u32 val, cur_target_mask, active_mask;
  698. BUG_ON(gic_nr >= CONFIG_ARM_GIC_MAX_NR);
  699. dist_base = gic_data_dist_base(&gic_data[gic_nr]);
  700. if (!dist_base)
  701. return;
  702. gic_irqs = gic_data[gic_nr].gic_irqs;
  703. cur_cpu_id = __ffs(gic_cpu_map[cpu]);
  704. cur_target_mask = 0x01010101 << cur_cpu_id;
  705. ror_val = (cur_cpu_id - new_cpu_id) & 31;
  706. raw_spin_lock(&irq_controller_lock);
  707. /* Update the target interface for this logical CPU */
  708. gic_cpu_map[cpu] = 1 << new_cpu_id;
  709. /*
  710. * Find all the peripheral interrupts targetting the current
  711. * CPU interface and migrate them to the new CPU interface.
  712. * We skip DIST_TARGET 0 to 7 as they are read-only.
  713. */
  714. for (i = 8; i < DIV_ROUND_UP(gic_irqs, 4); i++) {
  715. val = readl_relaxed(dist_base + GIC_DIST_TARGET + i * 4);
  716. active_mask = val & cur_target_mask;
  717. if (active_mask) {
  718. val &= ~active_mask;
  719. val |= ror32(active_mask, ror_val);
  720. writel_relaxed(val, dist_base + GIC_DIST_TARGET + i*4);
  721. }
  722. }
  723. raw_spin_unlock(&irq_controller_lock);
  724. /*
  725. * Now let's migrate and clear any potential SGIs that might be
  726. * pending for us (cur_cpu_id). Since GIC_DIST_SGI_PENDING_SET
  727. * is a banked register, we can only forward the SGI using
  728. * GIC_DIST_SOFTINT. The original SGI source is lost but Linux
  729. * doesn't use that information anyway.
  730. *
  731. * For the same reason we do not adjust SGI source information
  732. * for previously sent SGIs by us to other CPUs either.
  733. */
  734. for (i = 0; i < 16; i += 4) {
  735. int j;
  736. val = readl_relaxed(dist_base + GIC_DIST_SGI_PENDING_SET + i);
  737. if (!val)
  738. continue;
  739. writel_relaxed(val, dist_base + GIC_DIST_SGI_PENDING_CLEAR + i);
  740. for (j = i; j < i + 4; j++) {
  741. if (val & 0xff)
  742. writel_relaxed((1 << (new_cpu_id + 16)) | j,
  743. dist_base + GIC_DIST_SOFTINT);
  744. val >>= 8;
  745. }
  746. }
  747. }
  748. /*
  749. * gic_get_sgir_physaddr - get the physical address for the SGI register
  750. *
  751. * REturn the physical address of the SGI register to be used
  752. * by some early assembly code when the kernel is not yet available.
  753. */
  754. static unsigned long gic_dist_physaddr;
  755. unsigned long gic_get_sgir_physaddr(void)
  756. {
  757. if (!gic_dist_physaddr)
  758. return 0;
  759. return gic_dist_physaddr + GIC_DIST_SOFTINT;
  760. }
  761. void __init gic_init_physaddr(struct device_node *node)
  762. {
  763. struct resource res;
  764. if (of_address_to_resource(node, 0, &res) == 0) {
  765. gic_dist_physaddr = res.start;
  766. pr_info("GIC physical location is %#lx\n", gic_dist_physaddr);
  767. }
  768. }
  769. #else
  770. #define gic_init_physaddr(node) do { } while (0)
  771. #endif
  772. static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
  773. irq_hw_number_t hw)
  774. {
  775. struct gic_chip_data *gic = d->host_data;
  776. if (hw < 32) {
  777. irq_set_percpu_devid(irq);
  778. irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data,
  779. handle_percpu_devid_irq, NULL, NULL);
  780. irq_set_status_flags(irq, IRQ_NOAUTOEN);
  781. } else {
  782. irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data,
  783. handle_fasteoi_irq, NULL, NULL);
  784. irq_set_probe(irq);
  785. }
  786. return 0;
  787. }
  788. static void gic_irq_domain_unmap(struct irq_domain *d, unsigned int irq)
  789. {
  790. }
  791. static int gic_irq_domain_translate(struct irq_domain *d,
  792. struct irq_fwspec *fwspec,
  793. unsigned long *hwirq,
  794. unsigned int *type)
  795. {
  796. if (is_of_node(fwspec->fwnode)) {
  797. if (fwspec->param_count < 3)
  798. return -EINVAL;
  799. /* Get the interrupt number and add 16 to skip over SGIs */
  800. *hwirq = fwspec->param[1] + 16;
  801. /*
  802. * For SPIs, we need to add 16 more to get the GIC irq
  803. * ID number
  804. */
  805. if (!fwspec->param[0])
  806. *hwirq += 16;
  807. *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
  808. return 0;
  809. }
  810. if (is_fwnode_irqchip(fwspec->fwnode)) {
  811. if(fwspec->param_count != 2)
  812. return -EINVAL;
  813. *hwirq = fwspec->param[0];
  814. *type = fwspec->param[1];
  815. return 0;
  816. }
  817. return -EINVAL;
  818. }
  819. #ifdef CONFIG_SMP
  820. static int gic_secondary_init(struct notifier_block *nfb, unsigned long action,
  821. void *hcpu)
  822. {
  823. if (action == CPU_STARTING || action == CPU_STARTING_FROZEN)
  824. gic_cpu_init(&gic_data[0]);
  825. return NOTIFY_OK;
  826. }
  827. /*
  828. * Notifier for enabling the GIC CPU interface. Set an arbitrarily high
  829. * priority because the GIC needs to be up before the ARM generic timers.
  830. */
  831. static struct notifier_block gic_cpu_notifier = {
  832. .notifier_call = gic_secondary_init,
  833. .priority = 100,
  834. };
  835. #endif
  836. static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
  837. unsigned int nr_irqs, void *arg)
  838. {
  839. int i, ret;
  840. irq_hw_number_t hwirq;
  841. unsigned int type = IRQ_TYPE_NONE;
  842. struct irq_fwspec *fwspec = arg;
  843. ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type);
  844. if (ret)
  845. return ret;
  846. for (i = 0; i < nr_irqs; i++)
  847. gic_irq_domain_map(domain, virq + i, hwirq + i);
  848. return 0;
  849. }
  850. static const struct irq_domain_ops gic_irq_domain_hierarchy_ops = {
  851. .translate = gic_irq_domain_translate,
  852. .alloc = gic_irq_domain_alloc,
  853. .free = irq_domain_free_irqs_top,
  854. };
  855. static const struct irq_domain_ops gic_irq_domain_ops = {
  856. .map = gic_irq_domain_map,
  857. .unmap = gic_irq_domain_unmap,
  858. };
  859. static int __init __gic_init_bases(struct gic_chip_data *gic, int irq_start,
  860. struct fwnode_handle *handle)
  861. {
  862. irq_hw_number_t hwirq_base;
  863. int gic_irqs, irq_base, i, ret;
  864. if (WARN_ON(!gic || gic->domain))
  865. return -EINVAL;
  866. /* Initialize irq_chip */
  867. gic->chip = gic_chip;
  868. if (static_key_true(&supports_deactivate) && gic == &gic_data[0]) {
  869. gic->chip.irq_mask = gic_eoimode1_mask_irq;
  870. gic->chip.irq_eoi = gic_eoimode1_eoi_irq;
  871. gic->chip.irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity;
  872. gic->chip.name = kasprintf(GFP_KERNEL, "GICv2");
  873. } else {
  874. gic->chip.name = kasprintf(GFP_KERNEL, "GIC-%d",
  875. (int)(gic - &gic_data[0]));
  876. }
  877. #ifdef CONFIG_SMP
  878. if (gic == &gic_data[0])
  879. gic->chip.irq_set_affinity = gic_set_affinity;
  880. #endif
  881. if (IS_ENABLED(CONFIG_GIC_NON_BANKED) && gic->percpu_offset) {
  882. /* Frankein-GIC without banked registers... */
  883. unsigned int cpu;
  884. gic->dist_base.percpu_base = alloc_percpu(void __iomem *);
  885. gic->cpu_base.percpu_base = alloc_percpu(void __iomem *);
  886. if (WARN_ON(!gic->dist_base.percpu_base ||
  887. !gic->cpu_base.percpu_base)) {
  888. ret = -ENOMEM;
  889. goto error;
  890. }
  891. for_each_possible_cpu(cpu) {
  892. u32 mpidr = cpu_logical_map(cpu);
  893. u32 core_id = MPIDR_AFFINITY_LEVEL(mpidr, 0);
  894. unsigned long offset = gic->percpu_offset * core_id;
  895. *per_cpu_ptr(gic->dist_base.percpu_base, cpu) =
  896. gic->raw_dist_base + offset;
  897. *per_cpu_ptr(gic->cpu_base.percpu_base, cpu) =
  898. gic->raw_cpu_base + offset;
  899. }
  900. gic_set_base_accessor(gic, gic_get_percpu_base);
  901. } else {
  902. /* Normal, sane GIC... */
  903. WARN(gic->percpu_offset,
  904. "GIC_NON_BANKED not enabled, ignoring %08x offset!",
  905. gic->percpu_offset);
  906. gic->dist_base.common_base = gic->raw_dist_base;
  907. gic->cpu_base.common_base = gic->raw_cpu_base;
  908. gic_set_base_accessor(gic, gic_get_common_base);
  909. }
  910. /*
  911. * Find out how many interrupts are supported.
  912. * The GIC only supports up to 1020 interrupt sources.
  913. */
  914. gic_irqs = readl_relaxed(gic_data_dist_base(gic) + GIC_DIST_CTR) & 0x1f;
  915. gic_irqs = (gic_irqs + 1) * 32;
  916. if (gic_irqs > 1020)
  917. gic_irqs = 1020;
  918. gic->gic_irqs = gic_irqs;
  919. if (handle) { /* DT/ACPI */
  920. gic->domain = irq_domain_create_linear(handle, gic_irqs,
  921. &gic_irq_domain_hierarchy_ops,
  922. gic);
  923. } else { /* Legacy support */
  924. /*
  925. * For primary GICs, skip over SGIs.
  926. * For secondary GICs, skip over PPIs, too.
  927. */
  928. if (gic == &gic_data[0] && (irq_start & 31) > 0) {
  929. hwirq_base = 16;
  930. if (irq_start != -1)
  931. irq_start = (irq_start & ~31) + 16;
  932. } else {
  933. hwirq_base = 32;
  934. }
  935. gic_irqs -= hwirq_base; /* calculate # of irqs to allocate */
  936. irq_base = irq_alloc_descs(irq_start, 16, gic_irqs,
  937. numa_node_id());
  938. if (IS_ERR_VALUE(irq_base)) {
  939. WARN(1, "Cannot allocate irq_descs @ IRQ%d, assuming pre-allocated\n",
  940. irq_start);
  941. irq_base = irq_start;
  942. }
  943. gic->domain = irq_domain_add_legacy(NULL, gic_irqs, irq_base,
  944. hwirq_base, &gic_irq_domain_ops, gic);
  945. }
  946. if (WARN_ON(!gic->domain)) {
  947. ret = -ENODEV;
  948. goto error;
  949. }
  950. if (gic == &gic_data[0]) {
  951. /*
  952. * Initialize the CPU interface map to all CPUs.
  953. * It will be refined as each CPU probes its ID.
  954. * This is only necessary for the primary GIC.
  955. */
  956. for (i = 0; i < NR_GIC_CPU_IF; i++)
  957. gic_cpu_map[i] = 0xff;
  958. #ifdef CONFIG_SMP
  959. set_smp_cross_call(gic_raise_softirq);
  960. register_cpu_notifier(&gic_cpu_notifier);
  961. #endif
  962. set_handle_irq(gic_handle_irq);
  963. if (static_key_true(&supports_deactivate))
  964. pr_info("GIC: Using split EOI/Deactivate mode\n");
  965. }
  966. gic_dist_init(gic);
  967. ret = gic_cpu_init(gic);
  968. if (ret)
  969. goto error;
  970. ret = gic_pm_init(gic);
  971. if (ret)
  972. goto error;
  973. return 0;
  974. error:
  975. if (IS_ENABLED(CONFIG_GIC_NON_BANKED) && gic->percpu_offset) {
  976. free_percpu(gic->dist_base.percpu_base);
  977. free_percpu(gic->cpu_base.percpu_base);
  978. }
  979. kfree(gic->chip.name);
  980. return ret;
  981. }
  982. void __init gic_init(unsigned int gic_nr, int irq_start,
  983. void __iomem *dist_base, void __iomem *cpu_base)
  984. {
  985. struct gic_chip_data *gic;
  986. if (WARN_ON(gic_nr >= CONFIG_ARM_GIC_MAX_NR))
  987. return;
  988. /*
  989. * Non-DT/ACPI systems won't run a hypervisor, so let's not
  990. * bother with these...
  991. */
  992. static_key_slow_dec(&supports_deactivate);
  993. gic = &gic_data[gic_nr];
  994. gic->raw_dist_base = dist_base;
  995. gic->raw_cpu_base = cpu_base;
  996. __gic_init_bases(gic, irq_start, NULL);
  997. }
  998. static void gic_teardown(struct gic_chip_data *gic)
  999. {
  1000. if (WARN_ON(!gic))
  1001. return;
  1002. if (gic->raw_dist_base)
  1003. iounmap(gic->raw_dist_base);
  1004. if (gic->raw_cpu_base)
  1005. iounmap(gic->raw_cpu_base);
  1006. }
  1007. #ifdef CONFIG_OF
  1008. static int gic_cnt __initdata;
  1009. static bool gic_check_eoimode(struct device_node *node, void __iomem **base)
  1010. {
  1011. struct resource cpuif_res;
  1012. of_address_to_resource(node, 1, &cpuif_res);
  1013. if (!is_hyp_mode_available())
  1014. return false;
  1015. if (resource_size(&cpuif_res) < SZ_8K)
  1016. return false;
  1017. if (resource_size(&cpuif_res) == SZ_128K) {
  1018. u32 val_low, val_high;
  1019. /*
  1020. * Verify that we have the first 4kB of a GIC400
  1021. * aliased over the first 64kB by checking the
  1022. * GICC_IIDR register on both ends.
  1023. */
  1024. val_low = readl_relaxed(*base + GIC_CPU_IDENT);
  1025. val_high = readl_relaxed(*base + GIC_CPU_IDENT + 0xf000);
  1026. if ((val_low & 0xffff0fff) != 0x0202043B ||
  1027. val_low != val_high)
  1028. return false;
  1029. /*
  1030. * Move the base up by 60kB, so that we have a 8kB
  1031. * contiguous region, which allows us to use GICC_DIR
  1032. * at its normal offset. Please pass me that bucket.
  1033. */
  1034. *base += 0xf000;
  1035. cpuif_res.start += 0xf000;
  1036. pr_warn("GIC: Adjusting CPU interface base to %pa",
  1037. &cpuif_res.start);
  1038. }
  1039. return true;
  1040. }
  1041. static int gic_of_setup(struct gic_chip_data *gic, struct device_node *node)
  1042. {
  1043. if (!gic || !node)
  1044. return -EINVAL;
  1045. gic->raw_dist_base = of_iomap(node, 0);
  1046. if (WARN(!gic->raw_dist_base, "unable to map gic dist registers\n"))
  1047. goto error;
  1048. gic->raw_cpu_base = of_iomap(node, 1);
  1049. if (WARN(!gic->raw_cpu_base, "unable to map gic cpu registers\n"))
  1050. goto error;
  1051. if (of_property_read_u32(node, "cpu-offset", &gic->percpu_offset))
  1052. gic->percpu_offset = 0;
  1053. return 0;
  1054. error:
  1055. gic_teardown(gic);
  1056. return -ENOMEM;
  1057. }
  1058. int __init
  1059. gic_of_init(struct device_node *node, struct device_node *parent)
  1060. {
  1061. struct gic_chip_data *gic;
  1062. int irq, ret;
  1063. if (WARN_ON(!node))
  1064. return -ENODEV;
  1065. if (WARN_ON(gic_cnt >= CONFIG_ARM_GIC_MAX_NR))
  1066. return -EINVAL;
  1067. gic = &gic_data[gic_cnt];
  1068. ret = gic_of_setup(gic, node);
  1069. if (ret)
  1070. return ret;
  1071. /*
  1072. * Disable split EOI/Deactivate if either HYP is not available
  1073. * or the CPU interface is too small.
  1074. */
  1075. if (gic_cnt == 0 && !gic_check_eoimode(node, &gic->raw_cpu_base))
  1076. static_key_slow_dec(&supports_deactivate);
  1077. ret = __gic_init_bases(gic, -1, &node->fwnode);
  1078. if (ret) {
  1079. gic_teardown(gic);
  1080. return ret;
  1081. }
  1082. if (!gic_cnt)
  1083. gic_init_physaddr(node);
  1084. if (parent) {
  1085. irq = irq_of_parse_and_map(node, 0);
  1086. gic_cascade_irq(gic_cnt, irq);
  1087. }
  1088. if (IS_ENABLED(CONFIG_ARM_GIC_V2M))
  1089. gicv2m_init(&node->fwnode, gic_data[gic_cnt].domain);
  1090. gic_cnt++;
  1091. return 0;
  1092. }
  1093. IRQCHIP_DECLARE(gic_400, "arm,gic-400", gic_of_init);
  1094. IRQCHIP_DECLARE(arm11mp_gic, "arm,arm11mp-gic", gic_of_init);
  1095. IRQCHIP_DECLARE(arm1176jzf_dc_gic, "arm,arm1176jzf-devchip-gic", gic_of_init);
  1096. IRQCHIP_DECLARE(cortex_a15_gic, "arm,cortex-a15-gic", gic_of_init);
  1097. IRQCHIP_DECLARE(cortex_a9_gic, "arm,cortex-a9-gic", gic_of_init);
  1098. IRQCHIP_DECLARE(cortex_a7_gic, "arm,cortex-a7-gic", gic_of_init);
  1099. IRQCHIP_DECLARE(msm_8660_qgic, "qcom,msm-8660-qgic", gic_of_init);
  1100. IRQCHIP_DECLARE(msm_qgic2, "qcom,msm-qgic2", gic_of_init);
  1101. IRQCHIP_DECLARE(pl390, "arm,pl390", gic_of_init);
  1102. #endif
  1103. #ifdef CONFIG_ACPI
  1104. static phys_addr_t cpu_phy_base __initdata;
  1105. static int __init
  1106. gic_acpi_parse_madt_cpu(struct acpi_subtable_header *header,
  1107. const unsigned long end)
  1108. {
  1109. struct acpi_madt_generic_interrupt *processor;
  1110. phys_addr_t gic_cpu_base;
  1111. static int cpu_base_assigned;
  1112. processor = (struct acpi_madt_generic_interrupt *)header;
  1113. if (BAD_MADT_GICC_ENTRY(processor, end))
  1114. return -EINVAL;
  1115. /*
  1116. * There is no support for non-banked GICv1/2 register in ACPI spec.
  1117. * All CPU interface addresses have to be the same.
  1118. */
  1119. gic_cpu_base = processor->base_address;
  1120. if (cpu_base_assigned && gic_cpu_base != cpu_phy_base)
  1121. return -EINVAL;
  1122. cpu_phy_base = gic_cpu_base;
  1123. cpu_base_assigned = 1;
  1124. return 0;
  1125. }
  1126. /* The things you have to do to just *count* something... */
  1127. static int __init acpi_dummy_func(struct acpi_subtable_header *header,
  1128. const unsigned long end)
  1129. {
  1130. return 0;
  1131. }
  1132. static bool __init acpi_gic_redist_is_present(void)
  1133. {
  1134. return acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR,
  1135. acpi_dummy_func, 0) > 0;
  1136. }
  1137. static bool __init gic_validate_dist(struct acpi_subtable_header *header,
  1138. struct acpi_probe_entry *ape)
  1139. {
  1140. struct acpi_madt_generic_distributor *dist;
  1141. dist = (struct acpi_madt_generic_distributor *)header;
  1142. return (dist->version == ape->driver_data &&
  1143. (dist->version != ACPI_MADT_GIC_VERSION_NONE ||
  1144. !acpi_gic_redist_is_present()));
  1145. }
  1146. #define ACPI_GICV2_DIST_MEM_SIZE (SZ_4K)
  1147. #define ACPI_GIC_CPU_IF_MEM_SIZE (SZ_8K)
  1148. static int __init gic_v2_acpi_init(struct acpi_subtable_header *header,
  1149. const unsigned long end)
  1150. {
  1151. struct acpi_madt_generic_distributor *dist;
  1152. struct fwnode_handle *domain_handle;
  1153. struct gic_chip_data *gic = &gic_data[0];
  1154. int count, ret;
  1155. /* Collect CPU base addresses */
  1156. count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
  1157. gic_acpi_parse_madt_cpu, 0);
  1158. if (count <= 0) {
  1159. pr_err("No valid GICC entries exist\n");
  1160. return -EINVAL;
  1161. }
  1162. gic->raw_cpu_base = ioremap(cpu_phy_base, ACPI_GIC_CPU_IF_MEM_SIZE);
  1163. if (!gic->raw_cpu_base) {
  1164. pr_err("Unable to map GICC registers\n");
  1165. return -ENOMEM;
  1166. }
  1167. dist = (struct acpi_madt_generic_distributor *)header;
  1168. gic->raw_dist_base = ioremap(dist->base_address,
  1169. ACPI_GICV2_DIST_MEM_SIZE);
  1170. if (!gic->raw_dist_base) {
  1171. pr_err("Unable to map GICD registers\n");
  1172. gic_teardown(gic);
  1173. return -ENOMEM;
  1174. }
  1175. /*
  1176. * Disable split EOI/Deactivate if HYP is not available. ACPI
  1177. * guarantees that we'll always have a GICv2, so the CPU
  1178. * interface will always be the right size.
  1179. */
  1180. if (!is_hyp_mode_available())
  1181. static_key_slow_dec(&supports_deactivate);
  1182. /*
  1183. * Initialize GIC instance zero (no multi-GIC support).
  1184. */
  1185. domain_handle = irq_domain_alloc_fwnode(gic->raw_dist_base);
  1186. if (!domain_handle) {
  1187. pr_err("Unable to allocate domain handle\n");
  1188. gic_teardown(gic);
  1189. return -ENOMEM;
  1190. }
  1191. ret = __gic_init_bases(gic, -1, domain_handle);
  1192. if (ret) {
  1193. pr_err("Failed to initialise GIC\n");
  1194. irq_domain_free_fwnode(domain_handle);
  1195. gic_teardown(gic);
  1196. return ret;
  1197. }
  1198. acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, domain_handle);
  1199. if (IS_ENABLED(CONFIG_ARM_GIC_V2M))
  1200. gicv2m_init(NULL, gic_data[0].domain);
  1201. return 0;
  1202. }
  1203. IRQCHIP_ACPI_DECLARE(gic_v2, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
  1204. gic_validate_dist, ACPI_MADT_GIC_VERSION_V2,
  1205. gic_v2_acpi_init);
  1206. IRQCHIP_ACPI_DECLARE(gic_v2_maybe, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
  1207. gic_validate_dist, ACPI_MADT_GIC_VERSION_NONE,
  1208. gic_v2_acpi_init);
  1209. #endif