context.c 7.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260
  1. /*
  2. * linux/arch/arm/mm/context.c
  3. *
  4. * Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved.
  5. * Copyright (C) 2012 ARM Limited
  6. *
  7. * Author: Will Deacon <will.deacon@arm.com>
  8. *
  9. * This program is free software; you can redistribute it and/or modify
  10. * it under the terms of the GNU General Public License version 2 as
  11. * published by the Free Software Foundation.
  12. */
  13. #include <linux/init.h>
  14. #include <linux/sched.h>
  15. #include <linux/mm.h>
  16. #include <linux/smp.h>
  17. #include <linux/percpu.h>
  18. #include <asm/mmu_context.h>
  19. #include <asm/smp_plat.h>
  20. #include <asm/thread_notify.h>
  21. #include <asm/tlbflush.h>
  22. #include <asm/proc-fns.h>
  23. /*
  24. * On ARMv6, we have the following structure in the Context ID:
  25. *
  26. * 31 7 0
  27. * +-------------------------+-----------+
  28. * | process ID | ASID |
  29. * +-------------------------+-----------+
  30. * | context ID |
  31. * +-------------------------------------+
  32. *
  33. * The ASID is used to tag entries in the CPU caches and TLBs.
  34. * The context ID is used by debuggers and trace logic, and
  35. * should be unique within all running processes.
  36. *
  37. * In big endian operation, the two 32 bit words are swapped if accessed
  38. * by non-64-bit operations.
  39. */
  40. #define ASID_FIRST_VERSION (1ULL << ASID_BITS)
  41. #define NUM_USER_ASIDS ASID_FIRST_VERSION
  42. static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
  43. static atomic64_t asid_generation = ATOMIC64_INIT(ASID_FIRST_VERSION);
  44. static DECLARE_BITMAP(asid_map, NUM_USER_ASIDS);
  45. static DEFINE_PER_CPU(atomic64_t, active_asids);
  46. static DEFINE_PER_CPU(u64, reserved_asids);
  47. static cpumask_t tlb_flush_pending;
  48. #ifdef CONFIG_ARM_ERRATA_798181
  49. void a15_erratum_get_cpumask(int this_cpu, struct mm_struct *mm,
  50. cpumask_t *mask)
  51. {
  52. int cpu;
  53. unsigned long flags;
  54. u64 context_id, asid;
  55. raw_spin_lock_irqsave(&cpu_asid_lock, flags);
  56. context_id = mm->context.id.counter;
  57. for_each_online_cpu(cpu) {
  58. if (cpu == this_cpu)
  59. continue;
  60. /*
  61. * We only need to send an IPI if the other CPUs are
  62. * running the same ASID as the one being invalidated.
  63. */
  64. asid = per_cpu(active_asids, cpu).counter;
  65. if (asid == 0)
  66. asid = per_cpu(reserved_asids, cpu);
  67. if (context_id == asid)
  68. cpumask_set_cpu(cpu, mask);
  69. }
  70. raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
  71. }
  72. #endif
  73. #ifdef CONFIG_ARM_LPAE
  74. /*
  75. * With LPAE, the ASID and page tables are updated atomicly, so there is
  76. * no need for a reserved set of tables (the active ASID tracking prevents
  77. * any issues across a rollover).
  78. */
  79. #define cpu_set_reserved_ttbr0()
  80. #else
  81. static void cpu_set_reserved_ttbr0(void)
  82. {
  83. u32 ttb;
  84. /*
  85. * Copy TTBR1 into TTBR0.
  86. * This points at swapper_pg_dir, which contains only global
  87. * entries so any speculative walks are perfectly safe.
  88. */
  89. asm volatile(
  90. " mrc p15, 0, %0, c2, c0, 1 @ read TTBR1\n"
  91. " mcr p15, 0, %0, c2, c0, 0 @ set TTBR0\n"
  92. : "=r" (ttb));
  93. isb();
  94. }
  95. #endif
  96. #ifdef CONFIG_PID_IN_CONTEXTIDR
  97. static int contextidr_notifier(struct notifier_block *unused, unsigned long cmd,
  98. void *t)
  99. {
  100. u32 contextidr;
  101. pid_t pid;
  102. struct thread_info *thread = t;
  103. if (cmd != THREAD_NOTIFY_SWITCH)
  104. return NOTIFY_DONE;
  105. pid = task_pid_nr(thread->task) << ASID_BITS;
  106. asm volatile(
  107. " mrc p15, 0, %0, c13, c0, 1\n"
  108. " and %0, %0, %2\n"
  109. " orr %0, %0, %1\n"
  110. " mcr p15, 0, %0, c13, c0, 1\n"
  111. : "=r" (contextidr), "+r" (pid)
  112. : "I" (~ASID_MASK));
  113. isb();
  114. return NOTIFY_OK;
  115. }
  116. static struct notifier_block contextidr_notifier_block = {
  117. .notifier_call = contextidr_notifier,
  118. };
  119. static int __init contextidr_notifier_init(void)
  120. {
  121. return thread_register_notifier(&contextidr_notifier_block);
  122. }
  123. arch_initcall(contextidr_notifier_init);
  124. #endif
  125. static void flush_context(unsigned int cpu)
  126. {
  127. int i;
  128. u64 asid;
  129. /* Update the list of reserved ASIDs and the ASID bitmap. */
  130. bitmap_clear(asid_map, 0, NUM_USER_ASIDS);
  131. for_each_possible_cpu(i) {
  132. if (i == cpu) {
  133. asid = 0;
  134. } else {
  135. asid = atomic64_xchg(&per_cpu(active_asids, i), 0);
  136. /*
  137. * If this CPU has already been through a
  138. * rollover, but hasn't run another task in
  139. * the meantime, we must preserve its reserved
  140. * ASID, as this is the only trace we have of
  141. * the process it is still running.
  142. */
  143. if (asid == 0)
  144. asid = per_cpu(reserved_asids, i);
  145. __set_bit(asid & ~ASID_MASK, asid_map);
  146. }
  147. per_cpu(reserved_asids, i) = asid;
  148. }
  149. /* Queue a TLB invalidate and flush the I-cache if necessary. */
  150. cpumask_setall(&tlb_flush_pending);
  151. if (icache_is_vivt_asid_tagged())
  152. __flush_icache_all();
  153. }
  154. static int is_reserved_asid(u64 asid)
  155. {
  156. int cpu;
  157. for_each_possible_cpu(cpu)
  158. if (per_cpu(reserved_asids, cpu) == asid)
  159. return 1;
  160. return 0;
  161. }
  162. static u64 new_context(struct mm_struct *mm, unsigned int cpu)
  163. {
  164. static u32 cur_idx = 1;
  165. u64 asid = atomic64_read(&mm->context.id);
  166. u64 generation = atomic64_read(&asid_generation);
  167. if (asid != 0 && is_reserved_asid(asid)) {
  168. /*
  169. * Our current ASID was active during a rollover, we can
  170. * continue to use it and this was just a false alarm.
  171. */
  172. asid = generation | (asid & ~ASID_MASK);
  173. } else {
  174. /*
  175. * Allocate a free ASID. If we can't find one, take a
  176. * note of the currently active ASIDs and mark the TLBs
  177. * as requiring flushes. We always count from ASID #1,
  178. * as we reserve ASID #0 to switch via TTBR0 and to
  179. * avoid speculative page table walks from hitting in
  180. * any partial walk caches, which could be populated
  181. * from overlapping level-1 descriptors used to map both
  182. * the module area and the userspace stack.
  183. */
  184. asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, cur_idx);
  185. if (asid == NUM_USER_ASIDS) {
  186. generation = atomic64_add_return(ASID_FIRST_VERSION,
  187. &asid_generation);
  188. flush_context(cpu);
  189. asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1);
  190. }
  191. __set_bit(asid, asid_map);
  192. cur_idx = asid;
  193. asid |= generation;
  194. cpumask_clear(mm_cpumask(mm));
  195. }
  196. return asid;
  197. }
  198. void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk)
  199. {
  200. unsigned long flags;
  201. unsigned int cpu = smp_processor_id();
  202. u64 asid;
  203. if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq))
  204. __check_vmalloc_seq(mm);
  205. /*
  206. * We cannot update the pgd and the ASID atomicly with classic
  207. * MMU, so switch exclusively to global mappings to avoid
  208. * speculative page table walking with the wrong TTBR.
  209. */
  210. cpu_set_reserved_ttbr0();
  211. asid = atomic64_read(&mm->context.id);
  212. if (!((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS)
  213. && atomic64_xchg(&per_cpu(active_asids, cpu), asid))
  214. goto switch_mm_fastpath;
  215. raw_spin_lock_irqsave(&cpu_asid_lock, flags);
  216. /* Check that our ASID belongs to the current generation. */
  217. asid = atomic64_read(&mm->context.id);
  218. if ((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS) {
  219. asid = new_context(mm, cpu);
  220. atomic64_set(&mm->context.id, asid);
  221. }
  222. if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending)) {
  223. local_flush_bp_all();
  224. local_flush_tlb_all();
  225. }
  226. atomic64_set(&per_cpu(active_asids, cpu), asid);
  227. cpumask_set_cpu(cpu, mm_cpumask(mm));
  228. raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
  229. switch_mm_fastpath:
  230. cpu_switch_mm(mm->pgd, mm);
  231. }