page.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682
  1. /*
  2. * This file is subject to the terms and conditions of the GNU General Public
  3. * License. See the file "COPYING" in the main directory of this archive
  4. * for more details.
  5. *
  6. * Copyright (C) 2003, 04, 05 Ralf Baechle (ralf@linux-mips.org)
  7. * Copyright (C) 2007 Maciej W. Rozycki
  8. * Copyright (C) 2008 Thiemo Seufer
  9. * Copyright (C) 2012 MIPS Technologies, Inc.
  10. */
  11. #include <linux/kernel.h>
  12. #include <linux/sched.h>
  13. #include <linux/smp.h>
  14. #include <linux/mm.h>
  15. #include <linux/module.h>
  16. #include <linux/proc_fs.h>
  17. #include <asm/bugs.h>
  18. #include <asm/cacheops.h>
  19. #include <asm/cpu-type.h>
  20. #include <asm/inst.h>
  21. #include <asm/io.h>
  22. #include <asm/page.h>
  23. #include <asm/pgtable.h>
  24. #include <asm/prefetch.h>
  25. #include <asm/bootinfo.h>
  26. #include <asm/mipsregs.h>
  27. #include <asm/mmu_context.h>
  28. #include <asm/cpu.h>
  29. #include <asm/war.h>
  30. #ifdef CONFIG_SIBYTE_DMA_PAGEOPS
  31. #include <asm/sibyte/sb1250.h>
  32. #include <asm/sibyte/sb1250_regs.h>
  33. #include <asm/sibyte/sb1250_dma.h>
  34. #endif
  35. #include <asm/uasm.h>
  36. /* Registers used in the assembled routines. */
  37. #define ZERO 0
  38. #define AT 2
  39. #define A0 4
  40. #define A1 5
  41. #define A2 6
  42. #define T0 8
  43. #define T1 9
  44. #define T2 10
  45. #define T3 11
  46. #define T9 25
  47. #define RA 31
  48. /* Handle labels (which must be positive integers). */
  49. enum label_id {
  50. label_clear_nopref = 1,
  51. label_clear_pref,
  52. label_copy_nopref,
  53. label_copy_pref_both,
  54. label_copy_pref_store,
  55. };
  56. UASM_L_LA(_clear_nopref)
  57. UASM_L_LA(_clear_pref)
  58. UASM_L_LA(_copy_nopref)
  59. UASM_L_LA(_copy_pref_both)
  60. UASM_L_LA(_copy_pref_store)
  61. /* We need one branch and therefore one relocation per target label. */
  62. static struct uasm_label labels[5];
  63. static struct uasm_reloc relocs[5];
  64. #define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010)
  65. #define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020)
  66. /*
  67. * R6 has a limited offset of the pref instruction.
  68. * Skip it if the offset is more than 9 bits.
  69. */
  70. #define _uasm_i_pref(a, b, c, d) \
  71. do { \
  72. if (cpu_has_mips_r6) { \
  73. if (c <= 0xff && c >= -0x100) \
  74. uasm_i_pref(a, b, c, d);\
  75. } else { \
  76. uasm_i_pref(a, b, c, d); \
  77. } \
  78. } while(0)
  79. static int pref_bias_clear_store;
  80. static int pref_bias_copy_load;
  81. static int pref_bias_copy_store;
  82. static u32 pref_src_mode;
  83. static u32 pref_dst_mode;
  84. static int clear_word_size;
  85. static int copy_word_size;
  86. static int half_clear_loop_size;
  87. static int half_copy_loop_size;
  88. static int cache_line_size;
  89. #define cache_line_mask() (cache_line_size - 1)
  90. static inline void
  91. pg_addiu(u32 **buf, unsigned int reg1, unsigned int reg2, unsigned int off)
  92. {
  93. if (cpu_has_64bit_gp_regs && DADDI_WAR && r4k_daddiu_bug()) {
  94. if (off > 0x7fff) {
  95. uasm_i_lui(buf, T9, uasm_rel_hi(off));
  96. uasm_i_addiu(buf, T9, T9, uasm_rel_lo(off));
  97. } else
  98. uasm_i_addiu(buf, T9, ZERO, off);
  99. uasm_i_daddu(buf, reg1, reg2, T9);
  100. } else {
  101. if (off > 0x7fff) {
  102. uasm_i_lui(buf, T9, uasm_rel_hi(off));
  103. uasm_i_addiu(buf, T9, T9, uasm_rel_lo(off));
  104. UASM_i_ADDU(buf, reg1, reg2, T9);
  105. } else
  106. UASM_i_ADDIU(buf, reg1, reg2, off);
  107. }
  108. }
  109. static void set_prefetch_parameters(void)
  110. {
  111. if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg)
  112. clear_word_size = 8;
  113. else
  114. clear_word_size = 4;
  115. if (cpu_has_64bit_gp_regs)
  116. copy_word_size = 8;
  117. else
  118. copy_word_size = 4;
  119. /*
  120. * The pref's used here are using "streaming" hints, which cause the
  121. * copied data to be kicked out of the cache sooner. A page copy often
  122. * ends up copying a lot more data than is commonly used, so this seems
  123. * to make sense in terms of reducing cache pollution, but I've no real
  124. * performance data to back this up.
  125. */
  126. if (cpu_has_prefetch) {
  127. /*
  128. * XXX: Most prefetch bias values in here are based on
  129. * guesswork.
  130. */
  131. cache_line_size = cpu_dcache_line_size();
  132. switch (current_cpu_type()) {
  133. case CPU_R5500:
  134. case CPU_TX49XX:
  135. /* These processors only support the Pref_Load. */
  136. pref_bias_copy_load = 256;
  137. break;
  138. case CPU_R10000:
  139. case CPU_R12000:
  140. case CPU_R14000:
  141. /*
  142. * Those values have been experimentally tuned for an
  143. * Origin 200.
  144. */
  145. pref_bias_clear_store = 512;
  146. pref_bias_copy_load = 256;
  147. pref_bias_copy_store = 256;
  148. pref_src_mode = Pref_LoadStreamed;
  149. pref_dst_mode = Pref_StoreStreamed;
  150. break;
  151. case CPU_SB1:
  152. case CPU_SB1A:
  153. pref_bias_clear_store = 128;
  154. pref_bias_copy_load = 128;
  155. pref_bias_copy_store = 128;
  156. /*
  157. * SB1 pass1 Pref_LoadStreamed/Pref_StoreStreamed
  158. * hints are broken.
  159. */
  160. if (current_cpu_type() == CPU_SB1 &&
  161. (current_cpu_data.processor_id & 0xff) < 0x02) {
  162. pref_src_mode = Pref_Load;
  163. pref_dst_mode = Pref_Store;
  164. } else {
  165. pref_src_mode = Pref_LoadStreamed;
  166. pref_dst_mode = Pref_StoreStreamed;
  167. }
  168. break;
  169. default:
  170. pref_bias_clear_store = 128;
  171. pref_bias_copy_load = 256;
  172. pref_bias_copy_store = 128;
  173. pref_src_mode = Pref_LoadStreamed;
  174. if (cpu_has_mips_r6)
  175. /*
  176. * Bit 30 (Pref_PrepareForStore) has been
  177. * removed from MIPS R6. Use bit 5
  178. * (Pref_StoreStreamed).
  179. */
  180. pref_dst_mode = Pref_StoreStreamed;
  181. else
  182. pref_dst_mode = Pref_PrepareForStore;
  183. break;
  184. }
  185. } else {
  186. if (cpu_has_cache_cdex_s)
  187. cache_line_size = cpu_scache_line_size();
  188. else if (cpu_has_cache_cdex_p)
  189. cache_line_size = cpu_dcache_line_size();
  190. }
  191. /*
  192. * Too much unrolling will overflow the available space in
  193. * clear_space_array / copy_page_array.
  194. */
  195. half_clear_loop_size = min(16 * clear_word_size,
  196. max(cache_line_size >> 1,
  197. 4 * clear_word_size));
  198. half_copy_loop_size = min(16 * copy_word_size,
  199. max(cache_line_size >> 1,
  200. 4 * copy_word_size));
  201. }
  202. static void build_clear_store(u32 **buf, int off)
  203. {
  204. if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg) {
  205. uasm_i_sd(buf, ZERO, off, A0);
  206. } else {
  207. uasm_i_sw(buf, ZERO, off, A0);
  208. }
  209. }
  210. static inline void build_clear_pref(u32 **buf, int off)
  211. {
  212. if (off & cache_line_mask())
  213. return;
  214. if (pref_bias_clear_store) {
  215. _uasm_i_pref(buf, pref_dst_mode, pref_bias_clear_store + off,
  216. A0);
  217. } else if (cache_line_size == (half_clear_loop_size << 1)) {
  218. if (cpu_has_cache_cdex_s) {
  219. uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0);
  220. } else if (cpu_has_cache_cdex_p) {
  221. if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) {
  222. uasm_i_nop(buf);
  223. uasm_i_nop(buf);
  224. uasm_i_nop(buf);
  225. uasm_i_nop(buf);
  226. }
  227. if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
  228. uasm_i_lw(buf, ZERO, ZERO, AT);
  229. uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0);
  230. }
  231. }
  232. }
  233. extern u32 __clear_page_start;
  234. extern u32 __clear_page_end;
  235. extern u32 __copy_page_start;
  236. extern u32 __copy_page_end;
  237. void build_clear_page(void)
  238. {
  239. int off;
  240. u32 *buf = &__clear_page_start;
  241. struct uasm_label *l = labels;
  242. struct uasm_reloc *r = relocs;
  243. int i;
  244. static atomic_t run_once = ATOMIC_INIT(0);
  245. if (atomic_xchg(&run_once, 1)) {
  246. return;
  247. }
  248. memset(labels, 0, sizeof(labels));
  249. memset(relocs, 0, sizeof(relocs));
  250. set_prefetch_parameters();
  251. /*
  252. * This algorithm makes the following assumptions:
  253. * - The prefetch bias is a multiple of 2 words.
  254. * - The prefetch bias is less than one page.
  255. */
  256. BUG_ON(pref_bias_clear_store % (2 * clear_word_size));
  257. BUG_ON(PAGE_SIZE < pref_bias_clear_store);
  258. off = PAGE_SIZE - pref_bias_clear_store;
  259. if (off > 0xffff || !pref_bias_clear_store)
  260. pg_addiu(&buf, A2, A0, off);
  261. else
  262. uasm_i_ori(&buf, A2, A0, off);
  263. if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
  264. uasm_i_lui(&buf, AT, uasm_rel_hi(0xa0000000));
  265. off = cache_line_size ? min(8, pref_bias_clear_store / cache_line_size)
  266. * cache_line_size : 0;
  267. while (off) {
  268. build_clear_pref(&buf, -off);
  269. off -= cache_line_size;
  270. }
  271. uasm_l_clear_pref(&l, buf);
  272. do {
  273. build_clear_pref(&buf, off);
  274. build_clear_store(&buf, off);
  275. off += clear_word_size;
  276. } while (off < half_clear_loop_size);
  277. pg_addiu(&buf, A0, A0, 2 * off);
  278. off = -off;
  279. do {
  280. build_clear_pref(&buf, off);
  281. if (off == -clear_word_size)
  282. uasm_il_bne(&buf, &r, A0, A2, label_clear_pref);
  283. build_clear_store(&buf, off);
  284. off += clear_word_size;
  285. } while (off < 0);
  286. if (pref_bias_clear_store) {
  287. pg_addiu(&buf, A2, A0, pref_bias_clear_store);
  288. uasm_l_clear_nopref(&l, buf);
  289. off = 0;
  290. do {
  291. build_clear_store(&buf, off);
  292. off += clear_word_size;
  293. } while (off < half_clear_loop_size);
  294. pg_addiu(&buf, A0, A0, 2 * off);
  295. off = -off;
  296. do {
  297. if (off == -clear_word_size)
  298. uasm_il_bne(&buf, &r, A0, A2,
  299. label_clear_nopref);
  300. build_clear_store(&buf, off);
  301. off += clear_word_size;
  302. } while (off < 0);
  303. }
  304. uasm_i_jr(&buf, RA);
  305. uasm_i_nop(&buf);
  306. BUG_ON(buf > &__clear_page_end);
  307. uasm_resolve_relocs(relocs, labels);
  308. pr_debug("Synthesized clear page handler (%u instructions).\n",
  309. (u32)(buf - &__clear_page_start));
  310. pr_debug("\t.set push\n");
  311. pr_debug("\t.set noreorder\n");
  312. for (i = 0; i < (buf - &__clear_page_start); i++)
  313. pr_debug("\t.word 0x%08x\n", (&__clear_page_start)[i]);
  314. pr_debug("\t.set pop\n");
  315. }
  316. static void build_copy_load(u32 **buf, int reg, int off)
  317. {
  318. if (cpu_has_64bit_gp_regs) {
  319. uasm_i_ld(buf, reg, off, A1);
  320. } else {
  321. uasm_i_lw(buf, reg, off, A1);
  322. }
  323. }
  324. static void build_copy_store(u32 **buf, int reg, int off)
  325. {
  326. if (cpu_has_64bit_gp_regs) {
  327. uasm_i_sd(buf, reg, off, A0);
  328. } else {
  329. uasm_i_sw(buf, reg, off, A0);
  330. }
  331. }
  332. static inline void build_copy_load_pref(u32 **buf, int off)
  333. {
  334. if (off & cache_line_mask())
  335. return;
  336. if (pref_bias_copy_load)
  337. _uasm_i_pref(buf, pref_src_mode, pref_bias_copy_load + off, A1);
  338. }
  339. static inline void build_copy_store_pref(u32 **buf, int off)
  340. {
  341. if (off & cache_line_mask())
  342. return;
  343. if (pref_bias_copy_store) {
  344. _uasm_i_pref(buf, pref_dst_mode, pref_bias_copy_store + off,
  345. A0);
  346. } else if (cache_line_size == (half_copy_loop_size << 1)) {
  347. if (cpu_has_cache_cdex_s) {
  348. uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0);
  349. } else if (cpu_has_cache_cdex_p) {
  350. if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) {
  351. uasm_i_nop(buf);
  352. uasm_i_nop(buf);
  353. uasm_i_nop(buf);
  354. uasm_i_nop(buf);
  355. }
  356. if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
  357. uasm_i_lw(buf, ZERO, ZERO, AT);
  358. uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0);
  359. }
  360. }
  361. }
  362. void build_copy_page(void)
  363. {
  364. int off;
  365. u32 *buf = &__copy_page_start;
  366. struct uasm_label *l = labels;
  367. struct uasm_reloc *r = relocs;
  368. int i;
  369. static atomic_t run_once = ATOMIC_INIT(0);
  370. if (atomic_xchg(&run_once, 1)) {
  371. return;
  372. }
  373. memset(labels, 0, sizeof(labels));
  374. memset(relocs, 0, sizeof(relocs));
  375. set_prefetch_parameters();
  376. /*
  377. * This algorithm makes the following assumptions:
  378. * - All prefetch biases are multiples of 8 words.
  379. * - The prefetch biases are less than one page.
  380. * - The store prefetch bias isn't greater than the load
  381. * prefetch bias.
  382. */
  383. BUG_ON(pref_bias_copy_load % (8 * copy_word_size));
  384. BUG_ON(pref_bias_copy_store % (8 * copy_word_size));
  385. BUG_ON(PAGE_SIZE < pref_bias_copy_load);
  386. BUG_ON(pref_bias_copy_store > pref_bias_copy_load);
  387. off = PAGE_SIZE - pref_bias_copy_load;
  388. if (off > 0xffff || !pref_bias_copy_load)
  389. pg_addiu(&buf, A2, A0, off);
  390. else
  391. uasm_i_ori(&buf, A2, A0, off);
  392. if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
  393. uasm_i_lui(&buf, AT, uasm_rel_hi(0xa0000000));
  394. off = cache_line_size ? min(8, pref_bias_copy_load / cache_line_size) *
  395. cache_line_size : 0;
  396. while (off) {
  397. build_copy_load_pref(&buf, -off);
  398. off -= cache_line_size;
  399. }
  400. off = cache_line_size ? min(8, pref_bias_copy_store / cache_line_size) *
  401. cache_line_size : 0;
  402. while (off) {
  403. build_copy_store_pref(&buf, -off);
  404. off -= cache_line_size;
  405. }
  406. uasm_l_copy_pref_both(&l, buf);
  407. do {
  408. build_copy_load_pref(&buf, off);
  409. build_copy_load(&buf, T0, off);
  410. build_copy_load_pref(&buf, off + copy_word_size);
  411. build_copy_load(&buf, T1, off + copy_word_size);
  412. build_copy_load_pref(&buf, off + 2 * copy_word_size);
  413. build_copy_load(&buf, T2, off + 2 * copy_word_size);
  414. build_copy_load_pref(&buf, off + 3 * copy_word_size);
  415. build_copy_load(&buf, T3, off + 3 * copy_word_size);
  416. build_copy_store_pref(&buf, off);
  417. build_copy_store(&buf, T0, off);
  418. build_copy_store_pref(&buf, off + copy_word_size);
  419. build_copy_store(&buf, T1, off + copy_word_size);
  420. build_copy_store_pref(&buf, off + 2 * copy_word_size);
  421. build_copy_store(&buf, T2, off + 2 * copy_word_size);
  422. build_copy_store_pref(&buf, off + 3 * copy_word_size);
  423. build_copy_store(&buf, T3, off + 3 * copy_word_size);
  424. off += 4 * copy_word_size;
  425. } while (off < half_copy_loop_size);
  426. pg_addiu(&buf, A1, A1, 2 * off);
  427. pg_addiu(&buf, A0, A0, 2 * off);
  428. off = -off;
  429. do {
  430. build_copy_load_pref(&buf, off);
  431. build_copy_load(&buf, T0, off);
  432. build_copy_load_pref(&buf, off + copy_word_size);
  433. build_copy_load(&buf, T1, off + copy_word_size);
  434. build_copy_load_pref(&buf, off + 2 * copy_word_size);
  435. build_copy_load(&buf, T2, off + 2 * copy_word_size);
  436. build_copy_load_pref(&buf, off + 3 * copy_word_size);
  437. build_copy_load(&buf, T3, off + 3 * copy_word_size);
  438. build_copy_store_pref(&buf, off);
  439. build_copy_store(&buf, T0, off);
  440. build_copy_store_pref(&buf, off + copy_word_size);
  441. build_copy_store(&buf, T1, off + copy_word_size);
  442. build_copy_store_pref(&buf, off + 2 * copy_word_size);
  443. build_copy_store(&buf, T2, off + 2 * copy_word_size);
  444. build_copy_store_pref(&buf, off + 3 * copy_word_size);
  445. if (off == -(4 * copy_word_size))
  446. uasm_il_bne(&buf, &r, A2, A0, label_copy_pref_both);
  447. build_copy_store(&buf, T3, off + 3 * copy_word_size);
  448. off += 4 * copy_word_size;
  449. } while (off < 0);
  450. if (pref_bias_copy_load - pref_bias_copy_store) {
  451. pg_addiu(&buf, A2, A0,
  452. pref_bias_copy_load - pref_bias_copy_store);
  453. uasm_l_copy_pref_store(&l, buf);
  454. off = 0;
  455. do {
  456. build_copy_load(&buf, T0, off);
  457. build_copy_load(&buf, T1, off + copy_word_size);
  458. build_copy_load(&buf, T2, off + 2 * copy_word_size);
  459. build_copy_load(&buf, T3, off + 3 * copy_word_size);
  460. build_copy_store_pref(&buf, off);
  461. build_copy_store(&buf, T0, off);
  462. build_copy_store_pref(&buf, off + copy_word_size);
  463. build_copy_store(&buf, T1, off + copy_word_size);
  464. build_copy_store_pref(&buf, off + 2 * copy_word_size);
  465. build_copy_store(&buf, T2, off + 2 * copy_word_size);
  466. build_copy_store_pref(&buf, off + 3 * copy_word_size);
  467. build_copy_store(&buf, T3, off + 3 * copy_word_size);
  468. off += 4 * copy_word_size;
  469. } while (off < half_copy_loop_size);
  470. pg_addiu(&buf, A1, A1, 2 * off);
  471. pg_addiu(&buf, A0, A0, 2 * off);
  472. off = -off;
  473. do {
  474. build_copy_load(&buf, T0, off);
  475. build_copy_load(&buf, T1, off + copy_word_size);
  476. build_copy_load(&buf, T2, off + 2 * copy_word_size);
  477. build_copy_load(&buf, T3, off + 3 * copy_word_size);
  478. build_copy_store_pref(&buf, off);
  479. build_copy_store(&buf, T0, off);
  480. build_copy_store_pref(&buf, off + copy_word_size);
  481. build_copy_store(&buf, T1, off + copy_word_size);
  482. build_copy_store_pref(&buf, off + 2 * copy_word_size);
  483. build_copy_store(&buf, T2, off + 2 * copy_word_size);
  484. build_copy_store_pref(&buf, off + 3 * copy_word_size);
  485. if (off == -(4 * copy_word_size))
  486. uasm_il_bne(&buf, &r, A2, A0,
  487. label_copy_pref_store);
  488. build_copy_store(&buf, T3, off + 3 * copy_word_size);
  489. off += 4 * copy_word_size;
  490. } while (off < 0);
  491. }
  492. if (pref_bias_copy_store) {
  493. pg_addiu(&buf, A2, A0, pref_bias_copy_store);
  494. uasm_l_copy_nopref(&l, buf);
  495. off = 0;
  496. do {
  497. build_copy_load(&buf, T0, off);
  498. build_copy_load(&buf, T1, off + copy_word_size);
  499. build_copy_load(&buf, T2, off + 2 * copy_word_size);
  500. build_copy_load(&buf, T3, off + 3 * copy_word_size);
  501. build_copy_store(&buf, T0, off);
  502. build_copy_store(&buf, T1, off + copy_word_size);
  503. build_copy_store(&buf, T2, off + 2 * copy_word_size);
  504. build_copy_store(&buf, T3, off + 3 * copy_word_size);
  505. off += 4 * copy_word_size;
  506. } while (off < half_copy_loop_size);
  507. pg_addiu(&buf, A1, A1, 2 * off);
  508. pg_addiu(&buf, A0, A0, 2 * off);
  509. off = -off;
  510. do {
  511. build_copy_load(&buf, T0, off);
  512. build_copy_load(&buf, T1, off + copy_word_size);
  513. build_copy_load(&buf, T2, off + 2 * copy_word_size);
  514. build_copy_load(&buf, T3, off + 3 * copy_word_size);
  515. build_copy_store(&buf, T0, off);
  516. build_copy_store(&buf, T1, off + copy_word_size);
  517. build_copy_store(&buf, T2, off + 2 * copy_word_size);
  518. if (off == -(4 * copy_word_size))
  519. uasm_il_bne(&buf, &r, A2, A0,
  520. label_copy_nopref);
  521. build_copy_store(&buf, T3, off + 3 * copy_word_size);
  522. off += 4 * copy_word_size;
  523. } while (off < 0);
  524. }
  525. uasm_i_jr(&buf, RA);
  526. uasm_i_nop(&buf);
  527. BUG_ON(buf > &__copy_page_end);
  528. uasm_resolve_relocs(relocs, labels);
  529. pr_debug("Synthesized copy page handler (%u instructions).\n",
  530. (u32)(buf - &__copy_page_start));
  531. pr_debug("\t.set push\n");
  532. pr_debug("\t.set noreorder\n");
  533. for (i = 0; i < (buf - &__copy_page_start); i++)
  534. pr_debug("\t.word 0x%08x\n", (&__copy_page_start)[i]);
  535. pr_debug("\t.set pop\n");
  536. }
  537. #ifdef CONFIG_SIBYTE_DMA_PAGEOPS
  538. extern void clear_page_cpu(void *page);
  539. extern void copy_page_cpu(void *to, void *from);
  540. /*
  541. * Pad descriptors to cacheline, since each is exclusively owned by a
  542. * particular CPU.
  543. */
  544. struct dmadscr {
  545. u64 dscr_a;
  546. u64 dscr_b;
  547. u64 pad_a;
  548. u64 pad_b;
  549. } ____cacheline_aligned_in_smp page_descr[DM_NUM_CHANNELS];
  550. void sb1_dma_init(void)
  551. {
  552. int i;
  553. for (i = 0; i < DM_NUM_CHANNELS; i++) {
  554. const u64 base_val = CPHYSADDR((unsigned long)&page_descr[i]) |
  555. V_DM_DSCR_BASE_RINGSZ(1);
  556. void *base_reg = IOADDR(A_DM_REGISTER(i, R_DM_DSCR_BASE));
  557. __raw_writeq(base_val, base_reg);
  558. __raw_writeq(base_val | M_DM_DSCR_BASE_RESET, base_reg);
  559. __raw_writeq(base_val | M_DM_DSCR_BASE_ENABL, base_reg);
  560. }
  561. }
  562. void clear_page(void *page)
  563. {
  564. u64 to_phys = CPHYSADDR((unsigned long)page);
  565. unsigned int cpu = smp_processor_id();
  566. /* if the page is not in KSEG0, use old way */
  567. if ((long)KSEGX((unsigned long)page) != (long)CKSEG0)
  568. return clear_page_cpu(page);
  569. page_descr[cpu].dscr_a = to_phys | M_DM_DSCRA_ZERO_MEM |
  570. M_DM_DSCRA_L2C_DEST | M_DM_DSCRA_INTERRUPT;
  571. page_descr[cpu].dscr_b = V_DM_DSCRB_SRC_LENGTH(PAGE_SIZE);
  572. __raw_writeq(1, IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_COUNT)));
  573. /*
  574. * Don't really want to do it this way, but there's no
  575. * reliable way to delay completion detection.
  576. */
  577. while (!(__raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE_DEBUG)))
  578. & M_DM_DSCR_BASE_INTERRUPT))
  579. ;
  580. __raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
  581. }
  582. void copy_page(void *to, void *from)
  583. {
  584. u64 from_phys = CPHYSADDR((unsigned long)from);
  585. u64 to_phys = CPHYSADDR((unsigned long)to);
  586. unsigned int cpu = smp_processor_id();
  587. /* if any page is not in KSEG0, use old way */
  588. if ((long)KSEGX((unsigned long)to) != (long)CKSEG0
  589. || (long)KSEGX((unsigned long)from) != (long)CKSEG0)
  590. return copy_page_cpu(to, from);
  591. page_descr[cpu].dscr_a = to_phys | M_DM_DSCRA_L2C_DEST |
  592. M_DM_DSCRA_INTERRUPT;
  593. page_descr[cpu].dscr_b = from_phys | V_DM_DSCRB_SRC_LENGTH(PAGE_SIZE);
  594. __raw_writeq(1, IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_COUNT)));
  595. /*
  596. * Don't really want to do it this way, but there's no
  597. * reliable way to delay completion detection.
  598. */
  599. while (!(__raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE_DEBUG)))
  600. & M_DM_DSCR_BASE_INTERRUPT))
  601. ;
  602. __raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
  603. }
  604. #endif /* CONFIG_SIBYTE_DMA_PAGEOPS */