dma-debug.c 42 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661
  1. /*
  2. * Copyright (C) 2008 Advanced Micro Devices, Inc.
  3. *
  4. * Author: Joerg Roedel <joerg.roedel@amd.com>
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms of the GNU General Public License version 2 as published
  8. * by the Free Software Foundation.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program; if not, write to the Free Software
  17. * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  18. */
  19. #include <linux/scatterlist.h>
  20. #include <linux/dma-mapping.h>
  21. #include <linux/stacktrace.h>
  22. #include <linux/dma-debug.h>
  23. #include <linux/spinlock.h>
  24. #include <linux/debugfs.h>
  25. #include <linux/uaccess.h>
  26. #include <linux/export.h>
  27. #include <linux/device.h>
  28. #include <linux/types.h>
  29. #include <linux/sched.h>
  30. #include <linux/ctype.h>
  31. #include <linux/list.h>
  32. #include <linux/slab.h>
  33. #include <asm/sections.h>
  34. #define HASH_SIZE 1024ULL
  35. #define HASH_FN_SHIFT 13
  36. #define HASH_FN_MASK (HASH_SIZE - 1)
  37. enum {
  38. dma_debug_single,
  39. dma_debug_page,
  40. dma_debug_sg,
  41. dma_debug_coherent,
  42. };
  43. enum map_err_types {
  44. MAP_ERR_CHECK_NOT_APPLICABLE,
  45. MAP_ERR_NOT_CHECKED,
  46. MAP_ERR_CHECKED,
  47. };
  48. #define DMA_DEBUG_STACKTRACE_ENTRIES 5
  49. /**
  50. * struct dma_debug_entry - track a dma_map* or dma_alloc_coherent mapping
  51. * @list: node on pre-allocated free_entries list
  52. * @dev: 'dev' argument to dma_map_{page|single|sg} or dma_alloc_coherent
  53. * @type: single, page, sg, coherent
  54. * @pfn: page frame of the start address
  55. * @offset: offset of mapping relative to pfn
  56. * @size: length of the mapping
  57. * @direction: enum dma_data_direction
  58. * @sg_call_ents: 'nents' from dma_map_sg
  59. * @sg_mapped_ents: 'mapped_ents' from dma_map_sg
  60. * @map_err_type: track whether dma_mapping_error() was checked
  61. * @stacktrace: support backtraces when a violation is detected
  62. */
  63. struct dma_debug_entry {
  64. struct list_head list;
  65. struct device *dev;
  66. int type;
  67. unsigned long pfn;
  68. size_t offset;
  69. u64 dev_addr;
  70. u64 size;
  71. int direction;
  72. int sg_call_ents;
  73. int sg_mapped_ents;
  74. enum map_err_types map_err_type;
  75. #ifdef CONFIG_STACKTRACE
  76. struct stack_trace stacktrace;
  77. unsigned long st_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
  78. #endif
  79. };
  80. typedef bool (*match_fn)(struct dma_debug_entry *, struct dma_debug_entry *);
  81. struct hash_bucket {
  82. struct list_head list;
  83. spinlock_t lock;
  84. } ____cacheline_aligned_in_smp;
  85. /* Hash list to save the allocated dma addresses */
  86. static struct hash_bucket dma_entry_hash[HASH_SIZE];
  87. /* List of pre-allocated dma_debug_entry's */
  88. static LIST_HEAD(free_entries);
  89. /* Lock for the list above */
  90. static DEFINE_SPINLOCK(free_entries_lock);
  91. /* Global disable flag - will be set in case of an error */
  92. static bool global_disable __read_mostly;
  93. /* Early initialization disable flag, set at the end of dma_debug_init */
  94. static bool dma_debug_initialized __read_mostly;
  95. static inline bool dma_debug_disabled(void)
  96. {
  97. return global_disable || !dma_debug_initialized;
  98. }
  99. /* Global error count */
  100. static u32 error_count;
  101. /* Global error show enable*/
  102. static u32 show_all_errors __read_mostly;
  103. /* Number of errors to show */
  104. static u32 show_num_errors = 1;
  105. static u32 num_free_entries;
  106. static u32 min_free_entries;
  107. static u32 nr_total_entries;
  108. /* number of preallocated entries requested by kernel cmdline */
  109. static u32 req_entries;
  110. /* debugfs dentry's for the stuff above */
  111. static struct dentry *dma_debug_dent __read_mostly;
  112. static struct dentry *global_disable_dent __read_mostly;
  113. static struct dentry *error_count_dent __read_mostly;
  114. static struct dentry *show_all_errors_dent __read_mostly;
  115. static struct dentry *show_num_errors_dent __read_mostly;
  116. static struct dentry *num_free_entries_dent __read_mostly;
  117. static struct dentry *min_free_entries_dent __read_mostly;
  118. static struct dentry *filter_dent __read_mostly;
  119. /* per-driver filter related state */
  120. #define NAME_MAX_LEN 64
  121. static char current_driver_name[NAME_MAX_LEN] __read_mostly;
  122. static struct device_driver *current_driver __read_mostly;
  123. static DEFINE_RWLOCK(driver_name_lock);
  124. static const char *const maperr2str[] = {
  125. [MAP_ERR_CHECK_NOT_APPLICABLE] = "dma map error check not applicable",
  126. [MAP_ERR_NOT_CHECKED] = "dma map error not checked",
  127. [MAP_ERR_CHECKED] = "dma map error checked",
  128. };
  129. static const char *type2name[4] = { "single", "page",
  130. "scather-gather", "coherent" };
  131. static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
  132. "DMA_FROM_DEVICE", "DMA_NONE" };
  133. /*
  134. * The access to some variables in this macro is racy. We can't use atomic_t
  135. * here because all these variables are exported to debugfs. Some of them even
  136. * writeable. This is also the reason why a lock won't help much. But anyway,
  137. * the races are no big deal. Here is why:
  138. *
  139. * error_count: the addition is racy, but the worst thing that can happen is
  140. * that we don't count some errors
  141. * show_num_errors: the subtraction is racy. Also no big deal because in
  142. * worst case this will result in one warning more in the
  143. * system log than the user configured. This variable is
  144. * writeable via debugfs.
  145. */
  146. static inline void dump_entry_trace(struct dma_debug_entry *entry)
  147. {
  148. #ifdef CONFIG_STACKTRACE
  149. if (entry) {
  150. pr_warning("Mapped at:\n");
  151. print_stack_trace(&entry->stacktrace, 0);
  152. }
  153. #endif
  154. }
  155. static bool driver_filter(struct device *dev)
  156. {
  157. struct device_driver *drv;
  158. unsigned long flags;
  159. bool ret;
  160. /* driver filter off */
  161. if (likely(!current_driver_name[0]))
  162. return true;
  163. /* driver filter on and initialized */
  164. if (current_driver && dev && dev->driver == current_driver)
  165. return true;
  166. /* driver filter on, but we can't filter on a NULL device... */
  167. if (!dev)
  168. return false;
  169. if (current_driver || !current_driver_name[0])
  170. return false;
  171. /* driver filter on but not yet initialized */
  172. drv = dev->driver;
  173. if (!drv)
  174. return false;
  175. /* lock to protect against change of current_driver_name */
  176. read_lock_irqsave(&driver_name_lock, flags);
  177. ret = false;
  178. if (drv->name &&
  179. strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) {
  180. current_driver = drv;
  181. ret = true;
  182. }
  183. read_unlock_irqrestore(&driver_name_lock, flags);
  184. return ret;
  185. }
  186. #define err_printk(dev, entry, format, arg...) do { \
  187. error_count += 1; \
  188. if (driver_filter(dev) && \
  189. (show_all_errors || show_num_errors > 0)) { \
  190. WARN(1, "%s %s: " format, \
  191. dev ? dev_driver_string(dev) : "NULL", \
  192. dev ? dev_name(dev) : "NULL", ## arg); \
  193. dump_entry_trace(entry); \
  194. } \
  195. if (!show_all_errors && show_num_errors > 0) \
  196. show_num_errors -= 1; \
  197. } while (0);
  198. /*
  199. * Hash related functions
  200. *
  201. * Every DMA-API request is saved into a struct dma_debug_entry. To
  202. * have quick access to these structs they are stored into a hash.
  203. */
  204. static int hash_fn(struct dma_debug_entry *entry)
  205. {
  206. /*
  207. * Hash function is based on the dma address.
  208. * We use bits 20-27 here as the index into the hash
  209. */
  210. return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;
  211. }
  212. /*
  213. * Request exclusive access to a hash bucket for a given dma_debug_entry.
  214. */
  215. static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,
  216. unsigned long *flags)
  217. __acquires(&dma_entry_hash[idx].lock)
  218. {
  219. int idx = hash_fn(entry);
  220. unsigned long __flags;
  221. spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);
  222. *flags = __flags;
  223. return &dma_entry_hash[idx];
  224. }
  225. /*
  226. * Give up exclusive access to the hash bucket
  227. */
  228. static void put_hash_bucket(struct hash_bucket *bucket,
  229. unsigned long *flags)
  230. __releases(&bucket->lock)
  231. {
  232. unsigned long __flags = *flags;
  233. spin_unlock_irqrestore(&bucket->lock, __flags);
  234. }
  235. static bool exact_match(struct dma_debug_entry *a, struct dma_debug_entry *b)
  236. {
  237. return ((a->dev_addr == b->dev_addr) &&
  238. (a->dev == b->dev)) ? true : false;
  239. }
  240. static bool containing_match(struct dma_debug_entry *a,
  241. struct dma_debug_entry *b)
  242. {
  243. if (a->dev != b->dev)
  244. return false;
  245. if ((b->dev_addr <= a->dev_addr) &&
  246. ((b->dev_addr + b->size) >= (a->dev_addr + a->size)))
  247. return true;
  248. return false;
  249. }
  250. /*
  251. * Search a given entry in the hash bucket list
  252. */
  253. static struct dma_debug_entry *__hash_bucket_find(struct hash_bucket *bucket,
  254. struct dma_debug_entry *ref,
  255. match_fn match)
  256. {
  257. struct dma_debug_entry *entry, *ret = NULL;
  258. int matches = 0, match_lvl, last_lvl = -1;
  259. list_for_each_entry(entry, &bucket->list, list) {
  260. if (!match(ref, entry))
  261. continue;
  262. /*
  263. * Some drivers map the same physical address multiple
  264. * times. Without a hardware IOMMU this results in the
  265. * same device addresses being put into the dma-debug
  266. * hash multiple times too. This can result in false
  267. * positives being reported. Therefore we implement a
  268. * best-fit algorithm here which returns the entry from
  269. * the hash which fits best to the reference value
  270. * instead of the first-fit.
  271. */
  272. matches += 1;
  273. match_lvl = 0;
  274. entry->size == ref->size ? ++match_lvl : 0;
  275. entry->type == ref->type ? ++match_lvl : 0;
  276. entry->direction == ref->direction ? ++match_lvl : 0;
  277. entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0;
  278. if (match_lvl == 4) {
  279. /* perfect-fit - return the result */
  280. return entry;
  281. } else if (match_lvl > last_lvl) {
  282. /*
  283. * We found an entry that fits better then the
  284. * previous one or it is the 1st match.
  285. */
  286. last_lvl = match_lvl;
  287. ret = entry;
  288. }
  289. }
  290. /*
  291. * If we have multiple matches but no perfect-fit, just return
  292. * NULL.
  293. */
  294. ret = (matches == 1) ? ret : NULL;
  295. return ret;
  296. }
  297. static struct dma_debug_entry *bucket_find_exact(struct hash_bucket *bucket,
  298. struct dma_debug_entry *ref)
  299. {
  300. return __hash_bucket_find(bucket, ref, exact_match);
  301. }
  302. static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket,
  303. struct dma_debug_entry *ref,
  304. unsigned long *flags)
  305. {
  306. unsigned int max_range = dma_get_max_seg_size(ref->dev);
  307. struct dma_debug_entry *entry, index = *ref;
  308. unsigned int range = 0;
  309. while (range <= max_range) {
  310. entry = __hash_bucket_find(*bucket, ref, containing_match);
  311. if (entry)
  312. return entry;
  313. /*
  314. * Nothing found, go back a hash bucket
  315. */
  316. put_hash_bucket(*bucket, flags);
  317. range += (1 << HASH_FN_SHIFT);
  318. index.dev_addr -= (1 << HASH_FN_SHIFT);
  319. *bucket = get_hash_bucket(&index, flags);
  320. }
  321. return NULL;
  322. }
  323. /*
  324. * Add an entry to a hash bucket
  325. */
  326. static void hash_bucket_add(struct hash_bucket *bucket,
  327. struct dma_debug_entry *entry)
  328. {
  329. list_add_tail(&entry->list, &bucket->list);
  330. }
  331. /*
  332. * Remove entry from a hash bucket list
  333. */
  334. static void hash_bucket_del(struct dma_debug_entry *entry)
  335. {
  336. list_del(&entry->list);
  337. }
  338. static unsigned long long phys_addr(struct dma_debug_entry *entry)
  339. {
  340. return page_to_phys(pfn_to_page(entry->pfn)) + entry->offset;
  341. }
  342. /*
  343. * Dump mapping entries for debugging purposes
  344. */
  345. void debug_dma_dump_mappings(struct device *dev)
  346. {
  347. int idx;
  348. for (idx = 0; idx < HASH_SIZE; idx++) {
  349. struct hash_bucket *bucket = &dma_entry_hash[idx];
  350. struct dma_debug_entry *entry;
  351. unsigned long flags;
  352. spin_lock_irqsave(&bucket->lock, flags);
  353. list_for_each_entry(entry, &bucket->list, list) {
  354. if (!dev || dev == entry->dev) {
  355. dev_info(entry->dev,
  356. "%s idx %d P=%Lx N=%lx D=%Lx L=%Lx %s %s\n",
  357. type2name[entry->type], idx,
  358. phys_addr(entry), entry->pfn,
  359. entry->dev_addr, entry->size,
  360. dir2name[entry->direction],
  361. maperr2str[entry->map_err_type]);
  362. }
  363. }
  364. spin_unlock_irqrestore(&bucket->lock, flags);
  365. }
  366. }
  367. EXPORT_SYMBOL(debug_dma_dump_mappings);
  368. /*
  369. * For each mapping (initial cacheline in the case of
  370. * dma_alloc_coherent/dma_map_page, initial cacheline in each page of a
  371. * scatterlist, or the cacheline specified in dma_map_single) insert
  372. * into this tree using the cacheline as the key. At
  373. * dma_unmap_{single|sg|page} or dma_free_coherent delete the entry. If
  374. * the entry already exists at insertion time add a tag as a reference
  375. * count for the overlapping mappings. For now, the overlap tracking
  376. * just ensures that 'unmaps' balance 'maps' before marking the
  377. * cacheline idle, but we should also be flagging overlaps as an API
  378. * violation.
  379. *
  380. * Memory usage is mostly constrained by the maximum number of available
  381. * dma-debug entries in that we need a free dma_debug_entry before
  382. * inserting into the tree. In the case of dma_map_page and
  383. * dma_alloc_coherent there is only one dma_debug_entry and one
  384. * dma_active_cacheline entry to track per event. dma_map_sg(), on the
  385. * other hand, consumes a single dma_debug_entry, but inserts 'nents'
  386. * entries into the tree.
  387. *
  388. * At any time debug_dma_assert_idle() can be called to trigger a
  389. * warning if any cachelines in the given page are in the active set.
  390. */
  391. static RADIX_TREE(dma_active_cacheline, GFP_NOWAIT);
  392. static DEFINE_SPINLOCK(radix_lock);
  393. #define ACTIVE_CACHELINE_MAX_OVERLAP ((1 << RADIX_TREE_MAX_TAGS) - 1)
  394. #define CACHELINE_PER_PAGE_SHIFT (PAGE_SHIFT - L1_CACHE_SHIFT)
  395. #define CACHELINES_PER_PAGE (1 << CACHELINE_PER_PAGE_SHIFT)
  396. static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry)
  397. {
  398. return (entry->pfn << CACHELINE_PER_PAGE_SHIFT) +
  399. (entry->offset >> L1_CACHE_SHIFT);
  400. }
  401. static int active_cacheline_read_overlap(phys_addr_t cln)
  402. {
  403. int overlap = 0, i;
  404. for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
  405. if (radix_tree_tag_get(&dma_active_cacheline, cln, i))
  406. overlap |= 1 << i;
  407. return overlap;
  408. }
  409. static int active_cacheline_set_overlap(phys_addr_t cln, int overlap)
  410. {
  411. int i;
  412. if (overlap > ACTIVE_CACHELINE_MAX_OVERLAP || overlap < 0)
  413. return overlap;
  414. for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
  415. if (overlap & 1 << i)
  416. radix_tree_tag_set(&dma_active_cacheline, cln, i);
  417. else
  418. radix_tree_tag_clear(&dma_active_cacheline, cln, i);
  419. return overlap;
  420. }
  421. static void active_cacheline_inc_overlap(phys_addr_t cln)
  422. {
  423. int overlap = active_cacheline_read_overlap(cln);
  424. overlap = active_cacheline_set_overlap(cln, ++overlap);
  425. /* If we overflowed the overlap counter then we're potentially
  426. * leaking dma-mappings. Otherwise, if maps and unmaps are
  427. * balanced then this overflow may cause false negatives in
  428. * debug_dma_assert_idle() as the cacheline may be marked idle
  429. * prematurely.
  430. */
  431. WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP,
  432. "DMA-API: exceeded %d overlapping mappings of cacheline %pa\n",
  433. ACTIVE_CACHELINE_MAX_OVERLAP, &cln);
  434. }
  435. static int active_cacheline_dec_overlap(phys_addr_t cln)
  436. {
  437. int overlap = active_cacheline_read_overlap(cln);
  438. return active_cacheline_set_overlap(cln, --overlap);
  439. }
  440. static int active_cacheline_insert(struct dma_debug_entry *entry)
  441. {
  442. phys_addr_t cln = to_cacheline_number(entry);
  443. unsigned long flags;
  444. int rc;
  445. /* If the device is not writing memory then we don't have any
  446. * concerns about the cpu consuming stale data. This mitigates
  447. * legitimate usages of overlapping mappings.
  448. */
  449. if (entry->direction == DMA_TO_DEVICE)
  450. return 0;
  451. spin_lock_irqsave(&radix_lock, flags);
  452. rc = radix_tree_insert(&dma_active_cacheline, cln, entry);
  453. if (rc == -EEXIST)
  454. active_cacheline_inc_overlap(cln);
  455. spin_unlock_irqrestore(&radix_lock, flags);
  456. return rc;
  457. }
  458. static void active_cacheline_remove(struct dma_debug_entry *entry)
  459. {
  460. phys_addr_t cln = to_cacheline_number(entry);
  461. unsigned long flags;
  462. /* ...mirror the insert case */
  463. if (entry->direction == DMA_TO_DEVICE)
  464. return;
  465. spin_lock_irqsave(&radix_lock, flags);
  466. /* since we are counting overlaps the final put of the
  467. * cacheline will occur when the overlap count is 0.
  468. * active_cacheline_dec_overlap() returns -1 in that case
  469. */
  470. if (active_cacheline_dec_overlap(cln) < 0)
  471. radix_tree_delete(&dma_active_cacheline, cln);
  472. spin_unlock_irqrestore(&radix_lock, flags);
  473. }
  474. /**
  475. * debug_dma_assert_idle() - assert that a page is not undergoing dma
  476. * @page: page to lookup in the dma_active_cacheline tree
  477. *
  478. * Place a call to this routine in cases where the cpu touching the page
  479. * before the dma completes (page is dma_unmapped) will lead to data
  480. * corruption.
  481. */
  482. void debug_dma_assert_idle(struct page *page)
  483. {
  484. static struct dma_debug_entry *ents[CACHELINES_PER_PAGE];
  485. struct dma_debug_entry *entry = NULL;
  486. void **results = (void **) &ents;
  487. unsigned int nents, i;
  488. unsigned long flags;
  489. phys_addr_t cln;
  490. if (dma_debug_disabled())
  491. return;
  492. if (!page)
  493. return;
  494. cln = (phys_addr_t) page_to_pfn(page) << CACHELINE_PER_PAGE_SHIFT;
  495. spin_lock_irqsave(&radix_lock, flags);
  496. nents = radix_tree_gang_lookup(&dma_active_cacheline, results, cln,
  497. CACHELINES_PER_PAGE);
  498. for (i = 0; i < nents; i++) {
  499. phys_addr_t ent_cln = to_cacheline_number(ents[i]);
  500. if (ent_cln == cln) {
  501. entry = ents[i];
  502. break;
  503. } else if (ent_cln >= cln + CACHELINES_PER_PAGE)
  504. break;
  505. }
  506. spin_unlock_irqrestore(&radix_lock, flags);
  507. if (!entry)
  508. return;
  509. cln = to_cacheline_number(entry);
  510. err_printk(entry->dev, entry,
  511. "DMA-API: cpu touching an active dma mapped cacheline [cln=%pa]\n",
  512. &cln);
  513. }
  514. /*
  515. * Wrapper function for adding an entry to the hash.
  516. * This function takes care of locking itself.
  517. */
  518. static void add_dma_entry(struct dma_debug_entry *entry)
  519. {
  520. struct hash_bucket *bucket;
  521. unsigned long flags;
  522. int rc;
  523. bucket = get_hash_bucket(entry, &flags);
  524. hash_bucket_add(bucket, entry);
  525. put_hash_bucket(bucket, &flags);
  526. rc = active_cacheline_insert(entry);
  527. if (rc == -ENOMEM) {
  528. pr_err("DMA-API: cacheline tracking ENOMEM, dma-debug disabled\n");
  529. global_disable = true;
  530. }
  531. /* TODO: report -EEXIST errors here as overlapping mappings are
  532. * not supported by the DMA API
  533. */
  534. }
  535. static struct dma_debug_entry *__dma_entry_alloc(void)
  536. {
  537. struct dma_debug_entry *entry;
  538. entry = list_entry(free_entries.next, struct dma_debug_entry, list);
  539. list_del(&entry->list);
  540. memset(entry, 0, sizeof(*entry));
  541. num_free_entries -= 1;
  542. if (num_free_entries < min_free_entries)
  543. min_free_entries = num_free_entries;
  544. return entry;
  545. }
  546. /* struct dma_entry allocator
  547. *
  548. * The next two functions implement the allocator for
  549. * struct dma_debug_entries.
  550. */
  551. static struct dma_debug_entry *dma_entry_alloc(void)
  552. {
  553. struct dma_debug_entry *entry;
  554. unsigned long flags;
  555. spin_lock_irqsave(&free_entries_lock, flags);
  556. if (list_empty(&free_entries)) {
  557. global_disable = true;
  558. spin_unlock_irqrestore(&free_entries_lock, flags);
  559. pr_err("DMA-API: debugging out of memory - disabling\n");
  560. return NULL;
  561. }
  562. entry = __dma_entry_alloc();
  563. spin_unlock_irqrestore(&free_entries_lock, flags);
  564. #ifdef CONFIG_STACKTRACE
  565. entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES;
  566. entry->stacktrace.entries = entry->st_entries;
  567. entry->stacktrace.skip = 2;
  568. save_stack_trace(&entry->stacktrace);
  569. #endif
  570. return entry;
  571. }
  572. static void dma_entry_free(struct dma_debug_entry *entry)
  573. {
  574. unsigned long flags;
  575. active_cacheline_remove(entry);
  576. /*
  577. * add to beginning of the list - this way the entries are
  578. * more likely cache hot when they are reallocated.
  579. */
  580. spin_lock_irqsave(&free_entries_lock, flags);
  581. list_add(&entry->list, &free_entries);
  582. num_free_entries += 1;
  583. spin_unlock_irqrestore(&free_entries_lock, flags);
  584. }
  585. int dma_debug_resize_entries(u32 num_entries)
  586. {
  587. int i, delta, ret = 0;
  588. unsigned long flags;
  589. struct dma_debug_entry *entry;
  590. LIST_HEAD(tmp);
  591. spin_lock_irqsave(&free_entries_lock, flags);
  592. if (nr_total_entries < num_entries) {
  593. delta = num_entries - nr_total_entries;
  594. spin_unlock_irqrestore(&free_entries_lock, flags);
  595. for (i = 0; i < delta; i++) {
  596. entry = kzalloc(sizeof(*entry), GFP_KERNEL);
  597. if (!entry)
  598. break;
  599. list_add_tail(&entry->list, &tmp);
  600. }
  601. spin_lock_irqsave(&free_entries_lock, flags);
  602. list_splice(&tmp, &free_entries);
  603. nr_total_entries += i;
  604. num_free_entries += i;
  605. } else {
  606. delta = nr_total_entries - num_entries;
  607. for (i = 0; i < delta && !list_empty(&free_entries); i++) {
  608. entry = __dma_entry_alloc();
  609. kfree(entry);
  610. }
  611. nr_total_entries -= i;
  612. }
  613. if (nr_total_entries != num_entries)
  614. ret = 1;
  615. spin_unlock_irqrestore(&free_entries_lock, flags);
  616. return ret;
  617. }
  618. EXPORT_SYMBOL(dma_debug_resize_entries);
  619. /*
  620. * DMA-API debugging init code
  621. *
  622. * The init code does two things:
  623. * 1. Initialize core data structures
  624. * 2. Preallocate a given number of dma_debug_entry structs
  625. */
  626. static int prealloc_memory(u32 num_entries)
  627. {
  628. struct dma_debug_entry *entry, *next_entry;
  629. int i;
  630. for (i = 0; i < num_entries; ++i) {
  631. entry = kzalloc(sizeof(*entry), GFP_KERNEL);
  632. if (!entry)
  633. goto out_err;
  634. list_add_tail(&entry->list, &free_entries);
  635. }
  636. num_free_entries = num_entries;
  637. min_free_entries = num_entries;
  638. pr_info("DMA-API: preallocated %d debug entries\n", num_entries);
  639. return 0;
  640. out_err:
  641. list_for_each_entry_safe(entry, next_entry, &free_entries, list) {
  642. list_del(&entry->list);
  643. kfree(entry);
  644. }
  645. return -ENOMEM;
  646. }
  647. static ssize_t filter_read(struct file *file, char __user *user_buf,
  648. size_t count, loff_t *ppos)
  649. {
  650. char buf[NAME_MAX_LEN + 1];
  651. unsigned long flags;
  652. int len;
  653. if (!current_driver_name[0])
  654. return 0;
  655. /*
  656. * We can't copy to userspace directly because current_driver_name can
  657. * only be read under the driver_name_lock with irqs disabled. So
  658. * create a temporary copy first.
  659. */
  660. read_lock_irqsave(&driver_name_lock, flags);
  661. len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name);
  662. read_unlock_irqrestore(&driver_name_lock, flags);
  663. return simple_read_from_buffer(user_buf, count, ppos, buf, len);
  664. }
  665. static ssize_t filter_write(struct file *file, const char __user *userbuf,
  666. size_t count, loff_t *ppos)
  667. {
  668. char buf[NAME_MAX_LEN];
  669. unsigned long flags;
  670. size_t len;
  671. int i;
  672. /*
  673. * We can't copy from userspace directly. Access to
  674. * current_driver_name is protected with a write_lock with irqs
  675. * disabled. Since copy_from_user can fault and may sleep we
  676. * need to copy to temporary buffer first
  677. */
  678. len = min(count, (size_t)(NAME_MAX_LEN - 1));
  679. if (copy_from_user(buf, userbuf, len))
  680. return -EFAULT;
  681. buf[len] = 0;
  682. write_lock_irqsave(&driver_name_lock, flags);
  683. /*
  684. * Now handle the string we got from userspace very carefully.
  685. * The rules are:
  686. * - only use the first token we got
  687. * - token delimiter is everything looking like a space
  688. * character (' ', '\n', '\t' ...)
  689. *
  690. */
  691. if (!isalnum(buf[0])) {
  692. /*
  693. * If the first character userspace gave us is not
  694. * alphanumerical then assume the filter should be
  695. * switched off.
  696. */
  697. if (current_driver_name[0])
  698. pr_info("DMA-API: switching off dma-debug driver filter\n");
  699. current_driver_name[0] = 0;
  700. current_driver = NULL;
  701. goto out_unlock;
  702. }
  703. /*
  704. * Now parse out the first token and use it as the name for the
  705. * driver to filter for.
  706. */
  707. for (i = 0; i < NAME_MAX_LEN - 1; ++i) {
  708. current_driver_name[i] = buf[i];
  709. if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0)
  710. break;
  711. }
  712. current_driver_name[i] = 0;
  713. current_driver = NULL;
  714. pr_info("DMA-API: enable driver filter for driver [%s]\n",
  715. current_driver_name);
  716. out_unlock:
  717. write_unlock_irqrestore(&driver_name_lock, flags);
  718. return count;
  719. }
  720. static const struct file_operations filter_fops = {
  721. .read = filter_read,
  722. .write = filter_write,
  723. .llseek = default_llseek,
  724. };
  725. static int dma_debug_fs_init(void)
  726. {
  727. dma_debug_dent = debugfs_create_dir("dma-api", NULL);
  728. if (!dma_debug_dent) {
  729. pr_err("DMA-API: can not create debugfs directory\n");
  730. return -ENOMEM;
  731. }
  732. global_disable_dent = debugfs_create_bool("disabled", 0444,
  733. dma_debug_dent,
  734. &global_disable);
  735. if (!global_disable_dent)
  736. goto out_err;
  737. error_count_dent = debugfs_create_u32("error_count", 0444,
  738. dma_debug_dent, &error_count);
  739. if (!error_count_dent)
  740. goto out_err;
  741. show_all_errors_dent = debugfs_create_u32("all_errors", 0644,
  742. dma_debug_dent,
  743. &show_all_errors);
  744. if (!show_all_errors_dent)
  745. goto out_err;
  746. show_num_errors_dent = debugfs_create_u32("num_errors", 0644,
  747. dma_debug_dent,
  748. &show_num_errors);
  749. if (!show_num_errors_dent)
  750. goto out_err;
  751. num_free_entries_dent = debugfs_create_u32("num_free_entries", 0444,
  752. dma_debug_dent,
  753. &num_free_entries);
  754. if (!num_free_entries_dent)
  755. goto out_err;
  756. min_free_entries_dent = debugfs_create_u32("min_free_entries", 0444,
  757. dma_debug_dent,
  758. &min_free_entries);
  759. if (!min_free_entries_dent)
  760. goto out_err;
  761. filter_dent = debugfs_create_file("driver_filter", 0644,
  762. dma_debug_dent, NULL, &filter_fops);
  763. if (!filter_dent)
  764. goto out_err;
  765. return 0;
  766. out_err:
  767. debugfs_remove_recursive(dma_debug_dent);
  768. return -ENOMEM;
  769. }
  770. static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry)
  771. {
  772. struct dma_debug_entry *entry;
  773. unsigned long flags;
  774. int count = 0, i;
  775. local_irq_save(flags);
  776. for (i = 0; i < HASH_SIZE; ++i) {
  777. spin_lock(&dma_entry_hash[i].lock);
  778. list_for_each_entry(entry, &dma_entry_hash[i].list, list) {
  779. if (entry->dev == dev) {
  780. count += 1;
  781. *out_entry = entry;
  782. }
  783. }
  784. spin_unlock(&dma_entry_hash[i].lock);
  785. }
  786. local_irq_restore(flags);
  787. return count;
  788. }
  789. static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data)
  790. {
  791. struct device *dev = data;
  792. struct dma_debug_entry *uninitialized_var(entry);
  793. int count;
  794. if (dma_debug_disabled())
  795. return 0;
  796. switch (action) {
  797. case BUS_NOTIFY_UNBOUND_DRIVER:
  798. count = device_dma_allocations(dev, &entry);
  799. if (count == 0)
  800. break;
  801. err_printk(dev, entry, "DMA-API: device driver has pending "
  802. "DMA allocations while released from device "
  803. "[count=%d]\n"
  804. "One of leaked entries details: "
  805. "[device address=0x%016llx] [size=%llu bytes] "
  806. "[mapped with %s] [mapped as %s]\n",
  807. count, entry->dev_addr, entry->size,
  808. dir2name[entry->direction], type2name[entry->type]);
  809. break;
  810. default:
  811. break;
  812. }
  813. return 0;
  814. }
  815. void dma_debug_add_bus(struct bus_type *bus)
  816. {
  817. struct notifier_block *nb;
  818. if (dma_debug_disabled())
  819. return;
  820. nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
  821. if (nb == NULL) {
  822. pr_err("dma_debug_add_bus: out of memory\n");
  823. return;
  824. }
  825. nb->notifier_call = dma_debug_device_change;
  826. bus_register_notifier(bus, nb);
  827. }
  828. /*
  829. * Let the architectures decide how many entries should be preallocated.
  830. */
  831. void dma_debug_init(u32 num_entries)
  832. {
  833. int i;
  834. /* Do not use dma_debug_initialized here, since we really want to be
  835. * called to set dma_debug_initialized
  836. */
  837. if (global_disable)
  838. return;
  839. for (i = 0; i < HASH_SIZE; ++i) {
  840. INIT_LIST_HEAD(&dma_entry_hash[i].list);
  841. spin_lock_init(&dma_entry_hash[i].lock);
  842. }
  843. if (dma_debug_fs_init() != 0) {
  844. pr_err("DMA-API: error creating debugfs entries - disabling\n");
  845. global_disable = true;
  846. return;
  847. }
  848. if (req_entries)
  849. num_entries = req_entries;
  850. if (prealloc_memory(num_entries) != 0) {
  851. pr_err("DMA-API: debugging out of memory error - disabled\n");
  852. global_disable = true;
  853. return;
  854. }
  855. nr_total_entries = num_free_entries;
  856. dma_debug_initialized = true;
  857. pr_info("DMA-API: debugging enabled by kernel config\n");
  858. }
  859. static __init int dma_debug_cmdline(char *str)
  860. {
  861. if (!str)
  862. return -EINVAL;
  863. if (strncmp(str, "off", 3) == 0) {
  864. pr_info("DMA-API: debugging disabled on kernel command line\n");
  865. global_disable = true;
  866. }
  867. return 0;
  868. }
  869. static __init int dma_debug_entries_cmdline(char *str)
  870. {
  871. int res;
  872. if (!str)
  873. return -EINVAL;
  874. res = get_option(&str, &req_entries);
  875. if (!res)
  876. req_entries = 0;
  877. return 0;
  878. }
  879. __setup("dma_debug=", dma_debug_cmdline);
  880. __setup("dma_debug_entries=", dma_debug_entries_cmdline);
  881. static void check_unmap(struct dma_debug_entry *ref)
  882. {
  883. struct dma_debug_entry *entry;
  884. struct hash_bucket *bucket;
  885. unsigned long flags;
  886. bucket = get_hash_bucket(ref, &flags);
  887. entry = bucket_find_exact(bucket, ref);
  888. if (!entry) {
  889. /* must drop lock before calling dma_mapping_error */
  890. put_hash_bucket(bucket, &flags);
  891. if (dma_mapping_error(ref->dev, ref->dev_addr)) {
  892. err_printk(ref->dev, NULL,
  893. "DMA-API: device driver tries to free an "
  894. "invalid DMA memory address\n");
  895. } else {
  896. err_printk(ref->dev, NULL,
  897. "DMA-API: device driver tries to free DMA "
  898. "memory it has not allocated [device "
  899. "address=0x%016llx] [size=%llu bytes]\n",
  900. ref->dev_addr, ref->size);
  901. }
  902. return;
  903. }
  904. if (ref->size != entry->size) {
  905. err_printk(ref->dev, entry, "DMA-API: device driver frees "
  906. "DMA memory with different size "
  907. "[device address=0x%016llx] [map size=%llu bytes] "
  908. "[unmap size=%llu bytes]\n",
  909. ref->dev_addr, entry->size, ref->size);
  910. }
  911. if (ref->type != entry->type) {
  912. err_printk(ref->dev, entry, "DMA-API: device driver frees "
  913. "DMA memory with wrong function "
  914. "[device address=0x%016llx] [size=%llu bytes] "
  915. "[mapped as %s] [unmapped as %s]\n",
  916. ref->dev_addr, ref->size,
  917. type2name[entry->type], type2name[ref->type]);
  918. } else if ((entry->type == dma_debug_coherent) &&
  919. (phys_addr(ref) != phys_addr(entry))) {
  920. err_printk(ref->dev, entry, "DMA-API: device driver frees "
  921. "DMA memory with different CPU address "
  922. "[device address=0x%016llx] [size=%llu bytes] "
  923. "[cpu alloc address=0x%016llx] "
  924. "[cpu free address=0x%016llx]",
  925. ref->dev_addr, ref->size,
  926. phys_addr(entry),
  927. phys_addr(ref));
  928. }
  929. if (ref->sg_call_ents && ref->type == dma_debug_sg &&
  930. ref->sg_call_ents != entry->sg_call_ents) {
  931. err_printk(ref->dev, entry, "DMA-API: device driver frees "
  932. "DMA sg list with different entry count "
  933. "[map count=%d] [unmap count=%d]\n",
  934. entry->sg_call_ents, ref->sg_call_ents);
  935. }
  936. /*
  937. * This may be no bug in reality - but most implementations of the
  938. * DMA API don't handle this properly, so check for it here
  939. */
  940. if (ref->direction != entry->direction) {
  941. err_printk(ref->dev, entry, "DMA-API: device driver frees "
  942. "DMA memory with different direction "
  943. "[device address=0x%016llx] [size=%llu bytes] "
  944. "[mapped with %s] [unmapped with %s]\n",
  945. ref->dev_addr, ref->size,
  946. dir2name[entry->direction],
  947. dir2name[ref->direction]);
  948. }
  949. if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
  950. err_printk(ref->dev, entry,
  951. "DMA-API: device driver failed to check map error"
  952. "[device address=0x%016llx] [size=%llu bytes] "
  953. "[mapped as %s]",
  954. ref->dev_addr, ref->size,
  955. type2name[entry->type]);
  956. }
  957. hash_bucket_del(entry);
  958. dma_entry_free(entry);
  959. put_hash_bucket(bucket, &flags);
  960. }
  961. static void check_for_stack(struct device *dev, void *addr)
  962. {
  963. if (object_is_on_stack(addr))
  964. err_printk(dev, NULL, "DMA-API: device driver maps memory from "
  965. "stack [addr=%p]\n", addr);
  966. }
  967. static inline bool overlap(void *addr, unsigned long len, void *start, void *end)
  968. {
  969. unsigned long a1 = (unsigned long)addr;
  970. unsigned long b1 = a1 + len;
  971. unsigned long a2 = (unsigned long)start;
  972. unsigned long b2 = (unsigned long)end;
  973. return !(b1 <= a2 || a1 >= b2);
  974. }
  975. static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len)
  976. {
  977. if (overlap(addr, len, _stext, _etext) ||
  978. overlap(addr, len, __start_rodata, __end_rodata))
  979. err_printk(dev, NULL, "DMA-API: device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len);
  980. }
  981. static void check_sync(struct device *dev,
  982. struct dma_debug_entry *ref,
  983. bool to_cpu)
  984. {
  985. struct dma_debug_entry *entry;
  986. struct hash_bucket *bucket;
  987. unsigned long flags;
  988. bucket = get_hash_bucket(ref, &flags);
  989. entry = bucket_find_contain(&bucket, ref, &flags);
  990. if (!entry) {
  991. err_printk(dev, NULL, "DMA-API: device driver tries "
  992. "to sync DMA memory it has not allocated "
  993. "[device address=0x%016llx] [size=%llu bytes]\n",
  994. (unsigned long long)ref->dev_addr, ref->size);
  995. goto out;
  996. }
  997. if (ref->size > entry->size) {
  998. err_printk(dev, entry, "DMA-API: device driver syncs"
  999. " DMA memory outside allocated range "
  1000. "[device address=0x%016llx] "
  1001. "[allocation size=%llu bytes] "
  1002. "[sync offset+size=%llu]\n",
  1003. entry->dev_addr, entry->size,
  1004. ref->size);
  1005. }
  1006. if (entry->direction == DMA_BIDIRECTIONAL)
  1007. goto out;
  1008. if (ref->direction != entry->direction) {
  1009. err_printk(dev, entry, "DMA-API: device driver syncs "
  1010. "DMA memory with different direction "
  1011. "[device address=0x%016llx] [size=%llu bytes] "
  1012. "[mapped with %s] [synced with %s]\n",
  1013. (unsigned long long)ref->dev_addr, entry->size,
  1014. dir2name[entry->direction],
  1015. dir2name[ref->direction]);
  1016. }
  1017. if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&
  1018. !(ref->direction == DMA_TO_DEVICE))
  1019. err_printk(dev, entry, "DMA-API: device driver syncs "
  1020. "device read-only DMA memory for cpu "
  1021. "[device address=0x%016llx] [size=%llu bytes] "
  1022. "[mapped with %s] [synced with %s]\n",
  1023. (unsigned long long)ref->dev_addr, entry->size,
  1024. dir2name[entry->direction],
  1025. dir2name[ref->direction]);
  1026. if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
  1027. !(ref->direction == DMA_FROM_DEVICE))
  1028. err_printk(dev, entry, "DMA-API: device driver syncs "
  1029. "device write-only DMA memory to device "
  1030. "[device address=0x%016llx] [size=%llu bytes] "
  1031. "[mapped with %s] [synced with %s]\n",
  1032. (unsigned long long)ref->dev_addr, entry->size,
  1033. dir2name[entry->direction],
  1034. dir2name[ref->direction]);
  1035. if (ref->sg_call_ents && ref->type == dma_debug_sg &&
  1036. ref->sg_call_ents != entry->sg_call_ents) {
  1037. err_printk(ref->dev, entry, "DMA-API: device driver syncs "
  1038. "DMA sg list with different entry count "
  1039. "[map count=%d] [sync count=%d]\n",
  1040. entry->sg_call_ents, ref->sg_call_ents);
  1041. }
  1042. out:
  1043. put_hash_bucket(bucket, &flags);
  1044. }
  1045. void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
  1046. size_t size, int direction, dma_addr_t dma_addr,
  1047. bool map_single)
  1048. {
  1049. struct dma_debug_entry *entry;
  1050. if (unlikely(dma_debug_disabled()))
  1051. return;
  1052. if (dma_mapping_error(dev, dma_addr))
  1053. return;
  1054. entry = dma_entry_alloc();
  1055. if (!entry)
  1056. return;
  1057. entry->dev = dev;
  1058. entry->type = dma_debug_page;
  1059. entry->pfn = page_to_pfn(page);
  1060. entry->offset = offset,
  1061. entry->dev_addr = dma_addr;
  1062. entry->size = size;
  1063. entry->direction = direction;
  1064. entry->map_err_type = MAP_ERR_NOT_CHECKED;
  1065. if (map_single)
  1066. entry->type = dma_debug_single;
  1067. if (!PageHighMem(page)) {
  1068. void *addr = page_address(page) + offset;
  1069. check_for_stack(dev, addr);
  1070. check_for_illegal_area(dev, addr, size);
  1071. }
  1072. add_dma_entry(entry);
  1073. }
  1074. EXPORT_SYMBOL(debug_dma_map_page);
  1075. void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
  1076. {
  1077. struct dma_debug_entry ref;
  1078. struct dma_debug_entry *entry;
  1079. struct hash_bucket *bucket;
  1080. unsigned long flags;
  1081. if (unlikely(dma_debug_disabled()))
  1082. return;
  1083. ref.dev = dev;
  1084. ref.dev_addr = dma_addr;
  1085. bucket = get_hash_bucket(&ref, &flags);
  1086. list_for_each_entry(entry, &bucket->list, list) {
  1087. if (!exact_match(&ref, entry))
  1088. continue;
  1089. /*
  1090. * The same physical address can be mapped multiple
  1091. * times. Without a hardware IOMMU this results in the
  1092. * same device addresses being put into the dma-debug
  1093. * hash multiple times too. This can result in false
  1094. * positives being reported. Therefore we implement a
  1095. * best-fit algorithm here which updates the first entry
  1096. * from the hash which fits the reference value and is
  1097. * not currently listed as being checked.
  1098. */
  1099. if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
  1100. entry->map_err_type = MAP_ERR_CHECKED;
  1101. break;
  1102. }
  1103. }
  1104. put_hash_bucket(bucket, &flags);
  1105. }
  1106. EXPORT_SYMBOL(debug_dma_mapping_error);
  1107. void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
  1108. size_t size, int direction, bool map_single)
  1109. {
  1110. struct dma_debug_entry ref = {
  1111. .type = dma_debug_page,
  1112. .dev = dev,
  1113. .dev_addr = addr,
  1114. .size = size,
  1115. .direction = direction,
  1116. };
  1117. if (unlikely(dma_debug_disabled()))
  1118. return;
  1119. if (map_single)
  1120. ref.type = dma_debug_single;
  1121. check_unmap(&ref);
  1122. }
  1123. EXPORT_SYMBOL(debug_dma_unmap_page);
  1124. void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
  1125. int nents, int mapped_ents, int direction)
  1126. {
  1127. struct dma_debug_entry *entry;
  1128. struct scatterlist *s;
  1129. int i;
  1130. if (unlikely(dma_debug_disabled()))
  1131. return;
  1132. for_each_sg(sg, s, mapped_ents, i) {
  1133. entry = dma_entry_alloc();
  1134. if (!entry)
  1135. return;
  1136. entry->type = dma_debug_sg;
  1137. entry->dev = dev;
  1138. entry->pfn = page_to_pfn(sg_page(s));
  1139. entry->offset = s->offset,
  1140. entry->size = sg_dma_len(s);
  1141. entry->dev_addr = sg_dma_address(s);
  1142. entry->direction = direction;
  1143. entry->sg_call_ents = nents;
  1144. entry->sg_mapped_ents = mapped_ents;
  1145. if (!PageHighMem(sg_page(s))) {
  1146. check_for_stack(dev, sg_virt(s));
  1147. check_for_illegal_area(dev, sg_virt(s), sg_dma_len(s));
  1148. }
  1149. add_dma_entry(entry);
  1150. }
  1151. }
  1152. EXPORT_SYMBOL(debug_dma_map_sg);
  1153. static int get_nr_mapped_entries(struct device *dev,
  1154. struct dma_debug_entry *ref)
  1155. {
  1156. struct dma_debug_entry *entry;
  1157. struct hash_bucket *bucket;
  1158. unsigned long flags;
  1159. int mapped_ents;
  1160. bucket = get_hash_bucket(ref, &flags);
  1161. entry = bucket_find_exact(bucket, ref);
  1162. mapped_ents = 0;
  1163. if (entry)
  1164. mapped_ents = entry->sg_mapped_ents;
  1165. put_hash_bucket(bucket, &flags);
  1166. return mapped_ents;
  1167. }
  1168. void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
  1169. int nelems, int dir)
  1170. {
  1171. struct scatterlist *s;
  1172. int mapped_ents = 0, i;
  1173. if (unlikely(dma_debug_disabled()))
  1174. return;
  1175. for_each_sg(sglist, s, nelems, i) {
  1176. struct dma_debug_entry ref = {
  1177. .type = dma_debug_sg,
  1178. .dev = dev,
  1179. .pfn = page_to_pfn(sg_page(s)),
  1180. .offset = s->offset,
  1181. .dev_addr = sg_dma_address(s),
  1182. .size = sg_dma_len(s),
  1183. .direction = dir,
  1184. .sg_call_ents = nelems,
  1185. };
  1186. if (mapped_ents && i >= mapped_ents)
  1187. break;
  1188. if (!i)
  1189. mapped_ents = get_nr_mapped_entries(dev, &ref);
  1190. check_unmap(&ref);
  1191. }
  1192. }
  1193. EXPORT_SYMBOL(debug_dma_unmap_sg);
  1194. void debug_dma_alloc_coherent(struct device *dev, size_t size,
  1195. dma_addr_t dma_addr, void *virt)
  1196. {
  1197. struct dma_debug_entry *entry;
  1198. if (unlikely(dma_debug_disabled()))
  1199. return;
  1200. if (unlikely(virt == NULL))
  1201. return;
  1202. entry = dma_entry_alloc();
  1203. if (!entry)
  1204. return;
  1205. entry->type = dma_debug_coherent;
  1206. entry->dev = dev;
  1207. entry->pfn = page_to_pfn(virt_to_page(virt));
  1208. entry->offset = (size_t) virt & ~PAGE_MASK;
  1209. entry->size = size;
  1210. entry->dev_addr = dma_addr;
  1211. entry->direction = DMA_BIDIRECTIONAL;
  1212. add_dma_entry(entry);
  1213. }
  1214. EXPORT_SYMBOL(debug_dma_alloc_coherent);
  1215. void debug_dma_free_coherent(struct device *dev, size_t size,
  1216. void *virt, dma_addr_t addr)
  1217. {
  1218. struct dma_debug_entry ref = {
  1219. .type = dma_debug_coherent,
  1220. .dev = dev,
  1221. .pfn = page_to_pfn(virt_to_page(virt)),
  1222. .offset = (size_t) virt & ~PAGE_MASK,
  1223. .dev_addr = addr,
  1224. .size = size,
  1225. .direction = DMA_BIDIRECTIONAL,
  1226. };
  1227. if (unlikely(dma_debug_disabled()))
  1228. return;
  1229. check_unmap(&ref);
  1230. }
  1231. EXPORT_SYMBOL(debug_dma_free_coherent);
  1232. void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
  1233. size_t size, int direction)
  1234. {
  1235. struct dma_debug_entry ref;
  1236. if (unlikely(dma_debug_disabled()))
  1237. return;
  1238. ref.type = dma_debug_single;
  1239. ref.dev = dev;
  1240. ref.dev_addr = dma_handle;
  1241. ref.size = size;
  1242. ref.direction = direction;
  1243. ref.sg_call_ents = 0;
  1244. check_sync(dev, &ref, true);
  1245. }
  1246. EXPORT_SYMBOL(debug_dma_sync_single_for_cpu);
  1247. void debug_dma_sync_single_for_device(struct device *dev,
  1248. dma_addr_t dma_handle, size_t size,
  1249. int direction)
  1250. {
  1251. struct dma_debug_entry ref;
  1252. if (unlikely(dma_debug_disabled()))
  1253. return;
  1254. ref.type = dma_debug_single;
  1255. ref.dev = dev;
  1256. ref.dev_addr = dma_handle;
  1257. ref.size = size;
  1258. ref.direction = direction;
  1259. ref.sg_call_ents = 0;
  1260. check_sync(dev, &ref, false);
  1261. }
  1262. EXPORT_SYMBOL(debug_dma_sync_single_for_device);
  1263. void debug_dma_sync_single_range_for_cpu(struct device *dev,
  1264. dma_addr_t dma_handle,
  1265. unsigned long offset, size_t size,
  1266. int direction)
  1267. {
  1268. struct dma_debug_entry ref;
  1269. if (unlikely(dma_debug_disabled()))
  1270. return;
  1271. ref.type = dma_debug_single;
  1272. ref.dev = dev;
  1273. ref.dev_addr = dma_handle;
  1274. ref.size = offset + size;
  1275. ref.direction = direction;
  1276. ref.sg_call_ents = 0;
  1277. check_sync(dev, &ref, true);
  1278. }
  1279. EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu);
  1280. void debug_dma_sync_single_range_for_device(struct device *dev,
  1281. dma_addr_t dma_handle,
  1282. unsigned long offset,
  1283. size_t size, int direction)
  1284. {
  1285. struct dma_debug_entry ref;
  1286. if (unlikely(dma_debug_disabled()))
  1287. return;
  1288. ref.type = dma_debug_single;
  1289. ref.dev = dev;
  1290. ref.dev_addr = dma_handle;
  1291. ref.size = offset + size;
  1292. ref.direction = direction;
  1293. ref.sg_call_ents = 0;
  1294. check_sync(dev, &ref, false);
  1295. }
  1296. EXPORT_SYMBOL(debug_dma_sync_single_range_for_device);
  1297. void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
  1298. int nelems, int direction)
  1299. {
  1300. struct scatterlist *s;
  1301. int mapped_ents = 0, i;
  1302. if (unlikely(dma_debug_disabled()))
  1303. return;
  1304. for_each_sg(sg, s, nelems, i) {
  1305. struct dma_debug_entry ref = {
  1306. .type = dma_debug_sg,
  1307. .dev = dev,
  1308. .pfn = page_to_pfn(sg_page(s)),
  1309. .offset = s->offset,
  1310. .dev_addr = sg_dma_address(s),
  1311. .size = sg_dma_len(s),
  1312. .direction = direction,
  1313. .sg_call_ents = nelems,
  1314. };
  1315. if (!i)
  1316. mapped_ents = get_nr_mapped_entries(dev, &ref);
  1317. if (i >= mapped_ents)
  1318. break;
  1319. check_sync(dev, &ref, true);
  1320. }
  1321. }
  1322. EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu);
  1323. void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
  1324. int nelems, int direction)
  1325. {
  1326. struct scatterlist *s;
  1327. int mapped_ents = 0, i;
  1328. if (unlikely(dma_debug_disabled()))
  1329. return;
  1330. for_each_sg(sg, s, nelems, i) {
  1331. struct dma_debug_entry ref = {
  1332. .type = dma_debug_sg,
  1333. .dev = dev,
  1334. .pfn = page_to_pfn(sg_page(s)),
  1335. .offset = s->offset,
  1336. .dev_addr = sg_dma_address(s),
  1337. .size = sg_dma_len(s),
  1338. .direction = direction,
  1339. .sg_call_ents = nelems,
  1340. };
  1341. if (!i)
  1342. mapped_ents = get_nr_mapped_entries(dev, &ref);
  1343. if (i >= mapped_ents)
  1344. break;
  1345. check_sync(dev, &ref, false);
  1346. }
  1347. }
  1348. EXPORT_SYMBOL(debug_dma_sync_sg_for_device);
  1349. static int __init dma_debug_driver_setup(char *str)
  1350. {
  1351. int i;
  1352. for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) {
  1353. current_driver_name[i] = *str;
  1354. if (*str == 0)
  1355. break;
  1356. }
  1357. if (current_driver_name[0])
  1358. pr_info("DMA-API: enable driver filter for driver [%s]\n",
  1359. current_driver_name);
  1360. return 1;
  1361. }
  1362. __setup("dma_debug_driver=", dma_debug_driver_setup);