slab.h 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390
  1. #ifndef MM_SLAB_H
  2. #define MM_SLAB_H
  3. /*
  4. * Internal slab definitions
  5. */
  6. #ifdef CONFIG_SLOB
  7. /*
  8. * Common fields provided in kmem_cache by all slab allocators
  9. * This struct is either used directly by the allocator (SLOB)
  10. * or the allocator must include definitions for all fields
  11. * provided in kmem_cache_common in their definition of kmem_cache.
  12. *
  13. * Once we can do anonymous structs (C11 standard) we could put a
  14. * anonymous struct definition in these allocators so that the
  15. * separate allocations in the kmem_cache structure of SLAB and
  16. * SLUB is no longer needed.
  17. */
  18. struct kmem_cache {
  19. unsigned int object_size;/* The original size of the object */
  20. unsigned int size; /* The aligned/padded/added on size */
  21. unsigned int align; /* Alignment as calculated */
  22. unsigned long flags; /* Active flags on the slab */
  23. const char *name; /* Slab name for sysfs */
  24. int refcount; /* Use counter */
  25. void (*ctor)(void *); /* Called on object slot creation */
  26. struct list_head list; /* List of all slab caches on the system */
  27. };
  28. #endif /* CONFIG_SLOB */
  29. #ifdef CONFIG_SLAB
  30. #include <linux/slab_def.h>
  31. #endif
  32. #ifdef CONFIG_SLUB
  33. #include <linux/slub_def.h>
  34. #endif
  35. #include <linux/memcontrol.h>
  36. /*
  37. * State of the slab allocator.
  38. *
  39. * This is used to describe the states of the allocator during bootup.
  40. * Allocators use this to gradually bootstrap themselves. Most allocators
  41. * have the problem that the structures used for managing slab caches are
  42. * allocated from slab caches themselves.
  43. */
  44. enum slab_state {
  45. DOWN, /* No slab functionality yet */
  46. PARTIAL, /* SLUB: kmem_cache_node available */
  47. PARTIAL_NODE, /* SLAB: kmalloc size for node struct available */
  48. UP, /* Slab caches usable but not all extras yet */
  49. FULL /* Everything is working */
  50. };
  51. extern enum slab_state slab_state;
  52. /* The slab cache mutex protects the management structures during changes */
  53. extern struct mutex slab_mutex;
  54. /* The list of all slab caches on the system */
  55. extern struct list_head slab_caches;
  56. /* The slab cache that manages slab cache information */
  57. extern struct kmem_cache *kmem_cache;
  58. unsigned long calculate_alignment(unsigned long flags,
  59. unsigned long align, unsigned long size);
  60. #ifndef CONFIG_SLOB
  61. /* Kmalloc array related functions */
  62. void setup_kmalloc_cache_index_table(void);
  63. void create_kmalloc_caches(unsigned long);
  64. /* Find the kmalloc slab corresponding for a certain size */
  65. struct kmem_cache *kmalloc_slab(size_t, gfp_t);
  66. #endif
  67. /* Functions provided by the slab allocators */
  68. extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags);
  69. extern struct kmem_cache *create_kmalloc_cache(const char *name, size_t size,
  70. unsigned long flags);
  71. extern void create_boot_cache(struct kmem_cache *, const char *name,
  72. size_t size, unsigned long flags);
  73. int slab_unmergeable(struct kmem_cache *s);
  74. struct kmem_cache *find_mergeable(size_t size, size_t align,
  75. unsigned long flags, const char *name, void (*ctor)(void *));
  76. #ifndef CONFIG_SLOB
  77. struct kmem_cache *
  78. __kmem_cache_alias(const char *name, size_t size, size_t align,
  79. unsigned long flags, void (*ctor)(void *));
  80. unsigned long kmem_cache_flags(unsigned long object_size,
  81. unsigned long flags, const char *name,
  82. void (*ctor)(void *));
  83. #else
  84. static inline struct kmem_cache *
  85. __kmem_cache_alias(const char *name, size_t size, size_t align,
  86. unsigned long flags, void (*ctor)(void *))
  87. { return NULL; }
  88. static inline unsigned long kmem_cache_flags(unsigned long object_size,
  89. unsigned long flags, const char *name,
  90. void (*ctor)(void *))
  91. {
  92. return flags;
  93. }
  94. #endif
  95. /* Legal flag mask for kmem_cache_create(), for various configurations */
  96. #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | SLAB_PANIC | \
  97. SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS )
  98. #if defined(CONFIG_DEBUG_SLAB)
  99. #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
  100. #elif defined(CONFIG_SLUB_DEBUG)
  101. #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
  102. SLAB_TRACE | SLAB_DEBUG_FREE)
  103. #else
  104. #define SLAB_DEBUG_FLAGS (0)
  105. #endif
  106. #if defined(CONFIG_SLAB)
  107. #define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD | SLAB_NOLEAKTRACE | \
  108. SLAB_RECLAIM_ACCOUNT | SLAB_TEMPORARY | SLAB_NOTRACK)
  109. #elif defined(CONFIG_SLUB)
  110. #define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \
  111. SLAB_TEMPORARY | SLAB_NOTRACK)
  112. #else
  113. #define SLAB_CACHE_FLAGS (0)
  114. #endif
  115. #define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS)
  116. int __kmem_cache_shutdown(struct kmem_cache *);
  117. int __kmem_cache_shrink(struct kmem_cache *, bool);
  118. void slab_kmem_cache_release(struct kmem_cache *);
  119. struct seq_file;
  120. struct file;
  121. struct slabinfo {
  122. unsigned long active_objs;
  123. unsigned long num_objs;
  124. unsigned long active_slabs;
  125. unsigned long num_slabs;
  126. unsigned long shared_avail;
  127. unsigned int limit;
  128. unsigned int batchcount;
  129. unsigned int shared;
  130. unsigned int objects_per_slab;
  131. unsigned int cache_order;
  132. };
  133. void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo);
  134. void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s);
  135. ssize_t slabinfo_write(struct file *file, const char __user *buffer,
  136. size_t count, loff_t *ppos);
  137. /*
  138. * Generic implementation of bulk operations
  139. * These are useful for situations in which the allocator cannot
  140. * perform optimizations. In that case segments of the objecct listed
  141. * may be allocated or freed using these operations.
  142. */
  143. void __kmem_cache_free_bulk(struct kmem_cache *, size_t, void **);
  144. bool __kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **);
  145. #ifdef CONFIG_MEMCG_KMEM
  146. /*
  147. * Iterate over all memcg caches of the given root cache. The caller must hold
  148. * slab_mutex.
  149. */
  150. #define for_each_memcg_cache(iter, root) \
  151. list_for_each_entry(iter, &(root)->memcg_params.list, \
  152. memcg_params.list)
  153. #define for_each_memcg_cache_safe(iter, tmp, root) \
  154. list_for_each_entry_safe(iter, tmp, &(root)->memcg_params.list, \
  155. memcg_params.list)
  156. static inline bool is_root_cache(struct kmem_cache *s)
  157. {
  158. return s->memcg_params.is_root_cache;
  159. }
  160. static inline bool slab_equal_or_root(struct kmem_cache *s,
  161. struct kmem_cache *p)
  162. {
  163. return p == s || p == s->memcg_params.root_cache;
  164. }
  165. /*
  166. * We use suffixes to the name in memcg because we can't have caches
  167. * created in the system with the same name. But when we print them
  168. * locally, better refer to them with the base name
  169. */
  170. static inline const char *cache_name(struct kmem_cache *s)
  171. {
  172. if (!is_root_cache(s))
  173. s = s->memcg_params.root_cache;
  174. return s->name;
  175. }
  176. /*
  177. * Note, we protect with RCU only the memcg_caches array, not per-memcg caches.
  178. * That said the caller must assure the memcg's cache won't go away by either
  179. * taking a css reference to the owner cgroup, or holding the slab_mutex.
  180. */
  181. static inline struct kmem_cache *
  182. cache_from_memcg_idx(struct kmem_cache *s, int idx)
  183. {
  184. struct kmem_cache *cachep;
  185. struct memcg_cache_array *arr;
  186. rcu_read_lock();
  187. arr = rcu_dereference(s->memcg_params.memcg_caches);
  188. /*
  189. * Make sure we will access the up-to-date value. The code updating
  190. * memcg_caches issues a write barrier to match this (see
  191. * memcg_create_kmem_cache()).
  192. */
  193. cachep = lockless_dereference(arr->entries[idx]);
  194. rcu_read_unlock();
  195. return cachep;
  196. }
  197. static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s)
  198. {
  199. if (is_root_cache(s))
  200. return s;
  201. return s->memcg_params.root_cache;
  202. }
  203. static __always_inline int memcg_charge_slab(struct kmem_cache *s,
  204. gfp_t gfp, int order)
  205. {
  206. if (!memcg_kmem_enabled())
  207. return 0;
  208. if (is_root_cache(s))
  209. return 0;
  210. return memcg_charge_kmem(s->memcg_params.memcg, gfp, 1 << order);
  211. }
  212. static __always_inline void memcg_uncharge_slab(struct kmem_cache *s, int order)
  213. {
  214. if (!memcg_kmem_enabled())
  215. return;
  216. if (is_root_cache(s))
  217. return;
  218. memcg_uncharge_kmem(s->memcg_params.memcg, 1 << order);
  219. }
  220. extern void slab_init_memcg_params(struct kmem_cache *);
  221. #else /* !CONFIG_MEMCG_KMEM */
  222. #define for_each_memcg_cache(iter, root) \
  223. for ((void)(iter), (void)(root); 0; )
  224. #define for_each_memcg_cache_safe(iter, tmp, root) \
  225. for ((void)(iter), (void)(tmp), (void)(root); 0; )
  226. static inline bool is_root_cache(struct kmem_cache *s)
  227. {
  228. return true;
  229. }
  230. static inline bool slab_equal_or_root(struct kmem_cache *s,
  231. struct kmem_cache *p)
  232. {
  233. return true;
  234. }
  235. static inline const char *cache_name(struct kmem_cache *s)
  236. {
  237. return s->name;
  238. }
  239. static inline struct kmem_cache *
  240. cache_from_memcg_idx(struct kmem_cache *s, int idx)
  241. {
  242. return NULL;
  243. }
  244. static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s)
  245. {
  246. return s;
  247. }
  248. static inline int memcg_charge_slab(struct kmem_cache *s, gfp_t gfp, int order)
  249. {
  250. return 0;
  251. }
  252. static inline void memcg_uncharge_slab(struct kmem_cache *s, int order)
  253. {
  254. }
  255. static inline void slab_init_memcg_params(struct kmem_cache *s)
  256. {
  257. }
  258. #endif /* CONFIG_MEMCG_KMEM */
  259. static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
  260. {
  261. struct kmem_cache *cachep;
  262. struct page *page;
  263. /*
  264. * When kmemcg is not being used, both assignments should return the
  265. * same value. but we don't want to pay the assignment price in that
  266. * case. If it is not compiled in, the compiler should be smart enough
  267. * to not do even the assignment. In that case, slab_equal_or_root
  268. * will also be a constant.
  269. */
  270. if (!memcg_kmem_enabled() && !unlikely(s->flags & SLAB_DEBUG_FREE))
  271. return s;
  272. page = virt_to_head_page(x);
  273. cachep = page->slab_cache;
  274. if (slab_equal_or_root(cachep, s))
  275. return cachep;
  276. pr_err("%s: Wrong slab cache. %s but object is from %s\n",
  277. __func__, s->name, cachep->name);
  278. WARN_ON_ONCE(1);
  279. return s;
  280. }
  281. #ifndef CONFIG_SLOB
  282. /*
  283. * The slab lists for all objects.
  284. */
  285. struct kmem_cache_node {
  286. spinlock_t list_lock;
  287. #ifdef CONFIG_SLAB
  288. struct list_head slabs_partial; /* partial list first, better asm code */
  289. struct list_head slabs_full;
  290. struct list_head slabs_free;
  291. unsigned long free_objects;
  292. unsigned int free_limit;
  293. unsigned int colour_next; /* Per-node cache coloring */
  294. struct array_cache *shared; /* shared per node */
  295. struct alien_cache **alien; /* on other nodes */
  296. unsigned long next_reap; /* updated without locking */
  297. int free_touched; /* updated without locking */
  298. #endif
  299. #ifdef CONFIG_SLUB
  300. unsigned long nr_partial;
  301. struct list_head partial;
  302. #ifdef CONFIG_SLUB_DEBUG
  303. atomic_long_t nr_slabs;
  304. atomic_long_t total_objects;
  305. struct list_head full;
  306. #endif
  307. #endif
  308. };
  309. static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
  310. {
  311. return s->node[node];
  312. }
  313. /*
  314. * Iterator over all nodes. The body will be executed for each node that has
  315. * a kmem_cache_node structure allocated (which is true for all online nodes)
  316. */
  317. #define for_each_kmem_cache_node(__s, __node, __n) \
  318. for (__node = 0; __node < nr_node_ids; __node++) \
  319. if ((__n = get_node(__s, __node)))
  320. #endif
  321. void *slab_start(struct seq_file *m, loff_t *pos);
  322. void *slab_next(struct seq_file *m, void *p, loff_t *pos);
  323. void slab_stop(struct seq_file *m, void *p);
  324. int memcg_slab_show(struct seq_file *m, void *p);
  325. #endif /* MM_SLAB_H */