|
@@ -100,7 +100,7 @@
|
|
* inode as:
|
|
* inode as:
|
|
*
|
|
*
|
|
* { page }
|
|
* { page }
|
|
- * [ group 0 buddy][ group 0 bitmap] [group 1][ group 1]...
|
|
|
|
|
|
+ * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]...
|
|
*
|
|
*
|
|
*
|
|
*
|
|
* one block each for bitmap and buddy information. So for each group we
|
|
* one block each for bitmap and buddy information. So for each group we
|
|
@@ -330,6 +330,18 @@
|
|
* object
|
|
* object
|
|
*
|
|
*
|
|
*/
|
|
*/
|
|
|
|
+static struct kmem_cache *ext4_pspace_cachep;
|
|
|
|
+static struct kmem_cache *ext4_ac_cachep;
|
|
|
|
+static struct kmem_cache *ext4_free_ext_cachep;
|
|
|
|
+static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap,
|
|
|
|
+ ext4_group_t group);
|
|
|
|
+static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap,
|
|
|
|
+ ext4_group_t group);
|
|
|
|
+static int ext4_mb_init_per_dev_proc(struct super_block *sb);
|
|
|
|
+static int ext4_mb_destroy_per_dev_proc(struct super_block *sb);
|
|
|
|
+static void release_blocks_on_commit(journal_t *journal, transaction_t *txn);
|
|
|
|
+
|
|
|
|
+
|
|
|
|
|
|
static inline void *mb_correct_addr_and_bit(int *bit, void *addr)
|
|
static inline void *mb_correct_addr_and_bit(int *bit, void *addr)
|
|
{
|
|
{
|
|
@@ -445,9 +457,9 @@ static void mb_free_blocks_double(struct inode *inode, struct ext4_buddy *e4b,
|
|
blocknr += first + i;
|
|
blocknr += first + i;
|
|
blocknr +=
|
|
blocknr +=
|
|
le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block);
|
|
le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block);
|
|
-
|
|
|
|
- ext4_error(sb, __func__, "double-free of inode"
|
|
|
|
- " %lu's block %llu(bit %u in group %lu)\n",
|
|
|
|
|
|
+ ext4_grp_locked_error(sb, e4b->bd_group,
|
|
|
|
+ __func__, "double-free of inode"
|
|
|
|
+ " %lu's block %llu(bit %u in group %u)",
|
|
inode ? inode->i_ino : 0, blocknr,
|
|
inode ? inode->i_ino : 0, blocknr,
|
|
first + i, e4b->bd_group);
|
|
first + i, e4b->bd_group);
|
|
}
|
|
}
|
|
@@ -477,7 +489,7 @@ static void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap)
|
|
b2 = (unsigned char *) bitmap;
|
|
b2 = (unsigned char *) bitmap;
|
|
for (i = 0; i < e4b->bd_sb->s_blocksize; i++) {
|
|
for (i = 0; i < e4b->bd_sb->s_blocksize; i++) {
|
|
if (b1[i] != b2[i]) {
|
|
if (b1[i] != b2[i]) {
|
|
- printk(KERN_ERR "corruption in group %lu "
|
|
|
|
|
|
+ printk(KERN_ERR "corruption in group %u "
|
|
"at byte %u(%u): %x in copy != %x "
|
|
"at byte %u(%u): %x in copy != %x "
|
|
"on disk/prealloc\n",
|
|
"on disk/prealloc\n",
|
|
e4b->bd_group, i, i * 8, b1[i], b2[i]);
|
|
e4b->bd_group, i, i * 8, b1[i], b2[i]);
|
|
@@ -690,8 +702,8 @@ static void ext4_mb_generate_buddy(struct super_block *sb,
|
|
grp->bb_fragments = fragments;
|
|
grp->bb_fragments = fragments;
|
|
|
|
|
|
if (free != grp->bb_free) {
|
|
if (free != grp->bb_free) {
|
|
- ext4_error(sb, __func__,
|
|
|
|
- "EXT4-fs: group %lu: %u blocks in bitmap, %u in gd\n",
|
|
|
|
|
|
+ ext4_grp_locked_error(sb, group, __func__,
|
|
|
|
+ "EXT4-fs: group %u: %u blocks in bitmap, %u in gd",
|
|
group, free, grp->bb_free);
|
|
group, free, grp->bb_free);
|
|
/*
|
|
/*
|
|
* If we intent to continue, we consider group descritor
|
|
* If we intent to continue, we consider group descritor
|
|
@@ -716,7 +728,7 @@ static void ext4_mb_generate_buddy(struct super_block *sb,
|
|
* stored in the inode as
|
|
* stored in the inode as
|
|
*
|
|
*
|
|
* { page }
|
|
* { page }
|
|
- * [ group 0 buddy][ group 0 bitmap] [group 1][ group 1]...
|
|
|
|
|
|
+ * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]...
|
|
*
|
|
*
|
|
*
|
|
*
|
|
* one block each for bitmap and buddy information.
|
|
* one block each for bitmap and buddy information.
|
|
@@ -782,25 +794,45 @@ static int ext4_mb_init_cache(struct page *page, char *incore)
|
|
if (bh[i] == NULL)
|
|
if (bh[i] == NULL)
|
|
goto out;
|
|
goto out;
|
|
|
|
|
|
- if (buffer_uptodate(bh[i]) &&
|
|
|
|
- !(desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)))
|
|
|
|
|
|
+ if (bitmap_uptodate(bh[i]))
|
|
continue;
|
|
continue;
|
|
|
|
|
|
lock_buffer(bh[i]);
|
|
lock_buffer(bh[i]);
|
|
|
|
+ if (bitmap_uptodate(bh[i])) {
|
|
|
|
+ unlock_buffer(bh[i]);
|
|
|
|
+ continue;
|
|
|
|
+ }
|
|
spin_lock(sb_bgl_lock(EXT4_SB(sb), first_group + i));
|
|
spin_lock(sb_bgl_lock(EXT4_SB(sb), first_group + i));
|
|
if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) {
|
|
if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) {
|
|
ext4_init_block_bitmap(sb, bh[i],
|
|
ext4_init_block_bitmap(sb, bh[i],
|
|
first_group + i, desc);
|
|
first_group + i, desc);
|
|
|
|
+ set_bitmap_uptodate(bh[i]);
|
|
set_buffer_uptodate(bh[i]);
|
|
set_buffer_uptodate(bh[i]);
|
|
- unlock_buffer(bh[i]);
|
|
|
|
spin_unlock(sb_bgl_lock(EXT4_SB(sb), first_group + i));
|
|
spin_unlock(sb_bgl_lock(EXT4_SB(sb), first_group + i));
|
|
|
|
+ unlock_buffer(bh[i]);
|
|
continue;
|
|
continue;
|
|
}
|
|
}
|
|
spin_unlock(sb_bgl_lock(EXT4_SB(sb), first_group + i));
|
|
spin_unlock(sb_bgl_lock(EXT4_SB(sb), first_group + i));
|
|
|
|
+ if (buffer_uptodate(bh[i])) {
|
|
|
|
+ /*
|
|
|
|
+ * if not uninit if bh is uptodate,
|
|
|
|
+ * bitmap is also uptodate
|
|
|
|
+ */
|
|
|
|
+ set_bitmap_uptodate(bh[i]);
|
|
|
|
+ unlock_buffer(bh[i]);
|
|
|
|
+ continue;
|
|
|
|
+ }
|
|
get_bh(bh[i]);
|
|
get_bh(bh[i]);
|
|
|
|
+ /*
|
|
|
|
+ * submit the buffer_head for read. We can
|
|
|
|
+ * safely mark the bitmap as uptodate now.
|
|
|
|
+ * We do it here so the bitmap uptodate bit
|
|
|
|
+ * get set with buffer lock held.
|
|
|
|
+ */
|
|
|
|
+ set_bitmap_uptodate(bh[i]);
|
|
bh[i]->b_end_io = end_buffer_read_sync;
|
|
bh[i]->b_end_io = end_buffer_read_sync;
|
|
submit_bh(READ, bh[i]);
|
|
submit_bh(READ, bh[i]);
|
|
- mb_debug("read bitmap for group %lu\n", first_group + i);
|
|
|
|
|
|
+ mb_debug("read bitmap for group %u\n", first_group + i);
|
|
}
|
|
}
|
|
|
|
|
|
/* wait for I/O completion */
|
|
/* wait for I/O completion */
|
|
@@ -814,6 +846,8 @@ static int ext4_mb_init_cache(struct page *page, char *incore)
|
|
|
|
|
|
err = 0;
|
|
err = 0;
|
|
first_block = page->index * blocks_per_page;
|
|
first_block = page->index * blocks_per_page;
|
|
|
|
+ /* init the page */
|
|
|
|
+ memset(page_address(page), 0xff, PAGE_CACHE_SIZE);
|
|
for (i = 0; i < blocks_per_page; i++) {
|
|
for (i = 0; i < blocks_per_page; i++) {
|
|
int group;
|
|
int group;
|
|
struct ext4_group_info *grinfo;
|
|
struct ext4_group_info *grinfo;
|
|
@@ -840,7 +874,6 @@ static int ext4_mb_init_cache(struct page *page, char *incore)
|
|
BUG_ON(incore == NULL);
|
|
BUG_ON(incore == NULL);
|
|
mb_debug("put buddy for group %u in page %lu/%x\n",
|
|
mb_debug("put buddy for group %u in page %lu/%x\n",
|
|
group, page->index, i * blocksize);
|
|
group, page->index, i * blocksize);
|
|
- memset(data, 0xff, blocksize);
|
|
|
|
grinfo = ext4_get_group_info(sb, group);
|
|
grinfo = ext4_get_group_info(sb, group);
|
|
grinfo->bb_fragments = 0;
|
|
grinfo->bb_fragments = 0;
|
|
memset(grinfo->bb_counters, 0,
|
|
memset(grinfo->bb_counters, 0,
|
|
@@ -848,7 +881,9 @@ static int ext4_mb_init_cache(struct page *page, char *incore)
|
|
/*
|
|
/*
|
|
* incore got set to the group block bitmap below
|
|
* incore got set to the group block bitmap below
|
|
*/
|
|
*/
|
|
|
|
+ ext4_lock_group(sb, group);
|
|
ext4_mb_generate_buddy(sb, data, incore, group);
|
|
ext4_mb_generate_buddy(sb, data, incore, group);
|
|
|
|
+ ext4_unlock_group(sb, group);
|
|
incore = NULL;
|
|
incore = NULL;
|
|
} else {
|
|
} else {
|
|
/* this is block of bitmap */
|
|
/* this is block of bitmap */
|
|
@@ -862,6 +897,7 @@ static int ext4_mb_init_cache(struct page *page, char *incore)
|
|
|
|
|
|
/* mark all preallocated blks used in in-core bitmap */
|
|
/* mark all preallocated blks used in in-core bitmap */
|
|
ext4_mb_generate_from_pa(sb, data, group);
|
|
ext4_mb_generate_from_pa(sb, data, group);
|
|
|
|
+ ext4_mb_generate_from_freelist(sb, data, group);
|
|
ext4_unlock_group(sb, group);
|
|
ext4_unlock_group(sb, group);
|
|
|
|
|
|
/* set incore so that the buddy information can be
|
|
/* set incore so that the buddy information can be
|
|
@@ -886,18 +922,20 @@ static noinline_for_stack int
|
|
ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group,
|
|
ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group,
|
|
struct ext4_buddy *e4b)
|
|
struct ext4_buddy *e4b)
|
|
{
|
|
{
|
|
- struct ext4_sb_info *sbi = EXT4_SB(sb);
|
|
|
|
- struct inode *inode = sbi->s_buddy_cache;
|
|
|
|
int blocks_per_page;
|
|
int blocks_per_page;
|
|
int block;
|
|
int block;
|
|
int pnum;
|
|
int pnum;
|
|
int poff;
|
|
int poff;
|
|
struct page *page;
|
|
struct page *page;
|
|
int ret;
|
|
int ret;
|
|
|
|
+ struct ext4_group_info *grp;
|
|
|
|
+ struct ext4_sb_info *sbi = EXT4_SB(sb);
|
|
|
|
+ struct inode *inode = sbi->s_buddy_cache;
|
|
|
|
|
|
- mb_debug("load group %lu\n", group);
|
|
|
|
|
|
+ mb_debug("load group %u\n", group);
|
|
|
|
|
|
blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize;
|
|
blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize;
|
|
|
|
+ grp = ext4_get_group_info(sb, group);
|
|
|
|
|
|
e4b->bd_blkbits = sb->s_blocksize_bits;
|
|
e4b->bd_blkbits = sb->s_blocksize_bits;
|
|
e4b->bd_info = ext4_get_group_info(sb, group);
|
|
e4b->bd_info = ext4_get_group_info(sb, group);
|
|
@@ -905,6 +943,15 @@ ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group,
|
|
e4b->bd_group = group;
|
|
e4b->bd_group = group;
|
|
e4b->bd_buddy_page = NULL;
|
|
e4b->bd_buddy_page = NULL;
|
|
e4b->bd_bitmap_page = NULL;
|
|
e4b->bd_bitmap_page = NULL;
|
|
|
|
+ e4b->alloc_semp = &grp->alloc_sem;
|
|
|
|
+
|
|
|
|
+ /* Take the read lock on the group alloc
|
|
|
|
+ * sem. This would make sure a parallel
|
|
|
|
+ * ext4_mb_init_group happening on other
|
|
|
|
+ * groups mapped by the page is blocked
|
|
|
|
+ * till we are done with allocation
|
|
|
|
+ */
|
|
|
|
+ down_read(e4b->alloc_semp);
|
|
|
|
|
|
/*
|
|
/*
|
|
* the buddy cache inode stores the block bitmap
|
|
* the buddy cache inode stores the block bitmap
|
|
@@ -920,6 +967,14 @@ ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group,
|
|
page = find_get_page(inode->i_mapping, pnum);
|
|
page = find_get_page(inode->i_mapping, pnum);
|
|
if (page == NULL || !PageUptodate(page)) {
|
|
if (page == NULL || !PageUptodate(page)) {
|
|
if (page)
|
|
if (page)
|
|
|
|
+ /*
|
|
|
|
+ * drop the page reference and try
|
|
|
|
+ * to get the page with lock. If we
|
|
|
|
+ * are not uptodate that implies
|
|
|
|
+ * somebody just created the page but
|
|
|
|
+ * is yet to initialize the same. So
|
|
|
|
+ * wait for it to initialize.
|
|
|
|
+ */
|
|
page_cache_release(page);
|
|
page_cache_release(page);
|
|
page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
|
|
page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
|
|
if (page) {
|
|
if (page) {
|
|
@@ -985,6 +1040,9 @@ err:
|
|
page_cache_release(e4b->bd_buddy_page);
|
|
page_cache_release(e4b->bd_buddy_page);
|
|
e4b->bd_buddy = NULL;
|
|
e4b->bd_buddy = NULL;
|
|
e4b->bd_bitmap = NULL;
|
|
e4b->bd_bitmap = NULL;
|
|
|
|
+
|
|
|
|
+ /* Done with the buddy cache */
|
|
|
|
+ up_read(e4b->alloc_semp);
|
|
return ret;
|
|
return ret;
|
|
}
|
|
}
|
|
|
|
|
|
@@ -994,6 +1052,9 @@ static void ext4_mb_release_desc(struct ext4_buddy *e4b)
|
|
page_cache_release(e4b->bd_bitmap_page);
|
|
page_cache_release(e4b->bd_bitmap_page);
|
|
if (e4b->bd_buddy_page)
|
|
if (e4b->bd_buddy_page)
|
|
page_cache_release(e4b->bd_buddy_page);
|
|
page_cache_release(e4b->bd_buddy_page);
|
|
|
|
+ /* Done with the buddy cache */
|
|
|
|
+ if (e4b->alloc_semp)
|
|
|
|
+ up_read(e4b->alloc_semp);
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -1031,7 +1092,10 @@ static void mb_clear_bits(spinlock_t *lock, void *bm, int cur, int len)
|
|
cur += 32;
|
|
cur += 32;
|
|
continue;
|
|
continue;
|
|
}
|
|
}
|
|
- mb_clear_bit_atomic(lock, cur, bm);
|
|
|
|
|
|
+ if (lock)
|
|
|
|
+ mb_clear_bit_atomic(lock, cur, bm);
|
|
|
|
+ else
|
|
|
|
+ mb_clear_bit(cur, bm);
|
|
cur++;
|
|
cur++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
@@ -1049,7 +1113,10 @@ static void mb_set_bits(spinlock_t *lock, void *bm, int cur, int len)
|
|
cur += 32;
|
|
cur += 32;
|
|
continue;
|
|
continue;
|
|
}
|
|
}
|
|
- mb_set_bit_atomic(lock, cur, bm);
|
|
|
|
|
|
+ if (lock)
|
|
|
|
+ mb_set_bit_atomic(lock, cur, bm);
|
|
|
|
+ else
|
|
|
|
+ mb_set_bit(cur, bm);
|
|
cur++;
|
|
cur++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
@@ -1094,12 +1161,11 @@ static void mb_free_blocks(struct inode *inode, struct ext4_buddy *e4b,
|
|
blocknr += block;
|
|
blocknr += block;
|
|
blocknr +=
|
|
blocknr +=
|
|
le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block);
|
|
le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block);
|
|
- ext4_unlock_group(sb, e4b->bd_group);
|
|
|
|
- ext4_error(sb, __func__, "double-free of inode"
|
|
|
|
- " %lu's block %llu(bit %u in group %lu)\n",
|
|
|
|
|
|
+ ext4_grp_locked_error(sb, e4b->bd_group,
|
|
|
|
+ __func__, "double-free of inode"
|
|
|
|
+ " %lu's block %llu(bit %u in group %u)",
|
|
inode ? inode->i_ino : 0, blocknr, block,
|
|
inode ? inode->i_ino : 0, blocknr, block,
|
|
e4b->bd_group);
|
|
e4b->bd_group);
|
|
- ext4_lock_group(sb, e4b->bd_group);
|
|
|
|
}
|
|
}
|
|
mb_clear_bit(block, EXT4_MB_BITMAP(e4b));
|
|
mb_clear_bit(block, EXT4_MB_BITMAP(e4b));
|
|
e4b->bd_info->bb_counters[order]++;
|
|
e4b->bd_info->bb_counters[order]++;
|
|
@@ -1296,13 +1362,20 @@ static void ext4_mb_use_best_found(struct ext4_allocation_context *ac,
|
|
ac->ac_tail = ret & 0xffff;
|
|
ac->ac_tail = ret & 0xffff;
|
|
ac->ac_buddy = ret >> 16;
|
|
ac->ac_buddy = ret >> 16;
|
|
|
|
|
|
- /* XXXXXXX: SUCH A HORRIBLE **CK */
|
|
|
|
- /*FIXME!! Why ? */
|
|
|
|
|
|
+ /*
|
|
|
|
+ * take the page reference. We want the page to be pinned
|
|
|
|
+ * so that we don't get a ext4_mb_init_cache_call for this
|
|
|
|
+ * group until we update the bitmap. That would mean we
|
|
|
|
+ * double allocate blocks. The reference is dropped
|
|
|
|
+ * in ext4_mb_release_context
|
|
|
|
+ */
|
|
ac->ac_bitmap_page = e4b->bd_bitmap_page;
|
|
ac->ac_bitmap_page = e4b->bd_bitmap_page;
|
|
get_page(ac->ac_bitmap_page);
|
|
get_page(ac->ac_bitmap_page);
|
|
ac->ac_buddy_page = e4b->bd_buddy_page;
|
|
ac->ac_buddy_page = e4b->bd_buddy_page;
|
|
get_page(ac->ac_buddy_page);
|
|
get_page(ac->ac_buddy_page);
|
|
-
|
|
|
|
|
|
+ /* on allocation we use ac to track the held semaphore */
|
|
|
|
+ ac->alloc_semp = e4b->alloc_semp;
|
|
|
|
+ e4b->alloc_semp = NULL;
|
|
/* store last allocated for subsequent stream allocation */
|
|
/* store last allocated for subsequent stream allocation */
|
|
if ((ac->ac_flags & EXT4_MB_HINT_DATA)) {
|
|
if ((ac->ac_flags & EXT4_MB_HINT_DATA)) {
|
|
spin_lock(&sbi->s_md_lock);
|
|
spin_lock(&sbi->s_md_lock);
|
|
@@ -1326,6 +1399,8 @@ static void ext4_mb_check_limits(struct ext4_allocation_context *ac,
|
|
struct ext4_free_extent ex;
|
|
struct ext4_free_extent ex;
|
|
int max;
|
|
int max;
|
|
|
|
|
|
|
|
+ if (ac->ac_status == AC_STATUS_FOUND)
|
|
|
|
+ return;
|
|
/*
|
|
/*
|
|
* We don't want to scan for a whole year
|
|
* We don't want to scan for a whole year
|
|
*/
|
|
*/
|
|
@@ -1575,8 +1650,9 @@ static void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac,
|
|
* free blocks even though group info says we
|
|
* free blocks even though group info says we
|
|
* we have free blocks
|
|
* we have free blocks
|
|
*/
|
|
*/
|
|
- ext4_error(sb, __func__, "%d free blocks as per "
|
|
|
|
- "group info. But bitmap says 0\n",
|
|
|
|
|
|
+ ext4_grp_locked_error(sb, e4b->bd_group,
|
|
|
|
+ __func__, "%d free blocks as per "
|
|
|
|
+ "group info. But bitmap says 0",
|
|
free);
|
|
free);
|
|
break;
|
|
break;
|
|
}
|
|
}
|
|
@@ -1584,8 +1660,9 @@ static void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac,
|
|
mb_find_extent(e4b, 0, i, ac->ac_g_ex.fe_len, &ex);
|
|
mb_find_extent(e4b, 0, i, ac->ac_g_ex.fe_len, &ex);
|
|
BUG_ON(ex.fe_len <= 0);
|
|
BUG_ON(ex.fe_len <= 0);
|
|
if (free < ex.fe_len) {
|
|
if (free < ex.fe_len) {
|
|
- ext4_error(sb, __func__, "%d free blocks as per "
|
|
|
|
- "group info. But got %d blocks\n",
|
|
|
|
|
|
+ ext4_grp_locked_error(sb, e4b->bd_group,
|
|
|
|
+ __func__, "%d free blocks as per "
|
|
|
|
+ "group info. But got %d blocks",
|
|
free, ex.fe_len);
|
|
free, ex.fe_len);
|
|
/*
|
|
/*
|
|
* The number of free blocks differs. This mostly
|
|
* The number of free blocks differs. This mostly
|
|
@@ -1692,6 +1769,173 @@ static int ext4_mb_good_group(struct ext4_allocation_context *ac,
|
|
return 0;
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+/*
|
|
|
|
+ * lock the group_info alloc_sem of all the groups
|
|
|
|
+ * belonging to the same buddy cache page. This
|
|
|
|
+ * make sure other parallel operation on the buddy
|
|
|
|
+ * cache doesn't happen whild holding the buddy cache
|
|
|
|
+ * lock
|
|
|
|
+ */
|
|
|
|
+int ext4_mb_get_buddy_cache_lock(struct super_block *sb, ext4_group_t group)
|
|
|
|
+{
|
|
|
|
+ int i;
|
|
|
|
+ int block, pnum;
|
|
|
|
+ int blocks_per_page;
|
|
|
|
+ int groups_per_page;
|
|
|
|
+ ext4_group_t first_group;
|
|
|
|
+ struct ext4_group_info *grp;
|
|
|
|
+
|
|
|
|
+ blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize;
|
|
|
|
+ /*
|
|
|
|
+ * the buddy cache inode stores the block bitmap
|
|
|
|
+ * and buddy information in consecutive blocks.
|
|
|
|
+ * So for each group we need two blocks.
|
|
|
|
+ */
|
|
|
|
+ block = group * 2;
|
|
|
|
+ pnum = block / blocks_per_page;
|
|
|
|
+ first_group = pnum * blocks_per_page / 2;
|
|
|
|
+
|
|
|
|
+ groups_per_page = blocks_per_page >> 1;
|
|
|
|
+ if (groups_per_page == 0)
|
|
|
|
+ groups_per_page = 1;
|
|
|
|
+ /* read all groups the page covers into the cache */
|
|
|
|
+ for (i = 0; i < groups_per_page; i++) {
|
|
|
|
+
|
|
|
|
+ if ((first_group + i) >= EXT4_SB(sb)->s_groups_count)
|
|
|
|
+ break;
|
|
|
|
+ grp = ext4_get_group_info(sb, first_group + i);
|
|
|
|
+ /* take all groups write allocation
|
|
|
|
+ * semaphore. This make sure there is
|
|
|
|
+ * no block allocation going on in any
|
|
|
|
+ * of that groups
|
|
|
|
+ */
|
|
|
|
+ down_write_nested(&grp->alloc_sem, i);
|
|
|
|
+ }
|
|
|
|
+ return i;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+void ext4_mb_put_buddy_cache_lock(struct super_block *sb,
|
|
|
|
+ ext4_group_t group, int locked_group)
|
|
|
|
+{
|
|
|
|
+ int i;
|
|
|
|
+ int block, pnum;
|
|
|
|
+ int blocks_per_page;
|
|
|
|
+ ext4_group_t first_group;
|
|
|
|
+ struct ext4_group_info *grp;
|
|
|
|
+
|
|
|
|
+ blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize;
|
|
|
|
+ /*
|
|
|
|
+ * the buddy cache inode stores the block bitmap
|
|
|
|
+ * and buddy information in consecutive blocks.
|
|
|
|
+ * So for each group we need two blocks.
|
|
|
|
+ */
|
|
|
|
+ block = group * 2;
|
|
|
|
+ pnum = block / blocks_per_page;
|
|
|
|
+ first_group = pnum * blocks_per_page / 2;
|
|
|
|
+ /* release locks on all the groups */
|
|
|
|
+ for (i = 0; i < locked_group; i++) {
|
|
|
|
+
|
|
|
|
+ grp = ext4_get_group_info(sb, first_group + i);
|
|
|
|
+ /* take all groups write allocation
|
|
|
|
+ * semaphore. This make sure there is
|
|
|
|
+ * no block allocation going on in any
|
|
|
|
+ * of that groups
|
|
|
|
+ */
|
|
|
|
+ up_write(&grp->alloc_sem);
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static int ext4_mb_init_group(struct super_block *sb, ext4_group_t group)
|
|
|
|
+{
|
|
|
|
+
|
|
|
|
+ int ret;
|
|
|
|
+ void *bitmap;
|
|
|
|
+ int blocks_per_page;
|
|
|
|
+ int block, pnum, poff;
|
|
|
|
+ int num_grp_locked = 0;
|
|
|
|
+ struct ext4_group_info *this_grp;
|
|
|
|
+ struct ext4_sb_info *sbi = EXT4_SB(sb);
|
|
|
|
+ struct inode *inode = sbi->s_buddy_cache;
|
|
|
|
+ struct page *page = NULL, *bitmap_page = NULL;
|
|
|
|
+
|
|
|
|
+ mb_debug("init group %lu\n", group);
|
|
|
|
+ blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize;
|
|
|
|
+ this_grp = ext4_get_group_info(sb, group);
|
|
|
|
+ /*
|
|
|
|
+ * This ensures we don't add group
|
|
|
|
+ * to this buddy cache via resize
|
|
|
|
+ */
|
|
|
|
+ num_grp_locked = ext4_mb_get_buddy_cache_lock(sb, group);
|
|
|
|
+ if (!EXT4_MB_GRP_NEED_INIT(this_grp)) {
|
|
|
|
+ /*
|
|
|
|
+ * somebody initialized the group
|
|
|
|
+ * return without doing anything
|
|
|
|
+ */
|
|
|
|
+ ret = 0;
|
|
|
|
+ goto err;
|
|
|
|
+ }
|
|
|
|
+ /*
|
|
|
|
+ * the buddy cache inode stores the block bitmap
|
|
|
|
+ * and buddy information in consecutive blocks.
|
|
|
|
+ * So for each group we need two blocks.
|
|
|
|
+ */
|
|
|
|
+ block = group * 2;
|
|
|
|
+ pnum = block / blocks_per_page;
|
|
|
|
+ poff = block % blocks_per_page;
|
|
|
|
+ page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
|
|
|
|
+ if (page) {
|
|
|
|
+ BUG_ON(page->mapping != inode->i_mapping);
|
|
|
|
+ ret = ext4_mb_init_cache(page, NULL);
|
|
|
|
+ if (ret) {
|
|
|
|
+ unlock_page(page);
|
|
|
|
+ goto err;
|
|
|
|
+ }
|
|
|
|
+ unlock_page(page);
|
|
|
|
+ }
|
|
|
|
+ if (page == NULL || !PageUptodate(page)) {
|
|
|
|
+ ret = -EIO;
|
|
|
|
+ goto err;
|
|
|
|
+ }
|
|
|
|
+ mark_page_accessed(page);
|
|
|
|
+ bitmap_page = page;
|
|
|
|
+ bitmap = page_address(page) + (poff * sb->s_blocksize);
|
|
|
|
+
|
|
|
|
+ /* init buddy cache */
|
|
|
|
+ block++;
|
|
|
|
+ pnum = block / blocks_per_page;
|
|
|
|
+ poff = block % blocks_per_page;
|
|
|
|
+ page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
|
|
|
|
+ if (page == bitmap_page) {
|
|
|
|
+ /*
|
|
|
|
+ * If both the bitmap and buddy are in
|
|
|
|
+ * the same page we don't need to force
|
|
|
|
+ * init the buddy
|
|
|
|
+ */
|
|
|
|
+ unlock_page(page);
|
|
|
|
+ } else if (page) {
|
|
|
|
+ BUG_ON(page->mapping != inode->i_mapping);
|
|
|
|
+ ret = ext4_mb_init_cache(page, bitmap);
|
|
|
|
+ if (ret) {
|
|
|
|
+ unlock_page(page);
|
|
|
|
+ goto err;
|
|
|
|
+ }
|
|
|
|
+ unlock_page(page);
|
|
|
|
+ }
|
|
|
|
+ if (page == NULL || !PageUptodate(page)) {
|
|
|
|
+ ret = -EIO;
|
|
|
|
+ goto err;
|
|
|
|
+ }
|
|
|
|
+ mark_page_accessed(page);
|
|
|
|
+err:
|
|
|
|
+ ext4_mb_put_buddy_cache_lock(sb, group, num_grp_locked);
|
|
|
|
+ if (bitmap_page)
|
|
|
|
+ page_cache_release(bitmap_page);
|
|
|
|
+ if (page)
|
|
|
|
+ page_cache_release(page);
|
|
|
|
+ return ret;
|
|
|
|
+}
|
|
|
|
+
|
|
static noinline_for_stack int
|
|
static noinline_for_stack int
|
|
ext4_mb_regular_allocator(struct ext4_allocation_context *ac)
|
|
ext4_mb_regular_allocator(struct ext4_allocation_context *ac)
|
|
{
|
|
{
|
|
@@ -1775,7 +2019,7 @@ repeat:
|
|
group = 0;
|
|
group = 0;
|
|
|
|
|
|
/* quick check to skip empty groups */
|
|
/* quick check to skip empty groups */
|
|
- grp = ext4_get_group_info(ac->ac_sb, group);
|
|
|
|
|
|
+ grp = ext4_get_group_info(sb, group);
|
|
if (grp->bb_free == 0)
|
|
if (grp->bb_free == 0)
|
|
continue;
|
|
continue;
|
|
|
|
|
|
@@ -1788,10 +2032,9 @@ repeat:
|
|
* we need full data about the group
|
|
* we need full data about the group
|
|
* to make a good selection
|
|
* to make a good selection
|
|
*/
|
|
*/
|
|
- err = ext4_mb_load_buddy(sb, group, &e4b);
|
|
|
|
|
|
+ err = ext4_mb_init_group(sb, group);
|
|
if (err)
|
|
if (err)
|
|
goto out;
|
|
goto out;
|
|
- ext4_mb_release_desc(&e4b);
|
|
|
|
}
|
|
}
|
|
|
|
|
|
/*
|
|
/*
|
|
@@ -1932,13 +2175,13 @@ static int ext4_mb_seq_history_show(struct seq_file *seq, void *v)
|
|
if (hs->op == EXT4_MB_HISTORY_ALLOC) {
|
|
if (hs->op == EXT4_MB_HISTORY_ALLOC) {
|
|
fmt = "%-5u %-8u %-23s %-23s %-23s %-5u %-5u %-2u "
|
|
fmt = "%-5u %-8u %-23s %-23s %-23s %-5u %-5u %-2u "
|
|
"%-5u %-5s %-5u %-6u\n";
|
|
"%-5u %-5s %-5u %-6u\n";
|
|
- sprintf(buf2, "%lu/%d/%u@%u", hs->result.fe_group,
|
|
|
|
|
|
+ sprintf(buf2, "%u/%d/%u@%u", hs->result.fe_group,
|
|
hs->result.fe_start, hs->result.fe_len,
|
|
hs->result.fe_start, hs->result.fe_len,
|
|
hs->result.fe_logical);
|
|
hs->result.fe_logical);
|
|
- sprintf(buf, "%lu/%d/%u@%u", hs->orig.fe_group,
|
|
|
|
|
|
+ sprintf(buf, "%u/%d/%u@%u", hs->orig.fe_group,
|
|
hs->orig.fe_start, hs->orig.fe_len,
|
|
hs->orig.fe_start, hs->orig.fe_len,
|
|
hs->orig.fe_logical);
|
|
hs->orig.fe_logical);
|
|
- sprintf(buf3, "%lu/%d/%u@%u", hs->goal.fe_group,
|
|
|
|
|
|
+ sprintf(buf3, "%u/%d/%u@%u", hs->goal.fe_group,
|
|
hs->goal.fe_start, hs->goal.fe_len,
|
|
hs->goal.fe_start, hs->goal.fe_len,
|
|
hs->goal.fe_logical);
|
|
hs->goal.fe_logical);
|
|
seq_printf(seq, fmt, hs->pid, hs->ino, buf, buf3, buf2,
|
|
seq_printf(seq, fmt, hs->pid, hs->ino, buf, buf3, buf2,
|
|
@@ -1947,20 +2190,20 @@ static int ext4_mb_seq_history_show(struct seq_file *seq, void *v)
|
|
hs->buddy ? 1 << hs->buddy : 0);
|
|
hs->buddy ? 1 << hs->buddy : 0);
|
|
} else if (hs->op == EXT4_MB_HISTORY_PREALLOC) {
|
|
} else if (hs->op == EXT4_MB_HISTORY_PREALLOC) {
|
|
fmt = "%-5u %-8u %-23s %-23s %-23s\n";
|
|
fmt = "%-5u %-8u %-23s %-23s %-23s\n";
|
|
- sprintf(buf2, "%lu/%d/%u@%u", hs->result.fe_group,
|
|
|
|
|
|
+ sprintf(buf2, "%u/%d/%u@%u", hs->result.fe_group,
|
|
hs->result.fe_start, hs->result.fe_len,
|
|
hs->result.fe_start, hs->result.fe_len,
|
|
hs->result.fe_logical);
|
|
hs->result.fe_logical);
|
|
- sprintf(buf, "%lu/%d/%u@%u", hs->orig.fe_group,
|
|
|
|
|
|
+ sprintf(buf, "%u/%d/%u@%u", hs->orig.fe_group,
|
|
hs->orig.fe_start, hs->orig.fe_len,
|
|
hs->orig.fe_start, hs->orig.fe_len,
|
|
hs->orig.fe_logical);
|
|
hs->orig.fe_logical);
|
|
seq_printf(seq, fmt, hs->pid, hs->ino, buf, "", buf2);
|
|
seq_printf(seq, fmt, hs->pid, hs->ino, buf, "", buf2);
|
|
} else if (hs->op == EXT4_MB_HISTORY_DISCARD) {
|
|
} else if (hs->op == EXT4_MB_HISTORY_DISCARD) {
|
|
- sprintf(buf2, "%lu/%d/%u", hs->result.fe_group,
|
|
|
|
|
|
+ sprintf(buf2, "%u/%d/%u", hs->result.fe_group,
|
|
hs->result.fe_start, hs->result.fe_len);
|
|
hs->result.fe_start, hs->result.fe_len);
|
|
seq_printf(seq, "%-5u %-8u %-23s discard\n",
|
|
seq_printf(seq, "%-5u %-8u %-23s discard\n",
|
|
hs->pid, hs->ino, buf2);
|
|
hs->pid, hs->ino, buf2);
|
|
} else if (hs->op == EXT4_MB_HISTORY_FREE) {
|
|
} else if (hs->op == EXT4_MB_HISTORY_FREE) {
|
|
- sprintf(buf2, "%lu/%d/%u", hs->result.fe_group,
|
|
|
|
|
|
+ sprintf(buf2, "%u/%d/%u", hs->result.fe_group,
|
|
hs->result.fe_start, hs->result.fe_len);
|
|
hs->result.fe_start, hs->result.fe_len);
|
|
seq_printf(seq, "%-5u %-8u %-23s free\n",
|
|
seq_printf(seq, "%-5u %-8u %-23s free\n",
|
|
hs->pid, hs->ino, buf2);
|
|
hs->pid, hs->ino, buf2);
|
|
@@ -2073,7 +2316,7 @@ static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos)
|
|
return NULL;
|
|
return NULL;
|
|
|
|
|
|
group = *pos + 1;
|
|
group = *pos + 1;
|
|
- return (void *) group;
|
|
|
|
|
|
+ return (void *) ((unsigned long) group);
|
|
}
|
|
}
|
|
|
|
|
|
static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos)
|
|
static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos)
|
|
@@ -2086,13 +2329,13 @@ static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos)
|
|
if (*pos < 0 || *pos >= sbi->s_groups_count)
|
|
if (*pos < 0 || *pos >= sbi->s_groups_count)
|
|
return NULL;
|
|
return NULL;
|
|
group = *pos + 1;
|
|
group = *pos + 1;
|
|
- return (void *) group;;
|
|
|
|
|
|
+ return (void *) ((unsigned long) group);
|
|
}
|
|
}
|
|
|
|
|
|
static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v)
|
|
static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v)
|
|
{
|
|
{
|
|
struct super_block *sb = seq->private;
|
|
struct super_block *sb = seq->private;
|
|
- long group = (long) v;
|
|
|
|
|
|
+ ext4_group_t group = (ext4_group_t) ((unsigned long) v);
|
|
int i;
|
|
int i;
|
|
int err;
|
|
int err;
|
|
struct ext4_buddy e4b;
|
|
struct ext4_buddy e4b;
|
|
@@ -2114,7 +2357,7 @@ static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v)
|
|
sizeof(struct ext4_group_info);
|
|
sizeof(struct ext4_group_info);
|
|
err = ext4_mb_load_buddy(sb, group, &e4b);
|
|
err = ext4_mb_load_buddy(sb, group, &e4b);
|
|
if (err) {
|
|
if (err) {
|
|
- seq_printf(seq, "#%-5lu: I/O error\n", group);
|
|
|
|
|
|
+ seq_printf(seq, "#%-5u: I/O error\n", group);
|
|
return 0;
|
|
return 0;
|
|
}
|
|
}
|
|
ext4_lock_group(sb, group);
|
|
ext4_lock_group(sb, group);
|
|
@@ -2122,7 +2365,7 @@ static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v)
|
|
ext4_unlock_group(sb, group);
|
|
ext4_unlock_group(sb, group);
|
|
ext4_mb_release_desc(&e4b);
|
|
ext4_mb_release_desc(&e4b);
|
|
|
|
|
|
- seq_printf(seq, "#%-5lu: %-5u %-5u %-5u [", group, sg.info.bb_free,
|
|
|
|
|
|
+ seq_printf(seq, "#%-5u: %-5u %-5u %-5u [", group, sg.info.bb_free,
|
|
sg.info.bb_fragments, sg.info.bb_first_free);
|
|
sg.info.bb_fragments, sg.info.bb_first_free);
|
|
for (i = 0; i <= 13; i++)
|
|
for (i = 0; i <= 13; i++)
|
|
seq_printf(seq, " %-5u", i <= sb->s_blocksize_bits + 1 ?
|
|
seq_printf(seq, " %-5u", i <= sb->s_blocksize_bits + 1 ?
|
|
@@ -2296,10 +2539,11 @@ int ext4_mb_add_groupinfo(struct super_block *sb, ext4_group_t group,
|
|
ext4_free_blocks_after_init(sb, group, desc);
|
|
ext4_free_blocks_after_init(sb, group, desc);
|
|
} else {
|
|
} else {
|
|
meta_group_info[i]->bb_free =
|
|
meta_group_info[i]->bb_free =
|
|
- le16_to_cpu(desc->bg_free_blocks_count);
|
|
|
|
|
|
+ ext4_free_blks_count(sb, desc);
|
|
}
|
|
}
|
|
|
|
|
|
INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list);
|
|
INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list);
|
|
|
|
+ init_rwsem(&meta_group_info[i]->alloc_sem);
|
|
meta_group_info[i]->bb_free_root.rb_node = NULL;;
|
|
meta_group_info[i]->bb_free_root.rb_node = NULL;;
|
|
|
|
|
|
#ifdef DOUBLE_CHECK
|
|
#ifdef DOUBLE_CHECK
|
|
@@ -2326,54 +2570,6 @@ exit_meta_group_info:
|
|
return -ENOMEM;
|
|
return -ENOMEM;
|
|
} /* ext4_mb_add_groupinfo */
|
|
} /* ext4_mb_add_groupinfo */
|
|
|
|
|
|
-/*
|
|
|
|
- * Add a group to the existing groups.
|
|
|
|
- * This function is used for online resize
|
|
|
|
- */
|
|
|
|
-int ext4_mb_add_more_groupinfo(struct super_block *sb, ext4_group_t group,
|
|
|
|
- struct ext4_group_desc *desc)
|
|
|
|
-{
|
|
|
|
- struct ext4_sb_info *sbi = EXT4_SB(sb);
|
|
|
|
- struct inode *inode = sbi->s_buddy_cache;
|
|
|
|
- int blocks_per_page;
|
|
|
|
- int block;
|
|
|
|
- int pnum;
|
|
|
|
- struct page *page;
|
|
|
|
- int err;
|
|
|
|
-
|
|
|
|
- /* Add group based on group descriptor*/
|
|
|
|
- err = ext4_mb_add_groupinfo(sb, group, desc);
|
|
|
|
- if (err)
|
|
|
|
- return err;
|
|
|
|
-
|
|
|
|
- /*
|
|
|
|
- * Cache pages containing dynamic mb_alloc datas (buddy and bitmap
|
|
|
|
- * datas) are set not up to date so that they will be re-initilaized
|
|
|
|
- * during the next call to ext4_mb_load_buddy
|
|
|
|
- */
|
|
|
|
-
|
|
|
|
- /* Set buddy page as not up to date */
|
|
|
|
- blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize;
|
|
|
|
- block = group * 2;
|
|
|
|
- pnum = block / blocks_per_page;
|
|
|
|
- page = find_get_page(inode->i_mapping, pnum);
|
|
|
|
- if (page != NULL) {
|
|
|
|
- ClearPageUptodate(page);
|
|
|
|
- page_cache_release(page);
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- /* Set bitmap page as not up to date */
|
|
|
|
- block++;
|
|
|
|
- pnum = block / blocks_per_page;
|
|
|
|
- page = find_get_page(inode->i_mapping, pnum);
|
|
|
|
- if (page != NULL) {
|
|
|
|
- ClearPageUptodate(page);
|
|
|
|
- page_cache_release(page);
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- return 0;
|
|
|
|
-}
|
|
|
|
-
|
|
|
|
/*
|
|
/*
|
|
* Update an existing group.
|
|
* Update an existing group.
|
|
* This function is used for online resize
|
|
* This function is used for online resize
|
|
@@ -2457,7 +2653,7 @@ static int ext4_mb_init_backend(struct super_block *sb)
|
|
desc = ext4_get_group_desc(sb, i, NULL);
|
|
desc = ext4_get_group_desc(sb, i, NULL);
|
|
if (desc == NULL) {
|
|
if (desc == NULL) {
|
|
printk(KERN_ERR
|
|
printk(KERN_ERR
|
|
- "EXT4-fs: can't read descriptor %lu\n", i);
|
|
|
|
|
|
+ "EXT4-fs: can't read descriptor %u\n", i);
|
|
goto err_freebuddy;
|
|
goto err_freebuddy;
|
|
}
|
|
}
|
|
if (ext4_mb_add_groupinfo(sb, i, desc) != 0)
|
|
if (ext4_mb_add_groupinfo(sb, i, desc) != 0)
|
|
@@ -2493,6 +2689,8 @@ int ext4_mb_init(struct super_block *sb, int needs_recovery)
|
|
if (sbi->s_mb_offsets == NULL) {
|
|
if (sbi->s_mb_offsets == NULL) {
|
|
return -ENOMEM;
|
|
return -ENOMEM;
|
|
}
|
|
}
|
|
|
|
+
|
|
|
|
+ i = (sb->s_blocksize_bits + 2) * sizeof(unsigned int);
|
|
sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL);
|
|
sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL);
|
|
if (sbi->s_mb_maxs == NULL) {
|
|
if (sbi->s_mb_maxs == NULL) {
|
|
kfree(sbi->s_mb_maxs);
|
|
kfree(sbi->s_mb_maxs);
|
|
@@ -2551,7 +2749,8 @@ int ext4_mb_init(struct super_block *sb, int needs_recovery)
|
|
ext4_mb_init_per_dev_proc(sb);
|
|
ext4_mb_init_per_dev_proc(sb);
|
|
ext4_mb_history_init(sb);
|
|
ext4_mb_history_init(sb);
|
|
|
|
|
|
- sbi->s_journal->j_commit_callback = release_blocks_on_commit;
|
|
|
|
|
|
+ if (sbi->s_journal)
|
|
|
|
+ sbi->s_journal->j_commit_callback = release_blocks_on_commit;
|
|
|
|
|
|
printk(KERN_INFO "EXT4-fs: mballoc enabled\n");
|
|
printk(KERN_INFO "EXT4-fs: mballoc enabled\n");
|
|
return 0;
|
|
return 0;
|
|
@@ -2652,7 +2851,7 @@ static void release_blocks_on_commit(journal_t *journal, transaction_t *txn)
|
|
list_for_each_safe(l, ltmp, &txn->t_private_list) {
|
|
list_for_each_safe(l, ltmp, &txn->t_private_list) {
|
|
entry = list_entry(l, struct ext4_free_data, list);
|
|
entry = list_entry(l, struct ext4_free_data, list);
|
|
|
|
|
|
- mb_debug("gonna free %u blocks in group %lu (0x%p):",
|
|
|
|
|
|
+ mb_debug("gonna free %u blocks in group %u (0x%p):",
|
|
entry->count, entry->group, entry);
|
|
entry->count, entry->group, entry);
|
|
|
|
|
|
err = ext4_mb_load_buddy(sb, entry->group, &e4b);
|
|
err = ext4_mb_load_buddy(sb, entry->group, &e4b);
|
|
@@ -2679,8 +2878,9 @@ static void release_blocks_on_commit(journal_t *journal, transaction_t *txn)
|
|
discard_block = (ext4_fsblk_t) entry->group * EXT4_BLOCKS_PER_GROUP(sb)
|
|
discard_block = (ext4_fsblk_t) entry->group * EXT4_BLOCKS_PER_GROUP(sb)
|
|
+ entry->start_blk
|
|
+ entry->start_blk
|
|
+ le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block);
|
|
+ le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block);
|
|
- trace_mark(ext4_discard_blocks, "dev %s blk %llu count %u", sb->s_id,
|
|
|
|
- (unsigned long long) discard_block, entry->count);
|
|
|
|
|
|
+ trace_mark(ext4_discard_blocks, "dev %s blk %llu count %u",
|
|
|
|
+ sb->s_id, (unsigned long long) discard_block,
|
|
|
|
+ entry->count);
|
|
sb_issue_discard(sb, discard_block, entry->count);
|
|
sb_issue_discard(sb, discard_block, entry->count);
|
|
|
|
|
|
kmem_cache_free(ext4_free_ext_cachep, entry);
|
|
kmem_cache_free(ext4_free_ext_cachep, entry);
|
|
@@ -2791,7 +2991,7 @@ void exit_ext4_mballoc(void)
|
|
*/
|
|
*/
|
|
static noinline_for_stack int
|
|
static noinline_for_stack int
|
|
ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac,
|
|
ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac,
|
|
- handle_t *handle, unsigned long reserv_blks)
|
|
|
|
|
|
+ handle_t *handle, unsigned int reserv_blks)
|
|
{
|
|
{
|
|
struct buffer_head *bitmap_bh = NULL;
|
|
struct buffer_head *bitmap_bh = NULL;
|
|
struct ext4_super_block *es;
|
|
struct ext4_super_block *es;
|
|
@@ -2824,7 +3024,7 @@ ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac,
|
|
if (!gdp)
|
|
if (!gdp)
|
|
goto out_err;
|
|
goto out_err;
|
|
|
|
|
|
- ext4_debug("using block group %lu(%d)\n", ac->ac_b_ex.fe_group,
|
|
|
|
|
|
+ ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group,
|
|
gdp->bg_free_blocks_count);
|
|
gdp->bg_free_blocks_count);
|
|
|
|
|
|
err = ext4_journal_get_write_access(handle, gdp_bh);
|
|
err = ext4_journal_get_write_access(handle, gdp_bh);
|
|
@@ -2843,8 +3043,8 @@ ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac,
|
|
in_range(block + len - 1, ext4_inode_table(sb, gdp),
|
|
in_range(block + len - 1, ext4_inode_table(sb, gdp),
|
|
EXT4_SB(sb)->s_itb_per_group)) {
|
|
EXT4_SB(sb)->s_itb_per_group)) {
|
|
ext4_error(sb, __func__,
|
|
ext4_error(sb, __func__,
|
|
- "Allocating block in system zone - block = %llu",
|
|
|
|
- block);
|
|
|
|
|
|
+ "Allocating block %llu in system zone of %d group\n",
|
|
|
|
+ block, ac->ac_b_ex.fe_group);
|
|
/* File system mounted not to panic on error
|
|
/* File system mounted not to panic on error
|
|
* Fix the bitmap and repeat the block allocation
|
|
* Fix the bitmap and repeat the block allocation
|
|
* We leak some of the blocks here.
|
|
* We leak some of the blocks here.
|
|
@@ -2852,7 +3052,7 @@ ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac,
|
|
mb_set_bits(sb_bgl_lock(sbi, ac->ac_b_ex.fe_group),
|
|
mb_set_bits(sb_bgl_lock(sbi, ac->ac_b_ex.fe_group),
|
|
bitmap_bh->b_data, ac->ac_b_ex.fe_start,
|
|
bitmap_bh->b_data, ac->ac_b_ex.fe_start,
|
|
ac->ac_b_ex.fe_len);
|
|
ac->ac_b_ex.fe_len);
|
|
- err = ext4_journal_dirty_metadata(handle, bitmap_bh);
|
|
|
|
|
|
+ err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh);
|
|
if (!err)
|
|
if (!err)
|
|
err = -EAGAIN;
|
|
err = -EAGAIN;
|
|
goto out_err;
|
|
goto out_err;
|
|
@@ -2866,18 +3066,17 @@ ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
#endif
|
|
#endif
|
|
- mb_set_bits(sb_bgl_lock(sbi, ac->ac_b_ex.fe_group), bitmap_bh->b_data,
|
|
|
|
- ac->ac_b_ex.fe_start, ac->ac_b_ex.fe_len);
|
|
|
|
-
|
|
|
|
spin_lock(sb_bgl_lock(sbi, ac->ac_b_ex.fe_group));
|
|
spin_lock(sb_bgl_lock(sbi, ac->ac_b_ex.fe_group));
|
|
|
|
+ mb_set_bits(NULL, bitmap_bh->b_data,
|
|
|
|
+ ac->ac_b_ex.fe_start, ac->ac_b_ex.fe_len);
|
|
if (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) {
|
|
if (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) {
|
|
gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT);
|
|
gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT);
|
|
- gdp->bg_free_blocks_count =
|
|
|
|
- cpu_to_le16(ext4_free_blocks_after_init(sb,
|
|
|
|
- ac->ac_b_ex.fe_group,
|
|
|
|
- gdp));
|
|
|
|
|
|
+ ext4_free_blks_set(sb, gdp,
|
|
|
|
+ ext4_free_blocks_after_init(sb,
|
|
|
|
+ ac->ac_b_ex.fe_group, gdp));
|
|
}
|
|
}
|
|
- le16_add_cpu(&gdp->bg_free_blocks_count, -ac->ac_b_ex.fe_len);
|
|
|
|
|
|
+ len = ext4_free_blks_count(sb, gdp) - ac->ac_b_ex.fe_len;
|
|
|
|
+ ext4_free_blks_set(sb, gdp, len);
|
|
gdp->bg_checksum = ext4_group_desc_csum(sbi, ac->ac_b_ex.fe_group, gdp);
|
|
gdp->bg_checksum = ext4_group_desc_csum(sbi, ac->ac_b_ex.fe_group, gdp);
|
|
spin_unlock(sb_bgl_lock(sbi, ac->ac_b_ex.fe_group));
|
|
spin_unlock(sb_bgl_lock(sbi, ac->ac_b_ex.fe_group));
|
|
percpu_counter_sub(&sbi->s_freeblocks_counter, ac->ac_b_ex.fe_len);
|
|
percpu_counter_sub(&sbi->s_freeblocks_counter, ac->ac_b_ex.fe_len);
|
|
@@ -2899,10 +3098,10 @@ ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac,
|
|
spin_unlock(sb_bgl_lock(sbi, flex_group));
|
|
spin_unlock(sb_bgl_lock(sbi, flex_group));
|
|
}
|
|
}
|
|
|
|
|
|
- err = ext4_journal_dirty_metadata(handle, bitmap_bh);
|
|
|
|
|
|
+ err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh);
|
|
if (err)
|
|
if (err)
|
|
goto out_err;
|
|
goto out_err;
|
|
- err = ext4_journal_dirty_metadata(handle, gdp_bh);
|
|
|
|
|
|
+ err = ext4_handle_dirty_metadata(handle, NULL, gdp_bh);
|
|
|
|
|
|
out_err:
|
|
out_err:
|
|
sb->s_dirt = 1;
|
|
sb->s_dirt = 1;
|
|
@@ -3031,7 +3230,7 @@ ext4_mb_normalize_request(struct ext4_allocation_context *ac,
|
|
/* check we don't cross already preallocated blocks */
|
|
/* check we don't cross already preallocated blocks */
|
|
rcu_read_lock();
|
|
rcu_read_lock();
|
|
list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) {
|
|
list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) {
|
|
- unsigned long pa_end;
|
|
|
|
|
|
+ ext4_lblk_t pa_end;
|
|
|
|
|
|
if (pa->pa_deleted)
|
|
if (pa->pa_deleted)
|
|
continue;
|
|
continue;
|
|
@@ -3075,7 +3274,7 @@ ext4_mb_normalize_request(struct ext4_allocation_context *ac,
|
|
/* XXX: extra loop to check we really don't overlap preallocations */
|
|
/* XXX: extra loop to check we really don't overlap preallocations */
|
|
rcu_read_lock();
|
|
rcu_read_lock();
|
|
list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) {
|
|
list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) {
|
|
- unsigned long pa_end;
|
|
|
|
|
|
+ ext4_lblk_t pa_end;
|
|
spin_lock(&pa->pa_lock);
|
|
spin_lock(&pa->pa_lock);
|
|
if (pa->pa_deleted == 0) {
|
|
if (pa->pa_deleted == 0) {
|
|
pa_end = pa->pa_lstart + pa->pa_len;
|
|
pa_end = pa->pa_lstart + pa->pa_len;
|
|
@@ -3306,6 +3505,32 @@ ext4_mb_use_preallocated(struct ext4_allocation_context *ac)
|
|
return 0;
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+/*
|
|
|
|
+ * the function goes through all block freed in the group
|
|
|
|
+ * but not yet committed and marks them used in in-core bitmap.
|
|
|
|
+ * buddy must be generated from this bitmap
|
|
|
|
+ * Need to be called with ext4 group lock (ext4_lock_group)
|
|
|
|
+ */
|
|
|
|
+static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap,
|
|
|
|
+ ext4_group_t group)
|
|
|
|
+{
|
|
|
|
+ struct rb_node *n;
|
|
|
|
+ struct ext4_group_info *grp;
|
|
|
|
+ struct ext4_free_data *entry;
|
|
|
|
+
|
|
|
|
+ grp = ext4_get_group_info(sb, group);
|
|
|
|
+ n = rb_first(&(grp->bb_free_root));
|
|
|
|
+
|
|
|
|
+ while (n) {
|
|
|
|
+ entry = rb_entry(n, struct ext4_free_data, node);
|
|
|
|
+ mb_set_bits(sb_bgl_lock(EXT4_SB(sb), group),
|
|
|
|
+ bitmap, entry->start_blk,
|
|
|
|
+ entry->count);
|
|
|
|
+ n = rb_next(n);
|
|
|
|
+ }
|
|
|
|
+ return;
|
|
|
|
+}
|
|
|
|
+
|
|
/*
|
|
/*
|
|
* the function goes through all preallocation in this group and marks them
|
|
* the function goes through all preallocation in this group and marks them
|
|
* used in in-core bitmap. buddy must be generated from this bitmap
|
|
* used in in-core bitmap. buddy must be generated from this bitmap
|
|
@@ -3346,7 +3571,7 @@ static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap,
|
|
preallocated += len;
|
|
preallocated += len;
|
|
count++;
|
|
count++;
|
|
}
|
|
}
|
|
- mb_debug("prellocated %u for group %lu\n", preallocated, group);
|
|
|
|
|
|
+ mb_debug("prellocated %u for group %u\n", preallocated, group);
|
|
}
|
|
}
|
|
|
|
|
|
static void ext4_mb_pa_callback(struct rcu_head *head)
|
|
static void ext4_mb_pa_callback(struct rcu_head *head)
|
|
@@ -3363,7 +3588,7 @@ static void ext4_mb_pa_callback(struct rcu_head *head)
|
|
static void ext4_mb_put_pa(struct ext4_allocation_context *ac,
|
|
static void ext4_mb_put_pa(struct ext4_allocation_context *ac,
|
|
struct super_block *sb, struct ext4_prealloc_space *pa)
|
|
struct super_block *sb, struct ext4_prealloc_space *pa)
|
|
{
|
|
{
|
|
- unsigned long grp;
|
|
|
|
|
|
+ ext4_group_t grp;
|
|
|
|
|
|
if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0)
|
|
if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0)
|
|
return;
|
|
return;
|
|
@@ -3473,6 +3698,10 @@ ext4_mb_new_inode_pa(struct ext4_allocation_context *ac)
|
|
|
|
|
|
mb_debug("new inode pa %p: %llu/%u for %u\n", pa,
|
|
mb_debug("new inode pa %p: %llu/%u for %u\n", pa,
|
|
pa->pa_pstart, pa->pa_len, pa->pa_lstart);
|
|
pa->pa_pstart, pa->pa_len, pa->pa_lstart);
|
|
|
|
+ trace_mark(ext4_mb_new_inode_pa,
|
|
|
|
+ "dev %s ino %lu pstart %llu len %u lstart %u",
|
|
|
|
+ sb->s_id, ac->ac_inode->i_ino,
|
|
|
|
+ pa->pa_pstart, pa->pa_len, pa->pa_lstart);
|
|
|
|
|
|
ext4_mb_use_inode_pa(ac, pa);
|
|
ext4_mb_use_inode_pa(ac, pa);
|
|
atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated);
|
|
atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated);
|
|
@@ -3530,7 +3759,9 @@ ext4_mb_new_group_pa(struct ext4_allocation_context *ac)
|
|
pa->pa_linear = 1;
|
|
pa->pa_linear = 1;
|
|
|
|
|
|
mb_debug("new group pa %p: %llu/%u for %u\n", pa,
|
|
mb_debug("new group pa %p: %llu/%u for %u\n", pa,
|
|
- pa->pa_pstart, pa->pa_len, pa->pa_lstart);
|
|
|
|
|
|
+ pa->pa_pstart, pa->pa_len, pa->pa_lstart);
|
|
|
|
+ trace_mark(ext4_mb_new_group_pa, "dev %s pstart %llu len %u lstart %u",
|
|
|
|
+ sb->s_id, pa->pa_pstart, pa->pa_len, pa->pa_lstart);
|
|
|
|
|
|
ext4_mb_use_group_pa(ac, pa);
|
|
ext4_mb_use_group_pa(ac, pa);
|
|
atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated);
|
|
atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated);
|
|
@@ -3579,16 +3810,18 @@ ext4_mb_release_inode_pa(struct ext4_buddy *e4b, struct buffer_head *bitmap_bh,
|
|
{
|
|
{
|
|
struct super_block *sb = e4b->bd_sb;
|
|
struct super_block *sb = e4b->bd_sb;
|
|
struct ext4_sb_info *sbi = EXT4_SB(sb);
|
|
struct ext4_sb_info *sbi = EXT4_SB(sb);
|
|
- unsigned long end;
|
|
|
|
- unsigned long next;
|
|
|
|
|
|
+ unsigned int end;
|
|
|
|
+ unsigned int next;
|
|
ext4_group_t group;
|
|
ext4_group_t group;
|
|
ext4_grpblk_t bit;
|
|
ext4_grpblk_t bit;
|
|
|
|
+ unsigned long long grp_blk_start;
|
|
sector_t start;
|
|
sector_t start;
|
|
int err = 0;
|
|
int err = 0;
|
|
int free = 0;
|
|
int free = 0;
|
|
|
|
|
|
BUG_ON(pa->pa_deleted == 0);
|
|
BUG_ON(pa->pa_deleted == 0);
|
|
ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);
|
|
ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);
|
|
|
|
+ grp_blk_start = pa->pa_pstart - bit;
|
|
BUG_ON(group != e4b->bd_group && pa->pa_len != 0);
|
|
BUG_ON(group != e4b->bd_group && pa->pa_len != 0);
|
|
end = bit + pa->pa_len;
|
|
end = bit + pa->pa_len;
|
|
|
|
|
|
@@ -3618,6 +3851,10 @@ ext4_mb_release_inode_pa(struct ext4_buddy *e4b, struct buffer_head *bitmap_bh,
|
|
ext4_mb_store_history(ac);
|
|
ext4_mb_store_history(ac);
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+ trace_mark(ext4_mb_release_inode_pa,
|
|
|
|
+ "dev %s ino %lu block %llu count %u",
|
|
|
|
+ sb->s_id, pa->pa_inode->i_ino, grp_blk_start + bit,
|
|
|
|
+ next - bit);
|
|
mb_free_blocks(pa->pa_inode, e4b, bit, next - bit);
|
|
mb_free_blocks(pa->pa_inode, e4b, bit, next - bit);
|
|
bit = next + 1;
|
|
bit = next + 1;
|
|
}
|
|
}
|
|
@@ -3626,8 +3863,9 @@ ext4_mb_release_inode_pa(struct ext4_buddy *e4b, struct buffer_head *bitmap_bh,
|
|
pa, (unsigned long) pa->pa_lstart,
|
|
pa, (unsigned long) pa->pa_lstart,
|
|
(unsigned long) pa->pa_pstart,
|
|
(unsigned long) pa->pa_pstart,
|
|
(unsigned long) pa->pa_len);
|
|
(unsigned long) pa->pa_len);
|
|
- ext4_error(sb, __func__, "free %u, pa_free %u\n",
|
|
|
|
- free, pa->pa_free);
|
|
|
|
|
|
+ ext4_grp_locked_error(sb, group,
|
|
|
|
+ __func__, "free %u, pa_free %u",
|
|
|
|
+ free, pa->pa_free);
|
|
/*
|
|
/*
|
|
* pa is already deleted so we use the value obtained
|
|
* pa is already deleted so we use the value obtained
|
|
* from the bitmap and continue.
|
|
* from the bitmap and continue.
|
|
@@ -3650,6 +3888,8 @@ ext4_mb_release_group_pa(struct ext4_buddy *e4b,
|
|
if (ac)
|
|
if (ac)
|
|
ac->ac_op = EXT4_MB_HISTORY_DISCARD;
|
|
ac->ac_op = EXT4_MB_HISTORY_DISCARD;
|
|
|
|
|
|
|
|
+ trace_mark(ext4_mb_release_group_pa, "dev %s pstart %llu len %d",
|
|
|
|
+ sb->s_id, pa->pa_pstart, pa->pa_len);
|
|
BUG_ON(pa->pa_deleted == 0);
|
|
BUG_ON(pa->pa_deleted == 0);
|
|
ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);
|
|
ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);
|
|
BUG_ON(group != e4b->bd_group && pa->pa_len != 0);
|
|
BUG_ON(group != e4b->bd_group && pa->pa_len != 0);
|
|
@@ -3692,7 +3932,7 @@ ext4_mb_discard_group_preallocations(struct super_block *sb,
|
|
int busy = 0;
|
|
int busy = 0;
|
|
int free = 0;
|
|
int free = 0;
|
|
|
|
|
|
- mb_debug("discard preallocation for group %lu\n", group);
|
|
|
|
|
|
+ mb_debug("discard preallocation for group %u\n", group);
|
|
|
|
|
|
if (list_empty(&grp->bb_prealloc_list))
|
|
if (list_empty(&grp->bb_prealloc_list))
|
|
return 0;
|
|
return 0;
|
|
@@ -3700,14 +3940,14 @@ ext4_mb_discard_group_preallocations(struct super_block *sb,
|
|
bitmap_bh = ext4_read_block_bitmap(sb, group);
|
|
bitmap_bh = ext4_read_block_bitmap(sb, group);
|
|
if (bitmap_bh == NULL) {
|
|
if (bitmap_bh == NULL) {
|
|
ext4_error(sb, __func__, "Error in reading block "
|
|
ext4_error(sb, __func__, "Error in reading block "
|
|
- "bitmap for %lu\n", group);
|
|
|
|
|
|
+ "bitmap for %u", group);
|
|
return 0;
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
|
|
err = ext4_mb_load_buddy(sb, group, &e4b);
|
|
err = ext4_mb_load_buddy(sb, group, &e4b);
|
|
if (err) {
|
|
if (err) {
|
|
ext4_error(sb, __func__, "Error in loading buddy "
|
|
ext4_error(sb, __func__, "Error in loading buddy "
|
|
- "information for %lu\n", group);
|
|
|
|
|
|
+ "information for %u", group);
|
|
put_bh(bitmap_bh);
|
|
put_bh(bitmap_bh);
|
|
return 0;
|
|
return 0;
|
|
}
|
|
}
|
|
@@ -3815,6 +4055,8 @@ void ext4_discard_preallocations(struct inode *inode)
|
|
}
|
|
}
|
|
|
|
|
|
mb_debug("discard preallocation for inode %lu\n", inode->i_ino);
|
|
mb_debug("discard preallocation for inode %lu\n", inode->i_ino);
|
|
|
|
+ trace_mark(ext4_discard_preallocations, "dev %s ino %lu", sb->s_id,
|
|
|
|
+ inode->i_ino);
|
|
|
|
|
|
INIT_LIST_HEAD(&list);
|
|
INIT_LIST_HEAD(&list);
|
|
|
|
|
|
@@ -3874,14 +4116,14 @@ repeat:
|
|
err = ext4_mb_load_buddy(sb, group, &e4b);
|
|
err = ext4_mb_load_buddy(sb, group, &e4b);
|
|
if (err) {
|
|
if (err) {
|
|
ext4_error(sb, __func__, "Error in loading buddy "
|
|
ext4_error(sb, __func__, "Error in loading buddy "
|
|
- "information for %lu\n", group);
|
|
|
|
|
|
+ "information for %u", group);
|
|
continue;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
|
|
bitmap_bh = ext4_read_block_bitmap(sb, group);
|
|
bitmap_bh = ext4_read_block_bitmap(sb, group);
|
|
if (bitmap_bh == NULL) {
|
|
if (bitmap_bh == NULL) {
|
|
ext4_error(sb, __func__, "Error in reading block "
|
|
ext4_error(sb, __func__, "Error in reading block "
|
|
- "bitmap for %lu\n", group);
|
|
|
|
|
|
+ "bitmap for %u", group);
|
|
ext4_mb_release_desc(&e4b);
|
|
ext4_mb_release_desc(&e4b);
|
|
continue;
|
|
continue;
|
|
}
|
|
}
|
|
@@ -4024,8 +4266,8 @@ ext4_mb_initialize_context(struct ext4_allocation_context *ac,
|
|
struct ext4_sb_info *sbi = EXT4_SB(sb);
|
|
struct ext4_sb_info *sbi = EXT4_SB(sb);
|
|
struct ext4_super_block *es = sbi->s_es;
|
|
struct ext4_super_block *es = sbi->s_es;
|
|
ext4_group_t group;
|
|
ext4_group_t group;
|
|
- unsigned long len;
|
|
|
|
- unsigned long goal;
|
|
|
|
|
|
+ unsigned int len;
|
|
|
|
+ ext4_fsblk_t goal;
|
|
ext4_grpblk_t block;
|
|
ext4_grpblk_t block;
|
|
|
|
|
|
/* we can't allocate > group size */
|
|
/* we can't allocate > group size */
|
|
@@ -4068,6 +4310,7 @@ ext4_mb_initialize_context(struct ext4_allocation_context *ac,
|
|
ac->ac_pa = NULL;
|
|
ac->ac_pa = NULL;
|
|
ac->ac_bitmap_page = NULL;
|
|
ac->ac_bitmap_page = NULL;
|
|
ac->ac_buddy_page = NULL;
|
|
ac->ac_buddy_page = NULL;
|
|
|
|
+ ac->alloc_semp = NULL;
|
|
ac->ac_lg = NULL;
|
|
ac->ac_lg = NULL;
|
|
|
|
|
|
/* we have to define context: we'll we work with a file or
|
|
/* we have to define context: we'll we work with a file or
|
|
@@ -4146,7 +4389,7 @@ ext4_mb_discard_lg_preallocations(struct super_block *sb,
|
|
ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, NULL);
|
|
ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, NULL);
|
|
if (ext4_mb_load_buddy(sb, group, &e4b)) {
|
|
if (ext4_mb_load_buddy(sb, group, &e4b)) {
|
|
ext4_error(sb, __func__, "Error in loading buddy "
|
|
ext4_error(sb, __func__, "Error in loading buddy "
|
|
- "information for %lu\n", group);
|
|
|
|
|
|
+ "information for %u", group);
|
|
continue;
|
|
continue;
|
|
}
|
|
}
|
|
ext4_lock_group(sb, group);
|
|
ext4_lock_group(sb, group);
|
|
@@ -4248,6 +4491,8 @@ static int ext4_mb_release_context(struct ext4_allocation_context *ac)
|
|
}
|
|
}
|
|
ext4_mb_put_pa(ac, ac->ac_sb, pa);
|
|
ext4_mb_put_pa(ac, ac->ac_sb, pa);
|
|
}
|
|
}
|
|
|
|
+ if (ac->alloc_semp)
|
|
|
|
+ up_read(ac->alloc_semp);
|
|
if (ac->ac_bitmap_page)
|
|
if (ac->ac_bitmap_page)
|
|
page_cache_release(ac->ac_bitmap_page);
|
|
page_cache_release(ac->ac_bitmap_page);
|
|
if (ac->ac_buddy_page)
|
|
if (ac->ac_buddy_page)
|
|
@@ -4264,6 +4509,8 @@ static int ext4_mb_discard_preallocations(struct super_block *sb, int needed)
|
|
int ret;
|
|
int ret;
|
|
int freed = 0;
|
|
int freed = 0;
|
|
|
|
|
|
|
|
+ trace_mark(ext4_mb_discard_preallocations, "dev %s needed %d",
|
|
|
|
+ sb->s_id, needed);
|
|
for (i = 0; i < EXT4_SB(sb)->s_groups_count && needed > 0; i++) {
|
|
for (i = 0; i < EXT4_SB(sb)->s_groups_count && needed > 0; i++) {
|
|
ret = ext4_mb_discard_group_preallocations(sb, i, needed);
|
|
ret = ext4_mb_discard_group_preallocations(sb, i, needed);
|
|
freed += ret;
|
|
freed += ret;
|
|
@@ -4286,12 +4533,24 @@ ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle,
|
|
struct ext4_sb_info *sbi;
|
|
struct ext4_sb_info *sbi;
|
|
struct super_block *sb;
|
|
struct super_block *sb;
|
|
ext4_fsblk_t block = 0;
|
|
ext4_fsblk_t block = 0;
|
|
- unsigned long inquota;
|
|
|
|
- unsigned long reserv_blks = 0;
|
|
|
|
|
|
+ unsigned int inquota;
|
|
|
|
+ unsigned int reserv_blks = 0;
|
|
|
|
|
|
sb = ar->inode->i_sb;
|
|
sb = ar->inode->i_sb;
|
|
sbi = EXT4_SB(sb);
|
|
sbi = EXT4_SB(sb);
|
|
|
|
|
|
|
|
+ trace_mark(ext4_request_blocks, "dev %s flags %u len %u ino %lu "
|
|
|
|
+ "lblk %llu goal %llu lleft %llu lright %llu "
|
|
|
|
+ "pleft %llu pright %llu ",
|
|
|
|
+ sb->s_id, ar->flags, ar->len,
|
|
|
|
+ ar->inode ? ar->inode->i_ino : 0,
|
|
|
|
+ (unsigned long long) ar->logical,
|
|
|
|
+ (unsigned long long) ar->goal,
|
|
|
|
+ (unsigned long long) ar->lleft,
|
|
|
|
+ (unsigned long long) ar->lright,
|
|
|
|
+ (unsigned long long) ar->pleft,
|
|
|
|
+ (unsigned long long) ar->pright);
|
|
|
|
+
|
|
if (!EXT4_I(ar->inode)->i_delalloc_reserved_flag) {
|
|
if (!EXT4_I(ar->inode)->i_delalloc_reserved_flag) {
|
|
/*
|
|
/*
|
|
* With delalloc we already reserved the blocks
|
|
* With delalloc we already reserved the blocks
|
|
@@ -4313,7 +4572,7 @@ ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle,
|
|
}
|
|
}
|
|
if (ar->len == 0) {
|
|
if (ar->len == 0) {
|
|
*errp = -EDQUOT;
|
|
*errp = -EDQUOT;
|
|
- return 0;
|
|
|
|
|
|
+ goto out3;
|
|
}
|
|
}
|
|
inquota = ar->len;
|
|
inquota = ar->len;
|
|
|
|
|
|
@@ -4348,10 +4607,14 @@ repeat:
|
|
ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len)
|
|
ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len)
|
|
ext4_mb_new_preallocation(ac);
|
|
ext4_mb_new_preallocation(ac);
|
|
}
|
|
}
|
|
-
|
|
|
|
if (likely(ac->ac_status == AC_STATUS_FOUND)) {
|
|
if (likely(ac->ac_status == AC_STATUS_FOUND)) {
|
|
*errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_blks);
|
|
*errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_blks);
|
|
if (*errp == -EAGAIN) {
|
|
if (*errp == -EAGAIN) {
|
|
|
|
+ /*
|
|
|
|
+ * drop the reference that we took
|
|
|
|
+ * in ext4_mb_use_best_found
|
|
|
|
+ */
|
|
|
|
+ ext4_mb_release_context(ac);
|
|
ac->ac_b_ex.fe_group = 0;
|
|
ac->ac_b_ex.fe_group = 0;
|
|
ac->ac_b_ex.fe_start = 0;
|
|
ac->ac_b_ex.fe_start = 0;
|
|
ac->ac_b_ex.fe_len = 0;
|
|
ac->ac_b_ex.fe_len = 0;
|
|
@@ -4382,6 +4645,26 @@ out2:
|
|
out1:
|
|
out1:
|
|
if (ar->len < inquota)
|
|
if (ar->len < inquota)
|
|
DQUOT_FREE_BLOCK(ar->inode, inquota - ar->len);
|
|
DQUOT_FREE_BLOCK(ar->inode, inquota - ar->len);
|
|
|
|
+out3:
|
|
|
|
+ if (!ar->len) {
|
|
|
|
+ if (!EXT4_I(ar->inode)->i_delalloc_reserved_flag)
|
|
|
|
+ /* release all the reserved blocks if non delalloc */
|
|
|
|
+ percpu_counter_sub(&sbi->s_dirtyblocks_counter,
|
|
|
|
+ reserv_blks);
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ trace_mark(ext4_allocate_blocks,
|
|
|
|
+ "dev %s block %llu flags %u len %u ino %lu "
|
|
|
|
+ "logical %llu goal %llu lleft %llu lright %llu "
|
|
|
|
+ "pleft %llu pright %llu ",
|
|
|
|
+ sb->s_id, (unsigned long long) block,
|
|
|
|
+ ar->flags, ar->len, ar->inode ? ar->inode->i_ino : 0,
|
|
|
|
+ (unsigned long long) ar->logical,
|
|
|
|
+ (unsigned long long) ar->goal,
|
|
|
|
+ (unsigned long long) ar->lleft,
|
|
|
|
+ (unsigned long long) ar->lright,
|
|
|
|
+ (unsigned long long) ar->pleft,
|
|
|
|
+ (unsigned long long) ar->pright);
|
|
|
|
|
|
return block;
|
|
return block;
|
|
}
|
|
}
|
|
@@ -4403,27 +4686,23 @@ static int can_merge(struct ext4_free_data *entry1,
|
|
|
|
|
|
static noinline_for_stack int
|
|
static noinline_for_stack int
|
|
ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b,
|
|
ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b,
|
|
- ext4_group_t group, ext4_grpblk_t block, int count)
|
|
|
|
|
|
+ struct ext4_free_data *new_entry)
|
|
{
|
|
{
|
|
|
|
+ ext4_grpblk_t block;
|
|
|
|
+ struct ext4_free_data *entry;
|
|
struct ext4_group_info *db = e4b->bd_info;
|
|
struct ext4_group_info *db = e4b->bd_info;
|
|
struct super_block *sb = e4b->bd_sb;
|
|
struct super_block *sb = e4b->bd_sb;
|
|
struct ext4_sb_info *sbi = EXT4_SB(sb);
|
|
struct ext4_sb_info *sbi = EXT4_SB(sb);
|
|
- struct ext4_free_data *entry, *new_entry;
|
|
|
|
struct rb_node **n = &db->bb_free_root.rb_node, *node;
|
|
struct rb_node **n = &db->bb_free_root.rb_node, *node;
|
|
struct rb_node *parent = NULL, *new_node;
|
|
struct rb_node *parent = NULL, *new_node;
|
|
|
|
|
|
-
|
|
|
|
|
|
+ BUG_ON(!ext4_handle_valid(handle));
|
|
BUG_ON(e4b->bd_bitmap_page == NULL);
|
|
BUG_ON(e4b->bd_bitmap_page == NULL);
|
|
BUG_ON(e4b->bd_buddy_page == NULL);
|
|
BUG_ON(e4b->bd_buddy_page == NULL);
|
|
|
|
|
|
- new_entry = kmem_cache_alloc(ext4_free_ext_cachep, GFP_NOFS);
|
|
|
|
- new_entry->start_blk = block;
|
|
|
|
- new_entry->group = group;
|
|
|
|
- new_entry->count = count;
|
|
|
|
- new_entry->t_tid = handle->h_transaction->t_tid;
|
|
|
|
new_node = &new_entry->node;
|
|
new_node = &new_entry->node;
|
|
|
|
+ block = new_entry->start_blk;
|
|
|
|
|
|
- ext4_lock_group(sb, group);
|
|
|
|
if (!*n) {
|
|
if (!*n) {
|
|
/* first free block exent. We need to
|
|
/* first free block exent. We need to
|
|
protect buddy cache from being freed,
|
|
protect buddy cache from being freed,
|
|
@@ -4441,10 +4720,9 @@ ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b,
|
|
else if (block >= (entry->start_blk + entry->count))
|
|
else if (block >= (entry->start_blk + entry->count))
|
|
n = &(*n)->rb_right;
|
|
n = &(*n)->rb_right;
|
|
else {
|
|
else {
|
|
- ext4_unlock_group(sb, group);
|
|
|
|
- ext4_error(sb, __func__,
|
|
|
|
- "Double free of blocks %d (%d %d)\n",
|
|
|
|
- block, entry->start_blk, entry->count);
|
|
|
|
|
|
+ ext4_grp_locked_error(sb, e4b->bd_group, __func__,
|
|
|
|
+ "Double free of blocks %d (%d %d)",
|
|
|
|
+ block, entry->start_blk, entry->count);
|
|
return 0;
|
|
return 0;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
@@ -4483,7 +4761,6 @@ ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b,
|
|
spin_lock(&sbi->s_md_lock);
|
|
spin_lock(&sbi->s_md_lock);
|
|
list_add(&new_entry->list, &handle->h_transaction->t_private_list);
|
|
list_add(&new_entry->list, &handle->h_transaction->t_private_list);
|
|
spin_unlock(&sbi->s_md_lock);
|
|
spin_unlock(&sbi->s_md_lock);
|
|
- ext4_unlock_group(sb, group);
|
|
|
|
return 0;
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
|
|
@@ -4499,7 +4776,7 @@ void ext4_mb_free_blocks(handle_t *handle, struct inode *inode,
|
|
struct ext4_allocation_context *ac = NULL;
|
|
struct ext4_allocation_context *ac = NULL;
|
|
struct ext4_group_desc *gdp;
|
|
struct ext4_group_desc *gdp;
|
|
struct ext4_super_block *es;
|
|
struct ext4_super_block *es;
|
|
- unsigned long overflow;
|
|
|
|
|
|
+ unsigned int overflow;
|
|
ext4_grpblk_t bit;
|
|
ext4_grpblk_t bit;
|
|
struct buffer_head *gd_bh;
|
|
struct buffer_head *gd_bh;
|
|
ext4_group_t block_group;
|
|
ext4_group_t block_group;
|
|
@@ -4522,6 +4799,10 @@ void ext4_mb_free_blocks(handle_t *handle, struct inode *inode,
|
|
}
|
|
}
|
|
|
|
|
|
ext4_debug("freeing block %lu\n", block);
|
|
ext4_debug("freeing block %lu\n", block);
|
|
|
|
+ trace_mark(ext4_free_blocks,
|
|
|
|
+ "dev %s block %llu count %lu metadata %d ino %lu",
|
|
|
|
+ sb->s_id, (unsigned long long) block, count, metadata,
|
|
|
|
+ inode ? inode->i_ino : 0);
|
|
|
|
|
|
ac = kmem_cache_alloc(ext4_ac_cachep, GFP_NOFS);
|
|
ac = kmem_cache_alloc(ext4_ac_cachep, GFP_NOFS);
|
|
if (ac) {
|
|
if (ac) {
|
|
@@ -4581,11 +4862,6 @@ do_more:
|
|
err = ext4_journal_get_write_access(handle, gd_bh);
|
|
err = ext4_journal_get_write_access(handle, gd_bh);
|
|
if (err)
|
|
if (err)
|
|
goto error_return;
|
|
goto error_return;
|
|
-
|
|
|
|
- err = ext4_mb_load_buddy(sb, block_group, &e4b);
|
|
|
|
- if (err)
|
|
|
|
- goto error_return;
|
|
|
|
-
|
|
|
|
#ifdef AGGRESSIVE_CHECK
|
|
#ifdef AGGRESSIVE_CHECK
|
|
{
|
|
{
|
|
int i;
|
|
int i;
|
|
@@ -4593,13 +4869,6 @@ do_more:
|
|
BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data));
|
|
BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data));
|
|
}
|
|
}
|
|
#endif
|
|
#endif
|
|
- mb_clear_bits(sb_bgl_lock(sbi, block_group), bitmap_bh->b_data,
|
|
|
|
- bit, count);
|
|
|
|
-
|
|
|
|
- /* We dirtied the bitmap block */
|
|
|
|
- BUFFER_TRACE(bitmap_bh, "dirtied bitmap block");
|
|
|
|
- err = ext4_journal_dirty_metadata(handle, bitmap_bh);
|
|
|
|
-
|
|
|
|
if (ac) {
|
|
if (ac) {
|
|
ac->ac_b_ex.fe_group = block_group;
|
|
ac->ac_b_ex.fe_group = block_group;
|
|
ac->ac_b_ex.fe_start = bit;
|
|
ac->ac_b_ex.fe_start = bit;
|
|
@@ -4607,19 +4876,41 @@ do_more:
|
|
ext4_mb_store_history(ac);
|
|
ext4_mb_store_history(ac);
|
|
}
|
|
}
|
|
|
|
|
|
- if (metadata) {
|
|
|
|
- /* blocks being freed are metadata. these blocks shouldn't
|
|
|
|
- * be used until this transaction is committed */
|
|
|
|
- ext4_mb_free_metadata(handle, &e4b, block_group, bit, count);
|
|
|
|
|
|
+ err = ext4_mb_load_buddy(sb, block_group, &e4b);
|
|
|
|
+ if (err)
|
|
|
|
+ goto error_return;
|
|
|
|
+ if (metadata && ext4_handle_valid(handle)) {
|
|
|
|
+ struct ext4_free_data *new_entry;
|
|
|
|
+ /*
|
|
|
|
+ * blocks being freed are metadata. these blocks shouldn't
|
|
|
|
+ * be used until this transaction is committed
|
|
|
|
+ */
|
|
|
|
+ new_entry = kmem_cache_alloc(ext4_free_ext_cachep, GFP_NOFS);
|
|
|
|
+ new_entry->start_blk = bit;
|
|
|
|
+ new_entry->group = block_group;
|
|
|
|
+ new_entry->count = count;
|
|
|
|
+ new_entry->t_tid = handle->h_transaction->t_tid;
|
|
|
|
+ ext4_lock_group(sb, block_group);
|
|
|
|
+ mb_clear_bits(sb_bgl_lock(sbi, block_group), bitmap_bh->b_data,
|
|
|
|
+ bit, count);
|
|
|
|
+ ext4_mb_free_metadata(handle, &e4b, new_entry);
|
|
|
|
+ ext4_unlock_group(sb, block_group);
|
|
} else {
|
|
} else {
|
|
ext4_lock_group(sb, block_group);
|
|
ext4_lock_group(sb, block_group);
|
|
|
|
+ /* need to update group_info->bb_free and bitmap
|
|
|
|
+ * with group lock held. generate_buddy look at
|
|
|
|
+ * them with group lock_held
|
|
|
|
+ */
|
|
|
|
+ mb_clear_bits(sb_bgl_lock(sbi, block_group), bitmap_bh->b_data,
|
|
|
|
+ bit, count);
|
|
mb_free_blocks(inode, &e4b, bit, count);
|
|
mb_free_blocks(inode, &e4b, bit, count);
|
|
ext4_mb_return_to_preallocation(inode, &e4b, block, count);
|
|
ext4_mb_return_to_preallocation(inode, &e4b, block, count);
|
|
ext4_unlock_group(sb, block_group);
|
|
ext4_unlock_group(sb, block_group);
|
|
}
|
|
}
|
|
|
|
|
|
spin_lock(sb_bgl_lock(sbi, block_group));
|
|
spin_lock(sb_bgl_lock(sbi, block_group));
|
|
- le16_add_cpu(&gdp->bg_free_blocks_count, count);
|
|
|
|
|
|
+ ret = ext4_free_blks_count(sb, gdp) + count;
|
|
|
|
+ ext4_free_blks_set(sb, gdp, ret);
|
|
gdp->bg_checksum = ext4_group_desc_csum(sbi, block_group, gdp);
|
|
gdp->bg_checksum = ext4_group_desc_csum(sbi, block_group, gdp);
|
|
spin_unlock(sb_bgl_lock(sbi, block_group));
|
|
spin_unlock(sb_bgl_lock(sbi, block_group));
|
|
percpu_counter_add(&sbi->s_freeblocks_counter, count);
|
|
percpu_counter_add(&sbi->s_freeblocks_counter, count);
|
|
@@ -4635,9 +4926,13 @@ do_more:
|
|
|
|
|
|
*freed += count;
|
|
*freed += count;
|
|
|
|
|
|
|
|
+ /* We dirtied the bitmap block */
|
|
|
|
+ BUFFER_TRACE(bitmap_bh, "dirtied bitmap block");
|
|
|
|
+ err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh);
|
|
|
|
+
|
|
/* And the group descriptor block */
|
|
/* And the group descriptor block */
|
|
BUFFER_TRACE(gd_bh, "dirtied group descriptor block");
|
|
BUFFER_TRACE(gd_bh, "dirtied group descriptor block");
|
|
- ret = ext4_journal_dirty_metadata(handle, gd_bh);
|
|
|
|
|
|
+ ret = ext4_handle_dirty_metadata(handle, NULL, gd_bh);
|
|
if (!err)
|
|
if (!err)
|
|
err = ret;
|
|
err = ret;
|
|
|
|
|