/fs/ext4/mballoc.c
C | 1776 lines | 1039 code | 187 blank | 550 comment | 223 complexity | 4a441cc9c9686842a9befa525253bec2 MD5 | raw file
- /*
- * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com
- * Written by Alex Tomas <alex@clusterfs.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public Licens
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-
- */
- /*
- * mballoc.c contains the multiblocks allocation routines
- */
- #include "mballoc.h"
- #include <linux/debugfs.h>
- #include <linux/slab.h>
- #include <trace/events/ext4.h>
- /*
- * MUSTDO:
- * - test ext4_ext_search_left() and ext4_ext_search_right()
- * - search for metadata in few groups
- *
- * TODO v4:
- * - normalization should take into account whether file is still open
- * - discard preallocations if no free space left (policy?)
- * - don't normalize tails
- * - quota
- * - reservation for superuser
- *
- * TODO v3:
- * - bitmap read-ahead (proposed by Oleg Drokin aka green)
- * - track min/max extents in each group for better group selection
- * - mb_mark_used() may allocate chunk right after splitting buddy
- * - tree of groups sorted by number of free blocks
- * - error handling
- */
- /*
- * The allocation request involve request for multiple number of blocks
- * near to the goal(block) value specified.
- *
- * During initialization phase of the allocator we decide to use the
- * group preallocation or inode preallocation depending on the size of
- * the file. The size of the file could be the resulting file size we
- * would have after allocation, or the current file size, which ever
- * is larger. If the size is less than sbi->s_mb_stream_request we
- * select to use the group preallocation. The default value of
- * s_mb_stream_request is 16 blocks. This can also be tuned via
- * /sys/fs/ext4/<partition>/mb_stream_req. The value is represented in
- * terms of number of blocks.
- *
- * The main motivation for having small file use group preallocation is to
- * ensure that we have small files closer together on the disk.
- *
- * First stage the allocator looks at the inode prealloc list,
- * ext4_inode_info->i_prealloc_list, which contains list of prealloc
- * spaces for this particular inode. The inode prealloc space is
- * represented as:
- *
- * pa_lstart -> the logical start block for this prealloc space
- * pa_pstart -> the physical start block for this prealloc space
- * pa_len -> length for this prealloc space
- * pa_free -> free space available in this prealloc space
- *
- * The inode preallocation space is used looking at the _logical_ start
- * block. If only the logical file block falls within the range of prealloc
- * space we will consume the particular prealloc space. This makes sure that
- * we have contiguous physical blocks representing the file blocks
- *
- * The important thing to be noted in case of inode prealloc space is that
- * we don't modify the values associated to inode prealloc space except
- * pa_free.
- *
- * If we are not able to find blocks in the inode prealloc space and if we
- * have the group allocation flag set then we look at the locality group
- * prealloc space. These are per CPU prealloc list represented as
- *
- * ext4_sb_info.s_locality_groups[smp_processor_id()]
- *
- * The reason for having a per cpu locality group is to reduce the contention
- * between CPUs. It is possible to get scheduled at this point.
- *
- * The locality group prealloc space is used looking at whether we have
- * enough free space (pa_free) within the prealloc space.
- *
- * If we can't allocate blocks via inode prealloc or/and locality group
- * prealloc then we look at the buddy cache. The buddy cache is represented
- * by ext4_sb_info.s_buddy_cache (struct inode) whose file offset gets
- * mapped to the buddy and bitmap information regarding different
- * groups. The buddy information is attached to buddy cache inode so that
- * we can access them through the page cache. The information regarding
- * each group is loaded via ext4_mb_load_buddy. The information involve
- * block bitmap and buddy information. The information are stored in the
- * inode as:
- *
- * { page }
- * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]...
- *
- *
- * one block each for bitmap and buddy information. So for each group we
- * take up 2 blocks. A page can contain blocks_per_page (PAGE_CACHE_SIZE /
- * blocksize) blocks. So it can have information regarding groups_per_page
- * which is blocks_per_page/2
- *
- * The buddy cache inode is not stored on disk. The inode is thrown
- * away when the filesystem is unmounted.
- *
- * We look for count number of blocks in the buddy cache. If we were able
- * to locate that many free blocks we return with additional information
- * regarding rest of the contiguous physical block available
- *
- * Before allocating blocks via buddy cache we normalize the request
- * blocks. This ensure we ask for more blocks that we needed. The extra
- * blocks that we get after allocation is added to the respective prealloc
- * list. In case of inode preallocation we follow a list of heuristics
- * based on file size. This can be found in ext4_mb_normalize_request. If
- * we are doing a group prealloc we try to normalize the request to
- * sbi->s_mb_group_prealloc. Default value of s_mb_group_prealloc is
- * 512 blocks. This can be tuned via
- * /sys/fs/ext4/<partition>/mb_group_prealloc. The value is represented in
- * terms of number of blocks. If we have mounted the file system with -O
- * stripe=<value> option the group prealloc request is normalized to the
- * the smallest multiple of the stripe value (sbi->s_stripe) which is
- * greater than the default mb_group_prealloc.
- *
- * The regular allocator (using the buddy cache) supports a few tunables.
- *
- * /sys/fs/ext4/<partition>/mb_min_to_scan
- * /sys/fs/ext4/<partition>/mb_max_to_scan
- * /sys/fs/ext4/<partition>/mb_order2_req
- *
- * The regular allocator uses buddy scan only if the request len is power of
- * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The
- * value of s_mb_order2_reqs can be tuned via
- * /sys/fs/ext4/<partition>/mb_order2_req. If the request len is equal to
- * stripe size (sbi->s_stripe), we try to search for contiguous block in
- * stripe size. This should result in better allocation on RAID setups. If
- * not, we search in the specific group using bitmap for best extents. The
- * tunable min_to_scan and max_to_scan control the behaviour here.
- * min_to_scan indicate how long the mballoc __must__ look for a best
- * extent and max_to_scan indicates how long the mballoc __can__ look for a
- * best extent in the found extents. Searching for the blocks starts with
- * the group specified as the goal value in allocation context via
- * ac_g_ex. Each group is first checked based on the criteria whether it
- * can be used for allocation. ext4_mb_good_group explains how the groups are
- * checked.
- *
- * Both the prealloc space are getting populated as above. So for the first
- * request we will hit the buddy cache which will result in this prealloc
- * space getting filled. The prealloc space is then later used for the
- * subsequent request.
- */
- /*
- * mballoc operates on the following data:
- * - on-disk bitmap
- * - in-core buddy (actually includes buddy and bitmap)
- * - preallocation descriptors (PAs)
- *
- * there are two types of preallocations:
- * - inode
- * assiged to specific inode and can be used for this inode only.
- * it describes part of inode's space preallocated to specific
- * physical blocks. any block from that preallocated can be used
- * independent. the descriptor just tracks number of blocks left
- * unused. so, before taking some block from descriptor, one must
- * make sure corresponded logical block isn't allocated yet. this
- * also means that freeing any block within descriptor's range
- * must discard all preallocated blocks.
- * - locality group
- * assigned to specific locality group which does not translate to
- * permanent set of inodes: inode can join and leave group. space
- * from this type of preallocation can be used for any inode. thus
- * it's consumed from the beginning to the end.
- *
- * relation between them can be expressed as:
- * in-core buddy = on-disk bitmap + preallocation descriptors
- *
- * this mean blocks mballoc considers used are:
- * - allocated blocks (persistent)
- * - preallocated blocks (non-persistent)
- *
- * consistency in mballoc world means that at any time a block is either
- * free or used in ALL structures. notice: "any time" should not be read
- * literally -- time is discrete and delimited by locks.
- *
- * to keep it simple, we don't use block numbers, instead we count number of
- * blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA.
- *
- * all operations can be expressed as:
- * - init buddy: buddy = on-disk + PAs
- * - new PA: buddy += N; PA = N
- * - use inode PA: on-disk += N; PA -= N
- * - discard inode PA buddy -= on-disk - PA; PA = 0
- * - use locality group PA on-disk += N; PA -= N
- * - discard locality group PA buddy -= PA; PA = 0
- * note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap
- * is used in real operation because we can't know actual used
- * bits from PA, only from on-disk bitmap
- *
- * if we follow this strict logic, then all operations above should be atomic.
- * given some of them can block, we'd have to use something like semaphores
- * killing performance on high-end SMP hardware. let's try to relax it using
- * the following knowledge:
- * 1) if buddy is referenced, it's already initialized
- * 2) while block is used in buddy and the buddy is referenced,
- * nobody can re-allocate that block
- * 3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has
- * bit set and PA claims same block, it's OK. IOW, one can set bit in
- * on-disk bitmap if buddy has same bit set or/and PA covers corresponded
- * block
- *
- * so, now we're building a concurrency table:
- * - init buddy vs.
- * - new PA
- * blocks for PA are allocated in the buddy, buddy must be referenced
- * until PA is linked to allocation group to avoid concurrent buddy init
- * - use inode PA
- * we need to make sure that either on-disk bitmap or PA has uptodate data
- * given (3) we care that PA-=N operation doesn't interfere with init
- * - discard inode PA
- * the simplest way would be to have buddy initialized by the discard
- * - use locality group PA
- * again PA-=N must be serialized with init
- * - discard locality group PA
- * the simplest way would be to have buddy initialized by the discard
- * - new PA vs.
- * - use inode PA
- * i_data_sem serializes them
- * - discard inode PA
- * discard process must wait until PA isn't used by another process
- * - use locality group PA
- * some mutex should serialize them
- * - discard locality group PA
- * discard process must wait until PA isn't used by another process
- * - use inode PA
- * - use inode PA
- * i_data_sem or another mutex should serializes them
- * - discard inode PA
- * discard process must wait until PA isn't used by another process
- * - use locality group PA
- * nothing wrong here -- they're different PAs covering different blocks
- * - discard locality group PA
- * discard process must wait until PA isn't used by another process
- *
- * now we're ready to make few consequences:
- * - PA is referenced and while it is no discard is possible
- * - PA is referenced until block isn't marked in on-disk bitmap
- * - PA changes only after on-disk bitmap
- * - discard must not compete with init. either init is done before
- * any discard or they're serialized somehow
- * - buddy init as sum of on-disk bitmap and PAs is done atomically
- *
- * a special case when we've used PA to emptiness. no need to modify buddy
- * in this case, but we should care about concurrent init
- *
- */
- /*
- * Logic in few words:
- *
- * - allocation:
- * load group
- * find blocks
- * mark bits in on-disk bitmap
- * release group
- *
- * - use preallocation:
- * find proper PA (per-inode or group)
- * load group
- * mark bits in on-disk bitmap
- * release group
- * release PA
- *
- * - free:
- * load group
- * mark bits in on-disk bitmap
- * release group
- *
- * - discard preallocations in group:
- * mark PAs deleted
- * move them onto local list
- * load on-disk bitmap
- * load group
- * remove PA from object (inode or locality group)
- * mark free blocks in-core
- *
- * - discard inode's preallocations:
- */
- /*
- * Locking rules
- *
- * Locks:
- * - bitlock on a group (group)
- * - object (inode/locality) (object)
- * - per-pa lock (pa)
- *
- * Paths:
- * - new pa
- * object
- * group
- *
- * - find and use pa:
- * pa
- *
- * - release consumed pa:
- * pa
- * group
- * object
- *
- * - generate in-core bitmap:
- * group
- * pa
- *
- * - discard all for given object (inode, locality group):
- * object
- * pa
- * group
- *
- * - discard all for given group:
- * group
- * pa
- * group
- * object
- *
- */
- static struct kmem_cache *ext4_pspace_cachep;
- static struct kmem_cache *ext4_ac_cachep;
- static struct kmem_cache *ext4_free_ext_cachep;
- /* We create slab caches for groupinfo data structures based on the
- * superblock block size. There will be one per mounted filesystem for
- * each unique s_blocksize_bits */
- #define NR_GRPINFO_CACHES 8
- static struct kmem_cache *ext4_groupinfo_caches[NR_GRPINFO_CACHES];
- static const char *ext4_groupinfo_slab_names[NR_GRPINFO_CACHES] = {
- "ext4_groupinfo_1k", "ext4_groupinfo_2k", "ext4_groupinfo_4k",
- "ext4_groupinfo_8k", "ext4_groupinfo_16k", "ext4_groupinfo_32k",
- "ext4_groupinfo_64k", "ext4_groupinfo_128k"
- };
- static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap,
- ext4_group_t group);
- static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap,
- ext4_group_t group);
- static void release_blocks_on_commit(journal_t *journal, transaction_t *txn);
- static inline void *mb_correct_addr_and_bit(int *bit, void *addr)
- {
- #if BITS_PER_LONG == 64
- *bit += ((unsigned long) addr & 7UL) << 3;
- addr = (void *) ((unsigned long) addr & ~7UL);
- #elif BITS_PER_LONG == 32
- *bit += ((unsigned long) addr & 3UL) << 3;
- addr = (void *) ((unsigned long) addr & ~3UL);
- #else
- #error "how many bits you are?!"
- #endif
- return addr;
- }
- static inline int mb_test_bit(int bit, void *addr)
- {
- /*
- * ext4_test_bit on architecture like powerpc
- * needs unsigned long aligned address
- */
- addr = mb_correct_addr_and_bit(&bit, addr);
- return ext4_test_bit(bit, addr);
- }
- static inline void mb_set_bit(int bit, void *addr)
- {
- addr = mb_correct_addr_and_bit(&bit, addr);
- ext4_set_bit(bit, addr);
- }
- static inline void mb_clear_bit(int bit, void *addr)
- {
- addr = mb_correct_addr_and_bit(&bit, addr);
- ext4_clear_bit(bit, addr);
- }
- static inline int mb_find_next_zero_bit(void *addr, int max, int start)
- {
- int fix = 0, ret, tmpmax;
- addr = mb_correct_addr_and_bit(&fix, addr);
- tmpmax = max + fix;
- start += fix;
- ret = ext4_find_next_zero_bit(addr, tmpmax, start) - fix;
- if (ret > max)
- return max;
- return ret;
- }
- static inline int mb_find_next_bit(void *addr, int max, int start)
- {
- int fix = 0, ret, tmpmax;
- addr = mb_correct_addr_and_bit(&fix, addr);
- tmpmax = max + fix;
- start += fix;
- ret = ext4_find_next_bit(addr, tmpmax, start) - fix;
- if (ret > max)
- return max;
- return ret;
- }
- static void *mb_find_buddy(struct ext4_buddy *e4b, int order, int *max)
- {
- char *bb;
- BUG_ON(EXT4_MB_BITMAP(e4b) == EXT4_MB_BUDDY(e4b));
- BUG_ON(max == NULL);
- if (order > e4b->bd_blkbits + 1) {
- *max = 0;
- return NULL;
- }
- /* at order 0 we see each particular block */
- if (order == 0) {
- *max = 1 << (e4b->bd_blkbits + 3);
- return EXT4_MB_BITMAP(e4b);
- }
- bb = EXT4_MB_BUDDY(e4b) + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order];
- *max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order];
- return bb;
- }
- #ifdef DOUBLE_CHECK
- static void mb_free_blocks_double(struct inode *inode, struct ext4_buddy *e4b,
- int first, int count)
- {
- int i;
- struct super_block *sb = e4b->bd_sb;
- if (unlikely(e4b->bd_info->bb_bitmap == NULL))
- return;
- assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));
- for (i = 0; i < count; i++) {
- if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) {
- ext4_fsblk_t blocknr;
- blocknr = ext4_group_first_block_no(sb, e4b->bd_group);
- blocknr += first + i;
- ext4_grp_locked_error(sb, e4b->bd_group,
- inode ? inode->i_ino : 0,
- blocknr,
- "freeing block already freed "
- "(bit %u)",
- first + i);
- }
- mb_clear_bit(first + i, e4b->bd_info->bb_bitmap);
- }
- }
- static void mb_mark_used_double(struct ext4_buddy *e4b, int first, int count)
- {
- int i;
- if (unlikely(e4b->bd_info->bb_bitmap == NULL))
- return;
- assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));
- for (i = 0; i < count; i++) {
- BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap));
- mb_set_bit(first + i, e4b->bd_info->bb_bitmap);
- }
- }
- static void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap)
- {
- if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) {
- unsigned char *b1, *b2;
- int i;
- b1 = (unsigned char *) e4b->bd_info->bb_bitmap;
- b2 = (unsigned char *) bitmap;
- for (i = 0; i < e4b->bd_sb->s_blocksize; i++) {
- if (b1[i] != b2[i]) {
- ext4_msg(e4b->bd_sb, KERN_ERR,
- "corruption in group %u "
- "at byte %u(%u): %x in copy != %x "
- "on disk/prealloc",
- e4b->bd_group, i, i * 8, b1[i], b2[i]);
- BUG();
- }
- }
- }
- }
- #else
- static inline void mb_free_blocks_double(struct inode *inode,
- struct ext4_buddy *e4b, int first, int count)
- {
- return;
- }
- static inline void mb_mark_used_double(struct ext4_buddy *e4b,
- int first, int count)
- {
- return;
- }
- static inline void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap)
- {
- return;
- }
- #endif
- #ifdef AGGRESSIVE_CHECK
- #define MB_CHECK_ASSERT(assert) \
- do { \
- if (!(assert)) { \
- printk(KERN_EMERG \
- "Assertion failure in %s() at %s:%d: \"%s\"\n", \
- function, file, line, # assert); \
- BUG(); \
- } \
- } while (0)
- static int __mb_check_buddy(struct ext4_buddy *e4b, char *file,
- const char *function, int line)
- {
- struct super_block *sb = e4b->bd_sb;
- int order = e4b->bd_blkbits + 1;
- int max;
- int max2;
- int i;
- int j;
- int k;
- int count;
- struct ext4_group_info *grp;
- int fragments = 0;
- int fstart;
- struct list_head *cur;
- void *buddy;
- void *buddy2;
- {
- static int mb_check_counter;
- if (mb_check_counter++ % 100 != 0)
- return 0;
- }
- while (order > 1) {
- buddy = mb_find_buddy(e4b, order, &max);
- MB_CHECK_ASSERT(buddy);
- buddy2 = mb_find_buddy(e4b, order - 1, &max2);
- MB_CHECK_ASSERT(buddy2);
- MB_CHECK_ASSERT(buddy != buddy2);
- MB_CHECK_ASSERT(max * 2 == max2);
- count = 0;
- for (i = 0; i < max; i++) {
- if (mb_test_bit(i, buddy)) {
- /* only single bit in buddy2 may be 1 */
- if (!mb_test_bit(i << 1, buddy2)) {
- MB_CHECK_ASSERT(
- mb_test_bit((i<<1)+1, buddy2));
- } else if (!mb_test_bit((i << 1) + 1, buddy2)) {
- MB_CHECK_ASSERT(
- mb_test_bit(i << 1, buddy2));
- }
- continue;
- }
- /* both bits in buddy2 must be 0 */
- MB_CHECK_ASSERT(mb_test_bit(i << 1, buddy2));
- MB_CHECK_ASSERT(mb_test_bit((i << 1) + 1, buddy2));
- for (j = 0; j < (1 << order); j++) {
- k = (i * (1 << order)) + j;
- MB_CHECK_ASSERT(
- !mb_test_bit(k, EXT4_MB_BITMAP(e4b)));
- }
- count++;
- }
- MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count);
- order--;
- }
- fstart = -1;
- buddy = mb_find_buddy(e4b, 0, &max);
- for (i = 0; i < max; i++) {
- if (!mb_test_bit(i, buddy)) {
- MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free);
- if (fstart == -1) {
- fragments++;
- fstart = i;
- }
- continue;
- }
- fstart = -1;
- /* check used bits only */
- for (j = 0; j < e4b->bd_blkbits + 1; j++) {
- buddy2 = mb_find_buddy(e4b, j, &max2);
- k = i >> j;
- MB_CHECK_ASSERT(k < max2);
- MB_CHECK_ASSERT(mb_test_bit(k, buddy2));
- }
- }
- MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info));
- MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments);
- grp = ext4_get_group_info(sb, e4b->bd_group);
- list_for_each(cur, &grp->bb_prealloc_list) {
- ext4_group_t groupnr;
- struct ext4_prealloc_space *pa;
- pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list);
- ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k);
- MB_CHECK_ASSERT(groupnr == e4b->bd_group);
- for (i = 0; i < pa->pa_len; i++)
- MB_CHECK_ASSERT(mb_test_bit(k + i, buddy));
- }
- return 0;
- }
- #undef MB_CHECK_ASSERT
- #define mb_check_buddy(e4b) __mb_check_buddy(e4b, \
- __FILE__, __func__, __LINE__)
- #else
- #define mb_check_buddy(e4b)
- #endif
- /*
- * Divide blocks started from @first with length @len into
- * smaller chunks with power of 2 blocks.
- * Clear the bits in bitmap which the blocks of the chunk(s) covered,
- * then increase bb_counters[] for corresponded chunk size.
- */
- static void ext4_mb_mark_free_simple(struct super_block *sb,
- void *buddy, ext4_grpblk_t first, ext4_grpblk_t len,
- struct ext4_group_info *grp)
- {
- struct ext4_sb_info *sbi = EXT4_SB(sb);
- ext4_grpblk_t min;
- ext4_grpblk_t max;
- ext4_grpblk_t chunk;
- unsigned short border;
- BUG_ON(len > EXT4_BLOCKS_PER_GROUP(sb));
- border = 2 << sb->s_blocksize_bits;
- while (len > 0) {
- /* find how many blocks can be covered since this position */
- max = ffs(first | border) - 1;
- /* find how many blocks of power 2 we need to mark */
- min = fls(len) - 1;
- if (max < min)
- min = max;
- chunk = 1 << min;
- /* mark multiblock chunks only */
- grp->bb_counters[min]++;
- if (min > 0)
- mb_clear_bit(first >> min,
- buddy + sbi->s_mb_offsets[min]);
- len -= chunk;
- first += chunk;
- }
- }
- /*
- * Cache the order of the largest free extent we have available in this block
- * group.
- */
- static void
- mb_set_largest_free_order(struct super_block *sb, struct ext4_group_info *grp)
- {
- int i;
- int bits;
- grp->bb_largest_free_order = -1; /* uninit */
- bits = sb->s_blocksize_bits + 1;
- for (i = bits; i >= 0; i--) {
- if (grp->bb_counters[i] > 0) {
- grp->bb_largest_free_order = i;
- break;
- }
- }
- }
- static noinline_for_stack
- void ext4_mb_generate_buddy(struct super_block *sb,
- void *buddy, void *bitmap, ext4_group_t group)
- {
- struct ext4_group_info *grp = ext4_get_group_info(sb, group);
- ext4_grpblk_t max = EXT4_BLOCKS_PER_GROUP(sb);
- ext4_grpblk_t i = 0;
- ext4_grpblk_t first;
- ext4_grpblk_t len;
- unsigned free = 0;
- unsigned fragments = 0;
- unsigned long long period = get_cycles();
- /* initialize buddy from bitmap which is aggregation
- * of on-disk bitmap and preallocations */
- i = mb_find_next_zero_bit(bitmap, max, 0);
- grp->bb_first_free = i;
- while (i < max) {
- fragments++;
- first = i;
- i = mb_find_next_bit(bitmap, max, i);
- len = i - first;
- free += len;
- if (len > 1)
- ext4_mb_mark_free_simple(sb, buddy, first, len, grp);
- else
- grp->bb_counters[0]++;
- if (i < max)
- i = mb_find_next_zero_bit(bitmap, max, i);
- }
- grp->bb_fragments = fragments;
- if (free != grp->bb_free) {
- ext4_grp_locked_error(sb, group, 0, 0,
- "%u blocks in bitmap, %u in gd",
- free, grp->bb_free);
- /*
- * If we intent to continue, we consider group descritor
- * corrupt and update bb_free using bitmap value
- */
- grp->bb_free = free;
- }
- mb_set_largest_free_order(sb, grp);
- clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state));
- period = get_cycles() - period;
- spin_lock(&EXT4_SB(sb)->s_bal_lock);
- EXT4_SB(sb)->s_mb_buddies_generated++;
- EXT4_SB(sb)->s_mb_generation_time += period;
- spin_unlock(&EXT4_SB(sb)->s_bal_lock);
- }
- /* The buddy information is attached the buddy cache inode
- * for convenience. The information regarding each group
- * is loaded via ext4_mb_load_buddy. The information involve
- * block bitmap and buddy information. The information are
- * stored in the inode as
- *
- * { page }
- * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]...
- *
- *
- * one block each for bitmap and buddy information.
- * So for each group we take up 2 blocks. A page can
- * contain blocks_per_page (PAGE_CACHE_SIZE / blocksize) blocks.
- * So it can have information regarding groups_per_page which
- * is blocks_per_page/2
- *
- * Locking note: This routine takes the block group lock of all groups
- * for this page; do not hold this lock when calling this routine!
- */
- static int ext4_mb_init_cache(struct page *page, char *incore)
- {
- ext4_group_t ngroups;
- int blocksize;
- int blocks_per_page;
- int groups_per_page;
- int err = 0;
- int i;
- ext4_group_t first_group;
- int first_block;
- struct super_block *sb;
- struct buffer_head *bhs;
- struct buffer_head **bh;
- struct inode *inode;
- char *data;
- char *bitmap;
- struct ext4_group_info *grinfo;
- mb_debug(1, "init page %lu\n", page->index);
- inode = page->mapping->host;
- sb = inode->i_sb;
- ngroups = ext4_get_groups_count(sb);
- blocksize = 1 << inode->i_blkbits;
- blocks_per_page = PAGE_CACHE_SIZE / blocksize;
- groups_per_page = blocks_per_page >> 1;
- if (groups_per_page == 0)
- groups_per_page = 1;
- /* allocate buffer_heads to read bitmaps */
- if (groups_per_page > 1) {
- err = -ENOMEM;
- i = sizeof(struct buffer_head *) * groups_per_page;
- bh = kzalloc(i, GFP_NOFS);
- if (bh == NULL)
- goto out;
- } else
- bh = &bhs;
- first_group = page->index * blocks_per_page / 2;
- /* read all groups the page covers into the cache */
- for (i = 0; i < groups_per_page; i++) {
- struct ext4_group_desc *desc;
- if (first_group + i >= ngroups)
- break;
- grinfo = ext4_get_group_info(sb, first_group + i);
- /*
- * If page is uptodate then we came here after online resize
- * which added some new uninitialized group info structs, so
- * we must skip all initialized uptodate buddies on the page,
- * which may be currently in use by an allocating task.
- */
- if (PageUptodate(page) && !EXT4_MB_GRP_NEED_INIT(grinfo)) {
- bh[i] = NULL;
- continue;
- }
- err = -EIO;
- desc = ext4_get_group_desc(sb, first_group + i, NULL);
- if (desc == NULL)
- goto out;
- err = -ENOMEM;
- bh[i] = sb_getblk(sb, ext4_block_bitmap(sb, desc));
- if (bh[i] == NULL)
- goto out;
- if (bitmap_uptodate(bh[i]))
- continue;
- lock_buffer(bh[i]);
- if (bitmap_uptodate(bh[i])) {
- unlock_buffer(bh[i]);
- continue;
- }
- ext4_lock_group(sb, first_group + i);
- if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) {
- ext4_init_block_bitmap(sb, bh[i],
- first_group + i, desc);
- set_bitmap_uptodate(bh[i]);
- set_buffer_uptodate(bh[i]);
- ext4_unlock_group(sb, first_group + i);
- unlock_buffer(bh[i]);
- continue;
- }
- ext4_unlock_group(sb, first_group + i);
- if (buffer_uptodate(bh[i])) {
- /*
- * if not uninit if bh is uptodate,
- * bitmap is also uptodate
- */
- set_bitmap_uptodate(bh[i]);
- unlock_buffer(bh[i]);
- continue;
- }
- get_bh(bh[i]);
- /*
- * submit the buffer_head for read. We can
- * safely mark the bitmap as uptodate now.
- * We do it here so the bitmap uptodate bit
- * get set with buffer lock held.
- */
- set_bitmap_uptodate(bh[i]);
- bh[i]->b_end_io = end_buffer_read_sync;
- submit_bh(READ, bh[i]);
- mb_debug(1, "read bitmap for group %u\n", first_group + i);
- }
- /* wait for I/O completion */
- for (i = 0; i < groups_per_page; i++)
- if (bh[i])
- wait_on_buffer(bh[i]);
- err = -EIO;
- for (i = 0; i < groups_per_page; i++)
- if (bh[i] && !buffer_uptodate(bh[i]))
- goto out;
- err = 0;
- first_block = page->index * blocks_per_page;
- for (i = 0; i < blocks_per_page; i++) {
- int group;
- group = (first_block + i) >> 1;
- if (group >= ngroups)
- break;
- if (!bh[group - first_group])
- /* skip initialized uptodate buddy */
- continue;
- /*
- * data carry information regarding this
- * particular group in the format specified
- * above
- *
- */
- data = page_address(page) + (i * blocksize);
- bitmap = bh[group - first_group]->b_data;
- /*
- * We place the buddy block and bitmap block
- * close together
- */
- if ((first_block + i) & 1) {
- /* this is block of buddy */
- BUG_ON(incore == NULL);
- mb_debug(1, "put buddy for group %u in page %lu/%x\n",
- group, page->index, i * blocksize);
- trace_ext4_mb_buddy_bitmap_load(sb, group);
- grinfo = ext4_get_group_info(sb, group);
- grinfo->bb_fragments = 0;
- memset(grinfo->bb_counters, 0,
- sizeof(*grinfo->bb_counters) *
- (sb->s_blocksize_bits+2));
- /*
- * incore got set to the group block bitmap below
- */
- ext4_lock_group(sb, group);
- /* init the buddy */
- memset(data, 0xff, blocksize);
- ext4_mb_generate_buddy(sb, data, incore, group);
- ext4_unlock_group(sb, group);
- incore = NULL;
- } else {
- /* this is block of bitmap */
- BUG_ON(incore != NULL);
- mb_debug(1, "put bitmap for group %u in page %lu/%x\n",
- group, page->index, i * blocksize);
- trace_ext4_mb_bitmap_load(sb, group);
- /* see comments in ext4_mb_put_pa() */
- ext4_lock_group(sb, group);
- memcpy(data, bitmap, blocksize);
- /* mark all preallocated blks used in in-core bitmap */
- ext4_mb_generate_from_pa(sb, data, group);
- ext4_mb_generate_from_freelist(sb, data, group);
- ext4_unlock_group(sb, group);
- /* set incore so that the buddy information can be
- * generated using this
- */
- incore = data;
- }
- }
- SetPageUptodate(page);
- out:
- if (bh) {
- for (i = 0; i < groups_per_page; i++)
- brelse(bh[i]);
- if (bh != &bhs)
- kfree(bh);
- }
- return err;
- }
- /*
- * Lock the buddy and bitmap pages. This make sure other parallel init_group
- * on the same buddy page doesn't happen whild holding the buddy page lock.
- * Return locked buddy and bitmap pages on e4b struct. If buddy and bitmap
- * are on the same page e4b->bd_buddy_page is NULL and return value is 0.
- */
- static int ext4_mb_get_buddy_page_lock(struct super_block *sb,
- ext4_group_t group, struct ext4_buddy *e4b)
- {
- struct inode *inode = EXT4_SB(sb)->s_buddy_cache;
- int block, pnum, poff;
- int blocks_per_page;
- struct page *page;
- e4b->bd_buddy_page = NULL;
- e4b->bd_bitmap_page = NULL;
- blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize;
- /*
- * the buddy cache inode stores the block bitmap
- * and buddy information in consecutive blocks.
- * So for each group we need two blocks.
- */
- block = group * 2;
- pnum = block / blocks_per_page;
- poff = block % blocks_per_page;
- page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
- if (!page)
- return -EIO;
- BUG_ON(page->mapping != inode->i_mapping);
- e4b->bd_bitmap_page = page;
- e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);
- if (blocks_per_page >= 2) {
- /* buddy and bitmap are on the same page */
- return 0;
- }
- block++;
- pnum = block / blocks_per_page;
- poff = block % blocks_per_page;
- page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
- if (!page)
- return -EIO;
- BUG_ON(page->mapping != inode->i_mapping);
- e4b->bd_buddy_page = page;
- return 0;
- }
- static void ext4_mb_put_buddy_page_lock(struct ext4_buddy *e4b)
- {
- if (e4b->bd_bitmap_page) {
- unlock_page(e4b->bd_bitmap_page);
- page_cache_release(e4b->bd_bitmap_page);
- }
- if (e4b->bd_buddy_page) {
- unlock_page(e4b->bd_buddy_page);
- page_cache_release(e4b->bd_buddy_page);
- }
- }
- /*
- * Locking note: This routine calls ext4_mb_init_cache(), which takes the
- * block group lock of all groups for this page; do not hold the BG lock when
- * calling this routine!
- */
- static noinline_for_stack
- int ext4_mb_init_group(struct super_block *sb, ext4_group_t group)
- {
- struct ext4_group_info *this_grp;
- struct ext4_buddy e4b;
- struct page *page;
- int ret = 0;
- mb_debug(1, "init group %u\n", group);
- this_grp = ext4_get_group_info(sb, group);
- /*
- * This ensures that we don't reinit the buddy cache
- * page which map to the group from which we are already
- * allocating. If we are looking at the buddy cache we would
- * have taken a reference using ext4_mb_load_buddy and that
- * would have pinned buddy page to page cache.
- */
- ret = ext4_mb_get_buddy_page_lock(sb, group, &e4b);
- if (ret || !EXT4_MB_GRP_NEED_INIT(this_grp)) {
- /*
- * somebody initialized the group
- * return without doing anything
- */
- goto err;
- }
- page = e4b.bd_bitmap_page;
- ret = ext4_mb_init_cache(page, NULL);
- if (ret)
- goto err;
- if (!PageUptodate(page)) {
- ret = -EIO;
- goto err;
- }
- mark_page_accessed(page);
- if (e4b.bd_buddy_page == NULL) {
- /*
- * If both the bitmap and buddy are in
- * the same page we don't need to force
- * init the buddy
- */
- ret = 0;
- goto err;
- }
- /* init buddy cache */
- page = e4b.bd_buddy_page;
- ret = ext4_mb_init_cache(page, e4b.bd_bitmap);
- if (ret)
- goto err;
- if (!PageUptodate(page)) {
- ret = -EIO;
- goto err;
- }
- mark_page_accessed(page);
- err:
- ext4_mb_put_buddy_page_lock(&e4b);
- return ret;
- }
- /*
- * Locking note: This routine calls ext4_mb_init_cache(), which takes the
- * block group lock of all groups for this page; do not hold the BG lock when
- * calling this routine!
- */
- static noinline_for_stack int
- ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group,
- struct ext4_buddy *e4b)
- {
- int blocks_per_page;
- int block;
- int pnum;
- int poff;
- struct page *page;
- int ret;
- struct ext4_group_info *grp;
- struct ext4_sb_info *sbi = EXT4_SB(sb);
- struct inode *inode = sbi->s_buddy_cache;
- mb_debug(1, "load group %u\n", group);
- blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize;
- grp = ext4_get_group_info(sb, group);
- e4b->bd_blkbits = sb->s_blocksize_bits;
- e4b->bd_info = grp;
- e4b->bd_sb = sb;
- e4b->bd_group = group;
- e4b->bd_buddy_page = NULL;
- e4b->bd_bitmap_page = NULL;
- if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) {
- /*
- * we need full data about the group
- * to make a good selection
- */
- ret = ext4_mb_init_group(sb, group);
- if (ret)
- return ret;
- }
- /*
- * the buddy cache inode stores the block bitmap
- * and buddy information in consecutive blocks.
- * So for each group we need two blocks.
- */
- block = group * 2;
- pnum = block / blocks_per_page;
- poff = block % blocks_per_page;
- /* we could use find_or_create_page(), but it locks page
- * what we'd like to avoid in fast path ... */
- page = find_get_page(inode->i_mapping, pnum);
- if (page == NULL || !PageUptodate(page)) {
- if (page)
- /*
- * drop the page reference and try
- * to get the page with lock. If we
- * are not uptodate that implies
- * somebody just created the page but
- * is yet to initialize the same. So
- * wait for it to initialize.
- */
- page_cache_release(page);
- page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
- if (page) {
- BUG_ON(page->mapping != inode->i_mapping);
- if (!PageUptodate(page)) {
- ret = ext4_mb_init_cache(page, NULL);
- if (ret) {
- unlock_page(page);
- goto err;
- }
- mb_cmp_bitmaps(e4b, page_address(page) +
- (poff * sb->s_blocksize));
- }
- unlock_page(page);
- }
- }
- if (page == NULL || !PageUptodate(page)) {
- ret = -EIO;
- goto err;
- }
- e4b->bd_bitmap_page = page;
- e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);
- mark_page_accessed(page);
- block++;
- pnum = block / blocks_per_page;
- poff = block % blocks_per_page;
- page = find_get_page(inode->i_mapping, pnum);
- if (page == NULL || !PageUptodate(page)) {
- if (page)
- page_cache_release(page);
- page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
- if (page) {
- BUG_ON(page->mapping != inode->i_mapping);
- if (!PageUptodate(page)) {
- ret = ext4_mb_init_cache(page, e4b->bd_bitmap);
- if (ret) {
- unlock_page(page);
- goto err;
- }
- }
- unlock_page(page);
- }
- }
- if (page == NULL || !PageUptodate(page)) {
- ret = -EIO;
- goto err;
- }
- e4b->bd_buddy_page = page;
- e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize);
- mark_page_accessed(page);
- BUG_ON(e4b->bd_bitmap_page == NULL);
- BUG_ON(e4b->bd_buddy_page == NULL);
- return 0;
- err:
- if (page)
- page_cache_release(page);
- if (e4b->bd_bitmap_page)
- page_cache_release(e4b->bd_bitmap_page);
- if (e4b->bd_buddy_page)
- page_cache_release(e4b->bd_buddy_page);
- e4b->bd_buddy = NULL;
- e4b->bd_bitmap = NULL;
- return ret;
- }
- static void ext4_mb_unload_buddy(struct ext4_buddy *e4b)
- {
- if (e4b->bd_bitmap_page)
- page_cache_release(e4b->bd_bitmap_page);
- if (e4b->bd_buddy_page)
- page_cache_release(e4b->bd_buddy_page);
- }
- static int mb_find_order_for_block(struct ext4_buddy *e4b, int block)
- {
- int order = 1;
- void *bb;
- BUG_ON(EXT4_MB_BITMAP(e4b) == EXT4_MB_BUDDY(e4b));
- BUG_ON(block >= (1 << (e4b->bd_blkbits + 3)));
- bb = EXT4_MB_BUDDY(e4b);
- while (order <= e4b->bd_blkbits + 1) {
- block = block >> 1;
- if (!mb_test_bit(block, bb)) {
- /* this block is part of buddy of order 'order' */
- return order;
- }
- bb += 1 << (e4b->bd_blkbits - order);
- order++;
- }
- return 0;
- }
- static void mb_clear_bits(void *bm, int cur, int len)
- {
- __u32 *addr;
- len = cur + len;
- while (cur < len) {
- if ((cur & 31) == 0 && (len - cur) >= 32) {
- /* fast path: clear whole word at once */
- addr = bm + (cur >> 3);
- *addr = 0;
- cur += 32;
- continue;
- }
- mb_clear_bit(cur, bm);
- cur++;
- }
- }
- void ext4_set_bits(void *bm, int cur, int len)
- {
- __u32 *addr;
- len = cur + len;
- while (cur < len) {
- if ((cur & 31) == 0 && (len - cur) >= 32) {
- /* fast path: set whole word at once */
- addr = bm + (cur >> 3);
- *addr = 0xffffffff;
- cur += 32;
- continue;
- }
- mb_set_bit(cur, bm);
- cur++;
- }
- }
- static void mb_free_blocks(struct inode *inode, struct ext4_buddy *e4b,
- int first, int count)
- {
- int block = 0;
- int max = 0;
- int order;
- void *buddy;
- void *buddy2;
- struct super_block *sb = e4b->bd_sb;
- BUG_ON(first + count > (sb->s_blocksize << 3));
- assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));
- mb_check_buddy(e4b);
- mb_free_blocks_double(inode, e4b, first, count);
- e4b->bd_info->bb_free += count;
- if (first < e4b->bd_info->bb_first_free)
- e4b->bd_info->bb_first_free = first;
- /* let's maintain fragments counter */
- if (first != 0)
- block = !mb_test_bit(first - 1, EXT4_MB_BITMAP(e4b));
- if (first + count < EXT4_SB(sb)->s_mb_maxs[0])
- max = !mb_test_bit(first + count, EXT4_MB_BITMAP(e4b));
- if (block && max)
- e4b->bd_info->bb_fragments--;
- else if (!block && !max)
- e4b->bd_info->bb_fragments++;
- /* let's maintain buddy itself */
- while (count-- > 0) {
- block = first++;
- order = 0;
- if (!mb_test_bit(block, EXT4_MB_BITMAP(e4b))) {
- ext4_fsblk_t blocknr;
- blocknr = ext4_group_first_block_no(sb, e4b->bd_group);
- blocknr += block;
- ext4_grp_locked_error(sb, e4b->bd_group,
- inode ? inode->i_ino : 0,
- blocknr,
- "freeing already freed block "
- "(bit %u)", block);
- }
- mb_clear_bit(block, EXT4_MB_BITMAP(e4b));
- e4b->bd_info->bb_counters[order]++;
- /* start of the buddy */
- buddy = mb_find_buddy(e4b, order, &max);
- do {
- block &= ~1UL;
- if (mb_test_bit(block, buddy) ||
- mb_test_bit(block + 1, buddy))
- break;
- /* both the buddies are free, try to coalesce them */
- buddy2 = mb_find_buddy(e4b, order + 1, &max);
- if (!buddy2)
- break;
- if (order > 0) {
- /* for special purposes, we don't set
- * free bits in bitmap */
- mb_set_bit(block, buddy);
- mb_set_bit(block + 1, buddy);
- }
- e4b->bd_info->bb_counters[order]--;
- e4b->bd_info->bb_counters[order]--;
- block = block >> 1;
- order++;
- e4b->bd_info->bb_counters[order]++;
- mb_clear_bit(block, buddy2);
- buddy = buddy2;
- } while (1);
- }
- mb_set_largest_free_order(sb, e4b->bd_info);
- mb_check_buddy(e4b);
- }
- static int mb_find_extent(struct ext4_buddy *e4b, int order, int block,
- int needed, struct ext4_free_extent *ex)
- {
- int next = block;
- int max;
- int ord;
- void *buddy;
- assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));
- BUG_ON(ex == NULL);
- buddy = mb_find_buddy(e4b, order, &max);
- BUG_ON(buddy == NULL);
- BUG_ON(block >= max);
- if (mb_test_bit(block, buddy)) {
- ex->fe_len = 0;
- ex->fe_start = 0;
- ex->fe_group = 0;
- return 0;
- }
- /* FIXME dorp order completely ? */
- if (likely(order == 0)) {
- /* find actual order */
- order = mb_find_order_for_block(e4b, block);
- block = block >> order;
- }
- ex->fe_len = 1 << order;
- ex->fe_start = block << order;
- ex->fe_group = e4b->bd_group;
- /* calc difference from given start */
- next = next - ex->fe_start;
- ex->fe_len -= next;
- ex->fe_start += next;
- while (needed > ex->fe_len &&
- (buddy = mb_find_buddy(e4b, order, &max))) {
- if (block + 1 >= max)
- break;
- next = (block + 1) * (1 << order);
- if (mb_test_bit(next, EXT4_MB_BITMAP(e4b)))
- break;
- ord = mb_find_order_for_block(e4b, next);
- order = ord;
- block = next >> order;
- ex->fe_len += 1 << order;
- }
- BUG_ON(ex->fe_start + ex->fe_len > (1 << (e4b->bd_blkbits + 3)));
- return ex->fe_len;
- }
- static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex)
- {
- int ord;
- int mlen = 0;
- int max = 0;
- int cur;
- int start = ex->fe_start;
- int len = ex->fe_len;
- unsigned ret = 0;
- int len0 = len;
- void *buddy;
- BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3));
- BUG_ON(e4b->bd_group != ex->fe_group);
- assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));
- mb_check_buddy(e4b);
- mb_mark_used_double(e4b, start, len);
- e4b->bd_info->bb_free -= len;
- if (e4b->bd_info->bb_first_free == start)
- e4b->bd_info->bb_first_free += len;
- /* let's maintain fragments counter */
- if (start != 0)
- mlen = !mb_test_bit(start - 1, EXT4_MB_BITMAP(e4b));
- if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0])
- max = !mb_test_bit(start + len, EXT4_MB_BITMAP(e4b));
- if (mlen && max)
- e4b->bd_info->bb_fragments++;
- else if (!mlen && !max)
- e4b->bd_info->bb_fragments--;
- /* let's maintain buddy itself */
- while (len) {
- ord = mb_find_order_for_block(e4b, start);
- if (((start >> ord) << ord) == start && len >= (1 << ord)) {
- /* the whole chunk may be allocated at once! */
- mlen = 1 << ord;
- buddy = mb_find_buddy(e4b, ord, &max);
- BUG_ON((start >> ord) >= max);
- mb_set_bit(start >> ord, buddy);
- e4b->bd_info->bb_counters[ord]--;
- start += mlen;
- len -= mlen;
- BUG_ON(len < 0);
- continue;
- }
- /* store for history */
- if (ret == 0)
- ret = len | (ord << 16);
- /* we have to split large buddy */
- BUG_ON(ord <= 0);
- buddy = mb_find_buddy(e4b, ord, &max);
- mb_set_bit(start >> ord, buddy);
- e4b->bd_info->bb_counters[ord]--;
- ord--;
- cur = (start >> ord) & ~1U;
- buddy = mb_find_buddy(e4b, ord, &max);
- mb_clear_bit(cur, buddy);
- mb_clear_bit(cur + 1, buddy);
- e4b->bd_info->bb_counters[ord]++;
- e4b->bd_info->bb_counters[ord]++;
- }
- mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info);
- ext4_set_bits(EXT4_MB_BITMAP(e4b), ex->fe_start, len0);
- mb_check_buddy(e4b);
- return ret;
- }
- /*
- * Must be called under group lock!
- */
- static void ext4_mb_use_best_found(struct ext4_allocation_context *ac,
- struct ext4_buddy *e4b)
- {
- struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
- int ret;
- BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group);
- BUG_ON(ac->ac_status == AC_STATUS_FOUND);
- ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len);
- ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical;
- ret = mb_mark_used(e4b, &ac->ac_b_ex);
- /* preallocation can change ac_b_ex, thus we store actually
- * allocated blocks for history */
- ac->ac_f_ex = ac->ac_b_ex;
- ac->ac_status = AC_STATUS_FOUND;
- ac->ac_tail = ret & 0xffff;
- ac->ac_buddy = ret >> 16;
- /*
- * take the page reference. We want the page to be pinned
- * so that we don't get a ext4_mb_init_cache_call for this
- * group until we update the bitmap. That would mean we
- * double allocate blocks. The reference is dropped
- * in ext4_mb_release_context
- */
- ac->ac_bitmap_page = e4b->bd_bitmap_page;
- get_page(ac->ac_bitmap_page);
- ac->ac_buddy_page = e4b->bd_buddy_page;
- get_page(ac->ac_buddy_page);
- /* store last allocated for subsequent stream allocation */
- if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {
- spin_lock(&sbi->s_md_lock);
- sbi->s_mb_last_group = ac->ac_f_ex.fe_group;
- sbi->s_mb_last_start = ac->ac_f_ex.fe_start;
- spin_unlock(&sbi->s_md_lock);
- }
- }
- /*
- * regular allocator, for general purposes allocation
- */
- static void ext4_mb_check_limits(struct ext4_allocation_context *ac,
- struct ext4_buddy *e4b,
- int finish_group)
- {
- struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
- struct ext4_free_extent *bex = &ac->ac_b_ex;
- struct ext4_free_extent *gex = &ac->ac_g_ex;
- struct ext4_free_extent ex;
- int max;
- if (ac->ac_status == AC_STATUS_FOUND)
- return;
- /*
- * We don't want to scan for a whole year
- */
- if (ac->ac_found > sbi->s_mb_max_to_scan &&
- !(ac->ac_flags & EXT4_MB_HINT_FIRST)) {
- ac->ac_status = AC_STATUS_BREAK;
- return;
- }
- /*
- * Haven't found good chunk so far, let's continue
- */
- if (bex->fe_len < gex->fe_len)
- return;
- if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan)
- && bex->fe_group == e4b->bd_group) {
- /* recheck chunk's availability - we don't know
- * when it was found (within this lock-unlock
- * period or not) */
- max = mb_find_extent(e4b, 0, bex->fe_start, gex->fe_len, &ex);
- if (max >= gex->fe_len) {
- ext4_mb_use_best_found(ac, e4b);
- return;
- }
- }
- }
- /*
- * The routine checks whether found extent is good enough. If it is,
- * then the extent gets marked used and flag is set to the context
- * to stop scanning. Otherwise, the extent is compared with the
- * previous found extent and if new one is better, then it's stored
- * in the context. Later, the best found extent will be used, if
- * mballoc can't find good enough extent.
- *
- * FIXME: real allocation policy is to be designed yet!
- */
- static void ext4_mb_measure_extent(struct ext4_allocation_context *ac,
- struct ext4_free_extent *ex,
- struct ext4_buddy *e4b)
- {
- struct ext4_free_extent *bex = &ac->ac_b_ex;
- struct ext4_free_extent *gex = &ac->ac_g_ex;
- BUG_ON(ex->fe_len <= 0);
- BUG_ON(ex->fe_len > EXT4_BLOCKS_PER_GROUP(ac->ac_sb));
- BUG_ON(ex->fe_start >= EXT4_BLOCKS_PER_GROUP(ac->ac_sb));
- BUG_ON(ac->ac_status != AC_STATUS_CONTINUE);
- ac->ac_found++;
- /*
- * The special case - take what you catch first
- */
- if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) {
- *bex = *ex;
- ext4_mb_use_best_found(ac, e4b);
- return;
- }
- /*
- * Let's check whether the chuck is good enough
- */
- if (ex->fe_len == gex->fe_len) {
- *bex = *ex;
- ext4_mb_use_best_found(ac, e4b);
- return;
- }
- /*
- * If this is first found extent, just store it in the context
- */
- if (bex->fe_len == 0) {
- *bex = *ex;
- return;
- }
- /*
- * If new found extent is better, store it in the context
- */
- if (bex->fe_len < gex->fe_len) {
- /* if the request isn't satisfied, any found extent
- * larger than previous best one is better */
- if (ex->fe_len > bex->fe_len)
- *bex = *ex;
- } else if (ex->fe_len > gex->fe_len) {
- /* if the request is satisfied, then we try to find
- * an extent that still satisfy the request, but is
- * smaller than previous one */
- if (ex->fe_len < bex->fe_len)
- *bex = *ex;
- }
- ext4_mb_check_limits(ac, e4b, 0);
- }
- static noinline_for_stack
- int ext4_mb_try_best_found(struct ext4_allocation_context *ac,
- struct ext4_buddy *e4b)
- {
- struct ext4_free_extent ex = ac->ac_b_ex;
- ext4_group_t group = ex.fe_group;
- int max;
- int err;
- BUG_ON(ex.fe_len <= 0);
- err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);
- if (err)
- return err;
- ext4_lock_group(ac->ac_sb, group);
- max = mb_find_extent(e4b, 0, ex.fe_start, ex.fe_len, &ex);
- if (max > 0) {
- ac->ac_b_ex = ex;
- ext4_mb_use_best_found(ac, e4b);
- }
- ext4_unlock_group(ac->ac_sb, group);
- ext4_mb_unload_buddy(e4b);
- return 0;
- }
- static noinline_for_stack
- int ext4_mb_find_by_goal(struct ext4_allocation_context *ac,
- struct ext4_buddy *e4b)
- {
- ext4_group_t group = ac->ac_g_ex.fe_group;
- int max;
- int err;
- struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
- struct ext4_free_extent ex;
- if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL))
- return 0;
- err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);
- if (err)
- return err;
- ext4_lock_group(ac->ac_sb, group);
- max = mb_find_extent(e4b, 0, ac->ac_g_ex.fe_start,
- ac->ac_g_ex.fe_len, &ex);
- if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) {
- ext4_fsblk_t start;
- start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) +
- ex.fe_start;
- /* use do_div to get remainder (would be 64-bit modulo) */
- if (do_div(start, sbi->s_stripe) == 0) {
- ac->ac_found++;
- ac->ac_b_ex = ex;
- ext4_mb_use_best_found(ac, e4b);
- }
- } else if (max >= ac->ac_g_ex.fe_len) {
- BUG_ON(ex.fe_len <= 0);
- BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);
- BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);
- ac->ac_found++;
- ac->ac_b_ex = ex;
- ext4_mb_use_best_found(ac, e4b);
- } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) {
- /* Sometimes, caller may want to merge even small
- * number of blocks to an existing extent */
- BUG_ON(ex.fe_len <= 0);
- BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);
- BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);
- ac->ac_found++;
- ac->ac_b_ex = ex;
- ext4_mb_use_best_found(ac, e4b);
- }
- ext4_unlock_group(ac->ac_sb, group);
- ext4_mb_unload_buddy(e4b);
- return 0;
- }
- /*
- * The routine scans buddy structures (not bitmap!) from given order
- * to max order and tries to find big enough chunk to satisfy the req
- */
- static noinline_for_stack
- void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac,
- struct ext4_buddy *e4b)
- {
- struct super_block *sb = ac->ac_sb;
- struct ext4_group_info *grp = e4b->bd_info;
- void *buddy;
- int i;
- int k;
- int max;
- BUG_ON