diff options
Diffstat (limited to 'lib/bitmap.c')
| -rw-r--r-- | lib/bitmap.c | 908 |
1 files changed, 288 insertions, 620 deletions
diff --git a/lib/bitmap.c b/lib/bitmap.c index 9a532805364b..9dc526507875 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -1,25 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * lib/bitmap.c * Helper functions for bitmap.h. - * - * This source code is licensed under the GNU General Public License, - * Version 2. See the file COPYING for more details. */ -#include <linux/export.h> -#include <linux/thread_info.h> -#include <linux/ctype.h> -#include <linux/errno.h> + #include <linux/bitmap.h> #include <linux/bitops.h> -#include <linux/bug.h> -#include <linux/kernel.h> -#include <linux/string.h> -#include <linux/uaccess.h> - -#include <asm/page.h> +#include <linux/ctype.h> +#include <linux/device.h> +#include <linux/export.h> +#include <linux/slab.h> -/* - * bitmaps provide an array of bits, implemented using an an +/** + * DOC: bitmap introduction + * + * bitmaps provide an array of bits, implemented using an * array of unsigned longs. The number of valid bits in a * given bitmap does _not_ need to be an exact multiple of * BITS_PER_LONG. @@ -33,41 +28,53 @@ * carefully filter out these unused bits from impacting their * results. * - * These operations actually hold to a slightly stronger rule: - * if you don't input any bitmaps to these ops that have some - * unused bits set, then they won't output any set unused bits - * in output bitmaps. - * * The byte ordering of bitmaps is more natural on little * endian architectures. See the big-endian headers * include/asm-ppc64/bitops.h and include/asm-s390/bitops.h * for the best explanations of this ordering. */ -int __bitmap_equal(const unsigned long *bitmap1, - const unsigned long *bitmap2, unsigned int bits) +bool __bitmap_equal(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits) { unsigned int k, lim = bits/BITS_PER_LONG; for (k = 0; k < lim; ++k) if (bitmap1[k] != bitmap2[k]) - return 0; + return false; if (bits % BITS_PER_LONG) if ((bitmap1[k] ^ bitmap2[k]) & BITMAP_LAST_WORD_MASK(bits)) - return 0; + return false; - return 1; + return true; } EXPORT_SYMBOL(__bitmap_equal); +bool __bitmap_or_equal(const unsigned long *bitmap1, + const unsigned long *bitmap2, + const unsigned long *bitmap3, + unsigned int bits) +{ + unsigned int k, lim = bits / BITS_PER_LONG; + unsigned long tmp; + + for (k = 0; k < lim; ++k) { + if ((bitmap1[k] | bitmap2[k]) != bitmap3[k]) + return false; + } + + if (!(bits % BITS_PER_LONG)) + return true; + + tmp = (bitmap1[k] | bitmap2[k]) ^ bitmap3[k]; + return (tmp & BITMAP_LAST_WORD_MASK(bits)) == 0; +} + void __bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int bits) { - unsigned int k, lim = bits/BITS_PER_LONG; + unsigned int k, lim = BITS_TO_LONGS(bits); for (k = 0; k < lim; ++k) dst[k] = ~src[k]; - - if (bits % BITS_PER_LONG) - dst[k] = ~src[k]; } EXPORT_SYMBOL(__bitmap_complement); @@ -152,7 +159,74 @@ void __bitmap_shift_left(unsigned long *dst, const unsigned long *src, } EXPORT_SYMBOL(__bitmap_shift_left); -int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, +/** + * bitmap_cut() - remove bit region from bitmap and right shift remaining bits + * @dst: destination bitmap, might overlap with src + * @src: source bitmap + * @first: start bit of region to be removed + * @cut: number of bits to remove + * @nbits: bitmap size, in bits + * + * Set the n-th bit of @dst iff the n-th bit of @src is set and + * n is less than @first, or the m-th bit of @src is set for any + * m such that @first <= n < nbits, and m = n + @cut. + * + * In pictures, example for a big-endian 32-bit architecture: + * + * The @src bitmap is:: + * + * 31 63 + * | | + * 10000000 11000001 11110010 00010101 10000000 11000001 01110010 00010101 + * | | | | + * 16 14 0 32 + * + * if @cut is 3, and @first is 14, bits 14-16 in @src are cut and @dst is:: + * + * 31 63 + * | | + * 10110000 00011000 00110010 00010101 00010000 00011000 00101110 01000010 + * | | | + * 14 (bit 17 0 32 + * from @src) + * + * Note that @dst and @src might overlap partially or entirely. + * + * This is implemented in the obvious way, with a shift and carry + * step for each moved bit. Optimisation is left as an exercise + * for the compiler. + */ +void bitmap_cut(unsigned long *dst, const unsigned long *src, + unsigned int first, unsigned int cut, unsigned int nbits) +{ + unsigned int len = BITS_TO_LONGS(nbits); + unsigned long keep = 0, carry; + int i; + + if (first % BITS_PER_LONG) { + keep = src[first / BITS_PER_LONG] & + (~0UL >> (BITS_PER_LONG - first % BITS_PER_LONG)); + } + + memmove(dst, src, len * sizeof(*dst)); + + while (cut--) { + for (i = first / BITS_PER_LONG; i < len; i++) { + if (i < len - 1) + carry = dst[i + 1] & 1UL; + else + carry = 0; + + dst[i] = (dst[i] >> 1) | (carry << (BITS_PER_LONG - 1)); + } + } + + dst[first / BITS_PER_LONG] &= ~0UL << (first % BITS_PER_LONG); + dst[first / BITS_PER_LONG] |= keep; +} +EXPORT_SYMBOL(bitmap_cut); + +bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits) { unsigned int k; @@ -190,7 +264,7 @@ void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1, } EXPORT_SYMBOL(__bitmap_xor); -int __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1, +bool __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits) { unsigned int k; @@ -206,50 +280,86 @@ int __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1, } EXPORT_SYMBOL(__bitmap_andnot); -int __bitmap_intersects(const unsigned long *bitmap1, - const unsigned long *bitmap2, unsigned int bits) +void __bitmap_replace(unsigned long *dst, + const unsigned long *old, const unsigned long *new, + const unsigned long *mask, unsigned int nbits) +{ + unsigned int k; + unsigned int nr = BITS_TO_LONGS(nbits); + + for (k = 0; k < nr; k++) + dst[k] = (old[k] & ~mask[k]) | (new[k] & mask[k]); +} +EXPORT_SYMBOL(__bitmap_replace); + +bool __bitmap_intersects(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits) { unsigned int k, lim = bits/BITS_PER_LONG; for (k = 0; k < lim; ++k) if (bitmap1[k] & bitmap2[k]) - return 1; + return true; if (bits % BITS_PER_LONG) if ((bitmap1[k] & bitmap2[k]) & BITMAP_LAST_WORD_MASK(bits)) - return 1; - return 0; + return true; + return false; } EXPORT_SYMBOL(__bitmap_intersects); -int __bitmap_subset(const unsigned long *bitmap1, - const unsigned long *bitmap2, unsigned int bits) +bool __bitmap_subset(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits) { unsigned int k, lim = bits/BITS_PER_LONG; for (k = 0; k < lim; ++k) if (bitmap1[k] & ~bitmap2[k]) - return 0; + return false; if (bits % BITS_PER_LONG) if ((bitmap1[k] & ~bitmap2[k]) & BITMAP_LAST_WORD_MASK(bits)) - return 0; - return 1; + return false; + return true; } EXPORT_SYMBOL(__bitmap_subset); -int __bitmap_weight(const unsigned long *bitmap, unsigned int bits) +#define BITMAP_WEIGHT(FETCH, bits) \ +({ \ + unsigned int __bits = (bits), idx, w = 0; \ + \ + for (idx = 0; idx < __bits / BITS_PER_LONG; idx++) \ + w += hweight_long(FETCH); \ + \ + if (__bits % BITS_PER_LONG) \ + w += hweight_long((FETCH) & BITMAP_LAST_WORD_MASK(__bits)); \ + \ + w; \ +}) + +unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int bits) { - unsigned int k, lim = bits/BITS_PER_LONG; - int w = 0; + return BITMAP_WEIGHT(bitmap[idx], bits); +} +EXPORT_SYMBOL(__bitmap_weight); - for (k = 0; k < lim; k++) - w += hweight_long(bitmap[k]); +unsigned int __bitmap_weight_and(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits) +{ + return BITMAP_WEIGHT(bitmap1[idx] & bitmap2[idx], bits); +} +EXPORT_SYMBOL(__bitmap_weight_and); - if (bits % BITS_PER_LONG) - w += hweight_long(bitmap[k] & BITMAP_LAST_WORD_MASK(bits)); +unsigned int __bitmap_weight_andnot(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits) +{ + return BITMAP_WEIGHT(bitmap1[idx] & ~bitmap2[idx], bits); +} +EXPORT_SYMBOL(__bitmap_weight_andnot); - return w; +unsigned int __bitmap_weighted_or(unsigned long *dst, const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits) +{ + return BITMAP_WEIGHT(({dst[idx] = bitmap1[idx] | bitmap2[idx]; dst[idx]; }), bits); } -EXPORT_SYMBOL(__bitmap_weight); void __bitmap_set(unsigned long *map, unsigned int start, int len) { @@ -332,335 +442,6 @@ again: } EXPORT_SYMBOL(bitmap_find_next_zero_area_off); -/* - * Bitmap printing & parsing functions: first version by Nadia Yvette Chambers, - * second version by Paul Jackson, third by Joe Korty. - */ - -#define CHUNKSZ 32 -#define nbits_to_hold_value(val) fls(val) -#define BASEDEC 10 /* fancier cpuset lists input in decimal */ - -/** - * __bitmap_parse - convert an ASCII hex string into a bitmap. - * @buf: pointer to buffer containing string. - * @buflen: buffer size in bytes. If string is smaller than this - * then it must be terminated with a \0. - * @is_user: location of buffer, 0 indicates kernel space - * @maskp: pointer to bitmap array that will contain result. - * @nmaskbits: size of bitmap, in bits. - * - * Commas group hex digits into chunks. Each chunk defines exactly 32 - * bits of the resultant bitmask. No chunk may specify a value larger - * than 32 bits (%-EOVERFLOW), and if a chunk specifies a smaller value - * then leading 0-bits are prepended. %-EINVAL is returned for illegal - * characters and for grouping errors such as "1,,5", ",44", "," and "". - * Leading and trailing whitespace accepted, but not embedded whitespace. - */ -int __bitmap_parse(const char *buf, unsigned int buflen, - int is_user, unsigned long *maskp, - int nmaskbits) -{ - int c, old_c, totaldigits, ndigits, nchunks, nbits; - u32 chunk; - const char __user __force *ubuf = (const char __user __force *)buf; - - bitmap_zero(maskp, nmaskbits); - - nchunks = nbits = totaldigits = c = 0; - do { - chunk = 0; - ndigits = totaldigits; - - /* Get the next chunk of the bitmap */ - while (buflen) { - old_c = c; - if (is_user) { - if (__get_user(c, ubuf++)) - return -EFAULT; - } - else - c = *buf++; - buflen--; - if (isspace(c)) - continue; - - /* - * If the last character was a space and the current - * character isn't '\0', we've got embedded whitespace. - * This is a no-no, so throw an error. - */ - if (totaldigits && c && isspace(old_c)) - return -EINVAL; - - /* A '\0' or a ',' signal the end of the chunk */ - if (c == '\0' || c == ',') - break; - - if (!isxdigit(c)) - return -EINVAL; - - /* - * Make sure there are at least 4 free bits in 'chunk'. - * If not, this hexdigit will overflow 'chunk', so - * throw an error. - */ - if (chunk & ~((1UL << (CHUNKSZ - 4)) - 1)) - return -EOVERFLOW; - - chunk = (chunk << 4) | hex_to_bin(c); - totaldigits++; - } - if (ndigits == totaldigits) - return -EINVAL; - if (nchunks == 0 && chunk == 0) - continue; - - __bitmap_shift_left(maskp, maskp, CHUNKSZ, nmaskbits); - *maskp |= chunk; - nchunks++; - nbits += (nchunks == 1) ? nbits_to_hold_value(chunk) : CHUNKSZ; - if (nbits > nmaskbits) - return -EOVERFLOW; - } while (buflen && c == ','); - - return 0; -} -EXPORT_SYMBOL(__bitmap_parse); - -/** - * bitmap_parse_user - convert an ASCII hex string in a user buffer into a bitmap - * - * @ubuf: pointer to user buffer containing string. - * @ulen: buffer size in bytes. If string is smaller than this - * then it must be terminated with a \0. - * @maskp: pointer to bitmap array that will contain result. - * @nmaskbits: size of bitmap, in bits. - * - * Wrapper for __bitmap_parse(), providing it with user buffer. - * - * We cannot have this as an inline function in bitmap.h because it needs - * linux/uaccess.h to get the access_ok() declaration and this causes - * cyclic dependencies. - */ -int bitmap_parse_user(const char __user *ubuf, - unsigned int ulen, unsigned long *maskp, - int nmaskbits) -{ - if (!access_ok(VERIFY_READ, ubuf, ulen)) - return -EFAULT; - return __bitmap_parse((const char __force *)ubuf, - ulen, 1, maskp, nmaskbits); - -} -EXPORT_SYMBOL(bitmap_parse_user); - -/** - * bitmap_print_to_pagebuf - convert bitmap to list or hex format ASCII string - * @list: indicates whether the bitmap must be list - * @buf: page aligned buffer into which string is placed - * @maskp: pointer to bitmap to convert - * @nmaskbits: size of bitmap, in bits - * - * Output format is a comma-separated list of decimal numbers and - * ranges if list is specified or hex digits grouped into comma-separated - * sets of 8 digits/set. Returns the number of characters written to buf. - * - * It is assumed that @buf is a pointer into a PAGE_SIZE area and that - * sufficient storage remains at @buf to accommodate the - * bitmap_print_to_pagebuf() output. - */ -int bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, - int nmaskbits) -{ - ptrdiff_t len = PTR_ALIGN(buf + PAGE_SIZE - 1, PAGE_SIZE) - buf; - int n = 0; - - if (len > 1) - n = list ? scnprintf(buf, len, "%*pbl\n", nmaskbits, maskp) : - scnprintf(buf, len, "%*pb\n", nmaskbits, maskp); - return n; -} -EXPORT_SYMBOL(bitmap_print_to_pagebuf); - -/** - * __bitmap_parselist - convert list format ASCII string to bitmap - * @buf: read nul-terminated user string from this buffer - * @buflen: buffer size in bytes. If string is smaller than this - * then it must be terminated with a \0. - * @is_user: location of buffer, 0 indicates kernel space - * @maskp: write resulting mask here - * @nmaskbits: number of bits in mask to be written - * - * Input format is a comma-separated list of decimal numbers and - * ranges. Consecutively set bits are shown as two hyphen-separated - * decimal numbers, the smallest and largest bit numbers set in - * the range. - * Optionally each range can be postfixed to denote that only parts of it - * should be set. The range will divided to groups of specific size. - * From each group will be used only defined amount of bits. - * Syntax: range:used_size/group_size - * Example: 0-1023:2/256 ==> 0,1,256,257,512,513,768,769 - * - * Returns: 0 on success, -errno on invalid input strings. Error values: - * - * - ``-EINVAL``: second number in range smaller than first - * - ``-EINVAL``: invalid character in string - * - ``-ERANGE``: bit number specified too large for mask - */ -static int __bitmap_parselist(const char *buf, unsigned int buflen, - int is_user, unsigned long *maskp, - int nmaskbits) -{ - unsigned int a, b, old_a, old_b; - unsigned int group_size, used_size; - int c, old_c, totaldigits, ndigits; - const char __user __force *ubuf = (const char __user __force *)buf; - int at_start, in_range, in_partial_range; - - totaldigits = c = 0; - old_a = old_b = 0; - group_size = used_size = 0; - bitmap_zero(maskp, nmaskbits); - do { - at_start = 1; - in_range = 0; - in_partial_range = 0; - a = b = 0; - ndigits = totaldigits; - - /* Get the next cpu# or a range of cpu#'s */ - while (buflen) { - old_c = c; - if (is_user) { - if (__get_user(c, ubuf++)) - return -EFAULT; - } else - c = *buf++; - buflen--; - if (isspace(c)) - continue; - - /* A '\0' or a ',' signal the end of a cpu# or range */ - if (c == '\0' || c == ',') - break; - /* - * whitespaces between digits are not allowed, - * but it's ok if whitespaces are on head or tail. - * when old_c is whilespace, - * if totaldigits == ndigits, whitespace is on head. - * if whitespace is on tail, it should not run here. - * as c was ',' or '\0', - * the last code line has broken the current loop. - */ - if ((totaldigits != ndigits) && isspace(old_c)) - return -EINVAL; - - if (c == '/') { - used_size = a; - at_start = 1; - in_range = 0; - a = b = 0; - continue; - } - - if (c == ':') { - old_a = a; - old_b = b; - at_start = 1; - in_range = 0; - in_partial_range = 1; - a = b = 0; - continue; - } - - if (c == '-') { - if (at_start || in_range) - return -EINVAL; - b = 0; - in_range = 1; - at_start = 1; - continue; - } - - if (!isdigit(c)) - return -EINVAL; - - b = b * 10 + (c - '0'); - if (!in_range) - a = b; - at_start = 0; - totaldigits++; - } - if (ndigits == totaldigits) - continue; - if (in_partial_range) { - group_size = a; - a = old_a; - b = old_b; - old_a = old_b = 0; - } - /* if no digit is after '-', it's wrong*/ - if (at_start && in_range) - return -EINVAL; - if (!(a <= b) || !(used_size <= group_size)) - return -EINVAL; - if (b >= nmaskbits) - return -ERANGE; - while (a <= b) { - if (in_partial_range) { - static int pos_in_group = 1; - - if (pos_in_group <= used_size) - set_bit(a, maskp); - - if (a == b || ++pos_in_group > group_size) - pos_in_group = 1; - } else - set_bit(a, maskp); - a++; - } - } while (buflen && c == ','); - return 0; -} - -int bitmap_parselist(const char *bp, unsigned long *maskp, int nmaskbits) -{ - char *nl = strchrnul(bp, '\n'); - int len = nl - bp; - - return __bitmap_parselist(bp, len, 0, maskp, nmaskbits); -} -EXPORT_SYMBOL(bitmap_parselist); - - -/** - * bitmap_parselist_user() - * - * @ubuf: pointer to user buffer containing string. - * @ulen: buffer size in bytes. If string is smaller than this - * then it must be terminated with a \0. - * @maskp: pointer to bitmap array that will contain result. - * @nmaskbits: size of bitmap, in bits. - * - * Wrapper for bitmap_parselist(), providing it with user buffer. - * - * We cannot have this as an inline function in bitmap.h because it needs - * linux/uaccess.h to get the access_ok() declaration and this causes - * cyclic dependencies. - */ -int bitmap_parselist_user(const char __user *ubuf, - unsigned int ulen, unsigned long *maskp, - int nmaskbits) -{ - if (!access_ok(VERIFY_READ, ubuf, ulen)) - return -EFAULT; - return __bitmap_parselist((const char __force *)ubuf, - ulen, 1, maskp, nmaskbits); -} -EXPORT_SYMBOL(bitmap_parselist_user); - - /** * bitmap_pos_to_ord - find ordinal of set bit at given position in bitmap * @buf: pointer to a bitmap @@ -684,37 +465,7 @@ static int bitmap_pos_to_ord(const unsigned long *buf, unsigned int pos, unsigne if (pos >= nbits || !test_bit(pos, buf)) return -1; - return __bitmap_weight(buf, pos); -} - -/** - * bitmap_ord_to_pos - find position of n-th set bit in bitmap - * @buf: pointer to bitmap - * @ord: ordinal bit position (n-th set bit, n >= 0) - * @nbits: number of valid bit positions in @buf - * - * Map the ordinal offset of bit @ord in @buf to its position in @buf. - * Value of @ord should be in range 0 <= @ord < weight(buf). If @ord - * >= weight(buf), returns @nbits. - * - * If for example, just bits 4 through 7 are set in @buf, then @ord - * values 0 through 3 will get mapped to 4 through 7, respectively, - * and all other @ord values returns @nbits. When @ord value 3 - * gets mapped to (returns) @pos value 7 in this example, that means - * that the 3rd set bit (starting with 0th) is at position 7 in @buf. - * - * The bit positions 0 through @nbits-1 are valid positions in @buf. - */ -unsigned int bitmap_ord_to_pos(const unsigned long *buf, unsigned int ord, unsigned int nbits) -{ - unsigned int pos; - - for (pos = find_first_bit(buf, nbits); - pos < nbits && ord; - pos = find_next_bit(buf, nbits, pos + 1)) - ord--; - - return pos; + return bitmap_weight(buf, pos); } /** @@ -737,7 +488,7 @@ unsigned int bitmap_ord_to_pos(const unsigned long *buf, unsigned int ord, unsig * to @dst. * * The positions of unset bits in @old are mapped to themselves - * (the identify map). + * (the identity map). * * Apply the above specified mapping to @src, placing the result in * @dst, clearing any bits previously set in @dst. @@ -766,7 +517,7 @@ void bitmap_remap(unsigned long *dst, const unsigned long *src, if (n < 0 || w == 0) set_bit(oldbit, dst); /* identity map */ else - set_bit(bitmap_ord_to_pos(new, n % w, nbits), dst); + set_bit(find_nth_bit(new, nbits, n % w), dst); } } EXPORT_SYMBOL(bitmap_remap); @@ -786,7 +537,7 @@ EXPORT_SYMBOL(bitmap_remap); * the position of the m-th set bit in @new, where m == n % w. * * The positions of unset bits in @old are mapped to themselves - * (the identify map). + * (the identity map). * * Apply the above specified mapping to bit position @oldbit, returning * the new bit position. @@ -805,10 +556,11 @@ int bitmap_bitremap(int oldbit, const unsigned long *old, if (n < 0 || w == 0) return oldbit; else - return bitmap_ord_to_pos(new, n % w, bits); + return find_nth_bit(new, bits, n % w); } EXPORT_SYMBOL(bitmap_bitremap); +#ifdef CONFIG_NUMA /** * bitmap_onto - translate one bitmap relative to another * @dst: resulting translated bitmap @@ -928,7 +680,7 @@ void bitmap_onto(unsigned long *dst, const unsigned long *orig, * The following code is a more efficient, but less * obvious, equivalent to the loop: * for (m = 0; m < bitmap_weight(relmap, bits); m++) { - * n = bitmap_ord_to_pos(orig, m, bits); + * n = find_nth_bit(orig, bits, m); * if (test_bit(m, orig)) * set_bit(n, dst); * } @@ -942,7 +694,6 @@ void bitmap_onto(unsigned long *dst, const unsigned long *orig, m++; } } -EXPORT_SYMBOL(bitmap_onto); /** * bitmap_fold - fold larger bitmap into smaller, modulo specified size @@ -967,254 +718,171 @@ void bitmap_fold(unsigned long *dst, const unsigned long *orig, for_each_set_bit(oldbit, orig, nbits) set_bit(oldbit % sz, dst); } -EXPORT_SYMBOL(bitmap_fold); +#endif /* CONFIG_NUMA */ -/* - * Common code for bitmap_*_region() routines. - * bitmap: array of unsigned longs corresponding to the bitmap - * pos: the beginning of the region - * order: region size (log base 2 of number of bits) - * reg_op: operation(s) to perform on that region of bitmap - * - * Can set, verify and/or release a region of bits in a bitmap, - * depending on which combination of REG_OP_* flag bits is set. - * - * A region of a bitmap is a sequence of bits in the bitmap, of - * some size '1 << order' (a power of two), aligned to that same - * '1 << order' power of two. - * - * Returns 1 if REG_OP_ISFREE succeeds (region is all zero bits). - * Returns 0 in all other cases and reg_ops. - */ +unsigned long *bitmap_alloc(unsigned int nbits, gfp_t flags) +{ + return kmalloc_array(BITS_TO_LONGS(nbits), sizeof(unsigned long), + flags); +} +EXPORT_SYMBOL(bitmap_alloc); -enum { - REG_OP_ISFREE, /* true if region is all zero bits */ - REG_OP_ALLOC, /* set all bits in region */ - REG_OP_RELEASE, /* clear all bits in region */ -}; +unsigned long *bitmap_zalloc(unsigned int nbits, gfp_t flags) +{ + return bitmap_alloc(nbits, flags | __GFP_ZERO); +} +EXPORT_SYMBOL(bitmap_zalloc); -static int __reg_op(unsigned long *bitmap, unsigned int pos, int order, int reg_op) +unsigned long *bitmap_alloc_node(unsigned int nbits, gfp_t flags, int node) { - int nbits_reg; /* number of bits in region */ - int index; /* index first long of region in bitmap */ - int offset; /* bit offset region in bitmap[index] */ - int nlongs_reg; /* num longs spanned by region in bitmap */ - int nbitsinlong; /* num bits of region in each spanned long */ - unsigned long mask; /* bitmask for one long of region */ - int i; /* scans bitmap by longs */ - int ret = 0; /* return value */ + return kmalloc_array_node(BITS_TO_LONGS(nbits), sizeof(unsigned long), + flags, node); +} +EXPORT_SYMBOL(bitmap_alloc_node); - /* - * Either nlongs_reg == 1 (for small orders that fit in one long) - * or (offset == 0 && mask == ~0UL) (for larger multiword orders.) - */ - nbits_reg = 1 << order; - index = pos / BITS_PER_LONG; - offset = pos - (index * BITS_PER_LONG); - nlongs_reg = BITS_TO_LONGS(nbits_reg); - nbitsinlong = min(nbits_reg, BITS_PER_LONG); +unsigned long *bitmap_zalloc_node(unsigned int nbits, gfp_t flags, int node) +{ + return bitmap_alloc_node(nbits, flags | __GFP_ZERO, node); +} +EXPORT_SYMBOL(bitmap_zalloc_node); - /* - * Can't do "mask = (1UL << nbitsinlong) - 1", as that - * overflows if nbitsinlong == BITS_PER_LONG. - */ - mask = (1UL << (nbitsinlong - 1)); - mask += mask - 1; - mask <<= offset; - - switch (reg_op) { - case REG_OP_ISFREE: - for (i = 0; i < nlongs_reg; i++) { - if (bitmap[index + i] & mask) - goto done; - } - ret = 1; /* all bits in region free (zero) */ - break; - - case REG_OP_ALLOC: - for (i = 0; i < nlongs_reg; i++) - bitmap[index + i] |= mask; - break; - - case REG_OP_RELEASE: - for (i = 0; i < nlongs_reg; i++) - bitmap[index + i] &= ~mask; - break; - } -done: - return ret; +void bitmap_free(const unsigned long *bitmap) +{ + kfree(bitmap); } +EXPORT_SYMBOL(bitmap_free); -/** - * bitmap_find_free_region - find a contiguous aligned mem region - * @bitmap: array of unsigned longs corresponding to the bitmap - * @bits: number of bits in the bitmap - * @order: region size (log base 2 of number of bits) to find - * - * Find a region of free (zero) bits in a @bitmap of @bits bits and - * allocate them (set them to one). Only consider regions of length - * a power (@order) of two, aligned to that power of two, which - * makes the search algorithm much faster. - * - * Return the bit offset in bitmap of the allocated region, - * or -errno on failure. - */ -int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order) +static void devm_bitmap_free(void *data) { - unsigned int pos, end; /* scans bitmap by regions of size order */ + unsigned long *bitmap = data; - for (pos = 0 ; (end = pos + (1U << order)) <= bits; pos = end) { - if (!__reg_op(bitmap, pos, order, REG_OP_ISFREE)) - continue; - __reg_op(bitmap, pos, order, REG_OP_ALLOC); - return pos; - } - return -ENOMEM; + bitmap_free(bitmap); } -EXPORT_SYMBOL(bitmap_find_free_region); -/** - * bitmap_release_region - release allocated bitmap region - * @bitmap: array of unsigned longs corresponding to the bitmap - * @pos: beginning of bit region to release - * @order: region size (log base 2 of number of bits) to release - * - * This is the complement to __bitmap_find_free_region() and releases - * the found region (by clearing it in the bitmap). - * - * No return value. - */ -void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order) +unsigned long *devm_bitmap_alloc(struct device *dev, + unsigned int nbits, gfp_t flags) { - __reg_op(bitmap, pos, order, REG_OP_RELEASE); + unsigned long *bitmap; + int ret; + + bitmap = bitmap_alloc(nbits, flags); + if (!bitmap) + return NULL; + + ret = devm_add_action_or_reset(dev, devm_bitmap_free, bitmap); + if (ret) + return NULL; + + return bitmap; } -EXPORT_SYMBOL(bitmap_release_region); +EXPORT_SYMBOL_GPL(devm_bitmap_alloc); -/** - * bitmap_allocate_region - allocate bitmap region - * @bitmap: array of unsigned longs corresponding to the bitmap - * @pos: beginning of bit region to allocate - * @order: region size (log base 2 of number of bits) to allocate - * - * Allocate (set bits in) a specified region of a bitmap. - * - * Return 0 on success, or %-EBUSY if specified region wasn't - * free (not all bits were zero). - */ -int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order) +unsigned long *devm_bitmap_zalloc(struct device *dev, + unsigned int nbits, gfp_t flags) { - if (!__reg_op(bitmap, pos, order, REG_OP_ISFREE)) - return -EBUSY; - return __reg_op(bitmap, pos, order, REG_OP_ALLOC); + return devm_bitmap_alloc(dev, nbits, flags | __GFP_ZERO); } -EXPORT_SYMBOL(bitmap_allocate_region); +EXPORT_SYMBOL_GPL(devm_bitmap_zalloc); +#if BITS_PER_LONG == 64 /** - * bitmap_from_u32array - copy the contents of a u32 array of bits to bitmap - * @bitmap: array of unsigned longs, the destination bitmap, non NULL + * bitmap_from_arr32 - copy the contents of u32 array of bits to bitmap + * @bitmap: array of unsigned longs, the destination bitmap + * @buf: array of u32 (in host byte order), the source bitmap * @nbits: number of bits in @bitmap - * @buf: array of u32 (in host byte order), the source bitmap, non NULL - * @nwords: number of u32 words in @buf - * - * copy min(nbits, 32*nwords) bits from @buf to @bitmap, remaining - * bits between nword and nbits in @bitmap (if any) are cleared. In - * last word of @bitmap, the bits beyond nbits (if any) are kept - * unchanged. - * - * Return the number of bits effectively copied. */ -unsigned int -bitmap_from_u32array(unsigned long *bitmap, unsigned int nbits, - const u32 *buf, unsigned int nwords) +void bitmap_from_arr32(unsigned long *bitmap, const u32 *buf, unsigned int nbits) { - unsigned int dst_idx, src_idx; - - for (src_idx = dst_idx = 0; dst_idx < BITS_TO_LONGS(nbits); ++dst_idx) { - unsigned long part = 0; + unsigned int i, halfwords; - if (src_idx < nwords) - part = buf[src_idx++]; + halfwords = DIV_ROUND_UP(nbits, 32); + for (i = 0; i < halfwords; i++) { + bitmap[i/2] = (unsigned long) buf[i]; + if (++i < halfwords) + bitmap[i/2] |= ((unsigned long) buf[i]) << 32; + } -#if BITS_PER_LONG == 64 - if (src_idx < nwords) - part |= ((unsigned long) buf[src_idx++]) << 32; -#endif + /* Clear tail bits in last word beyond nbits. */ + if (nbits % BITS_PER_LONG) + bitmap[(halfwords - 1) / 2] &= BITMAP_LAST_WORD_MASK(nbits); +} +EXPORT_SYMBOL(bitmap_from_arr32); - if (dst_idx < nbits/BITS_PER_LONG) - bitmap[dst_idx] = part; - else { - unsigned long mask = BITMAP_LAST_WORD_MASK(nbits); +/** + * bitmap_to_arr32 - copy the contents of bitmap to a u32 array of bits + * @buf: array of u32 (in host byte order), the dest bitmap + * @bitmap: array of unsigned longs, the source bitmap + * @nbits: number of bits in @bitmap + */ +void bitmap_to_arr32(u32 *buf, const unsigned long *bitmap, unsigned int nbits) +{ + unsigned int i, halfwords; - bitmap[dst_idx] = (bitmap[dst_idx] & ~mask) - | (part & mask); - } + halfwords = DIV_ROUND_UP(nbits, 32); + for (i = 0; i < halfwords; i++) { + buf[i] = (u32) (bitmap[i/2] & UINT_MAX); + if (++i < halfwords) + buf[i] = (u32) (bitmap[i/2] >> 32); } - return min_t(unsigned int, nbits, 32*nwords); + /* Clear tail bits in last element of array beyond nbits. */ + if (nbits % BITS_PER_LONG) + buf[halfwords - 1] &= (u32) (UINT_MAX >> ((-nbits) & 31)); } -EXPORT_SYMBOL(bitmap_from_u32array); +EXPORT_SYMBOL(bitmap_to_arr32); +#endif +#if BITS_PER_LONG == 32 /** - * bitmap_to_u32array - copy the contents of bitmap to a u32 array of bits - * @buf: array of u32 (in host byte order), the dest bitmap, non NULL - * @nwords: number of u32 words in @buf - * @bitmap: array of unsigned longs, the source bitmap, non NULL + * bitmap_from_arr64 - copy the contents of u64 array of bits to bitmap + * @bitmap: array of unsigned longs, the destination bitmap + * @buf: array of u64 (in host byte order), the source bitmap * @nbits: number of bits in @bitmap - * - * copy min(nbits, 32*nwords) bits from @bitmap to @buf. Remaining - * bits after nbits in @buf (if any) are cleared. - * - * Return the number of bits effectively copied. */ -unsigned int -bitmap_to_u32array(u32 *buf, unsigned int nwords, - const unsigned long *bitmap, unsigned int nbits) +void bitmap_from_arr64(unsigned long *bitmap, const u64 *buf, unsigned int nbits) { - unsigned int dst_idx = 0, src_idx = 0; + int n; - while (dst_idx < nwords) { - unsigned long part = 0; + for (n = nbits; n > 0; n -= 64) { + u64 val = *buf++; - if (src_idx < BITS_TO_LONGS(nbits)) { - part = bitmap[src_idx]; - if (src_idx >= nbits/BITS_PER_LONG) - part &= BITMAP_LAST_WORD_MASK(nbits); - src_idx++; - } - - buf[dst_idx++] = part & 0xffffffffUL; - -#if BITS_PER_LONG == 64 - if (dst_idx < nwords) { - part >>= 32; - buf[dst_idx++] = part & 0xffffffffUL; - } -#endif + *bitmap++ = val; + if (n > 32) + *bitmap++ = val >> 32; } - return min_t(unsigned int, nbits, 32*nwords); + /* + * Clear tail bits in the last word beyond nbits. + * + * Negative index is OK because here we point to the word next + * to the last word of the bitmap, except for nbits == 0, which + * is tested implicitly. + */ + if (nbits % BITS_PER_LONG) + bitmap[-1] &= BITMAP_LAST_WORD_MASK(nbits); } -EXPORT_SYMBOL(bitmap_to_u32array); +EXPORT_SYMBOL(bitmap_from_arr64); /** - * bitmap_copy_le - copy a bitmap, putting the bits into little-endian order. - * @dst: destination buffer - * @src: bitmap to copy - * @nbits: number of bits in the bitmap - * - * Require nbits % BITS_PER_LONG == 0. + * bitmap_to_arr64 - copy the contents of bitmap to a u64 array of bits + * @buf: array of u64 (in host byte order), the dest bitmap + * @bitmap: array of unsigned longs, the source bitmap + * @nbits: number of bits in @bitmap */ -#ifdef __BIG_ENDIAN -void bitmap_copy_le(unsigned long *dst, const unsigned long *src, unsigned int nbits) +void bitmap_to_arr64(u64 *buf, const unsigned long *bitmap, unsigned int nbits) { - unsigned int i; + const unsigned long *end = bitmap + BITS_TO_LONGS(nbits); - for (i = 0; i < nbits/BITS_PER_LONG; i++) { - if (BITS_PER_LONG == 64) - dst[i] = cpu_to_le64(src[i]); - else - dst[i] = cpu_to_le32(src[i]); + while (bitmap < end) { + *buf = *bitmap++; + if (bitmap < end) + *buf |= (u64)(*bitmap++) << 32; + buf++; } + + /* Clear tail bits in the last element of array beyond nbits. */ + if (nbits % 64) + buf[-1] &= GENMASK_ULL((nbits - 1) % 64, 0); } -EXPORT_SYMBOL(bitmap_copy_le); +EXPORT_SYMBOL(bitmap_to_arr64); #endif |