diff options
Diffstat (limited to 'mm')
| -rw-r--r-- | mm/kasan/kasan.h | 1 | ||||
| -rw-r--r-- | mm/mempolicy.c | 18 | ||||
| -rw-r--r-- | mm/mmap.c | 12 | ||||
| -rw-r--r-- | mm/nommu.c | 5 | ||||
| -rw-r--r-- | mm/percpu.c | 35 | ||||
| -rw-r--r-- | mm/readahead.c | 3 | ||||
| -rw-r--r-- | mm/shmem.c | 22 | ||||
| -rw-r--r-- | mm/slab_common.c | 48 | ||||
| -rw-r--r-- | mm/slub.c | 73 | ||||
| -rw-r--r-- | mm/swapfile.c | 23 | ||||
| -rw-r--r-- | mm/util.c | 4 |
11 files changed, 124 insertions, 120 deletions
diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h index d37831b8511c..8b06bab5c406 100644 --- a/mm/kasan/kasan.h +++ b/mm/kasan/kasan.h @@ -562,7 +562,6 @@ void kasan_restore_multi_shot(bool enabled); * code. Declared here to avoid warnings about missing declarations. */ -asmlinkage void kasan_unpoison_task_stack_below(const void *watermark); void __asan_register_globals(void *globals, ssize_t size); void __asan_unregister_globals(void *globals, ssize_t size); void __asan_handle_no_return(void); diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 5e472e6e0507..10a590ee1c89 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -132,22 +132,26 @@ static struct mempolicy default_policy = { static struct mempolicy preferred_node_policy[MAX_NUMNODES]; /** - * numa_map_to_online_node - Find closest online node + * numa_nearest_node - Find nearest node by state * @node: Node id to start the search + * @state: State to filter the search * - * Lookup the next closest node by distance if @nid is not online. + * Lookup the closest node by distance if @nid is not in state. * - * Return: this @node if it is online, otherwise the closest node by distance + * Return: this @node if it is in state, otherwise the closest node by distance */ -int numa_map_to_online_node(int node) +int numa_nearest_node(int node, unsigned int state) { int min_dist = INT_MAX, dist, n, min_node; - if (node == NUMA_NO_NODE || node_online(node)) + if (state >= NR_NODE_STATES) + return -EINVAL; + + if (node == NUMA_NO_NODE || node_state(node, state)) return node; min_node = node; - for_each_online_node(n) { + for_each_node_state(n, state) { dist = node_distance(node, n); if (dist < min_dist) { min_dist = dist; @@ -157,7 +161,7 @@ int numa_map_to_online_node(int node) return min_node; } -EXPORT_SYMBOL_GPL(numa_map_to_online_node); +EXPORT_SYMBOL_GPL(numa_nearest_node); struct mempolicy *get_task_policy(struct task_struct *p) { diff --git a/mm/mmap.c b/mm/mmap.c index 984804d77ae1..1971bfffcc03 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1949,9 +1949,9 @@ static int acct_stack_growth(struct vm_area_struct *vma, return 0; } -#if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64) +#if defined(CONFIG_STACK_GROWSUP) /* - * PA-RISC uses this for its stack; IA64 for its Register Backing Store. + * PA-RISC uses this for its stack. * vma is the last one with address > vma->vm_end. Have to extend vma. */ static int expand_upwards(struct vm_area_struct *vma, unsigned long address) @@ -2048,7 +2048,7 @@ static int expand_upwards(struct vm_area_struct *vma, unsigned long address) validate_mm(mm); return error; } -#endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */ +#endif /* CONFIG_STACK_GROWSUP */ /* * vma is the first one with address < vma->vm_start. Have to extend vma. @@ -3278,12 +3278,6 @@ limits_failed: } EXPORT_SYMBOL(vm_brk_flags); -int vm_brk(unsigned long addr, unsigned long len) -{ - return vm_brk_flags(addr, len, 0); -} -EXPORT_SYMBOL(vm_brk); - /* Release all mmaps. */ void exit_mmap(struct mm_struct *mm) { diff --git a/mm/nommu.c b/mm/nommu.c index fc4afe924ad5..b6dc558d3144 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1531,11 +1531,6 @@ void exit_mmap(struct mm_struct *mm) mmap_write_unlock(mm); } -int vm_brk(unsigned long addr, unsigned long len) -{ - return -ENOMEM; -} - /* * expand (or shrink) an existing mapping, potentially moving it at the same * time (controlled by the MREMAP_MAYMOVE flag and available VM space) diff --git a/mm/percpu.c b/mm/percpu.c index f53ba692d67a..7b97d31df767 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -2243,6 +2243,37 @@ static void pcpu_balance_workfn(struct work_struct *work) } /** + * pcpu_alloc_size - the size of the dynamic percpu area + * @ptr: pointer to the dynamic percpu area + * + * Returns the size of the @ptr allocation. This is undefined for statically + * defined percpu variables as there is no corresponding chunk->bound_map. + * + * RETURNS: + * The size of the dynamic percpu area. + * + * CONTEXT: + * Can be called from atomic context. + */ +size_t pcpu_alloc_size(void __percpu *ptr) +{ + struct pcpu_chunk *chunk; + unsigned long bit_off, end; + void *addr; + + if (!ptr) + return 0; + + addr = __pcpu_ptr_to_addr(ptr); + /* No pcpu_lock here: ptr has not been freed, so chunk is still alive */ + chunk = pcpu_chunk_addr_search(addr); + bit_off = (addr - chunk->base_addr) / PCPU_MIN_ALLOC_SIZE; + end = find_next_bit(chunk->bound_map, pcpu_chunk_map_bits(chunk), + bit_off + 1); + return (end - bit_off) * PCPU_MIN_ALLOC_SIZE; +} + +/** * free_percpu - free percpu area * @ptr: pointer to area to free * @@ -2265,12 +2296,10 @@ void free_percpu(void __percpu *ptr) kmemleak_free_percpu(ptr); addr = __pcpu_ptr_to_addr(ptr); - - spin_lock_irqsave(&pcpu_lock, flags); - chunk = pcpu_chunk_addr_search(addr); off = addr - chunk->base_addr; + spin_lock_irqsave(&pcpu_lock, flags); size = pcpu_free_area(chunk, off); pcpu_memcg_free_hook(chunk, off, size); diff --git a/mm/readahead.c b/mm/readahead.c index e815c114de21..6925e6959fd3 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -735,7 +735,8 @@ ssize_t ksys_readahead(int fd, loff_t offset, size_t count) */ ret = -EINVAL; if (!f.file->f_mapping || !f.file->f_mapping->a_ops || - !S_ISREG(file_inode(f.file)->i_mode)) + (!S_ISREG(file_inode(f.file)->i_mode) && + !S_ISBLK(file_inode(f.file)->i_mode))) goto out; ret = vfs_fadvise(f.file, offset, count, POSIX_FADV_WILLNEED); diff --git a/mm/shmem.c b/mm/shmem.c index a314a25aea8c..71b8d957b63b 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1094,7 +1094,7 @@ whole_folios: void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) { shmem_undo_range(inode, lstart, lend, false); - inode->i_mtime = inode_set_ctime_current(inode); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); inode_inc_iversion(inode); } EXPORT_SYMBOL_GPL(shmem_truncate_range); @@ -1205,7 +1205,7 @@ static int shmem_setattr(struct mnt_idmap *idmap, if (!error && update_ctime) { inode_set_ctime_current(inode); if (update_mtime) - inode->i_mtime = inode_get_ctime(inode); + inode_set_mtime_to_ts(inode, inode_get_ctime(inode)); inode_inc_iversion(inode); } return error; @@ -2468,7 +2468,7 @@ static struct inode *__shmem_get_inode(struct mnt_idmap *idmap, inode->i_ino = ino; inode_init_owner(idmap, inode, dir, mode); inode->i_blocks = 0; - inode->i_atime = inode->i_mtime = inode_set_ctime_current(inode); + simple_inode_init_ts(inode); inode->i_generation = get_random_u32(); info = SHMEM_I(inode); memset(info, 0, (char *)inode - (char *)info); @@ -2476,7 +2476,7 @@ static struct inode *__shmem_get_inode(struct mnt_idmap *idmap, atomic_set(&info->stop_eviction, 0); info->seals = F_SEAL_SEAL; info->flags = flags & VM_NORESERVE; - info->i_crtime = inode->i_mtime; + info->i_crtime = inode_get_mtime(inode); info->fsflags = (dir == NULL) ? 0 : SHMEM_I(dir)->fsflags & SHMEM_FL_INHERITED; if (info->fsflags) @@ -3241,7 +3241,7 @@ shmem_mknod(struct mnt_idmap *idmap, struct inode *dir, goto out_iput; dir->i_size += BOGO_DIRENT_SIZE; - dir->i_mtime = inode_set_ctime_current(dir); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); inode_inc_iversion(dir); d_instantiate(dentry, inode); dget(dentry); /* Extra count - pin the dentry in core */ @@ -3328,8 +3328,8 @@ static int shmem_link(struct dentry *old_dentry, struct inode *dir, } dir->i_size += BOGO_DIRENT_SIZE; - dir->i_mtime = inode_set_ctime_to_ts(dir, - inode_set_ctime_current(inode)); + inode_set_mtime_to_ts(dir, + inode_set_ctime_to_ts(dir, inode_set_ctime_current(inode))); inode_inc_iversion(dir); inc_nlink(inode); ihold(inode); /* New dentry reference */ @@ -3349,8 +3349,8 @@ static int shmem_unlink(struct inode *dir, struct dentry *dentry) simple_offset_remove(shmem_get_offset_ctx(dir), dentry); dir->i_size -= BOGO_DIRENT_SIZE; - dir->i_mtime = inode_set_ctime_to_ts(dir, - inode_set_ctime_current(inode)); + inode_set_mtime_to_ts(dir, + inode_set_ctime_to_ts(dir, inode_set_ctime_current(inode))); inode_inc_iversion(dir); drop_nlink(inode); dput(dentry); /* Undo the count from "create" - does all the work */ @@ -3497,7 +3497,7 @@ static int shmem_symlink(struct mnt_idmap *idmap, struct inode *dir, folio_put(folio); } dir->i_size += BOGO_DIRENT_SIZE; - dir->i_mtime = inode_set_ctime_current(dir); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); inode_inc_iversion(dir); d_instantiate(dentry, inode); dget(dentry); @@ -3721,7 +3721,7 @@ static const struct xattr_handler shmem_user_xattr_handler = { .set = shmem_xattr_handler_set, }; -static const struct xattr_handler *shmem_xattr_handlers[] = { +static const struct xattr_handler * const shmem_xattr_handlers[] = { &shmem_security_xattr_handler, &shmem_trusted_xattr_handler, &shmem_user_xattr_handler, diff --git a/mm/slab_common.c b/mm/slab_common.c index 8fda308e400d..8d431193c273 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -528,26 +528,6 @@ bool slab_is_available(void) } #ifdef CONFIG_PRINTK -/** - * kmem_valid_obj - does the pointer reference a valid slab object? - * @object: pointer to query. - * - * Return: %true if the pointer is to a not-yet-freed object from - * kmalloc() or kmem_cache_alloc(), either %true or %false if the pointer - * is to an already-freed object, and %false otherwise. - */ -bool kmem_valid_obj(void *object) -{ - struct folio *folio; - - /* Some arches consider ZERO_SIZE_PTR to be a valid address. */ - if (object < (void *)PAGE_SIZE || !virt_addr_valid(object)) - return false; - folio = virt_to_folio(object); - return folio_test_slab(folio); -} -EXPORT_SYMBOL_GPL(kmem_valid_obj); - static void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) { if (__kfence_obj_info(kpp, object, slab)) @@ -566,11 +546,11 @@ static void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab * * and, if available, the slab name, return address, and stack trace from * the allocation and last free path of that object. * - * This function will splat if passed a pointer to a non-slab object. - * If you are not sure what type of object you have, you should instead - * use mem_dump_obj(). + * Return: %true if the pointer is to a not-yet-freed object from + * kmalloc() or kmem_cache_alloc(), either %true or %false if the pointer + * is to an already-freed object, and %false otherwise. */ -void kmem_dump_obj(void *object) +bool kmem_dump_obj(void *object) { char *cp = IS_ENABLED(CONFIG_MMU) ? "" : "/vmalloc"; int i; @@ -578,13 +558,13 @@ void kmem_dump_obj(void *object) unsigned long ptroffset; struct kmem_obj_info kp = { }; - if (WARN_ON_ONCE(!virt_addr_valid(object))) - return; + /* Some arches consider ZERO_SIZE_PTR to be a valid address. */ + if (object < (void *)PAGE_SIZE || !virt_addr_valid(object)) + return false; slab = virt_to_slab(object); - if (WARN_ON_ONCE(!slab)) { - pr_cont(" non-slab memory.\n"); - return; - } + if (!slab) + return false; + kmem_obj_info(&kp, object, slab); if (kp.kp_slab_cache) pr_cont(" slab%s %s", cp, kp.kp_slab_cache->name); @@ -621,6 +601,7 @@ void kmem_dump_obj(void *object) pr_info(" %pS\n", kp.kp_free_stack[i]); } + return true; } EXPORT_SYMBOL_GPL(kmem_dump_obj); #endif @@ -895,10 +876,13 @@ void __init setup_kmalloc_cache_index_table(void) static unsigned int __kmalloc_minalign(void) { + unsigned int minalign = dma_get_cache_alignment(); + if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && is_swiotlb_allocated()) - return ARCH_KMALLOC_MINALIGN; - return dma_get_cache_alignment(); + minalign = ARCH_KMALLOC_MINALIGN; + + return max(minalign, arch_slab_minalign()); } void __init diff --git a/mm/slub.c b/mm/slub.c index f7940048138c..63d281dfacdb 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4110,17 +4110,12 @@ static unsigned int slub_min_objects; * the smallest order which will fit the object. */ static inline unsigned int calc_slab_order(unsigned int size, - unsigned int min_objects, unsigned int max_order, + unsigned int min_order, unsigned int max_order, unsigned int fract_leftover) { - unsigned int min_order = slub_min_order; unsigned int order; - if (order_objects(min_order, size) > MAX_OBJS_PER_PAGE) - return get_order(size * MAX_OBJS_PER_PAGE) - 1; - - for (order = max(min_order, (unsigned int)get_order(min_objects * size)); - order <= max_order; order++) { + for (order = min_order; order <= max_order; order++) { unsigned int slab_size = (unsigned int)PAGE_SIZE << order; unsigned int rem; @@ -4139,16 +4134,8 @@ static inline int calculate_order(unsigned int size) unsigned int order; unsigned int min_objects; unsigned int max_objects; - unsigned int nr_cpus; + unsigned int min_order; - /* - * Attempt to find best configuration for a slab. This - * works by first attempting to generate a layout with - * the best configuration and backing off gradually. - * - * First we increase the acceptable waste in a slab. Then - * we reduce the minimum objects required in a slab. - */ min_objects = slub_min_objects; if (!min_objects) { /* @@ -4160,40 +4147,46 @@ static inline int calculate_order(unsigned int size) * order on systems that appear larger than they are, and too * low order on systems that appear smaller than they are. */ - nr_cpus = num_present_cpus(); + unsigned int nr_cpus = num_present_cpus(); if (nr_cpus <= 1) nr_cpus = nr_cpu_ids; min_objects = 4 * (fls(nr_cpus) + 1); } - max_objects = order_objects(slub_max_order, size); + /* min_objects can't be 0 because get_order(0) is undefined */ + max_objects = max(order_objects(slub_max_order, size), 1U); min_objects = min(min_objects, max_objects); - while (min_objects > 1) { - unsigned int fraction; - - fraction = 16; - while (fraction >= 4) { - order = calc_slab_order(size, min_objects, - slub_max_order, fraction); - if (order <= slub_max_order) - return order; - fraction /= 2; - } - min_objects--; - } + min_order = max_t(unsigned int, slub_min_order, + get_order(min_objects * size)); + if (order_objects(min_order, size) > MAX_OBJS_PER_PAGE) + return get_order(size * MAX_OBJS_PER_PAGE) - 1; /* - * We were unable to place multiple objects in a slab. Now - * lets see if we can place a single object there. + * Attempt to find best configuration for a slab. This works by first + * attempting to generate a layout with the best possible configuration + * and backing off gradually. + * + * We start with accepting at most 1/16 waste and try to find the + * smallest order from min_objects-derived/slub_min_order up to + * slub_max_order that will satisfy the constraint. Note that increasing + * the order can only result in same or less fractional waste, not more. + * + * If that fails, we increase the acceptable fraction of waste and try + * again. The last iteration with fraction of 1/2 would effectively + * accept any waste and give us the order determined by min_objects, as + * long as at least single object fits within slub_max_order. */ - order = calc_slab_order(size, 1, slub_max_order, 1); - if (order <= slub_max_order) - return order; + for (unsigned int fraction = 16; fraction > 1; fraction /= 2) { + order = calc_slab_order(size, min_order, slub_max_order, + fraction); + if (order <= slub_max_order) + return order; + } /* * Doh this slab cannot be placed using slub_max_order. */ - order = calc_slab_order(size, 1, MAX_ORDER, 1); + order = get_order(size); if (order <= MAX_ORDER) return order; return -ENOSYS; @@ -4711,6 +4704,9 @@ static int __init setup_slub_min_order(char *str) { get_option(&str, (int *)&slub_min_order); + if (slub_min_order > slub_max_order) + slub_max_order = slub_min_order; + return 1; } @@ -4721,6 +4717,9 @@ static int __init setup_slub_max_order(char *str) get_option(&str, (int *)&slub_max_order); slub_max_order = min_t(unsigned int, slub_max_order, MAX_ORDER); + if (slub_min_order > slub_max_order) + slub_min_order = slub_max_order; + return 1; } diff --git a/mm/swapfile.c b/mm/swapfile.c index e52f486834eb..4bc70f459164 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2530,11 +2530,10 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile) exit_swap_address_space(p->type); inode = mapping->host; - if (S_ISBLK(inode->i_mode)) { - struct block_device *bdev = I_BDEV(inode); - - set_blocksize(bdev, old_block_size); - blkdev_put(bdev, p); + if (p->bdev_handle) { + set_blocksize(p->bdev, old_block_size); + bdev_release(p->bdev_handle); + p->bdev_handle = NULL; } inode_lock(inode); @@ -2764,13 +2763,14 @@ static int claim_swapfile(struct swap_info_struct *p, struct inode *inode) int error; if (S_ISBLK(inode->i_mode)) { - p->bdev = blkdev_get_by_dev(inode->i_rdev, + p->bdev_handle = bdev_open_by_dev(inode->i_rdev, BLK_OPEN_READ | BLK_OPEN_WRITE, p, NULL); - if (IS_ERR(p->bdev)) { - error = PTR_ERR(p->bdev); - p->bdev = NULL; + if (IS_ERR(p->bdev_handle)) { + error = PTR_ERR(p->bdev_handle); + p->bdev_handle = NULL; return error; } + p->bdev = p->bdev_handle->bdev; p->old_block_size = block_size(p->bdev); error = set_blocksize(p->bdev, PAGE_SIZE); if (error < 0) @@ -3206,9 +3206,10 @@ bad_swap: p->percpu_cluster = NULL; free_percpu(p->cluster_next_cpu); p->cluster_next_cpu = NULL; - if (inode && S_ISBLK(inode->i_mode) && p->bdev) { + if (p->bdev_handle) { set_blocksize(p->bdev, p->old_block_size); - blkdev_put(p->bdev, p); + bdev_release(p->bdev_handle); + p->bdev_handle = NULL; } inode = NULL; destroy_swap_extents(p); diff --git a/mm/util.c b/mm/util.c index eefa0336d38c..aa01f6ea5a75 100644 --- a/mm/util.c +++ b/mm/util.c @@ -1061,10 +1061,8 @@ void mem_dump_obj(void *object) { const char *type; - if (kmem_valid_obj(object)) { - kmem_dump_obj(object); + if (kmem_dump_obj(object)) return; - } if (vmalloc_dump_obj(object)) return; |