diff options
Diffstat (limited to 'drivers/base/node.c')
| -rw-r--r-- | drivers/base/node.c | 512 |
1 files changed, 225 insertions, 287 deletions
diff --git a/drivers/base/node.c b/drivers/base/node.c index 4a4ae868ad9f..00cf4532f121 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -7,6 +7,7 @@ #include <linux/init.h> #include <linux/mm.h> #include <linux/memory.h> +#include <linux/mempolicy.h> #include <linux/vmstat.h> #include <linux/notifier.h> #include <linux/node.h> @@ -20,49 +21,54 @@ #include <linux/pm_runtime.h> #include <linux/swap.h> #include <linux/slab.h> +#include <linux/memblock.h> -static struct bus_type node_subsys = { +static const struct bus_type node_subsys = { .name = "node", .dev_name = "node", }; - -static ssize_t node_read_cpumap(struct device *dev, bool list, char *buf) +static inline ssize_t cpumap_read(struct file *file, struct kobject *kobj, + const struct bin_attribute *attr, char *buf, + loff_t off, size_t count) { - ssize_t n; - cpumask_var_t mask; + struct device *dev = kobj_to_dev(kobj); struct node *node_dev = to_node(dev); - - /* 2008/04/07: buf currently PAGE_SIZE, need 9 chars per 32 bits. */ - BUILD_BUG_ON((NR_CPUS/32 * 9) > (PAGE_SIZE-1)); + cpumask_var_t mask; + ssize_t n; if (!alloc_cpumask_var(&mask, GFP_KERNEL)) return 0; cpumask_and(mask, cpumask_of_node(node_dev->dev.id), cpu_online_mask); - n = cpumap_print_to_pagebuf(list, buf, mask); + n = cpumap_print_bitmask_to_buf(buf, mask, off, count); free_cpumask_var(mask); return n; } -static inline ssize_t cpumap_show(struct device *dev, - struct device_attribute *attr, - char *buf) +static const BIN_ATTR_RO(cpumap, CPUMAP_FILE_MAX_BYTES); + +static inline ssize_t cpulist_read(struct file *file, struct kobject *kobj, + const struct bin_attribute *attr, char *buf, + loff_t off, size_t count) { - return node_read_cpumap(dev, false, buf); -} + struct device *dev = kobj_to_dev(kobj); + struct node *node_dev = to_node(dev); + cpumask_var_t mask; + ssize_t n; -static DEVICE_ATTR_RO(cpumap); + if (!alloc_cpumask_var(&mask, GFP_KERNEL)) + return 0; -static inline ssize_t cpulist_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - return node_read_cpumap(dev, true, buf); + cpumask_and(mask, cpumask_of_node(node_dev->dev.id), cpu_online_mask); + n = cpumap_print_list_to_buf(buf, mask, off, count); + free_cpumask_var(mask); + + return n; } -static DEVICE_ATTR_RO(cpulist); +static const BIN_ATTR_RO(cpulist, CPULIST_FILE_MAX_BYTES); /** * struct node_access_nodes - Access class device to hold user visible @@ -70,14 +76,14 @@ static DEVICE_ATTR_RO(cpulist); * @dev: Device for this memory access class * @list_node: List element in the node's access list * @access: The access class rank - * @hmem_attrs: Heterogeneous memory performance attributes + * @coord: Heterogeneous memory performance coordinates */ struct node_access_nodes { struct device dev; struct list_head list_node; - unsigned access; + unsigned int access; #ifdef CONFIG_HMEM_REPORTING - struct node_hmem_attrs hmem_attrs; + struct access_coordinate coord; #endif }; #define to_access_nodes(dev) container_of(dev, struct node_access_nodes, dev) @@ -106,6 +112,27 @@ static const struct attribute_group *node_access_node_groups[] = { NULL, }; +#ifdef CONFIG_MEMORY_HOTPLUG +static BLOCKING_NOTIFIER_HEAD(node_chain); + +int register_node_notifier(struct notifier_block *nb) +{ + return blocking_notifier_chain_register(&node_chain, nb); +} +EXPORT_SYMBOL(register_node_notifier); + +void unregister_node_notifier(struct notifier_block *nb) +{ + blocking_notifier_chain_unregister(&node_chain, nb); +} +EXPORT_SYMBOL(unregister_node_notifier); + +int node_notify(unsigned long val, void *v) +{ + return blocking_notifier_call_chain(&node_chain, val, v); +} +#endif + static void node_remove_accesses(struct node *node) { struct node_access_nodes *c, *cnext; @@ -122,7 +149,7 @@ static void node_access_release(struct device *dev) } static struct node_access_nodes *node_init_node_access(struct node *node, - unsigned access) + enum access_coordinate_class access) { struct node_access_nodes *access_node; struct device *dev; @@ -157,15 +184,15 @@ free: } #ifdef CONFIG_HMEM_REPORTING -#define ACCESS_ATTR(name) \ -static ssize_t name##_show(struct device *dev, \ +#define ACCESS_ATTR(property) \ +static ssize_t property##_show(struct device *dev, \ struct device_attribute *attr, \ char *buf) \ { \ return sysfs_emit(buf, "%u\n", \ - to_access_nodes(dev)->hmem_attrs.name); \ + to_access_nodes(dev)->coord.property); \ } \ -static DEVICE_ATTR_RO(name) +static DEVICE_ATTR_RO(property) ACCESS_ATTR(read_bandwidth); ACCESS_ATTR(read_latency); @@ -183,11 +210,11 @@ static struct attribute *access_attrs[] = { /** * node_set_perf_attrs - Set the performance values for given access class * @nid: Node identifier to be set - * @hmem_attrs: Heterogeneous memory performance attributes + * @coord: Heterogeneous memory performance coordinates * @access: The access class the for the given attributes */ -void node_set_perf_attrs(unsigned int nid, struct node_hmem_attrs *hmem_attrs, - unsigned access) +void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord, + enum access_coordinate_class access) { struct node_access_nodes *c; struct node *node; @@ -201,7 +228,7 @@ void node_set_perf_attrs(unsigned int nid, struct node_hmem_attrs *hmem_attrs, if (!c) return; - c->hmem_attrs = *hmem_attrs; + c->coord = *coord; for (i = 0; access_attrs[i] != NULL; i++) { if (sysfs_add_file_to_group(&c->dev.kobj, access_attrs[i], "initiators")) { @@ -210,7 +237,54 @@ void node_set_perf_attrs(unsigned int nid, struct node_hmem_attrs *hmem_attrs, break; } } + + /* When setting CPU access coordinates, update mempolicy */ + if (access == ACCESS_COORDINATE_CPU) { + if (mempolicy_set_node_perf(nid, coord)) { + pr_info("failed to set mempolicy attrs for node %d\n", + nid); + } + } } +EXPORT_SYMBOL_GPL(node_set_perf_attrs); + +/** + * node_update_perf_attrs - Update the performance values for given access class + * @nid: Node identifier to be updated + * @coord: Heterogeneous memory performance coordinates + * @access: The access class for the given attributes + */ +void node_update_perf_attrs(unsigned int nid, struct access_coordinate *coord, + enum access_coordinate_class access) +{ + struct node_access_nodes *access_node; + struct node *node; + int i; + + if (WARN_ON_ONCE(!node_online(nid))) + return; + + node = node_devices[nid]; + list_for_each_entry(access_node, &node->access_list, list_node) { + if (access_node->access != access) + continue; + + access_node->coord = *coord; + for (i = 0; access_attrs[i]; i++) { + sysfs_notify(&access_node->dev.kobj, + NULL, access_attrs[i]->name); + } + break; + } + + /* When setting CPU access coordinates, update mempolicy */ + if (access != ACCESS_COORDINATE_CPU) + return; + + if (mempolicy_set_node_perf(nid, coord)) + pr_info("failed to set mempolicy attrs for node %d\n", nid); +} +EXPORT_SYMBOL_GPL(node_update_perf_attrs); /** * struct node_cache_info - Internal tracking for memory node caches @@ -239,12 +313,14 @@ CACHE_ATTR(size, "%llu") CACHE_ATTR(line_size, "%u") CACHE_ATTR(indexing, "%u") CACHE_ATTR(write_policy, "%u") +CACHE_ATTR(address_mode, "%#x") static struct attribute *cache_attrs[] = { &dev_attr_indexing.attr, &dev_attr_size.attr, &dev_attr_line_size.attr, &dev_attr_write_policy.attr, + &dev_attr_address_mode.attr, NULL, }; ATTRIBUTE_GROUPS(cache); @@ -429,6 +505,7 @@ static ssize_t node_read_meminfo(struct device *dev, "Node %d ShadowCallStack:%8lu kB\n" #endif "Node %d PageTables: %8lu kB\n" + "Node %d SecPageTables: %8lu kB\n" "Node %d NFS_Unstable: %8lu kB\n" "Node %d Bounce: %8lu kB\n" "Node %d WritebackTmp: %8lu kB\n" @@ -440,8 +517,11 @@ static ssize_t node_read_meminfo(struct device *dev, "Node %d AnonHugePages: %8lu kB\n" "Node %d ShmemHugePages: %8lu kB\n" "Node %d ShmemPmdMapped: %8lu kB\n" - "Node %d FileHugePages: %8lu kB\n" - "Node %d FilePmdMapped: %8lu kB\n" + "Node %d FileHugePages: %8lu kB\n" + "Node %d FilePmdMapped: %8lu kB\n" +#endif +#ifdef CONFIG_UNACCEPTED_MEMORY + "Node %d Unaccepted: %8lu kB\n" #endif , nid, K(node_page_state(pgdat, NR_FILE_DIRTY)), @@ -455,9 +535,10 @@ static ssize_t node_read_meminfo(struct device *dev, nid, node_page_state(pgdat, NR_KERNEL_SCS_KB), #endif nid, K(node_page_state(pgdat, NR_PAGETABLE)), + nid, K(node_page_state(pgdat, NR_SECONDARY_PAGETABLE)), + nid, 0UL, + nid, 0UL, nid, 0UL, - nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)), - nid, K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), nid, K(sreclaimable + node_page_state(pgdat, NR_KERNEL_MISC_RECLAIMABLE)), nid, K(sreclaimable + sunreclaimable), @@ -471,6 +552,10 @@ static ssize_t node_read_meminfo(struct device *dev, nid, K(node_page_state(pgdat, NR_FILE_THPS)), nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED)) #endif +#ifdef CONFIG_UNACCEPTED_MEMORY + , + nid, K(sum_zone_node_page_state(nid, NR_UNACCEPTED)) +#endif ); len += hugetlb_report_node_meminfo(buf, len, nid); return len; @@ -557,115 +642,38 @@ static ssize_t node_read_distance(struct device *dev, static DEVICE_ATTR(distance, 0444, node_read_distance, NULL); static struct attribute *node_dev_attrs[] = { - &dev_attr_cpumap.attr, - &dev_attr_cpulist.attr, &dev_attr_meminfo.attr, &dev_attr_numastat.attr, &dev_attr_distance.attr, &dev_attr_vmstat.attr, NULL }; -ATTRIBUTE_GROUPS(node_dev); - -#ifdef CONFIG_HUGETLBFS -/* - * hugetlbfs per node attributes registration interface: - * When/if hugetlb[fs] subsystem initializes [sometime after this module], - * it will register its per node attributes for all online nodes with - * memory. It will also call register_hugetlbfs_with_node(), below, to - * register its attribute registration functions with this node driver. - * Once these hooks have been initialized, the node driver will call into - * the hugetlb module to [un]register attributes for hot-plugged nodes. - */ -static node_registration_func_t __hugetlb_register_node; -static node_registration_func_t __hugetlb_unregister_node; -static inline bool hugetlb_register_node(struct node *node) -{ - if (__hugetlb_register_node && - node_state(node->dev.id, N_MEMORY)) { - __hugetlb_register_node(node); - return true; - } - return false; -} - -static inline void hugetlb_unregister_node(struct node *node) -{ - if (__hugetlb_unregister_node) - __hugetlb_unregister_node(node); -} +static const struct bin_attribute *node_dev_bin_attrs[] = { + &bin_attr_cpumap, + &bin_attr_cpulist, + NULL +}; -void register_hugetlbfs_with_node(node_registration_func_t doregister, - node_registration_func_t unregister) -{ - __hugetlb_register_node = doregister; - __hugetlb_unregister_node = unregister; -} -#else -static inline void hugetlb_register_node(struct node *node) {} +static const struct attribute_group node_dev_group = { + .attrs = node_dev_attrs, + .bin_attrs = node_dev_bin_attrs, +}; -static inline void hugetlb_unregister_node(struct node *node) {} +static const struct attribute_group *node_dev_groups[] = { + &node_dev_group, +#ifdef CONFIG_HAVE_ARCH_NODE_DEV_GROUP + &arch_node_dev_group, #endif - -static void node_device_release(struct device *dev) -{ - struct node *node = to_node(dev); - -#if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) && defined(CONFIG_HUGETLBFS) - /* - * We schedule the work only when a memory section is - * onlined/offlined on this node. When we come here, - * all the memory on this node has been offlined, - * so we won't enqueue new work to this work. - * - * The work is using node->node_work, so we should - * flush work before freeing the memory. - */ - flush_work(&node->node_work); +#ifdef CONFIG_MEMORY_FAILURE + &memory_failure_attr_group, #endif - kfree(node); -} - -/* - * register_node - Setup a sysfs device for a node. - * @num - Node number to use when creating the device. - * - * Initialize and register the node device. - */ -static int register_node(struct node *node, int num) -{ - int error; - - node->dev.id = num; - node->dev.bus = &node_subsys; - node->dev.release = node_device_release; - node->dev.groups = node_dev_groups; - error = device_register(&node->dev); - - if (error) - put_device(&node->dev); - else { - hugetlb_register_node(node); - - compaction_register_node(node); - } - return error; -} + NULL +}; -/** - * unregister_node - unregister a node device - * @node: node going away - * - * Unregisters a node device @node. All the devices on the node must be - * unregistered before calling this function. - */ -void unregister_node(struct node *node) +static void node_device_release(struct device *dev) { - hugetlb_unregister_node(node); /* no-op, if memoryless node */ - node_remove_accesses(node); - node_remove_caches(node); - device_unregister(&node->dev); + kfree(to_node(dev)); } struct node *node_devices[MAX_NUMNODES]; @@ -711,7 +719,7 @@ int register_cpu_under_node(unsigned int cpu, unsigned int nid) */ int register_memory_node_under_compute_node(unsigned int mem_nid, unsigned int cpu_nid, - unsigned access) + enum access_coordinate_class access) { struct node *init_node, *targ_node; struct node_access_nodes *initiator, *target; @@ -765,29 +773,12 @@ int unregister_cpu_under_node(unsigned int cpu, unsigned int nid) return 0; } -#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE -static int __ref get_nid_for_pfn(unsigned long pfn) -{ - if (!pfn_valid_within(pfn)) - return -1; -#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT - if (system_state < SYSTEM_RUNNING) - return early_pfn_to_nid(pfn); -#endif - return pfn_to_nid(pfn); -} - +#ifdef CONFIG_MEMORY_HOTPLUG static void do_register_memory_block_under_node(int nid, struct memory_block *mem_blk) { int ret; - /* - * If this memory block spans multiple nodes, we only indicate - * the last processed node. - */ - mem_blk->nid = nid; - ret = sysfs_create_link_nowarn(&node_devices[nid]->dev.kobj, &mem_blk->dev.kobj, kobject_name(&mem_blk->dev.kobj)); @@ -806,46 +797,6 @@ static void do_register_memory_block_under_node(int nid, ret); } -/* register memory section under specified node if it spans that node */ -static int register_mem_block_under_node_early(struct memory_block *mem_blk, - void *arg) -{ - unsigned long memory_block_pfns = memory_block_size_bytes() / PAGE_SIZE; - unsigned long start_pfn = section_nr_to_pfn(mem_blk->start_section_nr); - unsigned long end_pfn = start_pfn + memory_block_pfns - 1; - int nid = *(int *)arg; - unsigned long pfn; - - for (pfn = start_pfn; pfn <= end_pfn; pfn++) { - int page_nid; - - /* - * memory block could have several absent sections from start. - * skip pfn range from absent section - */ - if (!pfn_in_present_section(pfn)) { - pfn = round_down(pfn + PAGES_PER_SECTION, - PAGES_PER_SECTION) - 1; - continue; - } - - /* - * We need to check if page belongs to nid only at the boot - * case because node's ranges can be interleaved. - */ - page_nid = get_nid_for_pfn(pfn); - if (page_nid < 0) - continue; - if (page_nid != nid) - continue; - - do_register_memory_block_under_node(nid, mem_blk); - return 0; - } - /* mem section does not span the specified node */ - return 0; -} - /* * During hotplug we know that all pages in the memory block belong to the same * node. @@ -874,99 +825,77 @@ void unregister_memory_block_under_nodes(struct memory_block *mem_blk) kobject_name(&node_devices[mem_blk->nid]->dev.kobj)); } -void link_mem_sections(int nid, unsigned long start_pfn, unsigned long end_pfn, - enum meminit_context context) +/* register all memory blocks under the corresponding nodes */ +static void register_memory_blocks_under_nodes(void) { - walk_memory_blocks_func_t func; + struct memblock_region *r; - if (context == MEMINIT_HOTPLUG) - func = register_mem_block_under_node_hotplug; - else - func = register_mem_block_under_node_early; + for_each_mem_region(r) { + const unsigned long start_block_id = phys_to_block_id(r->base); + const unsigned long end_block_id = phys_to_block_id(r->base + r->size - 1); + const int nid = memblock_get_region_node(r); + unsigned long block_id; - walk_memory_blocks(PFN_PHYS(start_pfn), PFN_PHYS(end_pfn - start_pfn), - (void *)&nid, func); - return; -} - -#ifdef CONFIG_HUGETLBFS -/* - * Handle per node hstate attribute [un]registration on transistions - * to/from memoryless state. - */ -static void node_hugetlb_work(struct work_struct *work) -{ - struct node *node = container_of(work, struct node, node_work); + if (!node_online(nid)) + continue; - /* - * We only get here when a node transitions to/from memoryless state. - * We can detect which transition occurred by examining whether the - * node has memory now. hugetlb_register_node() already check this - * so we try to register the attributes. If that fails, then the - * node has transitioned to memoryless, try to unregister the - * attributes. - */ - if (!hugetlb_register_node(node)) - hugetlb_unregister_node(node); -} + for (block_id = start_block_id; block_id <= end_block_id; block_id++) { + struct memory_block *mem; -static void init_node_hugetlb_work(int nid) -{ - INIT_WORK(&node_devices[nid]->node_work, node_hugetlb_work); -} + mem = find_memory_block_by_id(block_id); + if (!mem) + continue; -static int node_memory_callback(struct notifier_block *self, - unsigned long action, void *arg) -{ - struct memory_notify *mnb = arg; - int nid = mnb->status_change_nid; - - switch (action) { - case MEM_ONLINE: - case MEM_OFFLINE: - /* - * offload per node hstate [un]registration to a work thread - * when transitioning to/from memoryless state. - */ - if (nid != NUMA_NO_NODE) - schedule_work(&node_devices[nid]->node_work); - break; + memory_block_add_nid_early(mem, nid); + do_register_memory_block_under_node(nid, mem); + put_device(&mem->dev); + } - case MEM_GOING_ONLINE: - case MEM_GOING_OFFLINE: - case MEM_CANCEL_ONLINE: - case MEM_CANCEL_OFFLINE: - default: - break; } - - return NOTIFY_OK; } -#endif /* CONFIG_HUGETLBFS */ -#endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */ -#if !defined(CONFIG_MEMORY_HOTPLUG_SPARSE) || \ - !defined(CONFIG_HUGETLBFS) -static inline int node_memory_callback(struct notifier_block *self, - unsigned long action, void *arg) +void register_memory_blocks_under_node_hotplug(int nid, unsigned long start_pfn, + unsigned long end_pfn) { - return NOTIFY_OK; + walk_memory_blocks(PFN_PHYS(start_pfn), PFN_PHYS(end_pfn - start_pfn), + (void *)&nid, register_mem_block_under_node_hotplug); + return; } +#endif /* CONFIG_MEMORY_HOTPLUG */ -static void init_node_hugetlb_work(int nid) { } - -#endif - -int __register_one_node(int nid) +/** + * register_node - Initialize and register the node device. + * @nid: Node number to use when creating the device. + * + * Return: 0 on success, -errno otherwise + */ +int register_node(int nid) { int error; int cpu; + struct node *node; - node_devices[nid] = kzalloc(sizeof(struct node), GFP_KERNEL); - if (!node_devices[nid]) + node = kzalloc(sizeof(struct node), GFP_KERNEL); + if (!node) return -ENOMEM; - error = register_node(node_devices[nid], nid); + INIT_LIST_HEAD(&node->access_list); + + node->dev.id = nid; + node->dev.bus = &node_subsys; + node->dev.release = node_device_release; + node->dev.groups = node_dev_groups; + + error = device_register(&node->dev); + if (error) { + put_device(&node->dev); + return error; + } + + node_devices[nid] = node; + hugetlb_register_node(node); + compaction_register_node(node); + reclaim_register_node(node); /* link cpu under this node */ for_each_present_cpu(cpu) { @@ -974,20 +903,30 @@ int __register_one_node(int nid) register_cpu_under_node(cpu, nid); } - INIT_LIST_HEAD(&node_devices[nid]->access_list); - /* initialize work queue for memory hot plug */ - init_node_hugetlb_work(nid); node_init_caches(nid); return error; } - -void unregister_one_node(int nid) +/** + * unregister_node - unregister a node device + * @nid: nid of the node going away + * + * Unregisters the node device at node id @nid. All the devices on the + * node must be unregistered before calling this function. + */ +void unregister_node(int nid) { - if (!node_devices[nid]) + struct node *node = node_devices[nid]; + + if (!node) return; - unregister_node(node_devices[nid]); + hugetlb_unregister_node(node); + compaction_unregister_node(node); + reclaim_unregister_node(node); + node_remove_accesses(node); + node_remove_caches(node); + device_unregister(&node->dev); node_devices[nid] = NULL; } @@ -1047,27 +986,26 @@ static const struct attribute_group *cpu_root_attr_groups[] = { NULL, }; -#define NODE_CALLBACK_PRI 2 /* lower than SLAB */ -static int __init register_node_type(void) +void __init node_dev_init(void) { - int ret; + int ret, i; BUILD_BUG_ON(ARRAY_SIZE(node_state_attr) != NR_NODE_STATES); BUILD_BUG_ON(ARRAY_SIZE(node_state_attrs)-1 != NR_NODE_STATES); ret = subsys_system_register(&node_subsys, cpu_root_attr_groups); - if (!ret) { - static struct notifier_block node_memory_callback_nb = { - .notifier_call = node_memory_callback, - .priority = NODE_CALLBACK_PRI, - }; - register_hotmemory_notifier(&node_memory_callback_nb); - } + if (ret) + panic("%s() failed to register subsystem: %d\n", __func__, ret); /* - * Note: we're not going to unregister the node class if we fail - * to register the node state class attribute files. + * Create all node devices, which will properly link the node + * to already created cpu devices. */ - return ret; + for_each_online_node(i) { + ret = register_node(i); + if (ret) + panic("%s() failed to add node: %d\n", __func__, ret); + } + + register_memory_blocks_under_nodes(); } -postcore_initcall(register_node_type); |