diff options
Diffstat (limited to 'include/linux/balloon_compaction.h')
| -rw-r--r-- | include/linux/balloon_compaction.h | 266 |
1 files changed, 77 insertions, 189 deletions
diff --git a/include/linux/balloon_compaction.h b/include/linux/balloon_compaction.h index f7f1d7169b11..7cfe48769239 100644 --- a/include/linux/balloon_compaction.h +++ b/include/linux/balloon_compaction.h @@ -1,35 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * include/linux/balloon_compaction.h * * Common interface definitions for making balloon pages movable by compaction. * - * Despite being perfectly possible to perform ballooned pages migration, they - * make a special corner case to compaction scans because balloon pages are not - * enlisted at any LRU list like the other pages we do compact / migrate. + * Balloon page migration makes use of the general "movable_ops page migration" + * feature. + * + * page->private is used to reference the responsible balloon device. + * That these pages have movable_ops, and which movable_ops apply, + * is derived from the page type (PageOffline()) combined with the + * PG_movable_ops flag (PageMovableOps()). * * As the page isolation scanning step a compaction thread does is a lockless * procedure (from a page standpoint), it might bring some racy situations while * performing balloon page compaction. In order to sort out these racy scenarios * and safely perform balloon's page compaction and migration we must, always, - * ensure following these three simple rules: + * ensure following these simple rules: * - * i. when updating a balloon's page ->mapping element, strictly do it under - * the following lock order, independently of the far superior - * locking scheme (lru_lock, balloon_lock): + * i. Setting the PG_movable_ops flag and page->private with the following + * lock order * +-page_lock(page); * +--spin_lock_irq(&b_dev_info->pages_lock); - * ... page->mapping updates here ... - * - * ii. before isolating or dequeueing a balloon page from the balloon device - * pages list, the page reference counter must be raised by one and the - * extra refcount must be dropped when the page is enqueued back into - * the balloon device page list, thus a balloon page keeps its reference - * counter raised only while it is under our special handling; * - * iii. after the lockless scan step have selected a potential balloon page for - * isolation, re-test the page->mapping flags and the page ref counter - * under the proper page lock, to ensure isolating a valid balloon page - * (not yet isolated, nor under release procedure) + * ii. isolation or dequeueing procedure must remove the page from balloon + * device page list under b_dev_info->pages_lock. * * The functions provided by this interface are placed to help on coping with * the aforementioned balloon page corner case, as well as to ensure the simple @@ -45,6 +40,8 @@ #include <linux/migrate.h> #include <linux/gfp.h> #include <linux/err.h> +#include <linux/fs.h> +#include <linux/list.h> /* * Balloon device information descriptor. @@ -54,219 +51,110 @@ * balloon driver as a page book-keeper for its registered balloon devices. */ struct balloon_dev_info { - void *balloon_device; /* balloon device descriptor */ - struct address_space *mapping; /* balloon special page->mapping */ unsigned long isolated_pages; /* # of isolated pages for migration */ spinlock_t pages_lock; /* Protection to pages list */ struct list_head pages; /* Pages enqueued & handled to Host */ + int (*migratepage)(struct balloon_dev_info *, struct page *newpage, + struct page *page, enum migrate_mode mode); }; -extern struct page *balloon_page_enqueue(struct balloon_dev_info *b_dev_info); +extern struct page *balloon_page_alloc(void); +extern void balloon_page_enqueue(struct balloon_dev_info *b_dev_info, + struct page *page); extern struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info); -extern struct balloon_dev_info *balloon_devinfo_alloc( - void *balloon_dev_descriptor); +extern size_t balloon_page_list_enqueue(struct balloon_dev_info *b_dev_info, + struct list_head *pages); +extern size_t balloon_page_list_dequeue(struct balloon_dev_info *b_dev_info, + struct list_head *pages, size_t n_req_pages); -static inline void balloon_devinfo_free(struct balloon_dev_info *b_dev_info) +static inline void balloon_devinfo_init(struct balloon_dev_info *balloon) { - kfree(b_dev_info); + balloon->isolated_pages = 0; + spin_lock_init(&balloon->pages_lock); + INIT_LIST_HEAD(&balloon->pages); + balloon->migratepage = NULL; } +#ifdef CONFIG_BALLOON_COMPACTION +extern const struct movable_operations balloon_mops; /* - * balloon_page_free - release a balloon page back to the page free lists - * @page: ballooned page to be set free - * - * This function must be used to properly set free an isolated/dequeued balloon - * page at the end of a sucessful page migration, or at the balloon driver's - * page release procedure. + * balloon_page_device - get the b_dev_info descriptor for the balloon device + * that enqueues the given page. */ -static inline void balloon_page_free(struct page *page) -{ - /* - * Balloon pages always get an extra refcount before being isolated - * and before being dequeued to help on sorting out fortuite colisions - * between a thread attempting to isolate and another thread attempting - * to release the very same balloon page. - * - * Before we handle the page back to Buddy, lets drop its extra refcnt. - */ - put_page(page); - __free_page(page); -} - -#ifdef CONFIG_BALLOON_COMPACTION -extern bool balloon_page_isolate(struct page *page); -extern void balloon_page_putback(struct page *page); -extern int balloon_page_migrate(struct page *newpage, - struct page *page, enum migrate_mode mode); -extern struct address_space -*balloon_mapping_alloc(struct balloon_dev_info *b_dev_info, - const struct address_space_operations *a_ops); - -static inline void balloon_mapping_free(struct address_space *balloon_mapping) +static inline struct balloon_dev_info *balloon_page_device(struct page *page) { - kfree(balloon_mapping); + return (struct balloon_dev_info *)page_private(page); } +#endif /* CONFIG_BALLOON_COMPACTION */ /* - * page_flags_cleared - helper to perform balloon @page ->flags tests. + * balloon_page_insert - insert a page into the balloon's page list and make + * the page->private assignment accordingly. + * @balloon : pointer to balloon device + * @page : page to be assigned as a 'balloon page' * - * As balloon pages are obtained from buddy and we do not play with page->flags - * at driver level (exception made when we get the page lock for compaction), - * we can safely identify a ballooned page by checking if the - * PAGE_FLAGS_CHECK_AT_PREP page->flags are all cleared. This approach also - * helps us skip ballooned pages that are locked for compaction or release, thus - * mitigating their racy check at balloon_page_movable() + * Caller must ensure the page is locked and the spin_lock protecting balloon + * pages list is held before inserting a page into the balloon device. */ -static inline bool page_flags_cleared(struct page *page) +static inline void balloon_page_insert(struct balloon_dev_info *balloon, + struct page *page) { - return !(page->flags & PAGE_FLAGS_CHECK_AT_PREP); + __SetPageOffline(page); + if (IS_ENABLED(CONFIG_BALLOON_COMPACTION)) { + SetPageMovableOps(page); + set_page_private(page, (unsigned long)balloon); + } + list_add(&page->lru, &balloon->pages); } -/* - * __is_movable_balloon_page - helper to perform @page mapping->flags tests - */ -static inline bool __is_movable_balloon_page(struct page *page) +static inline gfp_t balloon_mapping_gfp_mask(void) { - struct address_space *mapping = page->mapping; - return mapping_balloon(mapping); + if (IS_ENABLED(CONFIG_BALLOON_COMPACTION)) + return GFP_HIGHUSER_MOVABLE; + return GFP_HIGHUSER; } /* - * balloon_page_movable - test page->mapping->flags to identify balloon pages - * that can be moved by compaction/migration. - * - * This function is used at core compaction's page isolation scheme, therefore - * most pages exposed to it are not enlisted as balloon pages and so, to avoid - * undesired side effects like racing against __free_pages(), we cannot afford - * holding the page locked while testing page->mapping->flags here. + * balloon_page_finalize - prepare a balloon page that was removed from the + * balloon list for release to the page allocator + * @page: page to be released to the page allocator * - * As we might return false positives in the case of a balloon page being just - * released under us, the page->mapping->flags need to be re-tested later, - * under the proper page lock, at the functions that will be coping with the - * balloon page case. + * Caller must ensure that the page is locked. */ -static inline bool balloon_page_movable(struct page *page) +static inline void balloon_page_finalize(struct page *page) { - /* - * Before dereferencing and testing mapping->flags, let's make sure - * this is not a page that uses ->mapping in a different way - */ - if (page_flags_cleared(page) && !page_mapped(page) && - page_count(page) == 1) - return __is_movable_balloon_page(page); - - return false; + if (IS_ENABLED(CONFIG_BALLOON_COMPACTION)) + set_page_private(page, 0); + /* PageOffline is sticky until the page is freed to the buddy. */ } /* - * balloon_page_insert - insert a page into the balloon's page list and make - * the page->mapping assignment accordingly. - * @page : page to be assigned as a 'balloon page' - * @mapping : allocated special 'balloon_mapping' - * @head : balloon's device page list head + * balloon_page_push - insert a page into a page list. + * @head : pointer to list + * @page : page to be added * - * Caller must ensure the page is locked and the spin_lock protecting balloon - * pages list is held before inserting a page into the balloon device. + * Caller must ensure the page is private and protect the list. */ -static inline void balloon_page_insert(struct page *page, - struct address_space *mapping, - struct list_head *head) +static inline void balloon_page_push(struct list_head *pages, struct page *page) { - page->mapping = mapping; - list_add(&page->lru, head); + list_add(&page->lru, pages); } /* - * balloon_page_delete - delete a page from balloon's page list and clear - * the page->mapping assignement accordingly. - * @page : page to be released from balloon's page list + * balloon_page_pop - remove a page from a page list. + * @head : pointer to list + * @page : page to be added * - * Caller must ensure the page is locked and the spin_lock protecting balloon - * pages list is held before deleting a page from the balloon device. + * Caller must ensure the page is private and protect the list. */ -static inline void balloon_page_delete(struct page *page) +static inline struct page *balloon_page_pop(struct list_head *pages) { - page->mapping = NULL; - list_del(&page->lru); -} + struct page *page = list_first_entry_or_null(pages, struct page, lru); -/* - * balloon_page_device - get the b_dev_info descriptor for the balloon device - * that enqueues the given page. - */ -static inline struct balloon_dev_info *balloon_page_device(struct page *page) -{ - struct address_space *mapping = page->mapping; - if (likely(mapping)) - return mapping->private_data; - - return NULL; -} + if (!page) + return NULL; -static inline gfp_t balloon_mapping_gfp_mask(void) -{ - return GFP_HIGHUSER_MOVABLE; -} - -static inline bool balloon_compaction_check(void) -{ - return true; -} - -#else /* !CONFIG_BALLOON_COMPACTION */ - -static inline void *balloon_mapping_alloc(void *balloon_device, - const struct address_space_operations *a_ops) -{ - return ERR_PTR(-EOPNOTSUPP); -} - -static inline void balloon_mapping_free(struct address_space *balloon_mapping) -{ - return; -} - -static inline void balloon_page_insert(struct page *page, - struct address_space *mapping, - struct list_head *head) -{ - list_add(&page->lru, head); -} - -static inline void balloon_page_delete(struct page *page) -{ list_del(&page->lru); + return page; } - -static inline bool balloon_page_movable(struct page *page) -{ - return false; -} - -static inline bool balloon_page_isolate(struct page *page) -{ - return false; -} - -static inline void balloon_page_putback(struct page *page) -{ - return; -} - -static inline int balloon_page_migrate(struct page *newpage, - struct page *page, enum migrate_mode mode) -{ - return 0; -} - -static inline gfp_t balloon_mapping_gfp_mask(void) -{ - return GFP_HIGHUSER; -} - -static inline bool balloon_compaction_check(void) -{ - return false; -} -#endif /* CONFIG_BALLOON_COMPACTION */ #endif /* _LINUX_BALLOON_COMPACTION_H */ |