diff options
Diffstat (limited to 'drivers/virtio/virtio_balloon.c')
| -rw-r--r-- | drivers/virtio/virtio_balloon.c | 234 |
1 files changed, 163 insertions, 71 deletions
diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 3f78a3a1eb75..74fe59f5a78c 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -33,7 +33,7 @@ #define VIRTIO_BALLOON_FREE_PAGE_ALLOC_FLAG (__GFP_NORETRY | __GFP_NOWARN | \ __GFP_NOMEMALLOC) /* The order of free page blocks to report to host */ -#define VIRTIO_BALLOON_HINT_BLOCK_ORDER (MAX_ORDER - 1) +#define VIRTIO_BALLOON_HINT_BLOCK_ORDER MAX_PAGE_ORDER /* The size of a free page block in bytes */ #define VIRTIO_BALLOON_HINT_BLOCK_BYTES \ (1 << (VIRTIO_BALLOON_HINT_BLOCK_ORDER + PAGE_SHIFT)) @@ -111,7 +111,7 @@ struct virtio_balloon { struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR]; /* Shrinker to return free pages - VIRTIO_BALLOON_F_FREE_PAGE_HINT */ - struct shrinker shrinker; + struct shrinker *shrinker; /* OOM notifier to deflate on OOM - VIRTIO_BALLOON_F_DEFLATE_ON_OOM */ struct notifier_block oom_nb; @@ -119,8 +119,16 @@ struct virtio_balloon { /* Free page reporting device */ struct virtqueue *reporting_vq; struct page_reporting_dev_info pr_dev_info; + + /* State for keeping the wakeup_source active while adjusting the balloon */ + spinlock_t wakeup_lock; + bool processing_wakeup_event; + u32 wakeup_signal_mask; }; +#define VIRTIO_BALLOON_WAKEUP_SIGNAL_ADJUST (1 << 0) +#define VIRTIO_BALLOON_WAKEUP_SIGNAL_STATS (1 << 1) + static const struct virtio_device_id id_table[] = { { VIRTIO_ID_BALLOON, VIRTIO_DEV_ANY_ID }, { 0 }, @@ -135,6 +143,36 @@ static u32 page_to_balloon_pfn(struct page *page) return pfn * VIRTIO_BALLOON_PAGES_PER_PAGE; } +static void start_wakeup_event(struct virtio_balloon *vb, u32 mask) +{ + unsigned long flags; + + spin_lock_irqsave(&vb->wakeup_lock, flags); + vb->wakeup_signal_mask |= mask; + if (!vb->processing_wakeup_event) { + vb->processing_wakeup_event = true; + pm_stay_awake(&vb->vdev->dev); + } + spin_unlock_irqrestore(&vb->wakeup_lock, flags); +} + +static void process_wakeup_event(struct virtio_balloon *vb, u32 mask) +{ + spin_lock_irq(&vb->wakeup_lock); + vb->wakeup_signal_mask &= ~mask; + spin_unlock_irq(&vb->wakeup_lock); +} + +static void finish_wakeup_event(struct virtio_balloon *vb) +{ + spin_lock_irq(&vb->wakeup_lock); + if (!vb->wakeup_signal_mask && vb->processing_wakeup_event) { + vb->processing_wakeup_event = false; + pm_relax(&vb->vdev->dev); + } + spin_unlock_irq(&vb->wakeup_lock); +} + static void balloon_ack(struct virtqueue *vq) { struct virtio_balloon *vb = vq->vdev->priv; @@ -167,7 +205,7 @@ static int virtballoon_free_page_report(struct page_reporting_dev_info *pr_dev_i unsigned int unused, err; /* We should always be able to add these buffers to an empty queue. */ - err = virtqueue_add_inbuf(vq, sg, nents, vb, GFP_NOWAIT | __GFP_NOWARN); + err = virtqueue_add_inbuf(vq, sg, nents, vb, GFP_NOWAIT); /* * In the extremely unlikely case that something has occurred and we @@ -213,7 +251,7 @@ static unsigned int fill_balloon(struct virtio_balloon *vb, size_t num) for (num_pfns = 0; num_pfns < num; num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) { - struct page *page = balloon_page_alloc(); + page = balloon_page_alloc(); if (!page) { dev_info_ratelimited(&vb->vdev->dev, @@ -311,34 +349,67 @@ static inline void update_stat(struct virtio_balloon *vb, int idx, #define pages_to_bytes(x) ((u64)(x) << PAGE_SHIFT) -static unsigned int update_balloon_stats(struct virtio_balloon *vb) +#ifdef CONFIG_VM_EVENT_COUNTERS +/* Return the number of entries filled by vm events */ +static inline unsigned int update_balloon_vm_stats(struct virtio_balloon *vb) { unsigned long events[NR_VM_EVENT_ITEMS]; - struct sysinfo i; unsigned int idx = 0; - long available; - unsigned long caches; + unsigned int zid; + unsigned long stall = 0; all_vm_events(events); - si_meminfo(&i); - - available = si_mem_available(); - caches = global_node_page_state(NR_FILE_PAGES); - -#ifdef CONFIG_VM_EVENT_COUNTERS update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN, - pages_to_bytes(events[PSWPIN])); + pages_to_bytes(events[PSWPIN])); update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT, - pages_to_bytes(events[PSWPOUT])); + pages_to_bytes(events[PSWPOUT])); update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]); update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]); + update_stat(vb, idx++, VIRTIO_BALLOON_S_OOM_KILL, events[OOM_KILL]); + + /* sum all the stall events */ + for (zid = 0; zid < MAX_NR_ZONES; zid++) + stall += events[ALLOCSTALL_NORMAL - ZONE_NORMAL + zid]; + + update_stat(vb, idx++, VIRTIO_BALLOON_S_ALLOC_STALL, stall); + + update_stat(vb, idx++, VIRTIO_BALLOON_S_ASYNC_SCAN, + pages_to_bytes(events[PGSCAN_KSWAPD])); + update_stat(vb, idx++, VIRTIO_BALLOON_S_DIRECT_SCAN, + pages_to_bytes(events[PGSCAN_DIRECT])); + update_stat(vb, idx++, VIRTIO_BALLOON_S_ASYNC_RECLAIM, + pages_to_bytes(events[PGSTEAL_KSWAPD])); + update_stat(vb, idx++, VIRTIO_BALLOON_S_DIRECT_RECLAIM, + pages_to_bytes(events[PGSTEAL_DIRECT])); + #ifdef CONFIG_HUGETLB_PAGE update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGALLOC, events[HTLB_BUDDY_PGALLOC]); update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGFAIL, events[HTLB_BUDDY_PGALLOC_FAIL]); -#endif -#endif +#endif /* CONFIG_HUGETLB_PAGE */ + + return idx; +} +#else /* CONFIG_VM_EVENT_COUNTERS */ +static inline unsigned int update_balloon_vm_stats(struct virtio_balloon *vb) +{ + return 0; +} +#endif /* CONFIG_VM_EVENT_COUNTERS */ + +static unsigned int update_balloon_stats(struct virtio_balloon *vb) +{ + struct sysinfo i; + unsigned int idx; + long available; + unsigned long caches; + + idx = update_balloon_vm_stats(vb); + + si_meminfo(&i); + available = si_mem_available(); + caches = global_node_page_state(NR_FILE_PAGES); update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE, pages_to_bytes(i.freeram)); update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT, @@ -365,8 +436,10 @@ static void stats_request(struct virtqueue *vq) struct virtio_balloon *vb = vq->vdev->priv; spin_lock(&vb->stop_update_lock); - if (!vb->stop_update) + if (!vb->stop_update) { + start_wakeup_event(vb, VIRTIO_BALLOON_WAKEUP_SIGNAL_STATS); queue_work(system_freezable_wq, &vb->update_balloon_stats_work); + } spin_unlock(&vb->stop_update_lock); } @@ -395,7 +468,11 @@ static inline s64 towards_target(struct virtio_balloon *vb) virtio_cread_le(vb->vdev, struct virtio_balloon_config, num_pages, &num_pages); - target = num_pages; + /* + * Aligned up to guest page size to avoid inflating and deflating + * balloon endlessly. + */ + target = ALIGN(num_pages, VIRTIO_BALLOON_PAGES_PER_PAGE); return target - vb->num_pages; } @@ -411,8 +488,7 @@ static unsigned long return_free_pages_to_mm(struct virtio_balloon *vb, page = balloon_page_pop(&vb->free_page_list); if (!page) break; - free_pages((unsigned long)page_address(page), - VIRTIO_BALLOON_HINT_BLOCK_ORDER); + __free_pages(page, VIRTIO_BALLOON_HINT_BLOCK_ORDER); } vb->num_free_page_blocks -= num_returned; spin_unlock_irq(&vb->free_page_list_lock); @@ -433,6 +509,12 @@ static void virtio_balloon_queue_free_page_work(struct virtio_balloon *vb) queue_work(vb->balloon_wq, &vb->report_free_page_work); } +static void start_update_balloon_size(struct virtio_balloon *vb) +{ + start_wakeup_event(vb, VIRTIO_BALLOON_WAKEUP_SIGNAL_ADJUST); + queue_work(system_freezable_wq, &vb->update_balloon_size_work); +} + static void virtballoon_changed(struct virtio_device *vdev) { struct virtio_balloon *vb = vdev->priv; @@ -440,8 +522,7 @@ static void virtballoon_changed(struct virtio_device *vdev) spin_lock_irqsave(&vb->stop_update_lock, flags); if (!vb->stop_update) { - queue_work(system_freezable_wq, - &vb->update_balloon_size_work); + start_update_balloon_size(vb); virtio_balloon_queue_free_page_work(vb); } spin_unlock_irqrestore(&vb->stop_update_lock, flags); @@ -462,7 +543,10 @@ static void update_balloon_stats_func(struct work_struct *work) vb = container_of(work, struct virtio_balloon, update_balloon_stats_work); + + process_wakeup_event(vb, VIRTIO_BALLOON_WAKEUP_SIGNAL_STATS); stats_handle_request(vb); + finish_wakeup_event(vb); } static void update_balloon_size_func(struct work_struct *work) @@ -472,26 +556,29 @@ static void update_balloon_size_func(struct work_struct *work) vb = container_of(work, struct virtio_balloon, update_balloon_size_work); - diff = towards_target(vb); - if (!diff) - return; + process_wakeup_event(vb, VIRTIO_BALLOON_WAKEUP_SIGNAL_ADJUST); - if (diff > 0) - diff -= fill_balloon(vb, diff); - else - diff += leak_balloon(vb, -diff); - update_balloon_size(vb); + diff = towards_target(vb); + + if (diff) { + if (diff > 0) + diff -= fill_balloon(vb, diff); + else + diff += leak_balloon(vb, -diff); + update_balloon_size(vb); + } if (diff) queue_work(system_freezable_wq, work); + else + finish_wakeup_event(vb); } static int init_vqs(struct virtio_balloon *vb) { + struct virtqueue_info vqs_info[VIRTIO_BALLOON_VQ_MAX] = {}; struct virtqueue *vqs[VIRTIO_BALLOON_VQ_MAX]; - vq_callback_t *callbacks[VIRTIO_BALLOON_VQ_MAX]; - const char *names[VIRTIO_BALLOON_VQ_MAX]; int err; /* @@ -499,33 +586,26 @@ static int init_vqs(struct virtio_balloon *vb) * will be NULL if the related feature is not enabled, which will * cause no allocation for the corresponding virtqueue in find_vqs. */ - callbacks[VIRTIO_BALLOON_VQ_INFLATE] = balloon_ack; - names[VIRTIO_BALLOON_VQ_INFLATE] = "inflate"; - callbacks[VIRTIO_BALLOON_VQ_DEFLATE] = balloon_ack; - names[VIRTIO_BALLOON_VQ_DEFLATE] = "deflate"; - callbacks[VIRTIO_BALLOON_VQ_STATS] = NULL; - names[VIRTIO_BALLOON_VQ_STATS] = NULL; - callbacks[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL; - names[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL; - names[VIRTIO_BALLOON_VQ_REPORTING] = NULL; + vqs_info[VIRTIO_BALLOON_VQ_INFLATE].callback = balloon_ack; + vqs_info[VIRTIO_BALLOON_VQ_INFLATE].name = "inflate"; + vqs_info[VIRTIO_BALLOON_VQ_DEFLATE].callback = balloon_ack; + vqs_info[VIRTIO_BALLOON_VQ_DEFLATE].name = "deflate"; if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) { - names[VIRTIO_BALLOON_VQ_STATS] = "stats"; - callbacks[VIRTIO_BALLOON_VQ_STATS] = stats_request; + vqs_info[VIRTIO_BALLOON_VQ_STATS].name = "stats"; + vqs_info[VIRTIO_BALLOON_VQ_STATS].callback = stats_request; } - if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) { - names[VIRTIO_BALLOON_VQ_FREE_PAGE] = "free_page_vq"; - callbacks[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL; - } + if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) + vqs_info[VIRTIO_BALLOON_VQ_FREE_PAGE].name = "free_page_vq"; if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) { - names[VIRTIO_BALLOON_VQ_REPORTING] = "reporting_vq"; - callbacks[VIRTIO_BALLOON_VQ_REPORTING] = balloon_ack; + vqs_info[VIRTIO_BALLOON_VQ_REPORTING].name = "reporting_vq"; + vqs_info[VIRTIO_BALLOON_VQ_REPORTING].callback = balloon_ack; } err = virtio_find_vqs(vb->vdev, VIRTIO_BALLOON_VQ_MAX, vqs, - callbacks, names, NULL); + vqs_info, NULL); if (err) return err; @@ -638,8 +718,7 @@ static int get_free_page_and_send(struct virtio_balloon *vb) if (vq->num_free > 1) { err = virtqueue_add_inbuf(vq, &sg, 1, p, GFP_KERNEL); if (unlikely(err)) { - free_pages((unsigned long)p, - VIRTIO_BALLOON_HINT_BLOCK_ORDER); + __free_pages(page, VIRTIO_BALLOON_HINT_BLOCK_ORDER); return err; } virtqueue_kick(vq); @@ -652,7 +731,7 @@ static int get_free_page_and_send(struct virtio_balloon *vb) * The vq has no available entry to add this page block, so * just free it. */ - free_pages((unsigned long)p, VIRTIO_BALLOON_HINT_BLOCK_ORDER); + __free_pages(page, VIRTIO_BALLOON_HINT_BLOCK_ORDER); } return 0; @@ -741,7 +820,7 @@ static void report_free_page_func(struct work_struct *work) * 2) update the host about the old page removed from vb->pages list; * * This function preforms the balloon page migration task. - * Called through balloon_mapping->a_ops->migratepage + * Called through movable_operations->migrate_page */ static int virtballoon_migratepage(struct balloon_dev_info *vb_dev_info, struct page *newpage, struct page *page, enum migrate_mode mode) @@ -785,18 +864,16 @@ static int virtballoon_migratepage(struct balloon_dev_info *vb_dev_info, tell_host(vb, vb->inflate_vq); /* balloon's page migration 2nd step -- deflate "page" */ - spin_lock_irqsave(&vb_dev_info->pages_lock, flags); - balloon_page_delete(page); - spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags); vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE; set_page_pfns(vb, vb->pfns, page); tell_host(vb, vb->deflate_vq); mutex_unlock(&vb->balloon_lock); + balloon_page_finalize(page); put_page(page); /* balloon reference */ - return MIGRATEPAGE_SUCCESS; + return 0; } #endif /* CONFIG_BALLOON_COMPACTION */ @@ -816,8 +893,7 @@ static unsigned long shrink_free_pages(struct virtio_balloon *vb, static unsigned long virtio_balloon_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) { - struct virtio_balloon *vb = container_of(shrinker, - struct virtio_balloon, shrinker); + struct virtio_balloon *vb = shrinker->private_data; return shrink_free_pages(vb, sc->nr_to_scan); } @@ -825,8 +901,7 @@ static unsigned long virtio_balloon_shrinker_scan(struct shrinker *shrinker, static unsigned long virtio_balloon_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) { - struct virtio_balloon *vb = container_of(shrinker, - struct virtio_balloon, shrinker); + struct virtio_balloon *vb = shrinker->private_data; return vb->num_free_page_blocks * VIRTIO_BALLOON_HINT_BLOCK_PAGES; } @@ -847,16 +922,22 @@ static int virtio_balloon_oom_notify(struct notifier_block *nb, static void virtio_balloon_unregister_shrinker(struct virtio_balloon *vb) { - unregister_shrinker(&vb->shrinker); + shrinker_free(vb->shrinker); } static int virtio_balloon_register_shrinker(struct virtio_balloon *vb) { - vb->shrinker.scan_objects = virtio_balloon_shrinker_scan; - vb->shrinker.count_objects = virtio_balloon_shrinker_count; - vb->shrinker.seeks = DEFAULT_SEEKS; + vb->shrinker = shrinker_alloc(0, "virtio-balloon"); + if (!vb->shrinker) + return -ENOMEM; + + vb->shrinker->scan_objects = virtio_balloon_shrinker_scan; + vb->shrinker->count_objects = virtio_balloon_shrinker_count; + vb->shrinker->private_data = vb; - return register_shrinker(&vb->shrinker, "virtio-balloon"); + shrinker_register(vb->shrinker); + + return 0; } static int virtballoon_probe(struct virtio_device *vdev) @@ -902,7 +983,8 @@ static int virtballoon_probe(struct virtio_device *vdev) goto out_del_vqs; } vb->balloon_wq = alloc_workqueue("balloon-wq", - WQ_FREEZABLE | WQ_CPU_INTENSIVE, 0); + WQ_FREEZABLE | WQ_CPU_INTENSIVE | WQ_PERCPU, + 0); if (!vb->balloon_wq) { err = -ENOMEM; goto out_del_vqs; @@ -984,6 +1066,17 @@ static int virtballoon_probe(struct virtio_device *vdev) goto out_unregister_oom; } + spin_lock_init(&vb->wakeup_lock); + + /* + * The virtio balloon itself can't wake up the device, but it is + * responsible for processing wakeup events passed up from the transport + * layer. Wakeup sources don't support nesting/chaining calls, so we use + * our own wakeup source to ensure wakeup events are properly handled + * without trampling on the transport layer's wakeup source. + */ + device_set_wakeup_capable(&vb->vdev->dev, true); + virtio_device_ready(vdev); if (towards_target(vb)) @@ -1110,7 +1203,6 @@ static struct virtio_driver virtio_balloon_driver = { .feature_table = features, .feature_table_size = ARRAY_SIZE(features), .driver.name = KBUILD_MODNAME, - .driver.owner = THIS_MODULE, .id_table = id_table, .validate = virtballoon_validate, .probe = virtballoon_probe, |