summaryrefslogtreecommitdiff
path: root/block
diff options
context:
space:
mode:
Diffstat (limited to 'block')
-rw-r--r--block/blk-cgroup.c91
-rw-r--r--block/blk-core.c7
-rw-r--r--block/blk-mq-cpumap.c2
-rw-r--r--block/blk-mq.c68
-rw-r--r--block/blk-sysfs.c2
-rw-r--r--block/bounce.c2
-rw-r--r--block/cfq-iosched.c125
-rw-r--r--block/elevator.c8
-rw-r--r--block/genhd.c13
9 files changed, 243 insertions, 75 deletions
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 31610ae0ebff..9f97da52d006 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -9,6 +9,10 @@
*
* Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
* Nauman Rafique <nauman@google.com>
+ *
+ * For policy-specific per-blkcg data:
+ * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
+ * Arianna Avanzini <avanzini.arianna@gmail.com>
*/
#include <linux/ioprio.h>
#include <linux/kdev_t.h>
@@ -27,8 +31,7 @@
static DEFINE_MUTEX(blkcg_pol_mutex);
-struct blkcg blkcg_root = { .cfq_weight = 2 * CFQ_WEIGHT_DEFAULT,
- .cfq_leaf_weight = 2 * CFQ_WEIGHT_DEFAULT, };
+struct blkcg blkcg_root;
EXPORT_SYMBOL_GPL(blkcg_root);
struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;
@@ -827,6 +830,8 @@ static struct cgroup_subsys_state *
blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
{
struct blkcg *blkcg;
+ struct cgroup_subsys_state *ret;
+ int i;
if (!parent_css) {
blkcg = &blkcg_root;
@@ -834,11 +839,35 @@ blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
}
blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
- if (!blkcg)
- return ERR_PTR(-ENOMEM);
+ if (!blkcg) {
+ ret = ERR_PTR(-ENOMEM);
+ goto free_blkcg;
+ }
+
+ for (i = 0; i < BLKCG_MAX_POLS ; i++) {
+ struct blkcg_policy *pol = blkcg_policy[i];
+ struct blkcg_policy_data *cpd;
+
+ /*
+ * If the policy hasn't been attached yet, wait for it
+ * to be attached before doing anything else. Otherwise,
+ * check if the policy requires any specific per-cgroup
+ * data: if it does, allocate and initialize it.
+ */
+ if (!pol || !pol->cpd_size)
+ continue;
+
+ BUG_ON(blkcg->pd[i]);
+ cpd = kzalloc(pol->cpd_size, GFP_KERNEL);
+ if (!cpd) {
+ ret = ERR_PTR(-ENOMEM);
+ goto free_pd_blkcg;
+ }
+ blkcg->pd[i] = cpd;
+ cpd->plid = i;
+ pol->cpd_init_fn(blkcg);
+ }
- blkcg->cfq_weight = CFQ_WEIGHT_DEFAULT;
- blkcg->cfq_leaf_weight = CFQ_WEIGHT_DEFAULT;
done:
spin_lock_init(&blkcg->lock);
INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_ATOMIC);
@@ -847,6 +876,14 @@ done:
INIT_LIST_HEAD(&blkcg->cgwb_list);
#endif
return &blkcg->css;
+
+free_pd_blkcg:
+ for (i--; i >= 0; i--)
+ kfree(blkcg->pd[i]);
+
+free_blkcg:
+ kfree(blkcg);
+ return ret;
}
/**
@@ -1000,8 +1037,10 @@ int blkcg_activate_policy(struct request_queue *q,
const struct blkcg_policy *pol)
{
LIST_HEAD(pds);
+ LIST_HEAD(cpds);
struct blkcg_gq *blkg;
- struct blkg_policy_data *pd, *n;
+ struct blkg_policy_data *pd, *nd;
+ struct blkcg_policy_data *cpd, *cnd;
int cnt = 0, ret;
if (blkcg_policy_enabled(q, pol))
@@ -1014,6 +1053,10 @@ int blkcg_activate_policy(struct request_queue *q,
cnt++;
spin_unlock_irq(q->queue_lock);
+ /*
+ * Allocate per-blkg and per-blkcg policy data
+ * for all existing blkgs.
+ */
while (cnt--) {
pd = kzalloc_node(pol->pd_size, GFP_KERNEL, q->node);
if (!pd) {
@@ -1021,26 +1064,50 @@ int blkcg_activate_policy(struct request_queue *q,
goto out_free;
}
list_add_tail(&pd->alloc_node, &pds);
+
+ if (!pol->cpd_size)
+ continue;
+ cpd = kzalloc_node(pol->cpd_size, GFP_KERNEL, q->node);
+ if (!cpd) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ list_add_tail(&cpd->alloc_node, &cpds);
}
/*
- * Install the allocated pds. With @q bypassing, no new blkg
+ * Install the allocated pds and cpds. With @q bypassing, no new blkg
* should have been created while the queue lock was dropped.
*/
spin_lock_irq(q->queue_lock);
list_for_each_entry(blkg, &q->blkg_list, q_node) {
- if (WARN_ON(list_empty(&pds))) {
+ if (WARN_ON(list_empty(&pds)) ||
+ WARN_ON(pol->cpd_size && list_empty(&cpds))) {
/* umm... this shouldn't happen, just abort */
ret = -ENOMEM;
goto out_unlock;
}
+ cpd = list_first_entry(&cpds, struct blkcg_policy_data,
+ alloc_node);
+ list_del_init(&cpd->alloc_node);
pd = list_first_entry(&pds, struct blkg_policy_data, alloc_node);
list_del_init(&pd->alloc_node);
/* grab blkcg lock too while installing @pd on @blkg */
spin_lock(&blkg->blkcg->lock);
+ if (!pol->cpd_size)
+ goto no_cpd;
+ if (!blkg->blkcg->pd[pol->plid]) {
+ /* Per-policy per-blkcg data */
+ blkg->blkcg->pd[pol->plid] = cpd;
+ cpd->plid = pol->plid;
+ pol->cpd_init_fn(blkg->blkcg);
+ } else { /* must free it as it has already been extracted */
+ kfree(cpd);
+ }
+no_cpd:
blkg->pd[pol->plid] = pd;
pd->blkg = blkg;
pd->plid = pol->plid;
@@ -1055,8 +1122,10 @@ out_unlock:
spin_unlock_irq(q->queue_lock);
out_free:
blk_queue_bypass_end(q);
- list_for_each_entry_safe(pd, n, &pds, alloc_node)
+ list_for_each_entry_safe(pd, nd, &pds, alloc_node)
kfree(pd);
+ list_for_each_entry_safe(cpd, cnd, &cpds, alloc_node)
+ kfree(cpd);
return ret;
}
EXPORT_SYMBOL_GPL(blkcg_activate_policy);
@@ -1093,6 +1162,8 @@ void blkcg_deactivate_policy(struct request_queue *q,
kfree(blkg->pd[pol->plid]);
blkg->pd[pol->plid] = NULL;
+ kfree(blkg->blkcg->pd[pol->plid]);
+ blkg->blkcg->pd[pol->plid] = NULL;
spin_unlock(&blkg->blkcg->lock);
}
diff --git a/block/blk-core.c b/block/blk-core.c
index a4a2dbe46fe3..688ae9482cb8 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -579,6 +579,8 @@ void blk_cleanup_queue(struct request_queue *q)
q->queue_lock = &q->__queue_lock;
spin_unlock_irq(lock);
+ bdi_destroy(&q->backing_dev_info);
+
/* @q is and will stay empty, shutdown and put */
blk_put_queue(q);
}
@@ -758,6 +760,8 @@ blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
}
EXPORT_SYMBOL(blk_init_queue_node);
+static void blk_queue_bio(struct request_queue *q, struct bio *bio);
+
struct request_queue *
blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn,
spinlock_t *lock)
@@ -1601,7 +1605,7 @@ void init_request_from_bio(struct request *req, struct bio *bio)
blk_rq_bio_prep(req->q, req, bio);
}
-void blk_queue_bio(struct request_queue *q, struct bio *bio)
+static void blk_queue_bio(struct request_queue *q, struct bio *bio)
{
const bool sync = !!(bio->bi_rw & REQ_SYNC);
struct blk_plug *plug;
@@ -1709,7 +1713,6 @@ out_unlock:
spin_unlock_irq(q->queue_lock);
}
}
-EXPORT_SYMBOL_GPL(blk_queue_bio); /* for device mapper only */
/*
* If bio->bi_dev is a partition, remap the location
diff --git a/block/blk-mq-cpumap.c b/block/blk-mq-cpumap.c
index 5f13f4d0bcce..1e28ddb656b8 100644
--- a/block/blk-mq-cpumap.c
+++ b/block/blk-mq-cpumap.c
@@ -24,7 +24,7 @@ static int get_first_sibling(unsigned int cpu)
{
unsigned int ret;
- ret = cpumask_first(topology_thread_cpumask(cpu));
+ ret = cpumask_first(topology_sibling_cpumask(cpu));
if (ret < nr_cpu_ids)
return ret;
diff --git a/block/blk-mq.c b/block/blk-mq.c
index ef100fd2cb86..f53779692c77 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -673,8 +673,11 @@ static void blk_mq_rq_timer(unsigned long priv)
data.next = blk_rq_timeout(round_jiffies_up(data.next));
mod_timer(&q->timeout, data.next);
} else {
- queue_for_each_hw_ctx(q, hctx, i)
- blk_mq_tag_idle(hctx);
+ queue_for_each_hw_ctx(q, hctx, i) {
+ /* the hctx may be unmapped, so check it here */
+ if (blk_mq_hw_queue_mapped(hctx))
+ blk_mq_tag_idle(hctx);
+ }
}
}
@@ -851,6 +854,16 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
spin_lock(&hctx->lock);
list_splice(&rq_list, &hctx->dispatch);
spin_unlock(&hctx->lock);
+ /*
+ * the queue is expected stopped with BLK_MQ_RQ_QUEUE_BUSY, but
+ * it's possible the queue is stopped and restarted again
+ * before this. Queue restart will dispatch requests. And since
+ * requests in rq_list aren't added into hctx->dispatch yet,
+ * the requests in rq_list might get lost.
+ *
+ * blk_mq_run_hw_queue() already checks the STOPPED bit
+ **/
+ blk_mq_run_hw_queue(hctx, true);
}
}
@@ -1600,22 +1613,6 @@ static int blk_mq_hctx_cpu_offline(struct blk_mq_hw_ctx *hctx, int cpu)
return NOTIFY_OK;
}
-static int blk_mq_hctx_cpu_online(struct blk_mq_hw_ctx *hctx, int cpu)
-{
- struct request_queue *q = hctx->queue;
- struct blk_mq_tag_set *set = q->tag_set;
-
- if (set->tags[hctx->queue_num])
- return NOTIFY_OK;
-
- set->tags[hctx->queue_num] = blk_mq_init_rq_map(set, hctx->queue_num);
- if (!set->tags[hctx->queue_num])
- return NOTIFY_STOP;
-
- hctx->tags = set->tags[hctx->queue_num];
- return NOTIFY_OK;
-}
-
static int blk_mq_hctx_notify(void *data, unsigned long action,
unsigned int cpu)
{
@@ -1623,12 +1620,16 @@ static int blk_mq_hctx_notify(void *data, unsigned long action,
if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
return blk_mq_hctx_cpu_offline(hctx, cpu);
- else if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
- return blk_mq_hctx_cpu_online(hctx, cpu);
+
+ /*
+ * In case of CPU online, tags may be reallocated
+ * in blk_mq_map_swqueue() after mapping is updated.
+ */
return NOTIFY_OK;
}
+/* hctx->ctxs will be freed in queue's release handler */
static void blk_mq_exit_hctx(struct request_queue *q,
struct blk_mq_tag_set *set,
struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
@@ -1647,7 +1648,6 @@ static void blk_mq_exit_hctx(struct request_queue *q,
blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
blk_free_flush_queue(hctx->fq);
- kfree(hctx->ctxs);
blk_mq_free_bitmap(&hctx->ctx_map);
}
@@ -1804,6 +1804,7 @@ static void blk_mq_map_swqueue(struct request_queue *q)
unsigned int i;
struct blk_mq_hw_ctx *hctx;
struct blk_mq_ctx *ctx;
+ struct blk_mq_tag_set *set = q->tag_set;
queue_for_each_hw_ctx(q, hctx, i) {
cpumask_clear(hctx->cpumask);
@@ -1833,16 +1834,20 @@ static void blk_mq_map_swqueue(struct request_queue *q)
* disable it and free the request entries.
*/
if (!hctx->nr_ctx) {
- struct blk_mq_tag_set *set = q->tag_set;
-
if (set->tags[i]) {
blk_mq_free_rq_map(set, set->tags[i], i);
set->tags[i] = NULL;
- hctx->tags = NULL;
}
+ hctx->tags = NULL;
continue;
}
+ /* unmapped hw queue can be remapped after CPU topo changed */
+ if (!set->tags[i])
+ set->tags[i] = blk_mq_init_rq_map(set, i);
+ hctx->tags = set->tags[i];
+ WARN_ON(!hctx->tags);
+
/*
* Set the map size to the number of mapped software queues.
* This is more accurate and more efficient than looping
@@ -1916,8 +1921,12 @@ void blk_mq_release(struct request_queue *q)
unsigned int i;
/* hctx kobj stays in hctx */
- queue_for_each_hw_ctx(q, hctx, i)
+ queue_for_each_hw_ctx(q, hctx, i) {
+ if (!hctx)
+ continue;
+ kfree(hctx->ctxs);
kfree(hctx);
+ }
kfree(q->queue_hw_ctx);
@@ -2120,9 +2129,16 @@ static int blk_mq_queue_reinit_notify(struct notifier_block *nb,
*/
list_for_each_entry(q, &all_q_list, all_q_node)
blk_mq_freeze_queue_start(q);
- list_for_each_entry(q, &all_q_list, all_q_node)
+ list_for_each_entry(q, &all_q_list, all_q_node) {
blk_mq_freeze_queue_wait(q);
+ /*
+ * timeout handler can't touch hw queue during the
+ * reinitialization
+ */
+ del_timer_sync(&q->timeout);
+ }
+
list_for_each_entry(q, &all_q_list, all_q_node)
blk_mq_queue_reinit(q);
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 1b60941dc4c6..6264b382d4d1 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -523,8 +523,6 @@ static void blk_release_queue(struct kobject *kobj)
blk_trace_shutdown(q);
- bdi_destroy(&q->backing_dev_info);
-
ida_simple_remove(&blk_queue_ida, q->id);
call_rcu(&q->rcu_head, blk_free_queue_rcu);
}
diff --git a/block/bounce.c b/block/bounce.c
index 072280b3dd13..b17311227c12 100644
--- a/block/bounce.c
+++ b/block/bounce.c
@@ -219,8 +219,8 @@ bounce:
if (page_to_pfn(page) <= queue_bounce_pfn(q) && !force)
continue;
- inc_zone_page_state(to->bv_page, NR_BOUNCE);
to->bv_page = mempool_alloc(pool, q->bounce_gfp);
+ inc_zone_page_state(to->bv_page, NR_BOUNCE);
if (rw == WRITE) {
char *vto, *vfrom;
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index bc8f42930773..c62bb2e650b8 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -67,6 +67,11 @@ static struct kmem_cache *cfq_pool;
#define sample_valid(samples) ((samples) > 80)
#define rb_entry_cfqg(node) rb_entry((node), struct cfq_group, rb_node)
+/* blkio-related constants */
+#define CFQ_WEIGHT_MIN 10
+#define CFQ_WEIGHT_MAX 1000
+#define CFQ_WEIGHT_DEFAULT 500
+
struct cfq_ttime {
unsigned long last_end_request;
@@ -212,6 +217,15 @@ struct cfqg_stats {
#endif /* CONFIG_CFQ_GROUP_IOSCHED */
};
+/* Per-cgroup data */
+struct cfq_group_data {
+ /* must be the first member */
+ struct blkcg_policy_data pd;
+
+ unsigned int weight;
+ unsigned int leaf_weight;
+};
+
/* This is per cgroup per device grouping structure */
struct cfq_group {
/* must be the first member */
@@ -446,16 +460,6 @@ CFQ_CFQQ_FNS(deep);
CFQ_CFQQ_FNS(wait_busy);
#undef CFQ_CFQQ_FNS
-static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd)
-{
- return pd ? container_of(pd, struct cfq_group, pd) : NULL;
-}
-
-static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)
-{
- return pd_to_blkg(&cfqg->pd);
-}
-
#if defined(CONFIG_CFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
/* cfqg stats flags */
@@ -600,6 +604,22 @@ static inline void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg) { }
#ifdef CONFIG_CFQ_GROUP_IOSCHED
+static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd)
+{
+ return pd ? container_of(pd, struct cfq_group, pd) : NULL;
+}
+
+static struct cfq_group_data
+*cpd_to_cfqgd(struct blkcg_policy_data *cpd)
+{
+ return cpd ? container_of(cpd, struct cfq_group_data, pd) : NULL;
+}
+
+static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)
+{
+ return pd_to_blkg(&cfqg->pd);
+}
+
static struct blkcg_policy blkcg_policy_cfq;
static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg)
@@ -607,6 +627,11 @@ static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg)
return pd_to_cfqg(blkg_to_pd(blkg, &blkcg_policy_cfq));
}
+static struct cfq_group_data *blkcg_to_cfqgd(struct blkcg *blkcg)
+{
+ return cpd_to_cfqgd(blkcg_to_cpd(blkcg, &blkcg_policy_cfq));
+}
+
static inline struct cfq_group *cfqg_parent(struct cfq_group *cfqg)
{
struct blkcg_gq *pblkg = cfqg_to_blkg(cfqg)->parent;
@@ -1544,13 +1569,28 @@ static void cfqg_stats_init(struct cfqg_stats *stats)
#endif
}
+static void cfq_cpd_init(const struct blkcg *blkcg)
+{
+ struct cfq_group_data *cgd =
+ cpd_to_cfqgd(blkcg->pd[blkcg_policy_cfq.plid]);
+
+ if (blkcg == &blkcg_root) {
+ cgd->weight = 2 * CFQ_WEIGHT_DEFAULT;
+ cgd->leaf_weight = 2 * CFQ_WEIGHT_DEFAULT;
+ } else {
+ cgd->weight = CFQ_WEIGHT_DEFAULT;
+ cgd->leaf_weight = CFQ_WEIGHT_DEFAULT;
+ }
+}
+
static void cfq_pd_init(struct blkcg_gq *blkg)
{
struct cfq_group *cfqg = blkg_to_cfqg(blkg);
+ struct cfq_group_data *cgd = blkcg_to_cfqgd(blkg->blkcg);
cfq_init_cfqg_base(cfqg);
- cfqg->weight = blkg->blkcg->cfq_weight;
- cfqg->leaf_weight = blkg->blkcg->cfq_leaf_weight;
+ cfqg->weight = cgd->weight;
+ cfqg->leaf_weight = cgd->leaf_weight;
cfqg_stats_init(&cfqg->stats);
cfqg_stats_init(&cfqg->dead_stats);
}
@@ -1673,13 +1713,27 @@ static int cfqg_print_leaf_weight_device(struct seq_file *sf, void *v)
static int cfq_print_weight(struct seq_file *sf, void *v)
{
- seq_printf(sf, "%u\n", css_to_blkcg(seq_css(sf))->cfq_weight);
+ struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
+ struct cfq_group_data *cgd = blkcg_to_cfqgd(blkcg);
+ unsigned int val = 0;
+
+ if (cgd)
+ val = cgd->weight;
+
+ seq_printf(sf, "%u\n", val);
return 0;
}
static int cfq_print_leaf_weight(struct seq_file *sf, void *v)
{
- seq_printf(sf, "%u\n", css_to_blkcg(seq_css(sf))->cfq_leaf_weight);
+ struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
+ struct cfq_group_data *cgd = blkcg_to_cfqgd(blkcg);
+ unsigned int val = 0;
+
+ if (cgd)
+ val = cgd->leaf_weight;
+
+ seq_printf(sf, "%u\n", val);
return 0;
}
@@ -1690,6 +1744,7 @@ static ssize_t __cfqg_set_weight_device(struct kernfs_open_file *of,
struct blkcg *blkcg = css_to_blkcg(of_css(of));
struct blkg_conf_ctx ctx;
struct cfq_group *cfqg;
+ struct cfq_group_data *cfqgd;
int ret;
ret = blkg_conf_prep(blkcg, &blkcg_policy_cfq, buf, &ctx);
@@ -1698,17 +1753,22 @@ static ssize_t __cfqg_set_weight_device(struct kernfs_open_file *of,
ret = -EINVAL;
cfqg = blkg_to_cfqg(ctx.blkg);
+ cfqgd = blkcg_to_cfqgd(blkcg);
+ if (!cfqg || !cfqgd)
+ goto err;
+
if (!ctx.v || (ctx.v >= CFQ_WEIGHT_MIN && ctx.v <= CFQ_WEIGHT_MAX)) {
if (!is_leaf_weight) {
cfqg->dev_weight = ctx.v;
- cfqg->new_weight = ctx.v ?: blkcg->cfq_weight;
+ cfqg->new_weight = ctx.v ?: cfqgd->weight;
} else {
cfqg->dev_leaf_weight = ctx.v;
- cfqg->new_leaf_weight = ctx.v ?: blkcg->cfq_leaf_weight;
+ cfqg->new_leaf_weight = ctx.v ?: cfqgd->leaf_weight;
}
ret = 0;
}
+err:
blkg_conf_finish(&ctx);
return ret ?: nbytes;
}
@@ -1730,16 +1790,23 @@ static int __cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft,
{
struct blkcg *blkcg = css_to_blkcg(css);
struct blkcg_gq *blkg;
+ struct cfq_group_data *cfqgd;
+ int ret = 0;
if (val < CFQ_WEIGHT_MIN || val > CFQ_WEIGHT_MAX)
return -EINVAL;
spin_lock_irq(&blkcg->lock);
+ cfqgd = blkcg_to_cfqgd(blkcg);
+ if (!cfqgd) {
+ ret = -EINVAL;
+ goto out;
+ }
if (!is_leaf_weight)
- blkcg->cfq_weight = val;
+ cfqgd->weight = val;
else
- blkcg->cfq_leaf_weight = val;
+ cfqgd->leaf_weight = val;
hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
struct cfq_group *cfqg = blkg_to_cfqg(blkg);
@@ -1749,15 +1816,16 @@ static int __cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft,
if (!is_leaf_weight) {
if (!cfqg->dev_weight)
- cfqg->new_weight = blkcg->cfq_weight;
+ cfqg->new_weight = cfqgd->weight;
} else {
if (!cfqg->dev_leaf_weight)
- cfqg->new_leaf_weight = blkcg->cfq_leaf_weight;
+ cfqg->new_leaf_weight = cfqgd->leaf_weight;
}
}
+out:
spin_unlock_irq(&blkcg->lock);
- return 0;
+ return ret;
}
static int cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft,
@@ -4477,6 +4545,18 @@ out_free:
return ret;
}
+static void cfq_registered_queue(struct request_queue *q)
+{
+ struct elevator_queue *e = q->elevator;
+ struct cfq_data *cfqd = e->elevator_data;
+
+ /*
+ * Default to IOPS mode with no idling for SSDs
+ */
+ if (blk_queue_nonrot(q))
+ cfqd->cfq_slice_idle = 0;
+}
+
/*
* sysfs parts below -->
*/
@@ -4592,6 +4672,7 @@ static struct elevator_type iosched_cfq = {
.elevator_may_queue_fn = cfq_may_queue,
.elevator_init_fn = cfq_init_queue,
.elevator_exit_fn = cfq_exit_queue,
+ .elevator_registered_fn = cfq_registered_queue,
},
.icq_size = sizeof(struct cfq_io_cq),
.icq_align = __alignof__(struct cfq_io_cq),
@@ -4603,8 +4684,10 @@ static struct elevator_type iosched_cfq = {
#ifdef CONFIG_CFQ_GROUP_IOSCHED
static struct blkcg_policy blkcg_policy_cfq = {
.pd_size = sizeof(struct cfq_group),
+ .cpd_size = sizeof(struct cfq_group_data),
.cftypes = cfq_blkcg_files,
+ .cpd_init_fn = cfq_cpd_init,
.pd_init_fn = cfq_pd_init,
.pd_offline_fn = cfq_pd_offline,
.pd_reset_stats_fn = cfq_pd_reset_stats,
diff --git a/block/elevator.c b/block/elevator.c
index 3bbb48f430e4..84d63943f2de 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -157,7 +157,7 @@ struct elevator_queue *elevator_alloc(struct request_queue *q,
eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, q->node);
if (unlikely(!eq))
- goto err;
+ return NULL;
eq->type = e;
kobject_init(&eq->kobj, &elv_ktype);
@@ -165,10 +165,6 @@ struct elevator_queue *elevator_alloc(struct request_queue *q,
hash_init(eq->hash);
return eq;
-err:
- kfree(eq);
- elevator_put(e);
- return NULL;
}
EXPORT_SYMBOL(elevator_alloc);
@@ -810,6 +806,8 @@ int elv_register_queue(struct request_queue *q)
}
kobject_uevent(&e->kobj, KOBJ_ADD);
e->registered = 1;
+ if (e->type->ops.elevator_registered_fn)
+ e->type->ops.elevator_registered_fn(q);
}
return error;
}
diff --git a/block/genhd.c b/block/genhd.c
index d46ba566d62f..59a1395eedac 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -423,9 +423,9 @@ int blk_alloc_devt(struct hd_struct *part, dev_t *devt)
/* allocate ext devt */
idr_preload(GFP_KERNEL);
- spin_lock(&ext_devt_lock);
+ spin_lock_bh(&ext_devt_lock);
idx = idr_alloc(&ext_devt_idr, part, 0, NR_EXT_DEVT, GFP_NOWAIT);
- spin_unlock(&ext_devt_lock);
+ spin_unlock_bh(&ext_devt_lock);
idr_preload_end();
if (idx < 0)
@@ -450,9 +450,9 @@ void blk_free_devt(dev_t devt)
return;
if (MAJOR(devt) == BLOCK_EXT_MAJOR) {
- spin_lock(&ext_devt_lock);
+ spin_lock_bh(&ext_devt_lock);
idr_remove(&ext_devt_idr, blk_mangle_minor(MINOR(devt)));
- spin_unlock(&ext_devt_lock);
+ spin_unlock_bh(&ext_devt_lock);
}
}
@@ -654,7 +654,6 @@ void del_gendisk(struct gendisk *disk)
disk->flags &= ~GENHD_FL_UP;
sysfs_remove_link(&disk_to_dev(disk)->kobj, "bdi");
- bdi_unregister(&disk->queue->backing_dev_info);
blk_unregister_queue(disk);
blk_unregister_region(disk_devt(disk), disk->minors);
@@ -692,13 +691,13 @@ struct gendisk *get_gendisk(dev_t devt, int *partno)
} else {
struct hd_struct *part;
- spin_lock(&ext_devt_lock);
+ spin_lock_bh(&ext_devt_lock);
part = idr_find(&ext_devt_idr, blk_mangle_minor(MINOR(devt)));
if (part && get_disk(part_to_disk(part))) {
*partno = part->partno;
disk = part_to_disk(part);
}
- spin_unlock(&ext_devt_lock);
+ spin_unlock_bh(&ext_devt_lock);
}
return disk;
generated by cgit (git 2.34.1) at 2024-06-14 08:54:57 +0000