diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c index ac83b06687640..1e7dc1a2d2fb9 100644 --- a/block/bfq-cgroup.c +++ b/block/bfq-cgroup.c @@ -940,7 +940,8 @@ void bfq_end_wr_async(struct bfq_data *bfqd) list_for_each_entry(blkg, &bfqd->queue->blkg_list, q_node) { struct bfq_group *bfqg = blkg_to_bfqg(blkg); - bfq_end_wr_async_queues(bfqd, bfqg); + if (bfqg) + bfq_end_wr_async_queues(bfqd, bfqg); } bfq_end_wr_async_queues(bfqd, bfqd->root_group); } diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index 141c602d5e858..42ccfd0c6140f 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -2645,6 +2645,9 @@ static void bfq_end_wr(struct bfq_data *bfqd) struct bfq_queue *bfqq; int i; +#ifdef CONFIG_BFQ_GROUP_IOSCHED + mutex_lock(&bfqd->queue->blkcg_mutex); +#endif spin_lock_irq(&bfqd->lock); for (i = 0; i < bfqd->num_actuators; i++) { @@ -2656,6 +2659,9 @@ static void bfq_end_wr(struct bfq_data *bfqd) bfq_end_wr_async(bfqd); spin_unlock_irq(&bfqd->lock); +#ifdef CONFIG_BFQ_GROUP_IOSCHED + mutex_unlock(&bfqd->queue->blkcg_mutex); +#endif } static sector_t bfq_io_struct_pos(void *io_struct, bool request) diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index bc63bd220865d..450ff7ff8ff84 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -575,6 +575,7 @@ static void blkg_destroy_all(struct gendisk *disk) int i; restart: + mutex_lock(&q->blkcg_mutex); spin_lock_irq(&q->queue_lock); list_for_each_entry(blkg, &q->blkg_list, q_node) { struct blkcg *blkcg = blkg->blkcg; @@ -593,6 +594,7 @@ static void blkg_destroy_all(struct gendisk *disk) if (!(--count)) { count = BLKG_DESTROY_BATCH_SIZE; spin_unlock_irq(&q->queue_lock); + mutex_unlock(&q->blkcg_mutex); cond_resched(); goto restart; } @@ -612,6 +614,7 @@ static void blkg_destroy_all(struct gendisk *disk) q->root_blkg = NULL; spin_unlock_irq(&q->queue_lock); + mutex_unlock(&q->blkcg_mutex); wake_up_var(&q->root_blkg); } @@ -1574,6 +1577,31 @@ struct cgroup_subsys io_cgrp_subsys = { }; EXPORT_SYMBOL_GPL(io_cgrp_subsys); +/* + * Tear down per-blkg policy data for @pol on @q. + */ +static void blkcg_policy_teardown_pds(struct request_queue *q, + const struct blkcg_policy *pol) +{ + struct blkcg_gq *blkg; + + list_for_each_entry(blkg, &q->blkg_list, q_node) { + struct blkcg *blkcg = blkg->blkcg; + struct blkg_policy_data *pd; + + spin_lock(&blkcg->lock); + pd = blkg->pd[pol->plid]; + if (pd) { + if (pd->online && pol->pd_offline_fn) + pol->pd_offline_fn(pd); + pd->online = false; + pol->pd_free_fn(pd); + blkg->pd[pol->plid] = NULL; + } + spin_unlock(&blkcg->lock); + } +} + /** * blkcg_activate_policy - activate a blkcg policy on a gendisk * @disk: gendisk of interest @@ -1611,6 +1639,8 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) if (queue_is_mq(q)) memflags = blk_mq_freeze_queue(q); + + mutex_lock(&q->blkcg_mutex); retry: spin_lock_irq(&q->queue_lock); @@ -1620,6 +1650,8 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) if (blkg->pd[pol->plid]) continue; + if (hlist_unhashed(&blkg->blkcg_node)) + continue; /* If prealloc matches, use it; otherwise try GFP_NOWAIT */ if (blkg == pinned_blkg) { @@ -1673,6 +1705,7 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) spin_unlock_irq(&q->queue_lock); out: + mutex_unlock(&q->blkcg_mutex); if (queue_is_mq(q)) blk_mq_unfreeze_queue(q, memflags); if (pinned_blkg) @@ -1684,21 +1717,7 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) enomem: /* alloc failed, take down everything */ spin_lock_irq(&q->queue_lock); - list_for_each_entry(blkg, &q->blkg_list, q_node) { - struct blkcg *blkcg = blkg->blkcg; - struct blkg_policy_data *pd; - - spin_lock(&blkcg->lock); - pd = blkg->pd[pol->plid]; - if (pd) { - if (pd->online && pol->pd_offline_fn) - pol->pd_offline_fn(pd); - pd->online = false; - pol->pd_free_fn(pd); - blkg->pd[pol->plid] = NULL; - } - spin_unlock(&blkcg->lock); - } + blkcg_policy_teardown_pds(q, pol); spin_unlock_irq(&q->queue_lock); ret = -ENOMEM; goto out; @@ -1717,7 +1736,6 @@ void blkcg_deactivate_policy(struct gendisk *disk, const struct blkcg_policy *pol) { struct request_queue *q = disk->queue; - struct blkcg_gq *blkg; unsigned int memflags; if (!blkcg_policy_enabled(q, pol)) @@ -1730,20 +1748,7 @@ void blkcg_deactivate_policy(struct gendisk *disk, spin_lock_irq(&q->queue_lock); __clear_bit(pol->plid, q->blkcg_pols); - - list_for_each_entry(blkg, &q->blkg_list, q_node) { - struct blkcg *blkcg = blkg->blkcg; - - spin_lock(&blkcg->lock); - if (blkg->pd[pol->plid]) { - if (blkg->pd[pol->plid]->online && pol->pd_offline_fn) - pol->pd_offline_fn(blkg->pd[pol->plid]); - pol->pd_free_fn(blkg->pd[pol->plid]); - blkg->pd[pol->plid] = NULL; - } - spin_unlock(&blkcg->lock); - } - + blkcg_policy_teardown_pds(q, pol); spin_unlock_irq(&q->queue_lock); mutex_unlock(&q->blkcg_mutex);