diff options
Diffstat (limited to 'drivers/mtd/ubi/wl.c')
| -rw-r--r-- | drivers/mtd/ubi/wl.c | 365 |
1 files changed, 293 insertions, 72 deletions
diff --git a/drivers/mtd/ubi/wl.c b/drivers/mtd/ubi/wl.c index 6f2ac865ff05..fbd399cf6503 100644 --- a/drivers/mtd/ubi/wl.c +++ b/drivers/mtd/ubi/wl.c @@ -1,20 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (c) International Business Machines Corp., 2006 * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See - * the GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * * Authors: Artem Bityutskiy (Битюцкий Артём), Thomas Gleixner */ @@ -178,7 +165,7 @@ static void wl_tree_add(struct ubi_wl_entry *e, struct rb_root *root) } /** - * wl_tree_destroy - destroy a wear-leveling entry. + * wl_entry_destroy - destroy a wear-leveling entry. * @ubi: UBI device description object * @e: the wear-leveling entry to add * @@ -194,11 +181,13 @@ static void wl_entry_destroy(struct ubi_device *ubi, struct ubi_wl_entry *e) /** * do_work - do one pending work. * @ubi: UBI device description object + * @executed: whether there is one work is executed * * This function returns zero in case of success and a negative error code in - * case of failure. + * case of failure. If @executed is not NULL and there is one work executed, + * @executed is set as %1, otherwise @executed is set as %0. */ -static int do_work(struct ubi_device *ubi) +static int do_work(struct ubi_device *ubi, int *executed) { int err; struct ubi_work *wrk; @@ -216,9 +205,13 @@ static int do_work(struct ubi_device *ubi) if (list_empty(&ubi->works)) { spin_unlock(&ubi->wl_lock); up_read(&ubi->work_sem); + if (executed) + *executed = 0; return 0; } + if (executed) + *executed = 1; wrk = list_entry(ubi->works.next, struct ubi_work, list); list_del(&wrk->list); ubi->works_count -= 1; @@ -278,6 +271,27 @@ static int in_wl_tree(struct ubi_wl_entry *e, struct rb_root *root) } /** + * in_pq - check if a wear-leveling entry is present in the protection queue. + * @ubi: UBI device description object + * @e: the wear-leveling entry to check + * + * This function returns non-zero if @e is in the protection queue and zero + * if it is not. + */ +static inline int in_pq(const struct ubi_device *ubi, struct ubi_wl_entry *e) +{ + struct ubi_wl_entry *p; + int i; + + for (i = 0; i < UBI_PROT_QUEUE_LEN; ++i) + list_for_each_entry(p, &ubi->pq[i], u.list) + if (p == e) + return 1; + + return 0; +} + +/** * prot_queue_add - add physical eraseblock to the protection queue. * @ubi: UBI device description object * @e: the physical eraseblock to add @@ -303,15 +317,17 @@ static void prot_queue_add(struct ubi_device *ubi, struct ubi_wl_entry *e) * @ubi: UBI device description object * @root: the RB-tree where to look for * @diff: maximum possible difference from the smallest erase counter + * @pick_max: pick PEB even its erase counter beyonds 'min_ec + @diff' * * This function looks for a wear leveling entry with erase counter closest to * min + @diff, where min is the smallest erase counter. */ static struct ubi_wl_entry *find_wl_entry(struct ubi_device *ubi, - struct rb_root *root, int diff) + struct rb_root *root, int diff, + int pick_max) { struct rb_node *p; - struct ubi_wl_entry *e, *prev_e = NULL; + struct ubi_wl_entry *e; int max; e = rb_entry(rb_first(root), struct ubi_wl_entry, u.rb); @@ -322,22 +338,16 @@ static struct ubi_wl_entry *find_wl_entry(struct ubi_device *ubi, struct ubi_wl_entry *e1; e1 = rb_entry(p, struct ubi_wl_entry, u.rb); - if (e1->ec >= max) + if (e1->ec >= max) { + if (pick_max) + e = e1; p = p->rb_left; - else { + } else { p = p->rb_right; - prev_e = e; e = e1; } } - /* If no fastmap has been written and this WL entry can be used - * as anchor PEB, hold it back and return the second best WL entry - * such that fastmap can use the anchor PEB later. */ - if (prev_e && !ubi->fm_disabled && - !ubi->fm && e->pnum < UBI_FM_MAX_START) - return prev_e; - return e; } @@ -361,12 +371,15 @@ static struct ubi_wl_entry *find_mean_wl_entry(struct ubi_device *ubi, if (last->ec - first->ec < WL_FREE_MAX_DIFF) { e = rb_entry(root->rb_node, struct ubi_wl_entry, u.rb); - /* If no fastmap has been written and this WL entry can be used - * as anchor PEB, hold it back and return the second best - * WL entry such that fastmap can use the anchor PEB later. */ + /* + * If no fastmap has been written and fm_anchor is not + * reserved and this WL entry can be used as anchor PEB + * hold it back and return the second best WL entry such + * that fastmap can use the anchor PEB later. + */ e = may_reserve_for_fm(ubi, e, root); } else - e = find_wl_entry(ubi, root, WL_FREE_MAX_DIFF/2); + e = find_wl_entry(ubi, root, WL_FREE_MAX_DIFF/2, 0); return e; } @@ -376,7 +389,7 @@ static struct ubi_wl_entry *find_mean_wl_entry(struct ubi_device *ubi, * refill_wl_user_pool(). * @ubi: UBI device description object * - * This function returns a a wear leveling entry in case of success and + * This function returns a wear leveling entry in case of success and * NULL in case of failure. */ static struct ubi_wl_entry *wl_get_wle(struct ubi_device *ubi) @@ -427,16 +440,15 @@ static int prot_queue_del(struct ubi_device *ubi, int pnum) } /** - * sync_erase - synchronously erase a physical eraseblock. + * ubi_sync_erase - synchronously erase a physical eraseblock. * @ubi: UBI device description object - * @e: the the physical eraseblock to erase + * @e: the physical eraseblock to erase * @torture: if the physical eraseblock has to be tortured * * This function returns zero in case of success and a negative error code in * case of failure. */ -static int sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, - int torture) +int ubi_sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, int torture) { int err; struct ubi_ec_hdr *ec_hdr; @@ -575,6 +587,7 @@ static int erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk, * @vol_id: the volume ID that last used this PEB * @lnum: the last used logical eraseblock number for the PEB * @torture: if the physical eraseblock has to be tortured + * @nested: denotes whether the work_sem is already held * * This function returns zero in case of success and a %-ENOMEM in case of * failure. @@ -648,9 +661,6 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, { int err, scrubbing = 0, torture = 0, protect = 0, erroneous = 0; int erase = 0, keep = 0, vol_id = -1, lnum = -1; -#ifdef CONFIG_MTD_UBI_FASTMAP - int anchor = wrk->anchor; -#endif struct ubi_wl_entry *e1, *e2; struct ubi_vid_io_buf *vidb; struct ubi_vid_hdr *vid_hdr; @@ -672,7 +682,11 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, ubi_assert(!ubi->move_from && !ubi->move_to); ubi_assert(!ubi->move_to_put); +#ifdef CONFIG_MTD_UBI_FASTMAP + if (!next_peb_for_wl(ubi, true) || +#else if (!ubi->free.rb_node || +#endif (!ubi->used.rb_node && !ubi->scrub.rb_node)) { /* * No free physical eraseblocks? Well, they must be waiting in @@ -690,12 +704,21 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, } #ifdef CONFIG_MTD_UBI_FASTMAP - /* Check whether we need to produce an anchor PEB */ - if (!anchor) - anchor = !anchor_pebs_available(&ubi->free); + e1 = find_anchor_wl_entry(&ubi->used); + if (e1 && ubi->fm_anchor && + (ubi->fm_anchor->ec - e1->ec >= UBI_WL_THRESHOLD)) { + ubi->fm_do_produce_anchor = 1; + /* + * fm_anchor is no longer considered a good anchor. + * NULL assignment also prevents multiple wear level checks + * of this PEB. + */ + wl_tree_add(ubi->fm_anchor, &ubi->free); + ubi->fm_anchor = NULL; + ubi->free_count++; + } - if (anchor) { - e1 = find_anchor_wl_entry(&ubi->used); + if (ubi->fm_do_produce_anchor) { if (!e1) goto out_cancel; e2 = get_peb_for_wl(ubi); @@ -705,6 +728,7 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, self_check_in_wl_tree(ubi, e1, &ubi->used); rb_erase(&e1->u.rb, &ubi->used); dbg_wl("anchor-move PEB %d to PEB %d", e1->pnum, e2->pnum); + ubi->fm_do_produce_anchor = 0; } else if (!ubi->scrub.rb_node) { #else if (!ubi->scrub.rb_node) { @@ -822,7 +846,14 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, goto out_not_moved; } if (err == MOVE_RETRY) { - scrubbing = 1; + /* + * For source PEB: + * 1. The scrubbing is set for scrub type PEB, it will + * be put back into ubi->scrub list. + * 2. Non-scrub type PEB will be put back into ubi->used + * list. + */ + keep = 1; dst_leb_clean = 1; goto out_not_moved; } @@ -878,8 +909,11 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, err = do_sync_erase(ubi, e1, vol_id, lnum, 0); if (err) { - if (e2) + if (e2) { + spin_lock(&ubi->wl_lock); wl_entry_destroy(ubi, e2); + spin_unlock(&ubi->wl_lock); + } goto out_ro; } @@ -961,11 +995,11 @@ out_error: spin_lock(&ubi->wl_lock); ubi->move_from = ubi->move_to = NULL; ubi->move_to_put = ubi->wl_scheduled = 0; + wl_entry_destroy(ubi, e1); + wl_entry_destroy(ubi, e2); spin_unlock(&ubi->wl_lock); ubi_free_vid_buf(vidb); - wl_entry_destroy(ubi, e1); - wl_entry_destroy(ubi, e2); out_ro: ubi_ro_mode(ubi); @@ -995,8 +1029,6 @@ out_cancel: static int ensure_wear_leveling(struct ubi_device *ubi, int nested) { int err = 0; - struct ubi_wl_entry *e1; - struct ubi_wl_entry *e2; struct ubi_work *wrk; spin_lock(&ubi->wl_lock); @@ -1006,9 +1038,16 @@ static int ensure_wear_leveling(struct ubi_device *ubi, int nested) /* * If the ubi->scrub tree is not empty, scrubbing is needed, and the - * the WL worker has to be scheduled anyway. + * WL worker has to be scheduled anyway. */ if (!ubi->scrub.rb_node) { +#ifdef CONFIG_MTD_UBI_FASTMAP + if (!need_wear_leveling(ubi)) + goto out_unlock; +#else + struct ubi_wl_entry *e1; + struct ubi_wl_entry *e2; + if (!ubi->used.rb_node || !ubi->free.rb_node) /* No physical eraseblocks - no deal */ goto out_unlock; @@ -1020,10 +1059,11 @@ static int ensure_wear_leveling(struct ubi_device *ubi, int nested) * %UBI_WL_THRESHOLD. */ e1 = rb_entry(rb_first(&ubi->used), struct ubi_wl_entry, u.rb); - e2 = find_wl_entry(ubi, &ubi->free, WL_FREE_MAX_DIFF); + e2 = find_wl_entry(ubi, &ubi->free, WL_FREE_MAX_DIFF, 0); if (!(e2->ec - e1->ec >= UBI_WL_THRESHOLD)) goto out_unlock; +#endif dbg_wl("schedule wear-leveling"); } else dbg_wl("schedule scrubbing"); @@ -1037,7 +1077,6 @@ static int ensure_wear_leveling(struct ubi_device *ubi, int nested) goto out_cancel; } - wrk->anchor = 0; wrk->func = &wear_leveling_worker; if (nested) __schedule_ubi_work(ubi, wrk); @@ -1057,8 +1096,6 @@ out_unlock: * __erase_worker - physical eraseblock erase worker function. * @ubi: UBI device description object * @wl_wrk: the work object - * @shutdown: non-zero if the worker has to free memory and exit - * because the WL sub-system is shutting down * * This function erases a physical eraseblock and perform torture testing if * needed. It also takes care about marking the physical eraseblock bad if @@ -1076,11 +1113,23 @@ static int __erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk) dbg_wl("erase PEB %d EC %d LEB %d:%d", pnum, e->ec, wl_wrk->vol_id, wl_wrk->lnum); - err = sync_erase(ubi, e, wl_wrk->torture); + err = ubi_sync_erase(ubi, e, wl_wrk->torture); if (!err) { spin_lock(&ubi->wl_lock); - wl_tree_add(e, &ubi->free); - ubi->free_count++; + + if (!ubi->fm_disabled && !ubi->fm_anchor && + e->pnum < UBI_FM_MAX_START) { + /* + * Abort anchor production, if needed it will be + * enabled again in the wear leveling started below. + */ + ubi->fm_anchor = e; + ubi->fm_do_produce_anchor = 0; + } else { + wl_tree_add(e, &ubi->free); + ubi->free_count++; + } + spin_unlock(&ubi->wl_lock); /* @@ -1101,16 +1150,20 @@ static int __erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk) int err1; /* Re-schedule the LEB for erasure */ - err1 = schedule_erase(ubi, e, vol_id, lnum, 0, false); + err1 = schedule_erase(ubi, e, vol_id, lnum, 0, true); if (err1) { + spin_lock(&ubi->wl_lock); wl_entry_destroy(ubi, e); + spin_unlock(&ubi->wl_lock); err = err1; goto out_ro; } return err; } + spin_lock(&ubi->wl_lock); wl_entry_destroy(ubi, e); + spin_unlock(&ubi->wl_lock); if (err != -EIO) /* * If this is not %-EIO, we have no idea what to do. Scheduling @@ -1226,6 +1279,18 @@ int ubi_wl_put_peb(struct ubi_device *ubi, int vol_id, int lnum, retry: spin_lock(&ubi->wl_lock); e = ubi->lookuptbl[pnum]; + if (!e) { + /* + * This wl entry has been removed for some errors by other + * process (eg. wear leveling worker), corresponding process + * (except __erase_worker, which cannot concurrent with + * ubi_wl_put_peb) will set ubi ro_mode at the same time, + * just ignore this wl entry. + */ + spin_unlock(&ubi->wl_lock); + up_read(&ubi->fm_protect); + return 0; + } if (e == ubi->move_from) { /* * User is putting the physical eraseblock which was selected to @@ -1419,6 +1484,150 @@ int ubi_wl_flush(struct ubi_device *ubi, int vol_id, int lnum) return err; } +static bool scrub_possible(struct ubi_device *ubi, struct ubi_wl_entry *e) +{ + if (in_wl_tree(e, &ubi->scrub)) + return false; + else if (in_wl_tree(e, &ubi->erroneous)) + return false; + else if (ubi->move_from == e) + return false; + else if (ubi->move_to == e) + return false; + + return true; +} + +/** + * ubi_bitflip_check - Check an eraseblock for bitflips and scrub it if needed. + * @ubi: UBI device description object + * @pnum: the physical eraseblock to schedule + * @force: don't read the block, assume bitflips happened and take action. + * + * This function reads the given eraseblock and checks if bitflips occured. + * In case of bitflips, the eraseblock is scheduled for scrubbing. + * If scrubbing is forced with @force, the eraseblock is not read, + * but scheduled for scrubbing right away. + * + * Returns: + * %EINVAL, PEB is out of range + * %ENOENT, PEB is no longer used by UBI + * %EBUSY, PEB cannot be checked now or a check is currently running on it + * %EAGAIN, bit flips happened but scrubbing is currently not possible + * %EUCLEAN, bit flips happened and PEB is scheduled for scrubbing + * %0, no bit flips detected + */ +int ubi_bitflip_check(struct ubi_device *ubi, int pnum, int force) +{ + int err = 0; + struct ubi_wl_entry *e; + + if (pnum < 0 || pnum >= ubi->peb_count) { + err = -EINVAL; + goto out; + } + + /* + * Pause all parallel work, otherwise it can happen that the + * erase worker frees a wl entry under us. + */ + down_write(&ubi->work_sem); + + /* + * Make sure that the wl entry does not change state while + * inspecting it. + */ + spin_lock(&ubi->wl_lock); + e = ubi->lookuptbl[pnum]; + if (!e) { + spin_unlock(&ubi->wl_lock); + err = -ENOENT; + goto out_resume; + } + + /* + * Does it make sense to check this PEB? + */ + if (!scrub_possible(ubi, e)) { + spin_unlock(&ubi->wl_lock); + err = -EBUSY; + goto out_resume; + } + spin_unlock(&ubi->wl_lock); + + if (!force) { + mutex_lock(&ubi->buf_mutex); + err = ubi_io_read(ubi, ubi->peb_buf, pnum, 0, ubi->peb_size); + mutex_unlock(&ubi->buf_mutex); + } + + if (force || err == UBI_IO_BITFLIPS) { + /* + * Okay, bit flip happened, let's figure out what we can do. + */ + spin_lock(&ubi->wl_lock); + + /* + * Recheck. We released wl_lock, UBI might have killed the + * wl entry under us. + */ + e = ubi->lookuptbl[pnum]; + if (!e) { + spin_unlock(&ubi->wl_lock); + err = -ENOENT; + goto out_resume; + } + + /* + * Need to re-check state + */ + if (!scrub_possible(ubi, e)) { + spin_unlock(&ubi->wl_lock); + err = -EBUSY; + goto out_resume; + } + + if (in_pq(ubi, e)) { + prot_queue_del(ubi, e->pnum); + wl_tree_add(e, &ubi->scrub); + spin_unlock(&ubi->wl_lock); + + err = ensure_wear_leveling(ubi, 1); + } else if (in_wl_tree(e, &ubi->used)) { + rb_erase(&e->u.rb, &ubi->used); + wl_tree_add(e, &ubi->scrub); + spin_unlock(&ubi->wl_lock); + + err = ensure_wear_leveling(ubi, 1); + } else if (in_wl_tree(e, &ubi->free)) { + rb_erase(&e->u.rb, &ubi->free); + ubi->free_count--; + spin_unlock(&ubi->wl_lock); + + /* + * This PEB is empty we can schedule it for + * erasure right away. No wear leveling needed. + */ + err = schedule_erase(ubi, e, UBI_UNKNOWN, UBI_UNKNOWN, + force ? 0 : 1, true); + } else { + spin_unlock(&ubi->wl_lock); + err = -EAGAIN; + } + + if (!err && !force) + err = -EUCLEAN; + } else { + err = 0; + } + +out_resume: + up_write(&ubi->work_sem); +out: + + return err; +} + /** * tree_destroy - destroy an RB-tree. * @ubi: UBI device description object @@ -1478,12 +1687,25 @@ int ubi_thread(void *u) !ubi->thread_enabled || ubi_dbg_is_bgt_disabled(ubi)) { set_current_state(TASK_INTERRUPTIBLE); spin_unlock(&ubi->wl_lock); + + /* + * Check kthread_should_stop() after we set the task + * state to guarantee that we either see the stop bit + * and exit or the task state is reset to runnable such + * that it's not scheduled out indefinitely and detects + * the stop bit at kthread_should_stop(). + */ + if (kthread_should_stop()) { + set_current_state(TASK_RUNNING); + break; + } + schedule(); continue; } spin_unlock(&ubi->wl_lock); - err = do_work(ubi); + err = do_work(ubi, NULL); if (err) { ubi_err(ubi, "%s: work failed with error code %d", ubi->bgt_name, err); @@ -1546,7 +1768,7 @@ static int erase_aeb(struct ubi_device *ubi, struct ubi_ainf_peb *aeb, bool sync ubi->lookuptbl[e->pnum] = e; if (sync) { - err = sync_erase(ubi, e, false); + err = ubi_sync_erase(ubi, e, false); if (err) goto out_free; @@ -1724,6 +1946,10 @@ int ubi_wl_init(struct ubi_device *ubi, struct ubi_attach_info *ai) if (err) goto out_free; +#ifdef CONFIG_MTD_UBI_FASTMAP + if (!ubi->ro_mode && !ubi->fm_disabled) + ubi_ensure_anchor_pebs(ubi); +#endif return 0; out_free: @@ -1848,16 +2074,11 @@ static int self_check_in_wl_tree(const struct ubi_device *ubi, static int self_check_in_pq(const struct ubi_device *ubi, struct ubi_wl_entry *e) { - struct ubi_wl_entry *p; - int i; - if (!ubi_dbg_chk_gen(ubi)) return 0; - for (i = 0; i < UBI_PROT_QUEUE_LEN; ++i) - list_for_each_entry(p, &ubi->pq[i], u.list) - if (p == e) - return 0; + if (in_pq(ubi, e)) + return 0; ubi_err(ubi, "self-check failed for PEB %d, EC %d, Protect queue", e->pnum, e->ec); @@ -1869,7 +2090,7 @@ static struct ubi_wl_entry *get_peb_for_wl(struct ubi_device *ubi) { struct ubi_wl_entry *e; - e = find_wl_entry(ubi, &ubi->free, WL_FREE_MAX_DIFF); + e = find_wl_entry(ubi, &ubi->free, WL_FREE_MAX_DIFF, 0); self_check_in_wl_tree(ubi, e, &ubi->free); ubi->free_count--; ubi_assert(ubi->free_count >= 0); @@ -1895,7 +2116,7 @@ static int produce_free_peb(struct ubi_device *ubi) spin_unlock(&ubi->wl_lock); dbg_wl("do one work synchronously"); - err = do_work(ubi); + err = do_work(ubi, NULL); spin_lock(&ubi->wl_lock); if (err) |