diff options
Diffstat (limited to 'drivers/mtd/ubi/wl.c')
| -rw-r--r-- | drivers/mtd/ubi/wl.c | 1246 |
1 files changed, 647 insertions, 599 deletions
diff --git a/drivers/mtd/ubi/wl.c b/drivers/mtd/ubi/wl.c index 5df49d3cb5c7..fbd399cf6503 100644 --- a/drivers/mtd/ubi/wl.c +++ b/drivers/mtd/ubi/wl.c @@ -1,20 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (c) International Business Machines Corp., 2006 * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See - * the GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * * Authors: Artem Bityutskiy (Битюцкий Артём), Thomas Gleixner */ @@ -103,6 +90,7 @@ #include <linux/freezer.h> #include <linux/kthread.h> #include "ubi.h" +#include "wl.h" /* Number of physical eraseblocks reserved for wear-leveling purposes */ #define WL_RESERVED_PEBS 1 @@ -140,42 +128,6 @@ static int self_check_in_wl_tree(const struct ubi_device *ubi, static int self_check_in_pq(const struct ubi_device *ubi, struct ubi_wl_entry *e); -#ifdef CONFIG_MTD_UBI_FASTMAP -/** - * update_fastmap_work_fn - calls ubi_update_fastmap from a work queue - * @wrk: the work description object - */ -static void update_fastmap_work_fn(struct work_struct *wrk) -{ - struct ubi_device *ubi = container_of(wrk, struct ubi_device, fm_work); - ubi_update_fastmap(ubi); -} - -/** - * ubi_ubi_is_fm_block - returns 1 if a PEB is currently used in a fastmap. - * @ubi: UBI device description object - * @pnum: the to be checked PEB - */ -static int ubi_is_fm_block(struct ubi_device *ubi, int pnum) -{ - int i; - - if (!ubi->fm) - return 0; - - for (i = 0; i < ubi->fm->used_blocks; i++) - if (ubi->fm->e[i]->pnum == pnum) - return 1; - - return 0; -} -#else -static int ubi_is_fm_block(struct ubi_device *ubi, int pnum) -{ - return 0; -} -#endif - /** * wl_tree_add - add a wear-leveling entry to a WL RB-tree. * @e: the wear-leveling entry to add @@ -213,13 +165,29 @@ static void wl_tree_add(struct ubi_wl_entry *e, struct rb_root *root) } /** + * wl_entry_destroy - destroy a wear-leveling entry. + * @ubi: UBI device description object + * @e: the wear-leveling entry to add + * + * This function destroys a wear leveling entry and removes + * the reference from the lookup table. + */ +static void wl_entry_destroy(struct ubi_device *ubi, struct ubi_wl_entry *e) +{ + ubi->lookuptbl[e->pnum] = NULL; + kmem_cache_free(ubi_wl_entry_slab, e); +} + +/** * do_work - do one pending work. * @ubi: UBI device description object + * @executed: whether there is one work is executed * * This function returns zero in case of success and a negative error code in - * case of failure. + * case of failure. If @executed is not NULL and there is one work executed, + * @executed is set as %1, otherwise @executed is set as %0. */ -static int do_work(struct ubi_device *ubi) +static int do_work(struct ubi_device *ubi, int *executed) { int err; struct ubi_work *wrk; @@ -237,9 +205,13 @@ static int do_work(struct ubi_device *ubi) if (list_empty(&ubi->works)) { spin_unlock(&ubi->wl_lock); up_read(&ubi->work_sem); + if (executed) + *executed = 0; return 0; } + if (executed) + *executed = 1; wrk = list_entry(ubi->works.next, struct ubi_work, list); list_del(&wrk->list); ubi->works_count -= 1; @@ -253,40 +225,13 @@ static int do_work(struct ubi_device *ubi) */ err = wrk->func(ubi, wrk, 0); if (err) - ubi_err("work failed with error code %d", err); + ubi_err(ubi, "work failed with error code %d", err); up_read(&ubi->work_sem); return err; } /** - * produce_free_peb - produce a free physical eraseblock. - * @ubi: UBI device description object - * - * This function tries to make a free PEB by means of synchronous execution of - * pending works. This may be needed if, for example the background thread is - * disabled. Returns zero in case of success and a negative error code in case - * of failure. - */ -static int produce_free_peb(struct ubi_device *ubi) -{ - int err; - - while (!ubi->free.rb_node) { - spin_unlock(&ubi->wl_lock); - - dbg_wl("do one work synchronously"); - err = do_work(ubi); - - spin_lock(&ubi->wl_lock); - if (err) - return err; - } - - return 0; -} - -/** * in_wl_tree - check if wear-leveling entry is present in a WL RB-tree. * @e: the wear-leveling entry to check * @root: the root of the tree @@ -326,6 +271,27 @@ static int in_wl_tree(struct ubi_wl_entry *e, struct rb_root *root) } /** + * in_pq - check if a wear-leveling entry is present in the protection queue. + * @ubi: UBI device description object + * @e: the wear-leveling entry to check + * + * This function returns non-zero if @e is in the protection queue and zero + * if it is not. + */ +static inline int in_pq(const struct ubi_device *ubi, struct ubi_wl_entry *e) +{ + struct ubi_wl_entry *p; + int i; + + for (i = 0; i < UBI_PROT_QUEUE_LEN; ++i) + list_for_each_entry(p, &ubi->pq[i], u.list) + if (p == e) + return 1; + + return 0; +} + +/** * prot_queue_add - add physical eraseblock to the protection queue. * @ubi: UBI device description object * @e: the physical eraseblock to add @@ -351,15 +317,17 @@ static void prot_queue_add(struct ubi_device *ubi, struct ubi_wl_entry *e) * @ubi: UBI device description object * @root: the RB-tree where to look for * @diff: maximum possible difference from the smallest erase counter + * @pick_max: pick PEB even its erase counter beyonds 'min_ec + @diff' * * This function looks for a wear leveling entry with erase counter closest to * min + @diff, where min is the smallest erase counter. */ static struct ubi_wl_entry *find_wl_entry(struct ubi_device *ubi, - struct rb_root *root, int diff) + struct rb_root *root, int diff, + int pick_max) { struct rb_node *p; - struct ubi_wl_entry *e, *prev_e = NULL; + struct ubi_wl_entry *e; int max; e = rb_entry(rb_first(root), struct ubi_wl_entry, u.rb); @@ -370,22 +338,16 @@ static struct ubi_wl_entry *find_wl_entry(struct ubi_device *ubi, struct ubi_wl_entry *e1; e1 = rb_entry(p, struct ubi_wl_entry, u.rb); - if (e1->ec >= max) + if (e1->ec >= max) { + if (pick_max) + e = e1; p = p->rb_left; - else { + } else { p = p->rb_right; - prev_e = e; e = e1; } } - /* If no fastmap has been written and this WL entry can be used - * as anchor PEB, hold it back and return the second best WL entry - * such that fastmap can use the anchor PEB later. */ - if (prev_e && !ubi->fm_disabled && - !ubi->fm && e->pnum < UBI_FM_MAX_START) - return prev_e; - return e; } @@ -409,119 +371,35 @@ static struct ubi_wl_entry *find_mean_wl_entry(struct ubi_device *ubi, if (last->ec - first->ec < WL_FREE_MAX_DIFF) { e = rb_entry(root->rb_node, struct ubi_wl_entry, u.rb); -#ifdef CONFIG_MTD_UBI_FASTMAP - /* If no fastmap has been written and this WL entry can be used - * as anchor PEB, hold it back and return the second best - * WL entry such that fastmap can use the anchor PEB later. */ - if (e && !ubi->fm_disabled && !ubi->fm && - e->pnum < UBI_FM_MAX_START) - e = rb_entry(rb_next(root->rb_node), - struct ubi_wl_entry, u.rb); -#endif + /* + * If no fastmap has been written and fm_anchor is not + * reserved and this WL entry can be used as anchor PEB + * hold it back and return the second best WL entry such + * that fastmap can use the anchor PEB later. + */ + e = may_reserve_for_fm(ubi, e, root); } else - e = find_wl_entry(ubi, root, WL_FREE_MAX_DIFF/2); + e = find_wl_entry(ubi, root, WL_FREE_MAX_DIFF/2, 0); return e; } -#ifdef CONFIG_MTD_UBI_FASTMAP -/** - * find_anchor_wl_entry - find wear-leveling entry to used as anchor PEB. - * @root: the RB-tree where to look for - */ -static struct ubi_wl_entry *find_anchor_wl_entry(struct rb_root *root) -{ - struct rb_node *p; - struct ubi_wl_entry *e, *victim = NULL; - int max_ec = UBI_MAX_ERASECOUNTER; - - ubi_rb_for_each_entry(p, e, root, u.rb) { - if (e->pnum < UBI_FM_MAX_START && e->ec < max_ec) { - victim = e; - max_ec = e->ec; - } - } - - return victim; -} - -static int anchor_pebs_avalible(struct rb_root *root) -{ - struct rb_node *p; - struct ubi_wl_entry *e; - - ubi_rb_for_each_entry(p, e, root, u.rb) - if (e->pnum < UBI_FM_MAX_START) - return 1; - - return 0; -} - /** - * ubi_wl_get_fm_peb - find a physical erase block with a given maximal number. + * wl_get_wle - get a mean wl entry to be used by ubi_wl_get_peb() or + * refill_wl_user_pool(). * @ubi: UBI device description object - * @anchor: This PEB will be used as anchor PEB by fastmap * - * The function returns a physical erase block with a given maximal number - * and removes it from the wl subsystem. - * Must be called with wl_lock held! + * This function returns a wear leveling entry in case of success and + * NULL in case of failure. */ -struct ubi_wl_entry *ubi_wl_get_fm_peb(struct ubi_device *ubi, int anchor) +static struct ubi_wl_entry *wl_get_wle(struct ubi_device *ubi) { - struct ubi_wl_entry *e = NULL; - - if (!ubi->free.rb_node || (ubi->free_count - ubi->beb_rsvd_pebs < 1)) - goto out; - - if (anchor) - e = find_anchor_wl_entry(&ubi->free); - else - e = find_mean_wl_entry(ubi, &ubi->free); - - if (!e) - goto out; - - self_check_in_wl_tree(ubi, e, &ubi->free); - - /* remove it from the free list, - * the wl subsystem does no longer know this erase block */ - rb_erase(&e->u.rb, &ubi->free); - ubi->free_count--; -out: - return e; -} -#endif - -/** - * __wl_get_peb - get a physical eraseblock. - * @ubi: UBI device description object - * - * This function returns a physical eraseblock in case of success and a - * negative error code in case of failure. - */ -static int __wl_get_peb(struct ubi_device *ubi) -{ - int err; struct ubi_wl_entry *e; -retry: - if (!ubi->free.rb_node) { - if (ubi->works_count == 0) { - ubi_err("no free eraseblocks"); - ubi_assert(list_empty(&ubi->works)); - return -ENOSPC; - } - - err = produce_free_peb(ubi); - if (err < 0) - return err; - goto retry; - } - e = find_mean_wl_entry(ubi, &ubi->free); if (!e) { - ubi_err("no free eraseblocks"); - return -ENOSPC; + ubi_err(ubi, "no free eraseblocks"); + return NULL; } self_check_in_wl_tree(ubi, e, &ubi->free); @@ -533,172 +411,10 @@ retry: rb_erase(&e->u.rb, &ubi->free); ubi->free_count--; dbg_wl("PEB %d EC %d", e->pnum, e->ec); -#ifndef CONFIG_MTD_UBI_FASTMAP - /* We have to enqueue e only if fastmap is disabled, - * is fastmap enabled prot_queue_add() will be called by - * ubi_wl_get_peb() after removing e from the pool. */ - prot_queue_add(ubi, e); -#endif - return e->pnum; -} - -#ifdef CONFIG_MTD_UBI_FASTMAP -/** - * return_unused_pool_pebs - returns unused PEB to the free tree. - * @ubi: UBI device description object - * @pool: fastmap pool description object - */ -static void return_unused_pool_pebs(struct ubi_device *ubi, - struct ubi_fm_pool *pool) -{ - int i; - struct ubi_wl_entry *e; - - for (i = pool->used; i < pool->size; i++) { - e = ubi->lookuptbl[pool->pebs[i]]; - wl_tree_add(e, &ubi->free); - ubi->free_count++; - } -} - -/** - * refill_wl_pool - refills all the fastmap pool used by the - * WL sub-system. - * @ubi: UBI device description object - */ -static void refill_wl_pool(struct ubi_device *ubi) -{ - struct ubi_wl_entry *e; - struct ubi_fm_pool *pool = &ubi->fm_wl_pool; - - return_unused_pool_pebs(ubi, pool); - - for (pool->size = 0; pool->size < pool->max_size; pool->size++) { - if (!ubi->free.rb_node || - (ubi->free_count - ubi->beb_rsvd_pebs < 5)) - break; - - e = find_wl_entry(ubi, &ubi->free, WL_FREE_MAX_DIFF); - self_check_in_wl_tree(ubi, e, &ubi->free); - rb_erase(&e->u.rb, &ubi->free); - ubi->free_count--; - - pool->pebs[pool->size] = e->pnum; - } - pool->used = 0; -} - -/** - * refill_wl_user_pool - refills all the fastmap pool used by ubi_wl_get_peb. - * @ubi: UBI device description object - */ -static void refill_wl_user_pool(struct ubi_device *ubi) -{ - struct ubi_fm_pool *pool = &ubi->fm_pool; - - return_unused_pool_pebs(ubi, pool); - - for (pool->size = 0; pool->size < pool->max_size; pool->size++) { - if (!ubi->free.rb_node || - (ubi->free_count - ubi->beb_rsvd_pebs < 1)) - break; - - pool->pebs[pool->size] = __wl_get_peb(ubi); - if (pool->pebs[pool->size] < 0) - break; - } - pool->used = 0; -} - -/** - * ubi_refill_pools - refills all fastmap PEB pools. - * @ubi: UBI device description object - */ -void ubi_refill_pools(struct ubi_device *ubi) -{ - spin_lock(&ubi->wl_lock); - refill_wl_pool(ubi); - refill_wl_user_pool(ubi); - spin_unlock(&ubi->wl_lock); -} - -/* ubi_wl_get_peb - works exaclty like __wl_get_peb but keeps track of - * the fastmap pool. - */ -int ubi_wl_get_peb(struct ubi_device *ubi) -{ - int ret; - struct ubi_fm_pool *pool = &ubi->fm_pool; - struct ubi_fm_pool *wl_pool = &ubi->fm_wl_pool; - - if (!pool->size || !wl_pool->size || pool->used == pool->size || - wl_pool->used == wl_pool->size) - ubi_update_fastmap(ubi); - - /* we got not a single free PEB */ - if (!pool->size) - ret = -ENOSPC; - else { - spin_lock(&ubi->wl_lock); - ret = pool->pebs[pool->used++]; - prot_queue_add(ubi, ubi->lookuptbl[ret]); - spin_unlock(&ubi->wl_lock); - } - - return ret; -} - -/* get_peb_for_wl - returns a PEB to be used internally by the WL sub-system. - * - * @ubi: UBI device description object - */ -static struct ubi_wl_entry *get_peb_for_wl(struct ubi_device *ubi) -{ - struct ubi_fm_pool *pool = &ubi->fm_wl_pool; - int pnum; - - if (pool->used == pool->size || !pool->size) { - /* We cannot update the fastmap here because this - * function is called in atomic context. - * Let's fail here and refill/update it as soon as possible. */ - schedule_work(&ubi->fm_work); - return NULL; - } else { - pnum = pool->pebs[pool->used++]; - return ubi->lookuptbl[pnum]; - } -} -#else -static struct ubi_wl_entry *get_peb_for_wl(struct ubi_device *ubi) -{ - struct ubi_wl_entry *e; - - e = find_wl_entry(ubi, &ubi->free, WL_FREE_MAX_DIFF); - self_check_in_wl_tree(ubi, e, &ubi->free); - rb_erase(&e->u.rb, &ubi->free); return e; } -int ubi_wl_get_peb(struct ubi_device *ubi) -{ - int peb, err; - - spin_lock(&ubi->wl_lock); - peb = __wl_get_peb(ubi); - spin_unlock(&ubi->wl_lock); - - err = ubi_self_check_all_ff(ubi, peb, ubi->vid_hdr_aloffset, - ubi->peb_size - ubi->vid_hdr_aloffset); - if (err) { - ubi_err("new PEB %d does not contain all 0xFF bytes", peb); - return err; - } - - return peb; -} -#endif - /** * prot_queue_del - remove a physical eraseblock from the protection queue. * @ubi: UBI device description object @@ -724,16 +440,15 @@ static int prot_queue_del(struct ubi_device *ubi, int pnum) } /** - * sync_erase - synchronously erase a physical eraseblock. + * ubi_sync_erase - synchronously erase a physical eraseblock. * @ubi: UBI device description object - * @e: the the physical eraseblock to erase + * @e: the physical eraseblock to erase * @torture: if the physical eraseblock has to be tortured * * This function returns zero in case of success and a negative error code in * case of failure. */ -static int sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, - int torture) +int ubi_sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, int torture) { int err; struct ubi_ec_hdr *ec_hdr; @@ -759,7 +474,7 @@ static int sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, * Erase counter overflow. Upgrade UBI and use 64-bit * erase counters internally. */ - ubi_err("erase counter overflow at PEB %d, EC %llu", + ubi_err(ubi, "erase counter overflow at PEB %d, EC %llu", e->pnum, ec); err = -EINVAL; goto out_free; @@ -834,7 +549,7 @@ repeat: * @wrk: the work to schedule * * This function adds a work defined by @wrk to the tail of the pending works - * list. Can only be used of ubi->work_sem is already held in read mode! + * list. Can only be used if ubi->work_sem is already held in read mode! */ static void __schedule_ubi_work(struct ubi_device *ubi, struct ubi_work *wrk) { @@ -863,18 +578,7 @@ static void schedule_ubi_work(struct ubi_device *ubi, struct ubi_work *wrk) } static int erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk, - int cancel); - -#ifdef CONFIG_MTD_UBI_FASTMAP -/** - * ubi_is_erase_work - checks whether a work is erase work. - * @wrk: The work object to be checked - */ -int ubi_is_erase_work(struct ubi_work *wrk) -{ - return wrk->func == erase_worker; -} -#endif + int shutdown); /** * schedule_erase - schedule an erase work. @@ -883,17 +587,17 @@ int ubi_is_erase_work(struct ubi_work *wrk) * @vol_id: the volume ID that last used this PEB * @lnum: the last used logical eraseblock number for the PEB * @torture: if the physical eraseblock has to be tortured + * @nested: denotes whether the work_sem is already held * * This function returns zero in case of success and a %-ENOMEM in case of * failure. */ static int schedule_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, - int vol_id, int lnum, int torture) + int vol_id, int lnum, int torture, bool nested) { struct ubi_work *wl_wrk; ubi_assert(e); - ubi_assert(!ubi_is_fm_block(ubi, e->pnum)); dbg_wl("schedule erasure of PEB %d, EC %d, torture %d", e->pnum, e->ec, torture); @@ -908,10 +612,14 @@ static int schedule_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, wl_wrk->lnum = lnum; wl_wrk->torture = torture; - schedule_ubi_work(ubi, wl_wrk); + if (nested) + __schedule_ubi_work(ubi, wl_wrk); + else + schedule_ubi_work(ubi, wl_wrk); return 0; } +static int __erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk); /** * do_sync_erase - run the erase worker synchronously. * @ubi: UBI device description object @@ -924,102 +632,61 @@ static int schedule_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, static int do_sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, int vol_id, int lnum, int torture) { - struct ubi_work *wl_wrk; + struct ubi_work wl_wrk; dbg_wl("sync erase of PEB %i", e->pnum); - wl_wrk = kmalloc(sizeof(struct ubi_work), GFP_NOFS); - if (!wl_wrk) - return -ENOMEM; - - wl_wrk->e = e; - wl_wrk->vol_id = vol_id; - wl_wrk->lnum = lnum; - wl_wrk->torture = torture; - - return erase_worker(ubi, wl_wrk, 0); -} - -#ifdef CONFIG_MTD_UBI_FASTMAP -/** - * ubi_wl_put_fm_peb - returns a PEB used in a fastmap to the wear-leveling - * sub-system. - * see: ubi_wl_put_peb() - * - * @ubi: UBI device description object - * @fm_e: physical eraseblock to return - * @lnum: the last used logical eraseblock number for the PEB - * @torture: if this physical eraseblock has to be tortured - */ -int ubi_wl_put_fm_peb(struct ubi_device *ubi, struct ubi_wl_entry *fm_e, - int lnum, int torture) -{ - struct ubi_wl_entry *e; - int vol_id, pnum = fm_e->pnum; - - dbg_wl("PEB %d", pnum); - - ubi_assert(pnum >= 0); - ubi_assert(pnum < ubi->peb_count); - - spin_lock(&ubi->wl_lock); - e = ubi->lookuptbl[pnum]; - - /* This can happen if we recovered from a fastmap the very - * first time and writing now a new one. In this case the wl system - * has never seen any PEB used by the original fastmap. - */ - if (!e) { - e = fm_e; - ubi_assert(e->ec >= 0); - ubi->lookuptbl[pnum] = e; - } else { - e->ec = fm_e->ec; - kfree(fm_e); - } - - spin_unlock(&ubi->wl_lock); + wl_wrk.e = e; + wl_wrk.vol_id = vol_id; + wl_wrk.lnum = lnum; + wl_wrk.torture = torture; - vol_id = lnum ? UBI_FM_DATA_VOLUME_ID : UBI_FM_SB_VOLUME_ID; - return schedule_erase(ubi, e, vol_id, lnum, torture); + return __erase_worker(ubi, &wl_wrk); } -#endif +static int ensure_wear_leveling(struct ubi_device *ubi, int nested); /** * wear_leveling_worker - wear-leveling worker function. * @ubi: UBI device description object * @wrk: the work object - * @cancel: non-zero if the worker has to free memory and exit + * @shutdown: non-zero if the worker has to free memory and exit + * because the WL-subsystem is shutting down * * This function copies a more worn out physical eraseblock to a less worn out * one. Returns zero in case of success and a negative error code in case of * failure. */ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, - int cancel) + int shutdown) { int err, scrubbing = 0, torture = 0, protect = 0, erroneous = 0; - int vol_id = -1, uninitialized_var(lnum); -#ifdef CONFIG_MTD_UBI_FASTMAP - int anchor = wrk->anchor; -#endif + int erase = 0, keep = 0, vol_id = -1, lnum = -1; struct ubi_wl_entry *e1, *e2; + struct ubi_vid_io_buf *vidb; struct ubi_vid_hdr *vid_hdr; + int dst_leb_clean = 0; kfree(wrk); - if (cancel) + if (shutdown) return 0; - vid_hdr = ubi_zalloc_vid_hdr(ubi, GFP_NOFS); - if (!vid_hdr) + vidb = ubi_alloc_vid_buf(ubi, GFP_NOFS); + if (!vidb) return -ENOMEM; + vid_hdr = ubi_get_vid_hdr(vidb); + + down_read(&ubi->fm_eba_sem); mutex_lock(&ubi->move_mutex); spin_lock(&ubi->wl_lock); ubi_assert(!ubi->move_from && !ubi->move_to); ubi_assert(!ubi->move_to_put); +#ifdef CONFIG_MTD_UBI_FASTMAP + if (!next_peb_for_wl(ubi, true) || +#else if (!ubi->free.rb_node || +#endif (!ubi->used.rb_node && !ubi->scrub.rb_node)) { /* * No free physical eraseblocks? Well, they must be waiting in @@ -1037,12 +704,21 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, } #ifdef CONFIG_MTD_UBI_FASTMAP - /* Check whether we need to produce an anchor PEB */ - if (!anchor) - anchor = !anchor_pebs_avalible(&ubi->free); + e1 = find_anchor_wl_entry(&ubi->used); + if (e1 && ubi->fm_anchor && + (ubi->fm_anchor->ec - e1->ec >= UBI_WL_THRESHOLD)) { + ubi->fm_do_produce_anchor = 1; + /* + * fm_anchor is no longer considered a good anchor. + * NULL assignment also prevents multiple wear level checks + * of this PEB. + */ + wl_tree_add(ubi->fm_anchor, &ubi->free); + ubi->fm_anchor = NULL; + ubi->free_count++; + } - if (anchor) { - e1 = find_anchor_wl_entry(&ubi->used); + if (ubi->fm_do_produce_anchor) { if (!e1) goto out_cancel; e2 = get_peb_for_wl(ubi); @@ -1052,6 +728,7 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, self_check_in_wl_tree(ubi, e1, &ubi->used); rb_erase(&e1->u.rb, &ubi->used); dbg_wl("anchor-move PEB %d to PEB %d", e1->pnum, e2->pnum); + ubi->fm_do_produce_anchor = 0; } else if (!ubi->scrub.rb_node) { #else if (!ubi->scrub.rb_node) { @@ -1069,6 +746,10 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, if (!(e2->ec - e1->ec >= UBI_WL_THRESHOLD)) { dbg_wl("no WL needed: min used EC %d, max free EC %d", e1->ec, e2->ec); + + /* Give the unused PEB back */ + wl_tree_add(e2, &ubi->free); + ubi->free_count++; goto out_cancel; } self_check_in_wl_tree(ubi, e1, &ubi->used); @@ -1103,8 +784,9 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, * which is being moved was unmapped. */ - err = ubi_io_read_vid_hdr(ubi, e1->pnum, vid_hdr, 0); + err = ubi_io_read_vid_hdr(ubi, e1->pnum, vidb, 0); if (err && err != UBI_IO_BITFLIPS) { + dst_leb_clean = 1; if (err == UBI_IO_FF) { /* * We are trying to move PEB without a VID header. UBI @@ -1129,9 +811,19 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, e1->pnum); scrubbing = 1; goto out_not_moved; + } else if (ubi->fast_attach && err == UBI_IO_BAD_HDR_EBADMSG) { + /* + * While a full scan would detect interrupted erasures + * at attach time we can face them here when attached from + * Fastmap. + */ + dbg_wl("PEB %d has ECC errors, maybe from an interrupted erasure", + e1->pnum); + erase = 1; + goto out_not_moved; } - ubi_err("error %d while reading VID header from PEB %d", + ubi_err(ubi, "error %d while reading VID header from PEB %d", err, e1->pnum); goto out_error; } @@ -1139,7 +831,7 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, vol_id = be32_to_cpu(vid_hdr->vol_id); lnum = be32_to_cpu(vid_hdr->lnum); - err = ubi_eba_copy_leb(ubi, e1->pnum, e2->pnum, vid_hdr); + err = ubi_eba_copy_leb(ubi, e1->pnum, e2->pnum, vidb); if (err) { if (err == MOVE_CANCEL_RACE) { /* @@ -1150,10 +842,19 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, * protection queue. */ protect = 1; + dst_leb_clean = 1; goto out_not_moved; } if (err == MOVE_RETRY) { - scrubbing = 1; + /* + * For source PEB: + * 1. The scrubbing is set for scrub type PEB, it will + * be put back into ubi->scrub list. + * 2. Non-scrub type PEB will be put back into ubi->used + * list. + */ + keep = 1; + dst_leb_clean = 1; goto out_not_moved; } if (err == MOVE_TARGET_BITFLIPS || err == MOVE_TARGET_WR_ERR || @@ -1162,6 +863,7 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, * Target PEB had bit-flips or write error - torture it. */ torture = 1; + keep = 1; goto out_not_moved; } @@ -1175,10 +877,11 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, * UBI from trying to move it over and over again. */ if (ubi->erroneous_peb_count > ubi->max_erroneous) { - ubi_err("too many erroneous eraseblocks (%d)", + ubi_err(ubi, "too many erroneous eraseblocks (%d)", ubi->erroneous_peb_count); goto out_error; } + dst_leb_clean = 1; erroneous = 1; goto out_not_moved; } @@ -1191,9 +894,9 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, /* The PEB has been successfully moved */ if (scrubbing) - ubi_msg("scrubbed PEB %d (LEB %d:%d), data moved to PEB %d", + ubi_msg(ubi, "scrubbed PEB %d (LEB %d:%d), data moved to PEB %d", e1->pnum, vol_id, lnum, e2->pnum); - ubi_free_vid_hdr(ubi, vid_hdr); + ubi_free_vid_buf(vidb); spin_lock(&ubi->wl_lock); if (!ubi->move_to_put) { @@ -1206,9 +909,11 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, err = do_sync_erase(ubi, e1, vol_id, lnum, 0); if (err) { - kmem_cache_free(ubi_wl_entry_slab, e1); - if (e2) - kmem_cache_free(ubi_wl_entry_slab, e2); + if (e2) { + spin_lock(&ubi->wl_lock); + wl_entry_destroy(ubi, e2); + spin_unlock(&ubi->wl_lock); + } goto out_ro; } @@ -1220,14 +925,13 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk, dbg_wl("PEB %d (LEB %d:%d) was put meanwhile, erase", e2->pnum, vol_id, lnum); err = do_sync_erase(ubi, e2, vol_id, lnum, 0); - if (err) { - kmem_cache_free(ubi_wl_entry_slab, e2); + if (err) goto out_ro; - } } dbg_wl("done"); mutex_unlock(&ubi->move_mutex); + up_read(&ubi->fm_eba_sem); return 0; /* @@ -1250,41 +954,57 @@ out_not_moved: ubi->erroneous_peb_count += 1; } else if (scrubbing) wl_tree_add(e1, &ubi->scrub); - else + else if (keep) wl_tree_add(e1, &ubi->used); + if (dst_leb_clean) { + wl_tree_add(e2, &ubi->free); + ubi->free_count++; + } + ubi_assert(!ubi->move_to_put); ubi->move_from = ubi->move_to = NULL; ubi->wl_scheduled = 0; spin_unlock(&ubi->wl_lock); - ubi_free_vid_hdr(ubi, vid_hdr); - err = do_sync_erase(ubi, e2, vol_id, lnum, torture); - if (err) { - kmem_cache_free(ubi_wl_entry_slab, e2); - goto out_ro; + ubi_free_vid_buf(vidb); + if (dst_leb_clean) { + ensure_wear_leveling(ubi, 1); + } else { + err = do_sync_erase(ubi, e2, vol_id, lnum, torture); + if (err) + goto out_ro; + } + + if (erase) { + err = do_sync_erase(ubi, e1, vol_id, lnum, 1); + if (err) + goto out_ro; } + mutex_unlock(&ubi->move_mutex); + up_read(&ubi->fm_eba_sem); return 0; out_error: if (vol_id != -1) - ubi_err("error %d while moving PEB %d to PEB %d", + ubi_err(ubi, "error %d while moving PEB %d to PEB %d", err, e1->pnum, e2->pnum); else - ubi_err("error %d while moving PEB %d (LEB %d:%d) to PEB %d", + ubi_err(ubi, "error %d while moving PEB %d (LEB %d:%d) to PEB %d", err, e1->pnum, vol_id, lnum, e2->pnum); spin_lock(&ubi->wl_lock); ubi->move_from = ubi->move_to = NULL; ubi->move_to_put = ubi->wl_scheduled = 0; + wl_entry_destroy(ubi, e1); + wl_entry_destroy(ubi, e2); spin_unlock(&ubi->wl_lock); - ubi_free_vid_hdr(ubi, vid_hdr); - kmem_cache_free(ubi_wl_entry_slab, e1); - kmem_cache_free(ubi_wl_entry_slab, e2); + ubi_free_vid_buf(vidb); out_ro: ubi_ro_mode(ubi); mutex_unlock(&ubi->move_mutex); + up_read(&ubi->fm_eba_sem); ubi_assert(err != 0); return err < 0 ? err : -EIO; @@ -1292,7 +1012,8 @@ out_cancel: ubi->wl_scheduled = 0; spin_unlock(&ubi->wl_lock); mutex_unlock(&ubi->move_mutex); - ubi_free_vid_hdr(ubi, vid_hdr); + up_read(&ubi->fm_eba_sem); + ubi_free_vid_buf(vidb); return 0; } @@ -1308,8 +1029,6 @@ out_cancel: static int ensure_wear_leveling(struct ubi_device *ubi, int nested) { int err = 0; - struct ubi_wl_entry *e1; - struct ubi_wl_entry *e2; struct ubi_work *wrk; spin_lock(&ubi->wl_lock); @@ -1319,9 +1038,16 @@ static int ensure_wear_leveling(struct ubi_device *ubi, int nested) /* * If the ubi->scrub tree is not empty, scrubbing is needed, and the - * the WL worker has to be scheduled anyway. + * WL worker has to be scheduled anyway. */ if (!ubi->scrub.rb_node) { +#ifdef CONFIG_MTD_UBI_FASTMAP + if (!need_wear_leveling(ubi)) + goto out_unlock; +#else + struct ubi_wl_entry *e1; + struct ubi_wl_entry *e2; + if (!ubi->used.rb_node || !ubi->free.rb_node) /* No physical eraseblocks - no deal */ goto out_unlock; @@ -1333,10 +1059,11 @@ static int ensure_wear_leveling(struct ubi_device *ubi, int nested) * %UBI_WL_THRESHOLD. */ e1 = rb_entry(rb_first(&ubi->used), struct ubi_wl_entry, u.rb); - e2 = find_wl_entry(ubi, &ubi->free, WL_FREE_MAX_DIFF); + e2 = find_wl_entry(ubi, &ubi->free, WL_FREE_MAX_DIFF, 0); if (!(e2->ec - e1->ec >= UBI_WL_THRESHOLD)) goto out_unlock; +#endif dbg_wl("schedule wear-leveling"); } else dbg_wl("schedule scrubbing"); @@ -1350,7 +1077,6 @@ static int ensure_wear_leveling(struct ubi_device *ubi, int nested) goto out_cancel; } - wrk->anchor = 0; wrk->func = &wear_leveling_worker; if (nested) __schedule_ubi_work(ubi, wrk); @@ -1366,51 +1092,17 @@ out_unlock: return err; } -#ifdef CONFIG_MTD_UBI_FASTMAP /** - * ubi_ensure_anchor_pebs - schedule wear-leveling to produce an anchor PEB. - * @ubi: UBI device description object - */ -int ubi_ensure_anchor_pebs(struct ubi_device *ubi) -{ - struct ubi_work *wrk; - - spin_lock(&ubi->wl_lock); - if (ubi->wl_scheduled) { - spin_unlock(&ubi->wl_lock); - return 0; - } - ubi->wl_scheduled = 1; - spin_unlock(&ubi->wl_lock); - - wrk = kmalloc(sizeof(struct ubi_work), GFP_NOFS); - if (!wrk) { - spin_lock(&ubi->wl_lock); - ubi->wl_scheduled = 0; - spin_unlock(&ubi->wl_lock); - return -ENOMEM; - } - - wrk->anchor = 1; - wrk->func = &wear_leveling_worker; - schedule_ubi_work(ubi, wrk); - return 0; -} -#endif - -/** - * erase_worker - physical eraseblock erase worker function. + * __erase_worker - physical eraseblock erase worker function. * @ubi: UBI device description object * @wl_wrk: the work object - * @cancel: non-zero if the worker has to free memory and exit * * This function erases a physical eraseblock and perform torture testing if * needed. It also takes care about marking the physical eraseblock bad if * needed. Returns zero in case of success and a negative error code in case of * failure. */ -static int erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk, - int cancel) +static int __erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk) { struct ubi_wl_entry *e = wl_wrk->e; int pnum = e->pnum; @@ -1418,26 +1110,26 @@ static int erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk, int lnum = wl_wrk->lnum; int err, available_consumed = 0; - if (cancel) { - dbg_wl("cancel erasure of PEB %d EC %d", pnum, e->ec); - kfree(wl_wrk); - kmem_cache_free(ubi_wl_entry_slab, e); - return 0; - } - dbg_wl("erase PEB %d EC %d LEB %d:%d", pnum, e->ec, wl_wrk->vol_id, wl_wrk->lnum); - ubi_assert(!ubi_is_fm_block(ubi, e->pnum)); - - err = sync_erase(ubi, e, wl_wrk->torture); + err = ubi_sync_erase(ubi, e, wl_wrk->torture); if (!err) { - /* Fine, we've erased it successfully */ - kfree(wl_wrk); - spin_lock(&ubi->wl_lock); - wl_tree_add(e, &ubi->free); - ubi->free_count++; + + if (!ubi->fm_disabled && !ubi->fm_anchor && + e->pnum < UBI_FM_MAX_START) { + /* + * Abort anchor production, if needed it will be + * enabled again in the wear leveling started below. + */ + ubi->fm_anchor = e; + ubi->fm_do_produce_anchor = 0; + } else { + wl_tree_add(e, &ubi->free); + ubi->free_count++; + } + spin_unlock(&ubi->wl_lock); /* @@ -1451,23 +1143,27 @@ static int erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk, return err; } - ubi_err("failed to erase PEB %d, error %d", pnum, err); - kfree(wl_wrk); + ubi_err(ubi, "failed to erase PEB %d, error %d", pnum, err); if (err == -EINTR || err == -ENOMEM || err == -EAGAIN || err == -EBUSY) { int err1; /* Re-schedule the LEB for erasure */ - err1 = schedule_erase(ubi, e, vol_id, lnum, 0); + err1 = schedule_erase(ubi, e, vol_id, lnum, 0, true); if (err1) { + spin_lock(&ubi->wl_lock); + wl_entry_destroy(ubi, e); + spin_unlock(&ubi->wl_lock); err = err1; goto out_ro; } return err; } - kmem_cache_free(ubi_wl_entry_slab, e); + spin_lock(&ubi->wl_lock); + wl_entry_destroy(ubi, e); + spin_unlock(&ubi->wl_lock); if (err != -EIO) /* * If this is not %-EIO, we have no idea what to do. Scheduling @@ -1479,7 +1175,7 @@ static int erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk, /* It is %-EIO, the PEB went bad */ if (!ubi->bad_allowed) { - ubi_err("bad physical eraseblock %d detected", pnum); + ubi_err(ubi, "bad physical eraseblock %d detected", pnum); goto out_ro; } @@ -1487,7 +1183,7 @@ static int erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk, if (ubi->beb_rsvd_pebs == 0) { if (ubi->avail_pebs == 0) { spin_unlock(&ubi->volumes_lock); - ubi_err("no reserved/available physical eraseblocks"); + ubi_err(ubi, "no reserved/available physical eraseblocks"); goto out_ro; } ubi->avail_pebs -= 1; @@ -1495,7 +1191,7 @@ static int erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk, } spin_unlock(&ubi->volumes_lock); - ubi_msg("mark PEB %d as bad", pnum); + ubi_msg(ubi, "mark PEB %d as bad", pnum); err = ubi_io_mark_bad(ubi, pnum); if (err) goto out_ro; @@ -1516,11 +1212,12 @@ static int erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk, ubi->good_peb_count -= 1; ubi_calculate_reserved(ubi); if (available_consumed) - ubi_warn("no PEBs in the reserved pool, used an available PEB"); + ubi_warn(ubi, "no PEBs in the reserved pool, used an available PEB"); else if (ubi->beb_rsvd_pebs) - ubi_msg("%d PEBs left in the reserve", ubi->beb_rsvd_pebs); + ubi_msg(ubi, "%d PEBs left in the reserve", + ubi->beb_rsvd_pebs); else - ubi_warn("last PEB from the reserve was used"); + ubi_warn(ubi, "last PEB from the reserve was used"); spin_unlock(&ubi->volumes_lock); return err; @@ -1535,6 +1232,25 @@ out_ro: return err; } +static int erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk, + int shutdown) +{ + int ret; + + if (shutdown) { + struct ubi_wl_entry *e = wl_wrk->e; + + dbg_wl("cancel erasure of PEB %d EC %d", e->pnum, e->ec); + kfree(wl_wrk); + wl_entry_destroy(ubi, e); + return 0; + } + + ret = __erase_worker(ubi, wl_wrk); + kfree(wl_wrk); + return ret; +} + /** * ubi_wl_put_peb - return a PEB to the wear-leveling sub-system. * @ubi: UBI device description object @@ -1558,9 +1274,23 @@ int ubi_wl_put_peb(struct ubi_device *ubi, int vol_id, int lnum, ubi_assert(pnum >= 0); ubi_assert(pnum < ubi->peb_count); + down_read(&ubi->fm_protect); + retry: spin_lock(&ubi->wl_lock); e = ubi->lookuptbl[pnum]; + if (!e) { + /* + * This wl entry has been removed for some errors by other + * process (eg. wear leveling worker), corresponding process + * (except __erase_worker, which cannot concurrent with + * ubi_wl_put_peb) will set ubi ro_mode at the same time, + * just ignore this wl entry. + */ + spin_unlock(&ubi->wl_lock); + up_read(&ubi->fm_protect); + return 0; + } if (e == ubi->move_from) { /* * User is putting the physical eraseblock which was selected to @@ -1588,6 +1318,7 @@ retry: ubi_assert(!ubi->move_to_put); ubi->move_to_put = 1; spin_unlock(&ubi->wl_lock); + up_read(&ubi->fm_protect); return 0; } else { if (in_wl_tree(e, &ubi->used)) { @@ -1606,22 +1337,24 @@ retry: } else { err = prot_queue_del(ubi, e->pnum); if (err) { - ubi_err("PEB %d not found", pnum); + ubi_err(ubi, "PEB %d not found", pnum); ubi_ro_mode(ubi); spin_unlock(&ubi->wl_lock); + up_read(&ubi->fm_protect); return err; } } } spin_unlock(&ubi->wl_lock); - err = schedule_erase(ubi, e, vol_id, lnum, torture); + err = schedule_erase(ubi, e, vol_id, lnum, torture, false); if (err) { spin_lock(&ubi->wl_lock); wl_tree_add(e, &ubi->used); spin_unlock(&ubi->wl_lock); } + up_read(&ubi->fm_protect); return err; } @@ -1639,7 +1372,7 @@ int ubi_wl_scrub_peb(struct ubi_device *ubi, int pnum) { struct ubi_wl_entry *e; - ubi_msg("schedule PEB %d for scrubbing", pnum); + ubi_msg(ubi, "schedule PEB %d for scrubbing", pnum); retry: spin_lock(&ubi->wl_lock); @@ -1671,7 +1404,7 @@ retry: err = prot_queue_del(ubi, e->pnum); if (err) { - ubi_err("PEB %d not found", pnum); + ubi_err(ubi, "PEB %d not found", pnum); ubi_ro_mode(ubi); spin_unlock(&ubi->wl_lock); return err; @@ -1713,12 +1446,12 @@ int ubi_wl_flush(struct ubi_device *ubi, int vol_id, int lnum) vol_id, lnum, ubi->works_count); while (found) { - struct ubi_work *wrk; + struct ubi_work *wrk, *tmp; found = 0; down_read(&ubi->work_sem); spin_lock(&ubi->wl_lock); - list_for_each_entry(wrk, &ubi->works, list) { + list_for_each_entry_safe(wrk, tmp, &ubi->works, list) { if ((vol_id == UBI_ALL || wrk->vol_id == vol_id) && (lnum == UBI_ALL || wrk->lnum == lnum)) { list_del(&wrk->list); @@ -1751,11 +1484,156 @@ int ubi_wl_flush(struct ubi_device *ubi, int vol_id, int lnum) return err; } +static bool scrub_possible(struct ubi_device *ubi, struct ubi_wl_entry *e) +{ + if (in_wl_tree(e, &ubi->scrub)) + return false; + else if (in_wl_tree(e, &ubi->erroneous)) + return false; + else if (ubi->move_from == e) + return false; + else if (ubi->move_to == e) + return false; + + return true; +} + +/** + * ubi_bitflip_check - Check an eraseblock for bitflips and scrub it if needed. + * @ubi: UBI device description object + * @pnum: the physical eraseblock to schedule + * @force: don't read the block, assume bitflips happened and take action. + * + * This function reads the given eraseblock and checks if bitflips occured. + * In case of bitflips, the eraseblock is scheduled for scrubbing. + * If scrubbing is forced with @force, the eraseblock is not read, + * but scheduled for scrubbing right away. + * + * Returns: + * %EINVAL, PEB is out of range + * %ENOENT, PEB is no longer used by UBI + * %EBUSY, PEB cannot be checked now or a check is currently running on it + * %EAGAIN, bit flips happened but scrubbing is currently not possible + * %EUCLEAN, bit flips happened and PEB is scheduled for scrubbing + * %0, no bit flips detected + */ +int ubi_bitflip_check(struct ubi_device *ubi, int pnum, int force) +{ + int err = 0; + struct ubi_wl_entry *e; + + if (pnum < 0 || pnum >= ubi->peb_count) { + err = -EINVAL; + goto out; + } + + /* + * Pause all parallel work, otherwise it can happen that the + * erase worker frees a wl entry under us. + */ + down_write(&ubi->work_sem); + + /* + * Make sure that the wl entry does not change state while + * inspecting it. + */ + spin_lock(&ubi->wl_lock); + e = ubi->lookuptbl[pnum]; + if (!e) { + spin_unlock(&ubi->wl_lock); + err = -ENOENT; + goto out_resume; + } + + /* + * Does it make sense to check this PEB? + */ + if (!scrub_possible(ubi, e)) { + spin_unlock(&ubi->wl_lock); + err = -EBUSY; + goto out_resume; + } + spin_unlock(&ubi->wl_lock); + + if (!force) { + mutex_lock(&ubi->buf_mutex); + err = ubi_io_read(ubi, ubi->peb_buf, pnum, 0, ubi->peb_size); + mutex_unlock(&ubi->buf_mutex); + } + + if (force || err == UBI_IO_BITFLIPS) { + /* + * Okay, bit flip happened, let's figure out what we can do. + */ + spin_lock(&ubi->wl_lock); + + /* + * Recheck. We released wl_lock, UBI might have killed the + * wl entry under us. + */ + e = ubi->lookuptbl[pnum]; + if (!e) { + spin_unlock(&ubi->wl_lock); + err = -ENOENT; + goto out_resume; + } + + /* + * Need to re-check state + */ + if (!scrub_possible(ubi, e)) { + spin_unlock(&ubi->wl_lock); + err = -EBUSY; + goto out_resume; + } + + if (in_pq(ubi, e)) { + prot_queue_del(ubi, e->pnum); + wl_tree_add(e, &ubi->scrub); + spin_unlock(&ubi->wl_lock); + + err = ensure_wear_leveling(ubi, 1); + } else if (in_wl_tree(e, &ubi->used)) { + rb_erase(&e->u.rb, &ubi->used); + wl_tree_add(e, &ubi->scrub); + spin_unlock(&ubi->wl_lock); + + err = ensure_wear_leveling(ubi, 1); + } else if (in_wl_tree(e, &ubi->free)) { + rb_erase(&e->u.rb, &ubi->free); + ubi->free_count--; + spin_unlock(&ubi->wl_lock); + + /* + * This PEB is empty we can schedule it for + * erasure right away. No wear leveling needed. + */ + err = schedule_erase(ubi, e, UBI_UNKNOWN, UBI_UNKNOWN, + force ? 0 : 1, true); + } else { + spin_unlock(&ubi->wl_lock); + err = -EAGAIN; + } + + if (!err && !force) + err = -EUCLEAN; + } else { + err = 0; + } + +out_resume: + up_write(&ubi->work_sem); +out: + + return err; +} + /** * tree_destroy - destroy an RB-tree. + * @ubi: UBI device description object * @root: the root of the tree to destroy */ -static void tree_destroy(struct rb_root *root) +static void tree_destroy(struct ubi_device *ubi, struct rb_root *root) { struct rb_node *rb; struct ubi_wl_entry *e; @@ -1777,7 +1655,7 @@ static void tree_destroy(struct rb_root *root) rb->rb_right = NULL; } - kmem_cache_free(ubi_wl_entry_slab, e); + wl_entry_destroy(ubi, e); } } } @@ -1791,7 +1669,7 @@ int ubi_thread(void *u) int failures = 0; struct ubi_device *ubi = u; - ubi_msg("background thread \"%s\" started, PID %d", + ubi_msg(ubi, "background thread \"%s\" started, PID %d", ubi->bgt_name, task_pid_nr(current)); set_freezable(); @@ -1809,21 +1687,34 @@ int ubi_thread(void *u) !ubi->thread_enabled || ubi_dbg_is_bgt_disabled(ubi)) { set_current_state(TASK_INTERRUPTIBLE); spin_unlock(&ubi->wl_lock); + + /* + * Check kthread_should_stop() after we set the task + * state to guarantee that we either see the stop bit + * and exit or the task state is reset to runnable such + * that it's not scheduled out indefinitely and detects + * the stop bit at kthread_should_stop(). + */ + if (kthread_should_stop()) { + set_current_state(TASK_RUNNING); + break; + } + schedule(); continue; } spin_unlock(&ubi->wl_lock); - err = do_work(ubi); + err = do_work(ubi, NULL); if (err) { - ubi_err("%s: work failed with error code %d", + ubi_err(ubi, "%s: work failed with error code %d", ubi->bgt_name, err); if (failures++ > WL_MAX_FAILURES) { /* * Too many failures, disable the thread and * switch to read-only mode. */ - ubi_msg("%s: %d consecutive failures", + ubi_msg(ubi, "%s: %d consecutive failures", ubi->bgt_name, WL_MAX_FAILURES); ubi_ro_mode(ubi); ubi->thread_enabled = 0; @@ -1836,14 +1727,15 @@ int ubi_thread(void *u) } dbg_wl("background thread \"%s\" is killed", ubi->bgt_name); + ubi->thread_enabled = 0; return 0; } /** - * cancel_pending - cancel all pending works. + * shutdown_work - shutdown all pending works. * @ubi: UBI device description object */ -static void cancel_pending(struct ubi_device *ubi) +static void shutdown_work(struct ubi_device *ubi) { while (!list_empty(&ubi->works)) { struct ubi_work *wrk; @@ -1857,6 +1749,46 @@ static void cancel_pending(struct ubi_device *ubi) } /** + * erase_aeb - erase a PEB given in UBI attach info PEB + * @ubi: UBI device description object + * @aeb: UBI attach info PEB + * @sync: If true, erase synchronously. Otherwise schedule for erasure + */ +static int erase_aeb(struct ubi_device *ubi, struct ubi_ainf_peb *aeb, bool sync) +{ + struct ubi_wl_entry *e; + int err; + + e = kmem_cache_alloc(ubi_wl_entry_slab, GFP_KERNEL); + if (!e) + return -ENOMEM; + + e->pnum = aeb->pnum; + e->ec = aeb->ec; + ubi->lookuptbl[e->pnum] = e; + + if (sync) { + err = ubi_sync_erase(ubi, e, false); + if (err) + goto out_free; + + wl_tree_add(e, &ubi->free); + ubi->free_count++; + } else { + err = schedule_erase(ubi, e, aeb->vol_id, aeb->lnum, 0, false); + if (err) + goto out_free; + } + + return 0; + +out_free: + wl_entry_destroy(ubi, e); + + return err; +} + +/** * ubi_wl_init - initialize the WL sub-system using attaching information. * @ubi: UBI device description object * @ai: attaching information @@ -1878,14 +1810,11 @@ int ubi_wl_init(struct ubi_device *ubi, struct ubi_attach_info *ai) init_rwsem(&ubi->work_sem); ubi->max_ec = ai->max_ec; INIT_LIST_HEAD(&ubi->works); -#ifdef CONFIG_MTD_UBI_FASTMAP - INIT_WORK(&ubi->fm_work, update_fastmap_work_fn); -#endif sprintf(ubi->bgt_name, UBI_BGT_NAME_PATTERN, ubi->ubi_num); err = -ENOMEM; - ubi->lookuptbl = kzalloc(ubi->peb_count * sizeof(void *), GFP_KERNEL); + ubi->lookuptbl = kcalloc(ubi->peb_count, sizeof(void *), GFP_KERNEL); if (!ubi->lookuptbl) return err; @@ -1893,37 +1822,29 @@ int ubi_wl_init(struct ubi_device *ubi, struct ubi_attach_info *ai) INIT_LIST_HEAD(&ubi->pq[i]); ubi->pq_head = 0; + ubi->free_count = 0; list_for_each_entry_safe(aeb, tmp, &ai->erase, u.list) { cond_resched(); - e = kmem_cache_alloc(ubi_wl_entry_slab, GFP_KERNEL); - if (!e) - goto out_free; - - e->pnum = aeb->pnum; - e->ec = aeb->ec; - ubi_assert(!ubi_is_fm_block(ubi, e->pnum)); - ubi->lookuptbl[e->pnum] = e; - if (schedule_erase(ubi, e, aeb->vol_id, aeb->lnum, 0)) { - kmem_cache_free(ubi_wl_entry_slab, e); + err = erase_aeb(ubi, aeb, false); + if (err) goto out_free; - } found_pebs++; } - ubi->free_count = 0; list_for_each_entry(aeb, &ai->free, u.list) { cond_resched(); e = kmem_cache_alloc(ubi_wl_entry_slab, GFP_KERNEL); - if (!e) + if (!e) { + err = -ENOMEM; goto out_free; + } e->pnum = aeb->pnum; e->ec = aeb->ec; ubi_assert(e->ec >= 0); - ubi_assert(!ubi_is_fm_block(ubi, e->pnum)); wl_tree_add(e, &ubi->free); ubi->free_count++; @@ -1938,8 +1859,10 @@ int ubi_wl_init(struct ubi_device *ubi, struct ubi_attach_info *ai) cond_resched(); e = kmem_cache_alloc(ubi_wl_entry_slab, GFP_KERNEL); - if (!e) + if (!e) { + err = -ENOMEM; goto out_free; + } e->pnum = aeb->pnum; e->ec = aeb->ec; @@ -1959,26 +1882,60 @@ int ubi_wl_init(struct ubi_device *ubi, struct ubi_attach_info *ai) } } + list_for_each_entry(aeb, &ai->fastmap, u.list) { + cond_resched(); + + e = ubi_find_fm_block(ubi, aeb->pnum); + + if (e) { + ubi_assert(!ubi->lookuptbl[e->pnum]); + ubi->lookuptbl[e->pnum] = e; + } else { + bool sync = false; + + /* + * Usually old Fastmap PEBs are scheduled for erasure + * and we don't have to care about them but if we face + * an power cut before scheduling them we need to + * take care of them here. + */ + if (ubi->lookuptbl[aeb->pnum]) + continue; + + /* + * The fastmap update code might not find a free PEB for + * writing the fastmap anchor to and then reuses the + * current fastmap anchor PEB. When this PEB gets erased + * and a power cut happens before it is written again we + * must make sure that the fastmap attach code doesn't + * find any outdated fastmap anchors, hence we erase the + * outdated fastmap anchor PEBs synchronously here. + */ + if (aeb->vol_id == UBI_FM_SB_VOLUME_ID) + sync = true; + + err = erase_aeb(ubi, aeb, sync); + if (err) + goto out_free; + } + + found_pebs++; + } + dbg_wl("found %i PEBs", found_pebs); - if (ubi->fm) - ubi_assert(ubi->good_peb_count == \ - found_pebs + ubi->fm->used_blocks); - else - ubi_assert(ubi->good_peb_count == found_pebs); + ubi_assert(ubi->good_peb_count == found_pebs); reserved_pebs = WL_RESERVED_PEBS; -#ifdef CONFIG_MTD_UBI_FASTMAP - /* Reserve enough LEBs to store two fastmaps. */ - reserved_pebs += (ubi->fm_size / ubi->leb_size) * 2; -#endif + ubi_fastmap_init(ubi, &reserved_pebs); if (ubi->avail_pebs < reserved_pebs) { - ubi_err("no enough physical eraseblocks (%d, need %d)", + ubi_err(ubi, "no enough physical eraseblocks (%d, need %d)", ubi->avail_pebs, reserved_pebs); if (ubi->corr_peb_count) - ubi_err("%d PEBs are corrupted and not used", + ubi_err(ubi, "%d PEBs are corrupted and not used", ubi->corr_peb_count); + err = -ENOSPC; goto out_free; } ubi->avail_pebs -= reserved_pebs; @@ -1989,13 +1946,17 @@ int ubi_wl_init(struct ubi_device *ubi, struct ubi_attach_info *ai) if (err) goto out_free; +#ifdef CONFIG_MTD_UBI_FASTMAP + if (!ubi->ro_mode && !ubi->fm_disabled) + ubi_ensure_anchor_pebs(ubi); +#endif return 0; out_free: - cancel_pending(ubi); - tree_destroy(&ubi->used); - tree_destroy(&ubi->free); - tree_destroy(&ubi->scrub); + shutdown_work(ubi); + tree_destroy(ubi, &ubi->used); + tree_destroy(ubi, &ubi->free); + tree_destroy(ubi, &ubi->scrub); kfree(ubi->lookuptbl); return err; } @@ -2012,7 +1973,7 @@ static void protection_queue_destroy(struct ubi_device *ubi) for (i = 0; i < UBI_PROT_QUEUE_LEN; ++i) { list_for_each_entry_safe(e, tmp, &ubi->pq[i], u.list) { list_del(&e->u.list); - kmem_cache_free(ubi_wl_entry_slab, e); + wl_entry_destroy(ubi, e); } } } @@ -2024,12 +1985,13 @@ static void protection_queue_destroy(struct ubi_device *ubi) void ubi_wl_close(struct ubi_device *ubi) { dbg_wl("close the WL sub-system"); - cancel_pending(ubi); + ubi_fastmap_close(ubi); + shutdown_work(ubi); protection_queue_destroy(ubi); - tree_destroy(&ubi->used); - tree_destroy(&ubi->erroneous); - tree_destroy(&ubi->free); - tree_destroy(&ubi->scrub); + tree_destroy(ubi, &ubi->used); + tree_destroy(ubi, &ubi->erroneous); + tree_destroy(ubi, &ubi->free); + tree_destroy(ubi, &ubi->scrub); kfree(ubi->lookuptbl); } @@ -2065,8 +2027,8 @@ static int self_check_ec(struct ubi_device *ubi, int pnum, int ec) read_ec = be64_to_cpu(ec_hdr->ec); if (ec != read_ec && read_ec - ec > 1) { - ubi_err("self-check failed for PEB %d", pnum); - ubi_err("read EC is %lld, should be %d", read_ec, ec); + ubi_err(ubi, "self-check failed for PEB %d", pnum); + ubi_err(ubi, "read EC is %lld, should be %d", read_ec, ec); dump_stack(); err = 1; } else @@ -2095,7 +2057,7 @@ static int self_check_in_wl_tree(const struct ubi_device *ubi, if (in_wl_tree(e, root)) return 0; - ubi_err("self-check failed for PEB %d, EC %d, RB-tree %p ", + ubi_err(ubi, "self-check failed for PEB %d, EC %d, RB-tree %p ", e->pnum, e->ec, root); dump_stack(); return -EINVAL; @@ -2112,19 +2074,105 @@ static int self_check_in_wl_tree(const struct ubi_device *ubi, static int self_check_in_pq(const struct ubi_device *ubi, struct ubi_wl_entry *e) { - struct ubi_wl_entry *p; - int i; - if (!ubi_dbg_chk_gen(ubi)) return 0; - for (i = 0; i < UBI_PROT_QUEUE_LEN; ++i) - list_for_each_entry(p, &ubi->pq[i], u.list) - if (p == e) - return 0; + if (in_pq(ubi, e)) + return 0; - ubi_err("self-check failed for PEB %d, EC %d, Protect queue", + ubi_err(ubi, "self-check failed for PEB %d, EC %d, Protect queue", e->pnum, e->ec); dump_stack(); return -EINVAL; } +#ifndef CONFIG_MTD_UBI_FASTMAP +static struct ubi_wl_entry *get_peb_for_wl(struct ubi_device *ubi) +{ + struct ubi_wl_entry *e; + + e = find_wl_entry(ubi, &ubi->free, WL_FREE_MAX_DIFF, 0); + self_check_in_wl_tree(ubi, e, &ubi->free); + ubi->free_count--; + ubi_assert(ubi->free_count >= 0); + rb_erase(&e->u.rb, &ubi->free); + + return e; +} + +/** + * produce_free_peb - produce a free physical eraseblock. + * @ubi: UBI device description object + * + * This function tries to make a free PEB by means of synchronous execution of + * pending works. This may be needed if, for example the background thread is + * disabled. Returns zero in case of success and a negative error code in case + * of failure. + */ +static int produce_free_peb(struct ubi_device *ubi) +{ + int err; + + while (!ubi->free.rb_node && ubi->works_count) { + spin_unlock(&ubi->wl_lock); + + dbg_wl("do one work synchronously"); + err = do_work(ubi, NULL); + + spin_lock(&ubi->wl_lock); + if (err) + return err; + } + + return 0; +} + +/** + * ubi_wl_get_peb - get a physical eraseblock. + * @ubi: UBI device description object + * + * This function returns a physical eraseblock in case of success and a + * negative error code in case of failure. + * Returns with ubi->fm_eba_sem held in read mode! + */ +int ubi_wl_get_peb(struct ubi_device *ubi) +{ + int err; + struct ubi_wl_entry *e; + +retry: + down_read(&ubi->fm_eba_sem); + spin_lock(&ubi->wl_lock); + if (!ubi->free.rb_node) { + if (ubi->works_count == 0) { + ubi_err(ubi, "no free eraseblocks"); + ubi_assert(list_empty(&ubi->works)); + spin_unlock(&ubi->wl_lock); + return -ENOSPC; + } + + err = produce_free_peb(ubi); + if (err < 0) { + spin_unlock(&ubi->wl_lock); + return err; + } + spin_unlock(&ubi->wl_lock); + up_read(&ubi->fm_eba_sem); + goto retry; + + } + e = wl_get_wle(ubi); + prot_queue_add(ubi, e); + spin_unlock(&ubi->wl_lock); + + err = ubi_self_check_all_ff(ubi, e->pnum, ubi->vid_hdr_aloffset, + ubi->peb_size - ubi->vid_hdr_aloffset); + if (err) { + ubi_err(ubi, "new PEB %d does not contain all 0xFF bytes", e->pnum); + return err; + } + + return e->pnum; +} +#else +#include "fastmap-wl.c" +#endif |