diff options
Diffstat (limited to 'drivers/mtd/nand/raw/nand_base.c')
| -rw-r--r-- | drivers/mtd/nand/raw/nand_base.c | 2861 |
1 files changed, 1777 insertions, 1084 deletions
diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c index cca4b24d2ffa..ad6d66309597 100644 --- a/drivers/mtd/nand/raw/nand_base.c +++ b/drivers/mtd/nand/raw/nand_base.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Overview: * This is the generic MTD driver for NAND flash devices. It should be @@ -20,11 +21,6 @@ * Check, if mtd->ecctype should be set to MTD_ECC_HW * if we have HW ECC support. * BBT table is not serialized, has to be fixed - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -38,8 +34,9 @@ #include <linux/mm.h> #include <linux/types.h> #include <linux/mtd/mtd.h> -#include <linux/mtd/nand_ecc.h> -#include <linux/mtd/nand_bch.h> +#include <linux/mtd/nand.h> +#include <linux/mtd/nand-ecc-sw-hamming.h> +#include <linux/mtd/nand-ecc-sw-bch.h> #include <linux/interrupt.h> #include <linux/bitops.h> #include <linux/io.h> @@ -49,164 +46,54 @@ #include "internals.h" -/* Define default oob placement schemes for large and small page devices */ -static int nand_ooblayout_ecc_sp(struct mtd_info *mtd, int section, - struct mtd_oob_region *oobregion) +static int nand_pairing_dist3_get_info(struct mtd_info *mtd, int page, + struct mtd_pairing_info *info) { - struct nand_chip *chip = mtd_to_nand(mtd); - struct nand_ecc_ctrl *ecc = &chip->ecc; + int lastpage = (mtd->erasesize / mtd->writesize) - 1; + int dist = 3; - if (section > 1) - return -ERANGE; + if (page == lastpage) + dist = 2; - if (!section) { - oobregion->offset = 0; - if (mtd->oobsize == 16) - oobregion->length = 4; - else - oobregion->length = 3; + if (!page || (page & 1)) { + info->group = 0; + info->pair = (page + 1) / 2; } else { - if (mtd->oobsize == 8) - return -ERANGE; - - oobregion->offset = 6; - oobregion->length = ecc->total - 4; + info->group = 1; + info->pair = (page + 1 - dist) / 2; } return 0; } -static int nand_ooblayout_free_sp(struct mtd_info *mtd, int section, - struct mtd_oob_region *oobregion) +static int nand_pairing_dist3_get_wunit(struct mtd_info *mtd, + const struct mtd_pairing_info *info) { - if (section > 1) - return -ERANGE; - - if (mtd->oobsize == 16) { - if (section) - return -ERANGE; - - oobregion->length = 8; - oobregion->offset = 8; - } else { - oobregion->length = 2; - if (!section) - oobregion->offset = 3; - else - oobregion->offset = 6; - } - - return 0; -} - -const struct mtd_ooblayout_ops nand_ooblayout_sp_ops = { - .ecc = nand_ooblayout_ecc_sp, - .free = nand_ooblayout_free_sp, -}; -EXPORT_SYMBOL_GPL(nand_ooblayout_sp_ops); - -static int nand_ooblayout_ecc_lp(struct mtd_info *mtd, int section, - struct mtd_oob_region *oobregion) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - struct nand_ecc_ctrl *ecc = &chip->ecc; - - if (section || !ecc->total) - return -ERANGE; - - oobregion->length = ecc->total; - oobregion->offset = mtd->oobsize - oobregion->length; - - return 0; -} - -static int nand_ooblayout_free_lp(struct mtd_info *mtd, int section, - struct mtd_oob_region *oobregion) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - struct nand_ecc_ctrl *ecc = &chip->ecc; - - if (section) - return -ERANGE; - - oobregion->length = mtd->oobsize - ecc->total - 2; - oobregion->offset = 2; + int lastpair = ((mtd->erasesize / mtd->writesize) - 1) / 2; + int page = info->pair * 2; + int dist = 3; - return 0; -} + if (!info->group && !info->pair) + return 0; -const struct mtd_ooblayout_ops nand_ooblayout_lp_ops = { - .ecc = nand_ooblayout_ecc_lp, - .free = nand_ooblayout_free_lp, -}; -EXPORT_SYMBOL_GPL(nand_ooblayout_lp_ops); + if (info->pair == lastpair && info->group) + dist = 2; -/* - * Support the old "large page" layout used for 1-bit Hamming ECC where ECC - * are placed at a fixed offset. - */ -static int nand_ooblayout_ecc_lp_hamming(struct mtd_info *mtd, int section, - struct mtd_oob_region *oobregion) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - struct nand_ecc_ctrl *ecc = &chip->ecc; - - if (section) - return -ERANGE; + if (!info->group) + page--; + else if (info->pair) + page += dist - 1; - switch (mtd->oobsize) { - case 64: - oobregion->offset = 40; - break; - case 128: - oobregion->offset = 80; - break; - default: + if (page >= mtd->erasesize / mtd->writesize) return -EINVAL; - } - - oobregion->length = ecc->total; - if (oobregion->offset + oobregion->length > mtd->oobsize) - return -ERANGE; - return 0; + return page; } -static int nand_ooblayout_free_lp_hamming(struct mtd_info *mtd, int section, - struct mtd_oob_region *oobregion) -{ - struct nand_chip *chip = mtd_to_nand(mtd); - struct nand_ecc_ctrl *ecc = &chip->ecc; - int ecc_offset = 0; - - if (section < 0 || section > 1) - return -ERANGE; - - switch (mtd->oobsize) { - case 64: - ecc_offset = 40; - break; - case 128: - ecc_offset = 80; - break; - default: - return -EINVAL; - } - - if (section == 0) { - oobregion->offset = 2; - oobregion->length = ecc_offset - 2; - } else { - oobregion->offset = ecc_offset + ecc->total; - oobregion->length = mtd->oobsize - oobregion->offset; - } - - return 0; -} - -static const struct mtd_ooblayout_ops nand_ooblayout_lp_hamming_ops = { - .ecc = nand_ooblayout_ecc_lp_hamming, - .free = nand_ooblayout_free_lp_hamming, +const struct mtd_pairing_scheme dist3_pairing_scheme = { + .ngroups = 2, + .get_info = nand_pairing_dist3_get_info, + .get_wunit = nand_pairing_dist3_get_wunit, }; static int check_offs_len(struct nand_chip *chip, loff_t ofs, uint64_t len) @@ -229,6 +116,50 @@ static int check_offs_len(struct nand_chip *chip, loff_t ofs, uint64_t len) } /** + * nand_extract_bits - Copy unaligned bits from one buffer to another one + * @dst: destination buffer + * @dst_off: bit offset at which the writing starts + * @src: source buffer + * @src_off: bit offset at which the reading starts + * @nbits: number of bits to copy from @src to @dst + * + * Copy bits from one memory region to another (overlap authorized). + */ +void nand_extract_bits(u8 *dst, unsigned int dst_off, const u8 *src, + unsigned int src_off, unsigned int nbits) +{ + unsigned int tmp, n; + + dst += dst_off / 8; + dst_off %= 8; + src += src_off / 8; + src_off %= 8; + + while (nbits) { + n = min3(8 - dst_off, 8 - src_off, nbits); + + tmp = (*src >> src_off) & GENMASK(n - 1, 0); + *dst &= ~GENMASK(n - 1 + dst_off, dst_off); + *dst |= tmp << dst_off; + + dst_off += n; + if (dst_off >= 8) { + dst++; + dst_off -= 8; + } + + src_off += n; + if (src_off >= 8) { + src++; + src_off -= 8; + } + + nbits -= n; + } +} +EXPORT_SYMBOL_GPL(nand_extract_bits); + +/** * nand_select_target() - Select a NAND target (A.K.A. die) * @chip: NAND chip object * @cs: the CS line to select. Note that this CS id is always from the chip @@ -240,10 +171,10 @@ static int check_offs_len(struct nand_chip *chip, loff_t ofs, uint64_t len) void nand_select_target(struct nand_chip *chip, unsigned int cs) { /* - * cs should always lie between 0 and chip->numchips, when that's not - * the case it's a bug and the caller should be fixed. + * cs should always lie between 0 and nanddev_ntargets(), when that's + * not the case it's a bug and the caller should be fixed. */ - if (WARN_ON(cs > chip->numchips)) + if (WARN_ON(cs > nanddev_ntargets(&chip->base))) return; chip->cur_cs = cs; @@ -278,11 +209,37 @@ EXPORT_SYMBOL_GPL(nand_deselect_target); static void nand_release_device(struct nand_chip *chip) { /* Release the controller and the chip */ - spin_lock(&chip->controller->lock); - chip->controller->active = NULL; - chip->state = FL_READY; - wake_up(&chip->controller->wq); - spin_unlock(&chip->controller->lock); + mutex_unlock(&chip->controller->lock); + mutex_unlock(&chip->lock); +} + +/** + * nand_bbm_get_next_page - Get the next page for bad block markers + * @chip: NAND chip object + * @page: First page to start checking for bad block marker usage + * + * Returns an integer that corresponds to the page offset within a block, for + * a page that is used to store bad block markers. If no more pages are + * available, -EINVAL is returned. + */ +int nand_bbm_get_next_page(struct nand_chip *chip, int page) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + int last_page = ((mtd->erasesize - mtd->writesize) >> + chip->page_shift) & chip->pagemask; + unsigned int bbm_flags = NAND_BBM_FIRSTPAGE | NAND_BBM_SECONDPAGE + | NAND_BBM_LASTPAGE; + + if (page == 0 && !(chip->options & bbm_flags)) + return 0; + if (page == 0 && chip->options & NAND_BBM_FIRSTPAGE) + return 0; + if (page <= 1 && chip->options & NAND_BBM_SECONDPAGE) + return 1; + if (page <= last_page && chip->options & NAND_BBM_LASTPAGE) + return last_page; + + return -EINVAL; } /** @@ -294,18 +251,15 @@ static void nand_release_device(struct nand_chip *chip) */ static int nand_block_bad(struct nand_chip *chip, loff_t ofs) { - struct mtd_info *mtd = nand_to_mtd(chip); - int page, page_end, res; + int first_page, page_offset; + int res; u8 bad; - if (chip->bbt_options & NAND_BBT_SCANLASTPAGE) - ofs += mtd->erasesize - mtd->writesize; - - page = (int)(ofs >> chip->page_shift) & chip->pagemask; - page_end = page + (chip->bbt_options & NAND_BBT_SCAN2NDPAGE ? 2 : 1); + first_page = (int)(ofs >> chip->page_shift) & chip->pagemask; + page_offset = nand_bbm_get_next_page(chip, 0); - for (; page < page_end; page++) { - res = chip->ecc.read_oob(chip, page); + while (page_offset >= 0) { + res = chip->ecc.read_oob(chip, first_page + page_offset); if (res < 0) return res; @@ -317,13 +271,58 @@ static int nand_block_bad(struct nand_chip *chip, loff_t ofs) res = hweight8(bad) < chip->badblockbits; if (res) return res; + + page_offset = nand_bbm_get_next_page(chip, page_offset + 1); } return 0; } +/** + * nand_region_is_secured() - Check if the region is secured + * @chip: NAND chip object + * @offset: Offset of the region to check + * @size: Size of the region to check + * + * Checks if the region is secured by comparing the offset and size with the + * list of secure regions obtained from DT. Returns true if the region is + * secured else false. + */ +static bool nand_region_is_secured(struct nand_chip *chip, loff_t offset, u64 size) +{ + int i; + + /* Skip touching the secure regions if present */ + for (i = 0; i < chip->nr_secure_regions; i++) { + const struct nand_secure_region *region = &chip->secure_regions[i]; + + if (offset + size <= region->offset || + offset >= region->offset + region->size) + continue; + + pr_debug("%s: Region 0x%llx - 0x%llx is secured!", + __func__, offset, offset + size); + + return true; + } + + return false; +} + static int nand_isbad_bbm(struct nand_chip *chip, loff_t ofs) { + struct mtd_info *mtd = nand_to_mtd(chip); + + if (chip->options & NAND_NO_BBM_QUIRK) + return 0; + + /* Check if the region is secured */ + if (nand_region_is_secured(chip, ofs, mtd->erasesize)) + return -EIO; + + if (mtd_check_expert_analysis_mode()) + return 0; + if (chip->legacy.block_bad) return chip->legacy.block_bad(chip, ofs); @@ -331,57 +330,24 @@ static int nand_isbad_bbm(struct nand_chip *chip, loff_t ofs) } /** - * panic_nand_get_device - [GENERIC] Get chip for selected access - * @chip: the nand chip descriptor - * @new_state: the state which is requested - * - * Used when in panic, no locks are taken. - */ -static void panic_nand_get_device(struct nand_chip *chip, int new_state) -{ - /* Hardware controller shared among independent devices */ - chip->controller->active = chip; - chip->state = new_state; -} - -/** * nand_get_device - [GENERIC] Get chip for selected access * @chip: NAND chip structure - * @new_state: the state which is requested * - * Get the device and lock it for exclusive access + * Lock the device and its controller for exclusive access */ -static int -nand_get_device(struct nand_chip *chip, int new_state) +static void nand_get_device(struct nand_chip *chip) { - spinlock_t *lock = &chip->controller->lock; - wait_queue_head_t *wq = &chip->controller->wq; - DECLARE_WAITQUEUE(wait, current); -retry: - spin_lock(lock); - - /* Hardware controller shared among independent devices */ - if (!chip->controller->active) - chip->controller->active = chip; - - if (chip->controller->active == chip && chip->state == FL_READY) { - chip->state = new_state; - spin_unlock(lock); - return 0; - } - if (new_state == FL_PM_SUSPENDED) { - if (chip->controller->active->state == FL_PM_SUSPENDED) { - chip->state = FL_PM_SUSPENDED; - spin_unlock(lock); - return 0; + /* Wait until the device is resumed. */ + while (1) { + mutex_lock(&chip->lock); + if (!chip->suspended) { + mutex_lock(&chip->controller->lock); + return; } + mutex_unlock(&chip->lock); + + wait_event(chip->resume_wq, !chip->suspended); } - set_current_state(TASK_UNINTERRUPTIBLE); - add_wait_queue(wq, &wait); - spin_unlock(lock); - schedule(); - remove_wait_queue(wq, &wait); - goto retry; } /** @@ -400,6 +366,10 @@ static int nand_check_wp(struct nand_chip *chip) if (chip->options & NAND_BROKEN_XD) return 0; + /* controller responsible for NAND write protect */ + if (chip->controller->controller_wp) + return 0; + /* Check the WP bit */ ret = nand_status_op(chip, &status); if (ret) @@ -410,6 +380,7 @@ static int nand_check_wp(struct nand_chip *chip) /** * nand_fill_oob - [INTERN] Transfer client buffer to oob + * @chip: NAND chip object * @oob: oob data buffer * @len: oob data write length * @ops: oob ops structure @@ -457,7 +428,7 @@ static int nand_do_write_oob(struct nand_chip *chip, loff_t to, struct mtd_oob_ops *ops) { struct mtd_info *mtd = nand_to_mtd(chip); - int chipnr, page, status, len; + int chipnr, page, status, len, ret; pr_debug("%s: to = 0x%08x, len = %i\n", __func__, (unsigned int)to, (int)ops->ooblen); @@ -471,6 +442,10 @@ static int nand_do_write_oob(struct nand_chip *chip, loff_t to, return -EINVAL; } + /* Check if the region is secured */ + if (nand_region_is_secured(chip, to, ops->ooblen)) + return -EIO; + chipnr = (int)(to >> chip->chip_shift); /* @@ -479,7 +454,9 @@ static int nand_do_write_oob(struct nand_chip *chip, loff_t to, * if we don't do this. I have no clue why, but I seem to have 'fixed' * it in the doc2000 driver in August 1999. dwmw2. */ - nand_reset(chip, chipnr); + ret = nand_reset(chip, chipnr); + if (ret) + return ret; nand_select_target(chip, chipnr); @@ -493,8 +470,8 @@ static int nand_do_write_oob(struct nand_chip *chip, loff_t to, } /* Invalidate the page cache, if we write to the cached page */ - if (page == chip->pagebuf) - chip->pagebuf = -1; + if (page == chip->pagecache.page) + chip->pagecache.page = -1; nand_fill_oob(chip, ops->oobbuf, ops->ooblen, ops); @@ -527,7 +504,7 @@ static int nand_default_block_markbad(struct nand_chip *chip, loff_t ofs) struct mtd_info *mtd = nand_to_mtd(chip); struct mtd_oob_ops ops; uint8_t buf[2] = { 0, 0 }; - int ret = 0, res, i = 0; + int ret = 0, res, page_offset; memset(&ops, 0, sizeof(ops)); ops.oobbuf = buf; @@ -540,17 +517,18 @@ static int nand_default_block_markbad(struct nand_chip *chip, loff_t ofs) } ops.mode = MTD_OPS_PLACE_OOB; - /* Write to first/last page(s) if necessary */ - if (chip->bbt_options & NAND_BBT_SCANLASTPAGE) - ofs += mtd->erasesize - mtd->writesize; - do { - res = nand_do_write_oob(chip, ofs, &ops); + page_offset = nand_bbm_get_next_page(chip, 0); + + while (page_offset >= 0) { + res = nand_do_write_oob(chip, + ofs + (page_offset * mtd->writesize), + &ops); + if (!ret) ret = res; - i++; - ofs += mtd->writesize; - } while ((chip->bbt_options & NAND_BBT_SCAN2NDPAGE) && i < 2); + page_offset = nand_bbm_get_next_page(chip, page_offset + 1); + } return ret; } @@ -602,7 +580,8 @@ static int nand_block_markbad_lowlevel(struct nand_chip *chip, loff_t ofs) nand_erase_nand(chip, &einfo, 0); /* Write bad block marker to OOB */ - nand_get_device(chip, FL_WRITING); + nand_get_device(chip); + ret = nand_markbad_bbm(chip, ofs); nand_release_device(chip); } @@ -674,7 +653,7 @@ static int nand_block_checkbad(struct nand_chip *chip, loff_t ofs, int allowbbt) */ int nand_soft_waitrdy(struct nand_chip *chip, unsigned long timeout_ms) { - const struct nand_sdr_timings *timings; + const struct nand_interface_config *conf; u8 status = 0; int ret; @@ -682,16 +661,22 @@ int nand_soft_waitrdy(struct nand_chip *chip, unsigned long timeout_ms) return -ENOTSUPP; /* Wait tWB before polling the STATUS reg. */ - timings = nand_get_sdr_timings(&chip->data_interface); - ndelay(PSEC_TO_NSEC(timings->tWB_max)); + conf = nand_get_interface_config(chip); + ndelay(NAND_COMMON_TIMING_NS(conf, tWB_max)); ret = nand_status_op(chip, NULL); if (ret) return ret; - timeout_ms = jiffies + msecs_to_jiffies(timeout_ms); + /* + * +1 below is necessary because if we are now in the last fraction + * of jiffy and msecs_to_jiffies is 1 then we will wait only that + * small jiffy fraction - possibly leading to false timeout + */ + timeout_ms = jiffies + msecs_to_jiffies(timeout_ms) + 1; do { - ret = nand_read_data_op(chip, &status, sizeof(status), true); + ret = nand_read_data_op(chip, &status, sizeof(status), true, + false); if (ret) break; @@ -737,8 +722,14 @@ EXPORT_SYMBOL_GPL(nand_soft_waitrdy); int nand_gpio_waitrdy(struct nand_chip *chip, struct gpio_desc *gpiod, unsigned long timeout_ms) { - /* Wait until R/B pin indicates chip is ready or timeout occurs */ - timeout_ms = jiffies + msecs_to_jiffies(timeout_ms); + + /* + * Wait until R/B pin indicates chip is ready or timeout occurs. + * +1 below is necessary because if we are now in the last fraction + * of jiffy and msecs_to_jiffies is 1 then we will wait only that + * small jiffy fraction - possibly leading to false timeout. + */ + timeout_ms = jiffies + msecs_to_jiffies(timeout_ms) + 1; do { if (gpiod_get_value_cansleep(gpiod)) return 0; @@ -771,7 +762,7 @@ void panic_nand_wait(struct nand_chip *chip, unsigned long timeo) u8 status; ret = nand_read_data_op(chip, &status, sizeof(status), - true); + true, false); if (ret) return; @@ -795,7 +786,7 @@ static bool nand_supports_set_features(struct nand_chip *chip, int addr) } /** - * nand_reset_data_interface - Reset data interface and timings + * nand_reset_interface - Reset data interface and timings * @chip: The NAND chip * @chipnr: Internal die id * @@ -803,11 +794,12 @@ static bool nand_supports_set_features(struct nand_chip *chip, int addr) * * Returns 0 for success or negative error code otherwise. */ -static int nand_reset_data_interface(struct nand_chip *chip, int chipnr) +static int nand_reset_interface(struct nand_chip *chip, int chipnr) { + const struct nand_controller_ops *ops = chip->controller->ops; int ret; - if (!nand_has_setup_data_iface(chip)) + if (!nand_controller_can_setup_interface(chip)) return 0; /* @@ -824,9 +816,9 @@ static int nand_reset_data_interface(struct nand_chip *chip, int chipnr) * timings to timing mode 0. */ - onfi_fill_data_interface(chip, NAND_SDR_IFACE, 0); - ret = chip->controller->ops->setup_data_interface(chip, chipnr, - &chip->data_interface); + chip->current_interface_config = nand_get_reset_interface_config(); + ret = ops->setup_interface(chip, chipnr, + chip->current_interface_config); if (ret) pr_err("Failed to configure data interface to SDR timing mode 0\n"); @@ -834,28 +826,41 @@ static int nand_reset_data_interface(struct nand_chip *chip, int chipnr) } /** - * nand_setup_data_interface - Setup the best data interface and timings + * nand_setup_interface - Setup the best data interface and timings * @chip: The NAND chip * @chipnr: Internal die id * - * Find and configure the best data interface and NAND timings supported by - * the chip and the driver. - * First tries to retrieve supported timing modes from ONFI information, - * and if the NAND chip does not support ONFI, relies on the - * ->onfi_timing_mode_default specified in the nand_ids table. + * Configure what has been reported to be the best data interface and NAND + * timings supported by the chip and the driver. * * Returns 0 for success or negative error code otherwise. */ -static int nand_setup_data_interface(struct nand_chip *chip, int chipnr) +static int nand_setup_interface(struct nand_chip *chip, int chipnr) { - u8 tmode_param[ONFI_SUBFEATURE_PARAM_LEN] = { - chip->onfi_timing_mode_default, - }; + const struct nand_controller_ops *ops = chip->controller->ops; + u8 tmode_param[ONFI_SUBFEATURE_PARAM_LEN] = { }, request; int ret; - if (!nand_has_setup_data_iface(chip)) + if (!nand_controller_can_setup_interface(chip)) return 0; + /* + * A nand_reset_interface() put both the NAND chip and the NAND + * controller in timings mode 0. If the default mode for this chip is + * also 0, no need to proceed to the change again. Plus, at probe time, + * nand_setup_interface() uses ->set/get_features() which would + * fail anyway as the parameter page is not available yet. + */ + if (!chip->best_interface_config) + return 0; + + request = chip->best_interface_config->timings.mode; + if (nand_interface_is_sdr(chip->best_interface_config)) + request |= ONFI_DATA_INTERFACE_SDR; + else + request |= ONFI_DATA_INTERFACE_NVDDR; + tmode_param[0] = request; + /* Change the mode on the chip side (if supported by the NAND chip) */ if (nand_supports_set_features(chip, ONFI_FEATURE_ADDR_TIMING_MODE)) { nand_select_target(chip, chipnr); @@ -867,14 +872,13 @@ static int nand_setup_data_interface(struct nand_chip *chip, int chipnr) } /* Change the mode on the controller side */ - ret = chip->controller->ops->setup_data_interface(chip, chipnr, - &chip->data_interface); + ret = ops->setup_interface(chip, chipnr, chip->best_interface_config); if (ret) return ret; /* Check the mode has been accepted by the chip, if supported */ if (!nand_supports_get_features(chip, ONFI_FEATURE_ADDR_TIMING_MODE)) - return 0; + goto update_interface_config; memset(tmode_param, 0, ONFI_SUBFEATURE_PARAM_LEN); nand_select_target(chip, chipnr); @@ -884,12 +888,19 @@ static int nand_setup_data_interface(struct nand_chip *chip, int chipnr) if (ret) goto err_reset_chip; - if (tmode_param[0] != chip->onfi_timing_mode_default) { - pr_warn("timing mode %d not acknowledged by the NAND chip\n", - chip->onfi_timing_mode_default); + if (request != tmode_param[0]) { + pr_warn("%s timing mode %d not acknowledged by the NAND chip\n", + nand_interface_is_nvddr(chip->best_interface_config) ? "NV-DDR" : "SDR", + chip->best_interface_config->timings.mode); + pr_debug("NAND chip would work in %s timing mode %d\n", + tmode_param[0] & ONFI_DATA_INTERFACE_NVDDR ? "NV-DDR" : "SDR", + (unsigned int)ONFI_TIMING_MODE_PARAM(tmode_param[0])); goto err_reset_chip; } +update_interface_config: + chip->current_interface_config = chip->best_interface_config; + return 0; err_reset_chip: @@ -897,7 +908,7 @@ err_reset_chip: * Fallback to mode 0 if the chip explicitly did not ack the chosen * timing mode. */ - nand_reset_data_interface(chip, chipnr); + nand_reset_interface(chip, chipnr); nand_select_target(chip, chipnr); nand_reset_op(chip); nand_deselect_target(chip); @@ -906,59 +917,164 @@ err_reset_chip: } /** - * nand_init_data_interface - find the best data interface and timings - * @chip: The NAND chip + * nand_choose_best_sdr_timings - Pick up the best SDR timings that both the + * NAND controller and the NAND chip support + * @chip: the NAND chip + * @iface: the interface configuration (can eventually be updated) + * @spec_timings: specific timings, when not fitting the ONFI specification * - * Find the best data interface and NAND timings supported by the chip - * and the driver. - * First tries to retrieve supported timing modes from ONFI information, - * and if the NAND chip does not support ONFI, relies on the - * ->onfi_timing_mode_default specified in the nand_ids table. After this - * function nand_chip->data_interface is initialized with the best timing mode - * available. + * If specific timings are provided, use them. Otherwise, retrieve supported + * timing modes from ONFI information. + */ +int nand_choose_best_sdr_timings(struct nand_chip *chip, + struct nand_interface_config *iface, + struct nand_sdr_timings *spec_timings) +{ + const struct nand_controller_ops *ops = chip->controller->ops; + int best_mode = 0, mode, ret = -EOPNOTSUPP; + + iface->type = NAND_SDR_IFACE; + + if (spec_timings) { + iface->timings.sdr = *spec_timings; + iface->timings.mode = onfi_find_closest_sdr_mode(spec_timings); + + /* Verify the controller supports the requested interface */ + ret = ops->setup_interface(chip, NAND_DATA_IFACE_CHECK_ONLY, + iface); + if (!ret) { + chip->best_interface_config = iface; + return ret; + } + + /* Fallback to slower modes */ + best_mode = iface->timings.mode; + } else if (chip->parameters.onfi) { + best_mode = fls(chip->parameters.onfi->sdr_timing_modes) - 1; + } + + for (mode = best_mode; mode >= 0; mode--) { + onfi_fill_interface_config(chip, iface, NAND_SDR_IFACE, mode); + + ret = ops->setup_interface(chip, NAND_DATA_IFACE_CHECK_ONLY, + iface); + if (!ret) { + chip->best_interface_config = iface; + break; + } + } + + return ret; +} + +/** + * nand_choose_best_nvddr_timings - Pick up the best NVDDR timings that both the + * NAND controller and the NAND chip support + * @chip: the NAND chip + * @iface: the interface configuration (can eventually be updated) + * @spec_timings: specific timings, when not fitting the ONFI specification * - * Returns 0 for success or negative error code otherwise. + * If specific timings are provided, use them. Otherwise, retrieve supported + * timing modes from ONFI information. */ -static int nand_init_data_interface(struct nand_chip *chip) +int nand_choose_best_nvddr_timings(struct nand_chip *chip, + struct nand_interface_config *iface, + struct nand_nvddr_timings *spec_timings) { - int modes, mode, ret; + const struct nand_controller_ops *ops = chip->controller->ops; + int best_mode = 0, mode, ret = -EOPNOTSUPP; - if (!nand_has_setup_data_iface(chip)) - return 0; + iface->type = NAND_NVDDR_IFACE; - /* - * First try to identify the best timings from ONFI parameters and - * if the NAND does not support ONFI, fallback to the default ONFI - * timing mode. - */ - if (chip->parameters.onfi) { - modes = chip->parameters.onfi->async_timing_mode; - } else { - if (!chip->onfi_timing_mode_default) - return 0; + if (spec_timings) { + iface->timings.nvddr = *spec_timings; + iface->timings.mode = onfi_find_closest_nvddr_mode(spec_timings); + + /* Verify the controller supports the requested interface */ + ret = ops->setup_interface(chip, NAND_DATA_IFACE_CHECK_ONLY, + iface); + if (!ret) { + chip->best_interface_config = iface; + return ret; + } - modes = GENMASK(chip->onfi_timing_mode_default, 0); + /* Fallback to slower modes */ + best_mode = iface->timings.mode; + } else if (chip->parameters.onfi) { + best_mode = fls(chip->parameters.onfi->nvddr_timing_modes) - 1; } - for (mode = fls(modes) - 1; mode >= 0; mode--) { - ret = onfi_fill_data_interface(chip, NAND_SDR_IFACE, mode); - if (ret) - continue; + for (mode = best_mode; mode >= 0; mode--) { + onfi_fill_interface_config(chip, iface, NAND_NVDDR_IFACE, mode); - /* - * Pass NAND_DATA_IFACE_CHECK_ONLY to only check if the - * controller supports the requested timings. - */ - ret = chip->controller->ops->setup_data_interface(chip, - NAND_DATA_IFACE_CHECK_ONLY, - &chip->data_interface); + ret = ops->setup_interface(chip, NAND_DATA_IFACE_CHECK_ONLY, + iface); if (!ret) { - chip->onfi_timing_mode_default = mode; + chip->best_interface_config = iface; break; } } - return 0; + return ret; +} + +/** + * nand_choose_best_timings - Pick up the best NVDDR or SDR timings that both + * NAND controller and the NAND chip support + * @chip: the NAND chip + * @iface: the interface configuration (can eventually be updated) + * + * If specific timings are provided, use them. Otherwise, retrieve supported + * timing modes from ONFI information. + */ +static int nand_choose_best_timings(struct nand_chip *chip, + struct nand_interface_config *iface) +{ + int ret; + + /* Try the fastest timings: NV-DDR */ + ret = nand_choose_best_nvddr_timings(chip, iface, NULL); + if (!ret) + return 0; + + /* Fallback to SDR timings otherwise */ + return nand_choose_best_sdr_timings(chip, iface, NULL); +} + +/** + * nand_choose_interface_config - find the best data interface and timings + * @chip: The NAND chip + * + * Find the best data interface and NAND timings supported by the chip + * and the driver. Eventually let the NAND manufacturer driver propose his own + * set of timings. + * + * After this function nand_chip->interface_config is initialized with the best + * timing mode available. + * + * Returns 0 for success or negative error code otherwise. + */ +static int nand_choose_interface_config(struct nand_chip *chip) +{ + struct nand_interface_config *iface; + int ret; + + if (!nand_controller_can_setup_interface(chip)) + return 0; + + iface = kzalloc(sizeof(*iface), GFP_KERNEL); + if (!iface) + return -ENOMEM; + + if (chip->ops.choose_interface_config) + ret = chip->ops.choose_interface_config(chip, iface); + else + ret = nand_choose_best_timings(chip, iface); + + if (ret) + kfree(iface); + + return ret; } /** @@ -977,28 +1093,32 @@ static int nand_fill_column_cycles(struct nand_chip *chip, u8 *addrs, unsigned int offset_in_page) { struct mtd_info *mtd = nand_to_mtd(chip); + bool ident_stage = !mtd->writesize; - /* Make sure the offset is less than the actual page size. */ - if (offset_in_page > mtd->writesize + mtd->oobsize) - return -EINVAL; + /* Bypass all checks during NAND identification */ + if (likely(!ident_stage)) { + /* Make sure the offset is less than the actual page size. */ + if (offset_in_page > mtd->writesize + mtd->oobsize) + return -EINVAL; - /* - * On small page NANDs, there's a dedicated command to access the OOB - * area, and the column address is relative to the start of the OOB - * area, not the start of the page. Asjust the address accordingly. - */ - if (mtd->writesize <= 512 && offset_in_page >= mtd->writesize) - offset_in_page -= mtd->writesize; + /* + * On small page NANDs, there's a dedicated command to access the OOB + * area, and the column address is relative to the start of the OOB + * area, not the start of the page. Asjust the address accordingly. + */ + if (mtd->writesize <= 512 && offset_in_page >= mtd->writesize) + offset_in_page -= mtd->writesize; - /* - * The offset in page is expressed in bytes, if the NAND bus is 16-bit - * wide, then it must be divided by 2. - */ - if (chip->options & NAND_BUSWIDTH_16) { - if (WARN_ON(offset_in_page % 2)) - return -EINVAL; + /* + * The offset in page is expressed in bytes, if the NAND bus is 16-bit + * wide, then it must be divided by 2. + */ + if (chip->options & NAND_BUSWIDTH_16) { + if (WARN_ON(offset_in_page % 2)) + return -EINVAL; - offset_in_page /= 2; + offset_in_page /= 2; + } } addrs[0] = offset_in_page; @@ -1007,7 +1127,7 @@ static int nand_fill_column_cycles(struct nand_chip *chip, u8 *addrs, * Small page NANDs use 1 cycle for the columns, while large page NANDs * need 2 */ - if (mtd->writesize <= 512) + if (!ident_stage && mtd->writesize <= 512) return 1; addrs[1] = offset_in_page >> 8; @@ -1019,15 +1139,15 @@ static int nand_sp_exec_read_page_op(struct nand_chip *chip, unsigned int page, unsigned int offset_in_page, void *buf, unsigned int len) { + const struct nand_interface_config *conf = + nand_get_interface_config(chip); struct mtd_info *mtd = nand_to_mtd(chip); - const struct nand_sdr_timings *sdr = - nand_get_sdr_timings(&chip->data_interface); u8 addrs[4]; struct nand_op_instr instrs[] = { NAND_OP_CMD(NAND_CMD_READ0, 0), - NAND_OP_ADDR(3, addrs, PSEC_TO_NSEC(sdr->tWB_max)), - NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tR_max), - PSEC_TO_NSEC(sdr->tRR_min)), + NAND_OP_ADDR(3, addrs, NAND_COMMON_TIMING_NS(conf, tWB_max)), + NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max), + NAND_COMMON_TIMING_NS(conf, tRR_min)), NAND_OP_DATA_IN(len, buf, 0), }; struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs); @@ -1062,15 +1182,15 @@ static int nand_lp_exec_read_page_op(struct nand_chip *chip, unsigned int page, unsigned int offset_in_page, void *buf, unsigned int len) { - const struct nand_sdr_timings *sdr = - nand_get_sdr_timings(&chip->data_interface); + const struct nand_interface_config *conf = + nand_get_interface_config(chip); u8 addrs[5]; struct nand_op_instr instrs[] = { NAND_OP_CMD(NAND_CMD_READ0, 0), NAND_OP_ADDR(4, addrs, 0), - NAND_OP_CMD(NAND_CMD_READSTART, PSEC_TO_NSEC(sdr->tWB_max)), - NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tR_max), - PSEC_TO_NSEC(sdr->tRR_min)), + NAND_OP_CMD(NAND_CMD_READSTART, NAND_COMMON_TIMING_NS(conf, tWB_max)), + NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max), + NAND_COMMON_TIMING_NS(conf, tRR_min)), NAND_OP_DATA_IN(len, buf, 0), }; struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs); @@ -1095,6 +1215,117 @@ static int nand_lp_exec_read_page_op(struct nand_chip *chip, unsigned int page, return nand_exec_op(chip, &op); } +static unsigned int rawnand_last_page_of_lun(unsigned int pages_per_lun, unsigned int lun) +{ + /* lun is expected to be very small */ + return (lun * pages_per_lun) + pages_per_lun - 1; +} + +static void rawnand_cap_cont_reads(struct nand_chip *chip) +{ + struct nand_memory_organization *memorg; + unsigned int ppl, first_lun, last_lun; + + memorg = nanddev_get_memorg(&chip->base); + ppl = memorg->pages_per_eraseblock * memorg->eraseblocks_per_lun; + first_lun = chip->cont_read.first_page / ppl; + last_lun = chip->cont_read.last_page / ppl; + + /* Prevent sequential cache reads across LUN boundaries */ + if (first_lun != last_lun) + chip->cont_read.pause_page = rawnand_last_page_of_lun(ppl, first_lun); + else + chip->cont_read.pause_page = chip->cont_read.last_page; + + if (chip->cont_read.first_page == chip->cont_read.pause_page) { + chip->cont_read.first_page++; + chip->cont_read.pause_page = min(chip->cont_read.last_page, + rawnand_last_page_of_lun(ppl, first_lun + 1)); + } + + if (chip->cont_read.first_page >= chip->cont_read.last_page) + chip->cont_read.ongoing = false; +} + +static int nand_lp_exec_cont_read_page_op(struct nand_chip *chip, unsigned int page, + unsigned int offset_in_page, void *buf, + unsigned int len, bool check_only) +{ + const struct nand_interface_config *conf = + nand_get_interface_config(chip); + u8 addrs[5]; + struct nand_op_instr start_instrs[] = { + NAND_OP_CMD(NAND_CMD_READ0, 0), + NAND_OP_ADDR(4, addrs, 0), + NAND_OP_CMD(NAND_CMD_READSTART, NAND_COMMON_TIMING_NS(conf, tWB_max)), + NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max), 0), + NAND_OP_CMD(NAND_CMD_READCACHESEQ, NAND_COMMON_TIMING_NS(conf, tWB_max)), + NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max), + NAND_COMMON_TIMING_NS(conf, tRR_min)), + NAND_OP_DATA_IN(len, buf, 0), + }; + struct nand_op_instr cont_instrs[] = { + NAND_OP_CMD(page == chip->cont_read.pause_page ? + NAND_CMD_READCACHEEND : NAND_CMD_READCACHESEQ, + NAND_COMMON_TIMING_NS(conf, tWB_max)), + NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max), + NAND_COMMON_TIMING_NS(conf, tRR_min)), + NAND_OP_DATA_IN(len, buf, 0), + }; + struct nand_operation start_op = NAND_OPERATION(chip->cur_cs, start_instrs); + struct nand_operation cont_op = NAND_OPERATION(chip->cur_cs, cont_instrs); + int ret; + + if (!len) { + start_op.ninstrs--; + cont_op.ninstrs--; + } + + ret = nand_fill_column_cycles(chip, addrs, offset_in_page); + if (ret < 0) + return ret; + + addrs[2] = page; + addrs[3] = page >> 8; + + if (chip->options & NAND_ROW_ADDR_3) { + addrs[4] = page >> 16; + start_instrs[1].ctx.addr.naddrs++; + } + + /* Check if cache reads are supported */ + if (check_only) { + if (nand_check_op(chip, &start_op) || nand_check_op(chip, &cont_op)) + return -EOPNOTSUPP; + + return 0; + } + + if (page == chip->cont_read.first_page) + ret = nand_exec_op(chip, &start_op); + else + ret = nand_exec_op(chip, &cont_op); + if (ret) + return ret; + + if (!chip->cont_read.ongoing) + return 0; + + if (page == chip->cont_read.last_page) { + chip->cont_read.ongoing = false; + } else if (page == chip->cont_read.pause_page) { + chip->cont_read.first_page++; + rawnand_cap_cont_reads(chip); + } + + return 0; +} + +static bool rawnand_cont_read_ongoing(struct nand_chip *chip, unsigned int page) +{ + return chip->cont_read.ongoing && page >= chip->cont_read.first_page; +} + /** * nand_read_page_op - Do a READ PAGE operation * @chip: The NAND chip @@ -1120,10 +1351,16 @@ int nand_read_page_op(struct nand_chip *chip, unsigned int page, return -EINVAL; if (nand_has_exec_op(chip)) { - if (mtd->writesize > 512) - return nand_lp_exec_read_page_op(chip, page, - offset_in_page, buf, - len); + if (mtd->writesize > 512) { + if (rawnand_cont_read_ongoing(chip, page)) + return nand_lp_exec_cont_read_page_op(chip, page, + offset_in_page, + buf, len, false); + else + return nand_lp_exec_read_page_op(chip, page, + offset_in_page, buf, + len); + } return nand_sp_exec_read_page_op(chip, page, offset_in_page, buf, len); @@ -1159,13 +1396,14 @@ int nand_read_param_page_op(struct nand_chip *chip, u8 page, void *buf, return -EINVAL; if (nand_has_exec_op(chip)) { - const struct nand_sdr_timings *sdr = - nand_get_sdr_timings(&chip->data_interface); + const struct nand_interface_config *conf = + nand_get_interface_config(chip); struct nand_op_instr instrs[] = { NAND_OP_CMD(NAND_CMD_PARAM, 0), - NAND_OP_ADDR(1, &page, PSEC_TO_NSEC(sdr->tWB_max)), - NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tR_max), - PSEC_TO_NSEC(sdr->tRR_min)), + NAND_OP_ADDR(1, &page, + NAND_COMMON_TIMING_NS(conf, tWB_max)), + NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tR_max), + NAND_COMMON_TIMING_NS(conf, tRR_min)), NAND_OP_8BIT_DATA_IN(len, buf, 0), }; struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs); @@ -1202,26 +1440,29 @@ int nand_change_read_column_op(struct nand_chip *chip, unsigned int len, bool force_8bit) { struct mtd_info *mtd = nand_to_mtd(chip); + bool ident_stage = !mtd->writesize; if (len && !buf) return -EINVAL; - if (offset_in_page + len > mtd->writesize + mtd->oobsize) - return -EINVAL; + if (!ident_stage) { + if (offset_in_page + len > mtd->writesize + mtd->oobsize) + return -EINVAL; - /* Small page NANDs do not support column change. */ - if (mtd->writesize <= 512) - return -ENOTSUPP; + /* Small page NANDs do not support column change. */ + if (mtd->writesize <= 512) + return -ENOTSUPP; + } if (nand_has_exec_op(chip)) { - const struct nand_sdr_timings *sdr = - nand_get_sdr_timings(&chip->data_interface); + const struct nand_interface_config *conf = + nand_get_interface_config(chip); u8 addrs[2] = {}; struct nand_op_instr instrs[] = { NAND_OP_CMD(NAND_CMD_RNDOUT, 0), NAND_OP_ADDR(2, addrs, 0), NAND_OP_CMD(NAND_CMD_RNDOUTSTART, - PSEC_TO_NSEC(sdr->tCCS_min)), + NAND_COMMON_TIMING_NS(conf, tCCS_min)), NAND_OP_DATA_IN(len, buf, 0), }; struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs); @@ -1289,9 +1530,9 @@ static int nand_exec_prog_page_op(struct nand_chip *chip, unsigned int page, unsigned int offset_in_page, const void *buf, unsigned int len, bool prog) { + const struct nand_interface_config *conf = + nand_get_interface_config(chip); struct mtd_info *mtd = nand_to_mtd(chip); - const struct nand_sdr_timings *sdr = - nand_get_sdr_timings(&chip->data_interface); u8 addrs[5] = {}; struct nand_op_instr instrs[] = { /* @@ -1301,15 +1542,15 @@ static int nand_exec_prog_page_op(struct nand_chip *chip, unsigned int page, */ NAND_OP_CMD(NAND_CMD_READ0, 0), NAND_OP_CMD(NAND_CMD_SEQIN, 0), - NAND_OP_ADDR(0, addrs, PSEC_TO_NSEC(sdr->tADL_min)), + NAND_OP_ADDR(0, addrs, NAND_COMMON_TIMING_NS(conf, tADL_min)), NAND_OP_DATA_OUT(len, buf, 0), - NAND_OP_CMD(NAND_CMD_PAGEPROG, PSEC_TO_NSEC(sdr->tWB_max)), - NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tPROG_max), 0), + NAND_OP_CMD(NAND_CMD_PAGEPROG, + NAND_COMMON_TIMING_NS(conf, tWB_max)), + NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tPROG_max), 0), }; - struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs); + struct nand_operation op = NAND_DESTRUCTIVE_OPERATION(chip->cur_cs, + instrs); int naddrs = nand_fill_column_cycles(chip, addrs, offset_in_page); - int ret; - u8 status; if (naddrs < 0) return naddrs; @@ -1349,15 +1590,7 @@ static int nand_exec_prog_page_op(struct nand_chip *chip, unsigned int page, op.ninstrs--; } - ret = nand_exec_op(chip, &op); - if (!prog || ret) - return ret; - - ret = nand_status_op(chip, &status); - if (ret) - return ret; - - return status; + return nand_exec_op(chip, &op); } /** @@ -1413,12 +1646,13 @@ int nand_prog_page_end_op(struct nand_chip *chip) u8 status; if (nand_has_exec_op(chip)) { - const struct nand_sdr_timings *sdr = - nand_get_sdr_timings(&chip->data_interface); + const struct nand_interface_config *conf = + nand_get_interface_config(chip); struct nand_op_instr instrs[] = { NAND_OP_CMD(NAND_CMD_PAGEPROG, - PSEC_TO_NSEC(sdr->tWB_max)), - NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tPROG_max), 0), + NAND_COMMON_TIMING_NS(conf, tWB_max)), + NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tPROG_max), + 0), }; struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs); @@ -1463,7 +1697,8 @@ int nand_prog_page_op(struct nand_chip *chip, unsigned int page, unsigned int len) { struct mtd_info *mtd = nand_to_mtd(chip); - int status; + u8 status; + int ret; if (!len || !buf) return -EINVAL; @@ -1472,14 +1707,24 @@ int nand_prog_page_op(struct nand_chip *chip, unsigned int page, return -EINVAL; if (nand_has_exec_op(chip)) { - status = nand_exec_prog_page_op(chip, page, offset_in_page, buf, + ret = nand_exec_prog_page_op(chip, page, offset_in_page, buf, len, true); + if (ret) + return ret; + + ret = nand_status_op(chip, &status); + if (ret) + return ret; } else { chip->legacy.cmdfunc(chip, NAND_CMD_SEQIN, offset_in_page, page); chip->legacy.write_buf(chip, buf, len); chip->legacy.cmdfunc(chip, NAND_CMD_PAGEPROG, -1, -1); - status = chip->legacy.waitfunc(chip); + ret = chip->legacy.waitfunc(chip); + if (ret < 0) + return ret; + + status = ret; } if (status & NAND_STATUS_FAIL) @@ -1520,12 +1765,12 @@ int nand_change_write_column_op(struct nand_chip *chip, return -ENOTSUPP; if (nand_has_exec_op(chip)) { - const struct nand_sdr_timings *sdr = - nand_get_sdr_timings(&chip->data_interface); + const struct nand_interface_config *conf = + nand_get_interface_config(chip); u8 addrs[2]; struct nand_op_instr instrs[] = { NAND_OP_CMD(NAND_CMD_RNDIN, 0), - NAND_OP_ADDR(2, addrs, PSEC_TO_NSEC(sdr->tCCS_min)), + NAND_OP_ADDR(2, addrs, NAND_COMMON_TIMING_NS(conf, tCCS_min)), NAND_OP_DATA_OUT(len, buf, 0), }; struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs); @@ -1569,26 +1814,46 @@ int nand_readid_op(struct nand_chip *chip, u8 addr, void *buf, unsigned int len) { unsigned int i; - u8 *id = buf; + u8 *id = buf, *ddrbuf = NULL; if (len && !buf) return -EINVAL; if (nand_has_exec_op(chip)) { - const struct nand_sdr_timings *sdr = - nand_get_sdr_timings(&chip->data_interface); + const struct nand_interface_config *conf = + nand_get_interface_config(chip); struct nand_op_instr instrs[] = { NAND_OP_CMD(NAND_CMD_READID, 0), - NAND_OP_ADDR(1, &addr, PSEC_TO_NSEC(sdr->tADL_min)), + NAND_OP_ADDR(1, &addr, + NAND_COMMON_TIMING_NS(conf, tADL_min)), NAND_OP_8BIT_DATA_IN(len, buf, 0), }; struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs); + int ret; + + /* READ_ID data bytes are received twice in NV-DDR mode */ + if (len && nand_interface_is_nvddr(conf)) { + ddrbuf = kcalloc(2, len, GFP_KERNEL); + if (!ddrbuf) + return -ENOMEM; + + instrs[2].ctx.data.len *= 2; + instrs[2].ctx.data.buf.in = ddrbuf; + } /* Drop the DATA_IN instruction if len is set to 0. */ if (!len) op.ninstrs--; - return nand_exec_op(chip, &op); + ret = nand_exec_op(chip, &op); + if (!ret && len && nand_interface_is_nvddr(conf)) { + for (i = 0; i < len; i++) + id[i] = ddrbuf[i * 2]; + } + + kfree(ddrbuf); + + return ret; } chip->legacy.cmdfunc(chip, NAND_CMD_READID, addr, -1); @@ -1614,19 +1879,31 @@ EXPORT_SYMBOL_GPL(nand_readid_op); int nand_status_op(struct nand_chip *chip, u8 *status) { if (nand_has_exec_op(chip)) { - const struct nand_sdr_timings *sdr = - nand_get_sdr_timings(&chip->data_interface); + const struct nand_interface_config *conf = + nand_get_interface_config(chip); + u8 ddrstatus[2]; struct nand_op_instr instrs[] = { NAND_OP_CMD(NAND_CMD_STATUS, - PSEC_TO_NSEC(sdr->tADL_min)), + NAND_COMMON_TIMING_NS(conf, tADL_min)), NAND_OP_8BIT_DATA_IN(1, status, 0), }; struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs); + int ret; + + /* The status data byte will be received twice in NV-DDR mode */ + if (status && nand_interface_is_nvddr(conf)) { + instrs[1].ctx.data.len *= 2; + instrs[1].ctx.data.buf.in = ddrstatus; + } if (!status) op.ninstrs--; - return nand_exec_op(chip, &op); + ret = nand_exec_op(chip, &op); + if (!ret && status && nand_interface_is_nvddr(conf)) + *status = ddrstatus[0]; + + return ret; } chip->legacy.cmdfunc(chip, NAND_CMD_STATUS, -1, -1); @@ -1663,6 +1940,7 @@ int nand_exit_status_op(struct nand_chip *chip) return 0; } +EXPORT_SYMBOL_GPL(nand_exit_status_op); /** * nand_erase_op - Do an erase operation @@ -1683,17 +1961,19 @@ int nand_erase_op(struct nand_chip *chip, unsigned int eraseblock) u8 status; if (nand_has_exec_op(chip)) { - const struct nand_sdr_timings *sdr = - nand_get_sdr_timings(&chip->data_interface); + const struct nand_interface_config *conf = + nand_get_interface_config(chip); u8 addrs[3] = { page, page >> 8, page >> 16 }; struct nand_op_instr instrs[] = { NAND_OP_CMD(NAND_CMD_ERASE1, 0), NAND_OP_ADDR(2, addrs, 0), NAND_OP_CMD(NAND_CMD_ERASE2, - PSEC_TO_MSEC(sdr->tWB_max)), - NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tBERS_max), 0), + NAND_COMMON_TIMING_NS(conf, tWB_max)), + NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tBERS_max), + 0), }; - struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs); + struct nand_operation op = NAND_DESTRUCTIVE_OPERATION(chip->cur_cs, + instrs); if (chip->options & NAND_ROW_ADDR_3) instrs[1].ctx.addr.naddrs++; @@ -1742,14 +2022,17 @@ static int nand_set_features_op(struct nand_chip *chip, u8 feature, int i, ret; if (nand_has_exec_op(chip)) { - const struct nand_sdr_timings *sdr = - nand_get_sdr_timings(&chip->data_interface); + const struct nand_interface_config *conf = + nand_get_interface_config(chip); struct nand_op_instr instrs[] = { NAND_OP_CMD(NAND_CMD_SET_FEATURES, 0), - NAND_OP_ADDR(1, &feature, PSEC_TO_NSEC(sdr->tADL_min)), + NAND_OP_ADDR(1, &feature, NAND_COMMON_TIMING_NS(conf, + tADL_min)), NAND_OP_8BIT_DATA_OUT(ONFI_SUBFEATURE_PARAM_LEN, data, - PSEC_TO_NSEC(sdr->tWB_max)), - NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tFEAT_max), 0), + NAND_COMMON_TIMING_NS(conf, + tWB_max)), + NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tFEAT_max), + 0), }; struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs); @@ -1785,23 +2068,37 @@ static int nand_set_features_op(struct nand_chip *chip, u8 feature, static int nand_get_features_op(struct nand_chip *chip, u8 feature, void *data) { - u8 *params = data; + u8 *params = data, ddrbuf[ONFI_SUBFEATURE_PARAM_LEN * 2]; int i; if (nand_has_exec_op(chip)) { - const struct nand_sdr_timings *sdr = - nand_get_sdr_timings(&chip->data_interface); + const struct nand_interface_config *conf = + nand_get_interface_config(chip); struct nand_op_instr instrs[] = { NAND_OP_CMD(NAND_CMD_GET_FEATURES, 0), - NAND_OP_ADDR(1, &feature, PSEC_TO_NSEC(sdr->tWB_max)), - NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tFEAT_max), - PSEC_TO_NSEC(sdr->tRR_min)), + NAND_OP_ADDR(1, &feature, + NAND_COMMON_TIMING_NS(conf, tWB_max)), + NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tFEAT_max), + NAND_COMMON_TIMING_NS(conf, tRR_min)), NAND_OP_8BIT_DATA_IN(ONFI_SUBFEATURE_PARAM_LEN, data, 0), }; struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs); + int ret; - return nand_exec_op(chip, &op); + /* GET_FEATURE data bytes are received twice in NV-DDR mode */ + if (nand_interface_is_nvddr(conf)) { + instrs[3].ctx.data.len *= 2; + instrs[3].ctx.data.buf.in = ddrbuf; + } + + ret = nand_exec_op(chip, &op); + if (nand_interface_is_nvddr(conf)) { + for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; i++) + params[i] = ddrbuf[i * 2]; + } + + return ret; } chip->legacy.cmdfunc(chip, NAND_CMD_GET_FEATURES, feature, -1); @@ -1846,11 +2143,13 @@ static int nand_wait_rdy_op(struct nand_chip *chip, unsigned int timeout_ms, int nand_reset_op(struct nand_chip *chip) { if (nand_has_exec_op(chip)) { - const struct nand_sdr_timings *sdr = - nand_get_sdr_timings(&chip->data_interface); + const struct nand_interface_config *conf = + nand_get_interface_config(chip); struct nand_op_instr instrs[] = { - NAND_OP_CMD(NAND_CMD_RESET, PSEC_TO_NSEC(sdr->tWB_max)), - NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tRST_max), 0), + NAND_OP_CMD(NAND_CMD_RESET, + NAND_COMMON_TIMING_NS(conf, tWB_max)), + NAND_OP_WAIT_RDY(NAND_COMMON_TIMING_MS(conf, tRST_max), + 0), }; struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs); @@ -1869,6 +2168,8 @@ EXPORT_SYMBOL_GPL(nand_reset_op); * @buf: buffer used to store the data * @len: length of the buffer * @force_8bit: force 8-bit bus access + * @check_only: do not actually run the command, only checks if the + * controller driver supports it * * This function does a raw data read on the bus. Usually used after launching * another NAND operation like nand_read_page_op(). @@ -1877,22 +2178,61 @@ EXPORT_SYMBOL_GPL(nand_reset_op); * Returns 0 on success, a negative error code otherwise. */ int nand_read_data_op(struct nand_chip *chip, void *buf, unsigned int len, - bool force_8bit) + bool force_8bit, bool check_only) { - if (!len || !buf) + if (!len || (!check_only && !buf)) return -EINVAL; if (nand_has_exec_op(chip)) { + const struct nand_interface_config *conf = + nand_get_interface_config(chip); struct nand_op_instr instrs[] = { NAND_OP_DATA_IN(len, buf, 0), }; struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs); + u8 *ddrbuf = NULL; + int ret, i; instrs[0].ctx.data.force_8bit = force_8bit; - return nand_exec_op(chip, &op); + /* + * Parameter payloads (ID, status, features, etc) do not go + * through the same pipeline as regular data, hence the + * force_8bit flag must be set and this also indicates that in + * case NV-DDR timings are being used the data will be received + * twice. + */ + if (force_8bit && nand_interface_is_nvddr(conf)) { + ddrbuf = kcalloc(2, len, GFP_KERNEL); + if (!ddrbuf) + return -ENOMEM; + + instrs[0].ctx.data.len *= 2; + instrs[0].ctx.data.buf.in = ddrbuf; + } + + if (check_only) { + ret = nand_check_op(chip, &op); + kfree(ddrbuf); + return ret; + } + + ret = nand_exec_op(chip, &op); + if (!ret && force_8bit && nand_interface_is_nvddr(conf)) { + u8 *dst = buf; + + for (i = 0; i < len; i++) + dst[i] = ddrbuf[i * 2]; + } + + kfree(ddrbuf); + + return ret; } + if (check_only) + return 0; + if (force_8bit) { u8 *p = buf; unsigned int i; @@ -2113,7 +2453,7 @@ static void nand_op_parser_trace(const struct nand_op_parser_ctx *ctx) char *prefix = " "; unsigned int i; - pr_debug("executing subop:\n"); + pr_debug("executing subop (CS%d):\n", ctx->subop.cs); for (i = 0; i < ctx->ninstrs; i++) { instr = &ctx->instrs[i]; @@ -2121,35 +2461,7 @@ static void nand_op_parser_trace(const struct nand_op_parser_ctx *ctx) if (instr == &ctx->subop.instrs[0]) prefix = " ->"; - switch (instr->type) { - case NAND_OP_CMD_INSTR: - pr_debug("%sCMD [0x%02x]\n", prefix, - instr->ctx.cmd.opcode); - break; - case NAND_OP_ADDR_INSTR: - pr_debug("%sADDR [%d cyc: %*ph]\n", prefix, - instr->ctx.addr.naddrs, - instr->ctx.addr.naddrs < 64 ? - instr->ctx.addr.naddrs : 64, - instr->ctx.addr.addrs); - break; - case NAND_OP_DATA_IN_INSTR: - pr_debug("%sDATA_IN [%d B%s]\n", prefix, - instr->ctx.data.len, - instr->ctx.data.force_8bit ? - ", force 8-bit" : ""); - break; - case NAND_OP_DATA_OUT_INSTR: - pr_debug("%sDATA_OUT [%d B%s]\n", prefix, - instr->ctx.data.len, - instr->ctx.data.force_8bit ? - ", force 8-bit" : ""); - break; - case NAND_OP_WAITRDY_INSTR: - pr_debug("%sWAITRDY [max %d ms]\n", prefix, - instr->ctx.waitrdy.timeout_ms); - break; - } + nand_op_trace(prefix, instr); if (instr == &ctx->subop.instrs[ctx->subop.ninstrs - 1]) prefix = " "; @@ -2162,6 +2474,22 @@ static void nand_op_parser_trace(const struct nand_op_parser_ctx *ctx) } #endif +static int nand_op_parser_cmp_ctx(const struct nand_op_parser_ctx *a, + const struct nand_op_parser_ctx *b) +{ + if (a->subop.ninstrs < b->subop.ninstrs) + return -1; + else if (a->subop.ninstrs > b->subop.ninstrs) + return 1; + + if (a->subop.last_instr_end_off < b->subop.last_instr_end_off) + return -1; + else if (a->subop.last_instr_end_off > b->subop.last_instr_end_off) + return 1; + + return 0; +} + /** * nand_op_parser_exec_op - exec_op parser * @chip: the NAND chip @@ -2189,6 +2517,7 @@ int nand_op_parser_exec_op(struct nand_chip *chip, const struct nand_operation *op, bool check_only) { struct nand_op_parser_ctx ctx = { + .subop.cs = op->cs, .subop.instrs = op->instrs, .instrs = op->instrs, .ninstrs = op->ninstrs, @@ -2196,32 +2525,40 @@ int nand_op_parser_exec_op(struct nand_chip *chip, unsigned int i; while (ctx.subop.instrs < op->instrs + op->ninstrs) { - int ret; + const struct nand_op_parser_pattern *pattern; + struct nand_op_parser_ctx best_ctx; + int ret, best_pattern = -1; for (i = 0; i < parser->npatterns; i++) { - const struct nand_op_parser_pattern *pattern; + struct nand_op_parser_ctx test_ctx = ctx; pattern = &parser->patterns[i]; - if (!nand_op_parser_match_pat(pattern, &ctx)) + if (!nand_op_parser_match_pat(pattern, &test_ctx)) continue; - nand_op_parser_trace(&ctx); - - if (check_only) - break; - - ret = pattern->exec(chip, &ctx.subop); - if (ret) - return ret; + if (best_pattern >= 0 && + nand_op_parser_cmp_ctx(&test_ctx, &best_ctx) <= 0) + continue; - break; + best_pattern = i; + best_ctx = test_ctx; } - if (i == parser->npatterns) { + if (best_pattern < 0) { pr_debug("->exec_op() parser: pattern not found!\n"); return -ENOTSUPP; } + ctx = best_ctx; + nand_op_parser_trace(&ctx); + + if (!check_only) { + pattern = &parser->patterns[best_pattern]; + ret = pattern->exec(chip, &ctx.subop); + if (ret) + return ret; + } + /* * Update the context structure by pointing to the start of the * next subop. @@ -2372,17 +2709,16 @@ EXPORT_SYMBOL_GPL(nand_subop_get_data_len); * @chipnr: Internal die id * * Save the timings data structure, then apply SDR timings mode 0 (see - * nand_reset_data_interface for details), do the reset operation, and - * apply back the previous timings. + * nand_reset_interface for details), do the reset operation, and apply + * back the previous timings. * * Returns 0 on success, a negative error code otherwise. */ int nand_reset(struct nand_chip *chip, int chipnr) { - struct nand_data_interface saved_data_intf = chip->data_interface; int ret; - ret = nand_reset_data_interface(chip, chipnr); + ret = nand_reset_interface(chip, chipnr); if (ret) return ret; @@ -2397,18 +2733,7 @@ int nand_reset(struct nand_chip *chip, int chipnr) if (ret) return ret; - /* - * A nand_reset_data_interface() put both the NAND chip and the NAND - * controller in timings mode 0. If the default mode for this chip is - * also 0, no need to proceed to the change again. Plus, at probe time, - * nand_setup_data_interface() uses ->set/get_features() which would - * fail anyway as the parameter page is not available yet. - */ - if (!chip->onfi_timing_mode_default) - return 0; - - chip->data_interface = saved_data_intf; - ret = nand_setup_data_interface(chip, chipnr); + ret = nand_setup_interface(chip, chipnr); if (ret) return ret; @@ -2459,137 +2784,6 @@ int nand_set_features(struct nand_chip *chip, int addr, } /** - * nand_check_erased_buf - check if a buffer contains (almost) only 0xff data - * @buf: buffer to test - * @len: buffer length - * @bitflips_threshold: maximum number of bitflips - * - * Check if a buffer contains only 0xff, which means the underlying region - * has been erased and is ready to be programmed. - * The bitflips_threshold specify the maximum number of bitflips before - * considering the region is not erased. - * Note: The logic of this function has been extracted from the memweight - * implementation, except that nand_check_erased_buf function exit before - * testing the whole buffer if the number of bitflips exceed the - * bitflips_threshold value. - * - * Returns a positive number of bitflips less than or equal to - * bitflips_threshold, or -ERROR_CODE for bitflips in excess of the - * threshold. - */ -static int nand_check_erased_buf(void *buf, int len, int bitflips_threshold) -{ - const unsigned char *bitmap = buf; - int bitflips = 0; - int weight; - - for (; len && ((uintptr_t)bitmap) % sizeof(long); - len--, bitmap++) { - weight = hweight8(*bitmap); - bitflips += BITS_PER_BYTE - weight; - if (unlikely(bitflips > bitflips_threshold)) - return -EBADMSG; - } - - for (; len >= sizeof(long); - len -= sizeof(long), bitmap += sizeof(long)) { - unsigned long d = *((unsigned long *)bitmap); - if (d == ~0UL) - continue; - weight = hweight_long(d); - bitflips += BITS_PER_LONG - weight; - if (unlikely(bitflips > bitflips_threshold)) - return -EBADMSG; - } - - for (; len > 0; len--, bitmap++) { - weight = hweight8(*bitmap); - bitflips += BITS_PER_BYTE - weight; - if (unlikely(bitflips > bitflips_threshold)) - return -EBADMSG; - } - - return bitflips; -} - -/** - * nand_check_erased_ecc_chunk - check if an ECC chunk contains (almost) only - * 0xff data - * @data: data buffer to test - * @datalen: data length - * @ecc: ECC buffer - * @ecclen: ECC length - * @extraoob: extra OOB buffer - * @extraooblen: extra OOB length - * @bitflips_threshold: maximum number of bitflips - * - * Check if a data buffer and its associated ECC and OOB data contains only - * 0xff pattern, which means the underlying region has been erased and is - * ready to be programmed. - * The bitflips_threshold specify the maximum number of bitflips before - * considering the region as not erased. - * - * Note: - * 1/ ECC algorithms are working on pre-defined block sizes which are usually - * different from the NAND page size. When fixing bitflips, ECC engines will - * report the number of errors per chunk, and the NAND core infrastructure - * expect you to return the maximum number of bitflips for the whole page. - * This is why you should always use this function on a single chunk and - * not on the whole page. After checking each chunk you should update your - * max_bitflips value accordingly. - * 2/ When checking for bitflips in erased pages you should not only check - * the payload data but also their associated ECC data, because a user might - * have programmed almost all bits to 1 but a few. In this case, we - * shouldn't consider the chunk as erased, and checking ECC bytes prevent - * this case. - * 3/ The extraoob argument is optional, and should be used if some of your OOB - * data are protected by the ECC engine. - * It could also be used if you support subpages and want to attach some - * extra OOB data to an ECC chunk. - * - * Returns a positive number of bitflips less than or equal to - * bitflips_threshold, or -ERROR_CODE for bitflips in excess of the - * threshold. In case of success, the passed buffers are filled with 0xff. - */ -int nand_check_erased_ecc_chunk(void *data, int datalen, - void *ecc, int ecclen, - void *extraoob, int extraooblen, - int bitflips_threshold) -{ - int data_bitflips = 0, ecc_bitflips = 0, extraoob_bitflips = 0; - - data_bitflips = nand_check_erased_buf(data, datalen, - bitflips_threshold); - if (data_bitflips < 0) - return data_bitflips; - - bitflips_threshold -= data_bitflips; - - ecc_bitflips = nand_check_erased_buf(ecc, ecclen, bitflips_threshold); - if (ecc_bitflips < 0) - return ecc_bitflips; - - bitflips_threshold -= ecc_bitflips; - - extraoob_bitflips = nand_check_erased_buf(extraoob, extraooblen, - bitflips_threshold); - if (extraoob_bitflips < 0) - return extraoob_bitflips; - - if (data_bitflips) - memset(data, 0xff, datalen); - - if (ecc_bitflips) - memset(ecc, 0xff, ecclen); - - if (extraoob_bitflips) - memset(extraoob, 0xff, extraooblen); - - return data_bitflips + ecc_bitflips + extraoob_bitflips; -} -EXPORT_SYMBOL(nand_check_erased_ecc_chunk); - -/** * nand_read_page_raw_notsupp - dummy read raw page function * @chip: nand chip info structure * @buf: buffer to store read data @@ -2625,7 +2819,7 @@ int nand_read_page_raw(struct nand_chip *chip, uint8_t *buf, int oob_required, if (oob_required) { ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, - false); + false, false); if (ret) return ret; } @@ -2635,6 +2829,47 @@ int nand_read_page_raw(struct nand_chip *chip, uint8_t *buf, int oob_required, EXPORT_SYMBOL(nand_read_page_raw); /** + * nand_monolithic_read_page_raw - Monolithic page read in raw mode + * @chip: NAND chip info structure + * @buf: buffer to store read data + * @oob_required: caller requires OOB data read to chip->oob_poi + * @page: page number to read + * + * This is a raw page read, ie. without any error detection/correction. + * Monolithic means we are requesting all the relevant data (main plus + * eventually OOB) to be loaded in the NAND cache and sent over the + * bus (from the NAND chip to the NAND controller) in a single + * operation. This is an alternative to nand_read_page_raw(), which + * first reads the main data, and if the OOB data is requested too, + * then reads more data on the bus. + */ +int nand_monolithic_read_page_raw(struct nand_chip *chip, u8 *buf, + int oob_required, int page) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + unsigned int size = mtd->writesize; + u8 *read_buf = buf; + int ret; + + if (oob_required) { + size += mtd->oobsize; + + if (buf != chip->data_buf) + read_buf = nand_get_data_buf(chip); + } + + ret = nand_read_page_op(chip, page, 0, read_buf, size); + if (ret) + return ret; + + if (buf != chip->data_buf) + memcpy(buf, read_buf, mtd->writesize); + + return 0; +} +EXPORT_SYMBOL(nand_monolithic_read_page_raw); + +/** * nand_read_page_raw_syndrome - [INTERN] read raw page data without ecc * @chip: nand chip info structure * @buf: buffer to store read data @@ -2657,7 +2892,7 @@ static int nand_read_page_raw_syndrome(struct nand_chip *chip, uint8_t *buf, return ret; for (steps = chip->ecc.steps; steps > 0; steps--) { - ret = nand_read_data_op(chip, buf, eccsize, false); + ret = nand_read_data_op(chip, buf, eccsize, false, false); if (ret) return ret; @@ -2665,14 +2900,14 @@ static int nand_read_page_raw_syndrome(struct nand_chip *chip, uint8_t *buf, if (chip->ecc.prepad) { ret = nand_read_data_op(chip, oob, chip->ecc.prepad, - false); + false, false); if (ret) return ret; oob += chip->ecc.prepad; } - ret = nand_read_data_op(chip, oob, eccbytes, false); + ret = nand_read_data_op(chip, oob, eccbytes, false, false); if (ret) return ret; @@ -2680,7 +2915,7 @@ static int nand_read_page_raw_syndrome(struct nand_chip *chip, uint8_t *buf, if (chip->ecc.postpad) { ret = nand_read_data_op(chip, oob, chip->ecc.postpad, - false); + false, false); if (ret) return ret; @@ -2690,7 +2925,7 @@ static int nand_read_page_raw_syndrome(struct nand_chip *chip, uint8_t *buf, size = mtd->oobsize - (oob - chip->oob_poi); if (size) { - ret = nand_read_data_op(chip, oob, size, false); + ret = nand_read_data_op(chip, oob, size, false, false); if (ret) return ret; } @@ -2883,14 +3118,15 @@ static int nand_read_page_hwecc(struct nand_chip *chip, uint8_t *buf, for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) { chip->ecc.hwctl(chip, NAND_ECC_READ); - ret = nand_read_data_op(chip, p, eccsize, false); + ret = nand_read_data_op(chip, p, eccsize, false, false); if (ret) return ret; chip->ecc.calculate(chip, p, &ecc_calc[i]); } - ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false); + ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false, + false); if (ret) return ret; @@ -2926,20 +3162,18 @@ static int nand_read_page_hwecc(struct nand_chip *chip, uint8_t *buf, } /** - * nand_read_page_hwecc_oob_first - [REPLACEABLE] hw ecc, read oob first + * nand_read_page_hwecc_oob_first - Hardware ECC page read with ECC + * data read from OOB area * @chip: nand chip info structure * @buf: buffer to store read data * @oob_required: caller requires OOB data read to chip->oob_poi * @page: page number to read * - * Hardware ECC for large page chips, require OOB to be read first. For this - * ECC mode, the write_page method is re-used from ECC_HW. These methods - * read/write ECC from the OOB area, unlike the ECC_HW_SYNDROME support with - * multiple ECC steps, follows the "infix ECC" scheme and reads/writes ECC from - * the data area, by overwriting the NAND manufacturer bad block markings. + * Hardware ECC for large page chips, which requires the ECC data to be + * extracted from the OOB before the actual data is read. */ -static int nand_read_page_hwecc_oob_first(struct nand_chip *chip, uint8_t *buf, - int oob_required, int page) +int nand_read_page_hwecc_oob_first(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) { struct mtd_info *mtd = nand_to_mtd(chip); int i, eccsize = chip->ecc.size, ret; @@ -2947,7 +3181,6 @@ static int nand_read_page_hwecc_oob_first(struct nand_chip *chip, uint8_t *buf, int eccsteps = chip->ecc.steps; uint8_t *p = buf; uint8_t *ecc_code = chip->ecc.code_buf; - uint8_t *ecc_calc = chip->ecc.calc_buf; unsigned int max_bitflips = 0; /* Read the OOB area first */ @@ -2955,7 +3188,8 @@ static int nand_read_page_hwecc_oob_first(struct nand_chip *chip, uint8_t *buf, if (ret) return ret; - ret = nand_read_page_op(chip, page, 0, NULL, 0); + /* Move read cursor to start of page */ + ret = nand_change_read_column_op(chip, 0, NULL, 0, false); if (ret) return ret; @@ -2969,20 +3203,18 @@ static int nand_read_page_hwecc_oob_first(struct nand_chip *chip, uint8_t *buf, chip->ecc.hwctl(chip, NAND_ECC_READ); - ret = nand_read_data_op(chip, p, eccsize, false); + ret = nand_read_data_op(chip, p, eccsize, false, false); if (ret) return ret; - chip->ecc.calculate(chip, p, &ecc_calc[i]); - stat = chip->ecc.correct(chip, p, &ecc_code[i], NULL); if (stat == -EBADMSG && (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) { /* check for empty pages with bitflips */ stat = nand_check_erased_ecc_chunk(p, eccsize, - &ecc_code[i], eccbytes, - NULL, 0, - chip->ecc.strength); + &ecc_code[i], + eccbytes, NULL, 0, + chip->ecc.strength); } if (stat < 0) { @@ -2994,6 +3226,7 @@ static int nand_read_page_hwecc_oob_first(struct nand_chip *chip, uint8_t *buf, } return max_bitflips; } +EXPORT_SYMBOL_GPL(nand_read_page_hwecc_oob_first); /** * nand_read_page_syndrome - [REPLACEABLE] hardware ECC syndrome based page read @@ -3026,13 +3259,13 @@ static int nand_read_page_syndrome(struct nand_chip *chip, uint8_t *buf, chip->ecc.hwctl(chip, NAND_ECC_READ); - ret = nand_read_data_op(chip, p, eccsize, false); + ret = nand_read_data_op(chip, p, eccsize, false, false); if (ret) return ret; if (chip->ecc.prepad) { ret = nand_read_data_op(chip, oob, chip->ecc.prepad, - false); + false, false); if (ret) return ret; @@ -3041,7 +3274,7 @@ static int nand_read_page_syndrome(struct nand_chip *chip, uint8_t *buf, chip->ecc.hwctl(chip, NAND_ECC_READSYN); - ret = nand_read_data_op(chip, oob, eccbytes, false); + ret = nand_read_data_op(chip, oob, eccbytes, false, false); if (ret) return ret; @@ -3051,7 +3284,7 @@ static int nand_read_page_syndrome(struct nand_chip *chip, uint8_t *buf, if (chip->ecc.postpad) { ret = nand_read_data_op(chip, oob, chip->ecc.postpad, - false); + false, false); if (ret) return ret; @@ -3079,7 +3312,7 @@ static int nand_read_page_syndrome(struct nand_chip *chip, uint8_t *buf, /* Calculate remaining oob bytes */ i = mtd->oobsize - (oob - chip->oob_poi); if (i) { - ret = nand_read_data_op(chip, oob, i, false); + ret = nand_read_data_op(chip, oob, i, false, false); if (ret) return ret; } @@ -3119,6 +3352,51 @@ static uint8_t *nand_transfer_oob(struct nand_chip *chip, uint8_t *oob, return NULL; } +static void rawnand_enable_cont_reads(struct nand_chip *chip, unsigned int page, + u32 readlen, int col) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + unsigned int first_page, last_page; + + chip->cont_read.ongoing = false; + + if (!chip->controller->supported_op.cont_read) + return; + + /* + * Don't bother making any calculations if the length is too small. + * Side effect: avoids possible integer underflows below. + */ + if (readlen < (2 * mtd->writesize)) + return; + + /* Derive the page where continuous read should start (the first full page read) */ + first_page = page; + if (col) + first_page++; + + /* Derive the page where continuous read should stop (the last full page read) */ + last_page = page + ((col + readlen) / mtd->writesize) - 1; + + /* Configure and enable continuous read when suitable */ + if (first_page < last_page) { + chip->cont_read.first_page = first_page; + chip->cont_read.last_page = last_page; + chip->cont_read.ongoing = true; + /* May reset the ongoing flag */ + rawnand_cap_cont_reads(chip); + } +} + +static void rawnand_cont_read_skip_first_page(struct nand_chip *chip, unsigned int page) +{ + if (!chip->cont_read.ongoing || page != chip->cont_read.first_page) + return; + + chip->cont_read.first_page++; + rawnand_cap_cont_reads(chip); +} + /** * nand_setup_read_retry - [INTERN] Set the READ RETRY mode * @chip: NAND chip object @@ -3135,21 +3413,21 @@ static int nand_setup_read_retry(struct nand_chip *chip, int retry_mode) if (retry_mode >= chip->read_retries) return -EINVAL; - if (!chip->setup_read_retry) + if (!chip->ops.setup_read_retry) return -EOPNOTSUPP; - return chip->setup_read_retry(chip, retry_mode); + return chip->ops.setup_read_retry(chip, retry_mode); } static void nand_wait_readrdy(struct nand_chip *chip) { - const struct nand_sdr_timings *sdr; + const struct nand_interface_config *conf; if (!(chip->options & NAND_NEED_READRDY)) return; - sdr = nand_get_sdr_timings(&chip->data_interface); - WARN_ON(nand_wait_rdy_op(chip, PSEC_TO_MSEC(sdr->tR_max), 0)); + conf = nand_get_interface_config(chip); + WARN_ON(nand_wait_rdy_op(chip, NAND_COMMON_TIMING_MS(conf, tR_max), 0)); } /** @@ -3171,11 +3449,15 @@ static int nand_do_read_ops(struct nand_chip *chip, loff_t from, uint32_t max_oobsize = mtd_oobavail(mtd, ops); uint8_t *bufpoi, *oob, *buf; - int use_bufpoi; + int use_bounce_buf; unsigned int max_bitflips = 0; int retry_mode = 0; bool ecc_fail = false; + /* Check if the region is secured */ + if (nand_region_is_secured(chip, from, readlen)) + return -EIO; + chipnr = (int)(from >> chip->chip_shift); nand_select_target(chip, chipnr); @@ -3188,26 +3470,29 @@ static int nand_do_read_ops(struct nand_chip *chip, loff_t from, oob = ops->oobbuf; oob_required = oob ? 1 : 0; + if (likely(ops->mode != MTD_OPS_RAW)) + rawnand_enable_cont_reads(chip, page, readlen, col); + while (1) { - unsigned int ecc_failures = mtd->ecc_stats.failed; + struct mtd_ecc_stats ecc_stats = mtd->ecc_stats; bytes = min(mtd->writesize - col, readlen); aligned = (bytes == mtd->writesize); if (!aligned) - use_bufpoi = 1; - else if (chip->options & NAND_USE_BOUNCE_BUFFER) - use_bufpoi = !virt_addr_valid(buf) || - !IS_ALIGNED((unsigned long)buf, - chip->buf_align); + use_bounce_buf = 1; + else if (chip->options & NAND_USES_DMA) + use_bounce_buf = !virt_addr_valid(buf) || + !IS_ALIGNED((unsigned long)buf, + chip->buf_align); else - use_bufpoi = 0; + use_bounce_buf = 0; /* Is the current page in the buffer? */ - if (realpage != chip->pagebuf || oob) { - bufpoi = use_bufpoi ? chip->data_buf : buf; + if (realpage != chip->pagecache.page || oob) { + bufpoi = use_bounce_buf ? chip->data_buf : buf; - if (use_bufpoi && aligned) + if (use_bounce_buf && aligned) pr_debug("%s: using read bounce buffer for buf@%p\n", __func__, buf); @@ -3228,24 +3513,27 @@ read_retry: ret = chip->ecc.read_page(chip, bufpoi, oob_required, page); if (ret < 0) { - if (use_bufpoi) + if (use_bounce_buf) /* Invalidate page cache */ - chip->pagebuf = -1; + chip->pagecache.page = -1; break; } - /* Transfer not aligned data */ - if (use_bufpoi) { + /* + * Copy back the data in the initial buffer when reading + * partial pages or when a bounce buffer is required. + */ + if (use_bounce_buf) { if (!NAND_HAS_SUBPAGE_READ(chip) && !oob && - !(mtd->ecc_stats.failed - ecc_failures) && + !(mtd->ecc_stats.failed - ecc_stats.failed) && (ops->mode != MTD_OPS_RAW)) { - chip->pagebuf = realpage; - chip->pagebuf_bitflips = ret; + chip->pagecache.page = realpage; + chip->pagecache.bitflips = ret; } else { /* Invalidate page cache */ - chip->pagebuf = -1; + chip->pagecache.page = -1; } - memcpy(buf, chip->data_buf + col, bytes); + memcpy(buf, bufpoi + col, bytes); } if (unlikely(oob)) { @@ -3260,7 +3548,7 @@ read_retry: nand_wait_readrdy(chip); - if (mtd->ecc_stats.failed - ecc_failures) { + if (mtd->ecc_stats.failed - ecc_stats.failed) { if (retry_mode + 1 < chip->read_retries) { retry_mode++; ret = nand_setup_read_retry(chip, @@ -3268,8 +3556,8 @@ read_retry: if (ret < 0) break; - /* Reset failures; retry */ - mtd->ecc_stats.failed = ecc_failures; + /* Reset ecc_stats; retry */ + mtd->ecc_stats = ecc_stats; goto read_retry; } else { /* No more retry modes; real failure */ @@ -3283,7 +3571,9 @@ read_retry: memcpy(buf, chip->data_buf + col, bytes); buf += bytes; max_bitflips = max_t(unsigned int, max_bitflips, - chip->pagebuf_bitflips); + chip->pagecache.bitflips); + + rawnand_cont_read_skip_first_page(chip, page); } readlen -= bytes; @@ -3314,6 +3604,9 @@ read_retry: } nand_deselect_target(chip); + if (WARN_ON_ONCE(chip->cont_read.ongoing)) + chip->cont_read.ongoing = false; + ops->retlen = ops->len - (size_t) readlen; if (oob) ops->oobretlen = ops->ooblen - oobreadlen; @@ -3378,7 +3671,7 @@ static int nand_read_oob_syndrome(struct nand_chip *chip, int page) sndrnd = 1; toread = min_t(int, length, chunk); - ret = nand_read_data_op(chip, bufpoi, toread, false); + ret = nand_read_data_op(chip, bufpoi, toread, false, false); if (ret) return ret; @@ -3386,7 +3679,7 @@ static int nand_read_oob_syndrome(struct nand_chip *chip, int page) length -= toread; } if (length > 0) { - ret = nand_read_data_op(chip, bufpoi, length, false); + ret = nand_read_data_op(chip, bufpoi, length, false, false); if (ret) return ret; } @@ -3504,6 +3797,10 @@ static int nand_do_read_oob(struct nand_chip *chip, loff_t from, pr_debug("%s: from = 0x%08Lx, len = %i\n", __func__, (unsigned long long)from, readlen); + /* Check if the region is secured */ + if (nand_region_is_secured(chip, from, readlen)) + return -EIO; + stats = mtd->ecc_stats; len = mtd_oobavail(mtd, ops); @@ -3571,6 +3868,7 @@ static int nand_read_oob(struct mtd_info *mtd, loff_t from, struct mtd_oob_ops *ops) { struct nand_chip *chip = mtd_to_nand(mtd); + struct mtd_ecc_stats old_stats; int ret; ops->retlen = 0; @@ -3580,13 +3878,22 @@ static int nand_read_oob(struct mtd_info *mtd, loff_t from, ops->mode != MTD_OPS_RAW) return -ENOTSUPP; - nand_get_device(chip, FL_READING); + nand_get_device(chip); + + old_stats = mtd->ecc_stats; if (!ops->datbuf) ret = nand_do_read_oob(chip, from, ops); else ret = nand_do_read_ops(chip, from, ops); + if (ops->stats) { + ops->stats->uncorrectable_errors += + mtd->ecc_stats.failed - old_stats.failed; + ops->stats->corrected_bitflips += + mtd->ecc_stats.corrected - old_stats.corrected; + } + nand_release_device(chip); return ret; } @@ -3637,6 +3944,42 @@ int nand_write_page_raw(struct nand_chip *chip, const uint8_t *buf, EXPORT_SYMBOL(nand_write_page_raw); /** + * nand_monolithic_write_page_raw - Monolithic page write in raw mode + * @chip: NAND chip info structure + * @buf: data buffer to write + * @oob_required: must write chip->oob_poi to OOB + * @page: page number to write + * + * This is a raw page write, ie. without any error detection/correction. + * Monolithic means we are requesting all the relevant data (main plus + * eventually OOB) to be sent over the bus and effectively programmed + * into the NAND chip arrays in a single operation. This is an + * alternative to nand_write_page_raw(), which first sends the main + * data, then eventually send the OOB data by latching more data + * cycles on the NAND bus, and finally sends the program command to + * synchronyze the NAND chip cache. + */ +int nand_monolithic_write_page_raw(struct nand_chip *chip, const u8 *buf, + int oob_required, int page) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + unsigned int size = mtd->writesize; + u8 *write_buf = (u8 *)buf; + + if (oob_required) { + size += mtd->oobsize; + + if (buf != chip->data_buf) { + write_buf = nand_get_data_buf(chip); + memcpy(write_buf, buf, mtd->writesize); + } + } + + return nand_prog_page_op(chip, page, 0, write_buf, size); +} +EXPORT_SYMBOL(nand_monolithic_write_page_raw); + +/** * nand_write_page_raw_syndrome - [INTERN] raw page write function * @chip: nand chip info structure * @buf: data buffer @@ -3987,6 +4330,10 @@ static int nand_do_write_ops(struct nand_chip *chip, loff_t to, return -EINVAL; } + /* Check if the region is secured */ + if (nand_region_is_secured(chip, to, writelen)) + return -EIO; + column = to & (mtd->writesize - 1); chipnr = (int)(to >> chip->chip_shift); @@ -4002,9 +4349,9 @@ static int nand_do_write_ops(struct nand_chip *chip, loff_t to, page = realpage & chip->pagemask; /* Invalidate the page cache, when we write to the cached page */ - if (to <= ((loff_t)chip->pagebuf << chip->page_shift) && - ((loff_t)chip->pagebuf << chip->page_shift) < (to + ops->len)) - chip->pagebuf = -1; + if (to <= ((loff_t)chip->pagecache.page << chip->page_shift) && + ((loff_t)chip->pagecache.page << chip->page_shift) < (to + ops->len)) + chip->pagecache.page = -1; /* Don't allow multipage oob writes with offset */ if (oob && ops->ooboffs && (ops->ooboffs + ops->ooblen > oobmaxlen)) { @@ -4015,28 +4362,30 @@ static int nand_do_write_ops(struct nand_chip *chip, loff_t to, while (1) { int bytes = mtd->writesize; uint8_t *wbuf = buf; - int use_bufpoi; + int use_bounce_buf; int part_pagewr = (column || writelen < mtd->writesize); if (part_pagewr) - use_bufpoi = 1; - else if (chip->options & NAND_USE_BOUNCE_BUFFER) - use_bufpoi = !virt_addr_valid(buf) || - !IS_ALIGNED((unsigned long)buf, - chip->buf_align); + use_bounce_buf = 1; + else if (chip->options & NAND_USES_DMA) + use_bounce_buf = !virt_addr_valid(buf) || + !IS_ALIGNED((unsigned long)buf, + chip->buf_align); else - use_bufpoi = 0; + use_bounce_buf = 0; - /* Partial page write?, or need to use bounce buffer */ - if (use_bufpoi) { + /* + * Copy the data from the initial buffer when doing partial page + * writes or when a bounce buffer is required. + */ + if (use_bounce_buf) { pr_debug("%s: using write bounce buffer for buf@%p\n", __func__, buf); if (part_pagewr) bytes = min_t(int, bytes - column, writelen); - chip->pagebuf = -1; - memset(chip->data_buf, 0xff, mtd->writesize); - memcpy(&chip->data_buf[column], buf, bytes); - wbuf = chip->data_buf; + wbuf = nand_get_data_buf(chip); + memset(wbuf, 0xff, mtd->writesize); + memcpy(&wbuf[column], buf, bytes); } if (unlikely(oob)) { @@ -4099,9 +4448,6 @@ static int panic_nand_write(struct mtd_info *mtd, loff_t to, size_t len, struct mtd_oob_ops ops; int ret; - /* Grab the device */ - panic_nand_get_device(chip, FL_WRITING); - nand_select_target(chip, chipnr); /* Wait for the device to get ready */ @@ -4128,11 +4474,11 @@ static int nand_write_oob(struct mtd_info *mtd, loff_t to, struct mtd_oob_ops *ops) { struct nand_chip *chip = mtd_to_nand(mtd); - int ret = -ENOTSUPP; + int ret = 0; ops->retlen = 0; - nand_get_device(chip, FL_WRITING); + nand_get_device(chip); switch (ops->mode) { case MTD_OPS_PLACE_OOB: @@ -4155,23 +4501,6 @@ out: } /** - * single_erase - [GENERIC] NAND standard block erase command function - * @chip: NAND chip object - * @page: the page address of the block which will be erased - * - * Standard erase command for NAND chips. Returns NAND status. - */ -static int single_erase(struct nand_chip *chip, int page) -{ - unsigned int eraseblock; - - /* Send commands to erase a block */ - eraseblock = page >> (chip->phys_erase_shift - chip->page_shift); - - return nand_erase_op(chip, eraseblock); -} - -/** * nand_erase - [MTD Interface] erase block(s) * @mtd: MTD device structure * @instr: erase instruction @@ -4194,7 +4523,7 @@ static int nand_erase(struct mtd_info *mtd, struct erase_info *instr) int nand_erase_nand(struct nand_chip *chip, struct erase_info *instr, int allowbbt) { - int page, status, pages_per_block, ret, chipnr; + int page, pages_per_block, ret, chipnr; loff_t len; pr_debug("%s: start = 0x%012llx, len = %llu\n", @@ -4204,8 +4533,12 @@ int nand_erase_nand(struct nand_chip *chip, struct erase_info *instr, if (check_offs_len(chip, instr->addr, instr->len)) return -EINVAL; + /* Check if the region is secured */ + if (nand_region_is_secured(chip, instr->addr, instr->len)) + return -EIO; + /* Grab the lock and see if the device is available */ - nand_get_device(chip, FL_ERASING); + nand_get_device(chip); /* Shift to get first page */ page = (int)(instr->addr >> chip->page_shift); @@ -4229,11 +4562,13 @@ int nand_erase_nand(struct nand_chip *chip, struct erase_info *instr, len = instr->len; while (len) { + loff_t ofs = (loff_t)page << chip->page_shift; + /* Check if we have a bad block, we do not erase bad blocks! */ if (nand_block_checkbad(chip, ((loff_t) page) << chip->page_shift, allowbbt)) { - pr_warn("%s: attempt to erase a bad block at page 0x%08x\n", - __func__, page); + pr_warn("%s: attempt to erase a bad block at 0x%08llx\n", + __func__, (unsigned long long)ofs); ret = -EIO; goto erase_exit; } @@ -4242,23 +4577,16 @@ int nand_erase_nand(struct nand_chip *chip, struct erase_info *instr, * Invalidate the page cache, if we erase the block which * contains the current cached page. */ - if (page <= chip->pagebuf && chip->pagebuf < + if (page <= chip->pagecache.page && chip->pagecache.page < (page + pages_per_block)) - chip->pagebuf = -1; - - if (chip->legacy.erase) - status = chip->legacy.erase(chip, - page & chip->pagemask); - else - status = single_erase(chip, page & chip->pagemask); + chip->pagecache.page = -1; - /* See if block erase succeeded */ - if (status) { + ret = nand_erase_op(chip, (page & chip->pagemask) >> + (chip->phys_erase_shift - chip->page_shift)); + if (ret) { pr_debug("%s: failed erase, page 0x%08x\n", __func__, page); - ret = -EIO; - instr->fail_addr = - ((loff_t)page << chip->page_shift); + instr->fail_addr = ofs; goto erase_exit; } @@ -4298,7 +4626,7 @@ static void nand_sync(struct mtd_info *mtd) pr_debug("%s: called\n", __func__); /* Grab the lock and see if the device is available */ - nand_get_device(chip, FL_SYNCING); + nand_get_device(chip); /* Release it and go back */ nand_release_device(chip); } @@ -4315,7 +4643,8 @@ static int nand_block_isbad(struct mtd_info *mtd, loff_t offs) int ret; /* Select the NAND device */ - nand_get_device(chip, FL_READING); + nand_get_device(chip); + nand_select_target(chip, chipnr); ret = nand_block_checkbad(chip, offs, 0); @@ -4347,73 +4676,88 @@ static int nand_block_markbad(struct mtd_info *mtd, loff_t ofs) } /** - * nand_max_bad_blocks - [MTD Interface] Max number of bad blocks for an mtd + * nand_suspend - [MTD Interface] Suspend the NAND flash * @mtd: MTD device structure - * @ofs: offset relative to mtd start - * @len: length of mtd + * + * Returns 0 for success or negative error code otherwise. */ -static int nand_max_bad_blocks(struct mtd_info *mtd, loff_t ofs, size_t len) +static int nand_suspend(struct mtd_info *mtd) { struct nand_chip *chip = mtd_to_nand(mtd); - u32 part_start_block; - u32 part_end_block; - u32 part_start_die; - u32 part_end_die; + int ret = 0; - /* - * max_bb_per_die and blocks_per_die used to determine - * the maximum bad block count. - */ - if (!chip->max_bb_per_die || !chip->blocks_per_die) - return -ENOTSUPP; + mutex_lock(&chip->lock); + if (chip->ops.suspend) + ret = chip->ops.suspend(chip); + if (!ret) + chip->suspended = 1; + mutex_unlock(&chip->lock); + + return ret; +} - /* Get the start and end of the partition in erase blocks. */ - part_start_block = mtd_div_by_eb(ofs, mtd); - part_end_block = mtd_div_by_eb(len, mtd) + part_start_block - 1; +/** + * nand_resume - [MTD Interface] Resume the NAND flash + * @mtd: MTD device structure + */ +static void nand_resume(struct mtd_info *mtd) +{ + struct nand_chip *chip = mtd_to_nand(mtd); - /* Get the start and end LUNs of the partition. */ - part_start_die = part_start_block / chip->blocks_per_die; - part_end_die = part_end_block / chip->blocks_per_die; + mutex_lock(&chip->lock); + if (chip->suspended) { + if (chip->ops.resume) + chip->ops.resume(chip); + chip->suspended = 0; + } else { + pr_err("%s called for a chip which is not in suspended state\n", + __func__); + } + mutex_unlock(&chip->lock); - /* - * Look up the bad blocks per unit and multiply by the number of units - * that the partition spans. - */ - return chip->max_bb_per_die * (part_end_die - part_start_die + 1); + wake_up_all(&chip->resume_wq); } /** - * nand_suspend - [MTD Interface] Suspend the NAND flash + * nand_shutdown - [MTD Interface] Finish the current NAND operation and + * prevent further operations * @mtd: MTD device structure */ -static int nand_suspend(struct mtd_info *mtd) +static void nand_shutdown(struct mtd_info *mtd) { - return nand_get_device(mtd_to_nand(mtd), FL_PM_SUSPENDED); + nand_suspend(mtd); } /** - * nand_resume - [MTD Interface] Resume the NAND flash + * nand_lock - [MTD Interface] Lock the NAND flash * @mtd: MTD device structure + * @ofs: offset byte address + * @len: number of bytes to lock (must be a multiple of block/page size) */ -static void nand_resume(struct mtd_info *mtd) +static int nand_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len) { struct nand_chip *chip = mtd_to_nand(mtd); - if (chip->state == FL_PM_SUSPENDED) - nand_release_device(chip); - else - pr_err("%s called for a chip which is not in suspended state\n", - __func__); + if (!chip->ops.lock_area) + return -ENOTSUPP; + + return chip->ops.lock_area(chip, ofs, len); } /** - * nand_shutdown - [MTD Interface] Finish the current NAND operation and - * prevent further operations + * nand_unlock - [MTD Interface] Unlock the NAND flash * @mtd: MTD device structure + * @ofs: offset byte address + * @len: number of bytes to unlock (must be a multiple of block/page size) */ -static void nand_shutdown(struct mtd_info *mtd) +static int nand_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len) { - nand_get_device(mtd_to_nand(mtd), FL_PM_SUSPENDED); + struct nand_chip *chip = mtd_to_nand(mtd); + + if (!chip->ops.unlock_area) + return -ENOTSUPP; + + return chip->ops.unlock_area(chip, ofs, len); } /* Set default functions */ @@ -4525,21 +4869,29 @@ static int nand_get_bits_per_cell(u8 cellinfo) */ void nand_decode_ext_id(struct nand_chip *chip) { + struct nand_memory_organization *memorg; struct mtd_info *mtd = nand_to_mtd(chip); int extid; u8 *id_data = chip->id.data; + + memorg = nanddev_get_memorg(&chip->base); + /* The 3rd id byte holds MLC / multichip data */ - chip->bits_per_cell = nand_get_bits_per_cell(id_data[2]); + memorg->bits_per_cell = nand_get_bits_per_cell(id_data[2]); /* The 4th id byte is the important one */ extid = id_data[3]; /* Calc pagesize */ - mtd->writesize = 1024 << (extid & 0x03); + memorg->pagesize = 1024 << (extid & 0x03); + mtd->writesize = memorg->pagesize; extid >>= 2; /* Calc oobsize */ - mtd->oobsize = (8 << (extid & 0x01)) * (mtd->writesize >> 9); + memorg->oobsize = (8 << (extid & 0x01)) * (mtd->writesize >> 9); + mtd->oobsize = memorg->oobsize; extid >>= 2; /* Calc blocksize. Blocksize is multiples of 64KiB */ + memorg->pages_per_eraseblock = ((64 * 1024) << (extid & 0x03)) / + memorg->pagesize; mtd->erasesize = (64 * 1024) << (extid & 0x03); extid >>= 2; /* Get buswidth information */ @@ -4556,13 +4908,19 @@ EXPORT_SYMBOL_GPL(nand_decode_ext_id); static void nand_decode_id(struct nand_chip *chip, struct nand_flash_dev *type) { struct mtd_info *mtd = nand_to_mtd(chip); + struct nand_memory_organization *memorg; + memorg = nanddev_get_memorg(&chip->base); + + memorg->pages_per_eraseblock = type->erasesize / type->pagesize; mtd->erasesize = type->erasesize; - mtd->writesize = type->pagesize; - mtd->oobsize = mtd->writesize / 32; + memorg->pagesize = type->pagesize; + mtd->writesize = memorg->pagesize; + memorg->oobsize = memorg->pagesize / 32; + mtd->oobsize = memorg->oobsize; /* All legacy ID NAND are small-page, SLC */ - chip->bits_per_cell = 1; + memorg->bits_per_cell = 1; } /* @@ -4576,9 +4934,9 @@ static void nand_decode_bbm_options(struct nand_chip *chip) /* Set the bad block position */ if (mtd->writesize > 512 || (chip->options & NAND_BUSWIDTH_16)) - chip->badblockpos = NAND_LARGE_BADBLOCK_POS; + chip->badblockpos = NAND_BBM_POS_LARGE; else - chip->badblockpos = NAND_SMALL_BADBLOCK_POS; + chip->badblockpos = NAND_BBM_POS_SMALL; } static inline bool is_full_id_nand(struct nand_flash_dev *type) @@ -4589,21 +4947,32 @@ static inline bool is_full_id_nand(struct nand_flash_dev *type) static bool find_full_id_nand(struct nand_chip *chip, struct nand_flash_dev *type) { + struct nand_device *base = &chip->base; + struct nand_ecc_props requirements; struct mtd_info *mtd = nand_to_mtd(chip); + struct nand_memory_organization *memorg; u8 *id_data = chip->id.data; + memorg = nanddev_get_memorg(&chip->base); + if (!strncmp(type->id, id_data, type->id_len)) { - mtd->writesize = type->pagesize; + memorg->pagesize = type->pagesize; + mtd->writesize = memorg->pagesize; + memorg->pages_per_eraseblock = type->erasesize / + type->pagesize; mtd->erasesize = type->erasesize; - mtd->oobsize = type->oobsize; - - chip->bits_per_cell = nand_get_bits_per_cell(id_data[2]); - chip->chipsize = (uint64_t)type->chipsize << 20; + memorg->oobsize = type->oobsize; + mtd->oobsize = memorg->oobsize; + + memorg->bits_per_cell = nand_get_bits_per_cell(id_data[2]); + memorg->eraseblocks_per_lun = + DIV_ROUND_DOWN_ULL((u64)type->chipsize << 20, + memorg->pagesize * + memorg->pages_per_eraseblock); chip->options |= type->options; - chip->ecc_strength_ds = NAND_ECC_STRENGTH(type); - chip->ecc_step_ds = NAND_ECC_STEP(type); - chip->onfi_timing_mode_default = - type->onfi_timing_mode_default; + requirements.strength = NAND_ECC_STRENGTH(type); + requirements.step_size = NAND_ECC_STEP(type); + nanddev_set_ecc_requirements(base, &requirements); chip->parameters.model = kstrdup(type->name, GFP_KERNEL); if (!chip->parameters.model) @@ -4627,8 +4996,12 @@ static void nand_manufacturer_detect(struct nand_chip *chip) */ if (chip->manufacturer.desc && chip->manufacturer.desc->ops && chip->manufacturer.desc->ops->detect) { + struct nand_memory_organization *memorg; + + memorg = nanddev_get_memorg(&chip->base); + /* The 3rd id byte holds MLC / multichip data */ - chip->bits_per_cell = nand_get_bits_per_cell(chip->id.data[2]); + memorg->bits_per_cell = nand_get_bits_per_cell(chip->id.data[2]); chip->manufacturer.desc->ops->detect(chip); } else { nand_decode_ext_id(chip); @@ -4665,9 +5038,70 @@ static void nand_manufacturer_cleanup(struct nand_chip *chip) } static const char * -nand_manufacturer_name(const struct nand_manufacturer *manufacturer) +nand_manufacturer_name(const struct nand_manufacturer_desc *manufacturer_desc) { - return manufacturer ? manufacturer->name : "Unknown"; + return manufacturer_desc ? manufacturer_desc->name : "Unknown"; +} + +static void rawnand_check_data_only_read_support(struct nand_chip *chip) +{ + /* Use an arbitrary size for the check */ + if (!nand_read_data_op(chip, NULL, SZ_512, true, true)) + chip->controller->supported_op.data_only_read = 1; +} + +static void rawnand_early_check_supported_ops(struct nand_chip *chip) +{ + /* The supported_op fields should not be set by individual drivers */ + WARN_ON_ONCE(chip->controller->supported_op.data_only_read); + + if (!nand_has_exec_op(chip)) + return; + + rawnand_check_data_only_read_support(chip); +} + +static void rawnand_check_cont_read_support(struct nand_chip *chip) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + + if (!chip->parameters.supports_read_cache) + return; + + if (chip->read_retries) + return; + + if (!nand_lp_exec_cont_read_page_op(chip, 0, 0, NULL, + mtd->writesize, true)) + chip->controller->supported_op.cont_read = 1; +} + +static void rawnand_late_check_supported_ops(struct nand_chip *chip) +{ + /* The supported_op fields should not be set by individual drivers */ + WARN_ON_ONCE(chip->controller->supported_op.cont_read); + + /* + * Too many devices do not support sequential cached reads with on-die + * ECC correction enabled, so in this case refuse to perform the + * automation. + */ + if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE) + return; + + if (!nand_has_exec_op(chip)) + return; + + /* + * For now, continuous reads can only be used with the core page helpers. + * This can be extended later. + */ + if (!(chip->ecc.read_page == nand_read_page_hwecc || + chip->ecc.read_page == nand_read_page_syndrome || + chip->ecc.read_page == nand_read_page_swecc)) + return; + + rawnand_check_cont_read_support(chip); } /* @@ -4675,11 +5109,21 @@ nand_manufacturer_name(const struct nand_manufacturer *manufacturer) */ static int nand_detect(struct nand_chip *chip, struct nand_flash_dev *type) { - const struct nand_manufacturer *manufacturer; + const struct nand_manufacturer_desc *manufacturer_desc; struct mtd_info *mtd = nand_to_mtd(chip); + struct nand_memory_organization *memorg; int busw, ret; u8 *id_data = chip->id.data; u8 maf_id, dev_id; + u64 targetsize; + + /* + * Let's start by initializing memorg fields that might be left + * unassigned by the ID-based detection logic. + */ + memorg = nanddev_get_memorg(&chip->base); + memorg->planes_per_lun = 1; + memorg->luns_per_target = 1; /* * Reset the chip, required by some chips (e.g. Micron MT29FxGxxxxx) @@ -4692,6 +5136,8 @@ static int nand_detect(struct nand_chip *chip, struct nand_flash_dev *type) /* Select the device */ nand_select_target(chip, 0); + rawnand_early_check_supported_ops(chip); + /* Send the command for reading device ID */ ret = nand_readid_op(chip, 0, id_data, 2); if (ret) @@ -4722,8 +5168,8 @@ static int nand_detect(struct nand_chip *chip, struct nand_flash_dev *type) chip->id.len = nand_id_len(id_data, ARRAY_SIZE(chip->id.data)); /* Try to identify manufacturer */ - manufacturer = nand_get_manufacturer(maf_id); - chip->manufacturer.desc = manufacturer; + manufacturer_desc = nand_get_manufacturer_desc(maf_id); + chip->manufacturer.desc = manufacturer_desc; if (!type) type = nand_flash_ids; @@ -4775,8 +5221,6 @@ static int nand_detect(struct nand_chip *chip, struct nand_flash_dev *type) if (!chip->parameters.model) return -ENOMEM; - chip->chipsize = (uint64_t)type->chipsize << 20; - if (!type->pagesize) nand_manufacturer_detect(chip); else @@ -4785,6 +5229,11 @@ static int nand_detect(struct nand_chip *chip, struct nand_flash_dev *type) /* Get chip options */ chip->options |= type->options; + memorg->eraseblocks_per_lun = + DIV_ROUND_DOWN_ULL((u64)type->chipsize << 20, + memorg->pagesize * + memorg->pages_per_eraseblock); + ident_done: if (!mtd->name) mtd->name = chip->parameters.model; @@ -4799,7 +5248,7 @@ ident_done: */ pr_info("device found, Manufacturer ID: 0x%02x, Chip ID: 0x%02x\n", maf_id, dev_id); - pr_info("%s %s\n", nand_manufacturer_name(manufacturer), + pr_info("%s %s\n", nand_manufacturer_name(manufacturer_desc), mtd->name); pr_warn("bus width %d instead of %d bits\n", busw ? 16 : 8, (chip->options & NAND_BUSWIDTH_16) ? 16 : 8); @@ -4813,14 +5262,15 @@ ident_done: /* Calculate the address shift from the page size */ chip->page_shift = ffs(mtd->writesize) - 1; /* Convert chipsize to number of pages per chip -1 */ - chip->pagemask = (chip->chipsize >> chip->page_shift) - 1; + targetsize = nanddev_target_size(&chip->base); + chip->pagemask = (targetsize >> chip->page_shift) - 1; chip->bbt_erase_shift = chip->phys_erase_shift = ffs(mtd->erasesize) - 1; - if (chip->chipsize & 0xffffffff) - chip->chip_shift = ffs((unsigned)chip->chipsize) - 1; + if (targetsize & 0xffffffff) + chip->chip_shift = ffs((unsigned)targetsize) - 1; else { - chip->chip_shift = ffs((unsigned)(chip->chipsize >> 32)); + chip->chip_shift = ffs((unsigned)(targetsize >> 32)); chip->chip_shift += 32 - 1; } @@ -4833,10 +5283,10 @@ ident_done: pr_info("device found, Manufacturer ID: 0x%02x, Chip ID: 0x%02x\n", maf_id, dev_id); - pr_info("%s %s\n", nand_manufacturer_name(manufacturer), + pr_info("%s %s\n", nand_manufacturer_name(manufacturer_desc), chip->parameters.model); pr_info("%d MiB, %s, erase size: %d KiB, page size: %d, OOB size: %d\n", - (int)(chip->chipsize >> 20), nand_is_slc(chip) ? "SLC" : "MLC", + (int)(targetsize >> 20), nand_is_slc(chip) ? "SLC" : "MLC", mtd->erasesize >> 10, mtd->writesize, mtd->oobsize); return 0; @@ -4846,149 +5296,233 @@ free_detect_allocation: return ret; } -static const char * const nand_ecc_modes[] = { - [NAND_ECC_NONE] = "none", - [NAND_ECC_SOFT] = "soft", - [NAND_ECC_HW] = "hw", - [NAND_ECC_HW_SYNDROME] = "hw_syndrome", - [NAND_ECC_HW_OOB_FIRST] = "hw_oob_first", - [NAND_ECC_ON_DIE] = "on-die", -}; - -static int of_get_nand_ecc_mode(struct device_node *np) +static enum nand_ecc_engine_type +of_get_rawnand_ecc_engine_type_legacy(struct device_node *np) { + enum nand_ecc_legacy_mode { + NAND_ECC_INVALID, + NAND_ECC_NONE, + NAND_ECC_SOFT, + NAND_ECC_SOFT_BCH, + NAND_ECC_HW, + NAND_ECC_HW_SYNDROME, + NAND_ECC_ON_DIE, + }; + const char * const nand_ecc_legacy_modes[] = { + [NAND_ECC_NONE] = "none", + [NAND_ECC_SOFT] = "soft", + [NAND_ECC_SOFT_BCH] = "soft_bch", + [NAND_ECC_HW] = "hw", + [NAND_ECC_HW_SYNDROME] = "hw_syndrome", + [NAND_ECC_ON_DIE] = "on-die", + }; + enum nand_ecc_legacy_mode eng_type; const char *pm; - int err, i; + int err; err = of_property_read_string(np, "nand-ecc-mode", &pm); - if (err < 0) - return err; - - for (i = 0; i < ARRAY_SIZE(nand_ecc_modes); i++) - if (!strcasecmp(pm, nand_ecc_modes[i])) - return i; - - /* - * For backward compatibility we support few obsoleted values that don't - * have their mappings into nand_ecc_modes_t anymore (they were merged - * with other enums). - */ - if (!strcasecmp(pm, "soft_bch")) - return NAND_ECC_SOFT; + if (err) + return NAND_ECC_ENGINE_TYPE_INVALID; + + for (eng_type = NAND_ECC_NONE; + eng_type < ARRAY_SIZE(nand_ecc_legacy_modes); eng_type++) { + if (!strcasecmp(pm, nand_ecc_legacy_modes[eng_type])) { + switch (eng_type) { + case NAND_ECC_NONE: + return NAND_ECC_ENGINE_TYPE_NONE; + case NAND_ECC_SOFT: + case NAND_ECC_SOFT_BCH: + return NAND_ECC_ENGINE_TYPE_SOFT; + case NAND_ECC_HW: + case NAND_ECC_HW_SYNDROME: + return NAND_ECC_ENGINE_TYPE_ON_HOST; + case NAND_ECC_ON_DIE: + return NAND_ECC_ENGINE_TYPE_ON_DIE; + default: + break; + } + } + } - return -ENODEV; + return NAND_ECC_ENGINE_TYPE_INVALID; } -static const char * const nand_ecc_algos[] = { - [NAND_ECC_HAMMING] = "hamming", - [NAND_ECC_BCH] = "bch", - [NAND_ECC_RS] = "rs", -}; - -static int of_get_nand_ecc_algo(struct device_node *np) +static enum nand_ecc_placement +of_get_rawnand_ecc_placement_legacy(struct device_node *np) { const char *pm; - int err, i; + int err; - err = of_property_read_string(np, "nand-ecc-algo", &pm); + err = of_property_read_string(np, "nand-ecc-mode", &pm); if (!err) { - for (i = NAND_ECC_HAMMING; i < ARRAY_SIZE(nand_ecc_algos); i++) - if (!strcasecmp(pm, nand_ecc_algos[i])) - return i; - return -ENODEV; + if (!strcasecmp(pm, "hw_syndrome")) + return NAND_ECC_PLACEMENT_INTERLEAVED; } - /* - * For backward compatibility we also read "nand-ecc-mode" checking - * for some obsoleted values that were specifying ECC algorithm. - */ - err = of_property_read_string(np, "nand-ecc-mode", &pm); - if (err < 0) - return err; + return NAND_ECC_PLACEMENT_UNKNOWN; +} - if (!strcasecmp(pm, "soft")) - return NAND_ECC_HAMMING; - else if (!strcasecmp(pm, "soft_bch")) - return NAND_ECC_BCH; +static enum nand_ecc_algo of_get_rawnand_ecc_algo_legacy(struct device_node *np) +{ + const char *pm; + int err; + + err = of_property_read_string(np, "nand-ecc-mode", &pm); + if (!err) { + if (!strcasecmp(pm, "soft")) + return NAND_ECC_ALGO_HAMMING; + else if (!strcasecmp(pm, "soft_bch")) + return NAND_ECC_ALGO_BCH; + } - return -ENODEV; + return NAND_ECC_ALGO_UNKNOWN; } -static int of_get_nand_ecc_step_size(struct device_node *np) +static void of_get_nand_ecc_legacy_user_config(struct nand_chip *chip) { - int ret; - u32 val; + struct device_node *dn = nand_get_flash_node(chip); + struct nand_ecc_props *user_conf = &chip->base.ecc.user_conf; + + if (user_conf->engine_type == NAND_ECC_ENGINE_TYPE_INVALID) + user_conf->engine_type = of_get_rawnand_ecc_engine_type_legacy(dn); - ret = of_property_read_u32(np, "nand-ecc-step-size", &val); - return ret ? ret : val; + if (user_conf->algo == NAND_ECC_ALGO_UNKNOWN) + user_conf->algo = of_get_rawnand_ecc_algo_legacy(dn); + + if (user_conf->placement == NAND_ECC_PLACEMENT_UNKNOWN) + user_conf->placement = of_get_rawnand_ecc_placement_legacy(dn); } -static int of_get_nand_ecc_strength(struct device_node *np) +static int of_get_nand_bus_width(struct nand_chip *chip) { - int ret; + struct device_node *dn = nand_get_flash_node(chip); u32 val; + int ret; - ret = of_property_read_u32(np, "nand-ecc-strength", &val); - return ret ? ret : val; + ret = of_property_read_u32(dn, "nand-bus-width", &val); + if (ret == -EINVAL) + /* Buswidth defaults to 8 if the property does not exist .*/ + return 0; + else if (ret) + return ret; + + if (val == 16) + chip->options |= NAND_BUSWIDTH_16; + else if (val != 8) + return -EINVAL; + return 0; } -static int of_get_nand_bus_width(struct device_node *np) +static int of_get_nand_secure_regions(struct nand_chip *chip) { - u32 val; + struct device_node *dn = nand_get_flash_node(chip); + struct property *prop; + int nr_elem, i, j; + + /* Only proceed if the "secure-regions" property is present in DT */ + prop = of_find_property(dn, "secure-regions", NULL); + if (!prop) + return 0; - if (of_property_read_u32(np, "nand-bus-width", &val)) - return 8; + nr_elem = of_property_count_elems_of_size(dn, "secure-regions", sizeof(u64)); + if (nr_elem <= 0) + return nr_elem; - switch (val) { - case 8: - case 16: - return val; - default: - return -EIO; + chip->nr_secure_regions = nr_elem / 2; + chip->secure_regions = kcalloc(chip->nr_secure_regions, sizeof(*chip->secure_regions), + GFP_KERNEL); + if (!chip->secure_regions) + return -ENOMEM; + + for (i = 0, j = 0; i < chip->nr_secure_regions; i++, j += 2) { + of_property_read_u64_index(dn, "secure-regions", j, + &chip->secure_regions[i].offset); + of_property_read_u64_index(dn, "secure-regions", j + 1, + &chip->secure_regions[i].size); } + + return 0; } -static bool of_get_nand_on_flash_bbt(struct device_node *np) +/** + * rawnand_dt_parse_gpio_cs - Parse the gpio-cs property of a controller + * @dev: Device that will be parsed. Also used for managed allocations. + * @cs_array: Array of GPIO desc pointers allocated on success + * @ncs_array: Number of entries in @cs_array updated on success. + * @return 0 on success, an error otherwise. + */ +int rawnand_dt_parse_gpio_cs(struct device *dev, struct gpio_desc ***cs_array, + unsigned int *ncs_array) { - return of_property_read_bool(np, "nand-on-flash-bbt"); + struct gpio_desc **descs; + int ndescs, i; + + ndescs = gpiod_count(dev, "cs"); + if (ndescs < 0) { + dev_dbg(dev, "No valid cs-gpios property\n"); + return 0; + } + + descs = devm_kcalloc(dev, ndescs, sizeof(*descs), GFP_KERNEL); + if (!descs) + return -ENOMEM; + + for (i = 0; i < ndescs; i++) { + descs[i] = gpiod_get_index_optional(dev, "cs", i, + GPIOD_OUT_HIGH); + if (IS_ERR(descs[i])) + return PTR_ERR(descs[i]); + } + + *ncs_array = ndescs; + *cs_array = descs; + + return 0; } +EXPORT_SYMBOL(rawnand_dt_parse_gpio_cs); -static int nand_dt_init(struct nand_chip *chip) +static int rawnand_dt_init(struct nand_chip *chip) { + struct nand_device *nand = mtd_to_nanddev(nand_to_mtd(chip)); struct device_node *dn = nand_get_flash_node(chip); - int ecc_mode, ecc_algo, ecc_strength, ecc_step; + int ret; if (!dn) return 0; - if (of_get_nand_bus_width(dn) == 16) - chip->options |= NAND_BUSWIDTH_16; + ret = of_get_nand_bus_width(chip); + if (ret) + return ret; if (of_property_read_bool(dn, "nand-is-boot-medium")) chip->options |= NAND_IS_BOOT_MEDIUM; - if (of_get_nand_on_flash_bbt(dn)) + if (of_property_read_bool(dn, "nand-on-flash-bbt")) chip->bbt_options |= NAND_BBT_USE_FLASH; - ecc_mode = of_get_nand_ecc_mode(dn); - ecc_algo = of_get_nand_ecc_algo(dn); - ecc_strength = of_get_nand_ecc_strength(dn); - ecc_step = of_get_nand_ecc_step_size(dn); + of_get_nand_ecc_user_config(nand); + of_get_nand_ecc_legacy_user_config(chip); - if (ecc_mode >= 0) - chip->ecc.mode = ecc_mode; - - if (ecc_algo >= 0) - chip->ecc.algo = ecc_algo; - - if (ecc_strength >= 0) - chip->ecc.strength = ecc_strength; + /* + * If neither the user nor the NAND controller have requested a specific + * ECC engine type, we will default to NAND_ECC_ENGINE_TYPE_ON_HOST. + */ + nand->ecc.defaults.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; - if (ecc_step > 0) - chip->ecc.size = ecc_step; + /* + * Use the user requested engine type, unless there is none, in this + * case default to the NAND controller choice, otherwise fallback to + * the raw NAND default one. + */ + if (nand->ecc.user_conf.engine_type != NAND_ECC_ENGINE_TYPE_INVALID) + chip->ecc.engine_type = nand->ecc.user_conf.engine_type; + if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_INVALID) + chip->ecc.engine_type = nand->ecc.defaults.engine_type; - if (of_property_read_bool(dn, "nand-ecc-maximize")) - chip->ecc.options |= NAND_ECC_MAXIMIZE; + chip->ecc.placement = nand->ecc.user_conf.placement; + chip->ecc.algo = nand->ecc.user_conf.algo; + chip->ecc.strength = nand->ecc.user_conf.strength; + chip->ecc.size = nand->ecc.user_conf.step_size; return 0; } @@ -5011,29 +5545,29 @@ static int nand_scan_ident(struct nand_chip *chip, unsigned int maxchips, struct nand_flash_dev *table) { struct mtd_info *mtd = nand_to_mtd(chip); + struct nand_memory_organization *memorg; int nand_maf_id, nand_dev_id; unsigned int i; int ret; + memorg = nanddev_get_memorg(&chip->base); + /* Assume all dies are deselected when we enter nand_scan_ident(). */ chip->cur_cs = -1; + mutex_init(&chip->lock); + init_waitqueue_head(&chip->resume_wq); + /* Enforce the right timings for reset/detection */ - onfi_fill_data_interface(chip, NAND_SDR_IFACE, 0); + chip->current_interface_config = nand_get_reset_interface_config(); - ret = nand_dt_init(chip); + ret = rawnand_dt_init(chip); if (ret) return ret; if (!mtd->name && mtd->dev.parent) mtd->name = dev_name(mtd->dev.parent); - /* - * Start with chips->numchips = maxchips to let nand_select_target() do - * its job. chip->numchips will be adjusted after. - */ - chip->numchips = maxchips; - /* Set the default functions */ nand_set_defaults(chip); @@ -5041,6 +5575,8 @@ static int nand_scan_ident(struct nand_chip *chip, unsigned int maxchips, if (ret) return ret; + memorg->ntargets = maxchips; + /* Read the flash type */ ret = nand_detect(chip, table); if (ret) { @@ -5060,11 +5596,15 @@ static int nand_scan_ident(struct nand_chip *chip, unsigned int maxchips, u8 id[2]; /* See comment in nand_get_flash_type for reset */ - nand_reset(chip, i); + ret = nand_reset(chip, i); + if (ret) + break; nand_select_target(chip, i); /* Send the command for reading device ID */ - nand_readid_op(chip, 0, id, sizeof(id)); + ret = nand_readid_op(chip, 0, id, sizeof(id)); + if (ret) + break; /* Read manufacturer and device IDs */ if (nand_maf_id != id[0] || nand_dev_id != id[1]) { nand_deselect_target(chip); @@ -5076,8 +5616,8 @@ static int nand_scan_ident(struct nand_chip *chip, unsigned int maxchips, pr_info("%d chips detected\n", i); /* Store the number of chips and calc total size for mtd */ - chip->numchips = i; - mtd->size = i * chip->chipsize; + memorg->ntargets = i; + mtd->size = i * nanddev_target_size(&chip->base); return 0; } @@ -5088,23 +5628,196 @@ static void nand_scan_ident_cleanup(struct nand_chip *chip) kfree(chip->parameters.onfi); } +int rawnand_sw_hamming_init(struct nand_chip *chip) +{ + struct nand_ecc_sw_hamming_conf *engine_conf; + struct nand_device *base = &chip->base; + int ret; + + base->ecc.user_conf.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + base->ecc.user_conf.algo = NAND_ECC_ALGO_HAMMING; + base->ecc.user_conf.strength = chip->ecc.strength; + base->ecc.user_conf.step_size = chip->ecc.size; + + ret = nand_ecc_sw_hamming_init_ctx(base); + if (ret) + return ret; + + engine_conf = base->ecc.ctx.priv; + + if (chip->ecc.options & NAND_ECC_SOFT_HAMMING_SM_ORDER) + engine_conf->sm_order = true; + + chip->ecc.size = base->ecc.ctx.conf.step_size; + chip->ecc.strength = base->ecc.ctx.conf.strength; + chip->ecc.total = base->ecc.ctx.total; + chip->ecc.steps = nanddev_get_ecc_nsteps(base); + chip->ecc.bytes = base->ecc.ctx.total / nanddev_get_ecc_nsteps(base); + + return 0; +} +EXPORT_SYMBOL(rawnand_sw_hamming_init); + +int rawnand_sw_hamming_calculate(struct nand_chip *chip, + const unsigned char *buf, + unsigned char *code) +{ + struct nand_device *base = &chip->base; + + return nand_ecc_sw_hamming_calculate(base, buf, code); +} +EXPORT_SYMBOL(rawnand_sw_hamming_calculate); + +int rawnand_sw_hamming_correct(struct nand_chip *chip, + unsigned char *buf, + unsigned char *read_ecc, + unsigned char *calc_ecc) +{ + struct nand_device *base = &chip->base; + + return nand_ecc_sw_hamming_correct(base, buf, read_ecc, calc_ecc); +} +EXPORT_SYMBOL(rawnand_sw_hamming_correct); + +void rawnand_sw_hamming_cleanup(struct nand_chip *chip) +{ + struct nand_device *base = &chip->base; + + nand_ecc_sw_hamming_cleanup_ctx(base); +} +EXPORT_SYMBOL(rawnand_sw_hamming_cleanup); + +int rawnand_sw_bch_init(struct nand_chip *chip) +{ + struct nand_device *base = &chip->base; + const struct nand_ecc_props *ecc_conf = nanddev_get_ecc_conf(base); + int ret; + + base->ecc.user_conf.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + base->ecc.user_conf.algo = NAND_ECC_ALGO_BCH; + base->ecc.user_conf.step_size = chip->ecc.size; + base->ecc.user_conf.strength = chip->ecc.strength; + + ret = nand_ecc_sw_bch_init_ctx(base); + if (ret) + return ret; + + chip->ecc.size = ecc_conf->step_size; + chip->ecc.strength = ecc_conf->strength; + chip->ecc.total = base->ecc.ctx.total; + chip->ecc.steps = nanddev_get_ecc_nsteps(base); + chip->ecc.bytes = base->ecc.ctx.total / nanddev_get_ecc_nsteps(base); + + return 0; +} +EXPORT_SYMBOL(rawnand_sw_bch_init); + +static int rawnand_sw_bch_calculate(struct nand_chip *chip, + const unsigned char *buf, + unsigned char *code) +{ + struct nand_device *base = &chip->base; + + return nand_ecc_sw_bch_calculate(base, buf, code); +} + +int rawnand_sw_bch_correct(struct nand_chip *chip, unsigned char *buf, + unsigned char *read_ecc, unsigned char *calc_ecc) +{ + struct nand_device *base = &chip->base; + + return nand_ecc_sw_bch_correct(base, buf, read_ecc, calc_ecc); +} +EXPORT_SYMBOL(rawnand_sw_bch_correct); + +void rawnand_sw_bch_cleanup(struct nand_chip *chip) +{ + struct nand_device *base = &chip->base; + + nand_ecc_sw_bch_cleanup_ctx(base); +} +EXPORT_SYMBOL(rawnand_sw_bch_cleanup); + +static int nand_set_ecc_on_host_ops(struct nand_chip *chip) +{ + struct nand_ecc_ctrl *ecc = &chip->ecc; + + switch (ecc->placement) { + case NAND_ECC_PLACEMENT_UNKNOWN: + case NAND_ECC_PLACEMENT_OOB: + /* Use standard hwecc read page function? */ + if (!ecc->read_page) + ecc->read_page = nand_read_page_hwecc; + if (!ecc->write_page) + ecc->write_page = nand_write_page_hwecc; + if (!ecc->read_page_raw) + ecc->read_page_raw = nand_read_page_raw; + if (!ecc->write_page_raw) + ecc->write_page_raw = nand_write_page_raw; + if (!ecc->read_oob) + ecc->read_oob = nand_read_oob_std; + if (!ecc->write_oob) + ecc->write_oob = nand_write_oob_std; + if (!ecc->read_subpage) + ecc->read_subpage = nand_read_subpage; + if (!ecc->write_subpage && ecc->hwctl && ecc->calculate) + ecc->write_subpage = nand_write_subpage_hwecc; + fallthrough; + + case NAND_ECC_PLACEMENT_INTERLEAVED: + if ((!ecc->calculate || !ecc->correct || !ecc->hwctl) && + (!ecc->read_page || + ecc->read_page == nand_read_page_hwecc || + !ecc->write_page || + ecc->write_page == nand_write_page_hwecc)) { + WARN(1, "No ECC functions supplied; hardware ECC not possible\n"); + return -EINVAL; + } + /* Use standard syndrome read/write page function? */ + if (!ecc->read_page) + ecc->read_page = nand_read_page_syndrome; + if (!ecc->write_page) + ecc->write_page = nand_write_page_syndrome; + if (!ecc->read_page_raw) + ecc->read_page_raw = nand_read_page_raw_syndrome; + if (!ecc->write_page_raw) + ecc->write_page_raw = nand_write_page_raw_syndrome; + if (!ecc->read_oob) + ecc->read_oob = nand_read_oob_syndrome; + if (!ecc->write_oob) + ecc->write_oob = nand_write_oob_syndrome; + break; + + default: + pr_warn("Invalid NAND_ECC_PLACEMENT %d\n", + ecc->placement); + return -EINVAL; + } + + return 0; +} + static int nand_set_ecc_soft_ops(struct nand_chip *chip) { struct mtd_info *mtd = nand_to_mtd(chip); + struct nand_device *nanddev = mtd_to_nanddev(mtd); struct nand_ecc_ctrl *ecc = &chip->ecc; + int ret; - if (WARN_ON(ecc->mode != NAND_ECC_SOFT)) + if (WARN_ON(ecc->engine_type != NAND_ECC_ENGINE_TYPE_SOFT)) return -EINVAL; switch (ecc->algo) { - case NAND_ECC_HAMMING: - ecc->calculate = nand_calculate_ecc; - ecc->correct = nand_correct_data; + case NAND_ECC_ALGO_HAMMING: + ecc->calculate = rawnand_sw_hamming_calculate; + ecc->correct = rawnand_sw_hamming_correct; ecc->read_page = nand_read_page_swecc; ecc->read_subpage = nand_read_subpage; ecc->write_page = nand_write_page_swecc; - ecc->read_page_raw = nand_read_page_raw; - ecc->write_page_raw = nand_write_page_raw; + if (!ecc->read_page_raw) + ecc->read_page_raw = nand_read_page_raw; + if (!ecc->write_page_raw) + ecc->write_page_raw = nand_write_page_raw; ecc->read_oob = nand_read_oob_std; ecc->write_oob = nand_write_oob_std; if (!ecc->size) @@ -5112,75 +5825,48 @@ static int nand_set_ecc_soft_ops(struct nand_chip *chip) ecc->bytes = 3; ecc->strength = 1; - if (IS_ENABLED(CONFIG_MTD_NAND_ECC_SMC)) + if (IS_ENABLED(CONFIG_MTD_NAND_ECC_SW_HAMMING_SMC)) ecc->options |= NAND_ECC_SOFT_HAMMING_SM_ORDER; + ret = rawnand_sw_hamming_init(chip); + if (ret) { + WARN(1, "Hamming ECC initialization failed!\n"); + return ret; + } + return 0; - case NAND_ECC_BCH: - if (!mtd_nand_has_bch()) { - WARN(1, "CONFIG_MTD_NAND_ECC_BCH not enabled\n"); + case NAND_ECC_ALGO_BCH: + if (!IS_ENABLED(CONFIG_MTD_NAND_ECC_SW_BCH)) { + WARN(1, "CONFIG_MTD_NAND_ECC_SW_BCH not enabled\n"); return -EINVAL; } - ecc->calculate = nand_bch_calculate_ecc; - ecc->correct = nand_bch_correct_data; + ecc->calculate = rawnand_sw_bch_calculate; + ecc->correct = rawnand_sw_bch_correct; ecc->read_page = nand_read_page_swecc; ecc->read_subpage = nand_read_subpage; ecc->write_page = nand_write_page_swecc; - ecc->read_page_raw = nand_read_page_raw; - ecc->write_page_raw = nand_write_page_raw; + if (!ecc->read_page_raw) + ecc->read_page_raw = nand_read_page_raw; + if (!ecc->write_page_raw) + ecc->write_page_raw = nand_write_page_raw; ecc->read_oob = nand_read_oob_std; ecc->write_oob = nand_write_oob_std; /* - * Board driver should supply ecc.size and ecc.strength - * values to select how many bits are correctable. - * Otherwise, default to 4 bits for large page devices. - */ - if (!ecc->size && (mtd->oobsize >= 64)) { - ecc->size = 512; - ecc->strength = 4; - } - - /* - * if no ecc placement scheme was provided pickup the default - * large page one. - */ - if (!mtd->ooblayout) { - /* handle large page devices only */ - if (mtd->oobsize < 64) { - WARN(1, "OOB layout is required when using software BCH on small pages\n"); - return -EINVAL; - } - - mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops); - - } - - /* * We can only maximize ECC config when the default layout is * used, otherwise we don't know how many bytes can really be * used. */ - if (mtd->ooblayout == &nand_ooblayout_lp_ops && - ecc->options & NAND_ECC_MAXIMIZE) { - int steps, bytes; + if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH && + mtd->ooblayout != nand_get_large_page_ooblayout()) + nanddev->ecc.user_conf.flags &= ~NAND_ECC_MAXIMIZE_STRENGTH; - /* Always prefer 1k blocks over 512bytes ones */ - ecc->size = 1024; - steps = mtd->writesize / ecc->size; - - /* Reserve 2 bytes for the BBM */ - bytes = (mtd->oobsize - 2) / steps; - ecc->strength = bytes * 8 / fls(8 * ecc->size); - } - - /* See nand_bch_init() for details. */ - ecc->bytes = 0; - ecc->priv = nand_bch_init(mtd); - if (!ecc->priv) { + ret = rawnand_sw_bch_init(chip); + if (ret) { WARN(1, "BCH ECC initialization failed!\n"); - return -EINVAL; + return ret; } + return 0; default: WARN(1, "Unsupported ECC algorithm!\n"); @@ -5256,12 +5942,14 @@ static int nand_match_ecc_req(struct nand_chip *chip, const struct nand_ecc_caps *caps, int oobavail) { + const struct nand_ecc_props *requirements = + nanddev_get_ecc_requirements(&chip->base); struct mtd_info *mtd = nand_to_mtd(chip); const struct nand_ecc_step_info *stepinfo; - int req_step = chip->ecc_step_ds; - int req_strength = chip->ecc_strength_ds; + int req_step = requirements->step_size; + int req_strength = requirements->strength; int req_corr, step_size, strength, nsteps, ecc_bytes, ecc_bytes_total; - int best_step, best_strength, best_ecc_bytes; + int best_step = 0, best_strength = 0, best_ecc_bytes = 0; int best_ecc_bytes_total = INT_MAX; int i, j; @@ -5342,7 +6030,7 @@ nand_maximize_ecc(struct nand_chip *chip, int step_size, strength, nsteps, ecc_bytes, corr; int best_corr = 0; int best_step = 0; - int best_strength, best_ecc_bytes; + int best_strength = 0, best_ecc_bytes = 0; int i, j; for (i = 0; i < caps->nstepinfos; i++) { @@ -5400,11 +6088,12 @@ nand_maximize_ecc(struct nand_chip *chip, * @caps: ECC engine caps info structure * @oobavail: OOB size that the ECC engine can use * - * Choose the ECC configuration according to following logic + * Choose the ECC configuration according to following logic. * * 1. If both ECC step size and ECC strength are already set (usually by DT) * then check if it is supported by this controller. - * 2. If NAND_ECC_MAXIMIZE is set, then select maximum ECC strength. + * 2. If the user provided the nand-ecc-maximize property, then select maximum + * ECC strength. * 3. Otherwise, try to match the ECC step size and ECC strength closest * to the chip's requirement. If available OOB size can't fit the chip * requirement then fallback to the maximum ECC step size and ECC strength. @@ -5415,6 +6104,7 @@ int nand_ecc_choose_conf(struct nand_chip *chip, const struct nand_ecc_caps *caps, int oobavail) { struct mtd_info *mtd = nand_to_mtd(chip); + struct nand_device *nanddev = mtd_to_nanddev(mtd); if (WARN_ON(oobavail < 0 || oobavail > mtd->oobsize)) return -EINVAL; @@ -5422,7 +6112,7 @@ int nand_ecc_choose_conf(struct nand_chip *chip, if (chip->ecc.size && chip->ecc.strength) return nand_check_ecc_caps(chip, caps, oobavail); - if (chip->ecc.options & NAND_ECC_MAXIMIZE) + if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) return nand_maximize_ecc(chip, caps, oobavail); if (!nand_match_ecc_req(chip, caps, oobavail)) @@ -5432,40 +6122,50 @@ int nand_ecc_choose_conf(struct nand_chip *chip, } EXPORT_SYMBOL_GPL(nand_ecc_choose_conf); -/* - * Check if the chip configuration meet the datasheet requirements. +static int rawnand_erase(struct nand_device *nand, const struct nand_pos *pos) +{ + struct nand_chip *chip = container_of(nand, struct nand_chip, + base); + unsigned int eb = nanddev_pos_to_row(nand, pos); + int ret; - * If our configuration corrects A bits per B bytes and the minimum - * required correction level is X bits per Y bytes, then we must ensure - * both of the following are true: - * - * (1) A / B >= X / Y - * (2) A >= X - * - * Requirement (1) ensures we can correct for the required bitflip density. - * Requirement (2) ensures we can correct even when all bitflips are clumped - * in the same sector. - */ -static bool nand_ecc_strength_good(struct nand_chip *chip) + eb >>= nand->rowconv.eraseblock_addr_shift; + + nand_select_target(chip, pos->target); + ret = nand_erase_op(chip, eb); + nand_deselect_target(chip); + + return ret; +} + +static int rawnand_markbad(struct nand_device *nand, + const struct nand_pos *pos) { - struct mtd_info *mtd = nand_to_mtd(chip); - struct nand_ecc_ctrl *ecc = &chip->ecc; - int corr, ds_corr; + struct nand_chip *chip = container_of(nand, struct nand_chip, + base); - if (ecc->size == 0 || chip->ecc_step_ds == 0) - /* Not enough information */ - return true; + return nand_markbad_bbm(chip, nanddev_pos_to_offs(nand, pos)); +} - /* - * We get the number of corrected bits per page to compare - * the correction density. - */ - corr = (mtd->writesize * ecc->strength) / ecc->size; - ds_corr = (mtd->writesize * chip->ecc_strength_ds) / chip->ecc_step_ds; +static bool rawnand_isbad(struct nand_device *nand, const struct nand_pos *pos) +{ + struct nand_chip *chip = container_of(nand, struct nand_chip, + base); + int ret; + + nand_select_target(chip, pos->target); + ret = nand_isbad_bbm(chip, nanddev_pos_to_offs(nand, pos)); + nand_deselect_target(chip); - return corr >= ds_corr && ecc->strength >= chip->ecc_strength_ds; + return ret; } +static const struct nand_ops rawnand_ops = { + .erase = rawnand_erase, + .markbad = rawnand_markbad, + .isbad = rawnand_isbad, +}; + /** * nand_scan_tail - Scan for the NAND device * @chip: NAND chip object @@ -5477,6 +6177,7 @@ static bool nand_ecc_strength_good(struct nand_chip *chip) static int nand_scan_tail(struct nand_chip *chip) { struct mtd_info *mtd = nand_to_mtd(chip); + struct nand_device *base = &chip->base; struct nand_ecc_ctrl *ecc = &chip->ecc; int ret, i; @@ -5509,15 +6210,19 @@ static int nand_scan_tail(struct nand_chip *chip) * If no default placement scheme is given, select an appropriate one. */ if (!mtd->ooblayout && - !(ecc->mode == NAND_ECC_SOFT && ecc->algo == NAND_ECC_BCH)) { + !(ecc->engine_type == NAND_ECC_ENGINE_TYPE_SOFT && + ecc->algo == NAND_ECC_ALGO_BCH) && + !(ecc->engine_type == NAND_ECC_ENGINE_TYPE_SOFT && + ecc->algo == NAND_ECC_ALGO_HAMMING)) { switch (mtd->oobsize) { case 8: case 16: - mtd_set_ooblayout(mtd, &nand_ooblayout_sp_ops); + mtd_set_ooblayout(mtd, nand_get_small_page_ooblayout()); break; case 64: case 128: - mtd_set_ooblayout(mtd, &nand_ooblayout_lp_hamming_ops); + mtd_set_ooblayout(mtd, + nand_get_large_page_hamming_ooblayout()); break; default: /* @@ -5527,9 +6232,9 @@ static int nand_scan_tail(struct nand_chip *chip) * page with ECC layout when ->oobsize <= 128 for * compatibility reasons. */ - if (ecc->mode == NAND_ECC_NONE) { + if (ecc->engine_type == NAND_ECC_ENGINE_TYPE_NONE) { mtd_set_ooblayout(mtd, - &nand_ooblayout_lp_ops); + nand_get_large_page_ooblayout()); break; } @@ -5545,59 +6250,11 @@ static int nand_scan_tail(struct nand_chip *chip) * selected and we have 256 byte pagesize fallback to software ECC */ - switch (ecc->mode) { - case NAND_ECC_HW_OOB_FIRST: - /* Similar to NAND_ECC_HW, but a separate read_page handle */ - if (!ecc->calculate || !ecc->correct || !ecc->hwctl) { - WARN(1, "No ECC functions supplied; hardware ECC not possible\n"); - ret = -EINVAL; - goto err_nand_manuf_cleanup; - } - if (!ecc->read_page) - ecc->read_page = nand_read_page_hwecc_oob_first; - - case NAND_ECC_HW: - /* Use standard hwecc read page function? */ - if (!ecc->read_page) - ecc->read_page = nand_read_page_hwecc; - if (!ecc->write_page) - ecc->write_page = nand_write_page_hwecc; - if (!ecc->read_page_raw) - ecc->read_page_raw = nand_read_page_raw; - if (!ecc->write_page_raw) - ecc->write_page_raw = nand_write_page_raw; - if (!ecc->read_oob) - ecc->read_oob = nand_read_oob_std; - if (!ecc->write_oob) - ecc->write_oob = nand_write_oob_std; - if (!ecc->read_subpage) - ecc->read_subpage = nand_read_subpage; - if (!ecc->write_subpage && ecc->hwctl && ecc->calculate) - ecc->write_subpage = nand_write_subpage_hwecc; - - case NAND_ECC_HW_SYNDROME: - if ((!ecc->calculate || !ecc->correct || !ecc->hwctl) && - (!ecc->read_page || - ecc->read_page == nand_read_page_hwecc || - !ecc->write_page || - ecc->write_page == nand_write_page_hwecc)) { - WARN(1, "No ECC functions supplied; hardware ECC not possible\n"); - ret = -EINVAL; + switch (ecc->engine_type) { + case NAND_ECC_ENGINE_TYPE_ON_HOST: + ret = nand_set_ecc_on_host_ops(chip); + if (ret) goto err_nand_manuf_cleanup; - } - /* Use standard syndrome read/write page function? */ - if (!ecc->read_page) - ecc->read_page = nand_read_page_syndrome; - if (!ecc->write_page) - ecc->write_page = nand_write_page_syndrome; - if (!ecc->read_page_raw) - ecc->read_page_raw = nand_read_page_raw_syndrome; - if (!ecc->write_page_raw) - ecc->write_page_raw = nand_write_page_raw_syndrome; - if (!ecc->read_oob) - ecc->read_oob = nand_read_oob_syndrome; - if (!ecc->write_oob) - ecc->write_oob = nand_write_oob_syndrome; if (mtd->writesize >= ecc->size) { if (!ecc->strength) { @@ -5609,18 +6266,17 @@ static int nand_scan_tail(struct nand_chip *chip) } pr_warn("%d byte HW ECC not possible on %d byte page size, fallback to SW ECC\n", ecc->size, mtd->writesize); - ecc->mode = NAND_ECC_SOFT; - ecc->algo = NAND_ECC_HAMMING; + ecc->engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + ecc->algo = NAND_ECC_ALGO_HAMMING; + fallthrough; - case NAND_ECC_SOFT: + case NAND_ECC_ENGINE_TYPE_SOFT: ret = nand_set_ecc_soft_ops(chip); - if (ret) { - ret = -EINVAL; + if (ret) goto err_nand_manuf_cleanup; - } break; - case NAND_ECC_ON_DIE: + case NAND_ECC_ENGINE_TYPE_ON_DIE: if (!ecc->read_page || !ecc->write_page) { WARN(1, "No ECC functions supplied; on-die ECC not possible\n"); ret = -EINVAL; @@ -5632,8 +6288,8 @@ static int nand_scan_tail(struct nand_chip *chip) ecc->write_oob = nand_write_oob_std; break; - case NAND_ECC_NONE: - pr_warn("NAND_ECC_NONE selected by board driver. This is not recommended!\n"); + case NAND_ECC_ENGINE_TYPE_NONE: + pr_warn("NAND_ECC_ENGINE_TYPE_NONE selected by board driver. This is not recommended!\n"); ecc->read_page = nand_read_page_raw; ecc->write_page = nand_write_page_raw; ecc->read_oob = nand_read_oob_std; @@ -5646,7 +6302,7 @@ static int nand_scan_tail(struct nand_chip *chip) break; default: - WARN(1, "Invalid NAND_ECC_MODE %d\n", ecc->mode); + WARN(1, "Invalid NAND_ECC_MODE %d\n", ecc->engine_type); ret = -EINVAL; goto err_nand_manuf_cleanup; } @@ -5666,21 +6322,36 @@ static int nand_scan_tail(struct nand_chip *chip) if (!ecc->write_oob_raw) ecc->write_oob_raw = ecc->write_oob; - /* propagate ecc info to mtd_info */ + /* Propagate ECC info to the generic NAND and MTD layers */ mtd->ecc_strength = ecc->strength; + if (!base->ecc.ctx.conf.strength) + base->ecc.ctx.conf.strength = ecc->strength; mtd->ecc_step_size = ecc->size; + if (!base->ecc.ctx.conf.step_size) + base->ecc.ctx.conf.step_size = ecc->size; /* * Set the number of read / write steps for one page depending on ECC * mode. */ - ecc->steps = mtd->writesize / ecc->size; - if (ecc->steps * ecc->size != mtd->writesize) { - WARN(1, "Invalid ECC parameters\n"); - ret = -EINVAL; - goto err_nand_manuf_cleanup; + if (!ecc->steps) + ecc->steps = mtd->writesize / ecc->size; + if (!base->ecc.ctx.nsteps) + base->ecc.ctx.nsteps = ecc->steps; + + /* + * Validity check: Warn if ECC parameters are not compatible with page size. + * Due to the custom handling of ECC blocks in certain controllers the check + * may result in an expected failure. + */ + if (ecc->steps * ecc->size != mtd->writesize) + pr_warn("ECC parameters may be invalid in reference to underlying NAND chip\n"); + + if (!ecc->total) { + ecc->total = ecc->steps * ecc->bytes; + chip->base.ecc.ctx.total = ecc->total; } - ecc->total = ecc->steps * ecc->bytes; + if (ecc->total > mtd->oobsize) { WARN(1, "Total number of ECC bytes exceeded oobsize\n"); ret = -EINVAL; @@ -5698,9 +6369,11 @@ static int nand_scan_tail(struct nand_chip *chip) mtd->oobavail = ret; /* ECC sanity check: warn if it's too weak */ - if (!nand_ecc_strength_good(chip)) - pr_warn("WARNING: %s: the ECC used on your system is too weak compared to the one required by the NAND chip\n", - mtd->name); + if (!nand_ecc_is_strong_enough(&chip->base)) + pr_warn("WARNING: %s: the ECC used on your system (%db/%dB) is too weak compared to the one required by the NAND chip (%db/%dB)\n", + mtd->name, chip->ecc.strength, chip->ecc.size, + nanddev_get_ecc_requirements(&chip->base)->strength, + nanddev_get_ecc_requirements(&chip->base)->step_size); /* Allow subpage writes up to ecc.steps. Not possible for MLC flash */ if (!(chip->options & NAND_NO_SUBPAGE_WRITE) && nand_is_slc(chip)) { @@ -5717,15 +6390,12 @@ static int nand_scan_tail(struct nand_chip *chip) } chip->subpagesize = mtd->writesize >> mtd->subpage_sft; - /* Initialize state */ - chip->state = FL_READY; - /* Invalidate the pagebuffer reference */ - chip->pagebuf = -1; + chip->pagecache.page = -1; /* Large page NAND with SOFT_ECC should support subpage reads */ - switch (ecc->mode) { - case NAND_ECC_SOFT: + switch (ecc->engine_type) { + case NAND_ECC_ENGINE_TYPE_SOFT: if (chip->page_shift > 9) chip->options |= NAND_SUBPAGE_READ; break; @@ -5734,10 +6404,15 @@ static int nand_scan_tail(struct nand_chip *chip) break; } + ret = nanddev_init(&chip->base, &rawnand_ops, mtd->owner); + if (ret) + goto err_nand_manuf_cleanup; + + /* Adjust the MTD_CAP_ flags when NAND_ROM is set. */ + if (chip->options & NAND_ROM) + mtd->flags = MTD_CAP_ROM; + /* Fill in remaining MTD driver data */ - mtd->type = nand_is_slc(chip) ? MTD_NANDFLASH : MTD_MLCNANDFLASH; - mtd->flags = (chip->options & NAND_ROM) ? MTD_CAP_ROM : - MTD_CAP_NANDFLASH; mtd->_erase = nand_erase; mtd->_point = NULL; mtd->_unpoint = NULL; @@ -5745,16 +6420,15 @@ static int nand_scan_tail(struct nand_chip *chip) mtd->_read_oob = nand_read_oob; mtd->_write_oob = nand_write_oob; mtd->_sync = nand_sync; - mtd->_lock = NULL; - mtd->_unlock = NULL; + mtd->_lock = nand_lock; + mtd->_unlock = nand_unlock; mtd->_suspend = nand_suspend; mtd->_resume = nand_resume; mtd->_reboot = nand_shutdown; mtd->_block_isreserved = nand_block_isreserved; mtd->_block_isbad = nand_block_isbad; mtd->_block_markbad = nand_block_markbad; - mtd->_max_bad_blocks = nand_max_bad_blocks; - mtd->writebufsize = mtd->writesize; + mtd->_max_bad_blocks = nanddev_mtd_max_bad_blocks; /* * Initialize bitflip_threshold to its default prior scan_bbt() call. @@ -5764,18 +6438,30 @@ static int nand_scan_tail(struct nand_chip *chip) if (!mtd->bitflip_threshold) mtd->bitflip_threshold = DIV_ROUND_UP(mtd->ecc_strength * 3, 4); - /* Initialize the ->data_interface field. */ - ret = nand_init_data_interface(chip); + /* Find the fastest data interface for this chip */ + ret = nand_choose_interface_config(chip); if (ret) - goto err_nand_manuf_cleanup; + goto err_nanddev_cleanup; /* Enter fastest possible mode on all dies. */ - for (i = 0; i < chip->numchips; i++) { - ret = nand_setup_data_interface(chip, i); + for (i = 0; i < nanddev_ntargets(&chip->base); i++) { + ret = nand_setup_interface(chip, i); if (ret) - goto err_nand_manuf_cleanup; + goto err_free_interface_config; } + rawnand_late_check_supported_ops(chip); + + /* + * Look for secure regions in the NAND chip. These regions are supposed + * to be protected by a secure element like Trustzone. So the read/write + * accesses to these regions will be blocked in the runtime by this + * driver. + */ + ret = of_get_nand_secure_regions(chip); + if (ret) + goto err_free_interface_config; + /* Check, if we should skip the bad block table scan */ if (chip->options & NAND_SKIP_BBTSCAN) return 0; @@ -5783,10 +6469,18 @@ static int nand_scan_tail(struct nand_chip *chip) /* Build bad block table */ ret = nand_create_bbt(chip); if (ret) - goto err_nand_manuf_cleanup; + goto err_free_secure_regions; return 0; +err_free_secure_regions: + kfree(chip->secure_regions); + +err_free_interface_config: + kfree(chip->best_interface_config); + +err_nanddev_cleanup: + nanddev_cleanup(&chip->base); err_nand_manuf_cleanup: nand_manufacturer_cleanup(chip); @@ -5860,9 +6554,17 @@ EXPORT_SYMBOL(nand_scan_with_ids); */ void nand_cleanup(struct nand_chip *chip) { - if (chip->ecc.mode == NAND_ECC_SOFT && - chip->ecc.algo == NAND_ECC_BCH) - nand_bch_free((struct nand_bch_control *)chip->ecc.priv); + if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_SOFT) { + if (chip->ecc.algo == NAND_ECC_ALGO_HAMMING) + rawnand_sw_hamming_cleanup(chip); + else if (chip->ecc.algo == NAND_ECC_ALGO_BCH) + rawnand_sw_bch_cleanup(chip); + } + + nanddev_cleanup(&chip->base); + + /* Free secure regions data */ + kfree(chip->secure_regions); /* Free bad block table memory */ kfree(chip->bbt); @@ -5875,6 +6577,9 @@ void nand_cleanup(struct nand_chip *chip) & NAND_BBT_DYNAMICSTRUCT) kfree(chip->badblock_pattern); + /* Free the data interface */ + kfree(chip->best_interface_config); + /* Free manufacturer priv data. */ nand_manufacturer_cleanup(chip); @@ -5887,18 +6592,6 @@ void nand_cleanup(struct nand_chip *chip) EXPORT_SYMBOL_GPL(nand_cleanup); -/** - * nand_release - [NAND Interface] Unregister the MTD device and free resources - * held by the NAND device - * @chip: NAND chip object - */ -void nand_release(struct nand_chip *chip) -{ - mtd_device_unregister(nand_to_mtd(chip)); - nand_cleanup(chip); -} -EXPORT_SYMBOL_GPL(nand_release); - MODULE_LICENSE("GPL"); MODULE_AUTHOR("Steven J. Hill <sjhill@realitydiluted.com>"); MODULE_AUTHOR("Thomas Gleixner <tglx@linutronix.de>"); |